X-Git-Url: http://git.megacz.com/?a=blobdiff_plain;f=src%2Forg%2Fibex%2Fmail%2FSMTP.java;h=9ec8b582a09b0c0129958112abb9badb40b41ec1;hb=cffb5785610760f454310af99bd6b88793f0fe8c;hp=a426987a7deee5c0f6dbd9876e8e06cf4500d1d0;hpb=36e19d8a6c5c04fc1d48c166ca7d951b763093a1;p=org.ibex.mail.git diff --git a/src/org/ibex/mail/SMTP.java b/src/org/ibex/mail/SMTP.java index a426987..9ec8b58 100644 --- a/src/org/ibex/mail/SMTP.java +++ b/src/org/ibex/mail/SMTP.java @@ -11,12 +11,14 @@ import java.net.*; import java.io.*; import java.util.*; import java.text.*; -import javax.naming.*; -import javax.naming.directory.*; // FIXME: inbound throttling/ratelimiting +// FIXME: probably need some throttling on outbound mail +// FEATURE: rate-limiting + +// "Address enumeration detection" -- notice when it looks like somebody +// is trying a raft of addresses. -// RFC's implemented // RFC2554: SMTP Service Extension for Authentication // - did not implement section 5, though // RFC4616: SASL PLAIN @@ -27,24 +29,75 @@ import javax.naming.directory.*; // FIXME: logging: current logging sucks // FIXME: loop prevention -// FIXME: probably need some throttling on outbound mail -// FEATURE: public static boolean validate(Address a) -// FEATURE: rate-limiting +// FEATURE: infer date if not present -// FEATURE: infer messageid, date, if not present (?) -// FEATURE: exponential backoff on retry time? +/* // FEATURE: RFC2822, section 4.5.1: special "postmaster" address -// FEATURE: RFC2822, section 4.5.4.1: retry strategies + Any system that includes an SMTP server supporting mail relaying or + delivery MUST support the reserved mailbox "postmaster" as a case- + insensitive local name. This postmaster address is not strictly + necessary if the server always returns 554 on connection opening (as + described in section 3.1). The requirement to accept mail for + postmaster implies that RCPT commands which specify a mailbox for + postmaster at any of the domains for which the SMTP server provides + mail service, as well as the special case of "RCPT TO:" + (with no domain specification), MUST be supported. + + */ // FEATURE: RFC2822, section 5, multiple MX records, preferences, ordering // FEATURE: RFC2822, end of 4.1.2: backslashes in headers +// FEATURE: batching retrys by host (retry multiple in one session, keep retry intervals on a host basis not a message basis) +// FEATURE: first two attempts should be close together (rec'd by 2821) +/* + // FEATURE: RFC2822, section 4.5.4.1: retry strategies + // per-command, per-attempt timeouts + Experience suggests that failures are typically transient (the target + system or its connection has crashed), favoring a policy of two + connection attempts in the first hour the message is in the queue, + and then backing off to one every two or three hours. + + The SMTP client can shorten the queuing delay in cooperation with the + SMTP server. For example, if mail is received from a particular + address, it is likely that mail queued for that host can now be sent. + Application of this principle may, in many cases, eliminate the + requirement for an explicit "send queues now" function such as ETRN + [9]. + + An SMTP client may have a large queue of messages for each + unavailable destination host. If all of these messages were retried + in every retry cycle, there would be excessive Internet overhead and + the sending system would be blocked for a long period. Note that an + SMTP client can generally determine that a delivery attempt has + failed only after a timeout of several minutes and even a one-minute + timeout per connection will result in a very large delay if retries + are repeated for dozens, or even hundreds, of queued messages to the + same host. + + When a mail message is to be delivered to multiple recipients, and + the SMTP server to which a copy of the message is to be sent is the + same for multiple recipients, then only one copy of the message + SHOULD be transmitted. That is, the SMTP client SHOULD use the + command sequence: MAIL, RCPT, RCPT,... RCPT, DATA instead of the + sequence: MAIL, RCPT, DATA, ..., MAIL, RCPT, DATA. However, if there + are very many addresses, a limit on the number of RCPT commands per + MAIL command MAY be imposed. Implementation of this efficiency + feature is strongly encouraged. + */ public class SMTP { public static final SimpleDateFormat dateFormat = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss Z"); - public static final int numOutgoingThreads = 5; - public static final int GRAYLIST_MINWAIT = 1000 * 60 * 60; // one hour - public static final int GRAYLIST_MAXWAIT = 1000 * 60 * 60 * 24 * 5; // five days + private static final SqliteMailbox allmail = + (SqliteMailbox)FileBasedMailbox + .getFileBasedMailbox("/afs/megacz.com/mail/user/megacz/allmail.sqlite", false); + + public static final int NUM_OUTGOING_THREADS = 5; + public static final int GRAYLIST_MINWAIT = 1000 * 60 * 60; // one hour + public static final int GRAYLIST_MAXWAIT = 1000 * 60 * 60 * 24 * 5; // five days + public static final int MAX_MESSAGE_SIZE = Integer.parseInt(System.getProperty("org.ibex.mail.smtp.maxMessageSize", "-1")); + public static final int RETRY_TIME = 1000 * 60 * 30; // 30min recommended by RFC + public static final int GIVE_UP_TIME = 1000 * 60 * 24 * 5; // FIXME: actually use this public static final Graylist graylist; public static final Whitelist whitelist; @@ -57,20 +110,22 @@ public class SMTP { } } - public static final int MAX_MESSAGE_SIZE = - Integer.parseInt(System.getProperty("org.ibex.mail.smtp.maxMessageSize", "-1")); - private static final Mailbox spool = FileBasedMailbox.getFileBasedMailbox(Mailbox.STORAGE_ROOT,false).slash("spool",true).slash("smtp",true).getMailbox(); - static { - for(int i=0; i") ? null : new Address(command); conn.println("250 " + from + " is syntactically correct"); - // FEATURE: perform SMTP validation on the address, reject if invalid + // Don't perform SAV; discouraged here + // http://blog.fastmail.fm/2007/12/05/sending-email-servers-best-practice/ } else if (c.startsWith("RCPT TO:")) { // some clients are broken and put RCPT first; we will tolerate this command = command.substring(8).trim(); @@ -288,9 +344,29 @@ public class SMTP { // Outgoing Mail Thread ////////////////////////////////////////////////////////////////////////////// + static { + for(int i=0; i threads = new HashSet(); private static final HashMap deadHosts = new HashMap(); + private static Map nextTry = Collections.synchronizedMap(new HashMap()); + + private Mailbox.Iterator it; + private final String name; + + public Outgoing(String name) { + this.name = name; + synchronized(Outgoing.class) { + threads.add(this); + } + } + + public String toString() { return name; } + public static void enqueue(Message m) throws IOException { if (m == null) { Log.warn(Outgoing.class, "attempted to enqueue(null)"); return; } String traces = m.headers.get("Received"); @@ -316,7 +392,7 @@ public class SMTP { Log.warn(SMTP.Outgoing.class, "aieeee, null envelopeTo: " + m.summary()); return false; } - InetAddress[] mx = getMailExchangerIPs(m.envelopeTo.host); + InetAddress[] mx = DNSUtil.getMailExchangerIPs(m.envelopeTo.host); if (mx.length == 0) { if (!noBounces) { enqueue(m.bounce("could not resolve " + m.envelopeTo.host)); @@ -353,14 +429,23 @@ public class SMTP { Connection conn = null; try { conn = new Connection(new Socket(mx, 25), InetAddress.getLocalHost().getHostName()); + InetAddress localAddress = conn.getSocket().getLocalAddress(); + String reverse = DNSUtil.reverseLookup(localAddress); + Log.info(SMTP.Outgoing.class, + "outbound connection to " + mx + " uses " + localAddress + " [reverse: " + reverse + "]"); + InetAddress relookup = InetAddress.getByName(reverse); + if (!relookup.equals(localAddress)) + Log.error(SMTP.Outgoing.class, + "Warning: local machine fails forward-confirmed-reverse; " + + reverse + " resolves to " + localAddress); conn.setNewline("\r\n"); conn.setTimeout(60 * 1000); check(conn.readln(), conn); // banner try { - conn.println("EHLO " + conn.vhost); + conn.println("EHLO " + reverse); check(conn.readln(), conn); } catch (SMTPException smtpe) { - conn.println("HELO " + conn.vhost); + conn.println("HELO " + reverse); check(conn.readln(), conn); } String envelopeFrom = m.envelopeFrom==null ? "" : m.envelopeFrom.toString(); @@ -423,20 +508,9 @@ public class SMTP { return accepted; } - private static HashSet threads = new HashSet(); - private static int serials = 1; - private int serial = serials++; - private Mailbox.Iterator it; - - public Outgoing() { - synchronized(Outgoing.class) { - threads.add(this); - } - } - public void wake() { int count = spool.count(Query.all()); - Log.info(SMTP.Outgoing.class, "outgoing thread #"+serial+" woke up; " + count + " messages to send"); + Log.info(SMTP.Outgoing.class, "outgoing thread "+name+" woke up; " + count + " messages to send"); try { while(true) { boolean good = false; @@ -452,7 +526,12 @@ public class SMTP { } if (!good) break; try { - if (attempt(it.cur())) it.delete(); + String messageid = it.cur().messageid; + if (nextTry.get(messageid) == null || System.currentTimeMillis() > nextTry.get(messageid)) { + boolean ok = attempt(it.cur()); + if (ok) it.delete(); + else nextTry.put(messageid, System.currentTimeMillis() + RETRY_TIME); + } } catch (Exception e) { Log.error(SMTP.Outgoing.class, e); } @@ -463,14 +542,14 @@ public class SMTP { //if (e instanceof InterruptedException) throw e; Log.error(SMTP.Outgoing.class, e); } - Log.info(SMTP.Outgoing.class, "outgoing thread #"+serial+" going back to sleep"); + Log.info(SMTP.Outgoing.class, "outgoing thread #"+name+" going back to sleep"); it = null; } public void run() { try { while(true) { - Log.setThreadAnnotation("[outgoing #"+serial+"] "); + Log.setThreadAnnotation("[outgoing #"+name+"] "); wake(); Thread.sleep(1000); synchronized(Outgoing.class) { @@ -481,42 +560,4 @@ public class SMTP { } } - public static InetAddress[] getMailExchangerIPs(String hostName) { - InetAddress[] ret; - try { - Hashtable env = new Hashtable(); - env.put("java.naming.factory.initial", "com.sun.jndi.dns.DnsContextFactory"); - DirContext ictx = new InitialDirContext(env); - Attributes attrs = ictx.getAttributes(hostName, new String[] { "MX" }); - Attribute attr = attrs.get("MX"); - if (attr == null) { - ret = new InetAddress[1]; - try { - ret[0] = InetAddress.getByName(hostName); - if (ret[0].equals(IP.getIP(127,0,0,1)) || ret[0].isLoopbackAddress()) throw new UnknownHostException(); - return ret; - } catch (UnknownHostException uhe) { - Log.warn(SMTP.class, "no MX hosts or A record for " + hostName); - return new InetAddress[0]; - } - } else { - ret = new InetAddress[attr.size()]; - NamingEnumeration ne = attr.getAll(); - for(int i=0; ne.hasMore();) { - String mx = (String)ne.next(); - // FIXME we should be sorting here - mx = mx.substring(mx.indexOf(" ") + 1); - if (mx.charAt(mx.length() - 1) == '.') mx = mx.substring(0, mx.length() - 1); - InetAddress ia = InetAddress.getByName(mx); - if (ia.equals(IP.getIP(127,0,0,1)) || ia.isLoopbackAddress()) continue; - ret[i++] = ia; - } - } - } catch (Exception e) { - Log.warn(SMTP.class, "couldn't find MX host for " + hostName + " due to"); - Log.warn(SMTP.class, e); - return new InetAddress[0]; - } - return ret; - } }