X-Git-Url: http://git.megacz.com/?a=blobdiff_plain;f=src%2Forg%2Fibex%2Fmail%2FSMTP.java;h=9ec8b582a09b0c0129958112abb9badb40b41ec1;hb=cffb5785610760f454310af99bd6b88793f0fe8c;hp=ba4069fc7b0208dca2cd5669945c52c7db880ccf;hpb=8b22f8f41cd85657ee7042ac3b7146ee327a309d;p=org.ibex.mail.git diff --git a/src/org/ibex/mail/SMTP.java b/src/org/ibex/mail/SMTP.java index ba4069f..9ec8b58 100644 --- a/src/org/ibex/mail/SMTP.java +++ b/src/org/ibex/mail/SMTP.java @@ -11,10 +11,14 @@ import java.net.*; import java.io.*; import java.util.*; import java.text.*; -import javax.naming.*; -import javax.naming.directory.*; -// RFC's implemented +// FIXME: inbound throttling/ratelimiting +// FIXME: probably need some throttling on outbound mail +// FEATURE: rate-limiting + +// "Address enumeration detection" -- notice when it looks like somebody +// is trying a raft of addresses. + // RFC2554: SMTP Service Extension for Authentication // - did not implement section 5, though // RFC4616: SASL PLAIN @@ -25,45 +29,103 @@ import javax.naming.directory.*; // FIXME: logging: current logging sucks // FIXME: loop prevention -// FIXME: probably need some throttling on outbound mail -// FEATURE: public static boolean validate(Address a) -// FEATURE: rate-limiting +// FEATURE: infer date if not present -// FEATURE: infer messageid, date, if not present (?) -// FEATURE: exponential backoff on retry time? +/* // FEATURE: RFC2822, section 4.5.1: special "postmaster" address -// FEATURE: RFC2822, section 4.5.4.1: retry strategies + Any system that includes an SMTP server supporting mail relaying or + delivery MUST support the reserved mailbox "postmaster" as a case- + insensitive local name. This postmaster address is not strictly + necessary if the server always returns 554 on connection opening (as + described in section 3.1). The requirement to accept mail for + postmaster implies that RCPT commands which specify a mailbox for + postmaster at any of the domains for which the SMTP server provides + mail service, as well as the special case of "RCPT TO:" + (with no domain specification), MUST be supported. + + */ // FEATURE: RFC2822, section 5, multiple MX records, preferences, ordering // FEATURE: RFC2822, end of 4.1.2: backslashes in headers -public class SMTP { +// FEATURE: batching retrys by host (retry multiple in one session, keep retry intervals on a host basis not a message basis) +// FEATURE: first two attempts should be close together (rec'd by 2821) +/* + // FEATURE: RFC2822, section 4.5.4.1: retry strategies + // per-command, per-attempt timeouts + Experience suggests that failures are typically transient (the target + system or its connection has crashed), favoring a policy of two + connection attempts in the first hour the message is in the queue, + and then backing off to one every two or three hours. - public static final SimpleDateFormat dateFormat = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss Z"); - public static final int numOutgoingThreads = 5; + The SMTP client can shorten the queuing delay in cooperation with the + SMTP server. For example, if mail is received from a particular + address, it is likely that mail queued for that host can now be sent. + Application of this principle may, in many cases, eliminate the + requirement for an explicit "send queues now" function such as ETRN + [9]. - public static final int GRAYLIST_MINWAIT = 1000 * 60 * 60; // one hour - public static final int GRAYLIST_MAXWAIT = 1000 * 60 * 60 * 24 * 5; // five days + An SMTP client may have a large queue of messages for each + unavailable destination host. If all of these messages were retried + in every retry cycle, there would be excessive Internet overhead and + the sending system would be blocked for a long period. Note that an + SMTP client can generally determine that a delivery attempt has + failed only after a timeout of several minutes and even a one-minute + timeout per connection will result in a very large delay if retries + are repeated for dozens, or even hundreds, of queued messages to the + same host. - public static final Graylist graylist = - new Graylist(Mailbox.STORAGE_ROOT+"/db/graylist.sqlite"); + When a mail message is to be delivered to multiple recipients, and + the SMTP server to which a copy of the message is to be sent is the + same for multiple recipients, then only one copy of the message + SHOULD be transmitted. That is, the SMTP client SHOULD use the + command sequence: MAIL, RCPT, RCPT,... RCPT, DATA instead of the + sequence: MAIL, RCPT, DATA, ..., MAIL, RCPT, DATA. However, if there + are very many addresses, a limit on the number of RCPT commands per + MAIL command MAY be imposed. Implementation of this efficiency + feature is strongly encouraged. + */ +public class SMTP { - public static final Whitelist whitelist = - new Whitelist(Mailbox.STORAGE_ROOT+"/db/whitelist.sqlite"); + public static final SimpleDateFormat dateFormat = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss Z"); - public static final int MAX_MESSAGE_SIZE = - Integer.parseInt(System.getProperty("org.ibex.mail.smtp.maxMessageSize", "-1")); + private static final SqliteMailbox allmail = + (SqliteMailbox)FileBasedMailbox + .getFileBasedMailbox("/afs/megacz.com/mail/user/megacz/allmail.sqlite", false); - private static final Mailbox spool = - FileBasedMailbox.getFileBasedMailbox(Mailbox.STORAGE_ROOT,false).slash("spool",true).slash("smtp",true); + public static final int NUM_OUTGOING_THREADS = 5; + public static final int GRAYLIST_MINWAIT = 1000 * 60 * 60; // one hour + public static final int GRAYLIST_MAXWAIT = 1000 * 60 * 60 * 24 * 5; // five days + public static final int MAX_MESSAGE_SIZE = Integer.parseInt(System.getProperty("org.ibex.mail.smtp.maxMessageSize", "-1")); + public static final int RETRY_TIME = 1000 * 60 * 30; // 30min recommended by RFC + public static final int GIVE_UP_TIME = 1000 * 60 * 24 * 5; // FIXME: actually use this + public static final Graylist graylist; + public static final Whitelist whitelist; static { - for(int i=0; i") ? null : new Address(command); conn.println("250 " + from + " is syntactically correct"); + // Don't perform SAV; discouraged here + // http://blog.fastmail.fm/2007/12/05/sending-email-servers-best-practice/ } else if (c.startsWith("RCPT TO:")) { // some clients are broken and put RCPT first; we will tolerate this command = command.substring(8).trim(); if(command.indexOf(' ') != -1) command = command.substring(0, command.indexOf(' ')); Address addr = new Address(command); - if (addr.isLocal()) { - // FEATURE: should check the address further and give 550 if undeliverable - conn.println("250 " + addr + " is on this machine; I will deliver it"); - to.addElement(addr); - } else if (conn.getRemoteAddress().isLoopbackAddress() || (from!=null&&from.toString().indexOf("johnw")!=-1)) { + if (conn.getRemoteAddress().isLoopbackAddress() || (from!=null&&from.toString().indexOf("johnw")!=-1)) { conn.println("250 you are connected locally, so I will let you send"); to.addElement(addr); - whitelist.addWhitelist(addr); + if (!whitelist.isWhitelisted(addr)) + whitelist.addWhitelist(addr); } else if (authenticatedAs!=null) { conn.println("250 you are authenticated as "+authenticatedAs+", so I will let you send"); to.addElement(addr); - whitelist.addWhitelist(addr); + if (!whitelist.isWhitelisted(addr)) + whitelist.addWhitelist(addr); + } else if (addr.isLocal()) { + if (to.size() > 3) { + conn.println("536 sorry, limit on 3 RCPT TO's per DATA"); + } else { + // FEATURE: should check the address further and give 550 if undeliverable + conn.println("250 " + addr + " is on this machine; I will deliver it"); + to.addElement(addr); + } } else { conn.println("535 sorry, " + addr + " is not on this machine, you are not connected from localhost, and I will not relay without SMTP AUTH"); Log.warn("","535 sorry, " + addr + " is not on this machine, you are not connected from localhost, and I will not relay without SMTP AUTH"); + failedRcptCount++; + if (failedRcptCount > 3) { + conn.close(); + return; + } } conn.flush(); } else if (c.startsWith("DATA")) { @@ -237,7 +313,7 @@ public class SMTP { if (s.equals(".")) break; if (s.startsWith(".")) s = s.substring(1); buf.append(s + "\r\n"); - if (MAX_MESSAGE_SIZE != -1 && buf.length() > MAX_MESSAGE_SIZE) { + if (MAX_MESSAGE_SIZE != -1 && buf.length() > MAX_MESSAGE_SIZE && (from+"").indexOf("paperless")==-1) { Log.error("**"+conn.getRemoteAddress()+"**", "sorry, this mail server only accepts messages of less than " + ByteSize.toString(MAX_MESSAGE_SIZE)); @@ -245,12 +321,10 @@ public class SMTP { ByteSize.toString(MAX_MESSAGE_SIZE)); } } - String body = buf.toString(); + String message = buf.toString(); Message m = null; - for(int i=0; i threads = new HashSet(); private static final HashMap deadHosts = new HashMap(); + private static Map nextTry = Collections.synchronizedMap(new HashMap()); + + private Mailbox.Iterator it; + private final String name; + + public Outgoing(String name) { + this.name = name; + synchronized(Outgoing.class) { + threads.add(this); + } + } + + public String toString() { return name; } + public static void enqueue(Message m) throws IOException { if (m == null) { Log.warn(Outgoing.class, "attempted to enqueue(null)"); return; } String traces = m.headers.get("Received"); @@ -298,7 +392,7 @@ public class SMTP { Log.warn(SMTP.Outgoing.class, "aieeee, null envelopeTo: " + m.summary()); return false; } - InetAddress[] mx = getMailExchangerIPs(m.envelopeTo.host); + InetAddress[] mx = DNSUtil.getMailExchangerIPs(m.envelopeTo.host); if (mx.length == 0) { if (!noBounces) { enqueue(m.bounce("could not resolve " + m.envelopeTo.host)); @@ -319,7 +413,7 @@ public class SMTP { } for(int i=0; i= 500 && e.code <= 599) { try { attempt(m.bounce("unable to deliver: " + e), true); @@ -388,13 +493,14 @@ public class SMTP { } return true; } + */ return false; } catch (Exception e) { if (accepted) return true; Log.warn(SMTP.Outgoing.class, " unable to send; error=" + e); Log.warn(SMTP.Outgoing.class, " message: " + m.summary()); Log.warn(SMTP.Outgoing.class, e); - if (conn != null) Log.warn(SMTP.Outgoing.class, conn.dumpLog()); + //if (conn != null) Log.warn(SMTP.Outgoing.class, conn.dumpLog()); return false; } finally { if (conn != null) conn.close(); @@ -402,20 +508,9 @@ public class SMTP { return accepted; } - private static HashSet threads = new HashSet(); - private static int serials = 1; - private int serial = serials++; - private Mailbox.Iterator it; - - public Outgoing() { - synchronized(Outgoing.class) { - threads.add(this); - } - } - public void wake() { int count = spool.count(Query.all()); - Log.info(SMTP.Outgoing.class, "outgoing thread #"+serial+" woke up; " + count + " messages to send"); + Log.info(SMTP.Outgoing.class, "outgoing thread "+name+" woke up; " + count + " messages to send"); try { while(true) { boolean good = false; @@ -431,7 +526,12 @@ public class SMTP { } if (!good) break; try { - if (attempt(it.cur())) it.delete(); + String messageid = it.cur().messageid; + if (nextTry.get(messageid) == null || System.currentTimeMillis() > nextTry.get(messageid)) { + boolean ok = attempt(it.cur()); + if (ok) it.delete(); + else nextTry.put(messageid, System.currentTimeMillis() + RETRY_TIME); + } } catch (Exception e) { Log.error(SMTP.Outgoing.class, e); } @@ -442,14 +542,14 @@ public class SMTP { //if (e instanceof InterruptedException) throw e; Log.error(SMTP.Outgoing.class, e); } - Log.info(SMTP.Outgoing.class, "outgoing thread #"+serial+" going back to sleep"); + Log.info(SMTP.Outgoing.class, "outgoing thread #"+name+" going back to sleep"); it = null; } public void run() { try { while(true) { - Log.setThreadAnnotation("[outgoing #"+serial+"] "); + Log.setThreadAnnotation("[outgoing #"+name+"] "); wake(); Thread.sleep(1000); synchronized(Outgoing.class) { @@ -460,42 +560,4 @@ public class SMTP { } } - public static InetAddress[] getMailExchangerIPs(String hostName) { - InetAddress[] ret; - try { - Hashtable env = new Hashtable(); - env.put("java.naming.factory.initial", "com.sun.jndi.dns.DnsContextFactory"); - DirContext ictx = new InitialDirContext(env); - Attributes attrs = ictx.getAttributes(hostName, new String[] { "MX" }); - Attribute attr = attrs.get("MX"); - if (attr == null) { - ret = new InetAddress[1]; - try { - ret[0] = InetAddress.getByName(hostName); - if (ret[0].equals(IP.getIP(127,0,0,1)) || ret[0].isLoopbackAddress()) throw new UnknownHostException(); - return ret; - } catch (UnknownHostException uhe) { - Log.warn(SMTP.class, "no MX hosts or A record for " + hostName); - return new InetAddress[0]; - } - } else { - ret = new InetAddress[attr.size()]; - NamingEnumeration ne = attr.getAll(); - for(int i=0; ne.hasMore();) { - String mx = (String)ne.next(); - // FIXME we should be sorting here - mx = mx.substring(mx.indexOf(" ") + 1); - if (mx.charAt(mx.length() - 1) == '.') mx = mx.substring(0, mx.length() - 1); - InetAddress ia = InetAddress.getByName(mx); - if (ia.equals(IP.getIP(127,0,0,1)) || ia.isLoopbackAddress()) continue; - ret[i++] = ia; - } - } - } catch (Exception e) { - Log.warn(SMTP.class, "couldn't find MX host for " + hostName + " due to"); - Log.warn(SMTP.class, e); - return new InetAddress[0]; - } - return ret; - } }