import java.io.*;
import java.util.*;
import java.text.*;
-import javax.naming.*;
-import javax.naming.directory.*;
-// RFC's implemented
+// FIXME: inbound throttling/ratelimiting
+// FIXME: probably need some throttling on outbound mail
+// FEATURE: rate-limiting
+
+// "Address enumeration detection" -- notice when it looks like somebody
+// is trying a raft of addresses.
+
// RFC2554: SMTP Service Extension for Authentication
// - did not implement section 5, though
// RFC4616: SASL PLAIN
// FIXME: logging: current logging sucks
// FIXME: loop prevention
-// FIXME: probably need some throttling on outbound mail
-// FEATURE: public static boolean validate(Address a)
-// FEATURE: rate-limiting
+// FEATURE: infer date if not present
-// FEATURE: infer messageid, date, if not present (?)
-// FEATURE: exponential backoff on retry time?
+/*
// FEATURE: RFC2822, section 4.5.1: special "postmaster" address
-// FEATURE: RFC2822, section 4.5.4.1: retry strategies
+ Any system that includes an SMTP server supporting mail relaying or
+ delivery MUST support the reserved mailbox "postmaster" as a case-
+ insensitive local name. This postmaster address is not strictly
+ necessary if the server always returns 554 on connection opening (as
+ described in section 3.1). The requirement to accept mail for
+ postmaster implies that RCPT commands which specify a mailbox for
+ postmaster at any of the domains for which the SMTP server provides
+ mail service, as well as the special case of "RCPT TO:<Postmaster>"
+ (with no domain specification), MUST be supported.
+
+ */
// FEATURE: RFC2822, section 5, multiple MX records, preferences, ordering
// FEATURE: RFC2822, end of 4.1.2: backslashes in headers
-public class SMTP {
+// FEATURE: batching retrys by host (retry multiple in one session, keep retry intervals on a host basis not a message basis)
+// FEATURE: first two attempts should be close together (rec'd by 2821)
+/*
+ // FEATURE: RFC2822, section 4.5.4.1: retry strategies
+ // per-command, per-attempt timeouts
+ Experience suggests that failures are typically transient (the target
+ system or its connection has crashed), favoring a policy of two
+ connection attempts in the first hour the message is in the queue,
+ and then backing off to one every two or three hours.
- public static final SimpleDateFormat dateFormat = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss Z");
- public static final int numOutgoingThreads = 5;
+ The SMTP client can shorten the queuing delay in cooperation with the
+ SMTP server. For example, if mail is received from a particular
+ address, it is likely that mail queued for that host can now be sent.
+ Application of this principle may, in many cases, eliminate the
+ requirement for an explicit "send queues now" function such as ETRN
+ [9].
- public static final int GRAYLIST_MINWAIT = 1000 * 60 * 60; // one hour
- public static final int GRAYLIST_MAXWAIT = 1000 * 60 * 60 * 24 * 5; // five days
+ An SMTP client may have a large queue of messages for each
+ unavailable destination host. If all of these messages were retried
+ in every retry cycle, there would be excessive Internet overhead and
+ the sending system would be blocked for a long period. Note that an
+ SMTP client can generally determine that a delivery attempt has
+ failed only after a timeout of several minutes and even a one-minute
+ timeout per connection will result in a very large delay if retries
+ are repeated for dozens, or even hundreds, of queued messages to the
+ same host.
- public static final Graylist graylist =
- new Graylist(Mailbox.STORAGE_ROOT+"/db/graylist.sqlite");
+ When a mail message is to be delivered to multiple recipients, and
+ the SMTP server to which a copy of the message is to be sent is the
+ same for multiple recipients, then only one copy of the message
+ SHOULD be transmitted. That is, the SMTP client SHOULD use the
+ command sequence: MAIL, RCPT, RCPT,... RCPT, DATA instead of the
+ sequence: MAIL, RCPT, DATA, ..., MAIL, RCPT, DATA. However, if there
+ are very many addresses, a limit on the number of RCPT commands per
+ MAIL command MAY be imposed. Implementation of this efficiency
+ feature is strongly encouraged.
+ */
+public class SMTP {
- public static final Whitelist whitelist =
- new Whitelist(Mailbox.STORAGE_ROOT+"/db/whitelist.sqlite");
+ public static final SimpleDateFormat dateFormat = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss Z");
- public static final int MAX_MESSAGE_SIZE =
- Integer.parseInt(System.getProperty("org.ibex.mail.smtp.maxMessageSize", "-1"));
+ private static final SqliteMailbox allmail =
+ (SqliteMailbox)FileBasedMailbox
+ .getFileBasedMailbox("/afs/megacz.com/mail/user/megacz/allmail.sqlite", false);
- private static final Mailbox spool =
- FileBasedMailbox.getFileBasedMailbox(Mailbox.STORAGE_ROOT,false).slash("spool",true).slash("smtp",true);
+ public static final int NUM_OUTGOING_THREADS = 5;
+ public static final int GRAYLIST_MINWAIT = 1000 * 60 * 60; // one hour
+ public static final int GRAYLIST_MAXWAIT = 1000 * 60 * 60 * 24 * 5; // five days
+ public static final int MAX_MESSAGE_SIZE = Integer.parseInt(System.getProperty("org.ibex.mail.smtp.maxMessageSize", "-1"));
+ public static final int RETRY_TIME = 1000 * 60 * 30; // 30min recommended by RFC
+ public static final int GIVE_UP_TIME = 1000 * 60 * 24 * 5; // FIXME: actually use this
+ public static final Graylist graylist;
+ public static final Whitelist whitelist;
static {
- for(int i=0; i<numOutgoingThreads; i++)
- new Outgoing().start();
+ try {
+ graylist = new Graylist(Mailbox.STORAGE_ROOT+"/db/graylist.sqlite");
+ whitelist = new Whitelist(Mailbox.STORAGE_ROOT+"/db/whitelist.sqlite");
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
+ private static final Mailbox spool =
+ FileBasedMailbox.getFileBasedMailbox(Mailbox.STORAGE_ROOT,false).slash("spool",true).slash("smtp",true).getMailbox();
+
+
public static void enqueue(Message m) throws IOException {
if (!m.envelopeTo.isLocal()) Outgoing.enqueue(m);
- else Target.root.accept(m);
+ else {
+ try {
+ allmail.accept(m);
+ } catch (Exception e) {
+ // FIXME incredibly gross hack
+ if (e.toString().indexOf("attempt to insert two messages with identical messageid")==-1)
+ Log.error(SMTP.class, e);
+ }
+ Target.root.accept(m);
+ }
}
public static class SMTPException extends MailException {
boolean ehlo = false;
String remotehost = null;
String authenticatedAs = null;
+ int failedRcptCount = 0;
for(String command = conn.readln(); ; command = conn.readln()) try {
if (command == null) return;
Log.warn("**"+conn.getRemoteAddress()+"**", command);
command = command.substring(10).trim();
from = command.equals("<>") ? null : new Address(command);
conn.println("250 " + from + " is syntactically correct");
+ // Don't perform SAV; discouraged here
+ // http://blog.fastmail.fm/2007/12/05/sending-email-servers-best-practice/
} else if (c.startsWith("RCPT TO:")) {
// some clients are broken and put RCPT first; we will tolerate this
command = command.substring(8).trim();
if(command.indexOf(' ') != -1) command = command.substring(0, command.indexOf(' '));
Address addr = new Address(command);
- if (addr.isLocal()) {
- // FEATURE: should check the address further and give 550 if undeliverable
- conn.println("250 " + addr + " is on this machine; I will deliver it");
- to.addElement(addr);
- } else if (conn.getRemoteAddress().isLoopbackAddress() || (from!=null&&from.toString().indexOf("johnw")!=-1)) {
+ if (conn.getRemoteAddress().isLoopbackAddress() || (from!=null&&from.toString().indexOf("johnw")!=-1)) {
conn.println("250 you are connected locally, so I will let you send");
to.addElement(addr);
- whitelist.addWhitelist(addr);
+ if (!whitelist.isWhitelisted(addr))
+ whitelist.addWhitelist(addr);
} else if (authenticatedAs!=null) {
conn.println("250 you are authenticated as "+authenticatedAs+", so I will let you send");
to.addElement(addr);
- whitelist.addWhitelist(addr);
+ if (!whitelist.isWhitelisted(addr))
+ whitelist.addWhitelist(addr);
+ } else if (addr.isLocal()) {
+ if (to.size() > 3) {
+ conn.println("536 sorry, limit on 3 RCPT TO's per DATA");
+ } else {
+ // FEATURE: should check the address further and give 550 if undeliverable
+ conn.println("250 " + addr + " is on this machine; I will deliver it");
+ to.addElement(addr);
+ }
} else {
conn.println("535 sorry, " + addr + " is not on this machine, you are not connected from localhost, and I will not relay without SMTP AUTH");
Log.warn("","535 sorry, " + addr + " is not on this machine, you are not connected from localhost, and I will not relay without SMTP AUTH");
+ failedRcptCount++;
+ if (failedRcptCount > 3) {
+ conn.close();
+ return;
+ }
}
conn.flush();
} else if (c.startsWith("DATA")) {
if (s.equals(".")) break;
if (s.startsWith(".")) s = s.substring(1);
buf.append(s + "\r\n");
- if (MAX_MESSAGE_SIZE != -1 && buf.length() > MAX_MESSAGE_SIZE) {
+ if (MAX_MESSAGE_SIZE != -1 && buf.length() > MAX_MESSAGE_SIZE && (from+"").indexOf("paperless")==-1) {
Log.error("**"+conn.getRemoteAddress()+"**",
"sorry, this mail server only accepts messages of less than " +
ByteSize.toString(MAX_MESSAGE_SIZE));
ByteSize.toString(MAX_MESSAGE_SIZE));
}
}
- String body = buf.toString();
+ String message = buf.toString();
Message m = null;
- for(int i=0; i<to.size(); i++) {
- m = Message.newMessage(Fountain.Util.create(body), from, (Address)to.elementAt(i));
- enqueue(m);
- }
+ for(int i=0; i<to.size(); i++)
+ enqueue(m = Message.newMessage(Fountain.Util.create(message)).withEnvelope(from, (Address)to.elementAt(i)));
if (m != null) Log.info(SMTP.class, "accepted message: " + m.summary());
conn.println("250 message accepted");
conn.flush();
// Outgoing Mail Thread //////////////////////////////////////////////////////////////////////////////
+ static {
+ for(int i=0; i<NUM_OUTGOING_THREADS; i++)
+ new Outgoing("#"+i).start();
+ }
+
public static class Outgoing extends Thread {
+ private static HashSet<Outgoing> threads = new HashSet<Outgoing>();
private static final HashMap deadHosts = new HashMap();
+ private static Map<String,Long> nextTry = Collections.synchronizedMap(new HashMap<String,Long>());
+
+ private Mailbox.Iterator it;
+ private final String name;
+
+ public Outgoing(String name) {
+ this.name = name;
+ synchronized(Outgoing.class) {
+ threads.add(this);
+ }
+ }
+
+ public String toString() { return name; }
+
public static void enqueue(Message m) throws IOException {
if (m == null) { Log.warn(Outgoing.class, "attempted to enqueue(null)"); return; }
String traces = m.headers.get("Received");
Log.warn(SMTP.Outgoing.class, "aieeee, null envelopeTo: " + m.summary());
return false;
}
- InetAddress[] mx = getMailExchangerIPs(m.envelopeTo.host);
+ InetAddress[] mx = DNSUtil.getMailExchangerIPs(m.envelopeTo.host);
if (mx.length == 0) {
if (!noBounces) {
enqueue(m.bounce("could not resolve " + m.envelopeTo.host));
}
for(int i=0; i<mx.length; i++) {
//if (deadHosts.contains(mx[i])) continue;
- if (attempt(m, mx[i])) { return true; }
+ if (attempt(m, mx[i])) return true;
}
return false;
}
private static void check(String s, Connection conn) {
if (s==null) return;
while (s.length() > 3 && s.charAt(3) == '-') s = conn.readln();
- if (s.startsWith("4")||s.startsWith("5")) throw new SMTPException(s);
+ //if (s.startsWith("4")||s.startsWith("5")) throw new SMTPException(s);
+ if (!s.startsWith("2")&&!s.startsWith("3")) throw new SMTPException(s);
}
private static boolean attempt(final Message m, final InetAddress mx) {
boolean accepted = false;
Connection conn = null;
try {
conn = new Connection(new Socket(mx, 25), InetAddress.getLocalHost().getHostName());
+ InetAddress localAddress = conn.getSocket().getLocalAddress();
+ String reverse = DNSUtil.reverseLookup(localAddress);
+ Log.info(SMTP.Outgoing.class,
+ "outbound connection to " + mx + " uses " + localAddress + " [reverse: " + reverse + "]");
+ InetAddress relookup = InetAddress.getByName(reverse);
+ if (!relookup.equals(localAddress))
+ Log.error(SMTP.Outgoing.class,
+ "Warning: local machine fails forward-confirmed-reverse; " +
+ reverse + " resolves to " + localAddress);
conn.setNewline("\r\n");
conn.setTimeout(60 * 1000);
check(conn.readln(), conn); // banner
try {
- conn.println("EHLO " + conn.vhost);
+ conn.println("EHLO " + reverse);
check(conn.readln(), conn);
} catch (SMTPException smtpe) {
- conn.println("HELO " + conn.vhost);
+ conn.println("HELO " + reverse);
check(conn.readln(), conn);
}
String envelopeFrom = m.envelopeFrom==null ? "" : m.envelopeFrom.toString();
conn.println("RCPT TO:<" + m.envelopeTo.toString()+">"); check(conn.readln(), conn);
conn.println("DATA"); check(conn.readln(), conn);
- Headers head = m.headers;
- head = head.remove("return-path");
- head = head.remove("bcc");
+ Headers head = new Headers(m.headers,
+ new String[] {
+ "return-path", null,
+ "bcc", null
+ });
Stream stream = head.getStream();
for(String s = stream.readln(); s!=null; s=stream.readln()) {
if (s.startsWith(".")) conn.print(".");
Log.warn(SMTP.Outgoing.class, " unable to send; error=" + e);
Log.warn(SMTP.Outgoing.class, " message: " + m.summary());
Log.warn(SMTP.Outgoing.class, e);
+ /*
+ // FIXME: we should not be bouncing here!
if (e.code >= 500 && e.code <= 599) {
try {
attempt(m.bounce("unable to deliver: " + e), true);
}
return true;
}
+ */
return false;
} catch (Exception e) {
if (accepted) return true;
Log.warn(SMTP.Outgoing.class, " unable to send; error=" + e);
Log.warn(SMTP.Outgoing.class, " message: " + m.summary());
Log.warn(SMTP.Outgoing.class, e);
- if (conn != null) Log.warn(SMTP.Outgoing.class, conn.dumpLog());
+ //if (conn != null) Log.warn(SMTP.Outgoing.class, conn.dumpLog());
return false;
} finally {
if (conn != null) conn.close();
return accepted;
}
- private static HashSet<Outgoing> threads = new HashSet<Outgoing>();
- private static int serials = 1;
- private int serial = serials++;
- private Mailbox.Iterator it;
-
- public Outgoing() {
- synchronized(Outgoing.class) {
- threads.add(this);
- }
- }
-
public void wake() {
int count = spool.count(Query.all());
- Log.info(SMTP.Outgoing.class, "outgoing thread #"+serial+" woke up; " + count + " messages to send");
+ Log.info(SMTP.Outgoing.class, "outgoing thread "+name+" woke up; " + count + " messages to send");
try {
while(true) {
boolean good = false;
}
if (!good) break;
try {
- if (attempt(it.cur())) it.delete();
+ String messageid = it.cur().messageid;
+ if (nextTry.get(messageid) == null || System.currentTimeMillis() > nextTry.get(messageid)) {
+ boolean ok = attempt(it.cur());
+ if (ok) it.delete();
+ else nextTry.put(messageid, System.currentTimeMillis() + RETRY_TIME);
+ }
} catch (Exception e) {
Log.error(SMTP.Outgoing.class, e);
}
//if (e instanceof InterruptedException) throw e;
Log.error(SMTP.Outgoing.class, e);
}
- Log.info(SMTP.Outgoing.class, "outgoing thread #"+serial+" going back to sleep");
+ Log.info(SMTP.Outgoing.class, "outgoing thread #"+name+" going back to sleep");
it = null;
}
public void run() {
try {
while(true) {
- Log.setThreadAnnotation("[outgoing #"+serial+"] ");
+ Log.setThreadAnnotation("[outgoing #"+name+"] ");
wake();
Thread.sleep(1000);
synchronized(Outgoing.class) {
}
}
- public static InetAddress[] getMailExchangerIPs(String hostName) {
- InetAddress[] ret;
- try {
- Hashtable env = new Hashtable();
- env.put("java.naming.factory.initial", "com.sun.jndi.dns.DnsContextFactory");
- DirContext ictx = new InitialDirContext(env);
- Attributes attrs = ictx.getAttributes(hostName, new String[] { "MX" });
- Attribute attr = attrs.get("MX");
- if (attr == null) {
- ret = new InetAddress[1];
- try {
- ret[0] = InetAddress.getByName(hostName);
- if (ret[0].equals(IP.getIP(127,0,0,1)) || ret[0].isLoopbackAddress()) throw new UnknownHostException();
- return ret;
- } catch (UnknownHostException uhe) {
- Log.warn(SMTP.class, "no MX hosts or A record for " + hostName);
- return new InetAddress[0];
- }
- } else {
- ret = new InetAddress[attr.size()];
- NamingEnumeration ne = attr.getAll();
- for(int i=0; ne.hasMore();) {
- String mx = (String)ne.next();
- // FIXME we should be sorting here
- mx = mx.substring(mx.indexOf(" ") + 1);
- if (mx.charAt(mx.length() - 1) == '.') mx = mx.substring(0, mx.length() - 1);
- InetAddress ia = InetAddress.getByName(mx);
- if (ia.equals(IP.getIP(127,0,0,1)) || ia.isLoopbackAddress()) continue;
- ret[i++] = ia;
- }
- }
- } catch (Exception e) {
- Log.warn(SMTP.class, "couldn't find MX host for " + hostName + " due to");
- Log.warn(SMTP.class, e);
- return new InetAddress[0];
- }
- return ret;
- }
}
projects / org.ibex.mail.git / blobdiff