1 // Copyright 2000-2005 the Contributors, as shown in the revision logs.
2 // Licensed under the Apache Public Source License 2.0 ("the License").
3 // You may not use this file except in compliance with the License.
6 import org.ibex.mail.target.*;
7 import org.ibex.util.*;
15 // FIXME: inbound throttling/ratelimiting
16 // FIXME: probably need some throttling on outbound mail
17 // FEATURE: rate-limiting
19 // "Address enumeration detection" -- notice when it looks like somebody
20 // is trying a raft of addresses.
22 // RFC2554: SMTP Service Extension for Authentication
23 // - did not implement section 5, though
24 // RFC4616: SASL PLAIN
26 // Note: we can't actually use status codes for feedback if we accept
27 // multiple destination addresses... a failure on one and success on
30 // FIXME: logging: current logging sucks
31 // FIXME: loop prevention
33 // FEATURE: infer date if not present
36 // FEATURE: RFC2822, section 4.5.1: special "postmaster" address
37 Any system that includes an SMTP server supporting mail relaying or
38 delivery MUST support the reserved mailbox "postmaster" as a case-
39 insensitive local name. This postmaster address is not strictly
40 necessary if the server always returns 554 on connection opening (as
41 described in section 3.1). The requirement to accept mail for
42 postmaster implies that RCPT commands which specify a mailbox for
43 postmaster at any of the domains for which the SMTP server provides
44 mail service, as well as the special case of "RCPT TO:<Postmaster>"
45 (with no domain specification), MUST be supported.
48 // FEATURE: RFC2822, section 5, multiple MX records, preferences, ordering
49 // FEATURE: RFC2822, end of 4.1.2: backslashes in headers
50 // FEATURE: batching retrys by host (retry multiple in one session, keep retry intervals on a host basis not a message basis)
51 // FEATURE: first two attempts should be close together (rec'd by 2821)
53 // FEATURE: RFC2822, section 4.5.4.1: retry strategies
54 // per-command, per-attempt timeouts
55 Experience suggests that failures are typically transient (the target
56 system or its connection has crashed), favoring a policy of two
57 connection attempts in the first hour the message is in the queue,
58 and then backing off to one every two or three hours.
60 The SMTP client can shorten the queuing delay in cooperation with the
61 SMTP server. For example, if mail is received from a particular
62 address, it is likely that mail queued for that host can now be sent.
63 Application of this principle may, in many cases, eliminate the
64 requirement for an explicit "send queues now" function such as ETRN
67 An SMTP client may have a large queue of messages for each
68 unavailable destination host. If all of these messages were retried
69 in every retry cycle, there would be excessive Internet overhead and
70 the sending system would be blocked for a long period. Note that an
71 SMTP client can generally determine that a delivery attempt has
72 failed only after a timeout of several minutes and even a one-minute
73 timeout per connection will result in a very large delay if retries
74 are repeated for dozens, or even hundreds, of queued messages to the
77 When a mail message is to be delivered to multiple recipients, and
78 the SMTP server to which a copy of the message is to be sent is the
79 same for multiple recipients, then only one copy of the message
80 SHOULD be transmitted. That is, the SMTP client SHOULD use the
81 command sequence: MAIL, RCPT, RCPT,... RCPT, DATA instead of the
82 sequence: MAIL, RCPT, DATA, ..., MAIL, RCPT, DATA. However, if there
83 are very many addresses, a limit on the number of RCPT commands per
84 MAIL command MAY be imposed. Implementation of this efficiency
85 feature is strongly encouraged.
89 public static final SimpleDateFormat dateFormat = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss Z");
91 private static final SqliteMailbox allmail =
92 (SqliteMailbox)FileBasedMailbox
93 .getFileBasedMailbox("/afs/megacz.com/mail/user/megacz/allmail.sqlite", false);
95 public static final int NUM_OUTGOING_THREADS = 5;
96 public static final int GRAYLIST_MINWAIT = 1000 * 60 * 60; // one hour
97 public static final int GRAYLIST_MAXWAIT = 1000 * 60 * 60 * 24 * 5; // five days
98 public static final int MAX_MESSAGE_SIZE = Integer.parseInt(System.getProperty("org.ibex.mail.smtp.maxMessageSize", "-1"));
99 public static final int RETRY_TIME = 1000 * 60 * 30; // 30min recommended by RFC
100 public static final int GIVE_UP_TIME = 1000 * 60 * 24 * 5; // FIXME: actually use this
102 public static final Graylist graylist;
103 public static final Whitelist whitelist;
106 graylist = new Graylist(Mailbox.STORAGE_ROOT+"/db/graylist.sqlite");
107 whitelist = new Whitelist(Mailbox.STORAGE_ROOT+"/db/whitelist.sqlite");
108 } catch (Exception e) {
109 throw new RuntimeException(e);
113 private static final Mailbox spool =
114 FileBasedMailbox.getFileBasedMailbox(Mailbox.STORAGE_ROOT,false).slash("spool",true).slash("smtp",true).getMailbox();
117 public static void enqueue(Message m) throws IOException {
118 if (!m.envelopeTo.isLocal()) Outgoing.enqueue(m);
122 } catch (Exception e) {
123 // FIXME incredibly gross hack
124 if (e.toString().indexOf("attempt to insert two messages with identical messageid")==-1)
125 Log.error(SMTP.class, e);
127 Target.root.accept(m);
131 public static class SMTPException extends MailException {
134 public SMTPException(String s) {
136 code = Integer.parseInt(s.substring(0, s.indexOf(' ')));
137 message = s.substring(s.indexOf(' ')+1);
138 } catch (NumberFormatException nfe) {
143 public String toString() { return "SMTP " + code + ": " + message; }
144 public String getMessage() { return toString(); }
147 // Server //////////////////////////////////////////////////////////////////////////////
149 public static class Server {
150 public void handleRequest(Connection conn) throws IOException {
151 conn.setTimeout(5 * 60 * 1000);
152 conn.setNewline("\r\n");
153 conn.println("220 " + conn.vhost + " ESMTP " + this.getClass().getName());
155 Vector to = new Vector();
156 boolean ehlo = false;
157 String remotehost = null;
158 String authenticatedAs = null;
159 int failedRcptCount = 0;
160 for(String command = conn.readln(); ; command = conn.readln()) try {
161 if (command == null) return;
162 Log.warn("**"+conn.getRemoteAddress()+"**", command);
163 String c = command.toUpperCase();
164 if (c.startsWith("HELO")) {
165 remotehost = c.substring(5).trim();
166 conn.println("250 HELO " + conn.vhost);
167 from = null; to = new Vector();
168 } else if (c.startsWith("EHLO")) {
169 remotehost = c.substring(5).trim();
170 conn.println("250-"+conn.vhost);
171 //conn.println("250-AUTH");
172 conn.println("250-AUTH PLAIN");
173 //conn.println("250-STARTTLS");
174 conn.println("250 HELP");
176 from = null; to = new Vector();
177 } else if (c.startsWith("RSET")) { conn.println("250 reset ok"); from = null; to = new Vector();
178 } else if (c.startsWith("HELP")) { conn.println("214 you are beyond help. see a trained professional.");
179 } else if (c.startsWith("VRFY")) { conn.println("502 VRFY not supported");
180 } else if (c.startsWith("EXPN")) { conn.println("502 EXPN not supported");
181 } else if (c.startsWith("NOOP")) { conn.println("250 OK");
182 } else if (c.startsWith("QUIT")) { conn.println("221 " + conn.vhost + " closing connection"); return;
183 } else if (c.startsWith("STARTTLS")) {
184 conn.println("220 starting TLS...");
186 conn = conn.negotiateSSL(true);
187 from = null; to = new Vector();
188 } else if (c.startsWith("AUTH")) {
189 if (authenticatedAs != null) {
190 conn.println("503 you are already authenticated; you must reconnect to reauth");
192 String mechanism = command.substring(4).trim();
194 if (mechanism.indexOf(' ')!=-1) {
195 rest = mechanism.substring(mechanism.indexOf(' ')+1).trim();
196 mechanism = mechanism.substring(0, mechanism.indexOf(' '));
198 if (mechanism.equals("PLAIN")) {
199 // 538 Encryption required for requested authentication mechanism?
200 byte[] bytes = Encode.fromBase64(rest);
201 String authenticateUser = null;
202 String authorizeUser = null;
203 String password = null;
205 for(int i=0; i<=bytes.length; i++) {
206 if (i<bytes.length && bytes[i]!=0) continue;
207 String result = new String(bytes, start, i-start, "UTF-8");
208 if (authenticateUser==null) authenticateUser = result;
209 else if (authorizeUser==null) authorizeUser = result;
210 else if (password==null) password = result;
213 // FIXME: be smarter here
214 if (Main.auth.login(authorizeUser, password)!=null)
215 authenticatedAs = authenticateUser;
216 conn.println("235 Authentication successful");
218 } else if (mechanism.equals("CRAM-MD5")) {
220 conn.println("334 "+challenge);
221 String resp = conn.readln();
222 if (resp.equals("*")) {
223 conn.println("501 client requested AUTH cancellation");
225 } else if (mechanism.equals("ANONYMOUS")) {
226 } else if (mechanism.equals("EXTERNAL")) {
227 } else if (mechanism.equals("DIGEST-MD5")) {
230 conn.println("504 unrecognized authentication type");
232 // on success, reset to initial state; client will EHLO again
233 from = null; to = new Vector();
235 } else if (c.startsWith("MAIL FROM:")) {
236 command = command.substring(10).trim();
237 from = command.equals("<>") ? null : new Address(command);
238 conn.println("250 " + from + " is syntactically correct");
239 // Don't perform SAV; discouraged here
240 // http://blog.fastmail.fm/2007/12/05/sending-email-servers-best-practice/
241 } else if (c.startsWith("RCPT TO:")) {
242 // some clients are broken and put RCPT first; we will tolerate this
243 command = command.substring(8).trim();
244 if(command.indexOf(' ') != -1) command = command.substring(0, command.indexOf(' '));
245 Address addr = new Address(command);
246 if (conn.getRemoteAddress().isLoopbackAddress() || (from!=null&&from.toString().indexOf("johnw")!=-1)) {
247 conn.println("250 you are connected locally, so I will let you send");
249 if (!whitelist.isWhitelisted(addr))
250 whitelist.addWhitelist(addr);
251 } else if (authenticatedAs!=null) {
252 conn.println("250 you are authenticated as "+authenticatedAs+", so I will let you send");
254 if (!whitelist.isWhitelisted(addr))
255 whitelist.addWhitelist(addr);
256 } else if (addr.isLocal()) {
258 conn.println("536 sorry, limit on 3 RCPT TO's per DATA");
260 // FEATURE: should check the address further and give 550 if undeliverable
261 conn.println("250 " + addr + " is on this machine; I will deliver it");
265 conn.println("535 sorry, " + addr + " is not on this machine, you are not connected from localhost, and I will not relay without SMTP AUTH");
266 Log.warn("","535 sorry, " + addr + " is not on this machine, you are not connected from localhost, and I will not relay without SMTP AUTH");
268 if (failedRcptCount > 3) {
274 } else if (c.startsWith("DATA")) {
275 //if (from == null) { conn.println("503 MAIL FROM command must precede DATA"); continue; }
276 if (to == null || to.size()==0) { conn.println("503 RCPT TO command must precede DATA"); continue; }
277 if (!graylist.isWhitelisted(conn.getRemoteAddress()) && !conn.getRemoteAddress().isLoopbackAddress() && authenticatedAs==null) {
278 long when = graylist.getGrayListTimestamp(conn.getRemoteAddress(), from+"", to+"");
279 if (when == 0 || System.currentTimeMillis() - when > GRAYLIST_MAXWAIT) {
280 graylist.setGrayListTimestamp(conn.getRemoteAddress(), from+"", to+"", System.currentTimeMillis());
281 conn.println("451 you are graylisted; please try back in one hour to be whitelisted");
282 Log.warn(conn.getRemoteAddress().toString(), "451 you are graylisted; please try back in one hour to be whitelisted");
285 } else if (System.currentTimeMillis() - when > GRAYLIST_MINWAIT) {
286 graylist.addWhitelist(conn.getRemoteAddress());
287 conn.println("354 (you have been whitelisted) Enter message, ending with \".\" on a line by itself");
288 Log.warn(conn.getRemoteAddress().toString(), "has been whitelisted");
290 conn.println("451 you are still graylisted (since "+new java.util.Date(when)+")");
292 Log.warn(conn.getRemoteAddress().toString(), "451 you are still graylisted (since "+new java.util.Date(when)+")");
296 conn.println("354 Enter message, ending with \".\" on a line by itself");
300 // FIXME: deal with messages larger than memory here?
301 StringBuffer buf = new StringBuffer();
302 buf.append("Received: from " + conn.getRemoteHostname() + " (" + remotehost + ")\r\n");
303 buf.append(" by "+conn.vhost+" ("+SMTP.class.getName()+") with "+(ehlo?"ESMTP":"SMTP") + "\r\n");
305 // FIXME: this is leaking BCC addrs
306 // for(int i=0; i<to.size(); i++) buf.append(to.elementAt(i) + " ");
307 buf.append("; " + dateFormat.format(new Date()) + "\r\n");
309 // FIXME: some sort of stream transformer here?
311 String s = conn.readln();
312 if (s == null) throw new RuntimeException("connection closed");
313 if (s.equals(".")) break;
314 if (s.startsWith(".")) s = s.substring(1);
315 buf.append(s + "\r\n");
316 if (MAX_MESSAGE_SIZE != -1 && buf.length() > MAX_MESSAGE_SIZE && (from+"").indexOf("paperless")==-1) {
317 Log.error("**"+conn.getRemoteAddress()+"**",
318 "sorry, this mail server only accepts messages of less than " +
319 ByteSize.toString(MAX_MESSAGE_SIZE));
320 throw new MailException.Malformed("sorry, this mail server only accepts messages of less than " +
321 ByteSize.toString(MAX_MESSAGE_SIZE));
324 String message = buf.toString();
326 for(int i=0; i<to.size(); i++)
327 enqueue(m = Message.newMessage(Fountain.Util.create(message)).withEnvelope(from, (Address)to.elementAt(i)));
328 if (m != null) Log.info(SMTP.class, "accepted message: " + m.summary());
329 conn.println("250 message accepted");
331 from = null; to = new Vector();
332 } catch (MailException.Malformed mfe) { conn.println("501 " + mfe.toString());
333 } catch (MailException.MailboxFull mbf) { conn.println("452 " + mbf);
334 } catch (Script.Later.LaterException le) { conn.println("453 try again later");
335 } catch (Script.Reject.RejectException re) {
336 Log.warn(SMTP.class, "rejecting message due to: " + re.reason + "\n " + re.m.summary());
337 conn.println("501 " + re.reason);
339 } else { conn.println("500 unrecognized command"); }
340 } catch (Message.Malformed e) { conn.println("501 " + e.toString()); }
345 // Outgoing Mail Thread //////////////////////////////////////////////////////////////////////////////
348 for(int i=0; i<NUM_OUTGOING_THREADS; i++)
349 new Outgoing("#"+i).start();
352 public static class Outgoing extends Thread {
354 private static HashSet<Outgoing> threads = new HashSet<Outgoing>();
355 private static final HashMap deadHosts = new HashMap();
356 private static Map<String,Long> nextTry = Collections.synchronizedMap(new HashMap<String,Long>());
358 private Mailbox.Iterator it;
359 private final String name;
361 public Outgoing(String name) {
363 synchronized(Outgoing.class) {
368 public String toString() { return name; }
370 public static void enqueue(Message m) throws IOException {
371 if (m == null) { Log.warn(Outgoing.class, "attempted to enqueue(null)"); return; }
372 String traces = m.headers.get("Received");
375 for(int i=0; i<traces.length(); i++)
376 if (traces.charAt(i)=='\n' || traces.charAt(i)=='\r')
378 if (lines > 100) { // required by rfc
379 Log.warn(SMTP.Outgoing.class, "Message with " + lines + " trace hops; dropping\n" + m.summary());
383 synchronized(Outgoing.class) {
384 spool.insert(m, Mailbox.Flag.defaultFlags);
385 Outgoing.class.notifyAll();
389 public static boolean attempt(Message m) throws IOException { return attempt(m, false); }
390 public static boolean attempt(Message m, boolean noBounces) throws IOException {
391 if (m.envelopeTo == null) {
392 Log.warn(SMTP.Outgoing.class, "aieeee, null envelopeTo: " + m.summary());
395 InetAddress[] mx = DNSUtil.getMailExchangerIPs(m.envelopeTo.host);
396 if (mx.length == 0) {
398 enqueue(m.bounce("could not resolve " + m.envelopeTo.host));
401 Log.warn(SMTP.Outgoing.class, "could not resolve " + m.envelopeTo.host);
405 if (new Date().getTime() - m.arrival.getTime() > 1000 * 60 * 60 * 24 * 5) {
407 enqueue(m.bounce("could not send for 5 days"));
410 Log.warn(SMTP.Outgoing.class, "could not send for 5 days: " + m.summary());
414 for(int i=0; i<mx.length; i++) {
415 //if (deadHosts.contains(mx[i])) continue;
416 if (attempt(m, mx[i])) return true;
421 private static void check(String s, Connection conn) {
423 while (s.length() > 3 && s.charAt(3) == '-') s = conn.readln();
424 //if (s.startsWith("4")||s.startsWith("5")) throw new SMTPException(s);
425 if (!s.startsWith("2")&&!s.startsWith("3")) throw new SMTPException(s);
427 private static boolean attempt(final Message m, final InetAddress mx) {
428 boolean accepted = false;
429 Connection conn = null;
431 conn = new Connection(new Socket(mx, 25), InetAddress.getLocalHost().getHostName());
432 InetAddress localAddress = conn.getSocket().getLocalAddress();
433 String reverse = DNSUtil.reverseLookup(localAddress);
434 Log.info(SMTP.Outgoing.class,
435 "outbound connection to " + mx + " uses " + localAddress + " [reverse: " + reverse + "]");
436 InetAddress relookup = InetAddress.getByName(reverse);
437 if (!relookup.equals(localAddress))
438 Log.error(SMTP.Outgoing.class,
439 "Warning: local machine fails forward-confirmed-reverse; " +
440 reverse + " resolves to " + localAddress);
441 conn.setNewline("\r\n");
442 conn.setTimeout(60 * 1000);
443 check(conn.readln(), conn); // banner
445 conn.println("EHLO " + reverse);
446 check(conn.readln(), conn);
447 } catch (SMTPException smtpe) {
448 conn.println("HELO " + reverse);
449 check(conn.readln(), conn);
451 String envelopeFrom = m.envelopeFrom==null ? "" : m.envelopeFrom.toString();
452 conn.println("MAIL FROM:<" + envelopeFrom +">"); check(conn.readln(), conn);
453 conn.println("RCPT TO:<" + m.envelopeTo.toString()+">"); check(conn.readln(), conn);
454 conn.println("DATA"); check(conn.readln(), conn);
456 Headers head = new Headers(m.headers,
461 Stream stream = head.getStream();
462 for(String s = stream.readln(); s!=null; s=stream.readln()) {
463 if (s.startsWith(".")) conn.print(".");
467 stream = m.getBody().getStream();
468 for(String s = stream.readln(); s!=null; s=stream.readln()) {
469 if (s.startsWith(".")) conn.print(".");
473 String resp = conn.readln();
475 throw new SMTPException("server " + mx + " closed connection without accepting message");
477 Log.warn(SMTP.Outgoing.class, "success: " + mx + " accepted " + m.summary() + "\n["+resp+"]");
480 } catch (SMTPException e) {
481 if (accepted) return true;
482 Log.warn(SMTP.Outgoing.class, " unable to send; error=" + e);
483 Log.warn(SMTP.Outgoing.class, " message: " + m.summary());
484 Log.warn(SMTP.Outgoing.class, e);
486 // FIXME: we should not be bouncing here!
487 if (e.code >= 500 && e.code <= 599) {
489 attempt(m.bounce("unable to deliver: " + e), true);
490 } catch (Exception ex) {
491 Log.error(SMTP.Outgoing.class, "exception while trying to deliver bounce; giving up completely");
492 Log.error(SMTP.Outgoing.class, ex);
498 } catch (Exception e) {
499 if (accepted) return true;
500 Log.warn(SMTP.Outgoing.class, " unable to send; error=" + e);
501 Log.warn(SMTP.Outgoing.class, " message: " + m.summary());
502 Log.warn(SMTP.Outgoing.class, e);
503 //if (conn != null) Log.warn(SMTP.Outgoing.class, conn.dumpLog());
506 if (conn != null) conn.close();
512 int count = spool.count(Query.all());
513 Log.info(SMTP.Outgoing.class, "outgoing thread "+name+" woke up; " + count + " messages to send");
516 boolean good = false;
517 synchronized(Outgoing.class) {
518 it = spool.iterator();
519 OUTER: for(; it.next(); ) {
520 for(Outgoing o : threads)
521 if (o!=this && o.it != null && o.it.uid()==it.uid())
529 String messageid = it.cur().messageid;
530 if (nextTry.get(messageid) == null || System.currentTimeMillis() > nextTry.get(messageid)) {
531 boolean ok = attempt(it.cur());
533 else nextTry.put(messageid, System.currentTimeMillis() + RETRY_TIME);
535 } catch (Exception e) {
536 Log.error(SMTP.Outgoing.class, e);
538 Log.info(this, "sleeping for 3s...");
541 } catch (Exception e) {
542 //if (e instanceof InterruptedException) throw e;
543 Log.error(SMTP.Outgoing.class, e);
545 Log.info(SMTP.Outgoing.class, "outgoing thread #"+name+" going back to sleep");
552 Log.setThreadAnnotation("[outgoing #"+name+"] ");
555 synchronized(Outgoing.class) {
556 Outgoing.class.wait(5 * 60 * 1000);
559 } catch (InterruptedException e) { Log.warn(this, e); }