1 // Copyright 2000-2005 the Contributors, as shown in the revision logs.
2 // Licensed under the Apache Public Source License 2.0 ("the License").
3 // You may not use this file except in compliance with the License.
10 import org.ibex.util.*;
11 import org.ibex.crypto.*;
14 * This object encapsulates a *single* HTTP connection. Multiple requests may be pipelined over a connection (thread-safe),
15 * although any IOException encountered in a request will invalidate all later requests.
19 public static InetAddress originAddr = null;
20 public static String originHost = null;
22 // FIXME: HACK: these shouldn't be set globally
23 public static String userAgent = "Ibex";
24 public static boolean allowRedirects = true;
26 // Cookies //////////////////////////////////////////////////////////////////////////////
28 public static class Cookie {
29 public final String name;
30 public final String value;
31 public final String domain;
32 public final String path;
33 public final Date expires;
34 public final boolean secure;
35 public Cookie(String name, String value, String domain, String path, Date expires, boolean secure) {
40 this.expires = expires;
44 // FIXME: this could be much more efficient
45 // FIXME currently only implements http://wp.netscape.com/newsref/std/cookie_spec.html
46 public static class Jar {
47 private Hash h = new Hash();
48 public String getCookieHeader(String domain, String path, boolean secure) {
49 StringBuffer ret = new StringBuffer("Cookie: ");
50 Enumeration e = h.enumerateKeys();
51 while (e.hasMoreElements()) {
52 Vec v = (Vec)h.get(e.nextElement());
54 for(int i=0; i<v.size(); i++) {
55 Cookie c = (Cookie)v.elementAt(i);
56 if (domain.endsWith(c.domain) &&
57 (c.path == null || path.startsWith(c.path)) &&
58 (cookie == null || c.domain.length() > cookie.domain.length()))
62 ret.append(cookie.name);
64 ret.append(cookie.value);
68 //ret.setLength(ret.length() - 2);
69 return ret.toString();
71 public void setCookie(String header, String defaultDomain) {
74 String domain = defaultDomain;
77 boolean secure = false;
78 StringTokenizer st = new StringTokenizer(header, ";");
79 while(st.hasMoreTokens()) {
80 String s = st.nextToken();
81 if (s.indexOf('=') == -1) {
82 if (s.equals("secure")) secure = true;
85 String start = s.substring(0, s.indexOf('='));
86 String end = s.substring(s.indexOf('=')+1);
92 if (start.toLowerCase().equals("domain")) domain = end;
93 else if (start.toLowerCase().equals("path")) path = end;
94 else if (start.toLowerCase().equals("expires")) expires = new Date(end);
96 if (h.get(name) == null) h.put(name, new Vec());
97 ((Vec)h.get(name)).addElement(new Cookie(name, value, domain, path, expires, secure));
102 // Public Methods ////////////////////////////////////////////////////////////////////////////////////////
104 public HTTP(String url) { this(url, false); }
105 public HTTP(String url, boolean skipResolveCheck) { this(url, skipResolveCheck, null); }
106 public HTTP(String url, Proxy p) { this(url, false, p); }
107 public HTTP(String url, boolean skipResolveCheck, Proxy p) { originalUrl = url; this.skipResolveCheck = skipResolveCheck; proxy = p; }
109 /** Performs an HTTP GET request */
110 public InputStream GET(String referer, Cookie.Jar cookies) throws IOException {
111 return makeRequest(null, null, referer, cookies); }
113 /** Performs an HTTP POST request; content is additional headers, blank line, and body */
114 public InputStream POST(String contentType, String content, String referer, Cookie.Jar cookies) throws IOException {
115 return makeRequest(contentType, content, referer, cookies); }
117 public static class HTTPException extends IOException { public HTTPException(String s) { super(s); } }
119 public static HTTP stdio = new HTTP("stdio:");
122 // Statics ///////////////////////////////////////////////////////////////////////////////////////////////
124 static Hash resolvedHosts = new Hash(); ///< cache for resolveAndCheckIfFirewalled()
125 private static Hash authCache = new Hash(); ///< cache of userInfo strings, keyed on originalUrl
128 // Instance Data ///////////////////////////////////////////////////////////////////////////////////////////////
130 final String originalUrl; ///< the URL as passed to the original constructor; this is never changed
131 String url = null; ///< the URL to connect to; this is munged when the url is parsed */
132 String host = null; ///< the host to connect to
133 int port = -1; ///< the port to connect on
134 boolean ssl = false; ///< true if SSL (HTTPS) should be used
135 String path = null; ///< the path (URI) to retrieve on the server
136 Socket sock = null; ///< the socket
137 InputStream in = null; ///< the socket's inputstream
138 String userInfo = null; ///< the username and password portions of the URL
139 boolean firstRequest = true; ///< true iff this is the first request to be made on this socket
140 boolean skipResolveCheck = false; ///< allowed to skip the resolve check when downloading PAC script
141 boolean proxied = false; ///< true iff we're using a proxy
144 /** this is null if the current request is the first request on
145 * this HTTP connection; otherwise it is a Semaphore which will be
146 * released once the request ahead of us has recieved its response
148 Semaphore okToRecieve = null;
151 * This method isn't synchronized; however, only one thread can be in the inner synchronized block at a time, and the rest of
152 * the method is protected by in-order one-at-a-time semaphore lock-steps
154 private InputStream makeRequest(String contentType, String content, String referer, Cookie.Jar cookies) throws IOException {
156 // Step 1: send the request and establish a semaphore to stop any requests that pipeline after us
157 Semaphore blockOn = null;
158 Semaphore releaseMe = null;
162 sendRequest(contentType, content, referer, cookies);
163 } catch (IOException e) {
167 blockOn = okToRecieve;
168 releaseMe = okToRecieve = new Semaphore();
171 // Step 2: wait for requests ahead of us to complete, then read the reply off the stream
172 boolean doRelease = true;
174 if (blockOn != null) blockOn.block();
176 // previous call wrecked the socket connection, but we already sent our request, so we can't just retry --
177 // this could cause the server to receive the request twice, which could be bad (think of the case where the
178 // server call causes Amazon.com to ship you an item with one-click purchasing).
180 throw new HTTPException("a previous pipelined call messed up the socket");
182 Hashtable h = in == null ? null : parseHeaders(in, cookies);
184 if (firstRequest) throw new HTTPException("server closed the socket with no response");
185 // sometimes the server chooses to close the stream between requests
188 return makeRequest(contentType, content, referer, cookies);
191 String reply = h.get("STATUSLINE").toString();
193 if (reply.startsWith("407") || reply.startsWith("401")) {
195 if (reply.startsWith("407")) doProxyAuth(h, content == null ? "GET" : "POST");
196 else doWebAuth(h, content == null ? "GET" : "POST");
198 if (h.get("HTTP").equals("1.0") && h.get("content-length") == null) {
199 if (Log.on) Log.info(this, "proxy returned an HTTP/1.0 reply with no content-length...");
202 int cl = h.get("content-length") == null ? -1 : Integer.parseInt(h.get("content-length").toString());
203 new HTTPInputStream(in, cl, releaseMe).close();
206 return makeRequest(contentType, content, referer, cookies);
208 } else if (reply.startsWith("3") && allowRedirects) {
209 String location = (String)h.get("location");
210 if (location == null) throw new HTTPException("Got HTTP " + reply.substring(0, 3) + " but no Location header");
211 Log.info(HTTP.class, "redirecting to " + location);
213 return new HTTP(location).POST(contentType, content, url, cookies);
215 return new HTTP(location).GET(url, cookies);
217 } else if (reply.startsWith("2")) {
218 if (h.get("HTTP").equals("1.0") && h.get("content-length") == null)
219 throw new HTTPException("Ibex does not support HTTP/1.0 servers which fail to return the Content-Length header");
220 int cl = h.get("content-length") == null ? -1 : Integer.parseInt(h.get("content-length").toString());
221 InputStream ret = new HTTPInputStream(in, cl, releaseMe);
222 if ("gzip".equals(h.get("content-encoding"))) ret = new java.util.zip.GZIPInputStream(ret);
227 throw new HTTPException("HTTP Error: " + reply);
231 } catch (IOException e) { reset(); throw e;
232 } finally { if (doRelease) releaseMe.release();
237 // Safeguarded DNS Resolver ///////////////////////////////////////////////////////////////////////////
240 * resolves the hostname and returns it as a string in the form "x.y.z.w"
241 * @throws HTTPException if the host falls within a firewalled netblock
243 private void resolveAndCheckIfFirewalled(String host) throws HTTPException {
246 if (resolvedHosts.get(host) != null) return;
248 // if all scripts are trustworthy (local FS), continue
249 if (originAddr == null) return;
253 InetAddress addr = InetAddress.getByName(host);
254 byte[] quadbyte = addr.getAddress();
255 if ((quadbyte[0] == 10 ||
256 (quadbyte[0] == 192 && quadbyte[1] == 168) ||
257 (quadbyte[0] == 172 && (quadbyte[1] & 0xF0) == 16)) && !addr.equals(originAddr))
258 throw new HTTPException("security violation: " + host + " [" + addr.getHostAddress() +
259 "] is in a firewalled netblock");
261 } catch (UnknownHostException uhe) { }
264 if (Platform.detectProxy() == null)
265 throw new HTTPException("could not resolve hostname \"" + host + "\" and no proxy configured");
270 // Methods to attempt socket creation /////////////////////////////////////////////////////////////////
272 private Socket getSocket(String host, int port, boolean ssl, boolean negotiate) throws IOException {
273 Socket ret = ssl ? new SSL(host, port, negotiate) : new Socket(java.net.InetAddress.getByName(host), port);
274 ret.setTcpNoDelay(true);
278 /** Attempts a direct connection */
279 Socket attemptDirect() {
281 Log.info(this, "attempting to create unproxied socket to " +
282 host + ":" + port + (ssl ? " [ssl]" : ""));
283 return getSocket(host, port, ssl, true);
284 } catch (IOException e) {
285 if (Log.on) Log.info(this, "exception in attemptDirect(): " + e);
292 // Everything Else ////////////////////////////////////////////////////////////////////////////
294 private synchronized void connect() throws IOException {
295 if (originalUrl.equals("stdio:")) { in = new BufferedInputStream(System.in); return; }
297 if (in == null) in = new BufferedInputStream(sock.getInputStream());
300 // grab the userinfo; gcj doesn't have java.net.URL.getUserInfo()
301 String url = originalUrl;
302 userInfo = url.substring(url.indexOf("://") + 3);
303 userInfo = userInfo.indexOf('/') == -1 ? userInfo : userInfo.substring(0, userInfo.indexOf('/'));
304 if (userInfo.indexOf('@') != -1) {
305 userInfo = userInfo.substring(0, userInfo.indexOf('@'));
306 url = url.substring(0, url.indexOf("://") + 3) + url.substring(url.indexOf('@') + 1);
311 if (url.startsWith("https:")) {
313 } else if (!url.startsWith("http:")) {
314 throw new IOException("HTTP only supports http/https urls");
316 if (url.indexOf("://") == -1) throw new IOException("URLs must contain a ://");
317 String temphost = url.substring(url.indexOf("://") + 3);
318 path = temphost.substring(temphost.indexOf('/'));
319 temphost = temphost.substring(0, temphost.indexOf('/'));
320 if (temphost.indexOf(':') != -1) {
321 port = Integer.parseInt(temphost.substring(temphost.indexOf(':')+1));
322 temphost = temphost.substring(0, temphost.indexOf(':'));
324 port = ssl ? 443 : 80;
326 if (!skipResolveCheck) resolveAndCheckIfFirewalled(temphost);
328 if (Log.verbose) Log.info(this, "creating HTTP object for connection to " + host + ":" + port);
330 proxied = proxy!=null;
331 if (proxied) sock = proxy.attempt(url, host, this);
334 sock = attemptDirect();
336 if (sock == null) throw new HTTPException("unable to contact host " + host);
337 if (in == null) in = new BufferedInputStream(sock.getInputStream());
340 private void sendRequest(String contentType, String content, String referer, Cookie.Jar cookies) throws IOException {
341 PrintWriter pw = new PrintWriter(new OutputStreamWriter(originalUrl.equals("stdio:") ?
342 System.out : sock.getOutputStream()));
343 if (content != null) {
344 pw.print("POST " + path + " HTTP/1.0\r\n"); // FIXME chunked encoding
345 int contentLength = content.substring(0, 2).equals("\r\n") ?
346 content.length() - 2 :
347 (content.length() - content.indexOf("\r\n\r\n") - 4);
348 pw.print("Content-Length: " + contentLength + "\r\n");
349 if (contentType != null) pw.print("Content-Type: " + contentType + "\r\n");
351 pw.print("GET " + path + " HTTP/1.1\r\n");
354 if (cookies != null) pw.print(cookies.getCookieHeader(host, path, ssl));
355 pw.print("User-Agent: " + userAgent + "\r\n");
356 pw.print("Accept-encoding: gzip\r\n");
357 pw.print("Host: " + (host + (port == 80 ? "" : (":" + port))) + "\r\n");
358 if (proxied) pw.print("X-RequestOrigin: " + originHost + "\r\n");
360 if (Proxy.Authorization.authorization != null) pw.print("Proxy-Authorization: "+Proxy.Authorization.authorization2+"\r\n");
361 if (authCache.get(originalUrl) != null) pw.print("Authorization: " + authCache.get(originalUrl) + "\r\n");
363 pw.print(content == null ? "\r\n" : content);
368 private void doWebAuth(Hashtable h0, String method) throws IOException {
369 if (userInfo == null) throw new HTTPException("web server demanded username/password, but none were supplied");
370 Hashtable h = parseAuthenticationChallenge(h0.get("www-authenticate").toString());
372 if (h.get("AUTHTYPE").equals("Basic")) {
373 if (authCache.get(originalUrl) != null) throw new HTTPException("username/password rejected");
374 authCache.put(originalUrl, "Basic " + new String(Encode.toBase64(userInfo.getBytes("UTF8"))));
376 } else if (h.get("AUTHTYPE").equals("Digest")) {
377 if (authCache.get(originalUrl) != null && !"true".equals(h.get("stale")))
378 throw new HTTPException("username/password rejected");
380 if (path2.startsWith("http://") || path2.startsWith("https://")) {
381 path2 = path2.substring(path2.indexOf("://") + 3);
382 path2 = path2.substring(path2.indexOf('/'));
384 String A1 = userInfo.substring(0, userInfo.indexOf(':')) + ":" + h.get("realm") + ":" +
385 userInfo.substring(userInfo.indexOf(':') + 1);
386 String A2 = method + ":" + path2;
387 authCache.put(originalUrl,
389 "username=\"" + userInfo.substring(0, userInfo.indexOf(':')) + "\", " +
390 "realm=\"" + h.get("realm") + "\", " +
391 "nonce=\"" + h.get("nonce") + "\", " +
392 "uri=\"" + path2 + "\", " +
393 (h.get("opaque") == null ? "" : ("opaque=\"" + h.get("opaque") + "\", ")) +
394 "response=\"" + H(H(A1) + ":" + h.get("nonce") + ":" + H(A2)) + "\", " +
399 throw new HTTPException("unknown authentication type: " + h.get("AUTHTYPE"));
403 private void doProxyAuth(Hashtable h0, String method) throws IOException {
404 if (Log.on) Log.info(this, "Proxy AuthChallenge: " + h0.get("proxy-authenticate"));
405 Hashtable h = parseAuthenticationChallenge(h0.get("proxy-authenticate").toString());
406 String style = h.get("AUTHTYPE").toString();
407 String realm = (String)h.get("realm");
409 if (style.equals("NTLM") && Proxy.Authorization.authorization2 == null) {
410 Log.info(this, "Proxy identified itself as NTLM, sending Type 1 packet");
411 Proxy.Authorization.authorization2 = "NTLM " + Encode.toBase64(Proxy.NTLM.type1);
415 if (!realm.equals("Digest") || Proxy.Authorization.authorization2 == null || !"true".equals(h.get("stale")))
416 Proxy.Authorization.getPassword(realm, style, sock.getInetAddress().getHostAddress(),
417 Proxy.Authorization.authorization);
419 if (style.equals("Basic")) {
420 Proxy.Authorization.authorization2 =
421 "Basic " + new String(Encode.toBase64(Proxy.Authorization.authorization.getBytes("UTF8")));
423 } else if (style.equals("Digest")) {
424 String A1 = Proxy.Authorization.authorization.substring(0, userInfo.indexOf(':')) + ":" + h.get("realm") + ":" +
425 Proxy.Authorization.authorization.substring(Proxy.Authorization.authorization.indexOf(':') + 1);
426 String A2 = method + ":" + path;
427 Proxy.Authorization.authorization2 =
429 "username=\"" + Proxy.Authorization.authorization.substring(0, Proxy.Authorization.authorization.indexOf(':')) +
431 "realm=\"" + h.get("realm") + "\", " +
432 "nonce=\"" + h.get("nonce") + "\", " +
433 "uri=\"" + path + "\", " +
434 (h.get("opaque") == null ? "" : ("opaque=\"" + h.get("opaque") + "\", ")) +
435 "response=\"" + H(H(A1) + ":" + h.get("nonce") + ":" + H(A2)) + "\", " +
438 } else if (style.equals("NTLM")) {
439 Log.info(this, "Proxy identified itself as NTLM, got Type 2 packet");
440 byte[] type2 = Encode.fromBase64(((String)h0.get("proxy-authenticate")).substring(5).trim());
441 for(int i=0; i<type2.length; i += 4) {
443 if (i<type2.length) log += Integer.toString(type2[i] & 0xff, 16) + " ";
444 if (i+1<type2.length) log += Integer.toString(type2[i+1] & 0xff, 16) + " ";
445 if (i+2<type2.length) log += Integer.toString(type2[i+2] & 0xff, 16) + " ";
446 if (i+3<type2.length) log += Integer.toString(type2[i+3] & 0xff, 16) + " ";
449 // FEATURE: need to keep the connection open between type1 and type3
450 // FEATURE: finish this
451 //byte[] type3 = Proxy.NTLM.getResponse(
452 //Proxy.Authorization.authorization2 = "NTLM " + Encode.toBase64(type3));
457 // HTTPInputStream ///////////////////////////////////////////////////////////////////////////////////
459 /** An input stream that represents a subset of a longer input stream. Supports HTTP chunking as well */
460 public class HTTPInputStream extends FilterInputStream implements KnownLength {
462 private int length = 0; ///< if chunking, numbytes left in this subset; else the remainder of the chunk
463 private Semaphore releaseMe = null; ///< this semaphore will be released when the stream is closed
464 boolean chunkedDone = false; ///< indicates that we have encountered the zero-length terminator chunk
465 boolean firstChunk = true; ///< if we're on the first chunk, we don't pre-read a CRLF
466 private int contentLength = 0; ///< the length of the entire content body; -1 if chunked
468 HTTPInputStream(InputStream in, int length, Semaphore releaseMe) throws IOException {
470 this.releaseMe = releaseMe;
471 this.contentLength = length;
472 this.length = length == -1 ? 0 : length;
475 public int getLength() { return contentLength; }
476 public boolean markSupported() { return false; }
477 public int read(byte[] b) throws IOException { return read(b, 0, b.length); }
478 public long skip(long n) throws IOException { return read(null, -1, (int)n); }
479 public int available() throws IOException {
480 if (contentLength == -1) return java.lang.Math.min(super.available(), length);
481 return super.available();
484 public int read() throws IOException {
485 byte[] b = new byte[1];
486 int ret = read(b, 0, 1);
487 return ret == -1 ? -1 : b[0] & 0xff;
490 private void readChunk() throws IOException {
491 if (chunkedDone) return;
492 if (!firstChunk) super.skip(2); // CRLF
494 String chunkLen = "";
496 int i = super.read();
497 if (i == -1) throw new HTTPException("encountered end of stream while reading chunk length");
499 // FEATURE: handle chunking extensions
507 length = Integer.parseInt(chunkLen.trim(), 16);
508 if (length == 0) chunkedDone = true;
511 public int read(byte[] b, int off, int len) throws IOException {
512 boolean good = false;
514 if (length == 0 && contentLength == -1) {
516 if (chunkedDone) { good = true; return -1; }
518 if (length == 0) { good = true; return -1; }
520 if (len > length) len = length;
521 int ret = b == null ? (int)super.skip(len) : super.read(b, off, len);
532 public void close() throws IOException {
533 if (contentLength == -1) {
534 while(!chunkedDone) {
535 if (length != 0) skip(length);
540 if (length != 0) skip(length);
542 if (releaseMe != null) releaseMe.release();
553 // Misc Helpers ///////////////////////////////////////////////////////////////////////////////////
555 /** reads a set of HTTP headers off of the input stream, returning null if the stream is already at its end */
556 private Hashtable parseHeaders(InputStream in, Cookie.Jar cookies) throws IOException {
557 Hashtable ret = new Hashtable();
559 // we can't use a BufferedReader directly on the input stream, since it will buffer past the end of the headers
560 byte[] buf = new byte[4096];
563 int read = in.read();
564 if (read == -1 && buflen == 0) return null;
565 if (read == -1) throw new HTTPException("stream closed while reading headers");
566 buf[buflen++] = (byte)read;
567 if (buflen >= 4 && buf[buflen - 4] == '\r' && buf[buflen - 3] == '\n' &&
568 buf[buflen - 2] == '\r' && buf[buflen - 1] == '\n')
570 if (buflen >=2 && buf[buflen - 1] == '\n' && buf[buflen - 2] == '\n')
571 break; // nice for people using stdio
572 if (buflen == buf.length) {
573 byte[] newbuf = new byte[buf.length * 2];
574 System.arraycopy(buf, 0, newbuf, 0, buflen);
579 BufferedReader br = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(buf, 0, buflen)));
580 String s = br.readLine();
581 if (!s.startsWith("HTTP/")) throw new HTTPException("Expected reply to start with \"HTTP/\", got: " + s);
582 ret.put("STATUSLINE", s.substring(s.indexOf(' ') + 1));
583 ret.put("HTTP", s.substring(5, s.indexOf(' ')));
585 while((s = br.readLine()) != null && s.length() > 0) {
586 String front = s.substring(0, s.indexOf(':')).toLowerCase();
587 String back = s.substring(s.indexOf(':') + 1).trim();
588 // ugly hack: we never replace a Digest-auth with a Basic-auth (proxy + www)
589 if (front.endsWith("-authenticate") && ret.get(front) != null && !back.equals("Digest")) continue;
590 if (front.equals("set-cookie")) cookies.setCookie(back, host);
591 ret.put(front, back);
596 private Hashtable parseAuthenticationChallenge(String s) {
597 Hashtable ret = new Hashtable();
600 ret.put("AUTHTYPE", s.substring(0, s.indexOf(' ')));
601 s = s.substring(s.indexOf(' ')).trim();
603 while (s.length() > 0) {
605 String key = s.substring(0, s.indexOf('='));
606 s = s.substring(s.indexOf('=') + 1);
607 if (s.charAt(0) == '\"') {
609 val = s.substring(0, s.indexOf('\"'));
610 s = s.substring(s.indexOf('\"') + 1);
612 val = s.indexOf(',') == -1 ? s : s.substring(0, s.indexOf(','));
613 s = s.indexOf(',') == -1 ? "" : s.substring(s.indexOf(',') + 1);
615 if (s.length() > 0 && s.charAt(0) == ',') s = s.substring(1);
622 private String H(String s) throws IOException {
623 byte[] b = s.getBytes("UTF8");
625 md5.update(b, 0, b.length);
626 byte[] out = new byte[md5.getDigestSize()];
629 for(int i=0; i<out.length; i++) {
630 ret += "0123456789abcdef".charAt((out[i] & 0xf0) >> 4);
631 ret += "0123456789abcdef".charAt(out[i] & 0x0f);
637 // Proxy ///////////////////////////////////////////////////////////
639 /** encapsulates most of the proxy logic; some is shared in HTTP.java */
640 public static class Proxy {
642 public String httpProxyHost = null; ///< the HTTP Proxy host to use
643 public int httpProxyPort = -1; ///< the HTTP Proxy port to use
644 public String httpsProxyHost = null; ///< seperate proxy for HTTPS
645 public int httpsProxyPort = -1;
646 public String socksProxyHost = null; ///< the SOCKS Proxy Host to use
647 public int socksProxyPort = -1; ///< the SOCKS Proxy Port to use
648 public String[] excluded = new String[] { }; ///< hosts to be excluded from proxy use; wildcards permitted
650 public Socket attempt(String url, String host, HTTP http) {
651 for(int i=0; i<excluded.length; i++) if (http.host.equals(excluded[i])) return null;
652 if (http.ssl && httpsProxyHost != null) return attemptHttpProxy(http, httpsProxyHost, httpsProxyPort);
653 if (httpProxyHost != null) return attemptHttpProxy(http, httpProxyHost, httpProxyPort);
654 if (socksProxyHost != null) return attemptSocksProxy(http, socksProxyHost, socksProxyPort);
658 public Proxy detectProxyViaManual(String http_proxy,
662 Proxy ret = new Proxy();
663 ret.httpProxyHost = http_proxy;
664 if (ret.httpProxyHost != null) {
665 if (ret.httpProxyHost.startsWith("http://")) ret.httpProxyHost = ret.httpProxyHost.substring(7);
666 if (ret.httpProxyHost.endsWith("/"))
667 ret.httpProxyHost = ret.httpProxyHost.substring(0, ret.httpProxyHost.length() - 1);
668 if (ret.httpProxyHost.indexOf(':') != -1) {
669 ret.httpProxyPort = Integer.parseInt(ret.httpProxyHost.substring(ret.httpProxyHost.indexOf(':') + 1));
670 ret.httpProxyHost = ret.httpProxyHost.substring(0, ret.httpProxyHost.indexOf(':'));
672 ret.httpProxyPort = 80;
676 ret.httpsProxyHost = https_proxy;
677 if (ret.httpsProxyHost != null) {
678 if (ret.httpsProxyHost.startsWith("https://")) ret.httpsProxyHost = ret.httpsProxyHost.substring(7);
679 if (ret.httpsProxyHost.endsWith("/"))
680 ret.httpsProxyHost = ret.httpsProxyHost.substring(0, ret.httpsProxyHost.length() - 1);
681 if (ret.httpsProxyHost.indexOf(':') != -1) {
682 ret.httpsProxyPort = Integer.parseInt(ret.httpsProxyHost.substring(ret.httpsProxyHost.indexOf(':') + 1));
683 ret.httpsProxyHost = ret.httpsProxyHost.substring(0, ret.httpsProxyHost.indexOf(':'));
685 ret.httpsProxyPort = 80;
689 ret.socksProxyHost = socks_proxy;
690 if (ret.socksProxyHost != null) {
691 if (ret.socksProxyHost.startsWith("socks://")) ret.socksProxyHost = ret.socksProxyHost.substring(7);
692 if (ret.socksProxyHost.endsWith("/"))
693 ret.socksProxyHost = ret.socksProxyHost.substring(0, ret.socksProxyHost.length() - 1);
694 if (ret.socksProxyHost.indexOf(':') != -1) {
695 ret.socksProxyPort = Integer.parseInt(ret.socksProxyHost.substring(ret.socksProxyHost.indexOf(':') + 1));
696 ret.socksProxyHost = ret.socksProxyHost.substring(0, ret.socksProxyHost.indexOf(':'));
698 ret.socksProxyPort = 80;
702 if (noproxy != null) {
703 StringTokenizer st = new StringTokenizer(noproxy, ",");
704 ret.excluded = new String[st.countTokens()];
705 for(int i=0; st.hasMoreTokens(); i++) ret.excluded[i] = st.nextToken();
708 if (ret.httpProxyHost == null && ret.socksProxyHost == null) return null;
712 // Attempts //////////////////////////////////////////////////////////////////////////////
714 protected Socket attemptDirect(HTTP http) { return http.attemptDirect(); }
716 /** Attempts to use an HTTP proxy, employing the CONNECT method if HTTPS is requested */
717 protected Socket attemptHttpProxy(HTTP http, String proxyHost, int proxyPort) {
719 if (Log.verbose) Log.info(this, "attempting to create HTTP proxied socket using proxy " + proxyHost + ":" + proxyPort);
720 Socket sock = http.getSocket(proxyHost, proxyPort, http.ssl, false);
723 if (!http.path.startsWith("http://")) http.path = "http://" + http.host + ":" + http.port + http.path;
727 PrintWriter pw = new PrintWriter(new OutputStreamWriter(sock.getOutputStream()));
728 BufferedReader br = new BufferedReader(new InputStreamReader(sock.getInputStream()));
729 pw.print("CONNECT " + http.host + ":" + http.port + " HTTP/1.1\r\n\r\n");
731 String s = br.readLine();
732 if (s.charAt(9) != '2') throw new HTTPException("proxy refused CONNECT method: \"" + s + "\"");
733 while (br.readLine().length() > 0) { };
734 ((SSL)sock).negotiate();
737 } catch (IOException e) {
738 if (Log.on) Log.info(this, "exception in attemptHttpProxy(): " + e);
744 * Implements SOCKSv4 with v4a DNS extension
745 * @see http://www.socks.nec.com/protocol/socks4.protocol
746 * @see http://www.socks.nec.com/protocol/socks4a.protocol
748 protected Socket attemptSocksProxy(HTTP http, String proxyHost, int proxyPort) {
750 // even if host is already a "x.y.z.w" string, we use this to parse it into bytes
751 InetAddress addr = null;
752 try { addr = InetAddress.getByName(http.host); } catch (Exception e) { }
754 if (Log.verbose) Log.info(this, "attempting to create SOCKSv4" + (addr == null ? "" : "a") +
755 " proxied socket using proxy " + proxyHost + ":" + proxyPort);
758 Socket sock = http.getSocket(proxyHost, proxyPort, http.ssl, false);
760 DataOutputStream dos = new DataOutputStream(sock.getOutputStream());
761 dos.writeByte(0x04); // SOCKSv4(a)
762 dos.writeByte(0x01); // CONNECT
763 dos.writeShort(http.port & 0xffff); // port
764 if (addr == null) dos.writeInt(0x00000001); // bogus IP
765 else dos.write(addr.getAddress()); // actual IP
766 dos.writeByte(0x00); // no userid
768 PrintWriter pw = new PrintWriter(new OutputStreamWriter(dos));
771 dos.writeByte(0x00); // hostname null terminator
775 DataInputStream dis = new DataInputStream(sock.getInputStream());
776 dis.readByte(); // reply version
777 byte success = dis.readByte(); // success/fail
778 dis.skip(6); // ip/port
780 if ((int)(success & 0xff) == 90) {
781 if (http.ssl) ((SSL)sock).negotiate();
784 if (Log.on) Log.info(this, "SOCKS server denied access, code " + (success & 0xff));
787 } catch (IOException e) {
788 if (Log.on) Log.info(this, "exception in attemptSocksProxy(): " + e);
793 // Authorization ///////////////////////////////////////////////////////////////////////////////////
795 public static class Authorization {
797 static public String authorization = null;
798 static public String authorization2 = null;
799 static public Semaphore waitingForUser = new Semaphore();
801 // FIXME: temporarily disabled so we can use HTTP outside the core
803 public static synchronized void getPassword(final String realm, final String style,
804 final String proxyIP, String oldAuth) throws IOException {
806 // this handles cases where multiple threads hit the proxy auth at the same time -- all but one will block on the
807 // synchronized keyword. If 'authorization' changed while the thread was blocked, it means that the user entered
808 // a password, so we should reattempt authorization.
810 if (authorization != oldAuth) return;
811 if (Log.on) Log.info(Authorization.class, "displaying proxy authorization dialog");
812 Scheduler.add(new Task() {
813 public void perform() throws IOException, JSExn {
817 //Template.buildTemplate("org/ibex/builtin/proxy_authorization.ibex", Stream.getInputStream((JS)Main.builtin.get("org/ibex/builtin/proxy_authorization.ibex")), new Ibex(null));
819 b.put("realm", realm);
820 b.put("proxyIP", proxyIP);
824 waitingForUser.block();
825 if (Log.on) Log.info(Authorization.class, "got proxy authorization info; re-attempting connection");
831 * An implementation of Microsoft's proprietary NTLM authentication protocol. This code was derived from Eric
832 * Glass's work, and is copyright as follows:
834 * Copyright (c) 2003 Eric Glass (eglass1 at comcast.net).
836 * Permission to use, copy, modify, and distribute this document for any purpose and without any fee is hereby
837 * granted, provided that the above copyright notice and this list of conditions appear in all copies.
838 * The most current version of this document may be obtained from http://davenport.sourceforge.net/ntlm.html .
840 public static class NTLM {
842 public static final byte[] type1 = new byte[] { 0x4e, 0x54, 0x4c, 0x4d, 0x53, 0x53, 0x50, 0x00, 0x01,
843 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x00 };
846 * Calculates the NTLM Response for the given challenge, using the
847 * specified password.
849 * @param password The user's password.
850 * @param challenge The Type 2 challenge from the server.
852 * @return The NTLM Response.
854 public static byte[] getNTLMResponse(String password, byte[] challenge)
855 throws UnsupportedEncodingException {
856 byte[] ntlmHash = ntlmHash(password);
857 return lmResponse(ntlmHash, challenge);
861 * Calculates the LM Response for the given challenge, using the specified
864 * @param password The user's password.
865 * @param challenge The Type 2 challenge from the server.
867 * @return The LM Response.
869 public static byte[] getLMResponse(String password, byte[] challenge)
871 byte[] lmHash = lmHash(password);
872 return lmResponse(lmHash, challenge);
876 * Calculates the NTLMv2 Response for the given challenge, using the
877 * specified authentication target, username, password, target information
878 * block, and client challenge.
880 * @param target The authentication target (i.e., domain).
881 * @param user The username.
882 * @param password The user's password.
883 * @param targetInformation The target information block from the Type 2
885 * @param challenge The Type 2 challenge from the server.
886 * @param clientChallenge The random 8-byte client challenge.
888 * @return The NTLMv2 Response.
890 public static byte[] getNTLMv2Response(String target, String user,
891 String password, byte[] targetInformation, byte[] challenge,
892 byte[] clientChallenge) throws UnsupportedEncodingException {
893 byte[] ntlmv2Hash = ntlmv2Hash(target, user, password);
894 byte[] blob = createBlob(targetInformation, clientChallenge);
895 return lmv2Response(ntlmv2Hash, blob, challenge);
899 * Calculates the LMv2 Response for the given challenge, using the
900 * specified authentication target, username, password, and client
903 * @param target The authentication target (i.e., domain).
904 * @param user The username.
905 * @param password The user's password.
906 * @param challenge The Type 2 challenge from the server.
907 * @param clientChallenge The random 8-byte client challenge.
909 * @return The LMv2 Response.
911 public static byte[] getLMv2Response(String target, String user,
912 String password, byte[] challenge, byte[] clientChallenge)
913 throws UnsupportedEncodingException {
914 byte[] ntlmv2Hash = ntlmv2Hash(target, user, password);
915 return lmv2Response(ntlmv2Hash, clientChallenge, challenge);
919 * Calculates the NTLM2 Session Response for the given challenge, using the
920 * specified password and client challenge.
922 * @param password The user's password.
923 * @param challenge The Type 2 challenge from the server.
924 * @param clientChallenge The random 8-byte client challenge.
926 * @return The NTLM2 Session Response. This is placed in the NTLM
927 * response field of the Type 3 message; the LM response field contains
928 * the client challenge, null-padded to 24 bytes.
930 public static byte[] getNTLM2SessionResponse(String password,
931 byte[] challenge, byte[] clientChallenge) throws UnsupportedEncodingException {
932 byte[] ntlmHash = ntlmHash(password);
934 md5.update(challenge, 0, challenge.length);
935 md5.update(clientChallenge, 0, clientChallenge.length);
936 byte[] sessionHash = new byte[8];
937 byte[] md5_out = new byte[md5.getDigestSize()];
938 md5.doFinal(md5_out, 0);
939 System.arraycopy(md5_out, 0, sessionHash, 0, 8);
940 return lmResponse(ntlmHash, sessionHash);
944 * Creates the LM Hash of the user's password.
946 * @param password The password.
948 * @return The LM Hash of the given password, used in the calculation
949 * of the LM Response.
951 private static byte[] lmHash(String password) {
953 byte[] oemPassword = password.toUpperCase().getBytes("UTF8");
954 int length = java.lang.Math.min(oemPassword.length, 14);
955 byte[] keyBytes = new byte[14];
956 System.arraycopy(oemPassword, 0, keyBytes, 0, length);
957 Key lowKey = createDESKey(keyBytes, 0);
958 Key highKey = createDESKey(keyBytes, 7);
959 byte[] magicConstant = "KGS!@#$%".getBytes("UTF8");
960 Cipher des = Cipher.getInstance("DES/ECB/NoPadding");
961 des.init(Cipher.ENCRYPT_MODE, lowKey);
962 byte[] lowHash = des.doFinal(magicConstant);
963 des.init(Cipher.ENCRYPT_MODE, highKey);
964 byte[] highHash = des.doFinal(magicConstant);
965 byte[] lmHash = new byte[16];
966 System.arraycopy(lowHash, 0, lmHash, 0, 8);
967 System.arraycopy(highHash, 0, lmHash, 8, 8);
974 * Creates the NTLM Hash of the user's password.
976 * @param password The password.
978 * @return The NTLM Hash of the given password, used in the calculation
979 * of the NTLM Response and the NTLMv2 and LMv2 Hashes.
981 private static byte[] ntlmHash(String password) throws UnsupportedEncodingException {
984 byte[] unicodePassword = password.getBytes("UnicodeLittleUnmarked");
986 md4.update(unicodePassword, 0, unicodePassword.length);
987 byte[] ret = new byte[md4.getDigestSize()];
994 * Creates the NTLMv2 Hash of the user's password.
996 * @param target The authentication target (i.e., domain).
997 * @param user The username.
998 * @param password The password.
1000 * @return The NTLMv2 Hash, used in the calculation of the NTLMv2
1001 * and LMv2 Responses.
1003 private static byte[] ntlmv2Hash(String target, String user,
1004 String password) throws UnsupportedEncodingException {
1005 byte[] ntlmHash = ntlmHash(password);
1006 String identity = user.toUpperCase() + target.toUpperCase();
1007 return hmacMD5(identity.getBytes("UnicodeLittleUnmarked"), ntlmHash);
1011 * Creates the LM Response from the given hash and Type 2 challenge.
1013 * @param hash The LM or NTLM Hash.
1014 * @param challenge The server challenge from the Type 2 message.
1016 * @return The response (either LM or NTLM, depending on the provided
1019 private static byte[] lmResponse(byte[] hash, byte[] challenge)
1022 byte[] keyBytes = new byte[21];
1023 System.arraycopy(hash, 0, keyBytes, 0, 16);
1024 Key lowKey = createDESKey(keyBytes, 0);
1025 Key middleKey = createDESKey(keyBytes, 7);
1026 Key highKey = createDESKey(keyBytes, 14);
1027 Cipher des = Cipher.getInstance("DES/ECB/NoPadding");
1028 des.init(Cipher.ENCRYPT_MODE, lowKey);
1029 byte[] lowResponse = des.doFinal(challenge);
1030 des.init(Cipher.ENCRYPT_MODE, middleKey);
1031 byte[] middleResponse = des.doFinal(challenge);
1032 des.init(Cipher.ENCRYPT_MODE, highKey);
1033 byte[] highResponse = des.doFinal(challenge);
1034 byte[] lmResponse = new byte[24];
1035 System.arraycopy(lowResponse, 0, lmResponse, 0, 8);
1036 System.arraycopy(middleResponse, 0, lmResponse, 8, 8);
1037 System.arraycopy(highResponse, 0, lmResponse, 16, 8);
1044 * Creates the LMv2 Response from the given hash, client data, and
1047 * @param hash The NTLMv2 Hash.
1048 * @param clientData The client data (blob or client challenge).
1049 * @param challenge The server challenge from the Type 2 message.
1051 * @return The response (either NTLMv2 or LMv2, depending on the
1054 private static byte[] lmv2Response(byte[] hash, byte[] clientData,
1056 byte[] data = new byte[challenge.length + clientData.length];
1057 System.arraycopy(challenge, 0, data, 0, challenge.length);
1058 System.arraycopy(clientData, 0, data, challenge.length,
1060 byte[] mac = hmacMD5(data, hash);
1061 byte[] lmv2Response = new byte[mac.length + clientData.length];
1062 System.arraycopy(mac, 0, lmv2Response, 0, mac.length);
1063 System.arraycopy(clientData, 0, lmv2Response, mac.length,
1065 return lmv2Response;
1069 * Creates the NTLMv2 blob from the given target information block and
1072 * @param targetInformation The target information block from the Type 2
1074 * @param clientChallenge The random 8-byte client challenge.
1076 * @return The blob, used in the calculation of the NTLMv2 Response.
1078 private static byte[] createBlob(byte[] targetInformation,
1079 byte[] clientChallenge) {
1080 byte[] blobSignature = new byte[] {
1081 (byte) 0x01, (byte) 0x01, (byte) 0x00, (byte) 0x00
1083 byte[] reserved = new byte[] {
1084 (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00
1086 byte[] unknown1 = new byte[] {
1087 (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00
1089 byte[] unknown2 = new byte[] {
1090 (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00
1092 long time = System.currentTimeMillis();
1093 time += 11644473600000l; // milliseconds from January 1, 1601 -> epoch.
1094 time *= 10000; // tenths of a microsecond.
1095 // convert to little-endian byte array.
1096 byte[] timestamp = new byte[8];
1097 for (int i = 0; i < 8; i++) {
1098 timestamp[i] = (byte) time;
1101 byte[] blob = new byte[blobSignature.length + reserved.length +
1102 timestamp.length + clientChallenge.length +
1103 unknown1.length + targetInformation.length +
1106 System.arraycopy(blobSignature, 0, blob, offset, blobSignature.length);
1107 offset += blobSignature.length;
1108 System.arraycopy(reserved, 0, blob, offset, reserved.length);
1109 offset += reserved.length;
1110 System.arraycopy(timestamp, 0, blob, offset, timestamp.length);
1111 offset += timestamp.length;
1112 System.arraycopy(clientChallenge, 0, blob, offset,
1113 clientChallenge.length);
1114 offset += clientChallenge.length;
1115 System.arraycopy(unknown1, 0, blob, offset, unknown1.length);
1116 offset += unknown1.length;
1117 System.arraycopy(targetInformation, 0, blob, offset,
1118 targetInformation.length);
1119 offset += targetInformation.length;
1120 System.arraycopy(unknown2, 0, blob, offset, unknown2.length);
1125 * Calculates the HMAC-MD5 hash of the given data using the specified
1128 * @param data The data for which the hash will be calculated.
1129 * @param key The hashing key.
1131 * @return The HMAC-MD5 hash of the given data.
1133 private static byte[] hmacMD5(byte[] data, byte[] key) {
1134 byte[] ipad = new byte[64];
1135 byte[] opad = new byte[64];
1136 for (int i = 0; i < 64; i++) {
1137 ipad[i] = (byte) 0x36;
1138 opad[i] = (byte) 0x5c;
1140 for (int i = key.length - 1; i >= 0; i--) {
1144 byte[] content = new byte[data.length + 64];
1145 System.arraycopy(ipad, 0, content, 0, 64);
1146 System.arraycopy(data, 0, content, 64, data.length);
1147 MD5 md5 = new MD5();
1148 md5.update(content, 0, content.length);
1149 data = new byte[md5.getDigestSize()];
1150 md5.doFinal(data, 0);
1151 content = new byte[data.length + 64];
1152 System.arraycopy(opad, 0, content, 0, 64);
1153 System.arraycopy(data, 0, content, 64, data.length);
1155 md5.update(content, 0, content.length);
1156 byte[] ret = new byte[md5.getDigestSize()];
1157 md5.doFinal(ret, 0);
1162 * Creates a DES encryption key from the given key material.
1164 * @param bytes A byte array containing the DES key material.
1165 * @param offset The offset in the given byte array at which
1166 * the 7-byte key material starts.
1168 * @return A DES encryption key created from the key material
1169 * starting at the specified offset in the given byte array.
1172 private static Key createDESKey(byte[] bytes, int offset) {
1173 byte[] keyBytes = new byte[7];
1174 System.arraycopy(bytes, offset, keyBytes, 0, 7);
1175 byte[] material = new byte[8];
1176 material[0] = keyBytes[0];
1177 material[1] = (byte) (keyBytes[0] << 7 | (keyBytes[1] & 0xff) >>> 1);
1178 material[2] = (byte) (keyBytes[1] << 6 | (keyBytes[2] & 0xff) >>> 2);
1179 material[3] = (byte) (keyBytes[2] << 5 | (keyBytes[3] & 0xff) >>> 3);
1180 material[4] = (byte) (keyBytes[3] << 4 | (keyBytes[4] & 0xff) >>> 4);
1181 material[5] = (byte) (keyBytes[4] << 3 | (keyBytes[5] & 0xff) >>> 5);
1182 material[6] = (byte) (keyBytes[5] << 2 | (keyBytes[6] & 0xff) >>> 6);
1183 material[7] = (byte) (keyBytes[6] << 1);
1184 oddParity(material);
1185 return new SecretKeySpec(material, "DES");
1190 * Applies odd parity to the given byte array.
1192 * @param bytes The data whose parity bits are to be adjusted for
1195 private static void oddParity(byte[] bytes) {
1196 for (int i = 0; i < bytes.length; i++) {
1198 boolean needsParity = (((b >>> 7) ^ (b >>> 6) ^ (b >>> 5) ^
1199 (b >>> 4) ^ (b >>> 3) ^ (b >>> 2) ^
1200 (b >>> 1)) & 0x01) == 0;
1202 bytes[i] |= (byte) 0x01;
1204 bytes[i] &= (byte) 0xfe;