HttpUrlConnection底层实现

转自:http://blog.csdn.net/zhongweijian/article/details/7619453


最近有个需求需要对于获取URL页面进行host绑定并且立即生效,在java里面实现可以用代理服务器来实现:因为在测试环境下可能需要通过绑定来访问测试环境的应用
实现代码如下:
 
    public static String getResponseText(String queryUrl,String host,String ip) { //queryUrl,完整的url,host和ip需要绑定的host和ip
       InputStream is = null;
       BufferedReader br = null;
       StringBuffer res = new StringBuffer();
       try {
           HttpURLConnection httpUrlConn = null;
           URL url = new URL(queryUrl);
           if(ip!=null){
               String str[] = ip.split("\\.");
               byte[] b =new byte[str.length];
               for(int i=0,len=str.length;i<len;i++){
                   b[i] = (byte)(Integer.parseInt(str[i],10));
               }
                Proxy proxy = new Proxy(Proxy.Type.HTTP,
                new InetSocketAddress(InetAddress.getByAddress(b), 80));  //b是绑定的ip,生成proxy代理对象,因为http底层是socket实现,
                httpUrlConn = (HttpURLConnection) url
                .openConnection(proxy);
           }else{
                httpUrlConn = (HttpURLConnection) url
                        .openConnection(); 
           }
           httpUrlConn.setRequestMethod("GET");
           httpUrlConn.setDoOutput(true);
           httpUrlConn.setConnectTimeout(2000);
           httpUrlConn.setReadTimeout(2000);
           httpUrlConn.setDefaultUseCaches(false);
           httpUrlConn.setUseCaches(false);
 
           is = httpUrlConn.getInputStream();
 
 
那么底层对于proxy对象到底是怎么处理,底层的socket实现到底怎么样,带着这个疑惑看了下jdk的rt.jar对于这块的处理
httpUrlConn = (HttpURLConnection) url.openConnection(proxy)
 
java.net.URL类里面的openConnection方法:
public URLConnection openConnection(Proxy proxy){
   …
   return handler.openConnection(this, proxy); Handler是sun.net.www.protocol.http.Handler.java类,继承java.net. URLStreamHandler.java类,用来处理http连接请求响应的。
}
 
Handler的方法:
protected java.net.URLConnection openConnection(URL u, Proxy p)
        throws IOException {
        return new HttpURLConnection(u, p, this);
    }
 
只是简单的生成sun.net.www.protocl.http.HttpURLConnection对象,并进行初始化
protected HttpURLConnection(URL u, Proxy p, Handler handler) {
        super(u);
        requests = new MessageHeader();  请求头信息生成类
        responses = new MessageHeader(); 响应头信息解析类
        this.handler = handler; 
        instProxy = p;  代理服务器对象
        cookieHandler = (CookieHandler)java.security.AccessController.doPrivileged(
            new java.security.PrivilegedAction() {
            public Object run() {
                return CookieHandler.getDefault();
            }
        });
        cacheHandler = (ResponseCache)java.security.AccessController.doPrivileged(
            new java.security.PrivilegedAction() {
            public Object run() {
                return ResponseCache.getDefault();
            }
        });
    }
  
 
最终在httpUrlConn.getInputStream();才进行socket连接,发送http请求,解析http响应信息。具体过程如下:
 
sun.net.www.protocl.http.HttpURLConnection.java的getInputStream方法:
 
public synchronized InputStream getInputStream() throws IOException {
   
     ...socket连接
     connect();
     ...
     ps = (PrintStream)http.getOutputStream(); 获得输出流,打开连接之后已经生成。
 
       if (!streaming()) {
             writeRequests();  输出http请求头信息
       }
     ...
     http.parseHTTP(responses, pi, this);  解析响应信息
                if(logger.isLoggable(Level.FINEST)) {
                    logger.fine(responses.toString());
                }
                inputStream = http.getInputStream();  获得输入流
}
 
其中connect()调用方法链:
plainConnect(){
...
                Proxy p = null;
                if (sel != null) {
                    URI uri = sun.net.www.ParseUtil.toURI(url);
                    Iterator<Proxy> it = sel.select(uri).iterator();
                    while (it.hasNext()) {
                        p = it.next();
                        try {
                            if (!failedOnce) {
                                http = getNewHttpClient(url, p, connectTimeout);
...
}
 
getNewHttpClient(){
...
        return HttpClient.New(url, p, connectTimeout, useCache);
...
}
 
下面跟进去最终建立socket连接的代码:
sun.net.www.http.HttpClient.java的openServer()方法建立socket连接:
 
    protected synchronized void openServer() throws IOException {
            ...
            if ((proxy != null) && (proxy.type() == Proxy.Type.HTTP)) {
                sun.net.www.URLConnection.setProxiedHost(host);
                if (security != null) {
                    security.checkConnect(host, port);
                }
                privilegedOpenServer((InetSocketAddress) proxy.address());最终socket连接的是设置的代理服务器的地址,
            ...
}
 
    private synchronized void privilegedOpenServer(final InetSocketAddress server)
         throws IOException
    {
        try {
            java.security.AccessController.doPrivileged(
                new java.security.PrivilegedExceptionAction() {
                public Object run() throws IOException {
                    openServer(server.getHostName(), server.getPort());  注意openserver函数  这里的server的getHostName是设置的代理服务器,(ip或者hostname,如果是host绑定设置的代理服务器的ip,那么这里getHostName出来的就是ip地址,可以去查看InetSocketAddress类的getHostName方法)
                    return null;
                }
            });
        } catch (java.security.PrivilegedActionException pae) {
            throw (IOException) pae.getException();
        }
    }
 
   public void openServer(String server, int port) throws IOException {
        serverSocket = doConnect(server, port);  生成的Socket连接对象
        try {
            serverOutput = new PrintStream(
                new BufferedOutputStream(serverSocket.getOutputStream()),
                                         false, encoding);   生成输出流,
        } catch (UnsupportedEncodingException e) {
            throw new InternalError(encoding+" encoding not found");
        }
        serverSocket.setTcpNoDelay(true);
    }
 
 
protected Socket doConnect (String server, int port)
    throws IOException, UnknownHostException {
        Socket s;
        if (proxy != null) {
            if (proxy.type() == Proxy.Type.SOCKS) {
                s = (Socket) AccessController.doPrivileged(
                               new PrivilegedAction() {
                                   public Object run() {
                                       return new Socket(proxy);
                                   }});
            } else
                s = new Socket(Proxy.NO_PROXY);
        } else
            s = new Socket();
        // Instance specific timeouts do have priority, that means
        // connectTimeout & readTimeout (-1 means not set)
        // Then global default timeouts
        // Then no timeout.
        if (connectTimeout >= 0) {
            s.connect(new InetSocketAddress(server, port), connectTimeout);
        } else {
            if (defaultConnectTimeout > 0) {
                s.connect(new InetSocketAddress(server, port), defaultConnectTimeout);//连接到代理服务器,看下面Socket类的connect方法代码
            } else {
                s.connect(new InetSocketAddress(server, port));
            }
        }
        if (readTimeout >= 0)
            s.setSoTimeout(readTimeout);
        else if (defaultSoTimeout > 0) {
            s.setSoTimeout(defaultSoTimeout);
        }
        return s;
}
 
上面的new InetSocketAddress(server, port)这里会涉及到java DNS cache的处理,
 
      public InetSocketAddress(String hostname, int port) {
        if (port < 0 || port > 0xFFFF) {
            throw new IllegalArgumentException("port out of range:" + port);
        }
        if (hostname == null) {
            throw new IllegalArgumentException("hostname can't be null");
        }
        try {
            addr = InetAddress.getByName(hostname);  //这里会有java DNS缓存的处理,先从缓存取hostname绑定的ip地址,如果取不到再通过OS的DNS cache机制去取,取不到再从DNS服务器上取。
        } catch(UnknownHostException e) {
            this.hostname = hostname;
            addr = null;
        }
        this.port = port;
    }
 
 
 
当然最终的Socket.java的connect方法
java.net.socket
            
   public void connect(SocketAddress endpoint, int timeout) throws IOException {
        if (endpoint == null)
           
        if (timeout < 0)
          throw new IllegalArgumentException("connect: timeout can't be negative");
 
        if (isClosed())
            throw new SocketException("Socket is closed");
 
        if (!oldImpl && isConnected())
            throw new SocketException("already connected");
 
        if (!(endpoint instanceof InetSocketAddress))
            throw new IllegalArgumentException("Unsupported address type");
 
        InetSocketAddress epoint = (InetSocketAddress) endpoint;
 
        SecurityManager security = System.getSecurityManager();
        if (security != null) {
            if (epoint.isUnresolved())
                security.checkConnect(epoint.getHostName(),
                                      epoint.getPort());
            else
                security.checkConnect(epoint.getAddress().getHostAddress(),
                                      epoint.getPort());
        }
        if (!created)
            createImpl(true);
        if (!oldImpl)
            impl.connect(epoint, timeout);
        else if (timeout == 0) {
            if (epoint.isUnresolved())  //如果没有设置SocketAddress的ip地址,则用域名去访问
                impl.connect(epoint.getAddress().getHostName(),
                             epoint.getPort());
            else
                impl.connect(epoint.getAddress(), epoint.getPort());  最终socket连接的是设置的SocketAddress的ip地址,
        } else
            throw new UnsupportedOperationException("SocketImpl.connect(addr, timeout)");
        connected = true;
        /*
         * If the socket was not bound before the connect, it is now because
         * the kernel will have picked an ephemeral port & a local address
         */
        bound = true;
    }
 
 
 
我们再看下通过socket来发送HTTP请求的处理代码,也就是sun.net.www.protocl.http.HttpURLConnection.java的getInputStream方法中调用的writeRequests()方法: 
private void writeRequests() throws IOException {  这段代码就是封装http请求的头请求信息,通过socket发送出去
        /* print all message headers in the MessageHeader
         * onto the wire - all the ones we've set and any
         * others that have been set
         */
        // send any pre-emptive authentication
        if (http.usingProxy) {
            setPreemptiveProxyAuthentication(requests);
        }
        if (!setRequests) {
 
            /* We're very particular about the order in which we
             * set the request headers here.  The order should not
             * matter, but some careless CGI programs have been
             * written to expect a very particular order of the
             * standard headers.  To name names, the order in which
             * Navigator3.0 sends them.  In particular, we make *sure*
             * to send Content-type: <> and Content-length:<> second
             * to last and last, respectively, in the case of a POST
             * request.
             */
            if (!failedOnce)
                requests.prepend(method + " " + http.getURLFile()+" "  +
                                 httpVersion, null);
            if (!getUseCaches()) {
                requests.setIfNotSet ("Cache-Control", "no-cache");
                requests.setIfNotSet ("Pragma", "no-cache");
            }
            requests.setIfNotSet("User-Agent", userAgent);
            int port = url.getPort();
            String host = url.getHost();
            if (port != -1 && port != url.getDefaultPort()) {
                host += ":" + String.valueOf(port);
            }
            requests.setIfNotSet("Host", host);
            requests.setIfNotSet("Accept", acceptString);
 
            /*
             * For HTTP/1.1 the default behavior is to keep connections alive.
             * However, we may be talking to a 1.0 server so we should set
             * keep-alive just in case, except if we have encountered an error
             * or if keep alive is disabled via a system property
             */
 
            // Try keep-alive only on first attempt
            if (!failedOnce && http.getHttpKeepAliveSet()) {
                if (http.usingProxy) {
                    requests.setIfNotSet("Proxy-Connection", "keep-alive");
                } else {
                    requests.setIfNotSet("Connection", "keep-alive");
                }
            } else {
                /*
                 * RFC 2616 HTTP/1.1 section 14.10 says:
                 * HTTP/1.1 applications that do not support persistent
                 * connections MUST include the "close" connection option
                 * in every message
                 */
                requests.setIfNotSet("Connection", "close");
            }
            // Set modified since if necessary
            long modTime = getIfModifiedSince();
            if (modTime != 0 ) {
                Date date = new Date(modTime);
                //use the preferred date format according to RFC 2068(HTTP1.1),
                // RFC 822 and RFC 1123
                SimpleDateFormat fo =
                  new SimpleDateFormat ("EEE, dd MMM yyyy HH:mm:ss 'GMT'", Locale.US);
                fo.setTimeZone(TimeZone.getTimeZone("GMT"));
                requests.setIfNotSet("If-Modified-Since", fo.format(date));
            }
            // check for preemptive authorization
            AuthenticationInfo sauth = AuthenticationInfo.getServerAuth(url);
            if (sauth != null && sauth.supportsPreemptiveAuthorization() ) {
                // Sets "Authorization"
                requests.setIfNotSet(sauth.getHeaderName(), sauth.getHeaderValue(url,method));
                currentServerCredentials = sauth;
            }
 
            if (!method.equals("PUT") && (poster != null || streaming())) {
                requests.setIfNotSet ("Content-type",
                        "application/x-www-form-urlencoded");
            }
 
            if (streaming()) {
                if (chunkLength != -1) {
                    requests.set ("Transfer-Encoding", "chunked");
                } else {
                    requests.set ("Content-Length", String.valueOf(fixedContentLength));
                }
            } else if (poster != null) {
                /* add Content-Length & POST/PUT data */
                synchronized (poster) {
                    /* close it, so no more data can be added */
                    poster.close();
                    requests.set("Content-Length",
                                 String.valueOf(poster.size()));
                }
            }
 
            // get applicable cookies based on the uri and request headers
            // add them to the existing request headers
            setCookieHeader();
…
}
 
 
再来看看把socket响应信息解析为http的响应信息的代码:
sun.net.www.http.HttpClient.java的parseHTTP方法:
private boolean parseHTTPHeader(MessageHeader responses, ProgressSource pi, HttpURLConnection httpuc)
    throws IOException {
        /* If "HTTP/*" is found in the beginning, return true.  Let
         * HttpURLConnection parse the mime header itself.
         *
         * If this isn't valid HTTP, then we don't try to parse a header
         * out of the beginning of the response into the responses,
         * and instead just queue up the output stream to it's very beginning.
         * This seems most reasonable, and is what the NN browser does.
         */
 
        keepAliveConnections = -1;
        keepAliveTimeout = 0;
 
        boolean ret = false;
        byte[] b = new byte[8];
 
        try {
            int nread = 0;
            serverInput.mark(10);
            while (nread < 8) {
                int r = serverInput.read(b, nread, 8 - nread);
                if (r < 0) {
                    break;
                }
                nread += r;
            }
            String keep=null;
            ret = b[0] == 'H' && b[1] == 'T'
                    && b[2] == 'T' && b[3] == 'P' && b[4] == '/' &&
                b[5] == '1' && b[6] == '.';
            serverInput.reset();
            if (ret) { // is valid HTTP - response started w/ "HTTP/1."
                responses.parseHeader(serverInput);
 
                // we've finished parsing http headers
                // check if there are any applicable cookies to set (in cache)
                if (cookieHandler != null) {
                    URI uri = ParseUtil.toURI(url);
                    // NOTE: That cast from Map shouldn't be necessary but
                    // a bug in javac is triggered under certain circumstances
                    // So we do put the cast in as a workaround until
                    // it is resolved.
                    if (uri != null)
                        cookieHandler.put(uri, (Map<java.lang.String,java.util.List<java.lang.String>>)responses.getHeaders());
                }
 
                /* decide if we're keeping alive:
                 * This is a bit tricky.  There's a spec, but most current
                 * servers (10/1/96) that support this differ in dialects.
                 * If the server/client misunderstand each other, the
                 * protocol should fall back onto HTTP/1.0, no keep-alive.
                 */
                if (usingProxy) { // not likely a proxy will return this
                    keep = responses.findValue("Proxy-Connection");
                }
                if (keep == null) {
                    keep = responses.findValue("Connection");
                }
                if (keep != null && keep.toLowerCase().equals("keep-alive")) {
                    /* some servers, notably Apache1.1, send something like:
                     * "Keep-Alive: timeout=15, max=1" which we should respect.
                     */
                    HeaderParser p = new HeaderParser(
                            responses.findValue("Keep-Alive"));
                    if (p != null) {
                        /* default should be larger in case of proxy */
                        keepAliveConnections = p.findInt("max", usingProxy?50:5);
                        keepAliveTimeout = p.findInt("timeout", usingProxy?60:5);
                    }
                } else if (b[7] != '0') {
                    /*
                     * We're talking 1.1 or later. Keep persistent until
                     * the server says to close.
                     */
                    if (keep != null) {
                        /*
                         * The only Connection token we understand is close.
                         * Paranoia: if there is any Connection header then
                         * treat as non-persistent.
                         */
                        keepAliveConnections = 1;
                    } else {
                        keepAliveConnections = 5;
                    }
                }
……
}
 
 
对于java.net包的http,ftp等各种协议的底层实现,可以参考rt.jar下面的几个包的代码:
sun.net.www.protocl下的几个包。
 
 
在http client中也可以设置代理:
               HostConfiguration conf = new HostConfiguration();
               conf.setHost(host);
               conf.setProxy(ip, 80);
               statusCode = httpclient.executeMethod(conf,getMethod);
 
httpclient自己也是基于socket封装的http处理的库。底层代理的实现是一样的。
 
 
另外一种不设置代理,通过反射修改InetAddress的cache也是ok的。但是这种方法非常不推荐,不要使用,因为对于proxy代理服务器概念了解不清楚,最开始还使用这种方法,
public static void jdkDnsNoCache(final String host, final String ip)
           throws SecurityException, NoSuchFieldException,
           IllegalArgumentException, IllegalAccessException {
       if (StringUtils.isBlank(host)) {
           return;
       }
       final Class clazz = java.net.InetAddress.class;
       final Field cacheField = clazz.getDeclaredField("addressCache");
       cacheField.setAccessible(true);
       final Object o = cacheField.get(clazz);
       Class clazz2 = o.getClass();
       final Field cacheMapField = clazz2.getDeclaredField("cache");
       cacheMapField.setAccessible(true);
       final Map cacheMap = (Map) cacheMapField.get(o);
       AccessController.doPrivileged(new PrivilegedAction() {
           public Object run() {
              try {
                  synchronized (o) {// 同步是必须的,因为o可能会有多个线程同时访问修改。
                     // cacheMap.clear();//这步比较关键,用于清除原来的缓存
//                   cacheMap.remove(host);
                     if (!StringUtils.isBlank(ip)) {
                         InetAddress inet = InetAddress.getByAddress(host,IPUtil.int2byte(ip));
                         InetAddress addressstart = InetAddress.getByName(host);
                         Object cacheEntry = cacheMap.get(host);
                         cacheMap.put(host,newCacheEntry(inet,cacheEntry));
//                       cacheMap.put(host,newCacheEntry(newInetAddress(host, ip)));
                     }else{
                         cacheMap.remove(host);
                     }
//                   System.out.println(getStaticProperty(
//                          "java.net.InetAddress", "addressCacheInit"));
                     // System.out.println(invokeStaticMethod("java.net.InetAddress","getCachedAddress",new
                     // Object[]{host}));
                  }
              } catch (Throwable te) {
                  throw new RuntimeException(te);
              }
              return null;
           }
       });
       final Map cacheMapafter = (Map) cacheMapField.get(o);
       System.out.println(cacheMapafter);
 
    }
 
关于java中对于DNS的缓存设置可以参考:
1.在${java_home}/jre/lib/secuiry/java.secuiry文件,修改下面为 
  networkaddress.cache.negative.ttl=0   DNS解析不成功的缓存时间
networkaddress.cache.ttl=0    DNS解析成功的缓存的时间
2.jvm启动时增加下面两个启动环境变量
  -Dsun.net.inetaddr.ttl=0
      -Dsun.net.inetaddr.negative.ttl=0
 
 
如果在java程序中使用,可以这么设置设置:
    java.security.Security.setProperty("networkaddress.cache.ttl" , "0");
        java.security.Security.setProperty("networkaddress.cache.negative.ttl" , "0");
 
   还有几篇文档链接可以查看:
       http://www.rgagnon.com/javadetails/java-0445.html
http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6247501
 
  linux下关于OS DNS设置的几个文件是
/etc/resolve.conf
/etc/nscd.conf
/etc/nsswitch.conf
 
http://www.linuxfly.org/post/543/
http://linux.die.net/man/5/nscd.conf
http://www.linuxhomenetworking.com/wiki/index.php/Quick_HOWTO_:_Ch18_:_Configuring_DNS
http://linux.die.net/man/5/nscd.conf

http://blog.csdn.net/zhongweijian/article/details/7619453



展开阅读全文

没有更多推荐了,返回首页