HttpWebRequest提高效率之连接数,代理，自动跳转,gzip请求等设置问题

    先设置4个：

                  

    [csharp] 
    webrequest.ServicePoint.Expect100Continue = false; 
                   //是否使用 Nagle 不使用 提高效率 
                   webrequest.ServicePoint.UseNagleAlgorithm = false; 
                   //最大连接数 
                   webrequest.ServicePoint.ConnectionLimit = 65500; 
                   //数据是否缓冲 false 提高效率  
                   webrequest.AllowWriteStreamBuffering = false; 

     

    代理的使用：我用的代理都是不要账号和密码的，因为要不是免费的，要是是我们自己的，对来访IP 有限制

               

    [csharp] 
    if (proxy!=null) 
                   { 
                       model.proxy = proxy; 
                       WebProxy myProxy = new WebProxy(proxy.ProxyInfo, false); 
                       myProxy.Credentials = new NetworkCredential("", ""); 
                       webrequest.Proxy = myProxy; 
                   } 
                   else 
                   { 
                       webrequest.Proxy = GlobalProxySelection.GetEmptyWebProxy(); 
                   } 
     

    webrequest.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");

    带上gzip，如果对方支持gzip，流量上可以节省五分之四，原来250KB，用gzip压缩有 大概在50KB，减少带宽压力，增加速度

    值得注意的是，如果是gzip，返回值必须要gzip解压的。

     

    [csharp] 
    if (response.ContentEncoding.ToLower().Contains("gzip")) 
                       { 
                           using (GZipStream stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress)) 
                           { 
                               using (StreamReader reader = new StreamReader(stream, Encoding.UTF8)) 
                               { 
     
                                   model.html = reader.ReadToEnd(); 
                               } 
                           } 
                       } 
     

    还有一点：gzip解压的时候回耗费CPU，所以这个需要注意下

     

    附完成方法

     

     

    [csharp]
    /// <summary> 
           /// Get 方式 获取数据 
           /// </summary> 
           /// <param name="webUrl"></param> 
           /// <returns></returns> 
           public Model.WebRequestReturnModel WebRequestGetHtmlByGet(string webUrl, CookieContainer cookieContainer, Encoding encoding, string refer,Model.Proxy proxy) 
           { 
               Model.WebRequestReturnModel model = new Model.WebRequestReturnModel(); 
               HttpWebRequest webrequest = null; 
               try 
               { 
                   webrequest = WebRequest.Create(webUrl) as HttpWebRequest; 
                    
                   webrequest.ServicePoint.Expect100Continue = false; 
                    webrequest.ServicePoint.UseNagleAlgorithm = false; 
                    webrequest.ServicePoint.ConnectionLimit = 65500; 
                   webrequest.AllowWriteStreamBuffering = false; 
                   if (proxy!=null) 
                   { 
                       model.proxy = proxy; 
                       WebProxy myProxy = new WebProxy(proxy.ProxyInfo, false); 
                       myProxy.Credentials = new NetworkCredential("", ""); 
                       webrequest.Proxy = myProxy; 
                   } 
                   else 
                   { 
                       webrequest.Proxy = GlobalProxySelection.GetEmptyWebProxy(); 
                   } 
                   //设置其他的herader 
                   webrequest =WebRequestSetHeaders(webrequest); 
                   if (!String.IsNullOrEmpty(refer)) 
                   { 
                       webrequest.Referer = refer; 
                   } 
                   else 
                   { 
                       webrequest.Referer = webUrl; 
                   } 
                   webrequest.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate"); 
                   webrequest.CookieContainer = cookieContainer; 
                   webrequest.Timeout = 5000; 
                   webrequest.UserAgent = "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)"; 
                   webrequest.Accept = "*/*"; 
                   webrequest.KeepAlive = true; 
                   webrequest.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5"); 
     
                   using (HttpWebResponse response = (HttpWebResponse)webrequest.GetResponse()) 
                   { 
     
                       if (response.ContentEncoding.ToLower().Contains("gzip")) 
                       { 
                           using (GZipStream stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress)) 
                           { 
                               using (StreamReader reader = new StreamReader(stream, Encoding.UTF8)) 
                               { 
     
                                   model.html = reader.ReadToEnd(); 
                               } 
                           } 
                       } 
                       else if (response.ContentEncoding.ToLower().Contains("deflate")) 
                       { 
                           using (DeflateStream stream = new DeflateStream(response.GetResponseStream(), CompressionMode.Decompress)) 
                           { 
                               using (StreamReader reader = new StreamReader(stream, Encoding.UTF8)) 
                               { 
     
                                   model.html = reader.ReadToEnd(); 
                               } 
     
                           } 
                       } 
                       else 
                       { 
                           using (Stream stream = response.GetResponseStream()) 
                           { 
                               using (StreamReader reader = new StreamReader(stream, Encoding.UTF8)) 
                               { 
     
                                   model.html = reader.ReadToEnd(); 
                               } 
                           } 
                       } 
                   } 
               } 
               catch (Exception ex) 
               { 
                   model.exception = ex; 
                   model.html = string.Empty; 
     
               } 
               model.cookieContainer = cookieContainer; 
               return model; 
           }