C# .net 集合-并发处理http请求之HttpClient

通常我们进行http请求使用WebRequest，这个建立的 http 连接数就只有2个或10个。对于一些诸如浏览器或网络蜘蛛的应用，2个或10个并发数量实在太少，大大影响应用的性能。
在net4.0以后新加了一个HttpClient，在.net 4.5 新加了 async/await 配合httpclient来进行多线程使用。

我们看一下原始 WebRequest进行get\post请求的方法

     /// <summary>
        /// 
        /// </summary>
        /// <param name="SourceUrl">原网页</param>
        /// <param name="UrlStr">请求网页</param>
        /// <param name="EnCode">编码</param>
        /// <param name="CookieContainer">cookie</param>
        /// <param name="proxy">代理</param>
        /// <returns></returns>
        public string GetUrlInfByGet(string SourceUrl, string UrlStr, string EnCode, ref CookieContainer CookieContainer, Configuration.NetConfigElement proxy)
        {
            string ContextStr = "";
            HttpWebRequest request = null;
            HttpWebResponse response = null;
            Stream s = null;
            MemoryStream ms = null;
            MemoryStream js = null;
            GZipStream g = null;
            int c = 1024 * 10;
            byte[] data = null;
            try
            {
                request = (HttpWebRequest)WebRequest.Create(UrlStr);
                if (Program.CurrSpiderTimes == 1) request.Timeout = Program.sysPara.BegTimeOut;
                else request.Timeout = Program.sysPara.BegTimeOut + Common.TypeConverter.ObjectToInt((Math.Pow(2, Program.CurrSpiderTimes - 1))) * Program.sysPara.IntervalTimeOut;
                request.KeepAlive = false;
                request.ServicePoint.Expect100Continue = false;
                request.ServicePoint.UseNagleAlgorithm = false;
                if (Program.CurrSpiderTimes == 1) request.ReadWriteTimeout = Program.sysPara.BegRWTimeOut;
                else request.ReadWriteTimeout = Program.sysPara.BegRWTimeOut + Common.TypeConverter.ObjectToInt((Math.Pow(2, Program.CurrSpiderTimes - 1))) * Program.sysPara.IntervalRWTimeOut;
                request.Method = "GET";
                request.Accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
                request.CookieContainer = CookieContainer;
                request.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)";
                request.Referer = SourceUrl;
                //request.KeepAlive = true;
                if (proxy.proxy)
                {
                    WebProxy webProxy = new WebProxy();                                      //定义一个网关对象
                    webProxy.Address = new Uri(proxy.proxyEntity.proxyAddess);              //网关服务器:端口
                    webProxy.Credentials = new NetworkCredential(proxy.proxyEntity.proxyUid, proxy.proxyEntity.proxyPwd);      //用戶名,密码
                    request.Proxy = webProxy;
                }
                try
                {
                    response = (HttpWebResponse)request.GetResponse();
                }
                catch (WebException)
                {
                    response = (HttpWebResponse)request.GetResponse();
                }
                string ce = response.Headers[HttpResponseHeader.ContentEncoding];
                response.Cookies = CookieContainer.GetCookies(request.RequestUri);
                int ContentLength = (int)response.ContentLength;
                s = response.GetResponseStream();
                c = 1024 * 10;
                if (ContentLength < 0)
                {
                    data = new byte[c];
                    ms = new MemoryStream();
                    int l = s.Read(data, 0, c);
                    while (l > 0)
                    {
                        ms.Write(data, 0, l);
                        l = s.Read(data, 0, c);
                    }
                    data = ms.ToArray();
                    ms.Close();
                }
                else
                {
                    data = new byte[ContentLength];
                    int pos = 0;
                    int l;
                    while (ContentLength > 0)
                    {
                        l = s.Read(data, pos, ContentLength);
                        pos += l;
                        ContentLength -= l;
                    }
                }
                if (s != null) s.Close();
                if (request != null) request.Abort();
                if (response != null) response.Close();
                if (ce == "gzip")
                {
                    js = new MemoryStream();           // 解压后的流  
                    ms = new MemoryStream(data);       // 用于解压的流  
                    g = new GZipStream(ms, CompressionMode.Decompress);
                    byte[] buffer = new byte[c];                    // 读数据缓冲区     
                    int l = g.Read(buffer, 0, c);                   // 一次读 10K     
                    while (l > 0)
                    {
                        js.Write(buffer, 0, l);
                        l = g.Read(buffer, 0, c);
                    }
                    g.Close();
                    ms.Close();
                    data = js.ToArray();
                    js.Close();
                }
                ContextStr = System.Text.Encoding.GetEncoding(EnCode).GetString(data);
            }
            catch (Exception ex)
            {
                if (request != null) request.Abort();
                if (response != null) response.Close();
                if (s != null) s.Close();
                if (ms != null) ms.Close();
                if (js != null) js.Close();
                if (g != null) g.Close();
                throw ex;
            }
            return ContextStr;
        }



  public string getUrlInfByPost(string SourceUrl, string UrlStr, string postDataStr, string EnCode, ref CookieContainer CookieContainer, Configuration.NetConfigElement proxy)
        {
            string ContextStr = "";
            HttpWebRequest request = null;
            HttpWebResponse response = null;
            Stream myRequestStream = null;
            StreamWriter myStreamWriter = null;
            Stream s = null;
            MemoryStream ms = null;
            MemoryStream js = null;
            GZipStream g = null;
            int c = 1024 * 10;
            byte[] data = null;
            try
            {
                ServicePointManager.Expect100Continue = false;
                request = (HttpWebRequest)WebRequest.Create(UrlStr);
                if (Program.CurrSpiderTimes == 1) request.Timeout = Program.sysPara.BegTimeOut;
                else request.Timeout = Program.sysPara.BegTimeOut + Common.TypeConverter.ObjectToInt((Math.Pow(2, Program.CurrSpiderTimes - 1))) * Program.sysPara.IntervalTimeOut;
                request.KeepAlive = false;
                request.ServicePoint.Expect100Continue = false;
                request.ServicePoint.UseNagleAlgorithm = false;
                if (Program.CurrSpiderTimes == 1) request.ReadWriteTimeout = Program.sysPara.BegRWTimeOut;
                else request.ReadWriteTimeout = Program.sysPara.BegRWTimeOut + Common.TypeConverter.ObjectToInt((Math.Pow(2, Program.CurrSpiderTimes - 1))) * Program.sysPara.IntervalRWTimeOut;
                request.Method = "POST";
                request.ContentType = "application/x-www-form-urlencoded";
                request.Accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
                request.CookieContainer = CookieContainer;
                request.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022)";
                request.Referer = SourceUrl;
                HttpRequestCachePolicy policy = new HttpRequestCachePolicy(HttpRequestCacheLevel.NoCacheNoStore);
                request.CachePolicy = policy;
                if (proxy.proxy)
                {
                    WebProxy webProxy = new WebProxy();                                      //定义一个网关对象
                    webProxy.Address = new Uri(proxy.proxyEntity.proxyAddess);              //网关服务器:端口
                    webProxy.Credentials = new NetworkCredential(proxy.proxyEntity.proxyUid, proxy.proxyEntity.proxyPwd);      //用戶名,密码
                    request.Proxy = webProxy;
                }
                ASCIIEncoding encoding = new ASCIIEncoding();
                byte[] byte1 = encoding.GetBytes(postDataStr);
                request.ContentLength = byte1.Length;
                myRequestStream = request.GetRequestStream();
                myRequestStream.Write(byte1, 0, byte1.Length);
                myRequestStream.Close();
                try
                {
                    response = (HttpWebResponse)request.GetResponse();
                }
                catch (WebException)
                {
                    response = (HttpWebResponse)request.GetResponse();
                }
                string ce = response.Headers[HttpResponseHeader.ContentEncoding];
                response.Cookies = CookieContainer.GetCookies(request.RequestUri);

                int ContentLength = (int)response.ContentLength;
                s = response.GetResponseStream();
                if (ContentLength < 0)
                {
                    data = new byte[c];
                    ms = new MemoryStream();
                    int l = s.Read(data, 0, c);
                    while (l > 0)
                    {
                        ms.Write(data, 0, l);
                        l = s.Read(data, 0, c);
                    }
                    data = ms.ToArray();
                    ms.Close();
                }
                else
                {
                    data = new byte[ContentLength];
                    int pos = 0;
                    int l;
                    while (ContentLength > 0)
                    {
                        l = s.Read(data, pos, ContentLength);
                        pos += l;
                        ContentLength -= l;
                    }
                }
                if (s != null) s.Close();
                if (request != null) request.Abort();
                string setCookieStr = response.GetResponseHeader("Set-Cookie");
                if (response != null) response.Close();
                if (ce == "gzip")
                {
                    js = new MemoryStream();           // 解压后的流  
                    ms = new MemoryStream(data);       // 用于解压的流  
                    g = new GZipStream(ms, CompressionMode.Decompress);
                    byte[] buffer = new byte[c];                    // 读数据缓冲区     
                    int l = g.Read(buffer, 0, c);                   // 一次读 10K     
                    while (l > 0)
                    {
                        js.Write(buffer, 0, l);
                        l = g.Read(buffer, 0, c);
                    }
                    g.Close();
                    ms.Close();
                    data = js.ToArray();
                    js.Close();
                }

                ContextStr = System.Text.Encoding.GetEncoding(EnCode).GetString(data);
            }
            catch (Exception ex)
            {
                if (request != null) request.Abort();
                if (response != null) response.Close();
                if (myRequestStream != null) myRequestStream.Close();
                if (myStreamWriter != null) myStreamWriter.Close();
                if (s != null) s.Close();
                if (ms != null) ms.Close();
                if (js != null) js.Close();
                if (g != null) g.Close();
                throw ex;
            }
            return ContextStr;
        }

这个是对get\post一种封装 我们看到每次进行请求需要重新实例化一次HttpWebRequest，HttpClient它可以只需实例化一次，就可以进行多次连接，大大减少了请求时间。直接贴代码

  public class HttpClient
    {
        private  readonly  System.Net.Http.HttpClient _client;//只需要实例化一次，就可以保持多次请求
        private HttpClientHandler handler;

        private string _userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36";
        private string _acceptLanguage = "zh-CN,zh;q=0.8";
        private string _accept = "text/html, application/xhtml+xml, */*";
        private string _acceptEncoding = "gzip, deflate";
        private string _charset = "GB2312";

        public string UserAgent { set { ChangeHeaders("User-Agent", value); } }
        public string AcceptLanguage { set { ChangeHeaders("Accept-Language", value); } }
        public string Accept { set { ChangeHeaders("Accept", value); } }
        public string Cookie { get; set; }
        public string AcceptEncoding { set { ChangeHeaders("Accept-Encoding", value); } }
        public string ContentType { get; set; } = "application/x-www-form-urlencoded";
        public string Charset { set { ChangeHeaders("Accept-Charset", value); } }
        public string TagCode { get; set; }

        public string exception;

        private void ChangeHeaders(string name, string newHeaders)
        {
            _client.DefaultRequestHeaders.Remove(name);
            _client.DefaultRequestHeaders.Add(name, newHeaders);
        }

        public HttpClient()
        {
            handler = new HttpClientHandler()
            {
                UseCookies = true,
                AllowAutoRedirect = true,
                AutomaticDecompression = DecompressionMethods.GZip
                //UseProxy = true,
                //Proxy = new WebProxy("218.64.147.211", 9000)
            };
            _client = new System.Net.Http.HttpClient(handler);
            _client.DefaultRequestHeaders.Add("Accept", _accept);
            _client.DefaultRequestHeaders.Add("Accept-Language", _acceptLanguage);
            _client.DefaultRequestHeaders.Add("Accept-Encoding", _acceptEncoding);
            _client.DefaultRequestHeaders.Add("User-Agent", _userAgent);
            _client.DefaultRequestHeaders.Add("Connection", "keep-alive");
            _client.DefaultRequestHeaders.Add("Accept-Charset", _charset);
        }

        public HttpClient(string cookie)
        {
            if (!string.IsNullOrEmpty(cookie))
            {
                CookieContainer cookieContainer = new CookieContainer();
                string[] coos = cookie.Split('|');
                foreach (var c in coos)
                {
                    string key = c.Split('=')[0];
                    string value = c.Split('=')[1];
                    string domain = c.Split('=')[2];
                    var coo = new Cookie(key, value, "/", domain);
                    cookieContainer.Add(coo);   // 加入Cookie
                }

                handler = new HttpClientHandler()
                {
                    CookieContainer = cookieContainer,
                    AllowAutoRedirect = true,
                    UseCookies = true,
                    //UseProxy=true,
                    //Proxy = new WebProxy("218.64.147.211",9000)
                };
                _client = new System.Net.Http.HttpClient(handler);
            }
            else
            {
                handler = new HttpClientHandler()
                {
                    UseCookies = true,
                    AllowAutoRedirect = true,
                    //UseProxy = true,
                    //Proxy = new WebProxy("218.64.147.211", 9000)
                };
                _client = new System.Net.Http.HttpClient(handler);
            }
            _client.DefaultRequestHeaders.Add("Accept", _accept);
            _client.DefaultRequestHeaders.Add("Accept-Language", _acceptLanguage);
            _client.DefaultRequestHeaders.Add("Accept-Encoding", _acceptEncoding);
            _client.DefaultRequestHeaders.Add("User-Agent", _userAgent);
            _client.DefaultRequestHeaders.Add("Connection", "keep-alive");
            _client.DefaultRequestHeaders.Add("Accept-Charset", _charset);
        }

        public bool useTagCode = false;

        public async Task<string> GetResponse( string referer = "",string url="", string method = "get", string data = "", string EnCode="")
        {
            if (!string.IsNullOrEmpty(referer)) _client.DefaultRequestHeaders.Referrer = new Uri(referer);

            HttpContent content = new StringContent(data);

            content.Headers.ContentType = new MediaTypeHeaderValue(ContentType);

            Uri uri = new Uri(url);

            //_client.DefaultRequestHeaders.Remove("Accept-Charset");
            //_client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Charset", EnCode);
            HttpResponseMessage res = null;
            try
            {
                if (method.ToLower().Equals("get"))
                {
                    res = await _client.GetAsync(uri);
                }
                else res = await _client.PostAsync(uri, content);

                if (res != null && res.IsSuccessStatusCode)
                {
                    try
                    {
                        CookieCollection cookieCollection = handler.CookieContainer.GetCookies(uri);
                        var list = new List<string>();
                        if (cookieCollection.Count > 0)
                        {
                            for (int i = 0; i < cookieCollection.Count; i++)
                            {
                                list.Add(cookieCollection[i].Name + "=" + cookieCollection[i].Value + "=" + cookieCollection[i].Domain);
                            }
                            Cookie = string.Join("|", list);
                        }
                    }
                    catch (Exception ex)
                    {
                        exception = ex.Message;
                        return null;
                    }

                    referer = res.RequestMessage.RequestUri.ToString();

                    if (referer.Contains("/(") && referer.Contains(")/"))
                    {
                        useTagCode = true;
                        TagCode = referer.Remove(0, referer.IndexOf("(") + 1);//获取识别码
                        TagCode = TagCode.Remove(TagCode.IndexOf(")"), TagCode.Length - TagCode.IndexOf(")"));
                    }

                    //res.Content.Headers.ContentType.CharSet = EnCode;
                    return await res.Content.ReadAsStringAsync();
                }
                else
                {
                    exception = res == null ? "服务器不响应" : Enum.GetName(typeof(HttpStatusCode), res.StatusCode);
                    return null;
                }
            }
            catch (Exception ex)
            {
                exception = ex.Message;
                return null;
            }
        }

        public async Task<Bitmap> GetPicture(string url, string method = "get", bool x2 = true, string data = "", string referer = "")
        {
            if (!string.IsNullOrEmpty(referer)) _client.DefaultRequestHeaders.Referrer = new Uri(referer);

            HttpContent content = new StringContent(data);
            content.Headers.ContentType = new MediaTypeHeaderValue("image/Gif");

            HttpResponseMessage res;

            if (method.ToLower().Equals("get"))
            {
                res = await _client.GetAsync(new Uri(url));
            }
            else res = await _client.PostAsync(new Uri(url), content);

            if (res.IsSuccessStatusCode)
            {
                var stream = await res.Content.ReadAsStreamAsync();
                Bitmap bitmap = new Bitmap(stream);
                return x2 ? ResizeImage(bitmap, bitmap.Width * 2, bitmap.Height * 2) : bitmap;
            }
            else return null;
        }

        /// <summary>
        /// 修改图片尺寸
        /// </summary>
        /// <param name="bmp"></param>
        /// <param name="newW"></param>
        /// <param name="newH"></param>
        /// <returns></returns>
        public static Bitmap ResizeImage(Bitmap bmp, int newW, int newH)
        {
            try
            {
                Bitmap b = new Bitmap(newW, newH);
                Graphics g = Graphics.FromImage(b);
                // 插值算法的质量
                g.InterpolationMode = InterpolationMode.HighQualityBicubic;
                g.DrawImage(bmp, new Rectangle(0, 0, newW, newH), new Rectangle(0, 0, bmp.Width, bmp.Height), GraphicsUnit.Pixel);
                g.Dispose();
                return b;
            }
            catch
            {
                return null;
            }
        }

        ~HttpClient()
        {
            _client.Dispose();
        }
    }

HttpClient httpClient = new HttpClient();
string content = httpClient.GetResponse(“”, “https://passport.alibaba.com/mini_login.htm?lang=zh_cn&appName=youmeng&appEntrance=default&styleType=auto&bizParams=&notLoadSsoView=true&notKeepLogin=false&isMobile=false&cssLink=https://passport.umeng.com/css/loginIframe.css&rnd=0.11412324061518286“, “”, “”);

你们的GetResponse

 public string GetResponse(string referer = "", string url = "", string method = "get", string data = "", string EnCode = "")
        {
            if (!string.IsNullOrEmpty(referer)) _client.DefaultRequestHeaders.Referrer = new Uri(referer);

            HttpContent content = new StringContent(data);

            content.Headers.ContentType = new MediaTypeHeaderValue(ContentType);

            Uri uri = new Uri(url);

            //_client.DefaultRequestHeaders.Remove("Accept-Charset");
            //_client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Charset", EnCode);
            HttpResponseMessage res = null;
            try
            {
                if (method.ToLower().Equals("get"))
                {
                    res =  _client.GetAsync(uri).Result;
                }
                else res =  _client.PostAsync(uri, content).Result;

                if (res != null && res.IsSuccessStatusCode)
                {
                    try
                    {
                        CookieCollection cookieCollection = handler.CookieContainer.GetCookies(uri);
                        var list = new List<string>();
                        if (cookieCollection.Count > 0)
                        {
                            for (int i = 0; i < cookieCollection.Count; i++)
                            {
                                list.Add(cookieCollection[i].Name + "=" + cookieCollection[i].Value + "=" + cookieCollection[i].Domain);
                            }
                            Cookie = string.Join("|", list);
                        }
                    }
                    catch (Exception ex)
                    {
                        exception = ex.Message;
                        return null;
                    }

                    referer = res.RequestMessage.RequestUri.ToString();

                    if (referer.Contains("/(") && referer.Contains(")/"))
                    {
                        useTagCode = true;
                        TagCode = referer.Remove(0, referer.IndexOf("(") + 1);//获取识别码
                        TagCode = TagCode.Remove(TagCode.IndexOf(")"), TagCode.Length - TagCode.IndexOf(")"));
                    }

                    //res.Content.Headers.ContentType.CharSet = EnCode;
                    return  res.Content.ReadAsStringAsync().Result;
                }
                else
                {
                    exception = res == null ? "服务器不响应" : Enum.GetName(typeof(HttpStatusCode), res.StatusCode);
                    return null;
                }
            }
            catch (Exception ex)
            {
                exception = ex.Message;
                return null;
            }
        }