C# 模拟 Http 请求

1.支持 Gzip 和 deflate 压缩方式的响应信息

2.POST 可扩展提交文件

3.可设置代理(未测试过)

4.自动判断响应内容的字符集(简单判断，真实情况很复杂，有些无字符集描述的，或者 Response.Charset 是 ISO-**** 页面是 UTF-8 输出是 GBK ，环境很复杂，只能做到初步判断与转换)。

using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Web;

namespace System
{
    /// <summary>
    /// WebClient 请求工具类
    /// </summary>
    public static class HttpHelperV1
    {
        /// <summary>
        /// 浏览器标识
        /// 0：手机，1:IE
        /// </summary>
        public const string[] UserAgents = { 
"Mozilla/5.0(Linux;U;Android4.1.1;MI 2S Build/JRO03L)AppleWebKit/534.30 (KHTML,like Gecko)Version/4.0 Mobile Safari/534.30 XiaoMi/MiuiBrowser/1.0",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)"
                                            };
        #region POST

        /// <summary>
        /// 发送 POST 请求，并获取响应对象
        /// </summary>
        /// <param name="url">URL</param>
        /// <param name="UseAgent">浏览器标识</param>
        /// <param name="body">POST数据主体</param>
        /// <param name="head">头部信息</param>
        /// <param name="Proxy">代理信息</param>
        /// <returns></returns>
        public static HttpWebResponse PostAndGetResponse(string url, string UseAgent, byte[] body, Dictionary<string, string> head, IWebProxy Proxy)
        {
            HttpWebRequest Request = CreateRequest(url, head, UseAgent, Proxy);
            Request.Method = "POST";

            #region 设置主体

            if (body != null)
            {
                Request.ContentLength = body.Length;
                if (Request.ContentLength > 0)
                {
                    using (var requestStream = Request.GetRequestStream())
                    {
                        requestStream.Write(body, 0, body.Length);
                        requestStream.Close();
                    }
                }
            }
            else Request.ContentLength = 0;

            #endregion

            return (HttpWebResponse)Request.GetResponse();
        }
        /// <summary>
        /// 发送 POST 请求，并获取响应流
        /// </summary>
        /// <param name="url">URL</param>
        /// <param name="UseAgent">浏览器标识</param>
        /// <param name="param">参数</param>
        /// <param name="head">头部信息</param>
        /// <param name="Proxy">代理信息</param>
        /// <returns></returns>
        public static Stream PostAndGetStream(string url, string UseAgent, Dictionary<string, string> param, Dictionary<string, string> head, IWebProxy Proxy)
        {
            return PostAndGetResponse(url, UseAgent, Encoding.UTF8.GetBytes(param.paramToURL()), head, Proxy).GetStream();
        }
        /// <summary>
        /// 发送 POST 请求，并获取响应的 string 值
        /// </summary>
        /// <param name="url">URL</param>
        /// <param name="UseAgent">浏览器标识</param>
        /// <param name="param">参数</param>
        /// <param name="head">头部信息</param>
        /// <param name="Proxy">代理信息</param>
        /// <returns></returns>
        public static string PostAndGetString(string url, string UseAgent, Dictionary<string, string> param, Dictionary<string, string> head, IWebProxy Proxy)
        {
            return PostAndGetResponse(url, UseAgent, Encoding.UTF8.GetBytes(param.paramToURL()), head, Proxy).GetString();
        }

        #endregion

        #region GET

        /// <summary>
        /// 发送 GET 请求并获取响应对象
        /// </summary>
        /// <param name="url">URL</param>
        /// <param name="UseAgent">浏览器标识</param>
        /// <param name="head">头部信息</param>
        /// <param name="Proxy">代理信息</param>
        /// <returns></returns>
        public static HttpWebResponse GetAndGetResponse(string url, string UseAgent, Dictionary<string, string> head, IWebProxy Proxy)
        {
            //创建请求对象
            HttpWebRequest Request = CreateRequest(url, head, UseAgent, Proxy);
            //设置请求的发送为 GET 形式
            Request.Method = "GET";
            //返回响应对象
            return (HttpWebResponse)Request.GetResponse();
        }
        /// <summary>
        /// 发送 GET 请求，并获取响应的 string 值
        /// </summary>
        /// <param name="url">请求的URL</param>
        /// <param name="UseAgent">浏览器标识</param>
        /// <param name="param">发送的参数</param>
        /// <param name="head">发送的头部</param>
        /// <param name="Proxy">代理信息</param>
        /// <returns></returns>
        public static String GetAndGetString(string url, string UseAgent, Dictionary<string, string> param, Dictionary<string, string> head, IWebProxy Proxy)
        {
            return GetAndGetResponse(param.ParamInsertToURL(url), UseAgent, head, Proxy).GetString();
        }
        /// <summary>
        /// 发送 GET 请求，并获取响应的流
        /// </summary>
        /// <param name="url">请求的URL</param>
        /// <param name="UseAgent">浏览器标识</param>
        /// <param name="param">发送的参数</param>
        /// <param name="head">发送的头部</param>
        /// <param name="Proxy">代理信息</param>
        /// <returns></returns>
        public static Stream GetAndGetStream(string url, string UseAgent, Dictionary<string, string> param, Dictionary<string, string> head, IWebProxy Proxy)
        {
            return GetAndGetResponse(param.ParamInsertToURL(url), UseAgent, head, Proxy).GetStream();
        }

        #endregion

        #region 通用函数

        /// <summary>
        /// 获取响应流的副本
        /// </summary>
        /// <param name="Response"></param>
        /// <returns></returns>
        public static Stream GetStream(this HttpWebResponse Response)
        {
            //将响应流复制到结果流里返回给函数
            Stream result = new MemoryStream();
            Response.GetResponseStream().CopyTo(result);
            result.Position = 0;
            return result;
        }
        /// <summary>
        /// 将响应流转换成 String
        /// 自动处理字符集转换问题(简单转换)
        /// </summary>
        /// <param name="Response"></param>
        /// <returns></returns>
        public static string GetString(this HttpWebResponse Response)
        {
            string result = null;
            //设置副本流
            using (Stream responseStream = new MemoryStream())
            {
                //将响应流复制到副本流
                Response.GetResponseStream().CopyTo(responseStream);
                //设置默认字符集名称
                string encodStr = string.IsNullOrEmpty(Response.CharacterSet) ? "UTF-8" : Response.CharacterSet;
                
                bool isReadHtmlCharset = false; //已读取 html 中的字符集名称


            ReadStream: //读取流锚点
                responseStream.Position = 0;                        //将副本流的起始位置设为原点
                Encoding encoding = Encoding.GetEncoding(encodStr); //设置读取字符集
                using (MemoryStream tmp = new MemoryStream())       //设置临时流
                {
                    responseStream.CopyTo(tmp);                     //将副本流复制到临时流
                    tmp.Position = 0;                               //设置临时流的起始位置为原点
                    //按照设定的字符集读取临时流
                    using (var StreamReader = new StreamReader(tmp, encoding))
                    {
                        result = StreamReader.ReadToEnd();          //读取全部字符串
                    }
                }
                //若没从结果里面读取过 html 的 charset 的，就执行
                if (!isReadHtmlCharset)
                {
                    //将 html 多余的空格、换行、制表符等 \s 符号通通替换为一个空格，并转换成小写。
                    string regexSource = Regex.Replace(result, "\\s+", " ").ToLower(); //正则拼配源
                    
                    //正则表达式， 获取html上的字符集名称
                    string[] regexStrs = new string[] 
                        {                             
                        "<meta charset=\\s?[\"]?\\s?(?<charset>([\\w\\-]+))\\s?[\"]?\\s?[\\/]?>",
                        "<meta http-equiv=\\s?[\"]?\\s?content-type\\s?[\"]?\\s? content=\\s?[\"]?\\s?text/html; charset=(?<charset>([\\w\\-]*))\\s?[\"]?\\s?[\\/]?>"
                        };

                    foreach (var regexStr in regexStrs)
                    {
                        //获取匹配项
                        var match = Regex.Match(regexSource, regexStr);
                        if (match.Success) //成功匹配
                        {
                            //获取字符集名称
                            string tmp = match.Groups["charset"].Value;
                            if (!string.IsNullOrEmpty(tmp)) //字符集名称有具体数值
                            {
                                //构建异常处理，防止页面上的字符集名称不规范引发的异常
                                try
                                {
                                    //获取页面上字符集
                                    Encoding tmpEncoding = Encoding.GetEncoding(tmp);

                                    //若页面的字符集跟响应流的字符集不一致，则将字符集改成
                                    //页面字符集，重新读取一次 html
                                    if (tmpEncoding.EncodingName != encoding.EncodingName)
                                    {
                                        encodStr = tmp;
                                        isReadHtmlCharset = true;
                                        goto ReadStream; //返回读取流锚点再执行一次读取
                                    }
                                }
                                catch (Exception ex)
                                {

                                }
                            }
                        }
                    }

                }


            }
            return result;
        }
        /// <summary>
        /// 创建 Request
        /// </summary>
        /// <param name="url">完整的链接地址</param>
        /// <param name="head">头部信息</param>
        /// <param name="UserAgent">浏览器信息</param>
        /// <param name="proxy">代理</param>
        /// <returns></returns>
        public static HttpWebRequest CreateRequest(string url, Dictionary<string, string> head, string UserAgent, IWebProxy proxy)
        {
            //创建 Request 
            HttpWebRequest Request = (HttpWebRequest)HttpWebRequest.Create(url);
            Request.ContentType = "application/x-www-form-urlencoded";                                      //接受响应流的类型
            Request.Timeout = 60 * 1000;                                                                    //一分钟超时 
            Request.Proxy = proxy ?? Request.Proxy;                                                         //设置代理
            Request.UserAgent = UserAgent ?? UserAgents[1];                                                 //设置浏览器标识
            Request.Headers.Set("Accept-Encoding", "gzip, deflate");                                        //设置接受响应流的压缩方式
            Request.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;      //设置自动解压方式
            

            #region 设置头部

            if (head != null && head.Count > 0)
            {
                foreach (var key in head.Keys)
                {
                    if (Request.Headers.AllKeys.Contains(key))
                        Request.Headers.Set(key, head[key]);
                }
            }

            #endregion

            return Request;
        }
        /// <summary>
        /// 将参数拼接到 URL 之后
        /// </summary>
        /// <param name="param"></param>
        /// <param name="url"></param>
        /// <returns></returns>
        private static string ParamInsertToURL(this Dictionary<string, string> param, string url)
        {
            string result = url;
            //URL 与 param 都有值才进行处理，否则返回原 URL
            if (!string.IsNullOrEmpty(url) && param != null && param.Count > 0)
            {
                //判断中间拼接字符串选择，如果URL已带有参数，则用 & ，否则为 ?
                result = url + (url.Contains("?") ? "&" : "?") + param.paramToURL();
            }
            return url;
        }
        /// <summary>
        /// 键值对参数转成 URL 拼接字符串，Value 将做 UrlEncode 处理
        /// </summary>
        /// <param name="param"></param>
        /// <returns></returns>
        private static string paramToURL(this Dictionary<string, string> param)
        {
            string paramStr = string.Empty;
            //参数有值才处理，为空或无值就返回 string.Empty
            if (param != null && param.Count > 0)
            {
                //拼接参数
                //兼容 .net3.5-4.5 ,使用 Linq 扩展函数
                //范例：  "param1=456¶m2=test¶m3=%2094"
                paramStr = string.Join("&", param.Select(s => string.Format("{0}={1}", s.Key, HttpUtility.UrlEncode(s.Value))).ToArray());
            }
            return paramStr;
        }

        #endregion
    }
}