.NET的HttpWebRequest请求

默认常量

//默认UserAgent
private const string default_userAgent = "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.94 Safari/537.36";

//默认请求类型
private const string default_ContentType = "application/x-www-form-urlencoded";

1.直接获取页面内容

		/// <summary>
        /// 获取当前网页的内容
        /// </summary>
        /// <param name="httpUrl">Url地址</param>
        /// <param name="timeOut">请求响应超时时间，单位秒，默认30s</param>
        /// <param name="encoding">编码方式</param>
        /// <param name="userAgent">HTTP User-agent值</param>
        /// <param name="cookies">发送Cookie信息,常用于身份验证</param>
        /// <param name="isProxy">是否使用代理</param>
        /// <returns></returns>
        public static string GetUrlString(string httpUrl, int timeOut = 30, Encoding encoding = null, string userAgent = default_userAgent, CookieCollection cookies = null, bool isProxy=false)
        {
            string result = "";

            try
            {
                encoding = encoding ?? Encoding.UTF8;

                HttpWebRequest request = WebRequest.Create(httpUrl) as HttpWebRequest;
                request.UserAgent = userAgent;
                request.Timeout = timeOut * 1000;
                //发送Cookie信息
                if (cookies != null)
                {
                    request.CookieContainer = new CookieContainer();
                    request.CookieContainer.Add(cookies);
                }
                //设置代理 
                if (isProxy)
                {
                    WebProxy proxyObject = new WebProxy("ip", 端口号);
                    proxyObject.Credentials = new NetworkCredential("代理账号", "代理账号密码");
                    request.Proxy = proxyObject; 
                }

                //获取响应内容
                using (WebResponse wr = request.GetResponse())
                {
                    //在这里对接收到的页面内容进行处理
                    using (StreamReader reader = new StreamReader(wr.GetResponseStream(), encoding))
                    {
                        result = reader.ReadToEnd();
                    }
                }
            }
            catch (Exception ex)
            {
                //日志记录代码省略
            }

            return result;
        }

2.获取post请求后返回的内容

		/// <summary>
        /// 获取Post提交数据后返回的字符串
        /// </summary>
        /// <param name="httpUrl">Url地址</param>
        /// <param name="dataStr">Post数据</param>
        /// <param name="timeOut">请求响应超时时间，单位秒，默认30s</param>
        /// <param name="encoding">编码方式</param>
        /// <param name="userAgent">HTTP User-agent值</param>
        /// <param name="cookies">发送Cookie信息,常用于身份验证</param>
        /// <returns></returns>
        public static string PostUrlString(string httpUrl, string dataStr, int timeOut = 30, Encoding encoding = null,string ContentType= default_ContentType, string userAgent = default_userAgent, CookieCollection cookies = null, bool isProxy = false)
        {
            string result = "";

            try
            {
                encoding = encoding ?? Encoding.UTF8;

                HttpWebRequest request = WebRequest.Create(httpUrl) as HttpWebRequest;
                request.Method = "POST";
                request.ContentType = ContentType;
                request.UserAgent = userAgent;
                request.Timeout = timeOut * 1000;
                //发送Cookie信息
                if (cookies != null)
                {
                    request.CookieContainer = new CookieContainer();
                    request.CookieContainer.Add(cookies);
                }
                //设置代理 
                if (isProxy)
                {
                    WebProxy proxyObject = new WebProxy("ip", 端口);
                    proxyObject.Credentials = new NetworkCredential("代理账号", "代理密码");
                    request.Proxy = proxyObject;
                }

                //发送提交内容
                byte[] data = encoding.GetBytes(dataStr);
                using (Stream stream = request.GetRequestStream())
                {
                    stream.Write(data, 0, data.Length);
                }
                //获取响应内容
                using (WebResponse wr = request.GetResponse())
                {
                    //在这里对接收到的页面内容进行处理
                    using (StreamReader reader = new StreamReader(wr.GetResponseStream(), encoding))
                    {
                        result = reader.ReadToEnd();
                    }
                }
            }
            catch (Exception ex)
            {
                //日志记录
            }

            return result;
        }

3.下载指定文件的字节流

		/// <summary>
        /// 下载指定文件的字节流
        /// </summary>
        /// <param name="httpUrl">下载文件的地址</param>
        /// <param name="timeOut">下载文件超时时间，单位秒，默认60s</param>
        /// <param name="userAgent">HTTP User-agent值</param>
        /// <param name="cookies">发送Cookie信息,常用于身份验证</param>
        /// <returns></returns>
        public static Byte[] GetUrlByte(string httpUrl, int timeOut = 60, string userAgent = "", CookieCollection cookies = null)
        {
            try
            {
                HttpWebRequest request = WebRequest.Create(httpUrl) as HttpWebRequest;
                request.UserAgent = userAgent;
                request.Timeout = timeOut * 1000;
                //发送Cookie信息
                if (cookies != null)
                {
                    request.CookieContainer = new CookieContainer();
                    request.CookieContainer.Add(cookies);
                }
                //获取响应内容流
                using (WebResponse wr = request.GetResponse())
                {
                    using (Stream reader = wr.GetResponseStream())
                    {
                        int tempByte;
                        var tempStream = new MemoryStream();
                        while ((tempByte = reader.ReadByte()) != -1)
                        {
                            tempStream.WriteByte(((byte)tempByte));
                        }
                        return tempStream.ToArray();
                        //return reader.ToByte();
                    }
                }
            }
            catch (Exception ex)
            {
                //日志记录
            }

            return null;
        }