使用HttpWebRequest与HttpWebResponse抓取网页数据

最近一个功能需要从其他网站上抓取数据存到本地数据库中，供后续展示与分析使用。这里通过使用HttpWebRequest模拟浏览器发送GET和POST请求到目标网站，然后通过HttpWebResponse获取目标服务器返回流作为字符串，然后通过html parser 获取需要保存的数据存储到数据库中。下面贴出两个获取页面的方法。
1，通过GET 获取页面

private static string getGETHtml(string url){
    string responsestr = "";
    HttpWebRequest req = HttpWebRequest.Create(url) as HttpWebRequest;
    req.Method = "GET";
    req.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
    req.ContentType = "application/x-www-form-urlencoded";
    // 模拟浏览器终端
    req.UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36";
    // 防止页面使用gzip压缩，获取数据乱码问题
    req.Headers["Accept-Encoding"] = "gzip, deflate";
    req.AutomaticDecompression = DecompressionMethods.GZip;
    using (HttpWebResponse response = req.GetResponse() as HttpWebResponse)
    {
                Stream stream;
                // 如果是gzip压缩，则对应解压缩
                if (response.ContentEncoding.ToLower().Contains("gzip"))
                {
                    stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress);
                }
                else if (response.ContentEncoding.ToLower().Contains("deflate"))
                {
                    stream = new DeflateStream(response.GetResponseStream(), CompressionMode.Decompress);
                }
                else
                {
                    stream = response.GetResponseStream();
                }
                using (StreamReader reader = new StreamReader(stream, System.Text.Encoding.Default))
                {
                    responsestr = reader.ReadToEnd();
                    stream.Dispose();
                }
            }
            return responsestr;
}

2，通过POST 获取页面

     private static string getPOSTHtml(string url,string postData) {
            string responsestr = "";
            HttpWebRequest req = HttpWebRequest.Create(url) as HttpWebRequest;
            req.Method = "POST";
            req.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
            req.ContentType = "application/x-www-form-urlencoded";
            req.UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36";
            req.Headers["Accept-Encoding"] = "gzip, deflate";
            req.AutomaticDecompression = DecompressionMethods.GZip;
            req.Timeout = 10000;
            req.KeepAlive = true;
            //req.ProtocolVersion = new Version("1.0"); 
            if(!string.IsNullOrEmpty(postData)){
                byte[] btBodys = Encoding.Default.GetBytes(postData);
                req.ContentLength = btBodys.Length;
                req.GetRequestStream().Write(btBodys, 0, btBodys.Length);
            }
            using (HttpWebResponse response = req.GetResponse() as HttpWebResponse)
            {
                Stream stream;
                if (response.ContentEncoding.ToLower().Contains("gzip"))
                {
                    stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress);
                }
                else if (response.ContentEncoding.ToLower().Contains("deflate"))
                {
                    stream = new DeflateStream(response.GetResponseStream(), CompressionMode.Decompress);
                }
                else
                {
                    stream = response.GetResponseStream();
                }
                using (StreamReader reader = new StreamReader(stream, System.Text.Encoding.Default))
                {
                    responsestr = reader.ReadToEnd();
                    stream.Dispose();
                }
            }
            return responsestr;
        }

获取页面之后就可以尽心html解析然后存储所需数据了。
html解析工具可以使用HtmlAgilityPack或者NSoup等。