C#网页数据采集（三）HttpWebRequest

1. <span style="font-family: Arial, Helvetica, sans-serif; background-color: rgb(255, 255, 255);">截取到网页数据是js加载完以后的</span>  

 

 

 

[csharp]  view plain  copy

 

1. <span style="white-space:pre">    </span>    HtmlWeb webClient = new HtmlWeb();  
2.           string _url = "http://news.baidu.com/";  
3.           HtmlAgilityPack.HtmlDocument html1 = webClient.Load(_url);//是你需要解析的url  
4.           var end3 = html1.Encoding.BodyName;//获取页面编码格式  
5.           string _htmlSource = GetHtmlSource(_url, System.Text.Encoding.GetEncoding(end3));//还是需要设置一次编码格式避免乱码 调用<span style="font-family: Arial, Helvetica, sans-serif;">GetHtmlSource方法</span>  

 

[csharp]  view plain  copy

 

1. 1. public static string GetHtmlSource(string url, Encoding charset)  
   2.         {  
   3.             string _html = string.Empty;  
   4.             try  
   5.             {  
   6.                 HttpWebRequest _request = (HttpWebRequest)WebRequest.Create(url);  
   7.                 HttpWebResponse _response = (HttpWebResponse)_request.GetResponse();  
   8.                 using (Stream _stream = _response.GetResponseStream())  
   9.                 {  
   10.                     using (StreamReader _reader = new StreamReader(_stream, charset))  
   11.                     {  
   12.                         _html = _reader.ReadToEnd();  
   13.                     }  
   14.                 }  
   15.             }  
   16.             catch (WebException ex)  
   17.             {  
   18.                 using (StreamReader sr = new StreamReader(ex.Response.GetResponseStream()))  
   19.                 {  
   20.                     _html = sr.ReadToEnd();  
   21.                 }  
   22.             }  
   23.             catch (Exception ex)  
   24.             {  
   25.                 _html = ex.Message;  
   26.             }  
   27.             return _html;  
   28.         }  

转载于:https://www.cnblogs.com/telwanggs/p/6477670.html