自动获取网页内容的方法
1.WebClient
{
if (args == null || args.Length == 0 )
{
throw new ApplicationException ( " Specify the URI of the resource to retrieve. " );
}
WebClient client = new WebClient ();
// Add a user agent header in case the
// requested URI contains a query.
client.Headers.Add ( " user-agent " , " Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;) " );
Stream data = client.OpenRead (args[ 0 ]);
StreamReader reader = new StreamReader (data);
string s = reader.ReadToEnd ();
Console.WriteLine (s);
data.Close ();
reader.Close ();
}
{
string contentHtml = "";
try
{
WebClient webClient = new WebClient();
//设置用于向Internet资源的请求进行身份验证的网络凭据
webClient.Credentials = CredentialCache.DefaultCredentials;
//设置下载数据的网址URL
Byte[] pageData = webClient.DownloadData("http://www.sowsoy.com");
//将网页数据转换为字符串形式,根据网页编码格式进行相应的转换
contentHtml = Encoding.Default.GetString(pageData);
//如果获取网站页面采用的是UTF-8,则使用这句
//contentHtml = Encoding.UTF8.GetString(pageData);
}
catch (WebException webEx)
{
Console.WriteLine(webEx.Message.ToString());
}
return contentHtml;
}
2.WebBrowser
public void GetHtmlContentFromWeb()
{
WebBrowser web = new WebBrowser();
web.Navigate("http://www.sowsoy.com");
web.DocumentCompleted += new WebBrowserDocumentCompletedEventHandler(web_DocumentCompleted);
}
void web_DocumentCompleted(object sender, WebBrowserDocumentCompletedEventArgs e)
{
string contentHtml = "";
WebBrowser web = (WebBrowser)sender;
HtmlElementCollection ElementCollection = web.Document.GetElementsByTagName("Table");
foreach (HtmlElement item in ElementCollection)
{
contentHtml += item.InnerText;
}
}
3.HttpWebRequest/HttpWebResponse
对于使用 HttpWebRequest 的客户端验证身份,客户端证书必须安装在当前用户的“我的证书”存储区中。
public string GetHtmlContentFromWeb()
{
string strBuff = "";
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create("http://www.sowsoy.com");
HttpWebResponse webResponse = (HttpWebResponse)webRequest.GetResponse();
Stream stream = webResponse.GetResponseStream();
StreamReader reader = new StreamReader(stream, System.Text.Encoding.GetEncoding("GB2312"));
strBuff = reader.ReadToEnd();
return strBuff;
}
private string GetWebContent(string sUrl)
{
string strResult = "";
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(sUrl);
//声明一个HttpWebRequest请求
request.Timeout = 3000000;
//设置连接超时时间
request.Headers.Set("Pragma", "no-cache");
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
if (response.ToString() != "")
{
Stream streamReceive = response.GetResponseStream();
Encoding encoding = Encoding.GetEncoding("UTF-8");
StreamReader streamReader = new StreamReader(streamReceive, encoding);
strResult = streamReader.ReadToEnd();
}
}
catch (Exception exp)
{
strResult = "";
}
return strResult;
}
public string GetWebContent3(string url)
{
string strBuff = "";
char[] cbuffer = new char[256];
int byteRead = 0;
Uri httpURL = new Uri(url);
//构造HttpWebRequest
HttpWebRequest httpReq = (HttpWebRequest)WebRequest.Create(httpURL);
//获取响应HttpWebResponse
HttpWebResponse httpResp = (HttpWebResponse)httpReq.GetResponse();
try
{
//GetResponseStream()方法获取HTTP响应的数据流,并尝试取得URL中所指定的网页内容
Stream respStream = httpResp.GetResponseStream();
StreamReader respStreamReader = new StreamReader(respStream, Encoding.UTF8);
//StreamReader类的Read方法依次读取网页源程序代码每一行的内容,直至行尾(读取的编码格式:UTF8)
byteRead = respStreamReader.Read(cbuffer, 0, 256);
while (byteRead != 0)
{
string strResp = new string(cbuffer, 0, byteRead);
strBuff = strBuff + strResp;
byteRead = respStreamReader.Read(cbuffer, 0, 256);
}
respStream.Close();
}
catch (Exception)
{
throw;
}
return strBuff;
}