linux 提取网页内容,c#简单实现提取网页内容

下面的代码是从一个网络爬虫程序中提取出来的，觉得有用，记录下来。

代码

using System;

using System.Collections;

using System.Collections.Generic;

using System.Reflection;

using System.IO;

using System.Net;

using System.Text;

namespace MyCsStudy

{

class Program

{

/// /// 简单网络爬虫程序

///

///

/// 编码 可以为空

///

public static string Fetch(string url, string charset)

{

Encoding encoding;

HttpWebRequest request;

HttpWebResponse response = null;

Stream resStream = null;

StreamReader sr = null;

string result = string.Empty;

try

{

request = (HttpWebRequest)HttpWebRequest.Create(url);

response = (HttpWebResponse)request.GetResponse();

resStream = response.GetResponseStream();

if (!string.IsNullOrEmpty(charset))

{

encoding = Encoding.GetEncoding(charset);

}

else if (!string.IsNullOrEmpty(response.CharacterSet))

{

encoding = Encoding.GetEncoding(response.CharacterSet);

}

else

{

encoding = Encoding.Default;

}

sr = new StreamReader(resStream, encoding);

result = sr.ReadToEnd();

}

//catch (Exception ex)

//{

//    throw ex;

//}

finally

{

if (sr != null)

{

sr.Close();

}

if (resStream != null)

{

resStream.Close();

}

if (response != null)

{

response.Close();

}

}

return result;

}

static void Main(string[] args)

{

string "; //这里url必须带上协议

string strHTML = Fetch(webSite,null);

Console.Write(strHTML);

Console.ReadLine();

}

}

}