用以获取网页源码并转码的方式。(新闻标题信息采集)

/**
 * 追加收录一转码方式 
 **/
var checker = new Unicode.Utf8Checker();
var isUtf8 = checker.Check(path);

System.IO.StreamReader reader;
if (isUtf8)
    reader = new System.IO.StreamReader(path);
else
    reader = new System.IO.StreamReader(path, System.Text.Encoding.GetEncoding("GB2312"));


/**
 * 用以获取请求页面源代码
 **/
private String getPageSource(String url) 
{
	String Source = "";
	try
	{
		//连接到网址
		WebClient web = new WebClient();
		Byte[] data = web.DownloadData(url);
		//获取响应报文头
		String sContentEncoding = web.ResponseHeaders["Content-Encoding"];
		//处理报文编码为gzip的情况
		if (sContentEncoding == "gzip")
		{
			MemoryStream ms = new MemoryStream(data);
			MemoryStream msTemp = new MemoryStream();
			int count = 0;
			GZipStream gzip = new GZipStream(ms, CompressionMode.Decompress);
			byte[] buf = new byte[10000];
			while ((count = gzip.Read(buf, 0, buf.Length)) > 0)
			{
				msTemp.Write(buf, 0, count);
			}
			data = msTemp.ToArray();
		}
		//获取网页编码格式
		String Content_Type = web.ResponseHeaders.Get("Content-Type");
		//获取流对象
		//StreamReader reader = null;
		Regex typeReg = new Regex("charset=.+", RegexOptions.IgnoreCase);
		if (typeReg.IsMatch(Content_Type))
		{
			String CodeType = Content_Type.Substring(Content_Type.IndexOf("=") + 1).ToUpper();
			//reader = new StreamReader(new MemoryStream(data), Encoding.GetEncoding(CodeType));
			//Source = reader.ReadToEnd();
			Source = Encoding.GetEncoding(CodeType).GetString(data);
		}
		else
		{
			//reader = new StreamReader(new MemoryStream(data), Encoding.UTF8);
			//Source = reader.ReadToEnd();
			Source = Encoding.GetEncoding("UTF-8").GetString(data);
			if (isGarbled(Source))
			{
				//reader = new StreamReader(new MemoryStream(data), Encoding.UTF8);
				//Source = reader.ReadToEnd();
				Source = Encoding.GetEncoding("GBK").GetString(data);
			}
		}
		Regex reg = new Regex(@"<(script|style)\s*.*?>\s*.*?\s*</(script|style)>", RegexOptions.IgnoreCase | RegexOptions.Singleline);
		Source = reg.Replace(Source, "");
		//关闭
		//reader.Close();
	}
	catch (WebException e)
	{
		throw e;
	}
	return Source.Replace("\"", "'");
}

/**
 * 获取所有新闻标题
 **/
private List<String> getNewsTitle(String url)
{
	List<String> strList = new List<String>();
	try
	{
		//<a\s*?(?=[^>]*?targer=){1}.*?href='[^']*?\.(html|shtml)'\s*?>.*?</a>
		//得到主页面
		string mainPage = getPageSource(url);
		//获取主页面所有新闻标题(可能一条匹配多条)
		//Regex mainReg = new Regex(@"<(li|h[1-9]?|p)\s*[^i]*?>\s*?(\[?<a\s*?(target='.*?')?\s*?href='http://.*?'\s*?
		//        (target='.*?')?\s*?>.*?</a>\]?)*?\s*?</\s*(li|h[1-9]?|p)\s*>", RegexOptions.IgnoreCase | RegexOptions.Singleline);
		//获取一级匹配结果所有新闻标题(单一匹配)
		//<a\s*?(target='_blank')?\s*?href='[^']*?news\.[^']*?\.(html|shtml)'\s*?(target='_blank')?[^>]*?>[^<]{3,}?</a>
		Regex titleReg = new Regex(@"<li>\s*?<a\s*?(target='?_blank'?)?\s*?href='[^']*?news\.[^']*?\.(htm|html|shtml)'\s*?" + 
			@"(target='?_blank'?)?[^>]*?>[^&<]{5,}?</a>\s*?</li>", RegexOptions.IgnoreCase);
		//迭代获取新闻标题存于集合中
		MatchCollection titleMc = titleReg.Matches(mainPage);
		foreach (Match t in titleMc)
		{
			if (!strList.Contains(t.Value))
			{
				strList.Add(t.Value);
			}
		}
	}
	catch (WebException e)
	{
		throw e;
	}
	return strList;
}

/**
 * 判断是否存在乱码(无详细解释，参考网络)
 **/
private Boolean isGarbled(String val)
{
	var bytes = Encoding.UTF8.GetBytes(val);
	//连续三个byte表示为239 191 189则为乱码
	for (var i = 0; i < bytes.Length; i++)
	{
		if (i < bytes.Length - 3)
		{
			if (bytes[i] == 239 && bytes[i + 1] == 191 && bytes[i + 2] == 189)
			{
				return true;
			}
		}
	}
	return false;
}