C#用HttpWebRequest获取网页源码并自动检测编码

最近在做一个多功能的书签网，首先要解决的就是根据网址获取网页源码。下面是我的实现方法：

 

public string GetWebPage(string uri, string method) { try { HttpWebRequest req = (HttpWebRequest)WebRequest.Create(uri); req.Method = "GET"; req.Timeout = timeout; req.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 5.2; zh-CN; rv:1.9.1.4) Gecko/20091016 Firefox/3.5.4 (.NET CLR 3.5.30729)"; String ReturnedEncoding = ""; HttpWebResponse res = req.GetResponse() as HttpWebResponse; Encoding defaultEncoding = Encoding.UTF8; //判断类型 string type = ""; type = res.ContentType; if (type.IndexOf(";") > -1) { type = type.Split(new char[] { ';' })[0].Trim().ToLower(); } if (type != "text/html") { return ""; } //先判断编码 if (res.ContentType.ToLower().Replace(" ", "").IndexOf("charset") > -1) { ReturnedEncoding = res.ContentType.ToLower().Replace(" ", "").Substring(res.ContentType.ToLower().Replace(" ", "").IndexOf("charset=") + 8); if (ReturnedEncoding != "") defaultEncoding = Encoding.GetEncoding(ReturnedEncoding); } if (ReturnedEncoding == "") { ReturnedEncoding = res.ContentEncoding; if (ReturnedEncoding != "") defaultEncoding = Encoding.GetEncoding(ReturnedEncoding); } if (ReturnedEncoding == "") { ReturnedEncoding = res.CharacterSet; if (ReturnedEncoding != "") defaultEncoding = Encoding.GetEncoding(ReturnedEncoding); } Stream ReceiveStream = res.GetResponseStream(); StreamReader sr = new StreamReader(ReceiveStream, defaultEncoding); string ReturnedContent = sr.ReadToEnd(); //string h = "<meta http-equiv='Content-Type' content='text/html; charset=big5'>"; Regex reg_charset = new Regex(@"charset/b/s*=/s*(?<charset>[^""|^'']*)"); if (reg_charset.IsMatch(ReturnedContent)) { ReturnedEncoding = reg_charset.Match(ReturnedContent).Groups["charset"].Value; } if (ReturnedEncoding != "" && Encoding.GetEncoding(ReturnedEncoding) != defaultEncoding) { //重新加载 defaultEncoding = Encoding.GetEncoding(ReturnedEncoding); req = (HttpWebRequest)WebRequest.Create(uri); req.Method = "GET"; req.Timeout = timeout; req.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 5.2; zh-CN; rv:1.9.1.4) Gecko/20091016 Firefox/3.5.4 (.NET CLR 3.5.30729)"; res = req.GetResponse() as HttpWebResponse; ReceiveStream = res.GetResponseStream(); sr = new StreamReader(ReceiveStream, defaultEncoding); ReturnedContent = sr.ReadToEnd(); } return ReturnedContent; } catch { return ""; } } 

 

这个代码自有其弊端，就是当网址不是一个html网页，而是一个下载文件，那在获取response的时候，时间就会过长，时间过长，会长时间占用服务器资源，非常不利。

所以，解决方法当然是获取整个链接内容前，先得到content-type，或是html，再下载源码。

 

(实现网址：http://www.hao9483.com:8080/BM/Resister.aspx 该域名没有备案，80端口暂时处于被封阶段)