asp.net中抓取远程页面，自动采集示例

由于远程网页可能采用多种编码，并且有可能采用Gzip格式来压缩数据，让我们采集页面时头疼不已，这里借鉴了一些网络上的资料，一段小程序，用来自动分析编码，自动解压缩gzip数据。

 

private   string  GetChartset( string  url)
         ... {
            string html = getHTML(url);
            Regex reg_charset = new Regex(@"charsets*=s*(?<charset>[^""]*)");
            string enconding = null;
            if (reg_charset.IsMatch(html))
            ...{
                enconding = reg_charset.Match(html).Groups["charset"].Value;
            }
            else
            ...{
                enconding = Encoding.Default.EncodingName;
            }
            if (enconding.ToLower().Contains("gb2312"))
                enconding = "gb2312";
            if (enconding.ToLower().Contains("utf-8"))
                enconding = "utf-8";
            return enconding;
        }

         private   string  getHTML( string  url)
         ... {

            try
            ...{
                WebRequest webRequest = WebRequest.Create(url);
                WebResponse webResponse = webRequest.GetResponse();
                Stream stream = webResponse.GetResponseStream();
                StreamReader sr = new StreamReader(stream, Encoding.GetEncoding(Encoding.ASCII.EncodingName));
                string html = sr.ReadToEnd();
                return html;
            }
            catch (UriFormatException ex)
            ...{

                Console.WriteLine(ex.Message);
                return null;
            }
            catch (WebException ex)
            ...{

                Console.WriteLine(ex.Message);
                return null;
            }
        }

         private   string  Html( string  Url)
         ... {
            string strResult = "";
            try
            ...{
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url);
                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
                string ce = response.ContentEncoding;
                Stream streamReceive = response.GetResponseStream();

                Encoding encoding = Encoding.GetEncoding(GetChartset(Url));
                if (ce.ToLower() == "gzip")//压缩的内容
                ...{
                    GZipStream gzip = new GZipStream(streamReceive, CompressionMode.Decompress);
                    using (StreamReader reader = new StreamReader(gzip,encoding))
                    ...{
                        strResult = reader.ReadToEnd();
                    }

                }
            }
            catch (Exception ex) ...{ HttpContext.Current.Response.Write(ex.ToString()); }
            return strResult;

        }



         protected   void  Button1_Click( object  sender, EventArgs e)
         ... {
            Response.Write(Html(TextBox1.Text));
        }



         /**/ /*
         使用起来很简单，下面的程序将字符串压缩入文件：

                using (DeflateStream gzip = new DeflateStream(fs, CompressionMode.Compress))
                {
                    byte[] buf = Encoding.UTF8.GetBytes(this.txbSource.Text);
                    gzip.Write(buf, 0, buf.Length);
                    gzip.Flush();
                }

        解压只需要这样：

            gzip = new GZipStream(new MemoryStream(buf), CompressionMode.Decompress);
            using (StreamReader reader = new StreamReader(gzip))
            {
                this.txbTarget.Text = reader.ReadToEnd();
            }

        如果从文件解压，只需要把MemoryStream换成一个FileStream就行了。


         */