网页内空为压缩行式的测试代码

            try
            {
                req = (HttpWebRequest)HttpWebRequest.Create(strUrl);

                req.UserAgent = "Mozilla/5.0 (Windows NT 5.2; rv:6.0) Gecko/20100101 Firefox/6.0";
                req.Accept = "*/*";
                req.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5");
                req.ContentType = "text/xml";
                req.Timeout = 20000;

                // 设置访问来源地址，避免被检测到非本地址访问
                if (String.IsNullOrEmpty(referer))
                {
                    req.Referer = req.RequestUri.Host;
                }
                else
                {
                    req.Referer = referer;
                }

                // 设置 cookie 验证信息
                CookieContainer cc = new CookieContainer();
                req.CookieContainer = cc;

                foreach (Cookie cook in cookies)
                {
                    Cookie c = new Cookie(cook.Name, cook.Value);
                    if (cookieDomain != null)
                        c.Domain = cookieDomain;
                    cc.Add(c);
                }

                resp = (HttpWebResponse)req.GetResponse();

                Encoding enc;

                // 解决 .NET 编码识别错误的问题
                if ("ISO-8859-1" == resp.CharacterSet)
                {
                    // 使用指定的编码格式校正
                    enc = encoding;
                }
                else
                {
                    // 自动使用识别出来的编码格式
                    enc = Encoding.GetEncoding(resp.CharacterSet);
                }

                //if (isAjax)
                //{
                //    return GetAjaxUseWebBrowser(strUrl);
                //}

                string sHTML = string.Empty;



                // 从网站Url获取内容流
                stream = resp.GetResponseStream();

                // 判断内容是否使用gzip压缩
                if (resp.ContentEncoding.ToLower().Contains("gzip"))
                {
                    // 设置gzip解压缩
                    stream = new GZipStream(stream, CompressionMode.Decompress);
                    read = new StreamReader(stream, enc);
                    sHTML = read.ReadToEnd();
                }
                else
                {
                    string sChartSet = "";
                    read = new StreamReader(stream, enc);
                    sHTML = read.ReadToEnd();

                    // 获取内容使用的编码格式
                    Match charSetMatch = Regex.Match(sHTML, "charset=(?<code>[a-zA-Z0-9\\-]+)", RegexOptions.IgnoreCase);
                    sChartSet = charSetMatch.Groups["code"].Value;

                    //if it's not utf-8,we should redecode the html.
                    Regex rx = new Regex("([\u4e00-\u9fa5]{2,4})");

                    // 如果内容为UTF-8编码格式则进行再次转码
                    if (!rx.IsMatch(sHTML))
                    {
                        if (!string.IsNullOrEmpty(sChartSet.Trim()))
                            sHTML = Encoding.GetEncoding(sChartSet).GetString(enc.GetBytes(sHTML));
                    }
                }

                // 去除换行符
                sHTML = sHTML.Replace("\n", "").Replace("\r", "").Replace("\t", "");

                // 设置抓取任务状态为 1:=成功
                crawlerState = IWOMWebCrawlerDbLayer.Common.CrawlerState.Success;

                return sHTML;
            }
            catch (Exception ex)
            {
                CommonFunction.logWirte(ex.ToString() + strUrl, LogGrade.Warning);

                if (ex.Message.ToString().IndexOf("远程服务器返回错误: (503) 服务器不可用") > -1)
                {
                    crawlerState = IWOMWebCrawlerDbLayer.Common.CrawlerState.Forbat;
                }
                else
                {
                    crawlerState = IWOMWebCrawlerDbLayer.Common.CrawlerState.Error;
                }
            }
            finally
            {
                if (resp != null)
                {
                    resp.Close();
                }
                if (stream != null)
                {
                    stream.Close();
                }
                if (read != null)
                {
                    read.Close();
                }
            }