asp.net 网页抓取内容

网页抓取代码

using System;
using System.Collections.Generic;
using System.Linq; using System.Web; // using System.Net; using System.IO; using System.Text.RegularExpressions; using System.Text; namespace WSYL.Web.Common { public static class GetSteamShipInfo { public static string GetWebSite(string steamshipname,int itype) { if (steamshipname == null || steamshipname.Trim() == "") return null; //step1: get html from url string urlToCrawl = @"网址"; //generate http request HttpWebRequest req = (HttpWebRequest)WebRequest.Create(urlToCrawl); //use GET method to get url's html req.Method = "GET"; //use request to get response HttpWebResponse resp = (HttpWebResponse)req.GetResponse(); // 二〇一五年八月十二日 18:14:45 需要增加判断网页解析超时问题 防止网页假死 // string htmlCharset = "UTF-8"; string htmlCharset = "utf-8"; //use songtaste's html's charset GB2312 to decode html //otherwise will return messy code Encoding htmlEncoding = Encoding.GetEncoding(htmlCharset); StreamReader sr = new StreamReader(resp.GetResponseStream(), htmlEncoding); //read out the returned html string respHtml = sr.ReadToEnd(); //第三种获取内容 //Match TitleMatch = Regex.Match(rtbExtractedHtml.Text.ToString(), "<td width=\"30%\">([^<]*)</td>", RegexOptions.IgnoreCase | RegexOptions.Multiline);
//需要获取的代码开始和结尾内容
 Match TitleMatch2 = Regex.Match(respHtml.ToString(), "<td align=\"left\" bgcolor=\"#EEEEEE\">([^<]*)</td>", RegexOptions.IgnoreCase | RegexOptions.Multiline); // txbExtractedInfo.Text = TitleMatch2.Groups[1].Value+"/"+ TitleMatch2.Groups[2].Value; if (TitleMatch2.Groups[1].Value.Length == 0 || TitleMatch2.Groups[1].Value=="") return respHtml = ""; if(itype==0) { respHtml = TitleMatch2.Groups[1].Value.ToString(); } if(itype==1) { respHtml = StripHtml(TitleMatch2.NextMatch().Value.ToString()); } if (itype == 2) { respHtml = TitleMatch2.Groups[1].Value + "/" + StripHtml(TitleMatch2.NextMatch().Value.ToString()); } return respHtml; } /// <summary> /// 去除html标签和空格有些例外会使得去除不干净，所以建议连续两次转化。这样将Html标签转化为了空格。太多连续的空格会影响之后对字符串的操作 /// </summary> /// <param name="strHtml">标签内容</param> /// <returns></returns> private static string StripHtml(string strHtml) { Regex objRegExp = new Regex("<(.|\n)+?>"); string strOutput = objRegExp.Replace(strHtml, ""); strOutput = strOutput.Replace("<", "&lt;"); strOutput = strOutput.Replace(">", "&gt;"); //把所有空格变为一个空格 Regex r = new Regex(@"\s+"); strOutput = r.Replace(strOutput, " "); return strOutput.Trim(); } } }

转载于:https://www.cnblogs.com/hs8888/p/5520564.html