html获取远程数据,c#远程html数据抓取实例分享

///

/// 获取远程html

///

///

///

///

///

///

public static bool GetHttp(string url, string methed, string param, out string html)

{

methed = methed.ToLower();

if (param != null && methed == "get" && param.Length > 0)

{

url += "?" + param;

}

try

{

MSXML2.XMLHTTP mx = new MSXML2.XMLHTTPClass();

mx.open(methed, url, false, null, null);

if (param != null && methed == "post" && param.Length > 0)

{

mx.setRequestHeader("Content-Length", param.Length.ToString());

mx.setRequestHeader("Content-Type", "application/x-www-form-urlencoded");

}

mx.send(param);

if (mx.readyState != 4)

{

html = "远程连接失败：-4";

return false;

}

html = mx.responseText;

return true;

}

catch (Exception ex)

{

html = "远程连接失败："+ex.Message;

return false;

}

}

public static bool GetHttp1(string url, string methed, string param, string referer, string encode, out string html)

{

//return GetHttp(url,methed,param,out html);

//string encode = "utf-8";

//string methed = sendType.ToString();

if (param != null && methed == "get" && param.Length > 0)

{

if (url.IndexOf("?") >= 0)

{

url += "&" + param;

}

else

{

url += "?" + param;

}

}

try

{

HttpWebRequest webreq = (HttpWebRequest)WebRequest.Create(url);

webreq.Proxy=null;

webreq.Timeout = 1000 * 6;

webreq.ContentType = "application/x-www-form-urlencoded";

webreq.UserAgent = "User-Agent:Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0";

//webreq.UserAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/6.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)";

//谷歌的：User-Agent:Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36

//火狐的：User-Agent:Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0

//标准格式为： 浏览器标识 (操作系统标识; 加密等级标识; 浏览器语言) 渲染引擎标识 版本信息

//webreq.AllowAutoRedirect = false;

//频繁请求一个网址时，过段时间就会出现“基础连接已经关闭”

//webreq.KeepAlive = false;

//webreq.ProtocolVersion = HttpVersion.Version10;

if (referer.Length > 0)

{

webreq.Referer = referer;

}

CookieContainer mycookies = new CookieContainer();

webreq.CookieContainer = mycookies;

//if (this.cookieList != null)

//{

//    webreq.CookieContainer.Add(this.GetCookies(webreq.RequestUri, this.cookieList));

//}

webreq.Method = methed;

//post 开始

if (param != null && methed == "post")

{

byte[] arrbyte = Encoding.GetEncoding(encode).GetBytes(param);

webreq.ContentLength = arrbyte.Length;

Stream newStream = webreq.GetRequestStream();

newStream.Write(arrbyte, 0, arrbyte.Length);

newStream.Close();

}

//post 结束

WebResponse w = webreq.GetResponse();

//返回HTML

using (HttpWebResponse webres = (HttpWebResponse)webreq.GetResponse())

{

using (Stream dataStream = webres.GetResponseStream())

{

using (StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding(encode)))

{

html = reader.ReadToEnd();

//this.cookieList = webreq.CookieContainer.GetCookies(webreq.RequestUri);

webreq.Abort();//可能会解决卡住或阻塞问题

}

}

}

}

catch (Exception ex)

{

html = "出现异常(HttpHelper.GetHTML)，远程连接失败：" + ex.Message + " url：" + url;

//System.Windows.Forms.MessageBox.Show(html);

return false;

}

return true;

}