using System;
using System.Collections.Generic;
using System.Text;
using System.Net;
using System.IO;
using System.Net.Security;
using System.Drawing;
namespace Public
{
public class HttpRequestHelper
{
private BLL.ParamHelper _paramHelper;
/// <summary>
/// 参数处理类
/// </summary>
public BLL.ParamHelper ParamHelper
{
get { return _paramHelper; }
set { _paramHelper = value; }
}
/// <summary>
/// 获取html 源代码
/// </summary>
/// <param name="url"></param>
/// <param name="encoding"></param>
/// <param name="allowAutoRedirect"></param>
/// <returns></returns>
public static string GetHtml(string url, string encoding, bool allowAutoRedirect)
{
try
{
HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create(url);
// myHttpWebRequest.ContentType = "text/html";
myHttpWebRequest.ContentType = "application/x-www-form-urlencoded";
myHttpWebRequest.ProtocolVersion = HttpVersion.Version10;
myHttpWebRequest.Method = "GET";
myHttpWebRequest.AllowAutoRedirect = allowAutoRedirect;
myHttpWebRequest.KeepAlive = false;
myHttpWebRequest.Timeout = 120 * 1000;
myHttpWebRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
myHttpWebRequest.UserAgent = "Mozilla/5.0 (Windows NT 5.2; rv:6.0.2) Gecko/20100101 Firefox/6.0.2";
myHttpWebRequest.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5");
HttpWebResponse response = (HttpWebResponse)myHttpWebRequest.GetResponse();
System.IO.StreamReader sr = new System.IO.StreamReader(response.GetResponseStream(), Encoding.GetEncoding(encoding)); //utf-8
string html = sr.ReadToEnd();
sr.Close();
response.Close();
return html;
}
catch (Exception e)
{
return null;
}
}
/// <summary>
/// 发送http 请求,获取html
/// </summary>
/// <param name="rc"></param>
/// <param name="cookie"></param>
/// <returns></returns>
public string SendRequest_Html(string url, string encoding, string referer,string setCookie, List<Model.CommonParams> listParam, out string cookie)
{
if(url==null || url.Length==0){
cookie = "";
return "";
}
try
{
string param = "";
param = GetParamString(listParam, encoding);
if (param.Length > 0)
{
if (url.Contains("?"))
{
url += "&" + param;
}
else
{
url += "?" + param;
}
}
HttpWebRequest myHttpWebRequest = null;
//如果是发送HTTPS请求
if (url.StartsWith("https", StringComparison.OrdinalIgnoreCase))
{
ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult);
myHttpWebRequest = WebRequest.Create(url) as HttpWebRequest;
}
else
{
myHttpWebRequest = (HttpWebRequest)WebRequest.Create(url);
}
myHttpWebRequest.ProtocolVersion = HttpVersion.Version10;
myHttpWebRequest.ContentType = "application/x-www-form-urlencoded";//application/x-www-form-urlencoded
myHttpWebRequest.Method = "get";
myHttpWebRequest.AllowAutoRedirect = false;
myHttpWebRequest.KeepAlive = false;
myHttpWebRequest.Timeout = 60 * 1000;
myHttpWebRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
myHttpWebRequest.UserAgent = "Mozilla/5.0 (Windows NT 5.2; rv:6.0.2) Gecko/20100101 Firefox/6.0.2";
myHttpWebRequest.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5");
if (!string.IsNullOrEmpty(referer))
myHttpWebRequest.Referer = referer;
myHttpWebRequest.Headers["Cookie"] = setCookie;
HttpWebResponse response = (HttpWebResponse)myHttpWebRequest.GetResponse();
System.IO.StreamReader sr = new System.IO.StreamReader(response.GetResponseStream(), Encoding.GetEncoding(encoding)); //utf-8
if (response.Headers != null && response.Headers.Keys.Count > 0)
{
try
{
cookie = FormatCookies(response.Headers["Set-Cookie"],ref setCookie);
cookie = AppendCookie(setCookie, cookie);
}
catch (Exception ex) { cookie = ""; }
}
else
{
cookie = "";
}
string html = sr.ReadToEnd();
if (response.Headers != null && response.Headers.Keys.Count > 0)
{
try
{
string loc = response.Headers["Location"];
if (loc != null && loc.Length > 0)
{
loc = CreateUrl(url, loc);
if (!loc.Contains("weibo.com/unfreeze"))
{
html = SendRequest_Html(loc, encoding, url, setCookie, null, out cookie);
}
}
}
catch (Exception) { cookie = ""; }
}
sr.Close();
response.Close();
return html;
}
catch (Exception e)
{
cookie = "";
return null;
}
}
/// <summary>
/// 发送http 请求,获取html
/// </summary>
/// <param name="rc"></param>
/// <param name="cookie"></param>
/// <returns></returns>
public string SendRequest_Html(Model.HttpRequestConfig rc, out string cookie)
{
string location;
string html = SendRequest_Html(rc, out cookie, out location);
return html;
}
/// <summary>
/// 发送http 请求,获取html
/// </summary>
/// <param name="rc"></param>
/// <param name="cookie"></param>
/// <returns></returns>
public string SendRequest_Html(Model.HttpRequestConfig rc, out string cookie, out string location)
{
try
{
location = "";
string param = "";
string url = rc.RequestUrl;
if (rc.Method.Equals("get", StringComparison.OrdinalIgnoreCase))
{
param = GetParamString(rc.ListParam, rc.Encoding);
if (param.Length > 0)
{
if (url.Contains("?"))
{
url += "&" + param;
}
else
{
url += "?" + param;
}
}
}
else
{
param = GetParamString(rc.ListParam);
if(rc.Remark.Length > 0)
{
param += "&" +_paramHelper.GetParamValueByPattern(rc.Remark);
// param = "formhash=406bbc41&message=你好!%3A(&referer=home.php%3Fmod%3Dspace%26amp;uid%3D%26amp;do%3Dwall&handlekey=qcwall_102&idtype=uid&quickcomment=true&commentsubmit=true&commentsubmit_btn=true&id=102";
}
}
HttpWebRequest myHttpWebRequest = null;
//如果是发送HTTPS请求
if (url.StartsWith("https", StringComparison.OrdinalIgnoreCase))
{
ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult);
myHttpWebRequest = WebRequest.Create(url) as HttpWebRequest;
}
else {
myHttpWebRequest = (HttpWebRequest)WebRequest.Create(url);
}
myHttpWebRequest.ProtocolVersion = HttpVersion.Version10;
myHttpWebRequest.ContentType = "application/x-www-form-urlencoded";//application/x-www-form-urlencoded
myHttpWebRequest.Method = rc.Method;
myHttpWebRequest.AllowAutoRedirect = false;
myHttpWebRequest.KeepAlive = false;
myHttpWebRequest.Timeout = 120 * 1000;
myHttpWebRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
myHttpWebRequest.UserAgent = "Mozilla/5.0 (Windows NT 5.2; rv:6.0.2) Gecko/20100101 Firefox/6.0.2";
myHttpWebRequest.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5");
if (!string.IsNullOrEmpty(rc.Referer))
myHttpWebRequest.Referer = rc.Referer;
if (rc.IsSetCookie)
{
myHttpWebRequest.Headers["Cookie"] = rc.Cookie;
}
if ( rc.Method.Equals("post",StringComparison.OrdinalIgnoreCase))
{
string encoding = rc.Encoding;
if (url.Contains("pcbaby.com.cn/action/topic/create.jsp") || url.Contains("pcbaby.com.cn/action/post/create.jsp") || url.Contains("pcbaby.com.cn/bbs6/action/user/update.jsp"))
{
encoding = "utf-8";
}
byte[] bs = Encoding.GetEncoding(encoding).GetBytes(param);
myHttpWebRequest.ContentLength = bs.Length;
using (Stream reqStream = myHttpWebRequest.GetRequestStream())
{
reqStream.Write(bs, 0, bs.Length);
}
}
HttpWebResponse response = (HttpWebResponse)myHttpWebRequest.GetResponse();
System.IO.StreamReader sr = new System.IO.StreamReader(response.GetResponseStream(), Encoding.GetEncoding(rc.Encoding)); //utf-8
if (rc.IsGetCookie)
{
if (response.Headers != null && response.Headers.Keys.Count > 0)
{
try
{
string orgCookie = rc.Cookie;
cookie = FormatCookies(response.Headers["Set-Cookie"], ref orgCookie);
rc.Cookie = cookie = AppendCookie(orgCookie, cookie);
}
catch (Exception ex) { cookie = ""; }
}
else{cookie = "";}
}
else {cookie = "";}
string html = sr.ReadToEnd();
if (response.Headers != null && response.Headers.Keys.Count > 0 )
{
try
{
string loc = response.Headers["Location"];
if (loc!=null && loc.Length > 0)
{
Model.HttpRequestConfig hrc = rc.Clone();
hrc.RequestUrl = CreateUrl(hrc.RequestUrl,loc);
if (!HtmlHelper.IsMatch(hrc.RequestUrl, "(weibo.com/unfreeze)|(sorry\\?pagenotfound)"))
{
hrc.Referer = rc.RequestUrl;
hrc.IsSetCookie = true;
hrc.Method = "get";
html = SendRequest_Html(hrc, out cookie);
}
location = hrc.RequestUrl;
}
}
catch (Exception) { cookie = ""; }
}
sr.Close();
response.Close();
return html;
}
catch (Exception e)
{
location = "";cookie = "";
return null;
}
}
private bool CheckValidationResult(object sender, System.Security.Cryptography.X509Certificates.X509Certificate certificate,
System.Security.Cryptography.X509Certificates.X509Chain chain, SslPolicyErrors errors)
{
return true; //总是接受
}
/// <summary>
/// 发送http 请求,获取iamge
/// </summary>
/// <param name="rc"></param>
/// <param name="cookie"></param>
/// <returns></returns>
public Image SendRequest_Img(Model.HttpRequestConfig rc, out string cookie)
{
try
{
string param = "";
string url = rc.RequestUrl;
if (rc.Method.Equals("get", StringComparison.OrdinalIgnoreCase))
{
param = GetParamString(rc.ListParam, rc.Encoding);
if (url.Contains("?"))
{
url += "&" + param;
}
else
{
url += "?" + param;
}
}
else
{
param = GetParamString(rc.ListParam) ;
if (rc.Remark.Length > 0)
{
param += "&" + _paramHelper.GetParamValueByPattern(rc.Remark);
}
}
HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create(url);
myHttpWebRequest.ContentType = "application/x-www-form-urlencoded";
myHttpWebRequest.Method = rc.Method;
myHttpWebRequest.AllowAutoRedirect = false;
myHttpWebRequest.KeepAlive = false;
myHttpWebRequest.Timeout = 60 * 1000;
if(!string.IsNullOrEmpty(rc.Referer))
myHttpWebRequest.Referer = rc.Referer;
myHttpWebRequest.Accept = "image/png,image/*;q=0.8,*/*;q=0.5";
myHttpWebRequest.UserAgent = "Mozilla/5.0 (Windows NT 5.2; rv:6.0.2) Gecko/20100101 Firefox/6.0.2";
myHttpWebRequest.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5");
if(rc.IsSetCookie)
{
myHttpWebRequest.Headers["Cookie"] = rc.Cookie;
}
if (rc.Method.Equals("post", StringComparison.OrdinalIgnoreCase))
{
byte[] bs = Encoding.ASCII.GetBytes(param);
myHttpWebRequest.ContentLength = bs.Length;
using (Stream reqStream = myHttpWebRequest.GetRequestStream())
{
reqStream.Write(bs, 0, bs.Length);
}
}
HttpWebResponse response = (HttpWebResponse)myHttpWebRequest.GetResponse();
System.IO.Stream s = response.GetResponseStream(); //utf-8
if (rc.IsGetCookie)
{
if (response.Headers != null && response.Headers.Keys.Count > 0)
{
try
{
string orgCookie = rc.Cookie;
cookie = FormatCookies(response.Headers["Set-Cookie"], ref orgCookie);
rc.Cookie = cookie = AppendCookie(orgCookie, cookie);
}
catch (Exception ex) { cookie = ""; }
}
else{ cookie = "";}
}else{ cookie = "";}
Image img = Image.FromStream(s);
s.Close();
response.Close();
return img;
}
catch (Exception e)
{
cookie = "";
return null;
}
}
/// <summary>
/// 获取参数字符串
/// </summary>
/// <param name="list"></param>
/// <param name="encoding">编码</param>
/// <returns></returns>
private string GetParamString(List<Model.CommonParams> list, string encoding)
{
if (ParamHelper==null || list == null || list.Count == 0)
{
return "";
}
StringBuilder sb = new StringBuilder();
for (int i = 0; i < list.Count;i++ )
{
sb.AppendFormat("{0}={1}", list[i].ParamName, HtmlHelper.GetEncoding(HtmlHelper.ReplaceSpecialChar( ParamHelper.GetParamValue(list[i] )), encoding));
if (i < list.Count-1)
{
sb.Append("&");
}
}
return sb.ToString();
}
/// <summary>
/// 获取参数字符串
/// </summary>
/// <param name="list"></param>
/// <returns></returns>
private string GetParamString(List<Model.CommonParams> list)
{
if (ParamHelper == null || list == null || list.Count == 0)
{
return "";
}
StringBuilder sb = new StringBuilder();
string value;
for (int i = 0; i < list.Count; i++)
{
value = ParamHelper.GetParamValue(list[i]);
if (list[i].SystemName == Public.BLL.SystemParams.VerifyCode && value.Length ==0)
{
continue;
}
sb.AppendFormat("{0}={1}", list[i].ParamName, HtmlHelper.ReplaceSpecialChar(value));
if (i < list.Count - 1)
{
sb.Append("&");
}
}
return sb.ToString();
}
/// <summary>
/// 格式化cookie
/// </summary>
/// <param name="setCookie"></param>
/// <returns></returns>
private string FormatCookies(string setCookie)
{
string cookie = "";
return FormatCookies(setCookie, ref cookie);
}
/// <summary>
/// 格式化cookie
/// </summary>
/// <param name="setCookie"></param>
/// <returns></returns>
private string FormatCookies(string setCookie,ref string cookie)
{
if (string.IsNullOrEmpty(setCookie) || setCookie.Length ==0) {
return "";
}
setCookie = Public.HtmlHelper.Replace(setCookie, "(((?:expires)|(?:path)|(?:domain))=((?:.*?[,;])|(?:.*?$)))|(?:httponly,?)", "");
string[] cookies = System.Text.RegularExpressions.Regex.Split(setCookie, ";");
StringBuilder sb = new StringBuilder();
string key = "";
for (int i = cookies.Length - 1; i >= 0;i-- )
{
if (cookies[i].Contains("=") && cookies[i].Length > 3 && !HtmlHelper.IsMatch(cookies[i], "(?:=deleted)|(?:expires=)|(?:path=)|(?:domain=)"))
{
if (sb.Length > 0)
{
sb.Append(";");
}
sb.Append(cookies[i].Trim());
}
}
return sb.ToString();
}
/// <summary>
/// 合并cookie
/// </summary>
/// <param name="orgCookie"></param>
/// <param name="cookie"></param>
/// <returns></returns>
public string AppendCookie(string orgCookie,string cookie)
{
StringBuilder sb = new StringBuilder(cookie);
if (string.IsNullOrEmpty(orgCookie) || orgCookie.Length == 0)
{
return sb.ToString();
}
if (string.IsNullOrEmpty(cookie) || cookie.Length ==0)
{
return orgCookie == null ? "" : orgCookie;
}
string[] cookies = System.Text.RegularExpressions.Regex.Split(orgCookie, ";");
foreach (string c in cookies)
{
if (!cookie.Contains(c.Split('=')[0]))
{
if (sb.Length > 0)
{
sb.Append(";");
}
sb.Append(c);
}
}
return sb.ToString();
}
/// <summary>
/// 根据原来url和loction 生成完整的url
/// </summary>
/// <param name="url"></param>
/// <param name="loc"></param>
/// <returns></returns>
public static string CreateUrl(string url, string loc)
{
if (loc==null)
{
loc = "";
}
if (url == null || url.Length == 0)
{
return loc;
}
if (loc.ToLower().IndexOf("http") == 0) // 绝对路径
{
return loc;
}
else if (loc.ToLower().IndexOf("/") == 0) // 相对站点根目录路径
{
return url.Substring(0, (url.IndexOf('/', 8))) + loc;
}
else // 相对路径
{
return url.Substring(0, (url.Split('?')[0].LastIndexOf('/') + 1)) + loc;
}
}
/// <summary>
/// 获取站点编码
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
public static string GetEncoding(string url)
{
HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create(url);
myHttpWebRequest.ProtocolVersion = HttpVersion.Version10;
myHttpWebRequest.ContentType = "application/x-www-form-urlencoded";//application/x-www-form-urlencoded
myHttpWebRequest.Method = "get";
myHttpWebRequest.AllowAutoRedirect = true;
myHttpWebRequest.KeepAlive = false;
myHttpWebRequest.Timeout = 120 * 1000;
myHttpWebRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
myHttpWebRequest.UserAgent = "Mozilla/5.0 (Windows NT 5.2; rv:6.0.2) Gecko/20100101 Firefox/6.0.2";
myHttpWebRequest.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5");
string encoding;
HttpWebResponse response ;
try
{
response = (HttpWebResponse)myHttpWebRequest.GetResponse();
encoding = response.Headers["Content-Type"]; // Content-Type text/html;charset=utf-8
if (encoding != null && encoding.Length > 0)
{
encoding = HtmlHelper.GetHtmlContent(encoding, "charset=([\\w\\-]+)");
}
if (encoding==null || encoding.Length == 0)
{
System.IO.StreamReader sr = new System.IO.StreamReader(response.GetResponseStream(), Encoding.GetEncoding("utf-8")); //utf-8
encoding = sr.ReadToEnd();
if (encoding.Length > 0)
{
encoding= HtmlHelper.GetHtmlContent(encoding, "charset=[\"']?([\\w\\-]+)[\"']?");
}
sr.Close();
}
response.Close();
if (encoding == null)
{
return "";
}
encoding = encoding.ToLower();
if (encoding == "gbk")
{
return "gb2312";
}
return encoding;
}
catch (Exception ex)
{
return "";
}
}
/// <summary>
/// 检测网址是否discuz
/// </summary>
/// <param name="url"></param>
/// <param name="encoding"></param>
/// <param name="discuzRule"></param>
/// <returns></returns>
public static bool IsDicuz(string url, string encoding, string discuzRule)
{
string html = GetHtml(url, encoding, false);
if (html == null || html.Length<200) // 内容太少
{
return false;
}
return HtmlHelper.IsMatch(html, discuzRule);
}
/// <summary>
/// 将本地或者网络图片上传到指定的服务器(HttpWebRequest方法)
/// </summary>
/// <param name="address">文件上传到的服务器</param>
/// <param name="fileNamePath">要上传的本地文件(全路径)或者网络图片</param>
/// <param name="inputParamter">其他表单元素</param>
/// <param name="cookie"></param>
/// <returns>返回服务器的响应文本</returns>
public static string UploadIamge(string url, string fileNamePath, Dictionary<string, string> inputDic, string setCookie,string fileKey,out string location)
{
// 要上传的文件
byte[] bytes = GetBinaryReader(fileNamePath);
location = "";
if (bytes == null)
{
return "";
}
int index = fileNamePath.LastIndexOf("\\");
if (index==-1)
{
index = fileNamePath.LastIndexOf("/");
}
string saveName = fileNamePath.Substring(index + 1);
//时间戳
string strBoundary = "----------" + DateTime.Now.Ticks.ToString("x");
byte[] boundaryBytes = Encoding.ASCII.GetBytes("\r\n--" + strBoundary + "\r\n");
//请求头部信息
StringBuilder sb = new StringBuilder();
if (inputDic != null && inputDic.Count > 0)
{
//遍历字典取出表单普通空间的健和值
foreach (KeyValuePair<string, string> dicItem in inputDic)
{
sb.Append("--" + strBoundary);
sb.Append("\r\nContent-Disposition: form-data; name=\"" + dicItem.Key + "\"");
sb.Append("\r\n\r\n");
sb.Append(dicItem.Value);//value前面必须有2个换行
sb.Append("\r\n");
}
}
sb.Append("--");
sb.Append(strBoundary);
sb.AppendFormat("\r\nContent-Disposition: form-data; name=\"{0}\"; filename=\"",fileKey);
sb.Append(saveName);
sb.Append("\"\r\nContent-Type: image/jpeg\r\n\r\n");
string strPostHeader = sb.ToString();
byte[] postHeaderBytes = Encoding.UTF8.GetBytes(strPostHeader);
// 根据uri创建HttpWebRequest对象
HttpWebRequest httpReq = (HttpWebRequest)WebRequest.Create(new Uri(url));
httpReq.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
httpReq.UserAgent = "Mozilla/5.0 (Windows NT 5.2; rv:6.0.2) Gecko/20100101 Firefox/6.0.2";
httpReq.Method = "POST";
httpReq.AllowAutoRedirect = false;
//对发送的数据不使用缓存
httpReq.AllowWriteStreamBuffering = false;
httpReq.Headers["Cookie"] = setCookie == null ? "" : setCookie;
// httpReq.Referer = "http://weibo.com/rmmcn";
//设置获得响应的超时时间(300秒)
httpReq.Timeout = 300000;
httpReq.ContentType = "multipart/form-data; boundary=" + strBoundary;
long length = bytes.Length + postHeaderBytes.Length + boundaryBytes.Length;
long fileLength = bytes.Length;
httpReq.ContentLength = length;
try
{
//开始上传时间
DateTime startTime = DateTime.Now;
Stream postStream = httpReq.GetRequestStream();
//发送请求头部消息
postStream.Write(postHeaderBytes, 0, postHeaderBytes.Length);
postStream.Write(bytes, 0, bytes.Length);
//添加尾部的时间戳
postStream.Write(boundaryBytes, 0, boundaryBytes.Length);
postStream.Close();
//获取服务器端的响应
WebResponse webRespon = httpReq.GetResponse();
Stream s = webRespon.GetResponseStream();
StreamReader sr = new StreamReader(s);
//读取服务器端返回的消息
String html = sr.ReadLine();
if (webRespon.Headers != null && webRespon.Headers.Keys.Count > 0)
{
try
{
string loc = webRespon.Headers["Location"];
if (loc != null && loc.Length > 0)
{
location = CreateUrl(url, loc);
}
}
catch (Exception) { location = ""; }
}
sr.Close();
webRespon.Close();
return html;
}
catch { }
return "";
}
/// <summary>
/// 获取文件2进制流
/// </summary>
/// <param name="path"></param>
/// <returns></returns>
private static byte[] GetBinaryReader(string path)
{
try
{
if (HtmlHelper.IsMatch(path,"http://"))
{
return new WebClient().DownloadData(path);
}
else
{
FileStream fs = new FileStream(path, FileMode.Open, FileAccess.Read);
byte[] bytes = new byte[(int)fs.Length];
fs.Read(bytes, 0, bytes.Length);
fs.Close();
return bytes;
}
}
catch (Exception ex)
{
return null;
}
}
}
}
using System.Collections.Generic;
using System.Text;
using System.Net;
using System.IO;
using System.Net.Security;
using System.Drawing;
namespace Public
{
public class HttpRequestHelper
{
private BLL.ParamHelper _paramHelper;
/// <summary>
/// 参数处理类
/// </summary>
public BLL.ParamHelper ParamHelper
{
get { return _paramHelper; }
set { _paramHelper = value; }
}
/// <summary>
/// 获取html 源代码
/// </summary>
/// <param name="url"></param>
/// <param name="encoding"></param>
/// <param name="allowAutoRedirect"></param>
/// <returns></returns>
public static string GetHtml(string url, string encoding, bool allowAutoRedirect)
{
try
{
HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create(url);
// myHttpWebRequest.ContentType = "text/html";
myHttpWebRequest.ContentType = "application/x-www-form-urlencoded";
myHttpWebRequest.ProtocolVersion = HttpVersion.Version10;
myHttpWebRequest.Method = "GET";
myHttpWebRequest.AllowAutoRedirect = allowAutoRedirect;
myHttpWebRequest.KeepAlive = false;
myHttpWebRequest.Timeout = 120 * 1000;
myHttpWebRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
myHttpWebRequest.UserAgent = "Mozilla/5.0 (Windows NT 5.2; rv:6.0.2) Gecko/20100101 Firefox/6.0.2";
myHttpWebRequest.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5");
HttpWebResponse response = (HttpWebResponse)myHttpWebRequest.GetResponse();
System.IO.StreamReader sr = new System.IO.StreamReader(response.GetResponseStream(), Encoding.GetEncoding(encoding)); //utf-8
string html = sr.ReadToEnd();
sr.Close();
response.Close();
return html;
}
catch (Exception e)
{
return null;
}
}
/// <summary>
/// 发送http 请求,获取html
/// </summary>
/// <param name="rc"></param>
/// <param name="cookie"></param>
/// <returns></returns>
public string SendRequest_Html(string url, string encoding, string referer,string setCookie, List<Model.CommonParams> listParam, out string cookie)
{
if(url==null || url.Length==0){
cookie = "";
return "";
}
try
{
string param = "";
param = GetParamString(listParam, encoding);
if (param.Length > 0)
{
if (url.Contains("?"))
{
url += "&" + param;
}
else
{
url += "?" + param;
}
}
HttpWebRequest myHttpWebRequest = null;
//如果是发送HTTPS请求
if (url.StartsWith("https", StringComparison.OrdinalIgnoreCase))
{
ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult);
myHttpWebRequest = WebRequest.Create(url) as HttpWebRequest;
}
else
{
myHttpWebRequest = (HttpWebRequest)WebRequest.Create(url);
}
myHttpWebRequest.ProtocolVersion = HttpVersion.Version10;
myHttpWebRequest.ContentType = "application/x-www-form-urlencoded";//application/x-www-form-urlencoded
myHttpWebRequest.Method = "get";
myHttpWebRequest.AllowAutoRedirect = false;
myHttpWebRequest.KeepAlive = false;
myHttpWebRequest.Timeout = 60 * 1000;
myHttpWebRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
myHttpWebRequest.UserAgent = "Mozilla/5.0 (Windows NT 5.2; rv:6.0.2) Gecko/20100101 Firefox/6.0.2";
myHttpWebRequest.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5");
if (!string.IsNullOrEmpty(referer))
myHttpWebRequest.Referer = referer;
myHttpWebRequest.Headers["Cookie"] = setCookie;
HttpWebResponse response = (HttpWebResponse)myHttpWebRequest.GetResponse();
System.IO.StreamReader sr = new System.IO.StreamReader(response.GetResponseStream(), Encoding.GetEncoding(encoding)); //utf-8
if (response.Headers != null && response.Headers.Keys.Count > 0)
{
try
{
cookie = FormatCookies(response.Headers["Set-Cookie"],ref setCookie);
cookie = AppendCookie(setCookie, cookie);
}
catch (Exception ex) { cookie = ""; }
}
else
{
cookie = "";
}
string html = sr.ReadToEnd();
if (response.Headers != null && response.Headers.Keys.Count > 0)
{
try
{
string loc = response.Headers["Location"];
if (loc != null && loc.Length > 0)
{
loc = CreateUrl(url, loc);
if (!loc.Contains("weibo.com/unfreeze"))
{
html = SendRequest_Html(loc, encoding, url, setCookie, null, out cookie);
}
}
}
catch (Exception) { cookie = ""; }
}
sr.Close();
response.Close();
return html;
}
catch (Exception e)
{
cookie = "";
return null;
}
}
/// <summary>
/// 发送http 请求,获取html
/// </summary>
/// <param name="rc"></param>
/// <param name="cookie"></param>
/// <returns></returns>
public string SendRequest_Html(Model.HttpRequestConfig rc, out string cookie)
{
string location;
string html = SendRequest_Html(rc, out cookie, out location);
return html;
}
/// <summary>
/// 发送http 请求,获取html
/// </summary>
/// <param name="rc"></param>
/// <param name="cookie"></param>
/// <returns></returns>
public string SendRequest_Html(Model.HttpRequestConfig rc, out string cookie, out string location)
{
try
{
location = "";
string param = "";
string url = rc.RequestUrl;
if (rc.Method.Equals("get", StringComparison.OrdinalIgnoreCase))
{
param = GetParamString(rc.ListParam, rc.Encoding);
if (param.Length > 0)
{
if (url.Contains("?"))
{
url += "&" + param;
}
else
{
url += "?" + param;
}
}
}
else
{
param = GetParamString(rc.ListParam);
if(rc.Remark.Length > 0)
{
param += "&" +_paramHelper.GetParamValueByPattern(rc.Remark);
// param = "formhash=406bbc41&message=你好!%3A(&referer=home.php%3Fmod%3Dspace%26amp;uid%3D%26amp;do%3Dwall&handlekey=qcwall_102&idtype=uid&quickcomment=true&commentsubmit=true&commentsubmit_btn=true&id=102";
}
}
HttpWebRequest myHttpWebRequest = null;
//如果是发送HTTPS请求
if (url.StartsWith("https", StringComparison.OrdinalIgnoreCase))
{
ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult);
myHttpWebRequest = WebRequest.Create(url) as HttpWebRequest;
}
else {
myHttpWebRequest = (HttpWebRequest)WebRequest.Create(url);
}
myHttpWebRequest.ProtocolVersion = HttpVersion.Version10;
myHttpWebRequest.ContentType = "application/x-www-form-urlencoded";//application/x-www-form-urlencoded
myHttpWebRequest.Method = rc.Method;
myHttpWebRequest.AllowAutoRedirect = false;
myHttpWebRequest.KeepAlive = false;
myHttpWebRequest.Timeout = 120 * 1000;
myHttpWebRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
myHttpWebRequest.UserAgent = "Mozilla/5.0 (Windows NT 5.2; rv:6.0.2) Gecko/20100101 Firefox/6.0.2";
myHttpWebRequest.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5");
if (!string.IsNullOrEmpty(rc.Referer))
myHttpWebRequest.Referer = rc.Referer;
if (rc.IsSetCookie)
{
myHttpWebRequest.Headers["Cookie"] = rc.Cookie;
}
if ( rc.Method.Equals("post",StringComparison.OrdinalIgnoreCase))
{
string encoding = rc.Encoding;
if (url.Contains("pcbaby.com.cn/action/topic/create.jsp") || url.Contains("pcbaby.com.cn/action/post/create.jsp") || url.Contains("pcbaby.com.cn/bbs6/action/user/update.jsp"))
{
encoding = "utf-8";
}
byte[] bs = Encoding.GetEncoding(encoding).GetBytes(param);
myHttpWebRequest.ContentLength = bs.Length;
using (Stream reqStream = myHttpWebRequest.GetRequestStream())
{
reqStream.Write(bs, 0, bs.Length);
}
}
HttpWebResponse response = (HttpWebResponse)myHttpWebRequest.GetResponse();
System.IO.StreamReader sr = new System.IO.StreamReader(response.GetResponseStream(), Encoding.GetEncoding(rc.Encoding)); //utf-8
if (rc.IsGetCookie)
{
if (response.Headers != null && response.Headers.Keys.Count > 0)
{
try
{
string orgCookie = rc.Cookie;
cookie = FormatCookies(response.Headers["Set-Cookie"], ref orgCookie);
rc.Cookie = cookie = AppendCookie(orgCookie, cookie);
}
catch (Exception ex) { cookie = ""; }
}
else{cookie = "";}
}
else {cookie = "";}
string html = sr.ReadToEnd();
if (response.Headers != null && response.Headers.Keys.Count > 0 )
{
try
{
string loc = response.Headers["Location"];
if (loc!=null && loc.Length > 0)
{
Model.HttpRequestConfig hrc = rc.Clone();
hrc.RequestUrl = CreateUrl(hrc.RequestUrl,loc);
if (!HtmlHelper.IsMatch(hrc.RequestUrl, "(weibo.com/unfreeze)|(sorry\\?pagenotfound)"))
{
hrc.Referer = rc.RequestUrl;
hrc.IsSetCookie = true;
hrc.Method = "get";
html = SendRequest_Html(hrc, out cookie);
}
location = hrc.RequestUrl;
}
}
catch (Exception) { cookie = ""; }
}
sr.Close();
response.Close();
return html;
}
catch (Exception e)
{
location = "";cookie = "";
return null;
}
}
private bool CheckValidationResult(object sender, System.Security.Cryptography.X509Certificates.X509Certificate certificate,
System.Security.Cryptography.X509Certificates.X509Chain chain, SslPolicyErrors errors)
{
return true; //总是接受
}
/// <summary>
/// 发送http 请求,获取iamge
/// </summary>
/// <param name="rc"></param>
/// <param name="cookie"></param>
/// <returns></returns>
public Image SendRequest_Img(Model.HttpRequestConfig rc, out string cookie)
{
try
{
string param = "";
string url = rc.RequestUrl;
if (rc.Method.Equals("get", StringComparison.OrdinalIgnoreCase))
{
param = GetParamString(rc.ListParam, rc.Encoding);
if (url.Contains("?"))
{
url += "&" + param;
}
else
{
url += "?" + param;
}
}
else
{
param = GetParamString(rc.ListParam) ;
if (rc.Remark.Length > 0)
{
param += "&" + _paramHelper.GetParamValueByPattern(rc.Remark);
}
}
HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create(url);
myHttpWebRequest.ContentType = "application/x-www-form-urlencoded";
myHttpWebRequest.Method = rc.Method;
myHttpWebRequest.AllowAutoRedirect = false;
myHttpWebRequest.KeepAlive = false;
myHttpWebRequest.Timeout = 60 * 1000;
if(!string.IsNullOrEmpty(rc.Referer))
myHttpWebRequest.Referer = rc.Referer;
myHttpWebRequest.Accept = "image/png,image/*;q=0.8,*/*;q=0.5";
myHttpWebRequest.UserAgent = "Mozilla/5.0 (Windows NT 5.2; rv:6.0.2) Gecko/20100101 Firefox/6.0.2";
myHttpWebRequest.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5");
if(rc.IsSetCookie)
{
myHttpWebRequest.Headers["Cookie"] = rc.Cookie;
}
if (rc.Method.Equals("post", StringComparison.OrdinalIgnoreCase))
{
byte[] bs = Encoding.ASCII.GetBytes(param);
myHttpWebRequest.ContentLength = bs.Length;
using (Stream reqStream = myHttpWebRequest.GetRequestStream())
{
reqStream.Write(bs, 0, bs.Length);
}
}
HttpWebResponse response = (HttpWebResponse)myHttpWebRequest.GetResponse();
System.IO.Stream s = response.GetResponseStream(); //utf-8
if (rc.IsGetCookie)
{
if (response.Headers != null && response.Headers.Keys.Count > 0)
{
try
{
string orgCookie = rc.Cookie;
cookie = FormatCookies(response.Headers["Set-Cookie"], ref orgCookie);
rc.Cookie = cookie = AppendCookie(orgCookie, cookie);
}
catch (Exception ex) { cookie = ""; }
}
else{ cookie = "";}
}else{ cookie = "";}
Image img = Image.FromStream(s);
s.Close();
response.Close();
return img;
}
catch (Exception e)
{
cookie = "";
return null;
}
}
/// <summary>
/// 获取参数字符串
/// </summary>
/// <param name="list"></param>
/// <param name="encoding">编码</param>
/// <returns></returns>
private string GetParamString(List<Model.CommonParams> list, string encoding)
{
if (ParamHelper==null || list == null || list.Count == 0)
{
return "";
}
StringBuilder sb = new StringBuilder();
for (int i = 0; i < list.Count;i++ )
{
sb.AppendFormat("{0}={1}", list[i].ParamName, HtmlHelper.GetEncoding(HtmlHelper.ReplaceSpecialChar( ParamHelper.GetParamValue(list[i] )), encoding));
if (i < list.Count-1)
{
sb.Append("&");
}
}
return sb.ToString();
}
/// <summary>
/// 获取参数字符串
/// </summary>
/// <param name="list"></param>
/// <returns></returns>
private string GetParamString(List<Model.CommonParams> list)
{
if (ParamHelper == null || list == null || list.Count == 0)
{
return "";
}
StringBuilder sb = new StringBuilder();
string value;
for (int i = 0; i < list.Count; i++)
{
value = ParamHelper.GetParamValue(list[i]);
if (list[i].SystemName == Public.BLL.SystemParams.VerifyCode && value.Length ==0)
{
continue;
}
sb.AppendFormat("{0}={1}", list[i].ParamName, HtmlHelper.ReplaceSpecialChar(value));
if (i < list.Count - 1)
{
sb.Append("&");
}
}
return sb.ToString();
}
/// <summary>
/// 格式化cookie
/// </summary>
/// <param name="setCookie"></param>
/// <returns></returns>
private string FormatCookies(string setCookie)
{
string cookie = "";
return FormatCookies(setCookie, ref cookie);
}
/// <summary>
/// 格式化cookie
/// </summary>
/// <param name="setCookie"></param>
/// <returns></returns>
private string FormatCookies(string setCookie,ref string cookie)
{
if (string.IsNullOrEmpty(setCookie) || setCookie.Length ==0) {
return "";
}
setCookie = Public.HtmlHelper.Replace(setCookie, "(((?:expires)|(?:path)|(?:domain))=((?:.*?[,;])|(?:.*?$)))|(?:httponly,?)", "");
string[] cookies = System.Text.RegularExpressions.Regex.Split(setCookie, ";");
StringBuilder sb = new StringBuilder();
string key = "";
for (int i = cookies.Length - 1; i >= 0;i-- )
{
if (cookies[i].Contains("=") && cookies[i].Length > 3 && !HtmlHelper.IsMatch(cookies[i], "(?:=deleted)|(?:expires=)|(?:path=)|(?:domain=)"))
{
if (sb.Length > 0)
{
sb.Append(";");
}
sb.Append(cookies[i].Trim());
}
}
return sb.ToString();
}
/// <summary>
/// 合并cookie
/// </summary>
/// <param name="orgCookie"></param>
/// <param name="cookie"></param>
/// <returns></returns>
public string AppendCookie(string orgCookie,string cookie)
{
StringBuilder sb = new StringBuilder(cookie);
if (string.IsNullOrEmpty(orgCookie) || orgCookie.Length == 0)
{
return sb.ToString();
}
if (string.IsNullOrEmpty(cookie) || cookie.Length ==0)
{
return orgCookie == null ? "" : orgCookie;
}
string[] cookies = System.Text.RegularExpressions.Regex.Split(orgCookie, ";");
foreach (string c in cookies)
{
if (!cookie.Contains(c.Split('=')[0]))
{
if (sb.Length > 0)
{
sb.Append(";");
}
sb.Append(c);
}
}
return sb.ToString();
}
/// <summary>
/// 根据原来url和loction 生成完整的url
/// </summary>
/// <param name="url"></param>
/// <param name="loc"></param>
/// <returns></returns>
public static string CreateUrl(string url, string loc)
{
if (loc==null)
{
loc = "";
}
if (url == null || url.Length == 0)
{
return loc;
}
if (loc.ToLower().IndexOf("http") == 0) // 绝对路径
{
return loc;
}
else if (loc.ToLower().IndexOf("/") == 0) // 相对站点根目录路径
{
return url.Substring(0, (url.IndexOf('/', 8))) + loc;
}
else // 相对路径
{
return url.Substring(0, (url.Split('?')[0].LastIndexOf('/') + 1)) + loc;
}
}
/// <summary>
/// 获取站点编码
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
public static string GetEncoding(string url)
{
HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create(url);
myHttpWebRequest.ProtocolVersion = HttpVersion.Version10;
myHttpWebRequest.ContentType = "application/x-www-form-urlencoded";//application/x-www-form-urlencoded
myHttpWebRequest.Method = "get";
myHttpWebRequest.AllowAutoRedirect = true;
myHttpWebRequest.KeepAlive = false;
myHttpWebRequest.Timeout = 120 * 1000;
myHttpWebRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
myHttpWebRequest.UserAgent = "Mozilla/5.0 (Windows NT 5.2; rv:6.0.2) Gecko/20100101 Firefox/6.0.2";
myHttpWebRequest.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5");
string encoding;
HttpWebResponse response ;
try
{
response = (HttpWebResponse)myHttpWebRequest.GetResponse();
encoding = response.Headers["Content-Type"]; // Content-Type text/html;charset=utf-8
if (encoding != null && encoding.Length > 0)
{
encoding = HtmlHelper.GetHtmlContent(encoding, "charset=([\\w\\-]+)");
}
if (encoding==null || encoding.Length == 0)
{
System.IO.StreamReader sr = new System.IO.StreamReader(response.GetResponseStream(), Encoding.GetEncoding("utf-8")); //utf-8
encoding = sr.ReadToEnd();
if (encoding.Length > 0)
{
encoding= HtmlHelper.GetHtmlContent(encoding, "charset=[\"']?([\\w\\-]+)[\"']?");
}
sr.Close();
}
response.Close();
if (encoding == null)
{
return "";
}
encoding = encoding.ToLower();
if (encoding == "gbk")
{
return "gb2312";
}
return encoding;
}
catch (Exception ex)
{
return "";
}
}
/// <summary>
/// 检测网址是否discuz
/// </summary>
/// <param name="url"></param>
/// <param name="encoding"></param>
/// <param name="discuzRule"></param>
/// <returns></returns>
public static bool IsDicuz(string url, string encoding, string discuzRule)
{
string html = GetHtml(url, encoding, false);
if (html == null || html.Length<200) // 内容太少
{
return false;
}
return HtmlHelper.IsMatch(html, discuzRule);
}
/// <summary>
/// 将本地或者网络图片上传到指定的服务器(HttpWebRequest方法)
/// </summary>
/// <param name="address">文件上传到的服务器</param>
/// <param name="fileNamePath">要上传的本地文件(全路径)或者网络图片</param>
/// <param name="inputParamter">其他表单元素</param>
/// <param name="cookie"></param>
/// <returns>返回服务器的响应文本</returns>
public static string UploadIamge(string url, string fileNamePath, Dictionary<string, string> inputDic, string setCookie,string fileKey,out string location)
{
// 要上传的文件
byte[] bytes = GetBinaryReader(fileNamePath);
location = "";
if (bytes == null)
{
return "";
}
int index = fileNamePath.LastIndexOf("\\");
if (index==-1)
{
index = fileNamePath.LastIndexOf("/");
}
string saveName = fileNamePath.Substring(index + 1);
//时间戳
string strBoundary = "----------" + DateTime.Now.Ticks.ToString("x");
byte[] boundaryBytes = Encoding.ASCII.GetBytes("\r\n--" + strBoundary + "\r\n");
//请求头部信息
StringBuilder sb = new StringBuilder();
if (inputDic != null && inputDic.Count > 0)
{
//遍历字典取出表单普通空间的健和值
foreach (KeyValuePair<string, string> dicItem in inputDic)
{
sb.Append("--" + strBoundary);
sb.Append("\r\nContent-Disposition: form-data; name=\"" + dicItem.Key + "\"");
sb.Append("\r\n\r\n");
sb.Append(dicItem.Value);//value前面必须有2个换行
sb.Append("\r\n");
}
}
sb.Append("--");
sb.Append(strBoundary);
sb.AppendFormat("\r\nContent-Disposition: form-data; name=\"{0}\"; filename=\"",fileKey);
sb.Append(saveName);
sb.Append("\"\r\nContent-Type: image/jpeg\r\n\r\n");
string strPostHeader = sb.ToString();
byte[] postHeaderBytes = Encoding.UTF8.GetBytes(strPostHeader);
// 根据uri创建HttpWebRequest对象
HttpWebRequest httpReq = (HttpWebRequest)WebRequest.Create(new Uri(url));
httpReq.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
httpReq.UserAgent = "Mozilla/5.0 (Windows NT 5.2; rv:6.0.2) Gecko/20100101 Firefox/6.0.2";
httpReq.Method = "POST";
httpReq.AllowAutoRedirect = false;
//对发送的数据不使用缓存
httpReq.AllowWriteStreamBuffering = false;
httpReq.Headers["Cookie"] = setCookie == null ? "" : setCookie;
// httpReq.Referer = "http://weibo.com/rmmcn";
//设置获得响应的超时时间(300秒)
httpReq.Timeout = 300000;
httpReq.ContentType = "multipart/form-data; boundary=" + strBoundary;
long length = bytes.Length + postHeaderBytes.Length + boundaryBytes.Length;
long fileLength = bytes.Length;
httpReq.ContentLength = length;
try
{
//开始上传时间
DateTime startTime = DateTime.Now;
Stream postStream = httpReq.GetRequestStream();
//发送请求头部消息
postStream.Write(postHeaderBytes, 0, postHeaderBytes.Length);
postStream.Write(bytes, 0, bytes.Length);
//添加尾部的时间戳
postStream.Write(boundaryBytes, 0, boundaryBytes.Length);
postStream.Close();
//获取服务器端的响应
WebResponse webRespon = httpReq.GetResponse();
Stream s = webRespon.GetResponseStream();
StreamReader sr = new StreamReader(s);
//读取服务器端返回的消息
String html = sr.ReadLine();
if (webRespon.Headers != null && webRespon.Headers.Keys.Count > 0)
{
try
{
string loc = webRespon.Headers["Location"];
if (loc != null && loc.Length > 0)
{
location = CreateUrl(url, loc);
}
}
catch (Exception) { location = ""; }
}
sr.Close();
webRespon.Close();
return html;
}
catch { }
return "";
}
/// <summary>
/// 获取文件2进制流
/// </summary>
/// <param name="path"></param>
/// <returns></returns>
private static byte[] GetBinaryReader(string path)
{
try
{
if (HtmlHelper.IsMatch(path,"http://"))
{
return new WebClient().DownloadData(path);
}
else
{
FileStream fs = new FileStream(path, FileMode.Open, FileAccess.Read);
byte[] bytes = new byte[(int)fs.Length];
fs.Read(bytes, 0, bytes.Length);
fs.Close();
return bytes;
}
}
catch (Exception ex)
{
return null;
}
}
}
}