在某些网站没有会发生"远程服务器返回错误: (403) 已禁止。"错误
则要加上request.userAgent属性
HttpWebRequest request = null; //提交
HttpWebResponse response = null; //返回
Request.Method = "POST";//数据提交方式
request.Host = "user.51sole.com"; //提交网址前缀
request.ContentType = "application/x-www-form-urlencoded; charset=UTF-8"; //提交的值编码形式
request.Headers.Add("Cookie:lang=zh-cn;" ) //手动添加提交cookie值
Request.AllowAutoRedirect = false; // 是否需要自动跳转
关于Cookie:
变量:
/// <summary>
/// Cookies 字符串,用来提交POSTCookie头,如果不加,默认提交是没有Cookie的
/// </summary>
public static string CookiesString
{
get;
set;
}
/// <summary>
/// 浏览器Cookie标识,new一次代表一个新的页面
/// </summary>
public static CookieContainer CookieContainer
{
get;
set;
}
代码里写:
if (CookieContainer != null)
{
//设置本次提交为一个页面
Request.CookieContainer = CookieContainer;
}
else
{
//new一个新页面
Request.CookieContainer = new CookieContainer();
//保存当前页面
CookieContainer = Request.CookieContainer;
}
//接收Response返回响应之前,保存返回cookie
Response.Cookies = Request.CookieContainer.GetCookies(Request.RequestUri); //给Response返回设置Cookie
CookiesString = Request.CookieContainer.GetCookieHeader(Request.RequestUri); //保存本次浏览器提交头的Cookie
//在提交头里加入,表示一个页面的提交
Request.Headers.Add("Cookie:" + CookiesString);
//如果是GET设置好提交头,和必要cookie返回数据即可
response = (HttpWebResponse)request.GetResponse();
StreamReader sr = new StreamReader(response.GetResponseStream(), Encoding.UTF8);
string content = sr.ReadToEnd(); //返回response数据
sr.Close();
if (response != null)
{
response.Close();
response = null;
}
request.Abort();
request= null;
如果是post,提交参数:
string postdata = "ProductName=咸丰重宝拍卖行情一路看涨&ProductID=0";//模拟请求数据
//提交请求
byte[] postdatabytes = Encoding.UTF8.GetBytes(postdata);
request.ContentLength = postdatabytes.Length;
using (Stream stream = request.GetRequestStream())
{
//设置POST 数据
stream.Write(postdatabytes, 0, postdatabytes.Length);
stream.Close();
}
response = (HttpWebResponse)request.GetResponse();
StreamReader sr = new StreamReader(response.GetResponseStream(), Encoding.UTF8);
string content = sr.ReadToEnd(); //返回response
sr.Close();
if (response != null)
{
response.Close();
response = null;
}
request.Abort();
request= null;
常用参数Post\GET封装:
/// <summary>
/// 浏览器Cookie标识,new一次代表一个新的页面
/// </summary>
public CookieContainer CookieContainer
{
get;
set;
}
/// <summary>
/// Cookies 字符创
/// </summary>
public string CookiesString
{
get;
set;
}
#region Post
public string Post(string url, Dictionary<object, object> param)
{
HttpWebRequest Request = null;
HttpWebResponse Response = null;
WebProxy proxy;
//如果需要到代理IP,可到这里检查返回IPhttp://ip.chinaz.com/
proxy = new WebProxy("http://123.57.76.102:80", true);
//如果需要到代理IP
GlobalProxySelection.Select = proxy;
try
{
//HttpWebResponse Response ;
Request = (HttpWebRequest)HttpWebRequest.Create(url);
Request.Method = "POST";
Request.ContentType = "application/x-www-form-urlencoded";
//Request.UserAgent = "RK_C# 1.1";
Request.UserAgent = "Mozilla / 5.0(Windows NT 6.1; Win64; x64) AppleWebKit / 537.36(KHTML, like Gecko) Chrome / 68.0.3440.75 Safari / 537.36";
//Request.CookieContainer = new CookieContainer();
Request.Headers.Add("Cookie:" + CookiesString);
String test = Request.Headers.Get("Cookie");
//Request.Timeout = 10000;
if (CookieContainer != null)
{
Request.CookieContainer = CookieContainer;
}
else
{
Request.CookieContainer = new CookieContainer();
CookieContainer = Request.CookieContainer;
}
#region POST方法
//如果需要POST数据
if (!(param == null || param.Count == 0))
{
StringBuilder buffer = new StringBuilder();
int i = 0;
foreach (string key in param.Keys)
{
if (i > 0)
{
buffer.AppendFormat("&{0}={1}", key, param[key]);
}
else
{
buffer.AppendFormat("{0}={1}", key, param[key]);
}
i++;
}
byte[] data = System.Text.Encoding.UTF8.GetBytes(buffer.ToString());
try
{
using (Stream stream = Request.GetRequestStream())
{
stream.Write(data, 0, data.Length);
}
}
catch (Exception e)
{
File.AppendAllText("错误url:" + url + "\r\n" + e.Message, System.Windows.Forms.Application.StartupPath + "\\error.txt");
Console.WriteLine(e.Message);
return e.Message;
//return "无法连接.请检查网络.";
//return e.Message;
}
}
#endregion
try
{
string responseStr = string.Empty;
using (Response = (HttpWebResponse)Request.GetResponse())
{
//Response = (HttpWebResponse)Request.GetResponse();
//保存返回cookie
Response.Cookies = Request.CookieContainer.GetCookies(Request.RequestUri);
CookiesString = Request.CookieContainer.GetCookieHeader(Request.RequestUri);
//CookieContainer.Add(Response.Cookies);
if (Response != null)
{
StreamReader reader = new StreamReader(Response.GetResponseStream(), Encoding.UTF8);
responseStr = reader.ReadToEnd();
reader.Close();
}
}
return responseStr;
}
catch (Exception e)
{
Console.WriteLine(e.Message);
return e.Message;
//return e.Message;
//return "无法连接.请检查网络.";
}
}
catch (Exception e)
{
return e.Message;
}
finally
{
if (Request != null)
{
Request.Abort();
}
if (Response != null)
{
Response.Close();
}
}
}
#endregion
#region GET
/// <summary>
/// 创建GET方式的HTTP请求
/// </summary>
/// <param name="url">请求的URL</param>
/// <param name="timeout">请求的超时时间,以毫秒为单位</param>
/// <param name="userAgent">请求的客户端浏览器信息,可以为空</param>
/// <param name="cookies">随同HTTP请求发送的Cookie信息,如果不需要身份验证可以为空</param>
/// <returns></returns>
public static String CreateGetHttpResponse(string url, int? timeout, CookieCollection cookies)
{
WebProxy proxy;
//如果需要到代理IP,可到这里检查返回IPhttp://ip.chinaz.com/
proxy = new WebProxy("http://123.57.76.102:80", true);
//如果需要到代理IP
GlobalProxySelection.Select = proxy;
if (string.IsNullOrEmpty(url))
{
throw new ArgumentNullException("url");
}
HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest;
request.Method = "GET";
request.UserAgent = "RK_C# 1.1";
if (timeout.HasValue)
{
request.Timeout = timeout.Value;
}
if (cookies != null)
{
request.CookieContainer = new CookieContainer();
request.CookieContainer.Add(cookies);
}
HttpWebResponse Response = (HttpWebResponse)request.GetResponse();
StreamReader reader = new StreamReader(Response.GetResponseStream(), Encoding.UTF8);
String value = reader.ReadToEnd();
reader.Close();
return value;
}
#endregion
}
常用参数Post调用:
var param = new Dictionary<object, object>
{
{"username","a1003434346"},
{"password","a2912695"},
{"typeid","5000"},
{"timeout","90"},
{"softid","102925"},
{"softkey","af8b9ca0addc4e0d8096891cc8466b5e"}
};
string httpResult = Post("http://api.ruokuai.com/create.xml", param, data);
带图片Post封装:
/// <summary>
/// HTTP POST方式请求数据(带图片)
/// </summary>
/// <param name="url">URL</param>
/// <param name="param">POST的数据</param>
/// <param name="fileByte">图片Byte</param>
/// <returns></returns>
public static string Post(string url, IDictionary<object, object> param, byte[] fileByte)
{
string boundary = "---------------------------" + DateTime.Now.Ticks.ToString("x");
byte[] boundarybytes = System.Text.Encoding.ASCII.GetBytes("\r\n--" + boundary + "\r\n");
HttpWebRequest wr = (HttpWebRequest)WebRequest.Create(url);
wr.ContentType = "multipart/form-data; boundary=" + boundary;
wr.UserAgent = "RK_C# 1.2";
wr.Method = "POST";
//wr.Timeout = 150000;
//wr.KeepAlive = true;
//wr.Credentials = System.Net.CredentialCache.DefaultCredentials;
Stream rs = null;
try
{
rs = wr.GetRequestStream();
}
catch { return "无法连接.请检查网络."; }
string responseStr = null;
string formdataTemplate = "Content-Disposition: form-data; name=\"{0}\"\r\n\r\n{1}";
foreach (string key in param.Keys)
{
rs.Write(boundarybytes, 0, boundarybytes.Length);
string formitem = string.Format(formdataTemplate, key, param[key]);
byte[] formitembytes = System.Text.Encoding.UTF8.GetBytes(formitem);
rs.Write(formitembytes, 0, formitembytes.Length);
}
rs.Write(boundarybytes, 0, boundarybytes.Length);
string headerTemplate = "Content-Disposition: form-data; name=\"{0}\"; filename=\"{1}\"\r\nContent-Type: {2}\r\n\r\n";
string header = string.Format(headerTemplate, "image", "i.gif", "image/gif");//image/jpeg
byte[] headerbytes = System.Text.Encoding.UTF8.GetBytes(header);
rs.Write(headerbytes, 0, headerbytes.Length);
rs.Write(fileByte, 0, fileByte.Length);
byte[] trailer = System.Text.Encoding.ASCII.GetBytes("\r\n--" + boundary + "--");
rs.Write(trailer, 0, trailer.Length);
rs.Close();
WebResponse wresp = null;
try
{
wresp = wr.GetResponse();
Stream stream2 = wresp.GetResponseStream();
StreamReader reader2 = new StreamReader(stream2);
responseStr = reader2.ReadToEnd();
}
catch
{
//throw;
}
finally
{
if (wresp != null)
{
wresp.Close();
wresp = null;
}
wr.Abort();
wr = null;
}
return responseStr;
}
获取图片:
HttpWebRequest request = null;
HttpWebResponse response = null;
request = (HttpWebRequest)WebRequest.Create(@"http://user.51sole.com/user/web/validatecode.aspx?tn=2018/5/4%208:44:59");
request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36";
request.Method = "GET";
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";
request.Headers.Add("Cookie:" + Cookie);
Stream stream = null;
response = (HttpWebResponse)request.GetResponse();
stream = response.GetResponseStream();
Image img = Image.FromStream(stream);
img.Save(Environment.CurrentDirectory + "\\test.gif");
带图片调用:
byte[] data;
//把Image转换为byte
using (MemoryStream ms = new MemoryStream())
{
img.Save(ms, img.RawFormat);
ms.Position = 0;
data = new byte[ms.Length];
ms.Read(data, 0, Convert.ToInt32(ms.Length));
ms.Flush();
}
var param = new Dictionary<object, object>
{
{"username","a1003434346"},
{"password","a2912695"},
{"typeid","5000"},
{"timeout","90"},
{"softid","102925"},
{"softkey","af8b9ca0addc4e0d8096891cc8466b5e"}
};
string httpResult = Post("http://api.ruokuai.com/create.xml", param, data);
未完待续...