先设置4个:
[csharp]
webrequest.ServicePoint.Expect100Continue = false;
//是否使用 Nagle 不使用 提高效率
webrequest.ServicePoint.UseNagleAlgorithm = false;
//最大连接数
webrequest.ServicePoint.ConnectionLimit = 65500;
//数据是否缓冲 false 提高效率
webrequest.AllowWriteStreamBuffering = false;
代理的使用:我用的代理都是不要账号和密码的,因为要不是免费的,要是是我们自己的,对来访IP 有限制
[csharp]
if (proxy!=null)
{
model.proxy = proxy;
WebProxy myProxy = new WebProxy(proxy.ProxyInfo, false);
myProxy.Credentials = new NetworkCredential("", "");
webrequest.Proxy = myProxy;
}
else
{
webrequest.Proxy = GlobalProxySelection.GetEmptyWebProxy();
}
webrequest.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");
带上gzip,如果对方支持gzip,流量上可以节省五分之四,原来250KB,用gzip压缩有 大概在50KB,减少带宽压力,增加速度
值得注意的是,如果是gzip,返回值必须要gzip解压的。
[csharp]
if (response.ContentEncoding.ToLower().Contains("gzip"))
{
using (GZipStream stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress))
{
using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
{
model.html = reader.ReadToEnd();
}
}
}
还有一点:gzip解压的时候回耗费CPU,所以这个需要注意下
附完成方法
[csharp]
/// <summary>
/// Get 方式 获取数据
/// </summary>
/// <param name="webUrl"></param>
/// <returns></returns>
public Model.WebRequestReturnModel WebRequestGetHtmlByGet(string webUrl, CookieContainer cookieContainer, Encoding encoding, string refer,Model.Proxy proxy)
{
Model.WebRequestReturnModel model = new Model.WebRequestReturnModel();
HttpWebRequest webrequest = null;
try
{
webrequest = WebRequest.Create(webUrl) as HttpWebRequest;
webrequest.ServicePoint.Expect100Continue = false;
webrequest.ServicePoint.UseNagleAlgorithm = false;
webrequest.ServicePoint.ConnectionLimit = 65500;
webrequest.AllowWriteStreamBuffering = false;
if (proxy!=null)
{
model.proxy = proxy;
WebProxy myProxy = new WebProxy(proxy.ProxyInfo, false);
myProxy.Credentials = new NetworkCredential("", "");
webrequest.Proxy = myProxy;
}
else
{
webrequest.Proxy = GlobalProxySelection.GetEmptyWebProxy();
}
//设置其他的herader
webrequest =WebRequestSetHeaders(webrequest);
if (!String.IsNullOrEmpty(refer))
{
webrequest.Referer = refer;
}
else
{
webrequest.Referer = webUrl;
}
webrequest.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");
webrequest.CookieContainer = cookieContainer;
webrequest.Timeout = 5000;
webrequest.UserAgent = "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)";
webrequest.Accept = "*/*";
webrequest.KeepAlive = true;
webrequest.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5");
using (HttpWebResponse response = (HttpWebResponse)webrequest.GetResponse())
{
if (response.ContentEncoding.ToLower().Contains("gzip"))
{
using (GZipStream stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress))
{
using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
{
model.html = reader.ReadToEnd();
}
}
}
else if (response.ContentEncoding.ToLower().Contains("deflate"))
{
using (DeflateStream stream = new DeflateStream(response.GetResponseStream(), CompressionMode.Decompress))
{
using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
{
model.html = reader.ReadToEnd();
}
}
}
else
{
using (Stream stream = response.GetResponseStream())
{
using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
{
model.html = reader.ReadToEnd();
}
}
}
}
}
catch (Exception ex)
{
model.exception = ex;
model.html = string.Empty;
}
model.cookieContainer = cookieContainer;
return model;
}
HttpWebRequest提高效率之连接数,代理,自动跳转,gzip请求等设置问题
最新推荐文章于 2023-08-31 20:07:29 发布