c# HTTPHelper

c# 爬虫常用的3个方法，备份一下

  1 using System;
  2 using System.Collections.Generic;
  3 using System.IO;
  4 using System.Linq;
  5 using System.Net;
  6 using System.Text;
  7 
  8 namespace MiSuMi
  9 {
 10     public class HttpHelper
 11     {
 12         public CookieContainer cookie;
 13         public HttpHelper()
 14         {
 15             cookie = new CookieContainer();
 16         }
 17 
 18         public string Get_Request(
 19             string strUrl,
 20             CookieContainer _cookie = null,
 21             string strHost = "",
 22             string strRefer = "",
 23             string strOrigin = "",
 24             Dictionary<string, string> lstHeads = null,
 25             string strEncoding = "utf-8",
 26             string strContentType = "",
 27             string strAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
 28             string strUserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
 29             bool blnAllowAutoRedirect = true,
 30             int intTimeout = 1000 * 30)
 31         {
 32             HttpWebRequest request;
 33             HttpWebResponse response;
 34             request = (HttpWebRequest)WebRequest.Create(strUrl);
 35             request.Accept = strAccept;
 36             request.Timeout = intTimeout;
 37             request.Method = "GET";
 38             request.Credentials = CredentialCache.DefaultCredentials;
 39             request.UserAgent = strUserAgent;
 40             request.AllowAutoRedirect = blnAllowAutoRedirect;
 41             if (!string.IsNullOrEmpty(strContentType))
 42             {
 43                 request.ContentType = strContentType;
 44             }
 45             if (_cookie != null)
 46             {
 47                 request.CookieContainer = _cookie;
 48             }
 49             if (!string.IsNullOrEmpty(strHost))
 50             {
 51                 request.Host = strHost;
 52             }
 53             if (!string.IsNullOrEmpty(strRefer))
 54             {
 55                 request.Referer = strRefer;
 56             }
 57             if (!string.IsNullOrEmpty(strOrigin))
 58             {
 59                 request.Headers.Add("Origin", strOrigin);
 60             }
 61             if (lstHeads != null && lstHeads.Count > 0)
 62             {
 63                 foreach (var item in lstHeads)
 64                 {
 65                     request.Headers.Add(item.Key, item.Value);
 66                 }
 67             }
 68             response = (HttpWebResponse)request.GetResponse();
 69             var sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(strEncoding));
 70             string strResult = sr.ReadToEnd();
 71             sr.Close();
 72             request.Abort();
 73             response.Close();
 74             return strResult;
 75 
 76         }
 77 
 78         public string POST_Request(
 79             string strUrl,
 80             string postDataStr,
 81             CookieContainer _cookie = null,
 82             string strHost = "",
 83             string strRefer = "",
 84             string strOrigin = "",
 85             Dictionary<string, string> lstHeads = null,
 86             string strEncoding = "utf-8",
 87             string strContentType = "",
 88             string strAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
 89             string strUserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
 90             bool blnAllowAutoRedirect = true,
 91             int intTimeout = 1000 * 30)
 92         {
 93             HttpWebRequest request;
 94             HttpWebResponse response;
 95             request = (HttpWebRequest)WebRequest.Create(strUrl);
 96             request.Accept = strAccept;
 97             request.Timeout = intTimeout;
 98             request.Method = "POST";
 99             request.Host = strHost;
100             request.UserAgent = strUserAgent;
101             if (_cookie != null)
102             {
103                 request.CookieContainer = _cookie;
104             }
105             request.AllowAutoRedirect = blnAllowAutoRedirect;
106             if (!string.IsNullOrEmpty(strContentType))
107             {
108                 request.ContentType = strContentType;
109             }
110             if (!string.IsNullOrEmpty(strOrigin))
111             {
112                 request.Headers.Add("Origin", strOrigin);
113             }
114             if (!string.IsNullOrEmpty(strRefer))
115             {
116                 request.Referer = strRefer;
117             }
118             if (!string.IsNullOrEmpty(strHost))
119             {
120                 request.Host = strHost;
121             }
122             if (lstHeads != null && lstHeads.Count > 0)
123             {
124                 foreach (var item in lstHeads)
125                 {
126                     request.Headers.Add(item.Key, item.Value);
127                 }
128             }
129             if (!string.IsNullOrEmpty(postDataStr))
130             {
131                 request.ContentLength = postDataStr.Length;
132                 Stream myRequestStream = request.GetRequestStream();
133                 StreamWriter myStreamWriter = new StreamWriter(myRequestStream);
134                 myStreamWriter.Write(postDataStr);
135                 myStreamWriter.Close();
136             }
137 
138             response = (HttpWebResponse)request.GetResponse();
139             var sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(strEncoding));
140             string strResult = sr.ReadToEnd();
141             sr.Close();
142             request.Abort();
143             response.Close();
144             return strResult;
145         }
146 
147         private string DownloadFile(
148             string strURLAddress,
149             string strPath,
150             CookieContainer _cookie = null,
151             string strHost = "",
152             string strRefer = "",
153             string strOrigin = "",
154             Dictionary<string, string> lstHeads = null,
155             string strAccept = "",
156             string strUserAgent = "")
157         {
158             try
159             {
160                 // 设置参数
161                 HttpWebRequest request = WebRequest.Create(strURLAddress) as HttpWebRequest;
162                 if (!string.IsNullOrEmpty(strAccept))
163                 {
164                     request.Accept = strAccept;
165                 }
166                 if (!string.IsNullOrEmpty(strUserAgent))
167                 {
168                     request.UserAgent = strUserAgent;
169                 }
170                 if (_cookie != null)
171                 {
172                     request.CookieContainer = _cookie;
173                 }
174                 if (!string.IsNullOrEmpty(strOrigin))
175                 {
176                     request.Headers.Add("Origin", strOrigin);
177                 }
178                 if (!string.IsNullOrEmpty(strRefer))
179                 {
180                     request.Referer = strRefer;
181                 }
182                 if (!string.IsNullOrEmpty(strHost))
183                 {
184                     request.Host = strHost;
185                 }
186                 if (lstHeads != null && lstHeads.Count > 0)
187                 {
188                     foreach (var item in lstHeads)
189                     {
190                         request.Headers.Add(item.Key, item.Value);
191                     }
192                 }
193                 //发送请求并获取相应回应数据
194                 HttpWebResponse response = request.GetResponse() as HttpWebResponse;
195                 string strReceivePath = string.Empty;
196 
197                 //直到request.GetResponse()程序才开始向目标网页发送Post请求
198                 Stream responseStream = response.GetResponseStream();
199                 //创建本地文件写入流
200                 Stream stream = new FileStream(strPath, FileMode.Create);
201                 byte[] bArr = new byte[1024];
202                 int size = responseStream.Read(bArr, 0, (int)bArr.Length);
203                 while (size > 0)
204                 {
205                     stream.Write(bArr, 0, size);
206                     stream.Flush();
207                     size = responseStream.Read(bArr, 0, (int)bArr.Length);
208                 }
209                 stream.Close();
210                 responseStream.Close();
211                 return strPath;
212             }
213             catch (Exception ex)
214             {
215                 return "";
216             }
217         }
218 
219     }
220 }

View Code

常用的方法，参数就这些了，后面如果有用到其他的再补充了

转载于:https://www.cnblogs.com/bfyx/p/6827092.html