一般情况下,如果无特殊的需要,对于抓去网页数据,可以使用WebClient,也可以使用webrequest/webresponse这对组合,不过我使用webrequest/webresponse的时候比较多一些,所以,自己整理了一个网页抓取类,上代码:
Code
1/**//// <summary>
2 /// Web下载工具
3 /// </summary>
4 public class WebDownload : IDisposable
5 {
6 事件#region 事件
7
8 /**//// <summary>
9 /// 下载完成事件
10 /// </summary>
11 //public event EventHandler DownloadComplete;
12
13 #endregion
14
15 字段#region 字段
16 /**//// <summary>
17 /// 请求对象
18 /// </summary>
19 private HttpWebRequest m_request;
20
21 /**//// <summary>
22 /// 请求返回对象
23 /// </summary>
24 private HttpWebResponse m_response;
25
26 /**//// <summary>
27 /// 请求URL
28 /// </summary>
29 public string Url = string.Empty;
30
31 /**//// <summary>
32 /// Method
33 /// </summary>
34 public string Method = string.Empty;
35
36 /**//// <summary>
37 /// 数据类型
38 /// </summary>
39 public string ContentType = string.Empty;
40
41 /**//// <summary>
42 /// referer
43 /// </summary>
44 public string Referer = string.Empty;
45
46 /**//// <summary>
47 /// 是否允许重定向
48 /// </summary>
49 public bool AllowAutoRedirect = true;
50
51 /**//// <summary>
52 /// accept
53 /// </summary>
54 public string Accept = string.Empty;
55
56 /**//// <summary>
57 /// 提交的数据
58 /// </summary>
59 public string PostData = string.Empty;
60
61 /**//// <summary>
62 /// cookies
63 /// </summary>
64 public CookieContainer Cookies = null;
65
66 /**//// <summary>
67 /// 连接超时时间
68 /// </summary>
69 public int Timeout = 100000;
70
71 /**//// <summary>
72 /// 编码类型
73 /// </summary>
74 public Encoding Encode = Encoding.Default;
75
76 /**//// <summary>
77 /// useragent
78 /// </summary>
79 public string UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)";
80
81 /**//// <summary>
82 /// 请求返回的数据流
83 /// </summary>
84 //public Stream ReceiveData = null;
85 #endregion
86
87 构造函数#region 构造函数
88 /**//// <summary>
89 /// 下载工具
90 /// </summary>
91 /// <param name="url"></param>
92 /// <param name="method"></param>
93 public WebDownload(string url, string method) : this( url,method,string.Empty )
94 {
95 }
96
97 /**//// <summary>
98 /// 下载工具
99 /// </summary>
100 /// <param name="url"></param>
101 /// <param name="method"></param>
102 /// <param name="postData"></param>
103 public WebDownload(string url, string method, string postData) : this( url,method,postData,"application/x-www-form-urlencoded" )
104 {
105 }
106
107 /**//// <summary>
108 /// 下载工具
109 /// </summary>
110 /// <param name="url"></param>
111 /// <param name="method"></param>
112 /// <param name="postData"></param>
113 /// <param name="contentType"></param>
114 public WebDownload(string url, string method, string postData, string contentType) : this( url,method,postData,contentType,"image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, application/x-silverlight, */*" )
115 {
116 }
117
118 /**//// <summary>
119 /// 下载工具
120 /// </summary>
121 /// <param name="url"></param>
122 /// <param name="method"></param>
123 /// <param name="postData"></param>
124 /// <param name="contentType"></param>
125 /// <param name="accept"></param>
126 public WebDownload(string url, string method, string postData, string contentType, string accept) : this( url,method,postData,contentType,accept,string.Empty )
127 {
128 }
129
130 /**//// <summary>
131 /// 下载工具
132 /// </summary>
133 /// <param name="url"></param>
134 /// <param name="method"></param>
135 /// <param name="postData"></param>
136 /// <param name="contentType"></param>
137 /// <param name="accept"></param>
138 /// <param name="referer"></param>
139 public WebDownload(string url, string method, string postData, string contentType, string accept,string referer) : this( url,method,postData,contentType,accept,referer,true )
140 {
141 }
142
143 /**//// <summary>
144 /// 下载工具
145 /// </summary>
146 /// <param name="url"></param>
147 /// <param name="method"></param>
148 /// <param name="postData"></param>
149 /// <param name="contentType"></param>
150 /// <param name="accept"></param>
151 /// <param name="referer"></param>
152 /// <param name="autoredirect"></param>
153 public WebDownload(string url, string method, string postData, string contentType, string accept, string referer,bool autoredirect)
154 {
155 Url = url;
156 Method = method;
157 PostData = postData;
158 ContentType = contentType;
159 Accept = accept;
160 Referer = referer;
161 AllowAutoRedirect = autoredirect;
162 }
163 #endregion
164
165 方法#region 方法
166
167 /**//// <summary>
168 /// 下载数据
169 /// </summary>
170 public Stream Download()
171 {
172 m_request = (HttpWebRequest)WebRequest.Create(Url);
173
174 //创建cookies容器
175 CookieContainer cc = null;
176 if (Cookies != null)
177 cc = Cookies;
178 else
179 cc = new CookieContainer();
180 m_request.ContentType = ContentType;
181 m_request.UserAgent = UserAgent;
182 m_request.Referer = Referer;
183 m_request.AllowAutoRedirect = AllowAutoRedirect;
184 m_request.Accept = Accept;
185 m_request.Timeout = Timeout;
186 m_request.Method = Method;
187
188 //检查是否有提交数据,只有post才可提交数据
189 if (PostData.Length > 0 && Method == "POST")
190 {
191 byte[] bytes = Encode.GetBytes(PostData);
192 Stream postStream = m_request.GetRequestStream();
193 postStream.Write(bytes, 0, bytes.Length);
194 postStream.Close();
195 }
196
197 try
198 {
199 m_response = (HttpWebResponse)m_request.GetResponse();
200 }
201 catch (WebException ex)
202 {
203 if (ex.Response != null)
204 {
205 m_response = (HttpWebResponse)ex.Response;
206 }
207 else
208 {
209 throw ex;
210 }
211 }
212 return m_response.GetResponseStream();
213 //if (DownloadComplete != null)
214 //{
215 // DownloadComplete(this, EventArgs.Empty);
216 //}
217 }
218
219 #endregion
220
221 IDisposable 成员#region IDisposable 成员
222
223 public void Dispose()
224 {
225 if (m_response != null)
226 m_response.Close();
227 }
228
229 #endregion
230 }
1/**//// <summary>
2 /// Web下载工具
3 /// </summary>
4 public class WebDownload : IDisposable
5 {
6 事件#region 事件
7
8 /**//// <summary>
9 /// 下载完成事件
10 /// </summary>
11 //public event EventHandler DownloadComplete;
12
13 #endregion
14
15 字段#region 字段
16 /**//// <summary>
17 /// 请求对象
18 /// </summary>
19 private HttpWebRequest m_request;
20
21 /**//// <summary>
22 /// 请求返回对象
23 /// </summary>
24 private HttpWebResponse m_response;
25
26 /**//// <summary>
27 /// 请求URL
28 /// </summary>
29 public string Url = string.Empty;
30
31 /**//// <summary>
32 /// Method
33 /// </summary>
34 public string Method = string.Empty;
35
36 /**//// <summary>
37 /// 数据类型
38 /// </summary>
39 public string ContentType = string.Empty;
40
41 /**//// <summary>
42 /// referer
43 /// </summary>
44 public string Referer = string.Empty;
45
46 /**//// <summary>
47 /// 是否允许重定向
48 /// </summary>
49 public bool AllowAutoRedirect = true;
50
51 /**//// <summary>
52 /// accept
53 /// </summary>
54 public string Accept = string.Empty;
55
56 /**//// <summary>
57 /// 提交的数据
58 /// </summary>
59 public string PostData = string.Empty;
60
61 /**//// <summary>
62 /// cookies
63 /// </summary>
64 public CookieContainer Cookies = null;
65
66 /**//// <summary>
67 /// 连接超时时间
68 /// </summary>
69 public int Timeout = 100000;
70
71 /**//// <summary>
72 /// 编码类型
73 /// </summary>
74 public Encoding Encode = Encoding.Default;
75
76 /**//// <summary>
77 /// useragent
78 /// </summary>
79 public string UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)";
80
81 /**//// <summary>
82 /// 请求返回的数据流
83 /// </summary>
84 //public Stream ReceiveData = null;
85 #endregion
86
87 构造函数#region 构造函数
88 /**//// <summary>
89 /// 下载工具
90 /// </summary>
91 /// <param name="url"></param>
92 /// <param name="method"></param>
93 public WebDownload(string url, string method) : this( url,method,string.Empty )
94 {
95 }
96
97 /**//// <summary>
98 /// 下载工具
99 /// </summary>
100 /// <param name="url"></param>
101 /// <param name="method"></param>
102 /// <param name="postData"></param>
103 public WebDownload(string url, string method, string postData) : this( url,method,postData,"application/x-www-form-urlencoded" )
104 {
105 }
106
107 /**//// <summary>
108 /// 下载工具
109 /// </summary>
110 /// <param name="url"></param>
111 /// <param name="method"></param>
112 /// <param name="postData"></param>
113 /// <param name="contentType"></param>
114 public WebDownload(string url, string method, string postData, string contentType) : this( url,method,postData,contentType,"image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, application/x-silverlight, */*" )
115 {
116 }
117
118 /**//// <summary>
119 /// 下载工具
120 /// </summary>
121 /// <param name="url"></param>
122 /// <param name="method"></param>
123 /// <param name="postData"></param>
124 /// <param name="contentType"></param>
125 /// <param name="accept"></param>
126 public WebDownload(string url, string method, string postData, string contentType, string accept) : this( url,method,postData,contentType,accept,string.Empty )
127 {
128 }
129
130 /**//// <summary>
131 /// 下载工具
132 /// </summary>
133 /// <param name="url"></param>
134 /// <param name="method"></param>
135 /// <param name="postData"></param>
136 /// <param name="contentType"></param>
137 /// <param name="accept"></param>
138 /// <param name="referer"></param>
139 public WebDownload(string url, string method, string postData, string contentType, string accept,string referer) : this( url,method,postData,contentType,accept,referer,true )
140 {
141 }
142
143 /**//// <summary>
144 /// 下载工具
145 /// </summary>
146 /// <param name="url"></param>
147 /// <param name="method"></param>
148 /// <param name="postData"></param>
149 /// <param name="contentType"></param>
150 /// <param name="accept"></param>
151 /// <param name="referer"></param>
152 /// <param name="autoredirect"></param>
153 public WebDownload(string url, string method, string postData, string contentType, string accept, string referer,bool autoredirect)
154 {
155 Url = url;
156 Method = method;
157 PostData = postData;
158 ContentType = contentType;
159 Accept = accept;
160 Referer = referer;
161 AllowAutoRedirect = autoredirect;
162 }
163 #endregion
164
165 方法#region 方法
166
167 /**//// <summary>
168 /// 下载数据
169 /// </summary>
170 public Stream Download()
171 {
172 m_request = (HttpWebRequest)WebRequest.Create(Url);
173
174 //创建cookies容器
175 CookieContainer cc = null;
176 if (Cookies != null)
177 cc = Cookies;
178 else
179 cc = new CookieContainer();
180 m_request.ContentType = ContentType;
181 m_request.UserAgent = UserAgent;
182 m_request.Referer = Referer;
183 m_request.AllowAutoRedirect = AllowAutoRedirect;
184 m_request.Accept = Accept;
185 m_request.Timeout = Timeout;
186 m_request.Method = Method;
187
188 //检查是否有提交数据,只有post才可提交数据
189 if (PostData.Length > 0 && Method == "POST")
190 {
191 byte[] bytes = Encode.GetBytes(PostData);
192 Stream postStream = m_request.GetRequestStream();
193 postStream.Write(bytes, 0, bytes.Length);
194 postStream.Close();
195 }
196
197 try
198 {
199 m_response = (HttpWebResponse)m_request.GetResponse();
200 }
201 catch (WebException ex)
202 {
203 if (ex.Response != null)
204 {
205 m_response = (HttpWebResponse)ex.Response;
206 }
207 else
208 {
209 throw ex;
210 }
211 }
212 return m_response.GetResponseStream();
213 //if (DownloadComplete != null)
214 //{
215 // DownloadComplete(this, EventArgs.Empty);
216 //}
217 }
218
219 #endregion
220
221 IDisposable 成员#region IDisposable 成员
222
223 public void Dispose()
224 {
225 if (m_response != null)
226 m_response.Close();
227 }
228
229 #endregion
230 }