用.NET 抓取一个页面

/// <summary>  

02 /// Get a response as a string, given a uri string.  

03 /// </summary>  

04 /// <param name="uriArg">Specifies a uri such as "http://www.google.com" or @"file://X:\dir\myFile.html"</param>  

05 /// <returns>web response as a string.</returns>  

06 /// <example>  

07 /// try  

08 /// {  

09 ///    string uri = "http://www.google.zzz"; // note bad uri with zzz to exercise exception.  

10 ///    string s = GetWebPageResponse( uri );   

11 ///    Console.WriteLine( s );  

12 /// }  

13 /// catch ( WebException ex )  

14 /// {  

15 ///    // wex.Message could be something like: The remote server returned an error: (404) Not Found.  

16 ///    string s = string.Format( "Request: {0}\nResult: {1}", uri, ex.Message );  

17 ///    Console.WriteLine( s );  

18 /// }  

19 /// </example>  

20 static string GetWebPageResponse(string uriArg)  

21 {  

22     Stream responseStream = WebRequest.Create(uriArg).GetResponse().GetResponseStream();  

23    

24     StreamReader reader = new StreamReader(responseStream);  

25    

26     return reader.ReadToEnd();  

27 }  

28    

29 /// <summary>  

30 /// Similar to GetWebPageResponse(string uriArg), but uses a user/pw to log in.  

31 /// </summary>  

32 /// <param name="uriArg">e.g. "http://192.168.2.1"</param>  

33 /// <param name="userArg">e.g. "root"</param>  

34 /// <param name="pwArg">e.g. "admin"</param>  

35 /// <returns>string containing the http response.</returns>  

36 /// <example>  

37 /// // Example to get a response with DHCP table from my LinkSys router.  

38 /// string s = GetWebPageResponse( "http://192.168.2.1/DHCPTable.htm", "root", "admin" );  

39 /// </example>  

40 static string GetWebPageResponse(string uriArg, string userArg, string pwArg)  

41 {  

42     Uri uri = new Uri(uriArg);  

43     HttpWebRequest req = (HttpWebRequest)WebRequest.Create(uri);  

44     CredentialCache creds = new CredentialCache();  

45    

46     // See http://msdn.microsoft.com/en-us/library/system.directoryservices.protocols.authtype.aspx for list of types.  

47     const string authType = "basic";  

48    

49     creds.Add(uri, authType, new NetworkCredential(userArg, pwArg));  

50     req.PreAuthenticate = true;  

51     req.Credentials = creds.GetCredential(uri, authType);  

52    

53     Stream responseStream = req.GetResponse().GetResponseStream();  

54    

55     StreamReader reader = new StreamReader(responseStream);  

56    

57     return reader.ReadToEnd();  

58 }