TPL使用异步编程模型(APM)实现简单网页抓取

本文链接：https://blog.csdn.net/menghz/article/details/9021215

using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;

namespace fofo.NetSpider
{
    public class SimpleNetSpider
    {
        public void Start()
        {
            string[] urls = InitUrls();

            Task<WebResponse>[] tasks = urls.Select(url => GetResponseAsync(url)).ToArray();

            Task.WaitAll(tasks);
        }

        private string[] InitUrls()
        {
            string[] urls = 
            {
                "http://www.baidu.com/",
                "http://www.cnblogs.com",
                "http://www.csdn.com"
            };

            return urls;
        }

        // 启动任务
        private Task<WebResponse> GetResponseAsync(string strUrl)
        {
            WebRequest webRequest = WebRequest.Create(strUrl);
            Task<WebResponse> task = Task<WebResponse>.Factory.FromAsync(webRequest.BeginGetResponse, GetResponseAsyncCompleted, webRequest);

            return task;
        }

        // 抓取结束回调函数
        private WebResponse GetResponseAsyncCompleted(IAsyncResult result)
        {
            WebRequest webRequest = (WebRequest)result.AsyncState;
            WebResponse response = webRequest.EndGetResponse(result);

            using (System.IO.StreamReader reader = new System.IO.StreamReader(response.GetResponseStream()))
            {
                // 具体处理            
            }

            return response;
        }   
    }
}