C#采集CSDN单个博客所有文章

原理：

通过HtmlAgilityPack解析html源码得到所需的数据。

1、首先通过http://blog.csdn.net/gdjlc 底部的“xx条数据 共xx页”，获取得总页数；

2、获取每一页的所有文章URL，每一页的URL如下所示： http://blog.csdn.net/gdjlc/article/list/当前页索引,从1一直循环到总页数即可得.

3、获取单个文章的内容。

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using System.IO;
using System.Diagnostics;
using System.Collections;
using System.Threading;
using System.Threading.Tasks;
using System.Xml.Linq;
using System.Net;
using HtmlAgilityPack;

namespace Demo
{
    public partial class FrmCSDN : Form
    {       
        const string BLOGUSER = "gdjlc"; //博客用户名
        const string BLOGURL = "http://blog.csdn.net";
        const string PAGECOUNTPATH = "//div[@id='papelist']/span[1]";//总页数PATH
        const string ARTICLEURLPATH = "//span[@class='link_title']/a[1]"; //文章URL的PATH    
        const string ARTICLETITLEPATH = "//div[@class='article_title']/h3/span/a";//文章标题PATH
        const string POSTDATEPATH = "//span[@class='link_postdate']"; //文章创建日期PATH
        const string ARTICLECONTENTPATH = "//div[@id='article_content']"; //文章内容PATH
        List<string> articleUrlList = new List<string>(); //所有文章的URL
        private object moniter = new object();
        Stopwatch stopwatch = new Stopwatch();
        int cnt = 0;

        public FrmCSDN()
        {
            InitializeComponent();
        }

        //获取总页数
        private int GetPageCount(string pageCountUrl)
        {         
            HtmlNode rootNode = GetHtmlNodeByUrl(pageCountUrl, Encoding.UTF8);
            if (rootNode == null)
                return 0;            
            //形如“177条数据 共12页”
            string pageCountText = GetNodeInnerText(rootNode, PAGECOUNTPATH);
            int firstIndex = pageCountText.LastIndexOf("共") + 1;
            int lastIndex = pageCountText.LastIndexOf("页");
            string result = pageCountText.Substring(firstIndex, lastIndex - firstIndex);
            return Convert.ToInt32(result);
        }
        //开始采集按钮
        private void btnCollect_Click(object sender, EventArgs e)
        {
            stopwatch.Restart();         
            Task.Factory.StartNew(() =>
            {
                cnt = 0;
                int pageCount = GetPageCount(BLOGURL + "/" + BLOGUSER);
                if (pageCount == 0) 
                    return;
                
                //所有文章的URL
                for (int pageIndex = 1; pageIndex <= pageCount; pageIndex++)
                {
                    string pageIndexUrl = BLOGURL + "/" + BLOGUSER + "/article/list/" + pageIndex.ToString();
                    HtmlNode rootNode = GetHtmlNodeByUrl(pageIndexUrl, Encoding.UTF8);
                    if (rootNode == null)
                        continue;

                    HtmlNodeCollection ArticleUrlList = rootNode.SelectNodes(ARTICLEURLPATH);
                    foreach (HtmlNode articleUrlNode in ArticleUrlList)
                    {
                        string articleUrl = BLOGURL + articleUrlNode.Attributes["href"].Value;
                        articleUrlList.Add(articleUrl);
                    }
                }
            }).ContinueWith((x) =>
            {
                TaskFactory taskFactory = new TaskFactory();
                Task[] tasks = new Task[articleUrlList.Count];
                for (int i = 0; i < articleUrlList.Count; i++)
                {
                    tasks[i] = new Task(CollectArticle, articleUrlList[i]);
                    tasks[i].Start();
                }
                taskFactory.ContinueWhenAll(tasks, TaskEnded, TaskContinuationOptions.None);
            });
        }
        /// <summary>
        /// 单篇文章采集
        /// </summary>
        /// <param name="state"></param>
        void CollectArticle(object state)
        {
            Interlocked.Increment(ref cnt);
            lock (moniter)
            {
                SetStatuText(string.Format("总共{0}篇文章, 正在采集中第{1}篇.", articleUrlList.Count, cnt));
                 string articleUrl = (string)state;
                 string firstArticleHtml = GetHtmlSource(articleUrl, Encoding.UTF8);
                 if (string.IsNullOrEmpty(firstArticleHtml)) return;
                 HtmlNode rootNode = GetHtmlNodeByHtml(firstArticleHtml);
                 string articleTitle = GetNodeInnerText(rootNode, ARTICLETITLEPATH);
                 string postDate = GetNodeInnerText(rootNode, POSTDATEPATH);
                 string articleContent = GetNodeInnerText(rootNode, ARTICLECONTENTPATH);
                 //采集结果处理：保存到数据库或其它......
                 string blogFile = BLOGUSER + ".txt";
                 using (StreamWriter sw = new StreamWriter(blogFile, true))
                 {
                     sw.WriteLine(articleUrl);
                     sw.WriteLine(articleTitle);
                     sw.WriteLine(postDate);
                     sw.WriteLine(articleContent);
                 }         
            }                     
        }
        
        private void TaskEnded(Task[] task)
        {
            SetStatuText("采集结束，耗时 " + stopwatch.Elapsed.Minutes + "分" + stopwatch.Elapsed.Seconds + "秒");
        }
        //通过网页URL获取HtmlNode
        private HtmlNode GetHtmlNodeByUrl(string url, Encoding encoding)
        {
            string html = GetHtmlSource(url, encoding);
            if (string.IsNullOrEmpty(html)) return null;

            HtmlAgilityPack.HtmlDocument document = new HtmlAgilityPack.HtmlDocument();
            document.LoadHtml(html);
            HtmlNode rootNode = document.DocumentNode;
            return rootNode;
        }
        //通过网页html源代码获取HtmlNode
        private HtmlNode GetHtmlNodeByHtml(string htmlSource)
        {
            HtmlAgilityPack.HtmlDocument document = new HtmlAgilityPack.HtmlDocument();
            document.LoadHtml(htmlSource);
            HtmlNode rootNode = document.DocumentNode;
            return rootNode;
        }
        /// <summary>
        /// 获取网页源代码
        /// </summary>      
        private string GetHtmlSource(string url, Encoding encoding)
        {
            string result = "";
            try
            {
                WebRequest request = WebRequest.Create(url);
                using (WebResponse response = request.GetResponse())               
                    using (StreamReader reader = new StreamReader(response.GetResponseStream(), encoding))                   
                        result = reader.ReadToEnd(); 
            }
            catch
            {
                result = "";
            }
            return result;
        }
        private string GetNodeInnerText(HtmlNode srcNode, string path)
        {
            HtmlNode temp = srcNode.SelectSingleNode(path);
            if (temp == null)
                return null;
            return temp.InnerText;
        }
        private void SetStatuText(string s)
        {
            this.SafeCall(() =>
            {
                lblStatusInfo.Text = s;
            });
        }         
    }
    public static class Extenstions
    {
        public static void SafeCall(this Control ctrl, Action callback)
        {
            if (ctrl.InvokeRequired)
                ctrl.Invoke(callback);
            else
                callback();
        }
    }
}