利用爬虫-C#下载网络上的小说大部分网站可以用

本文纯属个人爱好不含任何商业用途：
基本思路：爬取网站内容的基本规则是利用标签来进行匹配，首先我们以一个小说网站为例http://www.jjwxc.net/onebook.php?novelid=3325239&chapterid=6这是某个小说的一个章节，大家可以看到novelid=3325239这个在该网站中对应的就是小说的唯一表示chapterid=6对应的是小说的章节号 那好根据这个我们可以访问打开任何小说包括他的任何章节。下载小说我们就需要拿到这个小说的名称查看网页源代码 这里我选择的是在这里截取小说的名称 源码

Regex ma_name = new Regex(@"<input type=""hidden"" id=""novel_name"" value=""(.+)""/>(.|\n)*?");
var mu_name = ma_name.Match(html);
text = mu_name.Groups[1].Value;

接下来获取章节名 我们可以发现章节名很明显在h2中 好那我们取出章节名 源码

Regex reg_zjm = new Regex(@"< h2 >(.|\n)*?</h2>");
var mat_zjm = reg_zjm.Match(html);
string zjm = mat_zjm.Groups[0].ToString();
zjm = zjm.Replace("< h2 >", " ");
zjm = zjm.Replace("</ h2>", " ");

最后在获取文章的内容就可以了同样的方法不细说了 直接上代码

Regex reg_mulu = new Regex(@"< br>(.|\n)*?</ div></td>");
var mat_mulu = reg_mulu.Match(html);
string nrong = mat_mulu.Groups[0].ToString();
// < br>< br/>替换为换行 
nrong = nrong.Replace("< br>", "\r\n");
// 去掉其它的<>之间的东西 
nrong = nrong.Split('d')[0];
nrong = nrong.Replace("<", "");
Content = "第" + i + "章" + zjm + "\r\n" + nrong + "\r\n";

最后输出文本

这里我是做成了一个winform输出的 可以浏览到本地的任何路径 选择路径的代码

    //选择文件框 对象
    FolderBrowserDialog ofd = new FolderBrowserDialog();
    //打开时指定默认路径
    ofd.SelectedPath = @"C:\Documents and Settings\Administrator.ICBCOA-6E96E6BE\桌面";
    //如果用户点击确定
    if (ofd.ShowDialog() == DialogResult.OK)
    {
        //将用户选择的文件路径 显示 在文本框中
        txtFilePathOpen.Text = ofd.SelectedPath;
    }
        文本框中输入小说的标识 选中下载路径点击下载就可以了  

完整代码
TextHelper.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;

namespace WindowsFormsApp1
{
    class TextHelper
    {
        /// <summary>
        /// 创建文本
        /// </summary>
        /// <param name="content">内容</param>
        /// <param name="name">名字</param>
        /// <param name="path">路径</param>
        public void Novel(string content, string name, string path)
        {
            string Log = content + "\r\n";
            // 创建文件夹，如果不存在就创建file文件夹
            if (Directory.Exists(path) == false)
            {
                Directory.CreateDirectory(path);
            }

            // 判断文件是否存在，不存在则创建
            if (!System.IO.File.Exists(path + "\\"+ name + ".txt"))
            {
                FileStream fs1 = new FileStream(path +"\\"+ name + ".txt", FileMode.Create, FileAccess.Write);// 创建写入文件 
                StreamWriter sw = new StreamWriter(fs1);
                sw.WriteLine(Log);// 开始写入值
                sw.Close();
                fs1.Close();
            }
            else
            {
                FileStream fs = new FileStream(path + "\\"+name + ".txt" + "", FileMode.Append, FileAccess.Write);
                StreamWriter sr = new StreamWriter(fs);
                sr.WriteLine(Log);// 开始写入值
                sr.Close();
                fs.Close();
            }
        }

        public string HttpPost(string Url, string postDataStr)
        {
            CookieContainer cookie = new CookieContainer();
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url);
            request.Method = "POST";
            request.ContentType = "application/x-www-form-urlencoded";
            request.ContentLength = Encoding.UTF8.GetByteCount(postDataStr);
            request.CookieContainer = cookie;
            Stream myRequestStream = request.GetRequestStream();
            StreamWriter myStreamWriter = new StreamWriter(myRequestStream, Encoding.GetEncoding("gb2312"));
            myStreamWriter.Write(postDataStr);
            myStreamWriter.Close();

            HttpWebResponse response = (HttpWebResponse)request.GetResponse();

            response.Cookies = cookie.GetCookies(response.ResponseUri);
            Stream myResponseStream = response.GetResponseStream();
            StreamReader myStreamReader = new StreamReader(myResponseStream, Encoding.GetEncoding("utf-8"));
            string retString = myStreamReader.ReadToEnd();
            myStreamReader.Close();
            myResponseStream.Close();

            return retString;
        }

        public string HttpGet(string Url, string postDataStr)
        {
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url);
            request.Method = "GET";
            //request.ContentType = "text/html;charset=GBK";
            HttpWebResponse response;
            try
            {
                response = (HttpWebResponse)request.GetResponse();
            }
            catch (WebException ex)
            {
                response = (HttpWebResponse)request.GetResponse();
            }


            Stream stm = new System.IO.Compression.GZipStream(response.GetResponseStream(), System.IO.Compression.CompressionMode.Decompress);
            StreamReader myStreamReader = new StreamReader(stm, Encoding.GetEncoding("GBK"));
            string retString = myStreamReader.ReadToEnd();
            myStreamReader.Close();

            return retString;
        }
    }
}

Form1

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;

namespace WindowsFormsApp1
{
    public partial class Form1 : Form
    {
        public Form1()
        {
            InitializeComponent();
        }

        private void Form1_Load(object sender, EventArgs e)
        {

        }
        public static string PathLog = "";
        private void button1_Click(object sender, EventArgs e)
        {
            //选择文件框 对象
            FolderBrowserDialog ofd = new FolderBrowserDialog();
            //打开时指定默认路径
            ofd.SelectedPath = @"C:\Documents and Settings\Administrator.ICBCOA-6E96E6BE\桌面";
            //如果用户点击确定
            if (ofd.ShowDialog() == DialogResult.OK)
            {
                //将用户选择的文件路径 显示 在文本框中
                txtFilePathOpen.Text = ofd.SelectedPath;
                PathLog = ofd.SelectedPath;
            }
        }

        private void button2_Click(object sender, EventArgs e)
        {
            var path = PathLog;
            //是否读取小说名称
            var IsText = true;
            //小说内容
            var Content = "";
            //小说标题
            var text = "";
            var novelid = textName.Text.ToString();
            //抓取整本小说
            //Thread thread = new Thread(() =>
            //{
            for (var i = 1; i < 2000; i++)
            {

                TextHelper cra = new TextHelper();
                try
                {
                    string html = cra.HttpGet("http://www.jjwxc.net/onebook.php?novelid=" + novelid + "&chapterid=" + i, "");

                    if (IsText)
                    {
                        //获取小说名字
                        Regex ma_name = new Regex(@"<input type=""hidden"" id=""novel_name"" value=""(.+)""/>(.|\n)*?");
                        var mu_name = ma_name.Match(html);
                        text = mu_name.Groups[1].Value;
                        IsText = false;
                    }

                    //小说章节
                    Regex reg_zjm = new Regex(@"<h2>(.|\n)*?</h2>");
                    var mat_zjm = reg_zjm.Match(html);
                    string zjm = mat_zjm.Groups[0].ToString();
                    zjm = zjm.Replace("<h2>", " ");
                    zjm = zjm.Replace("</h2>", " ");
                    if (zjm == "")
                    {
                        continue;
                    }

                    // 小说内容
                    Regex reg_mulu = new Regex(@"<br>(.|\n)*?</div></td>");
                    var mat_mulu = reg_mulu.Match(html);
                    string nrong = mat_mulu.Groups[0].ToString();
                    // <br><br/>替换为换行 
                    nrong = nrong.Replace("<br>", "\r\n");
                    // 去掉其它的<>之间的东西 
                    nrong = nrong.Split('d')[0];
                    nrong = nrong.Replace("<", "");
                    Content = "第" + i + "章" + zjm + "\r\n" + nrong + "\r\n";
                }
                catch
                {
                }
                // txt文本输出
                cra.Novel(Content, text, path);
            }

            //});
            //thread.IsBackground = true;
            //thread.Start();
        }

        private void textBox1_TextChanged(object sender, EventArgs e)
        {

        }
    }
}