C# 爬取小说 爬虫

最新推荐文章于 2024-03-19 16:08:32 发布
最新推荐文章于 2024-03-19 16:08:32 发布
阅读量557 收藏 2
点赞数
分类专栏： C#
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/u013628121/article/details/103983080
版权

 采用多线程爬取 , System.Data.SQLite 数据库存储

发现多线程 开太多 ,电脑性能不足 造成 部分下载不了,还占用过多资源

多线程 开少了 下载又慢, 只能说调到 中间部分 ,电脑刚刚能处理

代码是想到哪写到哪,架构混乱,只能说能跑起来

 NovelBookDetail

using System;
using System.Collections.Generic;
using System.Data.SQLite;
using System.Linq;
using System.Net.Http;
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;
namespace novelDownload
{
    public class NovelBookDetail
    {
        public static string mainIp = "网站不贴出来了";
        public static string dbPath = @"D:\DB\Demo.db3";
        public static string db_Xianxia_table_Name = "Xianxia";
        public static string db_table_Name = "NovelDetail";
        public static Action<string> Log_callback;
        StringBuilder log = new StringBuilder();
        public string Log
        {
            get
            {
                return log.ToString();
            }
            set
            {
                Log_callback(value);
                log.Append(value);
            }
        }
        private List<BookChapter> list =new List<BookChapter>();
        private SQLiteHelper sqlite;
        public NovelBookDetail(NovelBook book)
        {
            sqlite = new SQLiteHelper(dbPath);
            Log = book.book_url + "\r\n";
            Down(book.book_url, fileContent =>
            {
                string pattern = @"<meta property=""og:description"" content=""(.+)\/>";
                book.description = Regex.Match(fileContent, pattern).Groups[1].Value;
                pattern = @"og:novel:update_time"" content=""(.+)"" \/>";
                book.lastUpdateTime = Regex.Match(fileContent, pattern).Groups[1].Value;
                pattern = @"og:novel:latest_chapter_name"" content=""(.+)"" \/>";
                book.latest_chapter_name = Regex.Match(fileContent, pattern).Groups[1].Value;
                pattern = @"(?<=正文\<\/dt\>)([.\S\s]*)(?=\<\/dl\>)";
                var result = fileContent.get_Match_Groups_1(pattern);
                result = result.Trim();
                book.all_chapter_name = result;
                book.page_source = fileContent;
                Log = (book.ToString());
                Update_db(book);
            });
        }
        static SQLiteHelper CreatDb()
        {
            var sqlite = new SQLiteHelper(dbPath);
            var colNames = new string[]
            {
            "book_id","book_name", "chapter_num", "chapter_title", "chapter_Url", "chapter_content",
            };
            var colTypes = Enumerable.Repeat("TEXT", colNames.Length).ToArray();
            sqlite.CreateTable(db_table_Name, colNames, colTypes);
            return sqlite;
        }
        public static void _01_Down_book_name(TextBox textBox)
        {
            NovelBooks.All_Down_mian(db_Xianxia_table_Name);
        }
        public static void _02_Down_chapter_name(TextBox textBox)
        {
            var date = SQLiteHelper.ReadNullTable(dbPath, db_Xianxia_table_Name, "page_source");
            NovelBookDetail.Log_callback = (x) => { textBox.AppendShow(x); };
            textBox.AppendText("read db count:" + date.Rows.Count);
            for (int i = 0; i < date.Rows.Count; i++)
            {
                NovelBook book = new NovelBook();
                var line = date.Rows[i];
                string colField_Name = "";
                colField_Name = "book_id";
                book.book_id = line[colField_Name].ToString();
                colField_Name = "db_addDate";
                book.db_addDate = line[colField_Name].ToString();
                colField_Name = "book_name";
                book.book_name = line[colField_Name].ToString();
                colField_Name = "type";
                book.type = line[colField_Name].ToString();
                colField_Name = "book_url";
                book.book_url = line[colField_Name].ToString();
                colField_Name = "author";
                book.author = line[colField_Name].ToString();
                colField_Name = "authorUrl";
                book.authorUrl = line[colField_Name].ToString();
                colField_Name = "page_source";
                book.page_source = line[colField_Name].ToString();
                ThreadPool.QueueUserWorkItem(
                    p =>
                    {
                        // 将object转成数组
                        var objArr = (NovelBook)book;
                        var nbd = new NovelBookDetail(objArr);
                    }, book);
            }
        }
        async Task Down(string ip = "https://www.cnblogs.com", Action<string> func = null)
        {
            var client = new HttpClient();
            string response = await client.GetStringAsync(ip);
            if (func != null)
            {
                func(response);
            }
            //Console.WriteLine(response);
        }
        public static int downChapterNum = 0,JumpNum=0,jump_book;
        public static void _03_Down_all_chapter(TextBox textBox)
        {
            ThreadPool.SetMinThreads(1, 1);
            ThreadPool.SetMaxThreads(50, 50);
            string sql = "SELECT book_id,book_name,all_chapter_name FROM " + db_Xianxia_table_Name;
            var date = SQLiteHelper.ExecuteDataTable(dbPath, db_Xianxia_table_Name, sql);
            string mianCount = ("From Table '" + db_Xianxia_table_Name+"' read date count "+ date.Rows.Count);
            string chapterCount = "";
            var sqlite2 = CreatDb();
            StringBuilder stringBuilder =new StringBuilder();
            for (int i = 0; i < date.Rows.Count; i++)
            {
            
                var line = date.Rows[i];
                string colField_Name = "";
                colField_Name = "book_id";
                var book_id = line[colField_Name].ToString();
                colField_Name = "book_name";
                var book_name = line[colField_Name].ToString();
                colField_Name = "all_chapter_name";
                var all_chapter_name = line[colField_Name].ToString();
                var lines = all_chapter_name.Split(new string[] {"\r\n"}, System.StringSplitOptions.None);
                var chapters = lines.Where(x => x.Length > 75 && x.Contains(@"<dd> <a style="""" href="""))
                    .Select(x => x).ToArray();
                chapterCount =  ("book '" + book_name + "' chapters count " + chapters.Length);
                var pattern_url = @"href=""(.*)"">";
                string pattern_chapter_name = @""">(.*)<\/a>";
                
                for (int j = 0; j <  chapters.Length; j++)
                {
                    stringBuilder.Remove(0, stringBuilder.Length);//清空StringBuilder的方法
                    stringBuilder.Append(mianCount);
                    stringBuilder.Append("\r\n\r\n");
                    stringBuilder.Append(chapterCount);
                    stringBuilder.Append("\r\n\r\n");
                    BookChapter bookChapter = new BookChapter();
                    bookChapter.book_id = book_id;
                    bookChapter.book_name = book_name;
                    //如果最后一章都下载了话 跳过当前这本书
                    if (j == 0 && IsDownload(bookChapter.book_id, (chapters.Length - 1)+""))
                    {
                        jump_book++;
                        stringBuilder.Append("book all chapters is download");
                        textBox.SetShow(stringBuilder.ToString());
                        break;
                    }
                    var itemLine= chapters[j];
                    bookChapter.chapter_Url = mainIp + itemLine.get_Match_Groups_1(pattern_url);
                    bookChapter.chapter_title = itemLine.get_Match_Groups_1(pattern_chapter_name);
                    bookChapter.chapter_num = j + 1 + "";
    
                    if (IsDownload(bookChapter.book_id,bookChapter.chapter_num))
                    {
                        JumpNum++;
                        stringBuilder.Append("is download '" + bookChapter.book_name  + "' chapter " + bookChapter.chapter_num + "\r\n\r\n");
                        stringBuilder.Append("download chapter count " + downChapterNum + " ,jump chapter count " + JumpNum + " ,jump book count " + jump_book + "\r\n");
                        textBox.SetShow(stringBuilder.ToString());
                        continue;
                    }
                    else
                    {
                        downChapterNum++;
                        stringBuilder.Append("start download '" + bookChapter.book_name  + "' chapter " + bookChapter.chapter_num + "\r\n\r\n");
                        stringBuilder.Append("download chapter count " + downChapterNum + " ,jump chapter count " + JumpNum + " ,jump book count " +jump_book+ "\r\n");
                        textBox.SetShow(stringBuilder.ToString());
                        Down_chapter(bookChapter, sqlite2);
                    }
                }
            }
        }
      public static  bool IsDownload(string book_id, string chapter_num)
        {
            //SELECT * FROM NovelDetail WHERE book_id = '21663' AND chapter_num = '2'
            StringBuilder sql =new StringBuilder();
            sql.Append("SELECT * FROM ");
            sql.Append(db_table_Name);
            sql.Append(" WHERE book_id = :book_id");
            sql.Append(" AND chapter_num = :chapter_num");
            SQLiteParameter[] parameters = new SQLiteParameter[]
            {
                new SQLiteParameter(":book_id", book_id),
                new SQLiteParameter(":chapter_num",chapter_num),
            };
            var effctLines = SQLiteHelper.ExecuteDataTable(dbPath, sql.ToString(), parameters).Rows.Count;
            if (effctLines > 0)
            {
                return true;
            }
            return false;
        }
        void Update_db(NovelBook book)
        {
            StringBuilder sb = new StringBuilder();
            sb.Append("update Xianxia set ");
            sb.Append(" lastUpdateTime =:lastUpdateTime,");
            sb.Append(" latest_chapter_name =:latest_chapter_name,");
            sb.Append(" page_source =:page_source,");
            sb.Append(" all_chapter_name =:all_chapter_name,");
            sb.Append(" description =:description");
            sb.Append(" where book_id=:book_id");
            SQLiteParameter[] parameters = new SQLiteParameter[]
            {
            new SQLiteParameter(":lastUpdateTime", book.lastUpdateTime),
            new SQLiteParameter(":latest_chapter_name",book.latest_chapter_name),
            new SQLiteParameter(":page_source", book.page_source),
            new SQLiteParameter(":all_chapter_name",book.all_chapter_name),
            new SQLiteParameter(":description", book.description),
            new SQLiteParameter(":book_id", book.book_id)
            };
            int affectedRows = sqlite.ExecuteNonQuery(sb.ToString(), parameters);
            var b = affectedRows;
            Log = ("Update_db affectedRows:" + affectedRows + "\r\n");
            //  UPDATE Xianxia SET description = 'aa', lastUpdateTime = 'bb' WHERE book_id = '91791'
        }
        public static void Down_chapter(BookChapter bookChapter, SQLiteHelper sqlite)
        {
            Console.WriteLine();
            object[] objectArray = new object[3];//这里的2就是改成你要传递几个参数
                objectArray[0] = bookChapter;
                objectArray[1] = sqlite;
                objectArray[2] = db_table_Name;
                object param = (object)objectArray;
                ThreadPool.QueueUserWorkItem(p =>
                    {
                        // 将object转成数组
                        object[] objArr = (object[])param;
                        var param1 = (BookChapter)objArr[0];
                        var param2 = (SQLiteHelper)objArr[1];
                        var param3 = (string)objArr[2];
                        Down(param1, (fileContent, bookChapter2) =>
                        {
                            var cc = new BookChapter(bookChapter2, fileContent);
                            param2.InsertPublicField(param3, cc);
                        });
                    }, param);
        }
        public static async Task Down(BookChapter bc, Action<string, BookChapter> func)
        {
            try
            {
                var client = new HttpClient();
                string response = await client.GetStringAsync(bc.chapter_Url);
                if (func != null)
                {
                    func(response, bc);
                }
                Console.WriteLine("\r\nchapter_num:" + bc.chapter_Url + "\r\nresponse:" + response);
            }
            catch (Exception e)
            {
                Console.WriteLine(e);
                throw new Exception(e.Message);
            }
        }
        
    }
}

BookChapter 

using System.Text.RegularExpressions;

namespace novelDownload
{
    public class BookChapter
    {
        public string book_id;
        public string book_name;
        public string chapter_num;
        public string chapter_title;
        public string chapter_Url;
        public string chapter_content;


        public override string ToString()
        {
            return $"{nameof(book_id)}: {book_id}\r\n{nameof(book_name)}: {book_name}\r\n{nameof(chapter_num)}: {chapter_num}\r\n{nameof(chapter_title)}: {chapter_title}\r\n{nameof(chapter_Url)}: {chapter_Url}\r\n{nameof(chapter_content)}: {chapter_content}";
        }

        public BookChapter() { }
        public BookChapter(BookChapter bc, string fileContent)
        {
            bc.CopyPublicFieldTo(this);

            string pattern = @"(?<=<div id=""content"" deep=""3"">)([.\S\s]*)(?=<div align=""center"">)";
            var chapter_content = "";
            Match result = Regex.Match(fileContent, pattern);
            if (result.Success)
            {
                chapter_content = result.Value; //此为匹配出的值
                pattern = @"(?<=<p>)([.\S\s]*)(?=<\/p>)";
                chapter_content = Regex.Replace(chapter_content, pattern, "");
                pattern = @"&nbsp;&nbsp;&nbsp;&nbsp;";
                chapter_content = Regex.Replace(chapter_content, pattern, "\t");
                pattern = @"<br\/><br\/>";
                chapter_content = Regex.Replace(chapter_content, pattern, "\r\n");
                pattern = @"<p><\/p>";
                chapter_content = Regex.Replace(chapter_content, pattern, "");
                pattern = @"^\s*\n";
                chapter_content = Regex.Replace(chapter_content, pattern, "");
                pattern = @"(未完待续……)";
                chapter_content = Regex.Replace(chapter_content, pattern, "");

                this.chapter_content = chapter_content.Trim();

            }
        }
    }
}

 NovelBooks

using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;
namespace novelDownload
{
    public class NovelBooks
    {
        private string mainIp = "https://www.网址.com/";
        public static string dbPath = @"D:\DB\Demo.db3";
        List<NovelBook> list = new List<NovelBook>();
        public NovelBooks(string file_content, TextBox textBox2 = null)
        {
            var start = "<div class=\"l\">";
            var end = "<div class=\"page_b page_b2\">";
            var re = file_content.PickUpCenter(start, end);
            textBox2?.AppendShow(re);
            start = "<li>";
            string[] sArr = re.Split(new string[] { start }, StringSplitOptions.None);
            foreach (var VARIABLE in sArr)
            {
                textBox2?.AppendShow(VARIABLE);
                textBox2?.AppendShow("==================");
            }
            for (int i = 0; i < sArr.Length; i++)
            {
                var item = sArr[i];
                NovelBook book = new NovelBook();
                string pattern = @"<span class=""s1"">\[(.+)\]<\/span>";
                book.type = Regex.Match(item, pattern).Groups[1].Value;
                pattern = @"<span class=""s2""><a href=""\/\d+\/\d+\/"" target=""_blank"">(.+)<\/a><\/span>";
                book.book_name = Regex.Match(item, pattern).Groups[1].Value;
                pattern = @"<span class=""s2""><a href=""(.+)"" target=""_blank"">";
                book.book_url = mainIp + Regex.Match(item, pattern).Groups[1].Value;
                pattern = @"https:\/\/www.网址.com\/\d+\/(\d+)\/";
                book.book_id = Regex.Match(book.book_url, pattern).Groups[1].Value;
                pattern = @"<span class=""s4""><a href=""(.+)"">";
                book.authorUrl = mainIp + Regex.Match(item, pattern).Groups[1].Value;
                pattern = @"<span class=""s4""><a href=""\/author\/\d+\/"">(.+)<\/a><\/span>";
                book.author = Regex.Match(item, pattern).Groups[1].Value;
                textBox2?.AppendShow(book.ToString() + "\r\n");
                bool isHasDate = book.IsNoEmpty();
                if (isHasDate) list.Add(book);
            }
        }
        static SQLiteHelper CreatDb(string db_Table_Name)
        {
            var sqlite = new SQLiteHelper(dbPath);
            var colNames = new string[]
            {
                "book_id", "db_addDate", "book_name", "type", "book_url", "author", "authorUrl", "description",
                "lastUpdateTime", "latest_chapter_name", "all_chapter_name","page_source",
            };
            var colTypes = Enumerable.Repeat("TEXT", colNames.Length).ToArray();
            sqlite.CreateTable(db_Table_Name, colNames, colTypes);
            return sqlite;
        }
        public static void All_Down_mian(string db_Table_Name)
        {
            var sqlite = CreatDb(db_Table_Name);
            string ip_forward = "https://www.网址.com/xianxia_";
            string ip_center = "";
            string ip_end = ".html";
            string ip = "";
            for (int i = 1; i < 188; i++)
            {
                ip_center = i.ToString();
                ip = ip_forward + ip_center + ip_end;
                object[] objectArray = new object[3];//这里的2就是改成你要传递几个参数
                objectArray[0] = ip;
                objectArray[1] = sqlite;
                objectArray[2] = db_Table_Name;
                object param = (object)objectArray;
                ThreadPool.QueueUserWorkItem(
                    p =>
                    {
                        // 将object转成数组
                        object[] objArr = (object[])param;
                        var param1 = (string)objArr[0];
                        var param2 = (SQLiteHelper)objArr[1];
                        var param3 = (string)objArr[2];
                        Down(param1, fileContent =>
                        {
                            var books = new NovelBooks(fileContent);
                            param2.BatchInsertPublicFields(param3, books.list);
                        });
                    }, param);
            }
        }
        public static async Task Down(string ip = "https://www.cnblogs.com", Action<string> func = null)
        {
            NovelBookDetail.Log_callback("down ip:" + ip + "\r\n");
            var client = new HttpClient();
            string response = await client.GetStringAsync(ip);
            if (func != null)
            {
                func(response);
            }
            //Console.WriteLine(response);
        }
    }
}

NovelBook 

    public class NovelBook
    {
        public string book_id;
        public string db_addDate;
        public string book_name;
        public string type;
        public string book_url;
        public string author;
        public string authorUrl;
        public string description;// 简介
        public string lastUpdateTime; //最后一次更新时间
        public string latest_chapter_name; //最后一次章节名称
        public string all_chapter_name; //所有章节名称
        public string page_source; //所有章节名称
    }

 

绀目澄清
关注
  • 0
    点赞
  • 踩
  • 2
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
专栏目录
C#简单实现网络爬虫
03-06
简单的网络爬虫实现，下载源码可在本地运行使用。如有不足之处，请大家多多指教。谢谢。
C# 爬虫 抓取小说
zy5757的博客
09-05 4920
心血来潮，想研究下爬虫，爬点小说。 通过百度选择了个小说网站，随便找了一本小书http://www.23us.so/files/article/html/13/13655/index.html。 1、分析html规则 思路是获取小说章节目录，循环目录，抓取所有章节中的内容，拼到txt文本中。最后形成完本小说。 1、获取小说章节目录 通过分析，我在标注的地方获取小说名字及章节目录。
C#生成随机字符串
网络Online
06-30 3952
///<summary> ///生成随机字符串//转载请注明来自 http://www.uzhanbao.com ///</summary> ///<param name="length">目标字符串的长度</param> ///<param name="useN...
使用c#实现爬虫技术
热门推荐
qq_35500233的博客
05-23 1万+
这是我的第一个爬虫项目，也是我第一次接触c# 窗体程序。 我的需求：页面中有音频文件但是它时单个下载的，用户需要一个一个的去点击下载按钮进行下载，我的目的：根据用户的需求筛选出相关的数据，然后我拿到页面上用户筛选的数据，实现批量下载，然后将下载并存放到用户本地文件夹中，然后对下载下来的这些文件进行播放。 主要用到的插件有：CefSharp HtmlAgilityPack 将浏览器页面嵌入到wi...
C#实现简单爬虫
qq_40198590的博客
09-28 1万+
爬虫算是第一个小任务吧，因为每天要统计数据，比较繁琐，如果有一个程序能帮助我，解决这个问题就好了。 首先 要知道爬虫的流程 1、发送请求 2、响应数据 3、处理数据 4、存入数据库 也要看你要爬那个网站的信息，知道每个请求头里面包含哪些需要的内容，打开网页F12就能看到，苹果电脑就Fn+F12 我们要的cookie也在这里???? 先从第一步入手，发送请求，一个url，然后就第二步响应了数据返回一个json格式的字符串。 //这个url是你要爬的地址 public static str
c#爬虫爬取京东的商品信息
08-26
C# 爬虫爬取京东商品信息 在本篇文章中，我们将介绍如何使用 C# 语言编写一个爬虫来爬取京东的商品信息。通过示例代码，我们将详细地介绍爬虫的实现过程，包括下载网页、解析 HTML、获取商品分类和商品信息等步骤。...
C#网络爬虫代码分享 C#简单的爬取工具
09-01
下面将详细讲解这个简单的C#网络爬虫代码及其涉及的关键知识点。 首先，网络爬虫的核心是发送HTTP请求并接收响应。在C#中，我们可以使用`System.Net.WebRequest`和`System.Net.HttpWebResponse`类来实现这一功能。...
C#爬虫.ZIP_C# 图片爬虫_C#爬取_c#爬虫和python_c＃ 爬虫_图片爬虫
07-14
这个用C#实现和python一样的原理功能，通过一个连接地址不断爬取html中的图片路径，然后下载指定的文件夹中，希望对大家有帮助，该源码来源于网络。
京东商城商品信息爬取_爬虫_京东_
09-30
在IT行业中，网络爬虫是一项重要的技术，用于自动获取网页数据。京东商城商品信息爬取是这个项目的核心，它的目标是从京东商城的网站上抓取商品的相关信息，然后进行数据分析，以便于构建一个类似功能的商城网站。在...
C#爬虫开发小结
dotNET跨平台
01-26 1380
1前言2023年以来一直很忙，临近春节，各种琐事更多，但鸽了太久没写文章总是不舒坦，忙中偷闲来记录下最近用C#写爬虫的一些笔记。爬虫一般都是用Python来写，生态丰富，动态语言开发速度快，调试也很方便但是我要说但是，动态语言也有其局限性，笔者作为老爬虫带师，几乎各种语言都搞过，现在这个任务并不复杂，用我最喜欢的C#做小菜一碟~2开始之前做 OneCat 项目的时候，最开始的数据采集模块，就是用 ...
C#-Selenium爬虫抓取（一）
weixin_43736127的博客
03-02 2230
C# selenium 驱动配置 抓取 登录
C#爬取数据_详细篇
deer_soul的博客
11-09 4342
C#爬取数据的三种方法
C#实战分享--爬虫的基础原理及实现
最新发布
借雨醉东风的博客
03-19 4280
以实际的代码案例，分享博主如何写Spider的过程。
写爬虫，只是为了下小说
github_30830155的博客
12-17 731
整体思路 spider 大概兴趣是第一生产力吧 网上找的小说现成的txt排版都有问题，缺少章节，不过一些盗版网站章节倒是挺全 本来想着是找到index页，爬取所有章节连接，然后拼接成网址，再进行细节爬取，但是搞了一下发现不知道怎么弄 所以后来就变成了直接从第一章节开始爬，找到标题和正文，用xpath就可以了，然后找到下一章的地址 看着网上一个博客，用的 yield ，我的理解就是一个循
C#爬虫
Sky
04-07 1万+
前言最近两周在学习爬虫相关的知识，看过爬虫原理之后，就想自己也写一个简单的爬虫工具，实现功能，就是文本框中输入要爬取的基地址，然后点击开始爬取按钮，把爬取的内容保存到本地。虽然Python更是适合写爬虫，但是谁让我是搞C#的呢，只能用C#写了。知识准备写代码之前，需要了解基本的爬虫原理和步骤才能在写代码的时候，做到心中有数。调试的时候，才会知根知底。百度了两篇关于爬虫原理和代码的博客，写的非常好，通
C#常用爬虫框架
qq_42418169的博客
03-21 3827
1：.Net开源的跨平台爬虫框架 DotnetSpider Star:430 DotnetSpider这是国人开源的一个跨平台、高性能、轻量级的爬虫软件，采用 C# 开发。目前是.Net开源爬虫最为优秀的爬虫之一。 2：俄国牛人写的开源爬虫xNet Star:117 这个一...
使用C#写的爬虫
qq_44920920的博客
12-22 1077
使用C#写的爬虫示例
C# 爬取ajax后的页面数据
06-13
如果您想在 C# 中爬取 Ajax 后的页面数据，可以使用 HttpClient 类，以及 Newtonsoft.Json 库来处理返回的 JSON 数据。 以下是一个简单的示例代码，可以帮助您实现这个功能： ```csharp using System; using System.Net.Http; using Newtonsoft.Json; class Program { static async System.Threading.Tasks.Task Main(string[] args) { using (HttpClient client = new HttpClient()) { var url = "https://example.com/api/getdata"; var data = new FormUrlEncodedContent(new[] { new KeyValuePair<string, string>("param1", "value1"), new KeyValuePair<string, string>("param2", "value2"), // 添加需要传递的参数 }); var response = await client.PostAsync(url, data); var content = await response.Content.ReadAsStringAsync(); dynamic result = JsonConvert.DeserializeObject(content); // 处理返回的 JSON 数据 } } } ``` 在上面的代码中，我们使用 HttpClient 发送 POST 请求，并传递需要的参数。然后，我们通过读取响应内容，获取返回的 JSON 数据，并使用 Newtonsoft.Json 库将其转换为动态类型的对象，以便后续处理。 请注意，使用爬虫程序爬取网站数据需要遵守相关法律法规和网站的规定，否则可能会造成不良后果。

“相关推荐”对你有帮助么？

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值