Android 本地网络小说爬虫，基于 jsoup 及 xpath(1)

本文链接：https://blog.csdn.net/2401_83816970/article/details/137983367

put(SourceID.DINGDIAN, new Source(SourceID.DINGDIAN, “顶点小说”, “http://zhannei.baidu.com/cse/search?s=1682272515249779940&q=%s”));

put(SourceID.BIQUGER, new Source(SourceID.BIQUGER, “笔趣阁 2”, “http://zhannei.baidu.com/cse/search?s=7928441616248544648&ie=utf-8&q=%s”));

put(SourceID.CHINESEZHUOBI, new Source(SourceID.CHINESEZHUOBI, “着笔中文网”, “http://www.zbzw.com/s.php?ie=utf-8&s=4619765769851182557&q=%s”));

put(SourceID.DASHUBAO, new Source(SourceID.DASHUBAO, “大书包”, “http://zn.dashubao.net/cse/search?s=9410583021346449776&entry=1&ie=utf-8&q=%s”));

put(SourceID.CHINESEWUZHOU, new Source(SourceID.CHINESEWUZHOU, “梧州中文台”, “http://www.gxwztv.com/search.htm?keyword=%s”));

put(SourceID.UCSHUMENG, new Source(SourceID.UCSHUMENG, “UC 书盟”, “http://www.uctxt.com/modules/article/search.php?searchkey=%s”, 4));

put(SourceID.QUANXIAOSHUO, new Source(SourceID.QUANXIAOSHUO, “全小说”, “http://qxs.la/s_%s”));

put(SourceID.YANMOXUAN, new Source(SourceID.YANMOXUAN, “衍墨轩”, “http://www.ymoxuan.com/search.htm?keyword=%s”));

put(SourceID.AIQIWENXUE, new Source(SourceID.AIQIWENXUE, “爱奇文学”, “http://m.i7wx.com/?m=book/search&keyword=%s”));

put(SourceID.QIANQIANXIAOSHUO, new Source(SourceID.QIANQIANXIAOSHUO, “千千小说”, “http://www.xqqxs.com/modules/article/search.php?searchkey=%s”, 4));

put(SourceID.PIAOTIANWENXUE, new Source(SourceID.PIAOTIANWENXUE, “飘天文学网”, “http://www.piaotian.com/modules/article/search.php?searchtype=articlename&searchkey=%s”));

put(SourceID.SUIMENGXIAOSHUO, new Source(SourceID.SUIMENGXIAOSHUO, “随梦小说网”, “http://m.suimeng.la/modules/article/search.php?searchkey=%s”, 4));

put(SourceID.DAJIADUSHUYUAN, new Source(SourceID.DAJIADUSHUYUAN, “大家读书苑”, “http://www.dajiadu.net/modules/article/searchab.php?searchkey=%s”));

put(SourceID.SHUQIBA, new Source(SourceID.SHUQIBA, “书旗吧”, “http://www.shuqiba.com/modules/article/search.php?searchkey=%s”, 4));

put(SourceID.XIAOSHUO52, new Source(SourceID.XIAOSHUO52, “小说 52”, “http://m.xs52.com/search.php?searchkey=%s”));

}

};

模版示例

例如针对猎文网：

{

“id”: 1, //对应书源 id

“search”: { //搜索页解析规则

“charset”: “UTF-8”,

“xpath”: “//div[@class=‘result-item result-game-item’]”,

“coverXpath”: “//div[@class=‘result-game-item-pic’]//a//img/@src”,

“titleXpath”: “//div[@class=‘result-game-item-detail’]//h3//a/@title”,

“linkXpath”: “//div[@class=‘result-game-item-detail’]//h3//a/@href”,

“authorXpath”: “//div[@class=‘result-game-item-detail’]//div[@class=‘result-game-item-info’]//p[1]/span[2]/text()”,

“descXpath”: “//div[@class=‘result-game-item-detail’]//p[@class=‘result-game-item-desc’]/text()”

“catalog”: { //目录列表解析规则

“xpath”: “//div[@id=list]//dl//dd”,

“titleXpath”: “//a/text()”,

“linkXpath”: “//a/@href”

“content”: { //文章内容解析规则

“xpath”: “//div[@id=‘content’]/text()”

}

调用方式

目前虽然请求结果是通过 Callback 形式，因为搜多个源是分批返回结果。内部当仍是同步请求，没有做线程调度。

Crawler.search(“你好”, new SearchCallback() {

@Override

public void onResponse(String keyword, List appendList) {

}

@Override

public void onFinish() {

}

@Override

public void onError(String msg) {

}

});

Crawler.catalog(new SearchBook.SL(“https://www.liewen.cc/b/24/24934/”, SourceManager.SOURCES.get(1)), new ChapterCallback() {

@Override

public void onResponse(List chapters) {

}

@Override

public void onError(String msg) {

}

});

Crawler.content(new SearchBook.SL(“https://www.liewen.cc/b/24/24934/”, SourceManager.SOURCES.get(1)), “/b/24/24934/12212511.html”, new ContentCallback() {

@Override

public void onResponse(String content) {

}

@Override

public void onError(String msg) {

}

});

自我介绍一下，小编13年上海交大毕业，曾经在小公司待过，也去过华为、OPPO等大厂，18年进入阿里一直到现在。

深知大多数Python工程师，想要提升技能，往往是自己摸索成长或者是报班学习，但对于培训机构动则几千的学费，着实压力不小。自己不成体系的自学效果低效又漫长，而且极易碰到天花板技术停滞不前！

因此收集整理了一份《2024年Python开发全套学习资料》，初衷也很简单，就是希望能够帮助到想自学提升又不知道该从何学起的朋友，同时减轻大家的负担。