Java多线程爬取笔趣阁小说

最新推荐文章于 2024-03-22 11:01:19 发布

ak01_10

最新推荐文章于 2024-03-22 11:01:19 发布

阅读量267

点赞数

分类专栏： java

本文链接：https://blog.csdn.net/ak01_10/article/details/115422407

版权

java 专栏收录该内容

25 篇文章 0 订阅

订阅专栏

该代码实现了一个从xbiquge.la网站抓取特定小说章节内容的爬虫。首先，获取已存在章节的最大ID，然后通过发送HTTP请求获取所有章节链接。使用线程池并行处理每个章节，抓取内容并存入数据库。如果所有章节都已抓取，程序结束。爬虫过程中考虑了内容的编码、HTML解析和数据库操作。

摘要由CSDN通过智能技术生成

package com.framework.libInteresting.spider;

import java.sql.Connection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import com.framework.util.db.DbUtil;
import com.framework.util.html.HtmlUtil;
import com.framework.util.map.MapUtil;
import com.framework.util.string.StringUtil;

public class Biquge {
	public static void main(String[] args) throws Exception {
		Connection conn = DbUtil.getBestConn();
		try {
			DbUtil db = new DbUtil(conn);

			String baseUrl = "http://www.xbiquge.la";
			String name = "道君";
			String url = "http://www.xbiquge.la/15/15003/";
			
			int maxid = NovelUtil.getExistsId(db, url);
			if (maxid <= 0) {
				maxid = NovelUtil.newId(db, url, name);
			}
			
			// 获取所有url
			Map<String, String> heads = SpiderUtil.getParams();
			String category = SpiderUtil.sendGet(url, heads, "utf-8");
			Map<String, String> urlMap = new HashMap<String, String>();
			category = HtmlUtil.find(category, "div","class=box_con").get(0);
			List<String> dds = HtmlUtil.find(category,"dd");
			for (String dd : dds) {
				dd = HtmlUtil.innerHTML(dd).trim();
				String title = HtmlUtil.getValue(dd);
				String chaturl = HtmlUtil.propertyValue(dd, "href");
				urlMap.put(chaturl, title);
			}
				
			while(true){
				ExecutorService pool = Executors.newCachedThreadPool();
				pool = Executors.newFixedThreadPool(20);
				// 获取具体章节
				for (String purl : urlMap.keySet()) {
					pool.execute(new GetChapter(maxid, baseUrl + purl, urlMap.get(purl), heads));
				}
				pool.shutdown();
				while(true){
		           if(pool.isTerminated()){  
		                break;  
		           }  
		           Thread.sleep(1000);    
				}
				
				List<String> records=db.getListStr("select title from SPIDER_NOVEL_chapter where id=?",new Object[]{maxid});
				if(urlMap.size()>0){
					for(String key:MapUtil.getKeys(urlMap)){
						String v = urlMap.get(key);
						if(records.contains(v)){
							urlMap.remove(key);
						}
					}
				}
				
				if(urlMap.size()==0){
					System.out.println(db.queryForInt("select count(1) from spider_novel_chapter where id=?",new Object[]{maxid}));
					break;
				}
			}

		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			DbUtil.close(conn);
		}

	}
}

class GetChapter implements Runnable {
	int maxid;
	String purl;
	String title;
	Map<String, String> heads = new HashMap<String, String>();

	public GetChapter(int maxid, String purl, String title, Map<String, String> heads) {
		this.maxid = maxid;
		this.purl = purl;
		this.title = title;
		this.heads.putAll(heads);
	}

	public void run() {
		Connection conn = DbUtil.getBestConn();
		String content = "";
		try {
			DbUtil db = new DbUtil(conn);
			if (!NovelUtil.chapterExists(db, maxid, title)) {
//				System.out.println(maxid+" "+title);
				content = SpiderUtil.getContentByProxy("biquge", purl, heads, "utf-8");
				content = HtmlUtil.find(content, "div", "id=content").get(0);
				content=HtmlUtil.changeHtmlSymbol(content);
				content = StringUtil.kill(content, "<", ">").trim();
				content=StringUtil.trim(content);
				if(content!=null&&!"".equals(content))
					NovelUtil.insertChapter(db, maxid, title, content);
			}
		} catch (Exception e) {
//			FileUtil.writeTxt(content, "c:/error.txt");
		}finally{
			DbUtil.close(conn);
		}
	}
}