多线程爬虫

8g内存条，windows10环境，能爬取的连接在七百万左右，因为这时候list储存的url的占用内存太多导致频繁gc，cpu资源被沾满了，内存也接近极限了

package com.kk.demo.jsoup;

import java.io.IOException;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;

import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;

import org.apache.commons.logging.LogFactory;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.eclipse.collections.impl.list.mutable.primitive.IntArrayList;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.Test;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.google.common.base.Charsets;
import com.google.common.hash.BloomFilter;
import com.google.common.hash.Funnels;

/**
 * @author kk
 * 爬虫
 */
public class test {
	private static final Logger logger = LogManager.getLogger(test .class);
	
	//储存url 采用LinkedList是因为读写比较均匀 
	static List<String> linkList = Collections.synchronizedList(new LinkedList<>());
	
	//技术访问量
	static int count = 0;
	
	//种子url
	static String url = "http://www.youku.com";
	
	//布隆过滤器 初始化熔容量5千万+
	private static BloomFilter<String> boBloomFilter = 
												BloomFilter.create(Funnels.stringFunnel(Charsets.UTF_8), 1024*1024*20,0.00001);


public static void main(String[] args) throws InterruptedException, IOException {
		linkList.add(url);
		browse(linkList.get(linkList.size() - 1));
		//获取cpu核心数
		int processorsCount  = Runtime.getRuntime().availableProcessors();
		ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(
													processorsCount, //corePoolSize 线程池的大小
													processorsCount * 2, //maximumPoolSize 线程池的最大线程数
													100, //keepAliveTime 当线程数大于核心时，此为终止前多余的空闲线程等待新任务的最长时间。
													TimeUnit.SECONDS,
													new SynchronousQueue<Runnable>(), //用来储存等待执行任务的队列。
													new ThreadPoolExecutor.DiscardOldestPolicy() // 拒绝策略
													);
		for (int i = 0; i < processorsCount+1; i++) {
			threadPoolExecutor.execute(() -> {
				try {
					for (int j = 0; j < linkList.size(); j++) {
						 linkList.remove(j);
						 logger.info("线程池删除链接:{}",linkList.get(j));
						 requestList(linkList.get(j));
					}
				} catch (IOException | InterruptedException e) {
					e.printStackTrace();
				}
			});
		}



public static void requestList(String url) throws IOException, InterruptedException{
	try{
		Document doc = Jsoup.connect(url)
						.header("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:52.0) Gecko/20100101 Firefox/52.0")
						.header("Connection", "close")//如果是这种方式，这里务必带上
						.timeout(8000*5)//超时时间
						.get();
		Elements elements = doc.select("a[href]");

		//睡眠5毫秒
		TimeUnit.MICROSECONDS.sleep(5);
		
		String link = null;
	
		for (Element element : elements) {
			//获取绝对路径
			link = element.attr("abs:href");
			if(!boBloomFilter.mightContain(link) && 
			   link.length() > 0 &&
			   link.contains("youku")){
			   logger.info("*************访问连接:"+url);
				logger.info("第几个线程执行:"+Thread.currentThread().getName());
				logger.info("List大小:"+linkList.size());
				//添加到布隆过滤器
				boBloomFilter.put(link);
				//添加到list
				linkList.add(link);
				logger.info(link+"********************"+element.text());
				logger.info("count:"+count++);
				
			}
		}
	} catch (Exception e) {//可以精确处理timeoutException
			//超时处理
			e.printStackTrace();
			logger.info(e.getMessage());
		}
 }

依赖
	<!-- log4j2 -->
	<dependency>
	    <groupId>org.apache.logging.log4j</groupId>
	    <artifactId>log4j-api</artifactId>
	    <version>2.11.1</version>
	</dependency>
  	<dependency>
	    <groupId>org.apache.logging.log4j</groupId>
	    <artifactId>log4j-core</artifactId>
	    <version>2.11.1</version>
  	</dependency>

	<dependency>
	  <groupId>org.jsoup</groupId>
	  <artifactId>jsoup</artifactId>
	  <version>1.11.3</version>
	</dependency>

	 <!-- guava -->
  	<!-- https://mvnrepository.com/artifact/com.google.guava/guava -->
	<dependency>
	    <groupId>com.google.guava</groupId>
	    <artifactId>guava</artifactId>
	    <version>27.0-jre</version>
	</dependency>