java使用jsoup,多线程批量爬取天极网某分类下的美女图片

本例子只作为测试,页面个数直接设置了100个,可以可能会少或者多,容易报错,更优化的一种方式是获取“下一页”按钮的地址,然后再访问,当访问不到“下一页”的内容时跳出

多线程只体现在文件提取,也可以在elements循环中再加一个多线程访问页面的

本案例需要jsoup包的支持,可到下方url下载

jsoup jar包

Test.java==============>主方法

package com.test.main;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class Test {
	public static void main(String[] args) {
		ExecutorService executor = Executors.newFixedThreadPool(5);
		Document doc = null;
		FileWriter writer = null;
		String rui="index";
		List<String> alist = new  ArrayList<String>();
		//int keyword = 4;
		for(int keyword=4;keyword<100;keyword++){
		try {
			//创建页面对象
			doc = Jsoup.connect("http://pic.yesky.com/c/6_20491_"+keyword+".shtml").userAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36").timeout(10000).get();
			//根据标签和class id获取元素
			Elements div = doc.select("div.lb_box");
			//根据标签获取元素
			Elements dl = div.select("dl");
			Elements dd = div.select("dd");
			Elements pages = dd.select("a");
			for(Element e : pages){
				System.out.println(e.text());
				System.out.println(e.attr("href"));
				Document imgdoc = Jsoup.connect(e.attr("href")).userAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36").timeout(10000).get();	
				Elements scroll = imgdoc.select("div.effect_scroll");
				Elements li = scroll.select("li");
				Elements urls = li.select("a");
				int i=0;
				for(Element ipage : urls){
					Document imgpage = Jsoup.connect(ipage.attr("href")).userAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36").timeout(10000).get();	
					Elements imgediv = imgpage.select("div.l_effect_img_mid");
					Element img = imgediv.select("img").first();
					FileOutUtils fo =new FileOutUtils(img, e.text());
					fo.start();
					System.out.println(i);
					i++;
				}
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
		}
	}
	/**
	 * 单线程下载
	 * @author lcx 
	 * @param e
	 * @param filepath
	 */
	public static void savefile(Element e,String filepath){
        String src=e.attr("src");//获取img中的src路径
        // System.out.println(src);
        //获取后缀名
        String imageName = src.substring(src.lastIndexOf("/") + 1,src.length());
        //连接url
        
        URL url;
        System.out.println(src);
		try {
			url = new URL(src);
			URLConnection uri=url.openConnection();
	        //获取数据流
	        InputStream is=uri.getInputStream();
	        //写入数据流
	        File file = new File("E://imgs//"+filepath);
	        if(!file.exists()){
	        	file.mkdirs();
	        }
	        OutputStream os = new FileOutputStream(new File("E://imgs//"+filepath+"//", imageName)); 

	        byte[] buf = new byte[1024]; 

			int l=0; 

			while((l=is.read(buf))!=-1){
				os.write(buf, 0, l);
			} 
		} catch (MalformedURLException e1) {
			e1.printStackTrace();
		} catch (FileNotFoundException e1) {
			e1.printStackTrace();
		} catch (IOException e1) {
			e1.printStackTrace();
		}
	}
}

FileOutUtils.java  ==============>多线程保存到本地

package com.test.main;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;

import org.jsoup.nodes.Element;

public class FileOutUtils extends Thread {
	private Element e;
	private String filepath;
	
	
	
	public FileOutUtils(Element e, String filepath) {
		this.e = e;
		this.filepath = filepath;
	}
	/**
	 * 多线程下载
	 * @author lcx 
	 * @param e
	 * @param filepath
	 */
	public void savefile(Element e,String filepath){
        String src=e.attr("src");//获取img中的src路径
        // System.out.println(src);
        //获取后缀名
        String imageName = src.substring(src.lastIndexOf("/") + 1,src.length());
        //连接url
        URL url;
		try {
			url = new URL(src);
			URLConnection uri=url.openConnection();
	        //获取数据流
	        InputStream is=uri.getInputStream();
	        //写入数据流
	        File file = new File("E://imgs//"+filepath);
	        if(!file.exists()){
	        	file.mkdirs();
	        }
	        OutputStream os = new FileOutputStream(new File("E://imgs//"+filepath+"//", imageName)); 
	        byte[] buf = new byte[1024]; 
			int l=0; 
			while((l=is.read(buf))!=-1){
				os.write(buf, 0, l);
			} 

		} catch (MalformedURLException e1) {
			e1.printStackTrace();
		} catch (FileNotFoundException e1) {
			e1.printStackTrace();
		} catch (IOException e1) {
			e1.printStackTrace();
		}
	}
	public void run() {
		this.savefile(this.e,this.filepath);
	}
}


阅读更多
版权声明:本文为博主原创文章,未经博主允许不得转载。 https://blog.csdn.net/u010121228/article/details/54943731
文章标签: 多线程 java
上一篇Tomcat自定义源码路径和编译并运行
下一篇Spring mvc + mybatis web项目当系统启动时运行某方法查询数据库内容
想对作者说点什么? 我来说一句

没有更多推荐了,返回首页

关闭
关闭