webcrawler-----Jsoap爬虫美女图片多线程

文章来源：http://www.oschina.net/code/snippet_1447924_45939

多线程部分根据网络代码改编，自己添加Jsoap 模块支持代理，jsoup-1.8.1.jar（需要的jar包在网上）

1、java代码：

package constant;
 
public class Constant {
    public static final String proxyHost = "*.*.*.*";//代理IP地址
    public static final String proxyPort = "8080";//代理端口
    public static final String AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.79";
    public static final String IMAGE_PATH = "D:\\IMAGE";//图片存放地址
}

2、java代码：

package downloadImg;
 
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.List;
 
import constant.Constant;
import proxy.UseProxy;
 
public class DownloadImage implements Runnable {
 
    private int imageCount = 0;
    private File image = null;
    private URL imageUrl = null;
    private List<String> images = null;
    private BufferedInputStream inputStream = null;
    private BufferedOutputStream outputStream = null;
 
    public DownloadImage(List<String> image) {
        this.images = image;
    }
 
    @Override
    public void run() {
        // TODO Auto-generated method stub
        SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd_HHmmssSSS");
        try {
 
            while (!images.isEmpty()) {
                new UseProxy();
                imageUrl = new URL(images.remove(0));
                imageUrl.openConnection().setConnectTimeout(12000);
                imageUrl.openConnection().setReadTimeout(12000);
                inputStream = new BufferedInputStream(imageUrl.openStream());
                image = new File(Constant.IMAGE_PATH + "\\" /*+ dateFormat.format(new Date())*/+ getFileName(imageUrl));
                if (!image.getParentFile().exists()) {
                    image.getParentFile().mkdirs();
                }
                outputStream = new BufferedOutputStream(new FileOutputStream(image));
                byte[] buf = new byte[2048];
                int length = inputStream.read(buf);
                while (length != -1) {
                    outputStream.write(buf, 0, length);
                    length = inputStream.read(buf);
                }
                next();
            }
            // wait();
        } catch (MalformedURLException e) {
            e.printStackTrace();
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            System.out.println("链接解析失败---" + imageUrl);
            e.printStackTrace();
        } finally {
            try {
                next();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
 
    private String getFileName(URL url) {
        String fileName = url.getFile();
        return fileName.substring(fileName.lastIndexOf('/') + 1);
    }
 
    public void next() throws IOException {
        if (inputStream != null) {
            inputStream.close();
        }
        if (outputStream != null) {
            outputStream.close();
        }
        image = null;
        // images = null;
        imageUrl = null;
        inputStream = null;
        outputStream = null;
        System.gc();
        System.out.println("DownloadImage >>> " + ++imageCount);
    }
 
}

3、java代码：

package myjsoap;
 
import java.util.List;
 
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
 
import proxy.UseProxy;
import constant.Constant;
import downloadImg.DownloadImage;
 
public class DealUrl implements Runnable{
    // 已解析url队列
    private List<String> visited = null;
    // 未解析url队列
    private List<String> hrefs = null;
    // 图片链接队列
    private List<String> images = null;
     
    //已解析链接数
    private int analyze = 0;
    private int count   = 0;        
    public DealUrl(List<String> hrefs, List<String> visited, List<String> images) {
        this.hrefs = hrefs;
        this.visited = visited;
        this.images = images;
    }
 
    public void run() {
        while (!hrefs.isEmpty()) {
            // 把当前要解析的url字符串从hrefs移到visited
            String urlTmp = hrefs.remove(0);
            if (visited.indexOf(urlTmp) != -1)
                continue;
            visited.add(urlTmp);
            Document doc = getUrlDoc((String) visited.get(visited.size() - 1));
            if (doc == null)
                continue;
            System.out.println("已解析第 " + ++analyze + " 个连接。。。"+urlTmp);
            Elements hrefLinks = doc.select("a[href]");
            Elements imgLinks = doc.select("img[src]");
            if (hrefLinks != null)
                for (Element link : hrefLinks) {
                    String newUrl = link.attr("abs:href");
                    if (newUrl.indexOf("ququ") != -1)
                        hrefs.add(newUrl);
                    // System.out.println(++count + "  >>> " +
                    // link.attr("abs:href"));
                }
            if (imgLinks == null)
                continue;
            for (Element link : imgLinks) {
                String temImgUrl = link.attr("abs:src");
                if (temImgUrl.indexOf(".jpg") != -1 && images.indexOf(temImgUrl) == -1) {
                    images.add(link.attr("abs:src"));
                    System.out.println("img:"+link.attr("abs:src"));
                }
            }
             new Thread(new DownloadImage(images)).start();
        }
        System.gc();
    }
    public Document getUrlDoc(String url){
        Document doc = null;
        try {
            new UseProxy();//不是代理上网的可以注释掉
            Connection conneciton = Jsoup.connect(url);
            conneciton.userAgent(Constant.AGENT);
            doc = conneciton.get();
        } catch (Exception e) {
            System.out.println("connect fail!");
                        return null;
        }
        return doc;
    }
}

4、java代码：

package proxy;
 
import java.util.Properties;
 
import constant.Constant;
 
public class UseProxy {
    public UseProxy() {
        Properties prop = System.getProperties();
        prop.setProperty("http.proxyHost", Constant.proxyHost);
        prop.setProperty("http.proxyPort",  Constant.proxyPort);
    }
}

5、java代码：

package start;
 
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
 
import myjsoap.DealUrl;
 
public class MyRobot {
      
    private List<String> hrefs   = Collections.synchronizedList(new ArrayList<String>());
    private List<String> visited = Collections.synchronizedList(new ArrayList<String>());
    private List<String> images  = Collections.synchronizedList(new ArrayList<String>());
      
    public MyRobot(String href) {
        hrefs.add(href);
    }
      
    public void run() throws InterruptedException {
        ExecutorService pool = Executors.newFixedThreadPool(2);
        pool.execute(new DealUrl(hrefs, visited, images));
        Thread.sleep(8000);
        pool.execute(new DealUrl(hrefs, visited, images));
        pool.shutdown();
    }
      
    public static void main(String[] args) throws InterruptedException {
        MyRobot robot = new MyRobot("http://500ququ.com/");
        robot.run();
    }
  
}