Jsoup初接触-发一个Jsoup抓取图片的程序

主要有两个线程：图片url抓取线程、图片下载保存线程。

图片下载保存采用线程池处理，主要利用java的ThreadPoolExecutor实现。

url抓取线程：

package sys.gifspider;

import java.io.IOException;
import java.util.Properties;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import sys.gifspider.utils.PropertyUtil;

public class GifSpider implements Runnable
{

  volatile boolean isRunning = true;
  private ThreadPoolExecutor threadPool;
  BlockingQueue<String> queue;
  
  public GifSpider(BlockingQueue<String> queue)
  {
    this.queue = queue;
    this.init();
  }
  
  /**
   * 线程池初始化
   */
  private void init()
  {
    Properties pro = PropertyUtil.getProperties();
    int corePoolSize = Integer.parseInt(pro.getProperty("threadpool.corePoolSize"));
    int maxPoolSize = Integer.parseInt(pro.getProperty("threadpool.maxPoolSize"));
    int keepAliveSeconds = Integer.parseInt(pro.getProperty("threadpool.keepAliveSeconds"));
    int queueCap = Integer.parseInt(pro.getProperty("threadpool.queueCapacity"));
    BlockingQueue<Runnable> queue = new LinkedBlockingQueue<Runnable>(queueCap);
    this.threadPool = new ThreadPoolExecutor(
        corePoolSize, maxPoolSize, keepAliveSeconds, TimeUnit.SECONDS, 
        queue);
  }
  public boolean isRunning()
  {
    return isRunning;
  }

  public void setRunning(boolean isRunning)
  {
    this.isRunning = isRunning;
  }

  @Override
  public void run()
  {
    while (this.isRunning)
    {
      try
      {
        
        String url = this.queue.take();
        System.out.println("请求url：" + url);
        Document doc = Jsoup.connect(url).get();
        //获取所有<a href>
        Elements s = doc.select("div.pic_list2").first().select("a[href]");
        for (Element e : s)
        {
          //有img 和  文字 两种href，指向相同德图片，只过滤图片href就行了
          Elements s1 = e.select("img");
          if (s1.size() != 0)
          {
            String imgUrl = e.absUrl("href");
            String text = s1.attr("alt");
            Document doc1 = Jsoup.connect(imgUrl).get();
            Elements e1 = doc1.getElementById("endtext").select("img");
            //网页源码中是相对路径，要获取绝对路径
            String realUrl = e1.attr("abs:src");
            System.out.println("获取图片url：" + realUrl);
            //获取到图片url，扔给线程池处理
            GifProcessor pro = new GifProcessor(text,realUrl);
            this.threadPool.execute(pro);
          }
          
        }
        Thread.sleep(1000);
      } catch (InterruptedException e)
      {
        e.printStackTrace();
      } catch (IOException e)
      {
        e.printStackTrace();
      }
    }
    
  }
  
}

图片处理线程很简单，就是图片下载和保存：

package sys.gifspider;

import sys.gifspider.utils.FileProcessor;

public class GifProcessor implements Runnable
{

  private String imgName;
  private String imgUrl;
  
  public GifProcessor(String name,String url)
  {
    this.imgName = name;
    this.imgUrl = url;
  }
  @Override
  public void run()
  {
    FileProcessor fp = new FileProcessor(this.imgName,this.imgUrl);
    try
    {
      System.out.println("下载保存图片url："+this.imgUrl);
      fp.saveGif();
      
    }catch(Exception e)
    {
      System.out.println("下载保存图片失败，url："+this.imgUrl);
      e.printStackTrace();
    }
    
  }
  
}

下载保存：

package sys.gifspider.utils;

import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;

public class FileProcessor
{
  private String imgName;
  private String imgUrl;
  
  public FileProcessor(String name,String url)
  {
    this.imgName = name;
    this.imgUrl = url;
  }
  
  /**
   * 保存路径，不存在就创建
   * @return
   */
  private String makeDir()
  {
    String strdir = PropertyUtil.getProperties().getProperty("dir");
    File dir = new File(strdir);
    if (!dir.exists())
    {
      dir.mkdir();
    }
    return strdir;
  }
  
  /**
   * 保存
   * @throws Exception
   */
  public void saveGif() throws Exception
  {
    String dir = makeDir();
    String file = dir + this.imgName + this.imgUrl.substring(this.imgUrl.lastIndexOf("."));
    BufferedOutputStream out = null;
    byte[] bit = this.download();
    if (bit.length > 0)
    {
      try
      {
        out = new BufferedOutputStream(new FileOutputStream(file));
        out.write(bit);
        out.flush();
      } finally
      {
        if (out != null)
          out.close();
      }
    }
  }
  /**
   * 下载
   * @return
   * @throws Exception
   */
  private byte[] download() throws Exception  
  {
    URL url = new URL(this.imgUrl);  
    HttpURLConnection httpConn = (HttpURLConnection) url.openConnection();  
    httpConn.connect();  
    InputStream cin = httpConn.getInputStream();  
    ByteArrayOutputStream outStream = new ByteArrayOutputStream();  
    byte[] buffer = new byte[1024];  
    int len = 0;  
    while ((len = cin.read(buffer)) != -1) {  
        outStream.write(buffer, 0, len);  
    }  
    cin.close();  
    byte[] fileData = outStream.toByteArray();  
    outStream.close();  
    return fileData;  
  }
}

程序入口如下：

package sys.gifspider;

import java.io.IOException;
import java.util.Properties;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import sys.gifspider.utils.PropertyUtil;

public class Main
{
  public static void main(String[] args)
  {
    init();
    
  }
  public static void init()
  {
    Properties pro = PropertyUtil.getProperties();
    int startPage = Integer.parseInt(pro.getProperty("startPage"));
    int endPage = Integer.parseInt(pro.getProperty("endPage"));
    String url = pro.getProperty("url");
    int count = endPage - startPage +1;
    BlockingQueue<String> queue = new LinkedBlockingQueue<String>(count);
    for (int i = 1; i <= count; i++)
    {
      queue.add(String.format(url, i));
    }
    int spiderCount = Integer.parseInt(pro.getProperty("spiderThread"));
    for (int i = 0; i < spiderCount; i++)
    {
      GifSpider spider = new GifSpider(queue);
      Thread t = new Thread(spider);
      t.start();
    }
  }
  
}

配置文件：

spiderThread=1

threadpool.corePoolSize=8
threadpool.maxPoolSize=10
threadpool.keepAliveSeconds=600
threadpool.queueCapacity=1000

startPage=1
endPage=20
url=http://www.haha365.com/gxtp/index_gif_%d.htm

dir=E:/spider/

用haha365的动态gif做了下测试，如果想趴别的网站，自己根据人家的html结构，改一下爬取规则即可。

程序中没做过多的容错处理，可能存在一定的bug。

源码下载