Java爬虫爬取wallhaven的图片

Java爬虫爬取wallhaven的图片

参考文章：JAVA Jsoup爬取网页图片下载到本地

需要的jar包：jsuop

wallhaven网站拒绝java程序访问，所以要伪装报头。

发送请求时

Connection con = Jsoup.connect(imgSrc)
        			  .header("User-agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36")
        			  .timeout(10*1000);//伪装报头;

下载连接时

connection.setRequestProperty("User-agent", "Mozilla/4.0");//伪装报头

查看报头（以谷歌浏览器为例子）
F12 → Network → XHR → 点击Name下任意一个 → Request Payload

源码👇

package spider;

import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.*;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLEncoder;
import java.util.regex.*;

public class Robot {
    public static void downImages(String filePath, String imgUrl) {
        // 若指定文件夹没有，则先创建
        File dir = new File(filePath);
        if (!dir.exists()) {
            dir.mkdirs();
        }
        
        // 截取图片文件名
        String fileName = imgUrl.substring(imgUrl.lastIndexOf('/') + 1, imgUrl.length());

        try {
            // 文件名里面可能有中文或者空格，所以这里要进行处理。但空格又会被URLEncoder转义为加号
            String urlTail = URLEncoder.encode(fileName, "UTF-8");
            // 因此要将加号转化为UTF-8格式的%20
            imgUrl = imgUrl.substring(0, imgUrl.lastIndexOf('/') + 1) + urlTail.replaceAll("\\+", "\\%20");

        } catch (UnsupportedEncodingException e) {
            e.printStackTrace();
        }
        // 写出的路径
        File file = new File(filePath + File.separator + fileName);

        try {
            URL url = new URL(imgUrl);
            URLConnection connection = url.openConnection();
            connection.setRequestProperty("User-agent", "Mozilla/4.0");//伪装报头
            connection.setConnectTimeout(10 * 1000);
            InputStream in = connection.getInputStream();
            BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(file));
            byte[] buf = new byte[1024];
            int size;
            while (-1 != (size = in.read(buf))) {
                out.write(buf, 0, size);
            }
            out.close();
            in.close();
        } catch (MalformedURLException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    public static void main(String[] args) {
        // 利用Jsoup获得连接
    	String url = "https://wallhaven.cc/";
        Connection connect = Jsoup.connect(url)
        		.header("User-agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36");//伪装报头
      try {
      // 得到Document对象
      Document document = connect.get();
      // 查找所有img标签
      Elements imgs = document.getElementsByTag("a");
      String pattern = "https://wallhaven.cc/w/.*";
      // 遍历img标签并获得src的属性
      for (Element element : imgs) {
          //获取每个img标签URL "abs:"表示绝对路径
          String imgSrc = element.attr("abs:href");
          // 打印URL
          boolean isMatch = Pattern.matches(pattern, imgSrc);
          if (isMatch){
        	  System.out.println("下载链接："+imgSrc);
        	  Connection con = Jsoup.connect(imgSrc)
        			  .header("User-agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36")
        			  .timeout(10*1000);//伪装报头;
        	  Document doc = con.get();
        	  Elements pic = doc.getElementsByTag("img");
        	  System.out.println(pic);
              System.out.println("共检测到下列图片URL：");
              System.out.println("开始下载");
              for(Element p : pic)
              {
            	  Element picSrc = p.getElementById("wallpaper");
            	  if(picSrc!=null) 
            	  {
            		  String picsrc = picSrc.attr("abs:src");
                	  System.out.println("图片链接："+picsrc);
                	  Robot.downImages("wallpaper", picsrc);
            	  }
              }
          }
          System.out.println("——————————————————————————————————");
      }
      System.out.println("下载完成");
  } catch (IOException e) {
      e.printStackTrace();
  }
  }
 }