爬取 wallhaven图片到本地壁纸库

项目地址，另外知乎同名文章也是我发布的，大家可以多多关注

首先观察控制台

其次再看本地壁纸库

现在进入正题，这个小项目用到了 Jsoup具体版本见 POM），另外还用到了 JDK中的线程池、阻塞队列（生产-消费者模式）、NIO2（文件监听服务 API），所以至少要求 JDK版本为7或者以上

项目分为5个类和一个方法入口类

生产者类（任务：从列表页拿到详情页链接并放入阻塞队列）

public class Producer implements Runnable {

    private String name;
    private BlockingQueue<String> blockingQueue;

    public Producer(String name, BlockingQueue<String> blockingQueue) {
        this.name = name;
        this.blockingQueue = blockingQueue;
    }

    @Override
    public void run() {
        Document doc = null;
        try {
            for(int i = 1; i < 12018; i ++) {
                System.out.println();
                System.out.println();
                System.out.println("current page:" + i);
                System.out.println("-----------------------------------");
                if(i == 1) {
                    doc = Jsoup.connect("https://alpha.wallhaven.cc/latest").get();
                } else {
                    doc = Jsoup.connect("https://alpha.wallhaven.cc/latest?page=" + i).get();
                }
                Element div = doc.getElementById("thumbs");
                Elements sections = div.getElementsByTag("section");
                for (Element ele : sections) {
                    Elements links = ele.getElementsByClass("preview");
                    for (Element e : links) {
                        String href = e.attr("href");
                        blockingQueue.put(href);
                        System.out.println(name + " put " + href);
                    }
                }
            }
            blockingQueue.put("");
            System.out.println(name + " is over");
        } catch (IOException | InterruptedException e) {
            e.printStackTrace();
        } 
    }
}

消费者类（任务：从队列拿到链接并获取图片源地址并将下载任务交给一个缓存线程池）

public class Consumer implements Runnable {

    private String name;
    private BlockingQueue<String> blockingQueue;
    private ExecutorService taskPool;

    public Consumer(String name, BlockingQueue<String> blockingQueue, ExecutorService taskPool) {
        this.name = name;
        this.blockingQueue = blockingQueue;
        this.taskPool = taskPool;
    }

    @Override
    public void run() {
        Document doc = null;
        try {
            String href = null;
            while((href = blockingQueue.take()) != "") {
                System.out.println(name + " take " + href);
                doc = Jsoup.connect(href).get();
                Element img = doc.getElementById("wallpaper");
                String src = "https:" + img.attr("src");
                taskPool.submit(new DownloadTask(src));
            }
            System.out.println(name + " is over");
        } catch (IOException | InterruptedException e) {
            e.printStackTrace();
        } 
    }

}

下载任务执行类（任务：下载图片到本地）

public class DownloadTask implements Runnable {

    private static String path = "C:\\Users\\baiyapeng\\Desktop\\Paper\\";
    private String src;
    private String name;

    public DownloadTask(String src) {
        this.src = src;
        int n = src.lastIndexOf("/");
        this.name = src.substring(++n);
    }

    @Override
    public void run() {
        Response res = null;
        try {
            res = Jsoup.connect(src).ignoreContentType(true).timeout(30000).execute();
            byte[] bytes = res.bodyAsBytes();
            File file = new File(path + name);
            if (!file.exists()) {
                RandomAccessFile raf = new RandomAccessFile(file, "rw");
                raf.write(bytes);
                raf.close();
            }
        } catch (IOException e1) {
            e1.printStackTrace();
        }
    }

}

监听服务类（任务：将文件路径注册到监听服务上并开始监听）

public class ResourceListener {

    private static ExecutorService fixedThreadPool = Executors.newCachedThreadPool();

    private WatchService ws;

    private ResourceListener(String path) {
        try {
            ws = FileSystems.getDefault().newWatchService();
            start();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    private void start() {
        fixedThreadPool.execute(new Listener(ws));
    }

    public static void addListener(String path) {
        try {
            ResourceListener resourceListener = new ResourceListener(path);
            Path p = Paths.get(path);
            p.register(resourceListener.ws, StandardWatchEventKinds.ENTRY_CREATE);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

}

监听回调类（任务：执行回调任务）

public class Listener implements Runnable {

    private WatchService service;

    public Listener(WatchService service) {
        this.service = service;
    }

    @Override
    public void run() {
        try {
            while (true) {
                WatchKey watchKey = service.take();
                List<WatchEvent<?>> watchEvents = watchKey.pollEvents();
                for (WatchEvent<?> event : watchEvents) {
                    System.err.println(event.context() + "已下载");
                }
                watchKey.reset();
            }
        } catch (InterruptedException e) {
            e.printStackTrace();
        } 
    }
}

方法入口类

public class DownloadTaskExecutor {

    public static void main(String[] args) throws IOException {
        
        ResourceListener.addListener("C:\\Users\\baiyapeng\\Desktop\\Paper\\");
    
        BlockingQueue<String> blockingQueue = new SynchronousQueue<String>(true);
        ExecutorService proservice = Executors.newSingleThreadExecutor();
        ExecutorService conservice = Executors.newSingleThreadExecutor();
        ExecutorService taskPool = Executors.newCachedThreadPool();
        proservice.submit(new Producer("Producer", blockingQueue));
        conservice.submit(new Consumer("Consumer", blockingQueue, taskPool));
        proservice.shutdown();
        conservice.shutdown();
    }

}

最后就是设置壁纸库并设定更换频率

感谢大家，有问题可以再评论区留言~~