Java爬虫(httpClient+Jsoup)彼岸桌面壁纸GUI版

Java爬虫彼岸桌面壁纸GUI版

项目结构

package guiAndCode;

import guiAndCode.GUIMain;

import java.awt.TextArea;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;

import javax.swing.JTextArea;
import javax.swing.text.DefaultCaret;

import org.apache.commons.io.FileUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class DownLoadPicture {
    int sum = 0;
    static int page;
    GUIMain gui;
    int i;
    String typeName = null;
    String adress;
    String imageName;

    public DownLoadPicture() {//构造方法

    }

    //本类当中的一个普通方法
    public void downLoadPicture(int page, String typeName, GUIMain gui) throws UnsupportedOperationException, IOException {
        // 创建一个链接

        this.gui = gui;
        //setVisible(false);
        CloseableHttpClient httpClient = HttpClients.createDefault();
        // 利用链接发送请求index_2.htm


        if (page == 1)
            adress = "http://www.netbian.com/" + typeName + "/index.htm";
        else
            adress = "http://www.netbian.com/" + typeName + "/index_" + page + ".htm";
        HttpGet httpGet = new HttpGet(adress);
        // 接收回应
        CloseableHttpResponse httpResponse = httpClient.execute(httpGet);
        // 获取响应解析工具httpEntity
        HttpEntity httpEntity = httpResponse.getEntity();
        // content响应内容（网页源代码），并解决中文乱码用UTF-8
        String content = EntityUtils.toString(httpEntity, "gbk");
        // 现在得到了网页源代码，而图片就在源代码中
        // 拿到了脏数据----要清洗--像洗菜一样---
        // Jsoup解析器
        Document document = Jsoup.parse(content);
        // elements标签下的内容                                             //直接从浏览器复制过来
        Elements elements = document.select("div.wrap.clearfix #main div.list  ul  li  a ");// 选择，元素，标签
        //如果类名有空格继续用.链接

        // 进一步清洗
        for (int i = 0; i < 21; i++) {// 观察到一个有略缩图的网页一共有30张
            Element element = elements.get(i);
            // 拿到高清图的链接，这个链接在 href 标签里面，attr接收到高清图链接
            String attr = element.attr("href");
            if (attr.equals("https://pic.netbian.com/"))
                continue;
            // 重复上面的步骤 //创建一个链接
            CloseableHttpClient httpClient2 = HttpClients.createDefault();
            // 利用高清图链接发送请求
            HttpGet httpGet2 = new HttpGet("http://www.netbian.com" + attr + "");
            // 接收回应
            CloseableHttpResponse httpResponse2 = httpClient2.execute(httpGet2);
            // 获取响应解析工具httpEntity
            HttpEntity httpEntity2 = httpResponse2.getEntity();
            // content响应内容（网页源代码），并解决中文乱码用UTF-8
            String content2 = EntityUtils.toString(httpEntity2, "gbk");
            // Jsoup解析器                    再进一步的清洗
            Document document2 = Jsoup.parse(content2);//直接从浏览器复制过来
            Elements elements2 = document2.select("div.endpage div.pic p a img");// 选择，元素，标签
            // System.out.println( elements2);
            Element element2 = elements2.get(0);
            // src有图片的地址，得到了高清图片的地址src.....
            String attr3 = element2.attr("src");
             imageName = element2.attr("title");
            // 利用高清图地址发送请求
            HttpGet httpGet3 = new HttpGet(attr3);
            // 执行下载该高清原图
            CloseableHttpResponse httpResponse3 = httpClient.execute(httpGet3);
            HttpEntity httpEntity3 = httpResponse3.getEntity();
            //流入本地文件夹
            InputStream stream = httpEntity3.getContent();
            FileUtils.copyInputStreamToFile(stream, new File("" + gui.savePath + "\\" + typeName + "\\" + page + "-" + (i + 1) + ""+imageName+".jpg"));
            sum++;

            //Thread.sleep(1000);
            System.out.print("恭喜第" + page + " 页第" + (i + 1) + "图片正在下载...\n总共下载了" + sum + "张图;\n目录在" + gui.savePath + "\n" + typeName + "\n");

            gui.upDataGUI(page, i + 1, sum, typeName);


        }


        page = page + 1;//下一页
        downLoadPicture(page, typeName, gui);

    }

}
 
 

package guiAndCode;

import guiAndCode.DownLoadPicture;

import java.awt.BorderLayout;
import java.awt.EventQueue;

import javax.swing.*;
import javax.swing.border.EmptyBorder;
import javax.swing.text.DefaultCaret;
import javax.swing.GroupLayout.Alignment;

import java.awt.Font;
import java.awt.TextArea;

import java.awt.event.ActionListener;
import java.awt.event.ActionEvent;
import java.awt.event.MouseAdapter;
import java.awt.event.MouseEvent;
import java.io.IOException;
import java.awt.Color;

public class GUIMain extends JFrame {
    public static Thread t;
    private JPanel contentPane;
    int sel;
    JTextArea textArea;
    JButton jButtonSave;
    static GUIMain frame;
    SelectType selectType;
    DownLoadPicture downLoadPicture;
    JScrollPane scrollPane;
    JFileChooser jFileChooser;
    String savePath;

    /**
     * Launch the application.
     */
    public static void main(String[] args) {
        frame = new GUIMain();
        frame.setVisible(true);
    }

    /**
     * Create the frame.
     */

    public GUIMain() {


        jButtonSave = new JButton("选择保存路径");
        jButtonSave.setBounds(55, 60, 200, 50);
        jButtonSave.addActionListener(new ActionListener() {
            @Override
            public void actionPerformed(ActionEvent e) {

                jFileChooser = new JFileChooser();
                jFileChooser.setDialogType(JFileChooser.SAVE_DIALOG);
                jFileChooser.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY);
                jFileChooser.showSaveDialog(null);
                savePath = jFileChooser.getSelectedFile().getPath();
                jButtonSave.setEnabled(false);
            }
        });
        setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
        setBounds(100, 100, 817, 618);
        contentPane = new JPanel();
        contentPane.setBorder(new EmptyBorder(5, 5, 5, 5));
        setContentPane(contentPane);
        contentPane.setLayout(null);


        JPanel panel = new JPanel();
        panel.setBackground(new Color(154, 205, 50));
        panel.setBounds(10, 10, 803, 572);
        contentPane.add(panel);
        panel.setLayout(null);


        JLabel lblNewLabel = new JLabel("\u6B22\u8FCE\u4F7F\u7528\u5F7C\u5CB8\u58C1\u7EB8\u4E0B\u8F7D\u5DE5\u5177");
        lblNewLabel.setFont(new Font("宋体", Font.BOLD, 25));
        lblNewLabel.setBounds(209, 22, 397, 29);
        panel.add(lblNewLabel);
        panel.add(jButtonSave);
        JLabel lblNewLabel_1 = new JLabel("\u8BF7\u9009\u62E9\u4F60\u8981\u4E0B\u8F7D\u7684\u58C1\u7EB8\u7C7B\u578B");
        lblNewLabel_1.setFont(new Font("宋体", Font.BOLD, 20));
        lblNewLabel_1.setBounds(10, 131, 313, 29);
        panel.add(lblNewLabel_1);

        JComboBox comboBox = new JComboBox();
        comboBox.addActionListener(new ActionListener() {
            public void actionPerformed(ActionEvent e) {
                sel = comboBox.getSelectedIndex();

            }
        });

        comboBox.setModel(new DefaultComboBoxModel(new String[]{"", "\u65E5\u5386", "\u52A8\u6F2B", "\u98CE\u666F", "\u7F8E\u5973", "\u6E38\u620F", "\u5F71\u89C6", "\u52A8\u6001", "\u552F\u7F8E", "\u8BBE\u8BA1", "\u53EF\u7231", "\u6C7D\u8F66", "\u82B1\u5349", "\u52A8\u7269", "\u8282\u65E5", "\u4EBA\u7269", "\u7F8E\u98DF", "\u6C34\u679C", "\u5EFA\u7B51", "\u4F53\u80B2", "\u519B\u4E8B", "\u975E\u4E3B\u6D41", "\u5176\u5B83", "\u738B\u8005\u8363\u8000", "\u62A4\u773C", "LOL"}));
        comboBox.setBounds(50, 180, 114, 39);
        panel.add(comboBox);

        JButton btnNewButton = new JButton("\u5F00\u59CB\u4E0B\u8F7D");

        btnNewButton.setFont(new Font("宋体", Font.PLAIN, 25));

        scrollPane = new JScrollPane();
        scrollPane.setBounds(314, 100, 435, 317);

        panel.add(scrollPane);

        textArea = new JTextArea();
        textArea.setEditable(false);

        scrollPane.setViewportView(textArea);

        btnNewButton.addActionListener(new ActionListener() {

            public void actionPerformed(ActionEvent e) {
                selectType = new SelectType(sel);

                downLoadPicture = new DownLoadPicture();
                comboBox.setEnabled(false);
                btnNewButton.setEnabled(false);
                //GUI开始下载事件
                try {
                    //JOptionPane.showMessageDialog(btnNewButton, "\n下载进度请查看\nC盘图片爬虫/"+selectType.typeName+"/\n现在开始下载");


                    new Thread() {
                        public void run() {


                            textArea.append("准备开始下载\n");
                            textArea.paintImmediately(textArea.getBounds());
                            try {
                                downLoadPicture.downLoadPicture(1, selectType.typeName, frame);
                            } catch (UnsupportedOperationException e1) {
                                // TODO 自动生成的 catch 块
                                e1.printStackTrace();
                            } catch (IOException e1) {
                                // TODO 自动生成的 catch 块
                                e1.printStackTrace();
                            }
                            //textArea.append("开始下载\n"+downLoadPicture.page+"");
                            textArea.paintImmediately(textArea.getBounds());

                        }

                    }.start();
//				


                } catch (UnsupportedOperationException e1) {
                    // TODO 自动生成的 catch 块

                    JOptionPane.showMessageDialog(btnNewButton, "下载已经完成啦");
                    System.out.println("下载完成");

                    e1.printStackTrace();


                }
            }


        });
        btnNewButton.setBounds(105, 460, 141, 51);
        panel.add(btnNewButton);

        JButton btnNewButton_1 = new JButton("\u9000\u51FA");
        btnNewButton_1.setFont(new Font("宋体", Font.PLAIN, 25));
        btnNewButton_1.setBounds(520, 460, 141, 51);
        btnNewButton_1.addActionListener(new ActionListener() {
            @Override
            public void actionPerformed(ActionEvent e) {

                System.exit(0);
            }
        });
        panel.add(btnNewButton_1);


        //setVisible(false);
    }

    public void upDataGUI(int page, int i, int sum, String typeName) {

        textArea.append("\n\t图片正在下载...总共下载了" + sum +"张\n" +  downLoadPicture.imageName+".jpg");

        textArea.setCaretPosition(textArea.getDocument().getLength());
        //frame.textArea.paintImmediately(textArea.getBounds());

    }
}

package guiAndCode;

public class SelectType {
    int ch = 0;
    String typeName = null;

    String[] arry = {"rili", "dongman", "fengjing", "meinv", "youxi", "yingshi", "dongtai", "weimei", "sheji", "keai", "qiche", "huahui", "dongwu"

            , "jieri", "renwu", "meishi", "shuiguo", "jianzhu", "tiyou", "junshi", "feizhuliu", "qita", "s/wangzherongyao", "s/huyan", "s/lol"};

    public SelectType(int ch) {
        this.ch = ch;

        typeName = arry[ch-1];
    }


}

运行效果图