Java多线程爬虫和存储

import org.apache.http.HttpHeaders;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
import org.dom4j.Element;

import java.io.IOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class GetBookInfoThread extends Thread{
    private CloseableHttpClient httpClient;
    private String webAddress;
    private Element rootElement;
    private Pattern bookAuthorRegex;
    private Pattern bookPublishRegex;
    private Pattern bookIsbnRegex;
    private Pattern bookImgRegex;
    private String bookName;

    /**
     *
     * @param httpClient    用这个操作抓取
     * @param webAddress    这个是抓取的网址
     * @param rootElement   这个是一个xml文档的根节点,用这个来操作加入新的子节点
     */
    public GetBookInfoThread(CloseableHttpClient httpClient,String webAddress,String bookName,Element rootElement,Pattern bookAuthorRegex,Pattern bookPublishRegex,Pattern bookIsbnRegex,Pattern bookImgRegex) {
        this.httpClient = httpClient;
        this.webAddress = webAddress;
        this.rootElement = rootElement;
        this.bookAuthorRegex = bookAuthorRegex;
        this.bookPublishRegex = bookPublishRegex;
        this.bookIsbnRegex = bookIsbnRegex;
        this.bookName = bookName;
        this.bookImgRegex = bookImgRegex;
    }

    @Override
    public void run() {
        HttpGet getBookInfo = new HttpGet(webAddress);
        getBookInfo.addHeader(HttpHeaders.USER_AGENT, "Mozilla/5.0 (Windows NT 5.2) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.122 Safari/534.30");
        CloseableHttpResponse bookInfoResponse;
        String bookInfoCode = null;//书籍具体信息网页源码
        try {
            bookInfoResponse = httpClient.execute(getBookInfo);
            if (bookInfoResponse.getStatusLine().getStatusCode() != 200) {
                System.out.println("获取书本具体信息时出错,页面地址:" + webAddress + "错误信息" + bookInfoResponse.getStatusLine());
                return;
            }

            bookInfoCode = EntityUtils.toString(bookInfoResponse.getEntity());
        } catch (IOException e) {
            e.printStackTrace();
        }

        Matcher bookAuthorMatcher = bookAuthorRegex.matcher(bookInfoCode); //匹配作者
        Matcher bookPublishMatcher = bookPublishRegex.matcher(bookInfoCode);    //匹配出版商
        Matcher bookIsbnMatcher = bookIsbnRegex.matcher(bookInfoCode);  //匹配isbn
        Matcher bookImgMatcher = bookImgRegex.matcher(bookInfoCode);    //匹配图片地址

        String bookName = this.bookName;
        String bookAuthor = "";
        String bookPublish = "";
        String bookIsbn = "";
        String bookLink = webAddress;
        String bookImg = "";

        if (bookAuthorMatcher.find()) {
            bookAuthor = bookAuthorMatcher.group(1);
        }
        if (bookPublishMatcher.find()) {
            bookPublish = bookPublishMatcher.group(1);
        }
        if (bookIsbnMatcher.find()) {
            bookIsbn = bookIsbnMatcher.group(1);
        }
        if (bookImgMatcher.find()) {
            bookImg = bookImgMatcher.group(1);
        }

//                    System.out.println(bookName + "-" + bookAuthor + "-" + bookPublish + "-" + bookIsbn);

        Element bookElement = rootElement.addElement("book");//新建一个书的标签
        bookElement.addAttribute("id",String.valueOf(Main.bookId++));
        bookElement.addElement("name").setText(bookName);
        bookElement.addElement("author").setText(bookAuthor);
        bookElement.addElement("publish").setText(bookPublish);
        bookElement.addElement("isbn").setText(bookIsbn);
        bookElement.addElement("count").setText(String.valueOf((int)(Math.random() * 10) + 3));
        bookElement.addElement("link").setText(bookLink);
        bookElement.addElement("img").setText(bookImg);

        System.out.println("抓取了:" + webAddress + " " + bookName);
    }
}

import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.dom4j.Document;
import org.dom4j.DocumentHelper;
import org.dom4j.Element;

import java.io.IOException;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class Main {

    CloseableHttpClient httpClient;
    static int bookId = 496;
    Map<String,Integer> proxyMap;//ip->端口
    List<String> ipList;//从这个list中读出ip，再由ip从map中读出端口
    int i = 0;//根据这个从list中取出ip，换上对应的代理

    public static void main(String[] args) {
        Main m = new Main();

//        List<String> tagList = m.getTagList();
        List<String> tagList = new LinkedList<String>();
//        tagList.add("经典");
//        tagList.add("日本文学");
//        tagList.add("散文");
//        tagList.add("中国文学");
//        tagList.add("算法");
//        tagList.add("童话");
//        tagList.add("外国文学");
//        tagList.add("文学");
//        tagList.add("小说");
//        tagList.add("漫画");
//        tagList.add("诗词");
//        tagList.add("心理学");
        tagList.add("摄影");
        tagList.add("理财");
        tagList.add("经济学");
        m.pullAndWrite(tagList,10);
    }

    public Main() {
//        HttpHost proxy = new HttpHost("122.225.106.35",80);
//        httpClient = HttpClients.custom().setProxy(proxy).build();
        httpClient = HttpClients.createDefault();
        setProxyMap();
    }

    public void setProxyMap() {
        proxyMap = new HashMap<String, Integer>();
        ipList = new LinkedList<String>();
        proxyMap.put("211.68.122.171",80);ipList.add("211.68.122.171");
    }

    public List<String> getTagList() {
        HttpGet getTag = new HttpGet("http://book.douban.com/tag/");
        getTag.addHeader(HttpHeaders.USER_AGENT, "Mozilla/5.0 (Windows NT 5.2) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.122 Safari/534.30");
        CloseableHttpResponse tagPageResponse = null;
        String tagPageCode = null;//网页源码
        try {
            tagPageResponse = httpClient.execute(getTag);
            tagPageCode = EntityUtils.toString(tagPageResponse.getEntity());
            tagPageResponse.close();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            try {
                tagPageResponse.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

        Pattern p = Pattern.compile("class=\"tag\">(.*?)</a>");
        Matcher m = p.matcher(tagPageCode);
        List<String> resultTagList = new LinkedList<String>();
        while (m.find()) {
            resultTagList.add(m.group(1));
        }

        return resultTagList;
    }

    /**
     *
     * @param tagList  要抓的图书的类别
     * @param maxPageNum 每种图书最多抓取的页数
     */
    public void pullAndWrite(List<String> tagList,int maxPageNum) {
        Pattern bookAddressRegex = Pattern.compile("href=\"(.*?)\" class=\"title\" target=\"_blank\">(.*?)</a>");   //获取具体书籍网址的正则
        Pattern bookAuthorRegex = Pattern.compile("(?s)<span class=\"pl\"> 作者</span>:.*?>(.*?)</a>");//匹配作者
        Pattern bookPublishRegex = Pattern.compile("<span class=\"pl\">出版社:</span> (.*?)<br/>");
        Pattern bookIsbnRegex = Pattern.compile("<span class=\"pl\">ISBN:</span> (.*?)<br/>");
        Pattern bookImgRegex = Pattern.compile("<img src=\"(.*?)\" title=\"点击看大图\"");

        //分别抓取每一种类别的书籍
        for (String tag:tagList) {
            int nowPageNum = 0;//目前正在抓取的页数
            Document newDocument = DocumentHelper.createDocument();
            Element rootElement = newDocument.addElement("root");

            while (nowPageNum < maxPageNum) {
                System.out.println(1);
                String nowPageAddress = "http://www.douban.com/tag/" + tag + "/book?start=" + nowPageNum * 15;//当前页的网址
                HttpGet getBooksPage = new HttpGet(nowPageAddress);
                getBooksPage.addHeader(HttpHeaders.USER_AGENT, "Mozilla/5.0 (Windows NT 5.2) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.122 Safari/534.30");
                CloseableHttpResponse booksPageResponse;
                Matcher m = null;
                try {
                    System.out.println(2);
                    booksPageResponse = httpClient.execute(getBooksPage);
                    System.out.println(3);
                    m = bookAddressRegex.matcher(EntityUtils.toString(booksPageResponse.getEntity()));
                    booksPageResponse.close();
                    if (booksPageResponse.getStatusLine().getStatusCode() != 200) {
                        System.out.println("抓 " + nowPageAddress + " 时出错:");
                        System.out.println("错误信息:" + booksPageResponse.getStatusLine());
                        changeProxy();
                        continue;//换个代理继续爬当前页
                    }
                } catch (IOException e) {
                    e.printStackTrace();
                }
                //具体每一本书，具体抓取
                int findCount = 0;//找到的书籍的数目
                List<Thread> threadList = new LinkedList<Thread>();
                while (m.find()) {
                    threadList.add(new GetBookInfoThread(httpClient, m.group(1), m.group(2), rootElement, bookAuthorRegex, bookPublishRegex, bookIsbnRegex,bookImgRegex));
                    findCount++;
                }
                //没有知道到代表这种类别的书都找完了，那么直接退出此类书籍的查找
                if (findCount == 0) {
                    break;
                }

                for (Thread thread:threadList) {
                    thread.start();
                }
                for (Thread thread:threadList) {
                    try {
                        thread.join();
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                }
                nowPageNum++;
            }
            //一个类别爬完了再写入
            new WriteBookInfoToFile(rootElement,"/home/geekgao/book/" + tag + ".xml").start();  //另开一个线程写入文件

        }
    }

    private void changeProxy() {
        if (i >= ipList.size()) {
            System.out.println("代理用完了,退出");
            System.exit(0);
        }
        String ip = ipList.get(i++);
        httpClient = HttpClients.custom().setProxy(new HttpHost(ip,proxyMap.get(ip))).build();
        System.out.println("换代理啦,使用代理:" + ip + "，端口:" + proxyMap.get(ip));
    }

}

import org.dom4j.Element;
import org.dom4j.io.XMLWriter;

import java.io.FileWriter;
import java.io.IOException;
import java.io.Writer;

public class WriteBookInfoToFile extends Thread {
    private Element root;
    private String fileAddress;

    public WriteBookInfoToFile(Element root,String fileAddress) {
        this.root = root;
        this.fileAddress = fileAddress;
    }

    @Override
    public void run() {
        Writer fileWriter;
        try {
            fileWriter = new FileWriter(fileAddress);
            XMLWriter xmlWriter = new XMLWriter(fileWriter);
            xmlWriter.write(root);
            xmlWriter.close();
            System.out.println("[" + fileAddress + "]写入成功");
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

import org.dom4j.Document;
import org.dom4j.DocumentException;
import org.dom4j.Element;
import org.dom4j.io.SAXReader;

import java.io.File;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.List;

public class WriteInfoToDB {
    public static void main(String[] args) {
        File folder = new File("/home/geekgao/book");
        File[] XMLS = folder.listFiles();
        SAXReader reader = new SAXReader();
        Statement statement = null;    //用这个执行sql语句
        try {
            Class.forName("com.mysql.jdbc.Driver");// 动态加载mysql驱动
            statement = DriverManager.getConnection("jdbc:mysql://localhost:3306/BookManage?user=root&password=root").createStatement();
        } catch (SQLException e) {
            e.printStackTrace();
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        }

        for (File f:XMLS) {
            if (f.isDirectory()) {
                continue;
            }
            Document document = null;
            try {
                document = reader.read(f);
            } catch (DocumentException e) {
                e.printStackTrace();
            }

            Element root = document.getRootElement();
            List<Element> books = root.elements();
            for (Element book:books) {
                String name = null;
                String author = null;
                String publish = null;
                String isbn = null;
                String count = null;
                String link = null;
                String img = null;
                List<Element> b = book.elements();
                for (Element info:b) {
                    if (info.getName().equals("name")) {
                        name = info.getText();
                    } else if (info.getName().equals("author")) {
                        author = info.getText();
                    } else if (info.getName().equals("publish")) {
                        publish = info.getText();
                    } else if (info.getName().equals("isbn")) {
                        isbn = info.getText();
                    } else if (info.getName().equals("count")) {
                        count = info.getText();
                    } else if (info.getName().equals("link")) {
                        link = info.getText();
                    } else if (info.getName().equals("img")) {
                        img = info.getText();
                    }
//                    System.out.println(info.getName() + ": " + info.getText());
                }
                String sql = "INSERT INTO Book(bookPublish,bookName,bookAuthor,bookTag,bookIsbn,bookCount,bookRestCount,bookLink,bookImg) VALUES ('" + publish + "','" + name + "','" + author + "','" + f.getName().split("\\.")[0] + "','" + isbn + "','" + count + "','" + count + "','" + link + "','" + img + "');";
                try {
                    statement.execute(sql);
                } catch (SQLException e) {
                    System.err.println("sql语句处错误:" + e.getMessage());
                    System.err.println("sql语句:" + sql);
                }
            }
        }
    }
}