JAVA实现多线程生产者消费者模型

• 问题背景: 做一个爬虫,从网上爬书籍,爬下来以后插入数据库

 

• 问题设计: 用多个线程同时从网上爬书,爬下来的书使用一个线程作为插入,这些线程共享同一个书籍队列,并且同时开始运行

书籍POJO

package thread;
 
 public class Book {
     private Integer bookId;
     private String bookName;
 
     public Integer getBookId() {
         return bookId;
     }
 
     public void setBookId(Integer bookId) {
         this.bookId = bookId;
     }
 
     public String getBookName() {
         return bookName;
     }
 
     public void setBookName(String bookName) {
         this.bookName = bookName;
     }
 }

爬虫线程

package thread;
 
 import java.util.List;
 import java.util.Queue;
 
 public class CrawlerThread extends Thread {
     private List<Integer> bookIdList;   //待爬书籍的Id号
     private Queue<Book> bookQueue;  //爬完书籍的共享存储队列
 
     public CrawlerThread(List<Integer> bookIdList, Queue<Book> bookQueue) {
         this.bookIdList = bookIdList;
         this.bookQueue = bookQueue;
     }
 
     public void run() {
         for(int i = 0; i < bookIdList.size(); i++) {
             Book book = new Book(); //把这两步当做爬虫爬书的过程
             book.setBookId(bookIdList.get(i));
             book.setBookName("书名" + book.getBookId());
 
             try {
                 sleep((long) (Math.random() * 1000 * 1000));  //随机sleep 100 - 1000 秒,模拟爬虫爬书时的延时过程
             } catch (InterruptedException e) {
                 e.printStackTrace();
             }
 
             bookQueue.offer(book);  //爬完的书籍插入共享队列
         }
     }
 }

插入线程

package thread;
 
 import java.util.Queue;
 
 public class InsertThread extends Thread {
     private Queue<Book> bookQueue;  //爬完书籍的共享存储队列
 
     public InsertThread(Queue<Book> bookQueue) {
         this.bookQueue = bookQueue;
     }
 
     public void run() {
         int timer = 0; //超时计时器
 
         while(timer < 30) { //如果连续30分钟bookQueue均为空,则超时,线程结束
             if(bookQueue.size() != 0) { //如果队列不为空
                 Book book;
                 while((book = bookQueue.poll()) != null) {
                     System.out.println(book.getBookName()); //把这步当成插入数据库吧
                 }
                 timer = 0;  //超时计时器清零
             } else {
                 try {
                     sleep(60 * 1000);   //等待爬虫一分钟
                 } catch (InterruptedException e) {
                     e.printStackTrace();
                 }
                 timer++;    //timer时间+1
             }
         }
     }
 }

主程序

package thread;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
 
 public class Start {
     public void main(String[] args) {
         Queue<Book> bookQueue = new ConcurrentLinkedQueue<Book>();  //使用线程安全的队列作为共享书籍队列
 
         //构造待爬书籍Id号
         List<Integer> bookIdList = new ArrayList<Integer>();
         bookIdList.add(1);
         bookIdList.add(2);
         bookIdList.add(3);
         bookIdList.add(4);
         bookIdList.add(5);
 
         CrawlerThread ct = new CrawlerThread(bookIdList, bookQueue);    //此处你可以将Id列表分为多分,分配给多个线程爬,我就不写了,懒..- -
         InsertThread it = new InsertThread(bookQueue);  //插入数据库线程
 
         ct.start();
         it.start();
     }
 }