爬虫09-多线程爬虫

import threading
from queue import Queue
import requests
from lxml import etree
import json
import os


# 线程类 用于爬取页面的数据
class ThreadCrawl(threading.Thread):

    # 定义请求头
    def __init__(self, thread_id):
        threading.Thread.__init__(self)
        self.threadID = thread_id
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36',
            'Accept-Language': 'zh-CN,zh;q=0.8'
        }

    # 重写run方法 线程和进程类的run方法会自动运行 无需调用
    def run(self):
        print('Starting', self.threadID)
        self.qiushi_spider()
        print('Exiting', self.threadID)

    # 爬取网站数据的方法
    def qiushi_spider(self):
        # 判断是否还有需要爬取的页面
        while not page_queue.empty():
            page = page_queue.get()
            url = 'http://www.qiushibaike.com/8hr/page/' + s