python 爬虫 自编小项目可实现十几分钟下载网页数十页数据

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2020/11/24 18:33
# @Author  : huni
# @File    : 多线程+多页爬）.py
# @Software: PyCharm

from threading import Thread        # 多线程的包
from queue import Queue             # 队列
import requests
from lxml import etree
import os
import time


class CrawlInfo(Thread):

    # 重写构造函数
    def __init__(self,url_queue,html_queue):
        Thread.__init__(self)
        #声明两个类属性
        self.url_queue = url_queue
        self.html_queue = html_queue

    # 重写run方法
    def run(self):
        #爬虫代码
        headers = {
   
            'Connection': 'close',
            'referer': 'https://www.xiurenji.com/XiuRen/',
            'User-Agent': 'Mozilla/5.0(Windows NT 10.0;Win64;x64)AppleWebKit/537.36(KHTML,like Gecko)Chrome/86.0.4240.183 Safari/537.36'
        }
        while self.url_queue.empty() == False:          # 当url队列中不是空的就继续爬
            url = self.url_queue.get()                  # 从队列中获取一个url
            reponse = requests.get(url=url,headers=headers)
            # 处理中文乱码问题
            page_text = reponse.text.encode('ISO-8859-1').decode('GB18030')
            if reponse.status_code ==