python 爬虫-‘斗图啦’网站图片下载

python 爬虫练手，爬取 下载 斗图啦网站前100页最新图片。、

其中知识点启用线程安全及多线程的方式进行同时下载。目前跑代码未出现异常报错。

以下实例代码

# -*- encoding=utf8 -*-

import requests
from lxml import etree
import os
import re
import threading
from queue import Queue

class P(threading.Thread):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36'
              }
    def __init__(self,page_queue,img_queue,*args,**kwargs):
        super(P,self).__init__(*args,**kwargs)   ##继承类的方法
        self.page_queue = page_queue
        self.img_queue = img_queue

    def run(self):
        while True:
            if self.page_queue.empty():break
            url = self.page_queue.get()
            self.parse_page(url)

    def parse_page(self,url):
        text = requests.get(url, headers=self.headers).text
        texthtml = etree.HTML(text)
        imgs = texthtml.xpath("//div[@class='page-content text-center']//a[@class='col-xs-6 col-sm-3']//img")
        for img in imgs:
            imgurl = img.get('data-original')
            imgname = img.get('alt')
            reimg = re.sub(r'[、?（）@⊙\\{}？，:></【】》《！|+.,;*]', '', imgname)  #使用正则表达式过滤掉图名的特殊符号
            hz = os.path.splitext(imgurl)[1]
            filename = reimg + hz
            self.img_queue.put((filename,imgurl))

class C(threading.Thread):

    def __init__(self,page_queue,img_queue,*args,**kwargs):
        super(C,self).__init__(*args,**kwargs)  ##继承类的方法
        self.page_queue = page_queue
        self.img_queue = img_queue

    def run(self):
        while True:
            if self.page_queue.empty() and self.img_queue.empty():
                print("已下载完成")
                break
            filename,imgurl = self.img_queue.get()
            betyimg = requests.get(imgurl,P.headers)
            with open('D:\\test\\'+filename,'wb') as image:  #使用二进制的方法进行保存图片
                image.write(betyimg.content)
                print(filename+'下载完成')

def main():
    page_queue = Queue(100)   #开启线程安全
    img_queue = Queue(500)

    ##只下载前面100页
    for x in range(1,101):
        url = "https://www.doutula.com/photo/list/?page=%d"%x
        page_queue.put(url)  ##将获取到的链接保存到page_queue 
    for x in range(5):
        t=P(page_queue,img_queue)  
        t.start()
    for x in range(5):
        t = C(page_queue,img_queue)
        t.start()

if __name__ == '__main__':
    main()