九、学习分布式爬虫之多线程

实例：多线程实现爬取王者荣耀高清壁纸

import requests
from urllib import parse
from urllib import request
import os
import threading
from queue import Queue

headers = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.100 Safari/537.36',
    'referer': 'https://pvp.qq.com/web201605/wallpaper.shtml'
}

class Producer(threading.Thread):
    def __init__(self,page_queue,img_queue,*args,**kwargs):
        super(Producer,self).__init__(*args,**kwargs)
        self.page_queue = page_queue
        self.img_queue = img_queue

    def run(self):
        while not self.page_queue.empty():
            page_url = self.page_queue.get()
            resp = requests.get(page_url, headers=headers)
            resulte = resp.json()  # 将网页返回的内容进行json解析，这里得到一个字典
            datas = resulte['List']
            for data in datas:
                img_urls = extract_images(data)
                name = parse.unquote(data['sProdName']).replace('1:1','').strip()
                # print(name, img_urls)
                dir_path = os.path.join('images',name)  # 进行路径拼接
                if not os.path.exists(dir_path):  #如果目录存在就不创建
                    os.mkdir(dir_path)  # 创建文件夹
                for index,img_url in enumerate(img_urls):
                    img_path = os.path.join(dir_path, '%d.jpg' % (index + 1))
                    # print(img_path)
                    self.img_queue.put({"img_url":img_url,"dir_path":img_path})  #当消费者从这个中间容器拿值的时候，就会拿到两个值，一个是下载的地址，一个是存放的目录

class Consumer(threading.Thread):
    def __init__(self,img_queue,*args,**kwargs):
        super(Consumer, self).__init__(*args, **kwargs)
        self.img_queue = img_queue

    def run(self):
        while True:
            try:
                img_obj = self.img_queue.get(timeout=10)  #获得一个字典,如果队列当中没有数据，就会在这里进行阻塞,10s后还没有数据，就抛出异常
                img_url = img_obj.get('img_url')
                img_path = img_obj.get('dir_path')
                # print(img_url,img_path)
                try:
                    request.urlretrieve(img_url,img_path)  #下载
                    print(img_path+'下载完成')
                except:
                    print(img_path+'下载失败')
            except:
                print('退出')
                break

def extract_images(data):
    img_urls = []
    for x in range(1,9):
        img_url = parse.unquote(data['sProdImgNo_%d'%x]).replace('200', '0')
        img_urls.append(img_url)
    return img_urls

def main():
    page_queue = Queue(20)
    img_queue = Queue(1000) #中间容器，方便消费者从中取出数据下载
    for x in range(0,5):
        page_url = 'https://apps.game.qq.com/cgi-bin/ams/module/ishow/V1.0/query/workList_inc.cgi?activityId=2735&sVerifyCode=ABCD&sDataType=JSON&iListNum=20&totalpage=0&page={page}&iOrder=0&iSortNumClose=1&iAMSActivityId=51991&_everyRead=true&iTypeId=2&iFlowId=267733&iActId=2735&iModuleId=2735&_=1581737630573'.format(page=x)
        page_queue.put(page_url)

    for x in range(3): #创建三个线程生产者
        th = Producer(page_queue,img_queue,name='生产者%d号'%x)
        th.start()

    for x in range(5): #创建五个线程消费者
        th = Consumer(img_queue,name='消费者%d'%x)
        th.start()

if __name__ == '__main__':
    main()