python 多线程池爬虫

# -*- coding:utf8 -*-
import os.path
import re
import requests
import random
from concurrent.futures import ThreadPoolExecutor


class PictureDownload:
    user_agent = [
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1"
    ]

    headers = {
        "User-Agent": random.choice(user_agent),
        "sec-ch-ua-platform": "macOS"
    }

    @staticmethod
    def create_picture(name, content):
        name = os.path.join('/Users/wangze/PycharmProjects/Cloudwise/多线程/picture', name)
        with open(name, 'wb') as f:
            f.write(content)

    def response(self, url):
        response = requests.get(url=url, headers=self.headers)
        return response

    def download_picture(self, url):

        try:
            url_list = re.findall(r"<a href='javascript:void\(0\)' class='active2'>...</a> <a href=.*?>(\d+)</a>",
                                  self.response(url).text)[0]

            for current_url in range(1,int(url_list)+1):
                download_url = url.replace('.html','-{}.html'.format(current_url))
                pictures = re.findall('<img src="(.*?)" alt=(.*?) title="" /></a>',
                                      self.response(download_url).text)
                picture_url = pictures[0][0]
                picture_name = pictures[0][0].split('/')[-1]
                print('下载图片信息\n图片链接: {}\n图片名: {}\n'.format(picture_url, picture_name))
                picture_content = self.response(picture_url).content
                self.create_picture(picture_name, picture_content)
        except IndexError:
            pass

    def html_list(self, url):
        web_page = self.response(url).text
        htmls = list(set(re.findall('<li><a href=(.*?)target="_blank" class="dl-pic">', web_page)))
        html = []
        for page in htmls:
            html.append(page.replace('"', '').replace(' ', ''))
        return html

    def run(self):

        for number in range(1,291):
            url = 'https://www.2meinv.com/index-{}.html'.format(number)
            with ThreadPoolExecutor(50) as t:
                for download_url in self.html_list(url):
                    t.submit(self.download_picture,download_url)
                # self.download_picture(download_url)


if __name__ == '__main__':
    picture = PictureDownload()
    picture.run()