# -*- coding:utf8 -*-
import os.path
import re
import requests
import random
from concurrent.futures import ThreadPoolExecutor
class PictureDownload:
user_agent = [
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1"
]
headers = {
"User-Agent": random.choice(user_agent),
"sec-ch-ua-platform": "macOS"
}
@staticmethod
def create_picture(name, content):
name = os.path.join('/Users/wangze/PycharmProjects/Cloudwise/多线程/picture', name)
with open(name, 'wb') as f:
f.write(content)
def response(self, url):
response = requests.get(url=url, headers=self.headers)
return response
def download_picture(self, url):
try:
url_list = re.findall(r"<a href='javascript:void\(0\)' class='active2'>...</a> <a href=.*?>(\d+)</a>",
self.response(url).text)[0]
for current_url in range(1,int(url_list)+1):
download_url = url.replace('.html','-{}.html'.format(current_url))
pictures = re.findall('<img src="(.*?)" alt=(.*?) title="" /></a>',
self.response(download_url).text)
picture_url = pictures[0][0]
picture_name = pictures[0][0].split('/')[-1]
print('下载图片信息\n图片链接: {}\n图片名: {}\n'.format(picture_url, picture_name))
picture_content = self.response(picture_url).content
self.create_picture(picture_name, picture_content)
except IndexError:
pass
def html_list(self, url):
web_page = self.response(url).text
htmls = list(set(re.findall('<li><a href=(.*?)target="_blank" class="dl-pic">', web_page)))
html = []
for page in htmls:
html.append(page.replace('"', '').replace(' ', ''))
return html
def run(self):
for number in range(1,291):
url = 'https://www.2meinv.com/index-{}.html'.format(number)
with ThreadPoolExecutor(50) as t:
for download_url in self.html_list(url):
t.submit(self.download_picture,download_url)
# self.download_picture(download_url)
if __name__ == '__main__':
picture = PictureDownload()
picture.run()
python 多线程池爬虫
最新推荐文章于 2023-12-25 21:01:07 发布
![](https://img-home.csdnimg.cn/images/20240711042549.png)