# -*- encoding:utf-8 -*-
# --Author: Administrator
# --Time: 2023/03/17 14:59
import os
import parsel
import requests
import re
from concurrent.futures import ThreadPoolExecutor
import concurrent.futures
from tools.tool.founctions import save_directory
from utils.logging_tool.log_control import INFO
path = save_directory()
# 列表切割
def list_remove(children_list_len, filter_list):
"""
:param children_list_len: 指定切割的子列表的长度
:return: 新列表
"""
list_of_groups = zip(*(iter(filter_list),) * children_list_len)
end_list = [list(i) for i in list_of_groups]
count = len(filter_list) % children_list_len
end_list.append(filter_list[-count:]) if count != 0 else end_list
return end_list
def get_response(url):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'}
response = requests.get(url=url, headers=headers)
# 万能转码方式
response.encoding = response.apparent_encoding
return response
def get_hero_id():
hero_id_list = []
url = 'https://pvp.qq.com/web201605/js/herolist.json'
response = get_response(url).json()
for data in response:
hero_id = data['ename']
hero_id_list.append(hero_id)
return hero_id_list
def download_image_data(hero):
for hero_id in hero:
hero_url = f'https://pvp.qq.com/web201605/herodetail/{hero_id}.shtml'
response = get_response(hero_url)
selector = parsel.Selector(response.text)
hero_name = selector.css('.cover-name::text').get()
skin_name = re.findall('<ul class="pic-pf-list pic-pf-list3" data-imgname="(.*?)">', response.text)[0].split(
'|')
num = len(skin_name)
a = 0
for page in range(1, int(num) + 1):
a = a + 1
image_url = f'https://game.gtimg.cn/images/yxzj/img201606/skin/hero-info/{hero_id}/{hero_id}-bigskin-{page}.jpg'
image_content = get_response(image_url).content
path1 = path + f'\\{hero_name}\\'
if not os.path.exists(path1):
os.makedirs(path1)
filename = path1 + skin_name[int(page) - 1].split('&')[0] + '.jpg'
with open(filename, mode='wb') as f:
f.write(image_content)
INFO.logger.info(f"{a}: 正在保存:{hero_name}, {skin_name[int(page) - 1].split('&')[0]}")
def main():
data_list = list_remove(10, get_hero_id())
with concurrent.futures.ThreadPoolExecutor(max_workers=30) as executor:
for data in data_list:
executor.submit(download_image_data, data)
if __name__ == '__main__':
main()