import requests
from urllib import parse
import os
from urllib import request
headers={
'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36',
'referer':'https://pvp.qq.com/'
}
#发送请求
def send_requests(v):
print(v)
url=f'https://apps.game.qq.com/cgi-bin/ams/module/ishow/V1.0/query/workList_inc.cgi?activityId=2735&sVerifyCode=ABCD&sDataType=JSON&iListNum=20&totalpage={v}&page={v}&iOrder=0&iSortNumClose=1&jsoncallback=&iAMSActivityId=51991&_everyRead=true&iTypeId=2&iFlowId=267733&iActId=2735&iModuleId=2735&_=1661612201998'
resq=requests.get(url,headers=headers)
return resq.json()
#解析json
def parse_json(json_data):
data_lst=json_data['List']
d={}
for data in data_lst:
image_url_lst=exact_url(data)
sProdName=parse.unquote(data['sProdName'])
d[sProdName]=image_url_lst
'''for item in d:
print(item,d[item])'''
save_img(d)
#存储函数
def save_img(d):
a=1
for key in d:
#拼接路径 image/皮肤名称
dirpath=os.path.join('image',key.strip(' '))
if not os.path.exists(dirpath):
os.mkdir(dirpath)
#下载图片并且保存
print(a)
a+=1
for index,image_url in enumerate(d[key]):
request.urlretrieve(image_url,os.path.join(dirpath,'{}.jpg').format(index+1))
print('{}下载完毕'.format(d[key][index]))
#获取每张壁纸大小的url
def exact_url(data):
image_url_lst=[]
for i in range(1,9):
image_url=parse.unquote(data['sProdImgNo_{}'.format(i)]).replace('200','0')
image_url_lst.append(image_url)
return image_url_lst
#运行函数
def start():
for v in range(30):
json_data=send_requests(v)
parse_json(json_data)
if __name__ == '__main__':
start()
以上虽然可以下载,但运行速度过慢,可以使用多线程下载
import requests
from urllib import parse
from urllib import request
from queue import Queue
import threading
import os
headers={
'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36',
'referer':'https://pvp.qq.com/'
}
#获取每张壁纸大小的url
def exact_url(data):
image_url_lst=[]
for i in range(1,9):
image_url=parse.unquote(data['sProdImgNo_{}'.format(i)]).replace('200','0')
image_url_lst.append(image_url)
return image_url_lst
#生产者线程
class Producter(threading.Thread):
def __init__(self,page_queue,image_url_queue):
super().__init__()
self.page_queue=page_queue
self.image_url_queue=image_url_queue
def run(self):
while not self.page_queue.empty():
page_url=self.page_queue.get()
resp=requests.get(page_url,headers=headers)
json_data=resp.json()
data_lst = json_data['List']
d = {}
for data in data_lst:
image_url_lst=exact_url(data)
sProdName=parse.unquote(data['sProdName'])
d[sProdName]=image_url_lst
for key in d:
# 拼接路径 image/皮肤名称
dirpath = os.path.join('image', key.strip(' '))
if not os.path.exists(dirpath):
os.mkdir(dirpath)
# 下载图片并且保存
for index, image_url in enumerate(d[key]):
self.image_url_queue.put({'image_path':os.path.join(dirpath,'{}.jpg').format(index+1),'image_url':image_url})
#消费者线程
class Customer(threading.Thread):
def __init__(self,image_url_queue):
super().__init__()
self.image_url_queue = image_url_queue
def run(self):
while True:
try:
image_obj=self.image_url_queue.get(timeout=10)
request.urlretrieve(image_obj['image_url'],image_obj['image_path'])
print(f'{image_obj["image_path"]}下载完成')
except:
break
#启动线程的函数
def start():
page_queue=Queue(30)
image_url_queue=Queue(1000)
for i in range(0,30):
page_url=f'https://apps.game.qq.com/cgi-bin/ams/module/ishow/V1.0/query/workList_inc.cgi?activityId=2735&sVerifyCode=ABCD&sDataType=JSON&iListNum=20&totalpage={i}&page={i}&iOrder=0&iSortNumClose=1&jsoncallback=&iAMSActivityId=51991&_everyRead=true&iTypeId=2&iFlowId=267733&iActId=2735&iModuleId=2735&_=1661612201998'
# print(page_url)
page_queue.put(page_url)
#创建生产者线程对象
for i in range(30):
t=Producter(page_queue,image_url_queue)
t.start()
#创建消费者线程对象
for i in range(10):
t=Customer(image_url_queue)
t.start()
if __name__ == '__main__':
start()