异步协程下载全部图片，虽比线程慢但不会卡死，详见源码

青龙摄影

于 2024-08-02 07:21:11 发布

阅读量280

点赞数 8

文章标签： python

本文链接：https://blog.csdn.net/lfsysc/article/details/140862239

版权

import requests
from bs4 import BeautifulSoup
import asyncio
import aiohttp

url = 'https://www.umei.cc/meinvtupian/nayimeinv/'

async def aiodown(url):
    cookies = {
        '__51vcke__K0KOUvCHIpTH8Vt6': '81cf9431-fdbf-5504-93a6-08ac77e51820',
        '__51vuft__K0KOUvCHIpTH8Vt6': '1721268844171',
        '__51uvsct__K0KOUvCHIpTH8Vt6': '3',
        'gxgefecookieinforecord': '%2C67-317404%2C',
        '__vtins__K0KOUvCHIpTH8Vt6': '%7B%22sid%22%3A%20%223b0f6380-c11e-5ba6-af30-dcd1896efbea%22%2C%20%22vd%22%3A%205%2C%20%22stt%22%3A%20482411%2C%20%22dr%22%3A%2016586%2C%20%22expires%22%3A%201721271448467%2C%20%22ct%22%3A%201721269648467%7D',
    }

    headers = {
        'authority': 'www.umei.cc',
        'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
        'accept-language': 'zh-CN,zh;q=0.9',
        'cache-control': 'max-age=0',    

        'referer': 'https://www.umei.cc/bizhitupian/',
        'sec-ch-ua': '"Chromium";v="122", "Not(A:Brand";v="24", "Google Chrome";v="122"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"Windows"',
        'sec-fetch-dest': 'document',
        'sec-fetch-mode': 'navigate',
        'sec-fetch-site': 'same-origin',
        'sec-fetch-user': '?1',
        'upgrade-insecure-requests': '1',
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.95 Safari/537.36',
    }

    response=requests.get(url, cookies=cookies, headers=headers)
    response.encoding="utf-8"
    soup =BeautifulSoup(response.text,'lxml')
    photos = soup.select("div.title a")

    for p in photos:
        p_url = p['href'].split("=")[-1]
        pnurl = "https://www.umei.cc/"+p_url
        p_name =p.text
        print(pnurl,p_name)
        res = requests.get(url=pnurl,cookies=cookies, headers=headers)
        res.encoding=("utf-8"),
        soup = BeautifulSoup(res.text, 'lxml')
        pp = soup.select("div.big-pic a img")[-1]
        pp_url=pp.get('src')
        async  with aiohttp.ClientSession() as session:
            async with session.get(pp_url, cookies=cookies, headers=headers) as res1:
                with open(f'img/{p_name}.jpg','wb') as f:
                    f.write(await res1.content.read())
async def main():
    masks=[]
    for i in range(1, 216):
        url = f'https://www.umei.cc/meinvitupian/siwameinv/index_{i}.htm'
        masks.append(aiodown(url))
        await asyncio.wait(masks)
if __name__ == "__main__":
    asyncio.run(main())