爬虫 -- aiohttp解决证书错误以及使用代理

前言

• 以静态代理演示
• 下方代码中的代理皆无效

代码

# -*- coding: utf-8 -*-
# @Author   : zbz

import asyncio
import random

import aiohttp
from aiohttp import TCPConnector

username = "账号"
password = "密码"
proxy_auth = aiohttp.BasicAuth(username, password)


def get_proxy():
    proxies = [
        "http://113.2206.73.45:888",
        "http://124.21.73.13:888",
        "http://135.340.12.2512:888",
    ]
    return random.choice(proxies)


async def parse(url, delay):
    async with aiohttp.ClientSession(connector=TCPConnector(ssl=False)) as session:
        async with session.get(url, proxy=get_proxy(), proxy_auth=proxy_auth, timeout=3) as resp:
            print("start...")
            await asyncio.sleep(delay)
            html = await resp.text()
            print("end...")
            return delay, len(html)


async def run():
    url = "https://www.baidu.com"
    tasks = [
        asyncio.create_task(parse(url, delay))
        for delay in range(5)
    ]

    for coro in asyncio.as_completed(tasks):
        result = await coro
        print(result)


if __name__ == '__main__':
    asyncio.run(run())