爬虫学习线程池

线程池创建及例子

from concurrent.futures import ThreadPoolExecutor
def fn(name):
    for i  in range(100):
        print(name,i)

if __name__ == '__main__':
    with ThreadPoolExecutor(50) as t:  #创建线程池语法
        for i in range(100):
            t.submit(fn,f"线程{i}")  #执行任务

协程

当程序遇到IO操作时可以切换到其他任务中
充分利用cpu

写协程程序

import asyncio

async  def fun1():
    print("你好啊，我叫小笨猪")

if __name__ == '__main__':
    g=fun1()  #得到的是协程对象，此时函数是异步函数
    asyncio.run(g)

当程序出现同步操作 如： time.sleep() 就中断了
下面是测试例子：

import asyncio
import time
async  def fun1():
    print("超霸")
    await asyncio.sleep(2)  #挂起
    print("你好啊，我叫小笨猪")

async  def fun2():
    print("wudi 霸")
    await asyncio.sleep(3)
    print("你好啊，我叫潘金莲")
async  def fun3():
    print("asfafs霸")
    await asyncio.sleep(4)
    print("你好啊，我叫盘银联")
async  def fun4():
    print("adgfdg霸")
    await asyncio.sleep(5)
    print("你好啊，我叫猪八戒")
async def main():

     task = [
         asyncio.create_task(fun1()),
         asyncio.create_task(fun2()),
         asyncio.create_task(fun3()),
         asyncio.create_task(fun4())
     ]
     await asyncio.wait(task)


if __name__ == '__main__':
   
    t1 = time.time()
    asyncio.run(main())   #运行格式
    t2 = time.time()
    print(t2-t1)

异步操作爬取图片

例子：

import asyncio
import aiohttp   #异步操作获取请求需要的模块
urls = [
    "https://pic.netbian.com/uploads/allimg/220112/235701-1642003021a63b.jpg",
    "https://pic.netbian.com/uploads/allimg/220107/233112-164156947209db.jpg",
    "https://pic.netbian.com/uploads/allimg/211230/223710-1640875030af02.jpg"
]
async def getdownload(url):
    name = url.rsplit("/",1)[1]   #python基础 
    async  with aiohttp.ClientSession() as session:   #aiohttp.ClientSession()相当于requests
        async with session.get(url) as resp:         

            with open(name,mode="wb") as f:
                f.write(await resp.content.read())
async def main():
    tasks = []
    for url in urls:
        tasks.append(asyncio.create_task(getdownload(url)))
    await asyncio.wait(tasks)
if __name__ == '__main__':
    asyncio.get_event_loop().run_until_complete(main())