import asyncio
async def func1():
print("你好啊, 我叫潘金莲")
await asyncio.sleep(3)
print("你好啊, 我叫潘金莲")
async def func2():
print("你好啊, 我叫王建国")
await asyncio.sleep(2)
print("你好啊, 我叫王建国")
async def func3():
print("你好啊, 我叫李雪琴")
await asyncio.sleep(4)
print("你好啊, 我叫李雪琴")
async def main():
# 第一种写法
# f1 = func1()
# await f1 # 一般await挂起操作放在协程对象前面
# 第二种写法(推荐)
tasks1 = [
func1(),
func2(),
func3()
]
tasks2 = [
asyncio.create_task(func1()), # py3.8以后加上asyncio.create_task()
asyncio.create_task(func2()),
asyncio.create_task(func3())
]
await asyncio.wait(tasks1)
if __name__ == '__main__':
# 一次性启动多个任务(协程)
asyncio.run(main())
##################线程+多线程+协程三种方式下载图片应用
# requests.get() 同步的代码 -> 异步操作aiohttp
# pip install aiohttp
import time
import asyncio
import aiohttp
from threading import Thread
import requests
from concurrent.futures import ThreadPoolExecutor
urls = [
"http://kr.shanghai-jiuxin.com/file/2020/1031/191468637cab2f0206f7d1d9b175ac81.jpg",
"http://kr.shanghai-jiuxin.com/file/2020/1031/563337d07af599a9ea64e620729f367e.jpg",
"http://kr.shanghai-jiuxin.com/file/2020/1031/774218be86d832f359637ab120eba52d.jpg"
]
async def aiodownload(url):
# 发送请求.
# 得到图片内容
# 保存到文件
name = url.rsplit("/", 1)[1] # 从右边切, 切一次. 得到[1]位置的内容
async with aiohttp.ClientSession() as session: # requests
async with session.get(url) as resp: # resp = requests.get()
# 请求回来了. 写入文件
with open("./pic/"+name, mode="wb") as f: # 创建文件
f.write(await resp.content.read()) # 读取内容是异步的. 需要await挂起, resp.text()
print(name, "搞定")
async def main():
tasks = []
for url in urls:
#python 3.8版本以前这么用
tasks.append(aiodownload(url))
#ptyhon3.8以后这么用
# tasks.append(asyncio.create_task(aiodownload(url)))
await asyncio.wait(tasks)
def duoxiancheng_download(url):
resp = requests.get(url)
name = url.rsplit("/", 1)[1]
with open("./pic/"+name, mode="wb") as f: # 创建文件
f.write(resp.content) # 读取内容是异步的. 需要await挂起, resp.text()
print(name, "搞定")
if __name__ == '__main__':
#协程版本
# t1 = time.time()
# asyncio.run(main())
# t2 = time.time()
# print(t2-t1) #2.6049141883850098
#多线程版本
# t1 = time.time()
# for i in range(3):
# temp = Thread(target=duoxiancheng_download, args=(urls[i],))
# temp.start()
# # d1 = Thread(target=duoxiancheng_download, args=(urls[0],))
# # d1.start()
# # d2 = Thread(target=duoxiancheng_download, args=(urls[1],))
# # d2.start()
# # d3 = Thread(target=duoxiancheng_download, args=(urls[2],))
# # d3.start()
# t2 = time.time()
# print(t2-t1)
#利用线程池的多线程版本
t1 = time.time()
with ThreadPoolExecutor(5) as t:
for i in range(3):
# 把下载任务提交给线程池
# t.submit(duoxiancheng_download, urls[i])
t.submit(duoxiancheng_download, url=urls[i])
t2 = time.time()
print(t2-t1) #3.0072884559631348
############################用协程爬小说
# http://dushu.baidu.com/api/pc/getCatalog?data={"book_id":"4306063500"} => 所有章节的内容(名称, cid)
# 章节内部的内容
# http://dushu.baidu.com/api/pc/getChapterContent?data={"book_id":"4306063500","cid":"4306063500|11348571","need_bookinfo":1}
import requests
import asyncio
import aiohttp
import aiofiles
import json
import time
"""
1. 同步操作: 访问getCatalog 拿到所有章节的cid和名称
2. 异步操作: 访问getChapterContent 下载所有的文章内容
"""
#2. 异步操作:
async def aiodownload(cid, b_id, title):
data = {
"book_id":b_id,
"cid":f"{b_id}|{cid}",
"need_bookinfo":1
}
#用于拼凑章节内容的请求网址
data = json.dumps(data)
url = f"http://dushu.baidu.com/api/pc/getChapterContent?data={data}"
async with aiohttp.ClientSession() as session:
async with session.get(url) as resp:
dic = await resp.json()
#不使用异步写入 3.5019617080688477
# with open("./novel1/"+title, mode="w", encoding="utf-8") as f: # 创建文件
# f.write(dic['data']['novel']['content'])
#使用异步写入 1.4040799140930176
async with aiofiles.open("./novel1/"+title, mode="w", encoding="utf-8") as f:
await f.write(dic['data']['novel']['content']) # 把小说内容写出
#1. 同步操作:
async def getCatalog(url):
resp = requests.get(url)
#将获取到的页面目录数据转化为字典形式的数据,便于取数据
dic = resp.json()
tasks = []
for item in dic['data']['novel']['items']: # item就是对应每一个章节的名称和cid
title = item['title']
cid = item['cid']
# 准备异步任务
tasks.append(aiodownload(cid, b_id, title))
#开启协程异步任务
await asyncio.wait(tasks)
if __name__ == '__main__':
t1 = time.time()
b_id = "4306063500"
url = 'http://dushu.baidu.com/api/pc/getCatalog?data={"book_id":"' + b_id + '"}'
asyncio.run(getCatalog(url))
t2 = time.time()
print(t2-t1)
9_协程及应用
于 2022-03-09 17:08:45 首次发布
本文介绍了Python中使用asyncio进行协程编程,通过示例展示了并发下载图片的三种方式:同步、多线程和协程。接着,通过异步爬虫下载小说章节内容,对比了同步和异步操作的时间效率,突显了协程在提高性能上的优势。
摘要由CSDN通过智能技术生成