# 一、aiohttp模块引出
import asyncio
import time
import requests
async def getUrl(url):
print("正在下载:", url)
response = requests.get(url)
print("下载完成:", url)
urls = ["https://www.baidu.com", 'https://www.sougou.com', 'https://www.bilibili.com']
tasks = []
for url in urls:
rs = getUrl(url)
task = asyncio.ensure_future(rs)
tasks.append(task)
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))
end = time.time()
requests.get是基于同步, 必须使用基于异步的网络请求模块进行指定url的请求发送
二、aiohttp
基于异步网络请求的模块
1.aiohttp的使用
- 下载 : pip install aiohttp
- 导入 : import aiohttp
- 使用该模块中的ClientSession进行网络请求的发送
import asyncio
import time
import requests
import aiohttp
start = time.time()
async def getUrl(url):
# 获取session对象
async with aiohttp.ClientSession() as session:
# 使用session发起get请求
async with await session.get(url) as response:
# text() 返回字符串形式的响应数据
# 注意: 获取响应数据操作之前一定要使用await进行手动挂起
page_text = await response.text()
print(page_text)
urls = [
"http://127.0.0.1:5000/a1",
"http://127.0.0.1:5000/b1",
"http://127.0.0.1:5000/c1"
]
tasks = []
for url in urls:
rs = getUrl(url)
task = asyncio.ensure_future(rs)
tasks.append(task)
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))
end = time.time()
print(end-start)
1.get(),post()的使用
- 如果想要进行UA伪装, 只需要给get或post添加相应的参数即可
- headers : 请求头信息
- params/data : 请求参数
- proxy : 设置代理ip
proxy = 'http://ip:port'