# 学习aiohttp
# 第一步 安装
#pip install aiohttp -i https://pypi.douban.com/simple
#pip install cchardet -i https://pypi.douban.com/simple
# 客户端
import aiohttp
import asyncio
async def fetch(session,url):
async with session.get(url) as response:
return await response.text()
# 基本
# async def main():
# async with aiohttp.ClientSession() as session:
# html=await fetch(session,"http://httpbin.org/headers")
# print(html)
'''
{
"headers": {
"Accept": "*/*",
"Accept-Encoding": "gzip, deflate",
"Host": "httpbin.org",
"User-Agent": "Python/3.9 aiohttp/3.7.3",
"X-Amzn-Trace-Id": "Root=1-602b6a92-01f81ee520af7312137b8421"
}
}
'''
#读取网页内容
# async def main():
# async with aiohttp.ClientSession() as session:
# async with session.get('http://httpbin.org/get') as resp:
# print(resp.status) # 状态码
# print(await resp.text(encoding='utf-8'))
'''
200
{
"args": {},
"headers": {
"Accept": "*/*",
"Accept-Encoding": "gzip, deflate",
"Host": "httpbin.org",
"User-Agent": "Python/3.9 aiohttp/3.7.3",
"X-Amzn-Trace-Id": "Root=1-602b6c17-66a4d09013f941f320068d7f"
},
"origin": "171.44.106.55",
"url": "http://httpbin.org/get"
}
'''
# 非文本内容格式 只需把text方法改成read即可
#请求的自定义
# 自定义Headers
# async def main():
# async with aiohttp.ClientSession() as session:
# url="http://httpbin.org/"
# headers = {
# "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
# "AppleWebKit/537.36 (KHTML, like Gecko)"
# " Chrome/78.0.3904.108 Safari/537.36"
# }
# await session.post(url, headers=headers)
# 自定义cookie
async def main():
url = 'http://httpbin.org/cookies'
cookies = {'cookies_are': 'working'}
async with aiohttp.ClientSession(cookies=cookies) as session:
async with session.get(url) as resp:
assert await resp.json() == {
"cookies": {"cookies_are": "working"}}
asyncio.run(main())
# 一个函数用来发起请求，另外一个函数用来下载网页
# 同步爬虫
#爬取豆瓣电影
from datetime import datetime
import requests
from bs4 import BeautifulSoup
from lxml import etree
#请求头
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit"
"/537.36 (KHTML, like Gecko) "
"Chrome/72.0.3626.121 Safari/537.36"}
def get_movie_url():
req_url='https://movie.douban.com/chart'
response=requests.get(url=req_url,headers=headers)
html=etree.HTML(response.text)
movies_url=html.xpath('//*[@id="content"]/div/div[1]/div/div/table/tr/td/a/@href')
return movies_url
def get_movie_content(movie_url):
response=requests.get(movie_url,headers=headers)
result=etree.HTML(response.text)
movie=dict()
name=result.xpath('//*[@id="content"]/h1/span[1]/text()')
author =result.xpath('//*[@id="info"]/span[1]/span[2]//text()')
movie["name"] = name
movie["author"] = author
return movie
if __name__ == '__main__':
start = datetime.now()
movie_url_list = get_movie_url()
movies = dict()
for url in movie_url_list:
movies[url] = get_movie_content(url)
print(movies)
print("同步用时为：{}".format(datetime.now() - start))
'''
{'https://movie.douban.com/subject/30458949/': {'name': ['无依之地 Nomadland'], 'author': ['赵婷']}, 'https://movie.douban.com/subject/30257787/': {'name': ['一秒钟'], 'author': ['张艺谋']}, 'https://movie.douban.com/subject/30443686/': {'name': ['穷途鼠的奶酪梦 窮鼠はチーズの夢を見る'], 'author': ['行定勋']}, 'https://movie.douban.com/subject/34869387/': {'name': ['女人的碎片 Pieces of a Woman'], 'author': ['凯内尔·穆德卢佐']}, 'https://movie.douban.com/subject/33408026/': {'name': ['刻在你心底的名字'], 'author': ['柳广辉']}, 'https://movie.douban.com/subject/34894753/': {'name': ['沐浴之王'], 'author': ['易小星']}, 'https://movie.douban.com/subject/35211578/': {'name': ['逃避虽可耻但有用 新春特别篇 逃げるは恥だが役に立つ ガンバレ人類! 新春スペシャル!!'], 'author': ['金子文纪']}, 'https://movie.douban.com/subject/30450313/': {'name': ['前程似锦的女孩 Promising Young Woman'], 'author': ['埃默拉尔德·芬内尔']}, 'https://movie.douban.com/subject/35275115/': {'name': ['2020去死 Death to 2020'], 'author': ['阿尔·坎贝尔', ' / ', 'Alice Mathias']}, 'https://movie.douban.com/subject/34982759/': {'name': ["玫瑰岛的不可思议的历史 L'incredibile storia dell'isola delle rose"], 'author': ['希德尼·希比利亚']}}
同步用时为：0:00:08.478348
'''
# 将同步代码改成异步测试速度
import asyncio
from datetime import datetime
import aiohttp
from lxml import etree
#请求头
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit"
"/537.36 (KHTML, like Gecko) "
"Chrome/72.0.3626.121 Safari/537.36"}
async def get_movie_url():
req_url='https://movie.douban.com/chart'
async with aiohttp.ClientSession(headers=headers) as session:
async with session.get(url=req_url,headers=headers) as response:
result=await response.text()
result=etree.HTML(result)
return result.xpath('//*[@id="content"]/div/div[1]/div/div/table/tr/td/a/@href')
async def get_movie_content(movie_url):
async with aiohttp.ClientSession(headers=headers) as session:
async with session.get(url=movie_url,headers=headers) as response:
result=await response.text()
result=etree.HTML(result)
movie = dict()
name = result.xpath('//*[@id="content"]/h1/span[1]//text()')
author = result.xpath('//*[@id="info"]/span[1]/span[2]//text()')
movie["name"] = name
movie["author"] = author
return movie
if __name__ == '__main__':
start = datetime.now()
loop = asyncio.get_event_loop()
movie_url_list = loop.run_until_complete(get_movie_url()) # run_until_complete 来运行 loop ，等到 future 完成，run_until_complete 也就返回了。
tasks = [get_movie_content(url) for url in movie_url_list]
movies = loop.run_until_complete(asyncio.gather(*tasks))
print(movies)
print("异步用时为：{}".format(datetime.now()- start))
'''
[{'name': ['无依之地 Nomadland'], 'author': ['赵婷']}, {'name': ['一秒钟'], 'author': ['张艺谋']}, {'name': ['穷途鼠的奶酪梦 窮鼠はチーズの夢を見る'], 'author': ['行定勋']}, {'name': ['女人的碎片 Pieces of a Woman'], 'author': ['凯内尔·穆德卢佐']}, {'name': ['刻在你心底的名字'], 'author': ['柳广辉']}, {'name': ['沐浴之王'], 'author': ['易小星']}, {'name': ['逃避虽可耻但有用 新春特别篇 逃げるは恥だが役に立つ ガンバレ人類! 新春スペシャル!!'], 'author': ['金子文纪']}, {'name': ['前程似锦的女孩 Promising Young Woman'], 'author': ['埃默拉尔德·芬内尔']}, {'name': ['2020去死 Death to 2020'], 'author': ['阿尔·坎贝尔', ' / ', 'Alice Mathias']}, {'name': ["玫瑰岛的不可思议的历史 L'incredibile storia dell'isola delle rose"], 'author': ['希德尼·希比利亚']}]
异步用时为：0:00:01.431173
'''
# 异步理解
'''
参考博客 https://www.cnblogs.com/xinghun85/p/9937741.html
'''
import asyncio
from datetime import datetime
import requests
# 例子
async def test2(i):
r=await other_test(i)
print("1",i,r)
async def other_test(i):
r = requests.get(i)
print("2",i)
await asyncio.sleep(4)
print("3",datetime.now()-start)
return r
url = ["https://segmentfault.com/p/1210000013564725",
"https://www.jianshu.com/p/83badc8028bd",
"https://www.baidu.com/"]
loop = asyncio.get_event_loop()
task = [asyncio.ensure_future(test2(i)) for i in url]
start =datetime.now()
loop.run_until_complete(asyncio.wait(task))
endtime = datetime.now()-start
print(endtime)
loop.close()
# aiohttp 爬虫实战
# 参考博客 ：https://blog.csdn.net/qq_36772866/article/details/105355445?ops_request_misc=%257B%2522request%255Fid%2522%253A%2522161346654316780271553327%2522%252C%2522scm%2522%253A%252220140713.130102334.pc%255Fblog.%2522%257D&request_id=161346654316780271553327&biz_id=0&utm_medium=distribute.pc_search_result.none-task-blog-2~blog~first_rank_v1~rank_blog_v1-3-105355445.pc_v1_rank_blog_v1&utm_term=%E7%88%AC%E8%99%AB&spm=1018.2226.3001.4450
# 第一步 分析 页面 查看小说信息所在的div
# 导包
import asyncio
import csv
import aiohttp
from bs4 import BeautifulSoup
#爬取网站内容
async def getData(url,headers):
# 创建会话对象
async with aiohttp.ClientSession() as session:
# 发送get请求，设置请求头
async with session.get(url,headers=headers) as response:
#返回响应内容
return await response.text()
#保存数据
def saveData(result):
for i in result:
soup=BeautifulSoup(i,'lxml')
find_div=soup.find_all('div',class_='book-mid-info')
for d in find_div:
# 小说名
name = d.find('h4').getText()
# 作者
author = d.find('a', class_='name').getText()
# 更新时间
update = d.find('p', class_='update').getText()
# 写入 csv
csvFile = open(r'C:\Users\zxy\Desktop\data.csv', 'w', encoding='utf-8-sig', newline='')
writer = csv.writer(csvFile)
writer.writerow([name, author, update])
csvFile.close()
# 创建异步任务并保存数据
def run():
for i in range(25):
#构建不同的 url 传入 getData，最后由 asyncio 模块执行
task=asyncio.ensure_future(getData(url.format(i+1),headers))
#将所有请求都加入到列表 tasks
tasks.append(task)
# 当所有请求执行完成，返回响应结果
result = loop.run_until_complete(asyncio.gather(*tasks))
saveData(result)
print(len(result))
if __name__=='__main__':
#请求头
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36'
}
# 任务列表
tasks=[]
# url
url='https://www.qidian.com/rank/hotsales?page={}'
#loop对象
loop=asyncio.get_event_loop()
#调用run函数
run()
如何在scrapy中使用aiohttp
这里需要修改下载中间件
import requests
# 现在，我们创建一个中间件，在这个中间件里面，使用 requests 请求一个需要延迟 5 秒钟才会返回的网址：
# class TestAiohttp:
# def get_ip(self):
# requests.get('http://httpbin.org/delay/5').json()
#
# def process_request(self,request,spider):
# print("请求一个5秒才会返回的网址")
# self.get_ip()
#将requests换成aiohttp
import asyncio
import aiohttp
class TestAiohttp:
async def get_ip(self):
async with aiohttp.ClientSession() as client:
resp = await client.get('http://httpbin.org/delay/5')
result = await resp.json()
print(result)
async def process_request(self, request, spider):
print('请求一个延迟5秒的网址开始')
await asyncio.create_task(self.get_ip())
爬虫部分
from datetime import datetime
import scrapy
class ExerciseSpider(scrapy.Spider):
name = 'exercise'
#allowed_domains = ['v.qq.com']
# 跟进url
start_urls = ['http://exercise.com']
def start_requests(self):
for page in range(1, 10):
url=f'http://exercise.kingname.info/exercise_middleware_ip/{page}'
yield scrapy.Request(url)
def parse(self, response):
now=datetime.now()
print(f'现在时间是：',now,response.text)
'''
在没使用下载中间件的时候，每次请求只需要一秒钟。
我么那在下载中间件内 需要请求一个五秒才会给反馈的地址
如果用普通的requests 每次请求时，请求将会被卡五秒
换成aiohttp后，启动后会启动五个并发(这里设置的最大并发数是5)，所以会同时打印出请求一个延迟5秒的网址开始5次。然后稍稍停 5 秒，这 5 个请求几乎同时完成，于是同时打印出这个延迟网址的返回信息。接下来，后面的请求就是每秒一个。
'''
设置部分settings
# 开启自定义下载中间件
DOWNLOADER_MIDDLEWARES = {
'aioTest.middlewares.TestAiohttp': 543
}
# 开启异步
TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor'
启动类
import asyncio
from scrapy import cmdline
import sys
if sys.platform == 'win32':
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
# 我这里不加这句会报错
cmdline.execute('scrapy crawl exercise'.split())
1211
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
