python协程多任务爬取下载数据

WonderfulUUZ

已于 2024-02-07 17:42:56 修改

阅读量285

点赞数 6

文章标签： python 开发语言

于 2024-02-07 17:40:30 首次发布

本文链接：https://blog.csdn.net/WonderfulUUZ/article/details/136070660

版权

本文介绍了如何使用Python的requests库抓取网页数据，通过BeautifulSoup和XPath解析内容，实现异步下载图片并存储到本地文件，展示了从数据爬取到持久化的完整流程。

摘要由CSDN通过智能技术生成

1、首先实现爬取数据的基本功能

import requests
import os
from lxml import etree
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'
}
for page in range(2, 4):
    url = 'https://pic.netbian.com/4kqiche/index_%d.html'%page
    resp = requests.get(url=url, headers=headers)
    page_text = resp.text
    print(page_text)

打印一下，看一下页面内容有没有编码错误

有点小问题，设置一下编码

for page in range(2, 4):
    url = 'https://pic.netbian.com/4kqiche/index_%d.html'%page
    resp = requests.get(url=url, headers=headers)
    resp.encoding = 'gbk' #设置编码
    page_text = resp.text
    print(page_text)

2、xpath数据解析

打印一下，看一下有没有取到列表元素

    tree = etree.HTML(page_text)
    li_list = tree.xpath('//*[@id="main"]/div[3]/ul/li')
    print(li_list)

解析文件名称和下载地址

    for li in li_list:
        title = li.xpath('./a/b/text()')[0]
        print(title)
        pic_url = 'https://pic.netbian.com'+li.xpath('./a/img/@src')[0]
        print(pic_url)

3、封装第一个函数

用字典把title和pic_url都存储起来，同时加上函数返回值方便调用

def get_car():
    car_msg = [] #存储所有汽车图片信息
    for page in range(2, 4):
        url = 'https://xxx.xxxxxxx.com/4kqiche/index_%d.html'%page
        resp = requests.get(url=url, headers=headers)
        resp.encoding = 'gbk' #设置编码
        page_text = resp.text
        #print(page_text)
    #xpath数据解析
        tree = etree.HTML(page_text)
        li_list = tree.xpath('//*[@id="main"]/div[3]/ul/li')
        #print(li_list)
        for li in li_list:
            title = li.xpath('./a/b/text()')[0]
            #print(title)
            pic_url = 'https://xxx.xxxxxxx.com'+li.xpath('./a/img/@src')[0]
            #print(pic_url)
            #把信息存储到字典中
            dic = {
                'title': title,
                'pic_url': pic_url
            }
            print(dic)
            car_msg.append(dic)
        return car_msg
car_msg_list = get_car()

看运行的结果

4、阻塞操作（例如下载等等待时间较长的行为）通过特殊函数进行请求

把二进制数据写入dic,并留下返回值

async def get_request(dic):
    async with aiohttp.ClientSession() as sess:
        async with await sess.get(url=dic['pic_url'],headers=headers) as resp:
            car_data = await resp.read()
            dic['car_data'] = car_data
            return dic

5、存储函数

先创建文件夹

dirName = '汽车'
if not os.path.exists(dirName):
    os.mkdir(dirName)

持久化存储

def save_car(t):
    dic = t.result()
    car_data = dic['car_data']
    title = dic['title']
    dir_path = dirName+'/'+title
    with open(dir_path,'wb') as fp:
        fp.write(car_data)
    print(title,'爬取成功')

6、协程任务执行

#用for循环，让任务协程运行
tasks=[]
for dic in car_msg_list:
    c = get_request(dic)
    task = asyncio.ensure_future(c)
    task.add_done_callback(save_car)
    tasks.append(task)
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))

7、成功执行

总共耗时几秒钟

import asyncio

import aiohttp
import requests
import os
from lxml import etree
import time
start = time.time()
dirName = '汽车'
if not os.path.exists(dirName):
    os.mkdir(dirName)
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'
}
def get_car():
    car_msg = [] #存储所有汽车图片信息
    for page in range(2, 3):
        url = 'https://xxx.xxxxxx.com/4kqiche/index_%d.html'%page
        resp = requests.get(url=url, headers=headers)
        resp.encoding = 'gbk' #设置编码
        page_text = resp.text
        #print(page_text)
    #xpath数据解析
        tree = etree.HTML(page_text)
        li_list = tree.xpath('//*[@id="main"]/div[3]/ul/li')
        #print(li_list)
        for li in li_list:
            title = li.xpath('./a/b/text()')[0]
            #print(title)
            pic_url = 'https://xxx.xxxxxxx.com'+li.xpath('./a/img/@src')[0]
            #print(pic_url)
            #把信息存储到字典中
            dic = {
                'title': title,
                'pic_url': pic_url
            }
            #print(dic)
            car_msg.append(dic)
    return car_msg
car_msg_list = get_car()
print(car_msg_list)

#特殊函数
async def get_request(dic):
    async with aiohttp.ClientSession() as sess:
        async with await sess.get(url=dic['pic_url'],headers=headers) as resp:
            car_data = await resp.read()
            dic['car_data'] = car_data
            return dic
#存储函数
def save_car(t):
    dic = t.result()
    car_data = dic['car_data']
    title = dic['title']
    dir_path = dirName+'/'+title+'.jpg'
    with open(dir_path,'wb') as fp:
        fp.write(car_data)
    print(title,'爬取成功)
#用for循环，让任务协程运行
tasks=[]
for dic in car_msg_list:
    c = get_request(dic)
    task = asyncio.ensure_future(c)
    task.add_done_callback(save_car)
    tasks.append(task)
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))
print(time.time()-start)