FastAPI学习—异步爬虫—day02

from lxml import etree
from urllib.request import urlretrieve
import os
import asyncio

import aiohttp
import aiofiles
import requests


# 异步爬虫，不能使用requests库，因为requests是一个同步的，
# 想要异步发送网络请求，需要使用：aiohttp库，需要安装 pip install aiohttp

# 异步爬虫下载图片也不可以使用urlretrieve，因为它也是同步的
# 异步文件操作，需要用到aiofiles这个库，需要安装 pip install aiofiles

# 哪些地方需要加await
# 如果返回的是一个awaitable对象，就要加await, 例如：协程、Task、Future

# 同步
class TongbuDoutulaSpider(object):
    def __init__(self):
        self.base_url = "https://www.doutula.com/photo/list/?page={}"
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36"
        }

    def parse_page(self, url):
        response = requests.get(url, headers=self.headers)
        html = response.text
        praser = etree.HTML(html)
        emoji_url_list = praser.xpath("//div[contains(@class, page-content)]//a//img/@data-original")
        for emoji_url in emoji_url_list:
            # http://img.doutula.com/production/uploads/image/2020/08/29/20200829662145_CnAErN.jpg
            emoji_name = emoji_url.split("/")[-1]
            emoji_path = os.path.join("emoji", emoji_name)
            urlretrieve(emoji_url, emoji_path)
            print("{}表情下载成功".format(emoji_name))

    def run(self):
        # 爬取10页
        for i in range(1, 11):
            page_url = self.base_url.format(i)
            # 解析页面数据
            self.parse_page(page_url)
            break


class DoutulaSpider(object):
    def __init__(self):
        self.base_url = "https://www.doutula.com/photo/list/?page={}"
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36"
        }


    async def parse_page(self, url):
        # async with：是一个语法
        async with self.session.get(url) as response:
            html = await response.text()
            praser = etree.HTML(html)
            emoji_url_list = praser.xpath("//div[contains(@class, page-content)]//a//img/@data-original")
            for emoji_url in emoji_url_list:
                # http://img.doutula.com/production/uploads/image/2020/08/29/20200829662145_CnAErN.jpg
                emoji_name = emoji_url.split("/")[-1]
                emoji_path = os.path.join("emoji", emoji_name)
                async with self.session.get(emoji_url) as emoji_response:
                    # open("abc.jpg") 不能使用open打开文件写入，这样做是同步的，会导致异步代码不会异步执行
                    async with aiofiles.open(emoji_path, "wb") as f:
                        data = await emoji_response.read()
                        await f.write(data)
                print("{}表情下载成功".format(emoji_name))

    async def run(self):
        self.session = aiohttp.ClientSession(headers=self.headers)
        # 爬取10页
        for i in range(1, 11):
            page_url = self.base_url.format(i)
            # 解析页面数据
            await self.parse_page(page_url)
            break
        # 最后关闭session
        await self.session.close()


if __name__ == '__main__':
    # 同步启动方式
    # spider = TongbuDoutulaSpider()
    # spider.run()
    # 异步启动方式
    spider = DoutulaSpider()
    asyncio.run(spider.run())