pixabay网站图片下载

最新推荐文章于 2024-09-16 07:18:09 发布

funcdefmain

最新推荐文章于 2024-09-16 07:18:09 发布

阅读量1k

点赞数

分类专栏：爬虫文章标签：爬虫 python

本文链接：https://blog.csdn.net/qq_45444679/article/details/132670083

版权

爬虫专栏收录该内容

12 篇文章 1 订阅

订阅专栏

pixabay 网站图片下载

准备

目标网站：https://pixabay.com/

下载这个网站的图片需要先登录，可以用临时邮箱去注册一个账号。
使用浏览器插件Cookie-Editor导出json格式的cookies。
删除cookies 数据的 "sameSite": null

这个网站api请求会返回 Cloudflare 验证页面，它提示启用 JavaScript 和 cookies 以继续浏览，这是 Cloudflare 的安全机制，用于保护网站免受恶意访问。

没有办法跳过，于是使用pyppeteer模拟浏览器操作。

编码

import asyncio
import json

from pyppeteer import launch

from spider import Spider


#  https://cdn.pixabay.com/photo/2023/08/19/05/50/wolf-8199785_1280.png
#  https://pixabay.com/images/download/wolf-8199785.png
async def parse_one_page(url):
    options = {
        "executablePath": r"C:\Program Files\Google\Chrome\Application\chrome.exe",
        'userDataDir': './temp',
        "headless": False,
        "defaultViewport": {"width": 1920, "height": 1080},
        "autoClose": True,
        "dumpio": True,
        "args": [
            '--disable-infobars',
            '--no-sandbox',
            '--start-maximized',
        ],
    }
    browser = await launch(**options)
    page = await browser.newPage()
    # cookie导入前去掉 "sameSite": null
    with open('cookies.json', 'r', encoding='utf8') as f:
        cookies_data = json.load(f)

    for cookie in cookies_data:
        await page.setCookie(cookie)
    try:
        await page.goto(url, options={'timeout': 30000})
    except:
        pass
    ###############################################
    temp = None
    while True:
        y = await page.evaluate('document.documentElement.scrollTop')
        if y == temp:
            break
        else:
            print('滚动条刷新')
            temp = y
            await page.evaluate('window.scrollBy(0, document.documentElement.clientHeight)')
            await asyncio.sleep(2)
    ##############################################
    src_list = await page.xpath('//a[@class="link--WHWzm"]/img/@src')
    download_url_list = []
    for src in src_list:
        img_url = await (await src.getProperty('textContent')).jsonValue()
        print(img_url)
        pid = img_url.split('/')[-1].split('_')[0]
        download_url = f"https://pixabay.com/images/download/{pid}.jpg"
        download_url_list.append(download_url)
    print(download_url_list)
    return download_url_list


async def parse_download_url(url, save_path):
    options = {
        # "executablePath": r"C:\Program Files\Google\Chrome\Application\chrome.exe",
        'userDataDir': './temp',
        "headless": False,
        "defaultViewport": {"width": 1920, "height": 1080},
        "autoClose": True,
        "dumpio": True,
        "args": [
            '--disable-infobars',
            '--no-sandbox',
            '--start-maximized',
        ],
    }
    browser = await launch(**options)
    page = await browser.newPage()
    with open('cookies.json', 'r', encoding='utf8') as f:
        cookies_data = json.load(f)

    for cookie in cookies_data:
        await page.setCookie(cookie)

    response = await page.goto(url)
    print(response.url)
    await browser.close()
    # 下载
    pic_name = url.split('/')[-1]
    # os.makedirs(save_path, exist_ok=True)
    save_name = f'{save_path}/{pic_name}'
    return response.url, save_name



if __name__ == '__main__':
    keyword = 'face'
    for page in range(1, 406):
        print(f'下载第{page}页数据')
        page_url = f'https://pixabay.com/photos/search/{keyword}/?pagi={page}'
        loop1 = asyncio.get_event_loop()
        download_url_list = loop1.run_until_complete(parse_one_page(page_url))
        task_list = []
        for download_url in download_url_list:
            loop2 = asyncio.get_event_loop()
            one_task = loop2.run_until_complete(parse_download_url(download_url, f'./{keyword}'))
            task_list.append(one_task)
        spider = Spider(
            task_list=task_list,
            thread_num=3
        )
        spider.run()