获取网页相关图片

最新推荐文章于 2024-05-21 20:55:00 发布

Doctor_Wu_

最新推荐文章于 2024-05-21 20:55:00 发布

阅读量132

点赞数

分类专栏：开发: 脚本及小程序使用深度学习: 数据集介绍文章标签：爬虫

本文链接：https://blog.csdn.net/Boys_Wu/article/details/125855858

版权

深度学习: 数据集介绍同时被 2 个专栏收录

7 篇文章 0 订阅

订阅专栏

开发: 脚本及小程序使用

5 篇文章 0 订阅

订阅专栏

获取网页上相关图片.

# -*- coding: UTF-8 -*-"""
import requests
import tqdm


def configs(search, page, number):
    """

    :param search:
    :param page:
    :param number:
    :return:
    """
    url = 'https://image.baidu.com/search/acjson'
    # url = 'https://image.baidu.com/search/index?tn=baiduimage&ipn=r&ct=201326592&cl=2&lm=-1&st=-1&fm=result&fr=&sf=1&fmq=1657872367293_R&pv=&ic=&nc=1&z=0&hd=&latest=&copyright=&se=1&showtab=0&fb=0&width=0&height=0&face=0&istype=2&dyTabStr=MCwzLDgsNiwxLDQsNSw3LDIsOQ%3D%3D&ie=utf-8&sid=&word=%E5%87%8F%E9%80%9F%E5%B8%A6+%E5%A4%9C%E6%99%9A'
    params = {
        "tn": "resultjson_com",
        "logid": "11555092689241190059",
        "ipn": "rj",
        "ct": "201326592",
        "is": "",
        "fp": "result",
        "queryWord": search,
        "cl": "2",
        "lm": "-1",
        "ie": "utf-8",
        "oe": "utf-8",
        "adpicid": "",
        "st": "-1",
        "z": "",
        "ic": "0",
        "hd": "",
        "latest": "",
        "copyright": "",
        "word": search,
        "s": "",
        "se": "",
        "tab": "",
        "width": "",
        "height": "",
        "face": "0",
        "istype": "2",
        "qc": "",
        "nc": "1",
        "fr": "",
        "expermode": "",
        "force": "",
        "pn": str(60 * page),
        "rn": number,
        "gsm": "1e",
        "1617626956685": ""
    }
    return url, params


def loadpic(number, page):
    base = 1000                #修改文件起始编号
    """

    :param number:
    :param page:
    :return:
    """
    while (True):
        if number == 0:
            break
        url, params = configs(search, page, number)
        result = requests.get(url, headers=header, params=params).json()
        url_list = []
        for data in result['data'][:-1]:
            url_list.append(data['thumbURL'])
        for i in range(len(url_list)):
            getImg(url_list[i], 60 * page + i + base, path)
            bar.update(1)
            number -= 1
            if number == 0:
                break
        page += 1
    print("\nfinish!")


def getImg(url, idx, path):
    """

    :param url:
    :param idx:
    :param path:
    :return:
    """
    img = requests.get(url, headers=header)
    file = open(path + 'mark' + str(idx + 1) + '.jpg', 'wb')  #文件名称
    file.write(img.content)
    file.close()


if __name__ == '__main__':
    search = "交通红绿灯 白天"  #  搜索关键字
    number = 1000   #爬取数量
    path = '/home/wsy/data/wuxi/06_dataset_collection/dataset_traffic_light/data1/'  # 保存爬取的数据的文件夹
    header = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36'}

    bar = tqdm.tqdm(total=number)
    page = 0
    loadpic(number, page)