由于Python拥有强大且丰富的类库,语法简单,效率高而被广泛运用于网络爬虫,很多人都是通过爬虫认识Python。
因为小编最近正在做目标识别相关的项目,所以需要大量的训练样本集,从网页上一个个下载又太慢,所以就想到编写一个程序实现网页图片的批量下载。感兴趣的朋友可以将代码复制到本地运行试试,小编会在代码中写入一部分注释方便大家理解,如果有问题可以直接给我留言评论。
本篇文章只是分享代码,所以不会详细介绍爬虫的原理和结构,不过通过代码的注释也可以让新手轻松入门。
(一)获取请求头
首先你要知道爬虫就是模拟人去浏览网页,并通过代码实现批量获取信息的手段。所以我们在使用爬虫时,要先获取网页的请求头,你可以理解为你只有出示饭票,阿姨才会给你打饭。
1.打开你要爬取的网站,我这里是百度图片网页。然后点击键盘的“F12”进入控制台,接着按顺序点击“网络”(英文版的是Network),然后点击“XHR”,再点击左侧响应的名称,获取User-Agent、Host、Cookie信息。将这些信息制作成请求头的格式,如下述代码:
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome'
'/112.0.0.0 Safari/537.36 Edg/112.0.1722.58',
'Host': 'image.baidu.com',
'Cookie': 'BIDUPSID=6096EFD12C571F1D6231034147921FB8; PSTM=1682383713; BAIDUID=6096EFD12C571F1D5581B126'
'79EA8E7D:FG=1; BD_UPN=12314753; BDORZ=FFFB88E999055A3F8A630C64834BD6D0; delPer=0; BD_CK_SAM=1'
'; PSINO=5; BDRCVFR[dG2JNJb_ajR]=mk3SLVN4HKm; BAIDUID_BFESS=6096EFD12C571F1D5581B12679EA8E7D:FG'
'=1; userFrom=null; BDRCVFR[-pGxjrCMryR]=mk3SLVN4HKm; BDRCVFR[tox4WRQ4-Km]=mk3SLVN4HKm; BDRCVFR'
'[A24tJn4Wkd_]=mk3SLVN4HKm; shifen[598151295075_76725]=1682411441; BCLID=10450312018116497963; B'
'CLID_BFESS=10450312018116497963; BDSFRCVID=z9_OJeC62lsu0DJfOkenUsu36Pzw6K3TH6bHQI-qy-1kcJagoI4a'
'EG0PUx8g0KuMDFkVogKK0eOTHktF_2uxOjjg8UtVJeC6EG0Ptf8g0f5; BDSFRCVID_BFESS=z9_OJeC62lsu0DJfOkenUsu'
'36Pzw6K3TH6bHQI-qy-1kcJagoI4aEG0PUx8g0KuMDFkVogKK0eOTHktF_2uxOjjg8UtVJeC6EG0Ptf8g0f5; H_BDCLCKID'
'_SF=tbFqoK8bJKL3qJTph47hqR-8MxrK2JT3KC_X3b7Ef-FB_p7_bf--D4Ay5H3RBt592KTX-4OatKQmJ40CyTbxy5KVybQA'
'eRo8HR6W3hcq5b7zMbjHQT3m3JvbbN3i-xrR3D3pWb3cWKJq8UbSMnOPBTD02-nBat-OQ6npaJ5nJq5nhMJmb67JD-50exbH5'
'5uHtb-e3H; H_BDCLCKID_SF_BFESS=tbFqoK8bJKL3qJTph47hqR-8MxrK2JT3KC_X3b7Ef-FB_p7_bf--D4Ay5H3RBt592K'
'TX-4OatKQmJ40CyTbxy5KVybQAeRo8HR6W3hcq5b7zMbjHQT3m3JvbbN3i-xrR3D3pWb3cWKJq8UbSMnOPBTD02-nBat-OQ6n'
'paJ5nJq5nhMJmb67JD-50exbH55uHtb-e3H; BDRCVFR[Q5XHKaSBNfR]=mk3SLVN4HKm; BA_HECTOR=0005ah85ah01akak'
'ah218kdl1i4f4661m; ZFY=2wMwDt78vksPrYmFMrRHpQ0FDKAW:BwWKHieg1S7DwzI:C; Hm_lvt_aec699bb6442ba076c89'
'81c6dc490771=1682412342; Hm_lpvt_aec699bb6442ba076c8981c6dc490771=1682412342; COOKIE_SESSION=297_0'
'_8_8_21_12_1_0_8_6_0_1_4899_0_356_0_1682412370_0_1682412014%7C9%230_0_1682412014%7C1; ZD_ENTRY=bai'
'du; ab_sr=1.0.1_MTQ3MDNkZDUwMWVlMDBiOTUwOTNmZTIyZWYxOTI5MjA5OGY2ZDE3MjZhODhkZTNkMjg0YjY2MDMwYjhiZDI'
'2YTZhY2Y3MjRkZTQ0ZDVlNjJlNzQyZTg1NTYwMmU4MDg0MWVlOGYxYjljYzAxZmEyZTc1NDc2NTBjYjczMjBhZmY1MTcyYWQyYT'
'g0YTE1Mzc2NmUxODA3ZWU2YmE5MDM5MQ==; __bid_n=187b632a23dacdbd374207; H_PS_645EC=1b0eaVb4%2FdPDyC6op6N'
'C0mbno0FhzDP1g9C0LK2F9fx137fXB7h1o3RqkSjaSbV12NqWTbs; BD_HOME=1; H_PS_PSSID=38516_36554_38469_38368_'
'38468_38485_37928_37709_38356_26350_38546'
}
(二)获取图片的下载链接
这里没什么好说的,就直接上代码了(函数中的key是你要搜索的内容,后面的完整代码可以实现交互式输入,也可以手动输入)。
def Get_image_url(headers, key):
print("正在获取图片下载链接......")
url = 'https://image.baidu.com/search/index?tn=baiduimage&ie=utf-8&word={}'.format('%s' % key)
request = requests.get(url, headers=headers, timeout=10) # timeout实现网页未返回值的情况
# 获取网页的代码
image_urls = re.findall('"objURL":"(.*?)",', request.text, re.S)
# 获取图片的下载链接,列表形式
if not image_urls:
print("Error:图片下载链接获取失败!")
else:
return image_urls
注:这里使用的re正则解析的网页,还有很多种方式,感兴趣的朋友可以自己去了解。嫌费事就直接用我的代码就行。
(三)保存图片至本地
def Write_image(image_urls, num):
for i in range(0, num):
image_data = requests.get(image_urls[i])
# 获取下载链接中的图片信息
print("正在下载第%s张图片:" % (i + 1))
image_path = "G:/try/%s.jpg" % (i + 1)
with open(image_path, 'wb') as fp:
fp.write(image_data.content)
# 写入图片信息
这段代码中的num是可以自己选择需要下载的图片数量,可以直接改,也可以键盘输入(看后续完整代码)
(四)完整代码
# -*- coding: utf-8 -*-
"""
@Time : 2023/4/25 9:24
@Auth : RS迷途小书童
@File :Get_image_online.py
@IDE :PyCharm
"""
import requests
import re
def Get_image_url(headers, key):
print("正在获取图片下载链接......")
url = 'https://image.baidu.com/search/index?tn=baiduimage&ie=utf-8&word={}'.format('%s' % key)
request = requests.get(url, headers=headers, timeout=10) # timeout实现网页未返回值的情况
# 获取网页的代码
image_urls = re.findall('"objURL":"(.*?)",', request.text, re.S)
# 获取图片的下载链接,列表形式
if not image_urls:
print("Error:图片下载链接获取失败!")
else:
return image_urls
def Write_image(image_urls, num):
for i in range(0, num):
image_data = requests.get(image_urls[i])
# 获取下载链接中的图片信息
print("正在下载第%s张图片:" % (i + 1))
image_path = "G:/try/%s.jpg" % (i + 1)
# 保存图片的路径,可以自己修改
with open(image_path, 'wb') as fp:
fp.write(image_data.content)
# 写入图片信息
if __name__ == '__main__':
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome'
'/112.0.0.0 Safari/537.36 Edg/112.0.1722.58',
'Host': 'image.baidu.com',
'Cookie': 'BIDUPSID=6096EFD12C571F1D6231034147921FB8; PSTM=1682383713; BAIDUID=6096EFD12C571F1D5581B126'
'79EA8E7D:FG=1; BD_UPN=12314753; BDORZ=FFFB88E999055A3F8A630C64834BD6D0; delPer=0; BD_CK_SAM=1'
'; PSINO=5; BDRCVFR[dG2JNJb_ajR]=mk3SLVN4HKm; BAIDUID_BFESS=6096EFD12C571F1D5581B12679EA8E7D:FG'
'=1; userFrom=null; BDRCVFR[-pGxjrCMryR]=mk3SLVN4HKm; BDRCVFR[tox4WRQ4-Km]=mk3SLVN4HKm; BDRCVFR'
'[A24tJn4Wkd_]=mk3SLVN4HKm; shifen[598151295075_76725]=1682411441; BCLID=10450312018116497963; B'
'CLID_BFESS=10450312018116497963; BDSFRCVID=z9_OJeC62lsu0DJfOkenUsu36Pzw6K3TH6bHQI-qy-1kcJagoI4a'
'EG0PUx8g0KuMDFkVogKK0eOTHktF_2uxOjjg8UtVJeC6EG0Ptf8g0f5; BDSFRCVID_BFESS=z9_OJeC62lsu0DJfOkenUsu'
'36Pzw6K3TH6bHQI-qy-1kcJagoI4aEG0PUx8g0KuMDFkVogKK0eOTHktF_2uxOjjg8UtVJeC6EG0Ptf8g0f5; H_BDCLCKID'
'_SF=tbFqoK8bJKL3qJTph47hqR-8MxrK2JT3KC_X3b7Ef-FB_p7_bf--D4Ay5H3RBt592KTX-4OatKQmJ40CyTbxy5KVybQA'
'eRo8HR6W3hcq5b7zMbjHQT3m3JvbbN3i-xrR3D3pWb3cWKJq8UbSMnOPBTD02-nBat-OQ6npaJ5nJq5nhMJmb67JD-50exbH5'
'5uHtb-e3H; H_BDCLCKID_SF_BFESS=tbFqoK8bJKL3qJTph47hqR-8MxrK2JT3KC_X3b7Ef-FB_p7_bf--D4Ay5H3RBt592K'
'TX-4OatKQmJ40CyTbxy5KVybQAeRo8HR6W3hcq5b7zMbjHQT3m3JvbbN3i-xrR3D3pWb3cWKJq8UbSMnOPBTD02-nBat-OQ6n'
'paJ5nJq5nhMJmb67JD-50exbH55uHtb-e3H; BDRCVFR[Q5XHKaSBNfR]=mk3SLVN4HKm; BA_HECTOR=0005ah85ah01akak'
'ah218kdl1i4f4661m; ZFY=2wMwDt78vksPrYmFMrRHpQ0FDKAW:BwWKHieg1S7DwzI:C; Hm_lvt_aec699bb6442ba076c89'
'81c6dc490771=1682412342; Hm_lpvt_aec699bb6442ba076c8981c6dc490771=1682412342; COOKIE_SESSION=297_0'
'_8_8_21_12_1_0_8_6_0_1_4899_0_356_0_1682412370_0_1682412014%7C9%230_0_1682412014%7C1; ZD_ENTRY=bai'
'du; ab_sr=1.0.1_MTQ3MDNkZDUwMWVlMDBiOTUwOTNmZTIyZWYxOTI5MjA5OGY2ZDE3MjZhODhkZTNkMjg0YjY2MDMwYjhiZDI'
'2YTZhY2Y3MjRkZTQ0ZDVlNjJlNzQyZTg1NTYwMmU4MDg0MWVlOGYxYjljYzAxZmEyZTc1NDc2NTBjYjczMjBhZmY1MTcyYWQyYT'
'g0YTE1Mzc2NmUxODA3ZWU2YmE5MDM5MQ==; __bid_n=187b632a23dacdbd374207; H_PS_645EC=1b0eaVb4%2FdPDyC6op6N'
'C0mbno0FhzDP1g9C0LK2F9fx137fXB7h1o3RqkSjaSbV12NqWTbs; BD_HOME=1; H_PS_PSSID=38516_36554_38469_38368_'
'38468_38485_37928_37709_38356_26350_38546'
}
key = str(input("请输入你想要获取图片的关键字:"))
num = int(input("请输入你想获取图片的数量:"))
image_urls = Get_image_url(headers, key)
if image_urls:
Write_image(image_urls, num)
else:
print("程序结束!")
效果图:
代码中的image_path = "G:/try/%s.jpg" % (i + 1),可以自己修改路径。搜索的关键字和图片数量可以通过键盘输入。
20240805:更新代码库时,发现当时优化了一版,正好过来修改一下:
# -*- coding: utf-8 -*-
"""
@Time : 2023/4/25 9:24
@Auth : RS迷途小书童
@File :Get_image_online.py
@IDE :PyCharm
@Purpose: 批量获取网页上的图片制作数据集
"""
import requests
import re
from urllib import error
import urllib3
import time
urllib3.disable_warnings()
def Get_image_url(headers, key, num):
"""
:param headers: 请求头
:param key: 输入搜索的关键字
:param num: 输入图片数量
:return: 列表形式的下载链接
"""
page_all = 0
page_image = 0
page_sum = int(num/60)+1
image_urls = []
while page_image <= page_sum:
print("正在获取第%s页图片下载链接......" % (page_image+1))
url = 'https://image.baidu.com/search/flip?tn=baiduimage&ie=utf-8&word=' + str(key) + '&pn=%s' % page_all
request = requests.get(url, headers=headers, timeout=10, allow_redirects=False, verify=False)
# timeout实现网页未返回值的情况 # 获取网页的代码
image_urls_page = re.findall('"objURL":"(.*?)",', request.text, re.S)
if not image_urls_page:
print("Error:第%s页图片下载链接获取失败!" % (page_image + 1) + "错误链接:" + url)
num = page_image * 60 - 40
print("当前共有%s张图片" % num)
break
else:
image_urls += image_urls_page
# 获取图片的下载链接,列表形式
page_image += 1
page_all += 60
return image_urls
def Write_image(image_urls, num, path, header):
"""
:param image_urls: 以列表的形式输入下载链接
:param num: 输入图片数量
:param path: 输入存储路径
:param header: 请求头文件
:return: none,保存本地
"""
print("开始下载目标图片")
num_image = 0
num_all = 0
while num_image < num:
try:
if image_urls[num_image] is not None:
image_data = requests.get(image_urls[num_all], headers=header, verify=False, timeout=4)
# 获取下载链接中的图片信息
print("正在下载第%s张图片" % (num_image + 1) + ",下载链接:" + image_urls[num_all])
image_path = path + str(num_image + 1) + ".jpg"
with open(image_path, 'wb') as fp:
fp.write(image_data.content)
# 写入图片信息
num_image += 1
num_all += 1
else:
num_all += 1
continue
except BaseException:
print('Error:当前图片下载失败,下载链接:%s' % image_urls[num_all])
num_all += 1
continue
if __name__ == '__main__':
headers1 = {
'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
'Connection': 'keep-alive',
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0',
'Upgrade-Insecure-Requests': '1'
}
key1 = str(input("请输入你想要获取图片的关键字:"))
num1 = int(input("请输入你想获取图片的数量:"))
path1 = str(input("请输入存放图片的路径:"))
start = time.perf_counter()
image_urls1 = Get_image_url(headers1, key1, num1)
if image_urls1:
Write_image(image_urls1, num1, path1, headers1)
else:
print("Error:程序结束!")
end = time.perf_counter()
print("\n")
print("图片全部下载完成......")
print('共消耗: %s 秒' % (end - start))
本次批量下载网页图片的代码分享就到这,仅供大家学习参考,有问题可以留言!