python——爬百度图片
爬虫的本质:获取网页,解析网页,获取出自己想要的数据,对数据进行保存
import urllib.request
import urllib.parse
import re
import os
class BaiduPicture(object):
# User-Agent是必须的,伪装成浏览器进行访问
# Referer是必须的,否则会返回403错误
header = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',
"referer": "https://image.baidu.com"}
# 获取图片的网址
url = "https://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&is=&fp=result&queryWord=" \
"{word}&cl=2&lm=-1&ie=utf-8&oe=utf-8&adpicid=&st=-1&z=&ic=0&word={word}&s=&se=&tab=&width=&height=&" \
"face=0&istype=2&qc=&nc=1&fr=&cg=girl&pn={pageNum}&rn=30&gsm=1e00000000001e&1490169411926="
# 想要搜索的图片关键字
keyword = '王力宏'
# 对中文进行转码
keyword = urllib.parse.quote(keyword, 'utf-8')
# n用于去控制下载的图片的数量
n = 0
# j作为数字被加在图片的名称后面
j = 0
error = 0
while (n < 300):
n += 30
# format() 用于对字符串的格式化
new_url = url.format(word=keyword, pageNum=str(n))
rep = urllib.request.Request(new_url, headers=header)
rep = urllib.request.urlopen(rep)
try:
html = rep.read().decode('utf-8')
except:
print('wrong')
error = 1
print("page" + str(n))
if (error == 1):
continue
# compile() 函数,将一个字符串编译为字节代码
p = re.compile("thumbURL.*?\.jpg")
s = p.findall(html)
if os.path.isdir("Users/donghongyu/Downloads/picture") != True:
os.makedirs(r"Users/donghongyu/Downloads/picture")
with open("testPic1.txt", "w") as f:
for i in s:
i = i.replace("thumbURL\":\"", "")
print(i)
f.write(i)
f.write("\n")
urllib.request.urlretrieve(i, "/Users/donghongyu/Downloads/picture/pic{num}.jpg".format(num=j))
j += 1
if __name__ == '__main__':
BaiduPicture()