一、库的安装
首先我们要先安装我们所需要的库,否则我们所写的代码无法运行
pip install requests
二、爬取代码
import re # 正则表达式
import requests # 爬虫用的包
from urllib.request import urlretrieve # 本文用来下载图片
import os # 标准库,本文用来新建文件夹
class Xiaozhan(object):
def __init__(self):
self.base_url = "https://www.duitang.com/search/?kw=肖战&type=feed#!s-p{}"
self.header = {"user-agent":
"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Mobile Safari/537.36"
}
def Geturl(self):
all_url = []
for item in range(3):
all_url.append(self.base_url.format(item))
print(len(all_url))
return all_url # 返回全部网址
def Respons(self, all_url):
text_list = []
for url in all_url:
r = requests.get(url, headers=self.header)
text_list.append(r.text)
return text_list # 返回网址的源代码
def clear(self, respons):
for i in respons:
qqq = r'<img data-rootid=".*?" alt="肖战" data-iid="" src="(.*?)" height=".*?"/>'
pi = re.findall(qqq, i)
return pi # 返回图片的网址
def save(self, url_list):
i = 0
# print(url_list)
if not os.path.exists("肖战图片"):
os.mkdir("肖战图片") # 判断有没有该文件夹,如果没有就创建改文件夹
for url in url_list:
i = i + 1
print(url)
file_path = "./肖战图片" + "/" + str(i) + ".jpg" # 文件路径
urlretrieve(url, file_path) # 下载图片
print("第%d张图片下载成功" % i)
def run(self):
# 1.获取网址的列表
titile_url = self.Geturl()
# print(titile_url)
# 2.发送请求,获取响应
respons = self.Respons(titile_url)
# print(respons)
# 3.通过正则表达式清洗数据
picture = self.clear(respons)
print(picture)
# 4.保存数据
download = self.save(picture)
# print(download)
if __name__ == "__main__":
xiaozhan = Xiaozhan()
xiaozhan.run()