Python爬取表情包
任务:爬取某斗图网站的223页表情包:
一、获取图片URL
- 将url网址拼接成字符串在列表中并返回
url = "https://aidotu.com/search/0-0-0-1.html"
def getURLs():
# 获取所有的url列表
urls = []
for i in range(223):
url = "https://aidotu.com/search/0-0-0-" + str(i+1) + ".html"
urls.append(url)
return urls
二、发送请求获取图片数据并写入文件
import requests # 发送html请求
from fake_useragent import UserAgent # 反爬虫的工具包
from parsel import Selector
# 通过url进行解析并下载保存图片到指定目录
def download(url):
headers = {
# "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36"
"User-Agent": UserAgent().random # 反爬虫生成的随机User-Agent
}
res = requests.get(url=url, headers=headers)
# print(res.status_code)
# res.encoding = "utf-8"
# print(res.text)
# 解析返回值
select = Selector(res.text)
result_list = select.xpath("//div[@class='layui-col-sm3 layui-col-xs6']")
for i in result_list:
# 获取图片
img_url = i.xpath("./a/img/@src").extract_first()
# print("https:" + img_url)
# 获取图片名字
img_title = i.xpath("./a/img/@alt").extract_first()
# 字符串分割
img_name = img_title + "." + img_url.split(".")[-1]
img_data = requests.get("https:" + img_url, headers=headers).content
# # 写入数据到文件中
with open("/Users/yanzhuang/Downloads/img/" + img_name, "wb") as f:
f.write(img_data)
print(img_name + "===== 下载成功!")
三、线程池
- 由于数据量较大,需要使用多线程来提高效率,本次任务采用线程池 ThreadPoolExecutor 来解决。
from concurrent.futures import ThreadPoolExecutor #线程池的包
if __name__ == "__main__":
# 获取url列表
urls = getURLs()
# 创建最大线程为5的线程池
pool = ThreadPoolExecutor(max_workers=8)
# 遍历
for url in urls:
# 将任务提交给线程池
pool.submit(download, url)