Python爬虫爬取多张图片代码6

EYYLTV

于 2024-09-14 00:18:17 发布

阅读量523

点赞数 13

文章标签： python

本文链接：https://blog.csdn.net/qq_32257509/article/details/142193589

版权

python单线程爬取高清4k壁纸图片

import os
import re
import time
import requests
import parsel

url的规律

https://pic.netbian.com/new/index.html

https://pic.netbian.com/new/index_1.html

https://pic.netbian.com/new/index_2.html

…

start_time = time.time()
for page in range(2,10):
print(f’--------- 正在爬取第{page}的内容 ----------’)
url = f’https://pic.netbian.com/4kmeinv/index_{page}.html’ # 请求地址
# 模拟伪装
headers = {
‘user-agent’: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.139 Safari/537.36’}
# response = requests.get(url=url,headers=headers)
# response.encoding = response.apparent_encoding # 自动转码, 防止中文乱码
# print(response.text)
html_data = requests.get(url=url, headers=headers).content.decode(‘gbk’)
# print(html_data)
selector = parsel.Selector(html_data)
lis = selector.css(’.slist li’)
for li in lis:
# href = li.css(‘a::attr(href)’).get()
title = li.css(‘b::text’).get()
if title:
href = ‘https://pic.netbian.com’ + li.css(‘a::attr(href)’).get()
response = requests.get(url=href, headers=headers)
# print(href, title)
# 这里只是获取页面
# img_content = requests.get(url=href, headers=headers).content
# 不可行, 都是同一张图 https://pic.netbian.com/uploads/allimg/230813/221347-16919360273e05.jpg
href = ‘https://pic.netbian.com’ + li.css(‘a::attr(href)’).get()
response1 = requests.get(url=href, headers=headers).content.decode(‘gbk’)
selector1 = parsel.Selector(response1)
# 若要标题乱码，此处可不解码
# response1 = requests.get(url=href, headers=headers)
# selector1 = parsel.Selector(response1.text)

        # img_url = selector1.css('.slist li img::attr(src)').get()
        # 这一步错了, 要去href页面找img_url, 这是在原来的url页面找了
        img_url = 'https://pic.netbian.com' + selector1.css('.photo .photo-pic img::attr(src)').get()
        img_content = requests.get(url=img_url, headers=headers).content
        # 顺便更新一下title, 因为原来的是半截的, 不全
        title1 = selector1.css('.photo .photo-pic img::attr(title)').get()
        modified_title = re.split(r'\d', title1, 1)[0].strip()
        with open(modified_title + '.jpg', mode='wb') as f:
            f.write(img_content)
            # print(href, title)
            print('正在保存：', modified_title, img_url)

stop_time = time.time()
print(f’耗时：{int(stop_time)-int(start_time)}秒’)