Python3.6使用requests进行网页(Web)爬虫之批量下载图片

前段时间发现了一个超清图片的网站,然后闲着没事干就写了个爬虫,进行批量下载,拿来当壁纸切换,个人觉得很不错的,然后就开干 . . .

解析网页用的lmxl,这个一般需要手动先安装,中文注释是后来加的,方便大家学习理解,下面是我的源代码,其中很多变量命名我都一直遵循规范来的

#!/usr/bin/env python3.6
# -*-coding = 'utf-8'-*-
# @Time: 2020/2/28  19:39:26
# @File: wallhaven2.py

DOWN_URL = 'https://wallhaven.cc/toplist?page='
SAVE_PATH = 'photo_in_wallhaven/'  # 存放图片的路径
COUNT = 0
Temp_DATA = """
<div class="scrollbox" style="margin-right: -12px; margin-bottom: -12px;">fdsfds
    <img id="wallpaper" src="https://w.wallhaven.cc/full/96/wallhaven-96y88d.jpg" alt="General 1920x1080 nature landscape trees forest rocks mountains mist clouds Monsoon town fall river rock formation Kalampaka Greece" data-wallpaper-id="96y88d" data-wallpaper-width="1920" data-wallpaper-height="1080" crossorigin="anonymous" class="fill-vertical">
</div>
"""

import requests
from lxml import etree
from time import sleep
from random import randint
from os import mkdir
from os.path import exists

# 弄了两个请求头,随机算一个,因为访问的地址比较大,所以也可以多加几个
headers = [{
   
    "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:73.0) Gecko/20100101 Firefox/73.0"
},
    {
   
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3100.0 Safari/537.36'
    }]
if not exists(SAVE_PATH):
    mkdir(SAVE_PATH)


class DownWallhaven(object):
    def __init__(self, number, img_group: list = None):
        self.number = number
        self.img_group = img_group  # 因为返回lxml.etree的是一个列表,后面通过一个for循环对这个图片的下载列表进行依次下载
        self.folder = None
        self.new_img_folder = None
        self.headers