Python爬虫——批量爬取站长素材

import urllib.request
from lxml import etree

def create_request(page):
    if (page == 1):
        url = 'https://sc.chinaz.com/tupian/fengjingtupian.html'
    else:
        url = 'https://sc.chinaz.com/tupian/fengjingtupian_' + str(page) + '.html'

    headers = {
        'User-Agent': ''
    }

    request = urllib.request.Request(url, headers=headers)
    return request

def get_content(request):
    response = urllib.request.urlopen(request)
    content = response.read().decode('utf-8')
    return content

def down_load(content):
    # 下载图片
    tree = etree.HTML(content)

    name_list = tree.xpath('//img[@class="lazy"]/@alt')
    src_list = tree.xpath('//img[@class="lazy"]/@data-original')

    for i in range(len(name_list)):
        name = name_list[i]
        src = src_list[i]

        src_url = 'https:' + src
        urllib.request.urlretrieve(src_url, 'files/站长素材/' + name + '.jpg')


if __name__ == '__main__':
    start_page = 1
    end_page = 10
    for page in range(start_page, end_page+1):
        # 请求对象的定制
        request = create_request(page)
        # 获取网页源码
        content = get_content(request)
        # 下载
        down_load(content)