爬虫----072xpath解析站长素材

最新推荐文章于 2024-07-07 22:25:52 发布

编程菜鸟Q

最新推荐文章于 2024-07-07 22:25:52 发布

阅读量118

点赞数 2

分类专栏：爬虫小白学习文章标签：爬虫

本文链接：https://blog.csdn.net/m0_74151139/article/details/136966523

版权

爬虫小白学习专栏收录该内容

19 篇文章 0 订阅

订阅专栏

【注】部分网站的图片会进行懒加载，需注意找到正确的xpath路径


#(1)请求对象的定制
#(2)获取网页源码
#(3)下载

#需求  下载前十页的风景照片
#1 https://sc.chinaz.com/tupian/fengjing.html
#2 https://sc.chinaz.com/tupian/fengjing_2.html
#3 https://sc.chinaz.com/tupian/fengjing_3.html


import urllib.request
from lxml import etree

url = 'https://sc.chinaz.com/tupian/fengjing.html'

def create_request(page):
    if(page == 1):
        url = 'https://sc.chinaz.com/tupian/fengjing.html'
    else:
        url = 'https://sc.chinaz.com/tupian/fengjing_'+str(page)+'.html'

    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 SLBrowser/9.0.3.1311 SLBChan/103'
    }

    request = urllib.request.Request(url=url,headers=headers)
    return request

def get_content(request):
    response = urllib.request.urlopen(request)
    content = response.read().decode('utf-8')
    return content
def down_load(content):
    # 下载图片
    # urllib.request.urlretrieve('地址','名字')

    tree = etree.HTML(content)
    #src  //div[@class="container"]//img/@src
    #alt  //div[@class="container"]//img/@alt
    name_list = tree.xpath('//div[@class="container"]//img/@alt')
    #一般网站的图片都会进行懒加载
    # src_list = tree.xpath('//div[@class="container"]//img/@src')
    src_list = tree.xpath('//div[@class="container"]//img/@data-original')

    for i in range(len(name_list)):
        name = name_list[i]
        src = src_list[i]
        url = 'https:' + src

        urllib.request.urlretrieve(url=url,filename='./fengjing/'+name+'.jpg')



if __name__ == '__main__':
    start_page = int(input('起始：'))
    end_page = int(input('结束：'))

    for page in range(start_page,end_page+1):
        # (1)请求对象的定制
        request = create_request(page)
        # (2)获取网页源码
        content = get_content(request)
        # (3)下载
        down_load(content)