xpath的使用

基本使用

# _*_ coding : utf-8 _*_
# @Time : 2023/1/28 14:48
# @Author : 李阶熊
# @File : 解析xpath的基本使用
# @Project : pythonProject
from lxml import etree

# xpath 解析
# （1） 本地文件                                                              etree.parse
# （2） 服务器响应的数据  response.read().decode('utf-8')                       etree.HTML()

# xpath解析本地文件
tree = etree.parse('../html/xpath_test.html')
print(tree)

# tree.xpath('xpath路径')

# 查找ul下面的li
li_list = tree.xpath('//ul/li')

print(li_list)
print(len(li_list))


# 查找所有有id的属性的li标签
li_list = tree.xpath('//ul/li[@id]/text()')
print(li_list)
print(len(li_list))


# 找到id为l1的li标签
li_list = tree.xpath('//ul/li[@id="l1"]/text()')
print(li_list)
print(len(li_list))

# 查找id为l1的li标签的class的属性值
li = tree.xpath('//ul/li[@id="l1"]/@class')
print(li)
print(len(li))

# 查询id中包含l的li标签
li_list = tree.xpath('//ul/li[contains(@id,"l")]/text()')
print(li_list)
print(len(li_list))

# 查询id的值以l开头的li标签
li_list = tree.xpath('//ul/li[starts-with(@id,"l")]/text()')
print(li_list)
print(len(li_list))

# 查询id为l1喝class为a3的
li_list = tree.xpath('//ul/li[@id="a3" and @class="a3"]/text()')
print(li_list)
print(len(li_list))

li_list = tree.xpath('//ul/li[@id="a3"]/text()' | '//ul/li[@id="l1"]/text()')
print(li_list)
print(len(li_list))

使用xpath获取百度一下

# _*_ coding : utf-8 _*_
# @Time : 2023/1/28 15:50
# @Author : 李阶熊
# @File : 获取百度网站的百度一下
# @Project : pythonProject
# 1、获取网页原阿门
# 2、解析  解析的服务器响应的文件  etree.HTML
# 3、打印

import urllib.request

url = "https://www.baidu.com/"

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36'
}

request = urllib.request.Request(url=url, headers=headers)

# handler = urllib.request.HTTPHandler
#
# opener = urllib.request.build_opener(handler)
#
# response = opener.open(request)

response = urllib.request.urlopen(request)

# 获取网页源码
content = response.read().decode('utf-8')

# 解析网页源码  来获取我们想要的数据
from lxml import etree

# 解析服务器响应的文件
tree = etree.HTML(content)

# 获取想要的数据  xpath的返回值是一个列表类型的数据
result = tree.xpath('//input[@id="su"]/@value')

print(result[0])

站长素材图片获取和下载

# _*_ coding : utf-8 _*_
# @Time : 2023/1/28 16:12
# @Author : 李阶熊
# @File : 站长素材
# @Project : pythonProject
# （1） 请求对象的定制
# （2） 获取网页的源码
# （3） 下载
# https://sc.chinaz.com/tu/fengjing.html
# https://sc.chinaz.com/tu/fengjing-3-0-0.html
import urllib.request
from lxml import etree
import urllib.error

last = '.html'
url = 'https://sc.chinaz.com/tu/fengjing'

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36'
}


def Schedule(a, b, c):
    per = 100.0 * a * b / c
    if per > 100:
        per = 100
        print('完成！')
    print('%.2f%%' % per)


def create_request(p):
    if p == 1:
        current_url = url + last
    else:
        current_url = url + '-' + str(p) + '-0-0' + last
        print(current_url)
    request1 = urllib.request.Request(url=current_url, headers=headers)
    return request1


def get_content(request2):
    response = urllib.request.urlopen(request2)
    content1 = response.read().decode('utf-8')
    return content1


def down_load(content1):
    # 下载图片
    # urllib.request.urlretrieve('图片地址', '文件名称')
    tree = etree.HTML(content1)
    name_list = tree.xpath('//div[@id="ulcontent"]//a/img/@alt')

    # 一般设计到图片的网站，会有懒加载
    src_list = tree.xpath('//div[@id="ulcontent"]//a/img/@data-src')
    for i in range(len(name_list)):
        try:

            name = name_list[i] + '.jpg'
            src = src_list[i]
            url_1 = 'http:' + src
            url_1 = url_1.replace('\\', '/')
            print(url_1)
            urllib.request.urlretrieve(url_1, './images/' + name, Schedule)
        except urllib.error.HTTPError:
            print('图片路径有错误哦')
            continue


if __name__ == '__main__':
    start_page = int(input('请输入起始页码'))
    end_page = int(input('请输入结束页码'))
    for page in range(start_page, end_page + 1):
        # (1) 请求对象的定制
        request = create_request(page)
        # (2)获取网页的源码
        content = get_content(request)
        # （3） 下载
        down_load(content)