一、xpath的基本使用
(1)导入lxml.etree
from lxml import etree
(2)etree.parse() 解析本地文件
tree = etree.parse('xpath的基本使用.html')
print(tree)
(3)etree.HTML() 服务器响应文件
html_tree = etree.HTML(content)
(4)html_tree.xpath(xpath路径)
二、xpath语法
1、路径查找
//:查找所有子孙节点,不考虑层级关系
/:查找子节点
2、谓词查找
//div[@id]
//div[@id="maincontent"]
3、属性查找
//@class
4、模糊查找
//div[contains(@id,"he")]
//div[starts-with(@id,"he")]
5、内容查找
//div/ul/text()
6、逻辑运算
//div[@id="北京" and @class="A1"]
//title | //price
e.g.
from lxml import etree
#xpath解释
#(1)本地文件
#(2)服务器响应的数据 response.read().decode('utf-8')
#xpath解析本地文件
tree = etree.parse('xpath的基本使用.html')
#路径查找例子
# 查找ul下面的li
# li_list = tree.xpath('//body//li')
#谓词查找例子
# 查找所有有id属性的li标签
#text()获取标签中的内容
#li_list = tree.xpath('//ul/li[@id]/text()')
#找到id为l1的li标签
# li_list = tree.xpath('//ul/li[@id="l1"]/text()')
#属性查找例子
#查找id为l1的li标签的class的属性值
# li_list = tree.xpath('//ul/li[@id="l1"]/@class')
#模糊查找例子
#查找id中包含l的li标签
# li_list = tree.xpath('//ul/li[contains(@id,"l")]/text()')
#查找id的值以l开头的li 标签
li_list = tree.xpath('//ul/li[starts-with(@id,"l")]/text()')
print(li_list)
print(len(li_list))
附:xpath的基本使用.html
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8"/>
<title>Title</title>
</head>
<body>
<ul>
<li id="l1" class="A1">北京</li>
<li id="l2">上海</li>
<li>广州</li>
<li>深圳</li>
</ul>
</body>
</html>
三、用xpath解析来爬取图片
#(1)请求对象的定制
#(2)获取数据
#(3)下载
#下载前10页图片
#https://sc.chinaz.com/tupian/taikongkexuetupian.html
#https://sc.chinaz.com/tupian/taikongkexuetupian_page.html
import urllib.request
from lxml import etree
def create_request(page):
if (page == 1):
url = 'https://sc.chinaz.com/tupian/taikongkexuetupian.html'
else:
url = 'https://sc.chinaz.com/tupian/taikongkexuetupian_' + str(page) + '.html'
headers = {
'Cookie': 'cz_statistics_visitor=5c8d1db2-606a-6a71-1c2a-04d5f54fe634; Hm_lvt_398913ed58c9e7dfe9695953fb7b6799=1706184620; Hm_lpvt_398913ed58c9e7dfe9695953fb7b6799=1706188509; _clck=1elgg6s%7C2%7Cfip%7C0%7C1485',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
}
request = urllib.request.Request(url = url,headers = headers)
return request
def get_content(request):
response = urllib.request.urlopen(request)
content = response.read().decode('utf-8')
return content
def down_load(content):
#下载图片
#urllib.request.urlretrieve('图片地址','文件名字')
tree = etree.HTML(content)
name_list = tree.xpath('//div[@class="item masonry-brick"]//img/@src')
print(len(name_list))
if __name__ == '__main__':
start_page = int(input('请输入起始页码'))
end_page = int(input('请输入结束页码'))
for page in range(start_page,end_page+1):
#请求对象的定制
request = create_request(page)
#获取数据
content = get_content(request)
#下载
down_load(content)