使用Xpath

import requests
import re
from lxml import etree
th = """
    <div>
    <ul>
        <li class="item-0"><a href = "link1.html">first item</a></li>
        <li class="item-1"><a href = "link2.html">second item</a></li>
        <li class="item-inactive"><a href = "link3.html">third item</a></li>
        <li class="item-1"><a href = "link4.html">fourth item</a></li>
        <li class="item-0"><a href = "link5.html">fifth item</a></li>
    </ul>
</div>
"""

html = etree.HTML(th)

r1_all = html.xpath('//*') # 获取所有节点
print(r1_all)

r1_zijiedian = html.xpath('//li/a') # 获取li节点下直接子节点a
print(r1_zijiedian)

r1_fu = html.xpath('//a[@href="link4.html"]/../@class')
r1_fu = html.xpath('//a[@href="link4.html"]/parent::*/@class')
print(r1_fu)

r1_shuxingpipei = html.xpath('//li[@class = "item-0"]')
print(r1_shuxingpipei)   #选取属性class 为 "item-0"的li节点

###  文本获取
r1_wenbenhuoqu = html.xpath('//li[@class="item-0"]/a/text()')
print(r1_wenbenhuoqu)   # 获取li节点下,属性class为item-0,a节点内的文本

r1_wenbenhuoqu_1 = html.xpath('//li[@class="item-0"]//text()')
print(r1_wenbenhuoqu_1) # 获取li节点下,属性class为item-0,子孙节点内所有的文本

### 属性获取
r1_shuxinghuoqu = html.xpath('//li/a/@href')
print(r1_shuxinghuoqu)  #获取所有li节点下所有a节点的href属性

## 如果某些节点的某个属性有多个值,则用contains函数
text = '''
<li class = "li li-first"><a href="link.html">first item</a></li>
'''
html = etree.HTML(text)
r1_duogeshuxing = html.xpath('//li[contains(@class,"li")]/a/text()')
print(r1_duogeshuxing)

### 多属性匹配一个节点:
text = '''
<li class = "li li-first" name ="item"><a href="link.html">first item</a></li>
'''
html = etree.HTML(text)
re1_duoshuxingpipei = html.xpath('//li[contains(@class,"li") and @name="item"]/a/text()')
print(re1_duoshuxingpipei)