7.urllib-xpath插件

谷歌浏览器xpath安装

关闭浏览器，重新打开

测试xpath

ctrl+shift+x

xpath基本操作

# 解析本地文件
tree = etree.parse("lxmltext.html")

# 路径查询
# //：查找所有子孙节点，不考虑层级关系
# /：找直接子节点
li_list = tree.xpath("//body/ul/li")
print(li_list)
# 谓词查询
li_list = tree.xpath("//ul/li[@id]")
print(li_list)
li_list = tree.xpath("//ul/li[@id='l1']")
print(li_list)
# 内容查询
li_list = tree.xpath("//ul/li[@id]/text()")
print(li_list)
# 属性查询
li_list = tree.xpath("//ul/li[@id]/@class")
print(li_list)
# 模糊查询
li_list = tree.xpath("//ul/li[contains(@id,'l')]/text()")
print(li_list)
li_list = tree.xpath("//ul/li[starts-with(@id,'l')]/text()")
print(li_list)
# 逻辑运算
li_list = tree.xpath("//ul/li[@id='l1' and @class='c1']/text()")
print(li_list)
li_list = tree.xpath("//ul/li[@id='l1']/text() | //ul/li[@id='l2']/text()")
print(li_list)



# 解析网络文本
url = 'http://www.baidu.com'
headers = {
    "User-Agent": ..."
}
request = urllib.request.Request(url=url, headers=headers)
handler = urllib.request.HTTPHandler()
opener = urllib.request.build_opener(handler)
response = opener.open(request)
content = response.read().decode("utf8")

tree = etree.HTML(content)
print(tree.xpath("//input[@id='su']/@value")[0])

案例：爬取站长素材图片

import urllib.request

import lxml.etree
import tqdm


def create_request(page):
    if page == 1:
        url = "https://sc.chinaz.com/tupian/qinglvtupian.html"
    else:
        url = "https://sc.chinaz.com/tupian/qinglvtupian_" + str(page) + ".html"
    headers = {
        "User-Agent": "..."
    }
    request = urllib.request.Request(url=url, headers=headers)
    return request


def get_content(request):
    handler = urllib.request.HTTPHandler()
    opener = urllib.request.build_opener(handler)
    response = opener.open(request)
    content = response.read().decode('utf8')
    return content


def down_load(content):
    tree = lxml.etree.HTML(content)
    name_list = tree.xpath("//div[@id='container']//a/img/@alt")
    # 一般设计图片的网站都会进行懒加载，要注意加载前的图片属性
    src_list = tree.xpath("//div[@id='container']//a/img/@src2")
    for i in tqdm.tqdm(range(len(name_list))):
        name = name_list[i]
        src = src_list[i]
        url = "https:" + src
        urllib.request.urlretrieve(url=url, filename="data/" + name + ".jpg")


if __name__ == '__main__':
    start_page = int(input("请输入起始页码"))
    end_page = int(input("请输入结束页码"))
    for page in range(start_page, end_page + 1):
        request = create_request(page)
        content = get_content(request)
        down_load(content)