xpath 处理网页出现的问题总结
<div class="name">
<div class="title">
<div class="price">
<span>
<a href="">网页欣赏</a>
</span>
</div>
</div>
</div>
- 当文档为多层结构时,无法匹配内容
- 使用text()方法匹配不到内容,得到的只是空的 ‘\n ’
data.xpath('//div[@class="name"]//text()')
- 使用xpath(‘string(.)’)的方式,则可以正确匹配
data.xpath('//div[@class="name"]')[0].xpath('string(.)')
- 使用text()方法匹配不到内容,得到的只是空的 ‘\n ’
- 使用xpath得到的div对象,再次使用xpath匹配内容时,出现错误
- 使用 div对象 <class 'lxml.etree._Element'>,再次使用xpath 匹配其内容时,失败
-
html = etree.parse('./test_xpath.html', etree.HTMLParser()) strings = etree.HTML(etree.tounicode(html)) # print(strings) pp = strings.xpath('//div[@class="name"]')[0] print(type(strings)) # <class 'lxml.etree._Element'> print(type(pp)) # <class 'lxml.etree._Element'> print(pp.xpath('/div[@class="title"]')) # []
- 为何会匹配失败??
- print(pp.xpath('/div[@class="title"]')) # [], 前面使用的是 '/' 而不是 ‘//’, 匹配的是根路径,导致无法查找
- 使用xpath 打开html文件时,会遇到无法解码为中文的情况
html = etree.parse('./test_xpath.html', etree.HTMLParser()) strings = etree.tostring(html) print(strings) # get page like this ''' b'<!DOCTYPE html>\n<html lang="en"> \n<head> \n <meta charset="UTF-8"/> \n <title>Title</title> \n</head> \n<body> \n<div class="name"> \n <div class="title"> \n <div class="price"> \n <span> \n <a href="">网页欣赏</a> \n </span> \n </div> \n </div> \n</div> \n</body> \n</html>' ''' html = etree.parse('./test_xpath.html', etree.HTMLParser()) strings = etree.tostring(html).decode() print(strings) ''' <!DOCTYPE html> <html lang="en"> <head> <meta charset="UTF-8"/> <title>Title</title> </head> <body> <div class="name"> <div class="title"> <div class="price"> <span> <a href="">网页欣赏</a> </span> </div> </div> </div> </body> </html> ''' 但是还是不对,中文并没有解析出来
- 使用etree.tostring()
- 使用etree.tounicode(), 则正常解析,并且不需要使用decode,就能得到正常的html
-
html = etree.parse('./test_xpath.html', etree.HTMLParser()) strings = etree.tounicode(html) print(strings) <!DOCTYPE html> <html lang="en"> <head> <meta charset="UTF-8"/> <title>Title</title> </head> <body> <div class="name"> <div class="title"> <div class="price"> <span> <a href="">网页欣赏</a> </span> </div> </div> </div> </body> </html>
movie_div = strings.xpath("//div[contains(@class,'doulist-item')]//text()")
movie_div = strings.xpath("//div[contains(@class,'doulist-item')]//text()")
'\n '