【廖雪峰】python爬虫------------学习记录day06（xpath）

weixin_46061565

于 2021-05-07 23:33:01 发布

阅读量105

点赞数

分类专栏：笔记文章标签： python xpath

本文链接：https://blog.csdn.net/weixin_46061565/article/details/116458442

版权

笔记专栏收录该内容

9 篇文章 0 订阅

订阅专栏

import requests
from lxml import etree

url = 'http://news.baidu.com/'
headers = {
    "User-Agent":'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0'
}
data = requests.get(url,headers=headers).content.decode()

# 1.转解析类型
xpath_data = etree.HTML(data)


# xpath 语法 1. 节点 /
#            2. 跨节点: //
#            3. 精确的标签: //a[@属性="属性值"]
#            4. 标签包裹的内容 text()
#            5. 属性:@href
#              xpath--s数据类型---list
#            6.模糊查询 //div[contain(@class,"a")]
#            7.下一个节点（平级关系，例：兄弟节点）following-sibling::*
"""
html_data = etree.HTML(data)

        result_list = html_data.xpath('//div[contains(@id,"stickthread")]')
        result_list = html_data.xpath('//head/following-sibling::*[1]')
        print(len(result_list))
        print(result_list)
"""
# 2调用 xpath的方法
result = xpath_data.xpath('/html/head/title//text()')
result = xpath_data.xpath('//a/text()')
result = xpath_data.xpath('//a[@mon="ct=1&a=2&c=top&pn=18"]/text()')
result = xpath_data.xpath('//a[@mon="ct=1&a=2&c=top&pn=18"]/@href')
result = xpath_data.xpath('//li/a/text()')

print(result)

from lxml import etree

html = """
    <html>
    <body>
    <ul>
     <li>1
         <a href="">子</a>
     </li>
     <li>2
        <a href="">子</a>
     </li>
     <li>3
        <a href="">子</a>
     </li>
     <li>4
         <a href="">子</a>
     </li>
     <li>5
        <a href="">子</a>
     </li>

 </ul>
 </body>
 </html>
"""
# 1.转类型
x_data = etree.HTML(html)

# 2.xpath 下标 是从 1开始; 只能取 平级关系的标签
result = x_data.xpath('//li[5]/text()')
result = x_data.xpath('/html/body/ul/li/a/text()')
result = x_data.xpath('//a[2]')

print(result)

import requests
from lxml import etree
import json

class BctSpider(object):
    def __init__(self):
        self.base_url = 'https://www.chainnode.com/forum/61-'  #跳转
        self.headers = {
            "User-Agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0'
        }
    #1、发请求
    def get_response(self,url):
        response = requests.get(url,headers=self.headers)
        data = response.content.decode()
        return data
    #2、解析数据
    def parse_data(self,data):
        # 使用xpath 解析当前页面 所有的 新闻title 和url 保存
        # 1.转类型
        x_data = etree.HTML(data)
        #2、根据xpath路径解析
        title_list = x_data.xpath('//a[@class="link-dark-major font-bold bbt-block"]/span/text()')
        url_list = x_data.xpath('//a[@class="link-dark-major font-bold bbt-block"]/@href')
        data_list = []
        for index,title in enumerate(title_list):
            news = {}
            # print(index)
            # print(title)
            news['name'] = title
            news['url'] = url_list[index]
            data_list.append(news)
        return data_list
    #3、保存数据
    def save_data(self,data):
        # 将list----str
        data_str = json.dumps(data)
        with open('news01.html','w',encoding='utf-8') as f:
            f.write(data_str)
    #4、启动
    def run(self):
        #1、拼接完整的url
        url = self.base_url + '1'
        #2、发请求
        data = self.get_response(url)
        #3、做解析
        parse_data = self.parse_data(data)
        #4、保存
        self.save_data(parse_data)

BctSpider().run()