[B10]爬虫课程02

数据解析

1.Xpath语法和lxml模块

#使用方式：使用//获取整个页面当中的元素，然后写标签名，然后再写谓词进行提取。

//div[@clas='abc']

需要注意的知识点：

1./和//的区别：/只获取直接子节点，//可以获取子孙节点
2.contains:有时候某个属性包含多个值，可以使用cntains

//div[contains(@class,'job_detail')]

3.谓词的下标是从1开始

使用lxml解析HTML代码：

1.解析html字符串：使用’lxml.etree.HTML’进行

htmlElement = etree.HTML(text)
print(etree.tostring(htmlElement,encoding='utf-8').decode('utf-8'))

2.解析html文件：使用’lxml.etree.parse’进行，如果这个函数默认使用xml解析器，需要自己创建html解析器。

htmlElement = etree.parse('qingyunian.html')
print(etree.tostring(htmlElement,encoding='utf-8').decode('utf-8'))

实例

from lxml import etree

#解析庆余年短评
def parse_qyn_file():
    parser = etree.HTMLParser(encoding='utf-8')
    htmlElement = etree.parse('qingyunian.html',parser=parser)
    print(etree.tostring(htmlElement,encoding='utf-8').decode('utf-8'))
 
#解析拉钩网页
def parse_lagou_file():
    parser = etree.HTMLParser(encoding='utf-8')
    htmlElement = etree.parse('lagou.html',parser=parser)
    print(etree.tostring(htmlElement,encoding='utf-8').decode('utf-8'))

if __name__ == '__main__':
    #parse_lagou_file()
    #parse_text()
    parse_qyn_file()

实例：解析腾讯招聘的网页信息

from lxml import etree

parser = etree.HTMLParser(encoding='utf-8')
html = etree.parse("tencent.html",parser=parser)
#1.获取所有的a标签
#//a
#xpath返回的是一个列表
alis = html.xpath("//a[@class='recruit-list-link']")
for a in alis:
    print(etree.tostring(a,encoding='utf-8').decode("utf-8"))

#2.获取所有岗位名称
alis = html.xpath("//h4")
for a in alis:
    print(etree.tostring(a,encoding='utf-8').decode("utf-8"))

#3.获取所有职位信息
alis = html.xpath("//p[@class='recruit-text']")
for a in alis:
    print(etree.tostring(a,encoding='utf-8').decode("utf-8"))

#4.获取所有的纯文本信息
alis = html.xpath("//a[@class='recruit-list-link']")
positions = []
for a in alis:
    #在某个标签下，再执行xpath函数，获取子孙元素，那么应该在//前加一个点，代表在当前元素下获取
    title = a.xpath(".//h4[@class='recruit-title']/text()")
    daihao = a.xpath("./p[1]//span[1]/text()")
    address = a.xpath("./p[1]//span[2]/text()")
    category = a.xpath("./p[1]//span[3]/text()")
    time = a.xpath("./p[1]//span[5]/text()")
    needs = a.xpath("./p[2]/text()")  
    
    position = {
        'title':title,
        'daihao':daihao,
        'address':address,
        'category':category,
        'time':time,
        'needs':needs
    }
    positions.append(position)

    print(positions)

#写入Excel表格

#写入Excel表格
import pandas as pd

datadf = pd.DataFrame(positions)
 
datadf.to_excel('result.xlsx',sheet_name='pachong_cc')
# 导出excel