【Python爬虫】 爬取招聘信息（requests、lxml库）

开始前先引入requests库、lxml库中的etree类

import requests # 请求数据
from lxml import etree # 解析html代码

使用Python中的 requests 请求数据，headers值为浏览器头部信息，目的是为了将request请求伪装成浏览器的请求，返回的响应中使用“text”获取响应内容的文本形式，如果使用“content”则得到的是响应内容的二进制形式数据

def get_data():
    url = "https://km.58.com/fachuandan/1/?combination_id=1&utm_source=market&spm=u-2d2yxv86y3v43nkddh1.BDPCPZ_BT&PGTID=0d100000-0021-d747-e82a-1de4982080b4&ClickID=2"
    # 头部信息，将该请求伪装成浏览器的请求
    header = {
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36"}

    response = requests.get(url, headers=header)
    response.encoding = "utf-8" # 设置字符编码

使用 lxml 中的 etree 解析 html代码，解析后html代码呈现为树状结构，可以根据XPath获取想要获取的内容

    content = etree.HTML(response.text)
    data_dict = {}

    # 岗位名称
    name = content.xpath("/html/body/div[3]/div[1]/div[6]/div[1]/div[1]/div/div[1]/h2/a/text()")
    name_list = []
    for x in name:
        str_name = x.split("\r\n")[0]
        name_list.append(str_name)

    # 工资
    price = content.xpath("/html/body/div[3]/div[1]/div[6]/div[1]/div[1]/div/div[2]/div[2]/span[1]/em/text()")
    data_dict["price"] = price

（XPath的获取请看博文：【Python爬虫】 XPath 与 selector 的 获取_xpath selector_zhouwhui椆的博客-CSDN博客）

获取完数据后，组合输出：

for x,y in zip(name_list,price):
    print("name:",x,"price:",y)

综合代码：

import requests # 请求数据
from lxml import etree # 解析html代码

def get_data():
    url = "https://km.58.com/fachuandan/1/?combination_id=1&utm_source=market&spm=u-2d2yxv86y3v43nkddh1.BDPCPZ_BT&PGTID=0d100000-0021-d747-e82a-1de4982080b4&ClickID=2"
    # 头部信息，将该请求伪装成浏览器的请求
    header = {
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36"}

    response = requests.get(url, headers=header)
    response.encoding = "utf-8"
    content = etree.HTML(response.text)
    data_dict = {}

    # 岗位名称
    name = content.xpath("/html/body/div[3]/div[1]/div[6]/div[1]/div[1]/div/div[1]/h2/a/text()")
    name_list = []
    for x in name:
        str_name = x.split("\r\n")[0]
        name_list.append(str_name)

    # 工资
    price = content.xpath("/html/body/div[3]/div[1]/div[6]/div[1]/div[1]/div/div[2]/div[2]/span[1]/em/text()")
    data_dict["price"] = price

    for x,y in zip(name_list,price):
        print("name:",x,"price:",y)