网络爬虫初涉——用python爬取岗位招聘数据

招聘网站为：https://www.lagou.com/zhaopin/Python/?filterOption=3&sid=e4527f26930f4cfaba0ab9349911e6e7
代码：

import requests
from lxml import etree
# 请求头
headers = {
        'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36 Core/1.47.933.400 QQBrowser/9.4.8699.400',
    }

# 发出数据请求，返回response响应对象
r = requests.get('https://www.lagou.com/zhaopin/Python/',headers=headers)
# 进行utf-8编码
r.encoding = "utf-8"

# 构造一个xpath解析对象
selector = etree.HTML(r.text)
# 职位编号，一页共有14个职位，所以这里可以传递0~13任意数字
row_num = str(14)
'''获取职位概况信息，包括：职位名称、公司名称、公司简介、薪水、职位招聘对象'''
# p_name 职位名称
p_name = selector.xpath('//ul[@class="item_con_list"]/li[@data-index={}]//h3/text()'.format(row_num))[0]
# p_company 招聘公司名称
p_company = selector.xpath('//ul[@class="item_con_list"]/li[@data-index={}]//div[@class="company_name"]/a/text()'.format(row_num))[0]
# p_industry 招聘公司简介
p_industry = selector.xpath('//ul[@class="item_con_list"]/li[@data-index={}]//div[@class="industry"]/text()'.format(row_num))[0]
# p_money 职位薪资
p_money = selector.xpath('//ul[@class="item_con_list"]/li[@data-index={}]//span[@class="money"]/text()'.format(row_num))[0]
# p_require 职位招聘对象
p_require = selector.xpath('//ul[@class="item_con_list"]/li[@data-index={}]//div[@class="li_b_l"]/text()'.format(row_num))[2]
# p_address 地址
p_address = selector.xpath('//ul[@class="item_con_list"]/li[@data-index={}]//span[@class="add"]//em/text()'.format(row_num))[0]

''' 获取工作职责和工作要求等详细信息'''
# 该职位详细信息跳转链接
p_href = selector.xpath('//ul[@class="item_con_list"]/li[@data-index={}]//a[@class="position_link"]/@href'.format(row_num))[0]
print(p_href)
# 返回response响应对象
r_detail = requests.get(p_href, headers=headers)
# 进行utf-8编码
r_detail.encoding = "utf-8"
# 解析HTML对象
selector_detail = etree.HTML(r_detail.text)
# p_detail 工作职责和工作要求信息
p_detail = selector_detail.xpath('//div[@class="job-detail"]//text()')
'''将获取到的有效信息放到列表中'''
p_list = [p_name, p_company, p_industry.strip(), p_money, p_address, p_require.strip(), "\n ".join(p_detail)]
for i in p_list:
    print(i)

爬取结果：

参考

https://zhuanlan.zhihu.com/p/83968537