python爬取boss直聘职位数据，并保存到本地

代码环境

1. python 3.7
2. pip 19.0.3

---

主要引用的第三方库

1. requests，用于模拟http/https请求
   • 安装： pip install requests
   • 文档： requests中文文档
2. beautifulsoup4，用于解析网页，得出我们想要的内容。
   • 安装： pip install beautifulsoup4
   • 文档： bs4中文文档
3. xlwt，将爬到的结果以Excel的形式保存到本地
   • 安装： pip install xlwt
   • api： xlwt api

---

打开网页

首先打开boss直聘官网，选择一个地点，然后输入关键字，点击搜索，这里以深圳、python为例。

观察地址栏URL，可以发现有四个参数，分别是query,city,industry和position，query和city很明显是我输入的python和选择的地点深圳；而industry和position也就是公司行业和职位类型，这里没有选择这两项。

分析网页

F12打开开发者工具

每一条职位信息都在一个<li>标签中，<li>标签下的<div class=“job-primary”>就是我们要找的内容。

代码

• 获取城市编码

  url中的city=101280600，显示的是深圳，说明城市名有一个对应的编号，F12 点击Network选中XHR，有一个city.json
  

import requests
from bs4 import BeautifulSoup
import json
import xlwt
import time
import random

user_agent_list = [
    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; …) Gecko/20100101 Firefox/61.0",
    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)",
    "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10.5; en-US; rv:1.9.2.15) Gecko/20110303 Firefox/3.6.15"
]

headers = {
        "user-agent": random.choice(user_agent_list)
    }


# 获取指定城市的编码
def get_city_code(city_name):
    response = requests.get("https://www.zhipin.com/wapi/zpCommon/data/city.json")
    contents = json.loads(response.text)
    cities = contents["zpData"]["hotCityList"]
    city_code = contents["zpData"]["locationCity"]["code"]
    for city in cities:
        if city["name"] == city_name:
            city_code = city["code"]
    return city_code


def get_url(query="", city="", industry="", position="", page=1):
    base_url = "https://www.zhipin.com/job_detail/?query={}&city={}&industry={}&position={}&page={}"
    urls = []
    url = base_url.format(query, city, industry, position, page)
    response = requests.get(url, headers=headers)
    soup = BeautifulSoup(response.text, "lxml")
    page_list = soup.find("div", "page").find_all("a")
    urls.append(url)
    while page_list[len(page_list) - 1]["href"] != "javascript:;":
        page += 1
        url = base_url.format(query, city, industry, position, page)
        urls.append(url)
        response = requests.get(url, headers=headers)
        soup = BeautifulSoup(response.text, "lxml")
        page_list = soup.find("div", "page").find_all("a")
    return urls


def get_html(url):
    response = requests.get(url, headers=headers)
    return response.text


def job_info(job_name, company, industry, finance, staff_number, salary, site, work_experience, education_bak, job_desc):
    return {
        "job_name": job_name,
        "company": company,
        "industry": industry,
        "finance": finance,
        "staff_number": staff_number,
        "salary": salary,
        "site": site,
        "work_experience": work_experience,
        "education_bak": education_bak,
        "job_desc": job_desc
    }


def get_job_desc(jid, lid):
    url = "https://www.zhipin.com/wapi/zpgeek/view/job/card.json?jid={}&lid={}"
    response = requests.get(url.format(jid, lid), headers=headers)
    html = json.loads(response.text)["zpData"]["html"]
    soup = BeautifulSoup(html, "lxml")
    desc = soup.find("div", "detail-bottom-text").get_text()
    return desc


def get_content(html):
    bs = BeautifulSoup(html, 'lxml')
    contents = []
    for info in bs.find_all("div", "job-primary"):
        job_name = info.find("div", "job-title").get_text()
        company = info.find("div", "company-text").a.get_text()
        jid = info.find("div", "info-primary").a["data-jid"]
        lid = info.find("div", "info-primary").a["data-lid"]
        desc = get_job_desc(jid, lid)
        texts = [text for text in info.find("div", "info-primary").p.stripped_strings]
        site = texts[0]
        work_exp = texts[1]
        edu_bak = texts[2]
        salary = info.span.get_text()
        companies = [text for text in info.find("div", "company-text").p.stripped_strings]
        industry = companies[0]
        if len(companies) > 2:
            finance = companies[1]
            staff_num = companies[2]
        else:
            finance = None
            staff_num = companies[1]
        contents.append(job_info(job_name, company, industry, finance, staff_num, salary, site, work_exp, edu_bak, desc))
        time.sleep(1)
    return contents


def save_data(content, city, query):
    file = xlwt.Workbook(encoding="utf-8", style_compression=0)
    sheet = file.add_sheet("job_info", cell_overwrite_ok=True)
    sheet.write(0, 0, "职位名称")
    sheet.write(0, 1, "公司名称")
    sheet.write(0, 2, "行业")
    sheet.write(0, 3, "融资情况")
    sheet.write(0, 4, "公司人数")
    sheet.write(0, 5, "薪资")
    sheet.write(0, 6, "工作地点")
    sheet.write(0, 7, "工作经验")
    sheet.write(0, 8, "学历要求")
    sheet.write(0, 9, "职位描述")
    for i in range(len(content)):
        sheet.write(i+1, 0, content[i]["job_name"])
        sheet.write(i+1, 1, content[i]["company"])
        sheet.write(i+1, 2, content[i]["industry"])
        sheet.write(i+1, 3, content[i]["finance"])
        sheet.write(i+1, 4, content[i]["staff_number"])
        sheet.write(i+1, 5, content[i]["salary"])
        sheet.write(i+1, 6, content[i]["site"])
        sheet.write(i+1, 7, content[i]["work_experience"])
        sheet.write(i+1, 8, content[i]["education_bak"])
        sheet.write(i+1, 9, content[i]["job_desc"])
    file.save(r'c:\projects\{}_{}.xls'.format(city, query))


def main():
    city_name = "深圳"
    city = get_city_code(city_name)
    query = "python"
    urls = get_url(query=query, city=city)
    contents = []
    for url in urls:
        html = get_html(url)
        content = get_content(html)
        contents += content
        time.sleep(5)
    save_data(contents, city_name, query)


if __name__ == '__main__':
    main()