前程无忧岗位数据可视化分析报告

从入门到就业

已于 2023-03-01 23:18:09 修改

阅读量3.1k

点赞数 11

文章标签：信息可视化数据分析爬虫 Powered by 金山文档

于 2023-03-01 22:33:23 首次发布

本文链接：https://blog.csdn.net/m0_52382330/article/details/129287214

版权

1.数据爬取

数据爬取的内容主要包括30个岗位种类（数据分析，产品经理，产品助理，交互设计，前端开发，软件设计，IOS开发，业务分析，安卓开发，PHP开发，业务咨询，需求分析，流程设计，售后经理，售前经理，技术支持，ERP实施，实施工程师，IT项目经理，IT项目助理，信息咨询，数据挖掘，数据运营，网络营销，物流与供应链，渠道管理，电商运营，客户关系管理，新媒体运营，产品运营），每一个岗位爬取的信息包括：岗位名称、公司名称、公司规模、工作地点、薪资、工作要求、工作待遇等。

数据爬取代码展示：

1. import requests  
2. from bs4 import BeautifulSoup  
3. import pymysql  
4. import random  
5. from selenium import webdriver  
6. from lxml import etree  
7. import lxml  
8. from selenium.webdriver import ChromeOptions  
9. import re  
10. import time  
11. import  requests  
12. #定义函数，用于获取每个 url 的 html  
13. def spider(url):  
14.     headers = {  
15.         "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36 Edg/94.0.992.31"}  
16.     try:  
17.         rep = requests.get(url, headers=headers)  
18.         rep.raise_for_status()  
19.         rep.encoding = rep.apparent_encoding  
20.         txt = rep.text  
21.         return txt  
22.     except:  
23.         print("解析失败")  
24.   
25. #定义 jiexi()函数，用于解析得到的 html  
26. def jiexi(html, info,name):  
27.     soup = BeautifulSoup(html, "lxml")  
28.     text = soup.find_all("script", type="text/javascript")[3].string  
29.     #观察原始代码发现我们需要的数据在 engine_jds 后  
30.     data = eval(str(text).split("=", 1)[1])["engine_jds"]  
31.     for d in data:  
32.         try:  
33.             job_name = d["job_name"].replace("\\", "")  # 岗位名称  
34.         except:  
35.             job_name = " "  
36.         try:  
37.             company_name = d["company_name"].replace("\\", "")  # 公司名称  
38.         except:  
39.             company_name = " "  
40.         try:  
41.             providesalary_text = d["providesalary_text"].replace("\\", "")  # 薪资  
42.         except:  
43.             providesalary_text = " "  
44.         try:  
45.             workarea_text = d["workarea_text"].replace("\\", "")    #工作地点  
46.         except:  
47.             workarea_text = " "  
48.         try:  
49.             updatedate = d["updatedate"].replace("\\", "")  #更新时间  
50.         except:  
51.             updatedate = " "  
52.         try:  
53.             jobwelf = d["jobwelf"].replace("\\", "")    # 工作待遇  
54.         except:  
55.             jobwelf = " "  
56.         try:  
57.             companyind_text = d["companyind_text"].replace("\\", "")  # 公司类型  
58.         except:  
59.             companyind_text = " "  
60.         try:  
61.             companysize_text = d["companysize_text"].replace("\\", "") # 公司规模  
62.         except:  
63.             companysize_text = " "  
64.         try:  
65.             at = d["attribute_text"]    # 工作要求  
66.             s = ''  
67.             for i in range(0, len(at)):  
68.                 s = s + at[i] + ','  
69.                 attribute_text = s[:-1]  
70.         except:  
71.             attribute_text = " "  
72. #将每一条岗位数据爬取下的内容以及传入参数 name 作为一个列表，依此加入到 info 列表中  
73.         info.append( [ name,job_name, updatedate, company_name, companyind_text, companysize_text, workarea_text, providesalary_text, attribute_text, jobwelf])  
74. #将数据存到 MySQL 中名为“51job”的数据库中  
75. def save(info):  
76.     #将数据保存到数据库表中对应的列  
77.     for data in info :  
78.         present_job = data[0]  # 当前爬取岗位  
79.         job_name = data[1]  #岗位  
80.         updatedate = data[2]            #更新时间  
81.         company_name = data[3]  # 公司名称  
82.         companyind_text = data[4]           #公司类型  
83.         companysize_text = data[5]          #公司规模  
84.         workarea_text = data[6]                 #工作地点  
85.         providesalary_text = data[7]                    #薪资  
86.         attribute_text = data[8]        #工作要求  
87.         jobwelf = data[9]   #工作待遇  
88.         # 创建 sql 语句  
89.         sql = "insert into jobs(当前爬取岗位,岗位,更新时间,公司名称,公司类型,公司规模,工作地点,薪资,工作要求,工作待遇) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"  
90.         # 执行 sql 语句  
91.         cursor.execute(sql, [present_job, job_name, updatedate, company_name, companyind_text, companysize_text,  
92.                              workarea_text, providesalary_text, attribute_text, jobwelf])  
93.         db.commit()  # 提交数据  
94. if    __name__  == '__main__':  #主函数  
95.     job=["产品经理","产品助理","交互设计","前端开发","软件设计","IOS开发","业务分析","安卓开发","PHP开发","业务咨询","需求分析","流程设计"  
96.     ,"售后经理","售前经理","技术支持","ERP实施","实施工程师","IT项目经理","IT项目助理","信息咨询","数据挖掘","数据运营","数据分析","网络营销",  
97.     "物流与供应链","渠道管理","电商运营","客户关系管理","新媒体运营","产品运营"]  
98.     page_list=['1141', '62', '169', '619', '356', '61', '229', '64', '56', '356', '1379', '147', '62', '29', '2000', '173', '184', '10', '2', '396', '221', '115', '2000', '381', '5', '295', '1233', '280', '699', '352']  
99.     #https://www.pexels.com/  
100.     option = ChromeOptions()  
101.     UA="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36 Edg/94.0.992.31"  
102.     option.add_argument(f'user-agent={UA}')  
103.     option.add_experimental_option('useAutomationExtension', False)  
104.     option.add_experimental_option('excludeSwitches', ['enable-automation'])  
105.     web = webdriver.Chrome(chrome_options=option)  # chrome_options=chrome_opt,,options=option  
106.     web.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {  
107.         "source": """ 
108.         Object.defineProperty(navigator, 'webdriver', { 
109.           get: () => undefined 
110.         }) 
111.       """  
112.     })  
113.     web.implicitly_wait(3)  
114.     url='https://search.51job.com/list/000000,000000,0000,00,9,99,%E4%BA%A7%E5%93%81%E7%BB%8F%E7%90%86,2,2.html?'  
115.     web.get(url)  
116.     time.sleep(6)  
117.     le=len(job)  
118.     db = pymysql.connect(  # 连接数据库host="127.0.0.1", #MySQL 服务器名  
119.         user="root",  # 用户名  
120.         password="root",  # 密码  
121.         database="python上机",  # 操作的数据库名称charset="utf8"  
122.     )  
123.     cursor = db.cursor()  
124.     for j in range(23,le):  
125.         for i in range(1,int(page_list[j])):#页面  
126.             info = []  
127.             # url = "https://search.51job.com/list/000000,000000,0000,00,9,99," + j + ",2," + str(i) + ".html?"  
128.             url = "https://search.51job.com/list/000000,000000,0000,00,9,99,{},2,{}.html?".format(job[j], i)  
129.             web.get(url)  
130.             ht = web.page_source  
131.             soup = BeautifulSoup(ht, "lxml")  
132.             jiexi(ht, info,job[j])  
133.             print('岗位{}:{}/{}'.format(j,i,page_list[j]))  
134.             time.sleep(1)  
135.             save(info)  
136.         time.sleep(1)  
137.     cursor.close()  
138.     # 关闭连接  
139.     db.close()

数据爬取结果：