作者:一枚小菜鸟
博客:https://blog.csdn.net/u010154424
爬取某招聘网站java、python、c/c++,php四种语言在北京,上海,广州,深圳四个一线城市的公开职位发布信息进行分析,数据样本来自前30页的数据,样本大小大概6058个。
一、数据抓取
非常简单,基本上没有发抓取策略
def downloader(city, keyword, page):
'''
:param city:
:param keyword:
:param page:
:return:
'''
url = "https://www.lagou.com/jobs/positionAjax.json?city={}&needAddtionalResult=false" \
.format(quote(city))
data = {
"first": "false",
"pn": page,
"kd": keyword
}
headers = {
"Accept": "application/json, text/javascript, */*; q=0.01",
"Accept-Encoding": "gzip, deflate, br",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Connection": "keep-alive",
"Content-Length": "26",
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
"Host": "www.lagou.com",
"Cookie": "WEBTJ-ID=20181228093856-167f276e34849d-015bd2bf49274b-6114147a-1327104-167f276e34a334; _ga=GA1.2.651225173.1545961137; _gid=GA1.2.952777220.1545961137; user_trace_token=20181228093740-29e0dba1-0a41-11e9-b14d-525400f775ce; PRE_HOST=www.baidu.com; PRE_SITE=https%3A%2F%2Fwww.baidu.com%2Fs%3Fwd%3D%25E6%258B%2589%25E5%258B%25BE%25E7%25BD%2591%26rsv_spt%3D1%26rsv_iqid%3D0xdc8f964d00002f4f%26issp%3D1%26f%3D8%26rsv_bp%3D1%26rsv_idx%3D2%26ie%3Dutf-8%26rqlang%3Dcn%26tn%3Dbaiduhome_pg%26rsv_enter%3D1%26oq%3D%2525E4%2525B8%252593%2525E8%2525B5%252584%2525E5%25258A%25259E%26rsv_t%3Df7a1d2gJnPyNK%252FsS4vTWJ9EOKhzAsK05aVgqC43iWtqWmiKpIp0u6YQblMkUzbi3KwO7%26inputT%3D8441%26rsv_pq%3D9f44c2a800002af6%26rsv_sug3%3D57%26rsv_sug1%3D62%26rsv_sug7%3D101%26bs%3D%25E4%25B8%2593%25E8%25B5%2584%25E5%258A%259E; LGUID=20181228093740-29e0e252-0a41-11e9-b14d-525400f775ce; LGSID=20181228093745-2cd1a71c-0a41-11e9-b14d-525400f775ce; PRE_UTM=m_cf_cpc_baidu_pc; PRE_LAND=https%3A%2F%2Fwww.lagou.com%2Flp%2Fhtml%2Fcommon.html%3Futm_source%3Dm_cf_cpc_baidu_pc%26m_kw%3Dbaidu_cpc_bj_e110f9_d2162e_%25E6%258B%2589%25E5%258B%25BE%25E7%25BD%2591; JSESSIONID=ABAAABAAAGGABCB3EDF3AFE52B111A35A8BDCCF214C647F; Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1545961137,1545961142,1545961149; index_location_city=%E5%8C%97%E4%BA%AC; TG-TRACK-CODE=index_search; SEARCH_ID=832387387eb944a39636c9973cbd41c4; LGRID=20181228093800-3605ba8a-0a41-11e9-ad84-5254005c3644; Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1545961158",
"Origin": "https://www.lagou.com",
"Referer": "https://www.lagou.com/jobs/list_python?labelWords=&fromSearch=true&suginput=",
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
"X-Anit-Forge-Code": "0",
"X-Anit-Forge-Token": "None",
"X-Requested-With": "XMLHttpRequest"
}
proxies = {
"http": "****",
"https": "****",
}
while True:
try:
response = requests.post(url, data=data, headers=headers, proxies=proxies)
response.encoding = "utf-8"
if response.status_code == 200:
data = json.loads(response.text)
result = jsonpath.jsonpath(data, "$.content.positionResult.result")[0]
with MongodbTools("dataanalysis") as mongo:
lagou = mongo.db["lagou"]
for row in result:
row["_id"] = "{}".format(row["positionId"])
lagou.update_one({"_id": row["_id"]}, {"$set": row}, upsert=True)
print("update or insert data = {}".format(row["_id"]))
break
except BaseException as e:
print(e)
pass
直接保存数据到mongodb中。
二、数据分析
1、数据清洗,格式化
import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
from datetime import datetime,timedelta
from pymongo import MongoClient
import time
mongo = MongoClient()["dataanalysis"]["lagou"]
values = mongo.find({},{"_id":0,"positionAdvantage":1,"salary":1,"city":1,"positionName":1,"workYear":1,"education":1,"industryField":1,"companySize":1,"financeStage":1,"firstType":1,"secondType":1,"thirdType":1})
values = [row for row in values]
df = pd.DataFrame(values)
# 格式化公司规模
def length(data,type):
value = data.values
if not value:
return 0
value = value[0]
if not value:
return 0
if value.find("以上") != -1:
if type == 1:
return 2000
else:
return 10000
elif value.find("-") != -1:
t = value.replace("人","").split("-")
if type == 1:
return int(t[0])
else:
return int(t[1])
else:
if type == 1:
return 0
else:
return 15
def min_staff(data):
return length(data,1)
def max_staff(data):
return length(data,2)
df["min_staff"] = df[["companySize"]].apply(min_staff,axis=1)
df["max_staff"] = df[["companySize"]].apply(max_staff,axis=1)
df = df.drop(["companySize"],axis=1)
# 格式化薪资
def salary(data,type):
value = data.values
if not value:
return 0
value = value[0]
if not value:
return 0
if value.find("-") != -1:
t = value.replace("k","").replace("K","").split("-")
if type == 1:
return int(t[0])*1000
elif type == 2:
return int(t[1])*1000
else:
return (int(t[0])*1000+int(t[1])*1000)/2
else:
return 0
def min_salary(data):
return salary(data,1)
def max_salary(data):
return salary(data,2)
def avg_salary(data):
return salary(data,3)
df["min_salary"] = df[["salary"]].apply(min_salary,axis=1)
df["max_salary"] = df[["salary"]].apply(max_salary,axis=1)
df["avg_salary"] = df[["salary"]].apply(avg_salary,axis=1)
# 格式化语言
def language(data):
value = data.values
if not value:
return None
value = value[0]
if not value:
return None
value = value.upper()
if value.find("PYTHON") != -1:
return "python"
if value.find("C++") != -1:
return "c/c++"
if value.find("C") != -1:
return "c/c++"
if value.find("JAVA") != -1:
return "java"
if value.find("PHP") != -1:
return "php"
return None
df["language"] = df[["positionName"]].apply(language,axis=1)
df = df.dropna()
把薪资,语言,公司规模进行格式化数据,删除为Nan的数据。
2、每个城市地区的平均工资图
total_x = None
total_y = []
total_city = []
for city_name,data in df.groupby(by="city"):
result = data.groupby(by=["language"])["avg_salary"].mean().sort_index()
plt.figure(figsize=(20,8),dpi=80)
_x = result.index
_y = result.values
plt.bar(_x,_y)
total_x = _x
total_y.append(_y)
total_city.append(city_name)
plt.xlabel("语言")
plt.ylabel("平均薪资")
plt.title("{}地区编程语言平均薪资".format(city_name))
plt.grid()
plt
3、平均薪资城市之间的对比
plt.figure(figsize=(20,8),dpi=80)
interval = 6
ind = np.array(range(0,len(total_x) * interval,interval))
width = 1
for index in range(len(total_city)):
plt.bar(ind - (2 - index) * width + width/2,total_y[index],label=total_city[index],width=1)
plt.xticks(range(0,len(total_x) * interval,interval),total_x)
plt.xlabel("语言")
plt.ylabel("平均薪资")
plt.title("一线城市编程语言平均薪资")
plt.grid()
plt.legend()
plt
可见大帝都的平均工资最高
4、岗位优势的分析
import re
def position_advantage(data):
value = data.values
if not value:
return []
value = value[0]
if not value:
return []
value = re.sub(r"[.~]","",value)
return re.split(r'[,,; ;、+-]',value)
labels = list(set([i for row in df[["positionAdvantage"]].apply(position_advantage,axis=1).values for i in row if i]))
position_data = pd.DataFrame(np.zeros((df.shape[0],len(labels))).astype(int),columns=labels,index=df.index)
for label in labels:
position_data[label][df["positionAdvantage"].str.contains(label)] = 1
result = position_data.sum().sort_values(ascending=False)
size = result[:10].values
size = [row for row in size]
labels = result[:10].index
labels = [row for row in labels]
size.append(result.sum() - sum(size))
labels.append("其它")
explode = [0 for i in range(len(size))]
explode[0] = 0.1
plt.figure(figsize=(10,10),dpi=80)
plt.pie(size, explode=explode, labels=labels, autopct='%1.1f%%',
shadow=True, startangle=90)
plt.title("岗位优势百分比")
plt
5、城市之间的岗位优势对比
total_value = []
total_label = []
labels = [row for row in result[:10].index]
for index in range(len(total_city)):
city = total_city[index]
data = position_data[df["city"] == city]
total_size = data.sum().sum()
total_label.append(city)
total_value.append((data[labels].sum()/total_size*10000).values.tolist())
plt.figure(figsize=(20,8),dpi=80)
interval = 8
ind = np.array(range(0,len(labels) * interval,interval))
width = 1
for index in range(len(total_label)):
plt.bar(ind - (2 - index) * width + width/2,total_value[index],label=total_label[index],width=1)
plt.xticks(range(0,len(labels) * interval,interval),labels)
plt.xlabel("福利")
plt.ylabel("占比(*100)")
plt.title("岗位优势占比图")
plt.grid()
plt.legend()
plt
6、工作经验要求占比分析
#工作经验要求占比
for city_name,data in df.groupby(by="city"):
result = data.groupby(by=["workYear"])["avg_salary"].count().sort_values()
plt.figure(figsize=(8,8),dpi=80)
_x = result.index
_y = result.values
plt.pie(_y, labels=_x, autopct='%1.1f%%',shadow=True, startangle=90)
plt.title("{}地区编程语言学历要求占比".format(city_name))
plt.grid()
plt
7、学历经验要求占比分析
#学历要求占比
for city_name,data in df.groupby(by="city"):
result = data.groupby(by=["education"])["avg_salary"].count().sort_index()
plt.figure(figsize=(8,8),dpi=80)
_x = result.index
_y = result.values
plt.pie(_y, labels=_x, autopct='%1.1f%%',shadow=True, startangle=90)
plt.title("{}地区编程语言学历要求占比".format(city_name))
plt.grid()
plt
8、绘制岗位优势的词云图
# 生成词图
from scipy.misc import imread
from wordcloud import WordCloud
from wordcloud import ImageColorGenerator
import matplotlib.pyplot as plt
from os import path
cloud = WordCloud(
#设置字体,不指定就会出现乱码,文件名不支持中文
font_path="C:/simfang.ttf",
#font_path=path.join(d,'simsun.ttc'),
#设置背景色,默认为黑,可根据需要自定义为颜色
background_color='black',
#词云形状,
#mask=color_mask,
#允许最大词汇
max_words=400,
#最大号字体,如果不指定则为图像高度
max_font_size=100,
#画布宽度和高度,如果设置了msak则不会生效
width=1200,
height = 800,
margin = 2,
#词语水平摆放的频率,默认为0.9.即竖直摆放的频率为0.1
prefer_horizontal = 0.8
)
result = position_data.sum().sort_values(ascending=False)
_labels = [row for row in result.index]
_frequency = [row for row in result.values]
_data = { _labels[index]:_frequency[index] for index in range(len(_labels))}
wc = cloud.generate_from_frequencies(_data)
wc.to_file("cloud.jpg") #保存图片
#显示词云图片
plt.imshow(wc)
#不现实坐标轴
plt.axis('off')
plt
§ §
Python中文社区作为一个去中心化的全球技术社区,以成为全球20万Python中文开发者的精神部落为愿景,目前覆盖各大主流媒体和协作平台,与阿里、腾讯、百度、微软、亚马逊、开源中国、CSDN等业界知名公司和技术社区建立了广泛的联系,拥有来自十多个国家和地区数万名登记会员,会员来自以公安部、工信部、清华大学、北京大学、北京邮电大学、中国人民银行、中科院、中金、华为、BAT、谷歌、微软等为代表的政府机关、科研单位、金融机构以及海内外知名公司,全平台近20万开发者关注。
最近热门
投稿邮箱:pythonpost@163.com
▼点击下方阅读原文,免费成为数据科学俱乐部会员