数据挖掘岗位分析

import requests
from bs4 import BeautifulSoup
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from selenium import webdriver
import builtwith
import re

#检查网站采取了那些技术
html = requests.get('https://search.51job.com').content
builtwith.parse('https://search.51job.com',html=html.decode('gbk'))#因为51job是GBK格式，所以需要设置HTML的解码格式，模块默认是utf-8

#数据爬取过程，51job
occupation_infos = []
for page in range(1,262):
    url = 'https://search.51job.com/list/000000,000000,0000,00,9,99,%25E6%2595%25B0%25E6%258D%25AE%25E6%258C%2596%25E6%258E%2598,2,{}.html?\
    lang=c&stype=1&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&lonlat=0%2C0&radius=-1&ord_field=0&\
    confirmdate=9&fromType=1&dibiaoid=0&address=&line=&specialarea=00&from=&welfare='.format(page)
    #url中，
    res = requests.get(url).content.decode('gbk')
    soup = BeautifulSoup(res,'lxml')
    occupations = [i.get_text().strip() for i in soup.find_all('p',{'class':'t1'})]
    companys = [i.get_text().strip() for i in soup.find_all('span',{'class':'t2'})[1:]]
    regions  = [i.get_text().strip() for i in soup.find_all('span',{'class':'t3'})[1:]]
    salarys = [i.get_text().strip() for i in soup.find_all('span',{'class':'t4'})[1:]]
    times = [i.get_text() for i in soup.find_all('span',{'class':'t5'})[1:]]
    for occupation,company,region,salary,time in zip(occupations,companys,regions,salarys,times):
        occupation_infos.append([occupation,company,region,salary,time])       

infos_copy = pd.DataFrame(occupation_infos)
infos_copy.columns = ['职位','公司','工作地点','薪水','发布时间']
infos = infos_copy.copy()
infos.to_csv('occupation.csv')#数据备份到本地电脑

原始数据

#数据清洗
infos['工作地点'] = infos['工作地点'].map(lambda x:x.split('-')[0])
infos = infos[(infos['职位'].str.contains('挖掘')) | (infos['职位'].str.contains('数据'))]#剔除那些非数据挖掘的岗位
work_num_less5 = pd.value_counts(infos['工作地点'])[pd.value_counts(infos['工作地点']) <= 20].index#招聘职位少于20个地点
infos = infos[infos['工作地点'].map(lambda x: True if x not in work_num_less5 else False)] #剔除掉那些招聘少于20个工作地点
infos = infos[infos['薪水'].map(lambda x: True if len(x) != 0 else False)]#剔除那些没有注明薪水的岗位
def clean_salary(salary):
    '''该函数将薪水统一为数字，有很薪水是年薪，天薪，薪水格式很不规整'''
    if salary[-1] == '年':
        if salary[-3] == '万':
            salary = np.mean([float(i) * 10000 / 12 for i in salary[:len(salary)-3].split('-')])
        else:
            salary = np.mean([float(x)*10000 / 12 for x in re.findall('[0-9.]+',salary)])#正则匹配‘15万以上/年’这样的薪水格式
    elif salary[-1] == '月': 
        if salary[-3] == '万':
            salary = np.mean([float(i) * 10000  for i in salary[:len(salary)-3].split('-')])
        elif salary[-3] == '千':
            salary = np.mean([float(i) * 1000  for i in salary[:len(salary)-3].split('-')])
        else:salary = np.mean([float(x) for x in re.findall('[0-9.]+',salary)])
    elif salary[-1] == '天':
        salary = np.mean([float(i) * 30  for i in salary[:len(salary)-3].split('-')])
    return float(salary)
infos['薪水'] = infos['薪水'].map(clean_salary)#统一转化薪水为数字

看看数据清洗后的效果

#数据可视化
plt.rcParams['font.sans-serif']=['SimHei']
pd.value_counts(infos['工作地点']).plot(kind='bar',figsize=(12,7),fontsize=12)
for x,y in enumerate(pd.value_counts(infos['工作地点']).values):
    plt.text(x,y+1.5,'{}'.format(y),va='bottom',ha='center')
plt.ylabel('招聘岗位数',fontsize=15)
plt.xlabel('工作地点',fontsize=15)
plt.title('数据挖掘各个城市分布',fontsize=20)
plt.legend()
plt.show()

#将工作岗位很少的地点，归为一类，用其他表示
foo = pd.value_counts(infos['工作地点'])[pd.value_counts(infos['工作地点']) > 100]
other = pd.value_counts(infos['工作地点'])[pd.value_counts(infos['工作地点']) <= 100].sum()
add = pd.Series(other,index=['其他'])
foo_change = pd.concat([foo,add],axis=0).sort_values(ascending=False)
#饼图可视化
plt.figure(figsize=(6,6))
explode = [0]*len(foo_change)
explode[list(foo_change.index).index('武汉')] = 0.1#突出显示武汉地区岗位比例
plt.pie(foo_change,autopct='%0.1f%%',labels=foo_change.index,
        startangle=90,pctdistance=0.7,labeldistance=1.05,
       explode = explode,wedgeprops={'edgecolor':'w'},#让小饼边缘为白色
       textprops= {'fontsize':12})
#pctdistance文本里圆心距离，autopct文本格式化方式，startangle圆的起始角度,labeldistance控制标签距离,wedgeprops控制每个小饼参数
plt.axis('equal')#让饼图是圆的
plt.show()

salary_status = infos.groupby('工作地点').mean().sort_values('薪水')
salary_status.plot(kind='barh',figsize=(12,7),fontsize=12,color='m',label=[])
plt.xlabel('薪水',fontsize=15)
plt.ylabel('工作地点',fontsize=15)
plt.title('数据挖掘各城市薪水水平',fontsize=18)
for x,y in enumerate(salary_status.values):
    plt.text(y+400,x,'{}'.format(int(y)),va='center',ha='center',fontsize=10)
plt.show()