Python爬前程无忧 python职位情况

前几天有人在jb51net发布了python爬python开发现状，进一步查看时发现代码不完整，本人业余喜欢看下python,想着也学了这么久好像从没有完整来个一遍，于是尝试安那个大哥思路自己爬一遍看行不行，于是开始尝试，这次爬的是前程无忧的。直接上代码：写的有点丑，但是能用啊！
（这次爬的全国python职位 爬了200页 也有7千多个）

import urllib.request
from urllib.parse import urlencode
import time
import pandas as pd
from bs4 import BeautifulSoup
def get_content(i):
url0 =‘https://search.51job.com/list/000000,000000,0000,00,9,99,python,2,’+str(i)+’.html?lang=c&
stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&
providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&
address=&line=&specialarea=00&from=&welfare=’
headers = {
‘Host’: ‘search.51job.com’,
‘Referer’: ‘https://mkt.51job.com/tg/sem/pz_2018.html?from=baidupz’,
‘Upgrade-Insecure-Requests’: ‘1’,
‘User-Agent’:‘Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36’,
}
try:
req = urllib.request.Request(url0,headers=headers)
resp = urllib.request.urlopen(req)
return resp
except :
raise

def parser_content():
search_all_job =[]
for i in range(1,200):
html1 = get_content(i)
soup = BeautifulSoup(html1,“lxml”)
# 第一个没有引号
#print(soup)
cont = soup.find_all(‘div’,class_=“el”)
#print(cont)
total_job =[]
print(“开始爬第%d页：” % i)
for j in cont:
job = []
try:
jobname = j.find(‘p’,class_=“t1”).find(‘span’).find(‘a’)[‘title’]
#print(jobname)
job.append(jobname)
except:
jobname=None
try:
companyname = j.find(‘span’,class_=“t2”).find(‘a’).get_text()
#print(companyname)
job.append(companyname)
except:
companyame=None
#job.append(companyname)
try:
address = j.find(‘span’,class_=“t3”).get_text()
#print(address)
job.append(address)
except:
address=None
try:
salary = j.find(‘span’,class_=“t4”).get_text()
#print(salary)
job.append(salary)
except:
salary=None
try:
date = j.find(‘span’,class_=“t5”).get_text()
#print(date)
job.append(date)
except:
date=None
if len(job)!=0:
total_job.append(job)
#print(total_job)
else:
continue
search_all_job +=total_job
print(“第%d页爬完!” % i)
df = pd.DataFrame(data=search_all_job,
columns =[‘职位’,‘公司名字’,‘上班地点’,‘薪水’,‘发布日期’])
df.to_csv(‘前程无忧.csv’,index = False,encoding=‘utf-8_sig’)

if name ==‘main’:
parser_content()

好，数据已经爬好了该清理没用的数据了 如图：

直接给那些空着删除掉，发现中文名字有时候会乱码，我直接改了个名字，重新建了个py：
import pandas as pd
import os
import xlrd
import time
import re

data = pd.read_excel(‘noworry.xlsx’)
data.dropna(inplace=True)
data.to_excel(‘again.xls’)
现在已经把空格都去掉了，接着看薪资的大概分布吧
data = pd.read_excel(‘again.xls’)
data[‘salary’]=data[‘薪资’].str.findall(’[\d+.\d]+’)
#这个正则和那个不一样，和拉钩网站get的形式也不是一样，打开之后才发现
data.dropna(inplace=True)

[我以为这个正则表达式会匹配不到这种150/天的，以为会使这一行data[‘salary’]为空所以打算把空的又删掉，其实它也匹配到了，所以这个没用 也没删它
avg_salary = []
#count=0 在下面发现老是报错，才发现那个150/天的已经匹配上了 于是：
for i,k in enumerate(data[‘salary’]):
if len(k) ==2:
a = float(k[0])
b = float(k[1])
if ‘万’ in data[‘薪资’][i]:
avg = round((a + (b-a)/3)10,2)
else:
avg = round(a + (b-a)/3,2)
if ‘年’ in data[‘薪资’][i]:
avg = round(avg/12,2)
else:
avg = avg
else:
a = float(k[0])
if a>1500:
avg = 40
else:
avg = round(a21.75/1000,2)
avg_salary.append(avg)
‘’’
如果是千，直接相减取前30%
如果是万，取完之后还换成千
接着如果是月薪，不用动
如果不是，换成月薪
否则 是按日结算
每天超过1.5k的直接都当成40K
否则按照正常工资结算
‘’’
接着画直方图
import matplotlib.pyplot as plt
import matplotlib.font_manager as fm

myfont = fm.FontProperties(fname=r"D:\Software\Python\Lib\site-packages\matplotlib\mpl-data\fonts\ttf\chinese.ttf",size=36)
plt.figure(figsize=(8,12))

bins = np.linspace(0,60,16)
plt.hist(avg_salary, bins, histtype=‘bar’, rwidth=0.8)
plt.xlabel(u’区域’,fontproperties=myfont)
plt.ylabel(u’个数’,fontproperties=myfont)
plt.title(u’待遇分布’,fontproperties=myfont)
#plt.legend()
plt.show()
如图：

接着画区域分布饼图
from pylab import *
mpl.rcParams[‘font.sans-serif’] = [‘SimHei’]
mpl.rcParams[‘axes.unicode_minus’] = False
#这次在网上看到这种使用中文标签方法
count = data[‘地点’].values
count1 =[]
for i in range(len(count)):
cont = count[i][:2]
count1.append(cont)
#print(cont)

df = pd.Series(data=count1)
#先转成 pd.Series，才能用下面value_counts()方法
df = df.value_counts()
plt.pie(df,labels=df.keys(),autopct=’%2.1f%%’)
plt.axis(‘equal’)
plt.legend(loc=‘upper left’,bbox_to_anchor=(-0.01,1))
plt.savefig(‘addressjob.jpg’)
plt.show()
如图：

接着词云：
import jieba.analyse
from PIL import Image,ImageSequence
from wordcloud import WordCloud,ImageColorGenerator
words =’’
for line in data[‘职位’]:
words += line
result = jieba.analyse.textrank(words,topK=100,withWeight=True)
keywords = dict()
for i in result:
keywords[i[0]]=i[1]
#print(keywords)
image= Image.open(’./19.PNG’)
graph = np.array(image)
#graph图表 曲线图 2、array数组，数据类型是必须相同
wc = WordCloud(font_path=’./fonts/simhei.ttf’,background_color=‘white’,max_words=100,mask=graph)
wc.generate_from_frequencies(keywords)
image_color = ImageColorGenerator(graph)
plt.imshow(wc)
plt.imshow(wc.recolor(color_func=image_color))
plt.axis(“off”)
wc.to_file(‘python.png’)
plt.show()
如图：