数据分析行业数据爬虫及分析（新手3）

一、爬虫

from selenium import webdriver
import pandas as pd
from pandas import DataFrame
import time
bor= webdriver.Chrome()
def wab(www):
    bor.get(www)
    a1=[]         
    for i in range(90):
        b1=bor.find_elements_by_class_name('contentpile__content__wrapper__item__info__box__job__saray')
        a1.append(b1[i].text)
    a2=[]
    for i in range(90):
        #注意这里用了一个format()   因为每条数据的div{}不一样
        kk1=bor.find_element_by_xpath('//*[@id="listContent"]/div{}/div/a/div[1]/div[2]/a'.format([i+1]))
        a2.append( kk1.text)
    a3=[]
    for i in range(90):
        b1=bor.find_elements_by_class_name('contentpile__content__wrapper__item__info__box__job__demand')
        a3.append(b1[i].text)  #这里原来是list格式用b1[i] 提取原格式
  
    for i in range(180):
        b1=bor.find_elements_by_class_name('contentpile__content__wrapper__item__info__box__welfare__item')
        a4.append(b1[i].text)
    a5=[]
    for i in range(180):
        b1=bor.find_elements_by_class_name('contentpile__content__wrapper__item__info__box__job__comdec__item')
        a5.append(b1[i].text)
    a6=a5[::2]       #分割a5为a6和a7
    a7=a5[1::2]
    bb=DataFrame()
    bb['工资']=a1
    bb['公司']=a2
    bb['要求']=a3
    bb['公司性质']=a6
    bb['规模']=a7
   
    return bb
def page(n)
gg2=DataFrame()
for j in range(n): 
    try:
        mm='https://sou.zhaopin.com/?p={}&jl=854&sf=0&st=0&kw=数据分析&kt=3'.format(j+1)
        #print(aaaa) 
        df_page=wab(mm)
        gg2=gg2.append(df_page,ignore_index=True)
    except:
        pass
    return gg2
gg2.to_csv("E:\papapappapaap\西安数据.csv")   #保存数据
gg2.to_csv("E:\papapappapaap\北京数据.csv")

二、数据分析

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
data1=pd.read_csv("E:\papapappapaap\西安数据.csv")
data2=pd.read_csv("E:\papapappapaap\北京数据.csv")     #查看数据
data1.describe()
data2.info()
data1.head()
def palce(x,y):    # X='地址','要求'，'学历'位置(y)分别为0，1，2
    a4=data2['要求']
    a5=[]
    for i in a4:

        b2=i.split()     #切分数据data2['要求']
        a5.append(b2[y])
    data2[x]=a5
    return data2[x]

palce('要求',1)
data1.工资.unique()
data2.工资.unique(
data2.工资.replace(['6K-8K', '15K-25K', '15K-30K', '10K-20K', '薪资面议', '8K-15K',
       '10K-12K', '10K-15K', '15K-20K', '20K-30K', '10K-18K', '2K-4K',
       '8K-10K', '6K-12K', '6K-10K', '8K-16K', '5K-8K', '12K-24K',
       '15K-28K', '7K-10K', '5K-7K', '30K-50K', '12K-15K', '1K-2K',
       '15K-22K', '4K-6K', '9K-15K', '7K-14K'],[7000,20000,23000,15000,9000,12000,
                                               11000,13000,17000,25000,14000,3000,
                                               9000,9000,8000,12000,6600,18000,
                                               23000,8500,6000,100000,12500,1500,
                                               17000,5000,12000,10000],inplace=True)
                                             	    
data1.工资.replace(['4K-8K', '2K-4K', '6K-8K', '4K-6K', '5K-8K', '10K-18K', '10K-20K',
       '4K-4.5K', '5K-7K', '3K-4.5K', '8K-10K', '6K-10K', '8K-16K',
       '15K-25K', '20K-25K', '4K-7K', '3K-6K', '3K-5K', '3.5K-5K',
       '4.5K-7K', '5K-10K', '4.5K-6K', '6K-12K', '10K-15K', '薪资面议',
       '20K-35K', '2.6K-3K', '3K-4K', '10K-12K', '8K-12K', '8K-15K',
       '4K-5K', '3.5K-5.5K', '7K-9K', '5K-9K', '2K-3.5K', '7.5K-12K',
      '4.5K-5K', '3.3K-4.5K', '7K-10K', '12K-18K', '8K-13K', '3.3K-3.5K',
       '6.5K-10K', '12K-20K', '9K-16K', '7K-8K', '1.9K-2K', '3.5K-4K',
       '30K-50K', '3.5K-4.5K', '2.5K-3K', '15K-20K', '5K-6.5K', '12K-15K',
       '6K-11K', '25K-35K', '4.5K-5.5K', '3.5K-6K', '7K-14K', '20K-30K',
       '5.5K-11K', '8K-11K', '25K-30K', '2K-3K', '6K-8.5K', '5K-6K',
       '2.8K-4K', '6K-7K', '15K-30K', '14K-22K', '15K-22K', '3.8K-5K',
       '3.8K-4.5K', '3.5K-4.8K', '2K-2.5K', '25K-40K', '18K-30K',
       '7K-12K', '100K-150K', '2.5K-4K'],[6000,3000,7000,5000,6500,14000,15000,
                                         4250,6000,4800,9000,8000,12000,
                                         20000,22500,5500,4500,4000,4200,
                                         5700,7500,5200,9000,12500,9800,
                                         27500,2800,3500,11000,10000,11500,
                                         4500,4500,8000,7000,2750,9500,
                                         4750,3800,8500,15000,10500,3400,
                                         8300,16000,12500,7500,2000,3750,
                                        40000,3500,2700,17500,5800,13500,
                                         8500,30000,5000,4700,10500,25000,
                                         7700,9100,27500,2500,7000,5500,
                                          
                                          3500,6500,22500,18000,4000,
                                          4000,4000,2250,3000,32000,24000,
                                          
                                          9500,0,3000],inplace=True)


data1.地址.value_counts()
data2.地址.value_counts()
nnn=data2.groupby(['要求']).工资.mean()
nn=data1.groupby(['要求']).工资.mean()