基于Hive的招聘网站的大数据分析系统

1. 数据采集

首先，我们需要通过网络爬虫技术从招聘网站上获取数据。爬虫可以自动地访问网站并抓取所需的数据，例如职位信息、公司信息、薪资水平等。在选择爬虫工具时，需要考虑目标网站的结构和反爬虫机制，确保能够稳定高效地获取数据。

import csv
import time

import requests
import execjs

from storage.csv2mysql import sync_data2db


def read_js_code():
    f= open('/Users/shareit/workspace/chart_show/demo.js',encoding='utf-8')
    txt = f.read()
    js_code = execjs.compile(txt)
    ckId = js_code.call('r',32)
    return ckId
    

def post_data():
    read_js_code()
    url = "https://api-c.liepin.com/api/com.liepin.searchfront4c.pc-search-job"
    headers = {
        'Accept': 'application/json, text/plain, */*',
        'Accept-Encoding': 'gzip, deflate, br'
    }
    list = ["H01$H0001","H01$H0002",
            "H01$H0003","H01$H0004","H01$H0005",
            "H01$H0006","H01$H0007","H01$H0008",
            "H01$H0009","H01$H00010","H02$H0018","H02$H0019","H03$H0022",
            "H03$H0023","H03$H0024","H03$H0025","H04$H0030","H04$H0031",
            "H04$H0032","H05$H05","H06$H06","H07$H07","H08$H08"]
    for name in list:
        print("-------{}---------".format(name))
        for i in range(10):
            print("------------第{}页-----------".format(i))
            data = {"data": {"mainSearchPcConditionForm":
                                 {"city": "410", "dq": "410", "pubTime": "", "currentPage": i, "pageSize": 40, "key": "",
                                  "suggestTag": "", "workYearCode": "1", "compId": "", "compName": "", "compTag": "",
                                  "industry": name, "salary": "", "jobKind": "", "compScale": "", "compKind": "", "compStage": "",
                                  "eduLevel": ""},
                             "passThroughForm":
                                 {"scene": "page", "skId": "z33lm3jhwza7k1xjvcyn8lb8e9ghxx1b",
                                  "fkId": "z33lm3jhwza7k1xjvcyn8lb8e9ghxx1b",
                                  "ckId": read_js_code(),
                                  'sfrom': 'search_job_pc'}}}
            response = requests.post(url=url, json=data, headers=headers)
            time.sleep(2)
            parse_data(response)


def parse_data(response):
    try:
        jobCardList = response.json()['data']['data']['jobCardList']
        sync_data2db(jobCardList)
    except Exception as e:
        return

if __name__ == '__main__':
    post_data()

2. 数据预处理

获取到的原始数据往往杂乱无章，需要进行预处理才能进行后续的分析工作。预处理包括数据清洗、去重、缺失值处理、数据格式转换等环节，以确保数据的质量和一致性。在这一阶段，还可以利用自然语言处理技术对文本数据进行分词、词性标注等操作，为后续的分析提供更多维度的信息。然后将数据加载到hive中进行分析。

CREATE TABLE mydb.data (
  id INT,
  title STRING,
  city STRING,
  salary STRING,
  campus_job_kind STRING,
  labels STRING,
  compName STRING,
  compIndustry STRING,
  compScale STRING
)
COMMENT '数据表'
ROW FORMAT DELIMITED
FIELDS TERMINATED BY ','
STORED AS TEXTFILE;

LOAD DATA INPATH '/file.csv' OVERWRITE INTO TABLE mydb.data;

3. 数据分析

有了清洗和存储好的数据，接下来就是进行数据分析。数据分析的方法多种多样，可以根据具体的需求选择合适的分析技术和模型。常见的数据分析技术包括统计分析、机器学习、文本挖掘等。通过对招聘数据的分析，我们可以发现人才市场的热点行业、热门职位、薪资水平等信息，为企业招聘决策提供参考。

def city_count_from_db():
    tuple = []
    try:
        with connection.cursor() as cursor:
            select_query = "select * from (select city,count(1) cnt FROM data group by city)a limit 10"
            cursor.execute(select_query)
            result = cursor.fetchall()
            for row in result:
                tuple.append((row['city'],row['cnt']))
    except Exception as e:
        print(e)
    return tuple



def salary_avg_from_db():
    x=[]
    y=[]
    try:
        with connection.cursor() as cursor:
            select_query = "select * from (select city,avg(salary) avg FROM data group by city)a limit 20"
            cursor.execute(select_query)
            result = cursor.fetchall()
            for row in result:
                x.append(row['city'])
                y.append(int(row['avg']))
    except Exception as e:
        print(e)
    return x,y


def salary_industry_from_db():
    x=[]
    y=[]
    try:
        with connection.cursor() as cursor:
            select_query = "select * from (select compIndustry,avg(salary) avg FROM data group by compIndustry)a limit 20"
            cursor.execute(select_query)
            result = cursor.fetchall()
            for row in result:
                x.append(row['compIndustry'])
                y.append(int(row['avg']))
    except Exception as e:
        print(e)
    return x,y

def salary_title_from_db():
    x=[]
    y=[]
    try:
        with connection.cursor() as cursor:
            select_query = "select title,count(1) cnt from data group by title order by cnt desc limit 10"
            cursor.execute(select_query)
            result = cursor.fetchall()
            for row in result:
                x.append(row['title'])
                y.append(int(row['cnt']))
    except Exception as e:
        print(e)
    return x,y

def comany_from_db():
    tuple = []
    try:
        with connection.cursor() as cursor:
            select_query = "select compName,count(1) cnt FROM data group by compName order by cnt desc limit 10"
            cursor.execute(select_query)
            result = cursor.fetchall()
            for row in result:
                tuple.append((row['compName'], row['cnt']))
    except Exception as e:
        print(e)
    return tuple

标题4. 数据可视化

最后，为了更直观地展示分析结果，我们利用Django框架搭建了数据可视化的平台。Django是一个高效的Web开发框架，通过它可以快速构建出美观、易用的数据可视化界面。在可视化界面上，我们可以展示招聘数据的各种统计图表、热点地图、词云等，帮助用户更直观地理解数据背后的信息。

def bar_chart(request):
    line = Line()
    x,y=salary_avg()
    line.add_xaxis(x)
    line.add_yaxis("全国应届毕业生就业城市薪资分布图", y)

    line1 = Line().set_global_opts(
        xaxis_opts=opts.AxisOpts(axislabel_opts=opts.LabelOpts(rotate=90)),
    )
    x, y = title_count()
    line1.add_xaxis(x)
    line1.add_yaxis("全国应届毕业生就业岗位分布图", y)


    # 创建条形图
    bar = Bar().set_global_opts(
        xaxis_opts=opts.AxisOpts(axislabel_opts=opts.LabelOpts(rotate=90)),
    )
    x,y = industry_avg()
    bar.add_xaxis(x)
    bar.add_yaxis("全国应届毕业生就业领域薪资分布图", y)

    # 创建饼图
    pie = Pie()
    tuple = city_top()
    pie.add("全国应届毕业生就业城市top10",tuple)

    pie1 = Pie()
    tuple1 = comany_count()
    pie1.add("全国应届毕业生就业公司top10",tuple1)

    # 获取图表的JavaScript代码
    line_js = line.render_embed()
    bar_js = bar.render_embed()
    pie_js = pie.render_embed()
    line1_js = line1.render_embed()
    pie1_js = pie1.render_embed()
    return render(request, 'charts/bar_chart.html', {'line': line_js, 'bar': bar_js, 'pie': pie_js,'line1': line1_js,'pie1': pie1_js})

下面是数据分析的展示结果，喜欢的可以加个收藏点个关注哦，更多毕设相关内容，小编将持续分享哦





如有遇到问题可以找小编沟通交流哦。另外小编帮忙辅导大课作业，学生毕设等。不限于python，java，大数据，模型训练等。