大数据毕业设计Python+Vue(1)

最新推荐文章于 2024-05-21 15:39:16 发布
2401_84185626
最新推荐文章于 2024-05-21 15:39:16 发布
阅读量312
点赞数 3
分类专栏： 2024年程序员学习文章标签：课程设计 python vue.js
本文链接：https://blog.csdn.net/2401_84185626/article/details/137673383
版权
2024年程序员学习专栏收录该内容
46 篇文章 0 订阅
订阅专栏
核心代码分享如下：
import requests
import math
import time
import pandas as pd
import pymysql
from sqlalchemy import create_engine

def get_json(url, num):
    """
    从指定的url中通过requests请求携带请求头和请求体获取网页中的信息,
    :return:
    """
    #url1 = 'https://www.lagou.com/jobs/list_python/p-city_0?&cl=false&fromSearch=true&labelWords=&suginput='

    #####这个版本失效了
    # headers = {
    #     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36',
    #     'Host': 'www.lagou.com',
    #     'Referer': 'https://www.lagou.com/jobs/list_%E6%95%B0%E6%8D%AE%E5%88%86%E6%9E%90?labelWords=&fromSearch=true&suginput=',
    #     'X-Anit-Forge-Code': '0',
    #     'X-Anit-Forge-Token': 'None',
    #     'X-Requested-With': 'XMLHttpRequest',
    #    'Cookie': 'RECOMMEND_TIP=true; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%226450984%22%2C%22first_id%22%3A%22177e31c5c2f15a-0c263f345570168-1a367740-2073600-177e31c5c30426%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%2C%22%24os%22%3A%22Windows%22%2C%22%24browser%22%3A%22Firefox%22%2C%22%24browser_version%2…5f272cc3de2d-1b317140-2073600-17abeb169a41e2; LGSID=20210719201350-62445703-015e-4d7a-a521-9f9bb0264588; PRE_UTM=; PRE_HOST=; PRE_SITE=https%3A%2F%2Fwww.lagou.com; PRE_LAND=https%3A%2F%2Fwww.lagou.com%2F; LGRID=20210719201415-b14c318e-9024-422f-9f6f-eb317e8286f2; X_HTTP_TOKEN=5d3e2971ace082724586966261575a63c69b736c62; sensorsdata2015session=%7B%7D; _gat=1; Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1626696961; privacyPolicyPopup=false; _gid=GA1.2.1558591878.1626696939; index_location_city=%E5%85%A8%E5%9B%BD'
    # }
    # data = {
    #     'first': 'true',
    #     'pn': num,
    #     'kd': 'java'}
    ####这个版本失效了
    headers = {
        'authority': 'www.lagou.com',
        'sec-ch-ua': '^\\^',
        'accept': 'application/json, text/plain, */*',
        'x-anit-forge-code': 'a0555ed5-e147-492a-9d37-4fd12f268e70',
        'x-anit-forge-token': 'ab9e04ab-5b2c-499c-b8c3-1bf534b9be6b',
        'sec-ch-ua-mobile': '?0',
        'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
        'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
        'origin': 'https://www.lagou.com',
        'sec-fetch-site': 'same-origin',
        'sec-fetch-mode': 'cors',
        'sec-fetch-dest': 'empty',
        'referer': 'https://www.lagou.com/wn/jobs?cl=false&fromSearch=true&kd=vue',
        'accept-language': 'zh-CN,zh;q=0.9',
        #'cookie': 'RECOMMEND_TIP=true; user_trace_token=20210511084946-49c14b46-cb33-468c-8b8d-6857a3b2cfad; LGUID=20210511084946-f54c5440-e4bb-4554-83dd-47eec71f4e4c; _ga=GA1.2.835590744.1620694855; LG_HAS_LOGIN=1; JSESSIONID=ABAAAECABFAACEA84AC6A8F75C6A6FFBD5EA9A9A9CB6B59; WEBTJ-ID=20210719^%^E4^%^B8^%^8B^%^E5^%^8D^%^889:47:41214741-17abf05bc12949-015c51fd4d2569-3c3b5a0c-2073600-17abf05bc13807; privacyPolicyPopup=false; PRE_UTM=; PRE_HOST=; PRE_LAND=https^%^3A^%^2F^%^2Fwww.lagou.com^%^2F; LGSID=20210719214556-b1234dd2-6093-411c-902d-86332aad76c3; PRE_SITE=https^%^3A^%^2F^%^2Fwww.lagou.com; _gid=GA1.2.1678999665.1626702463; Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1626702463; sensorsdata2015session=^%^7B^%^7D; gate_login_token=8c829f21d23255c4cb8072f0fbc3487b9556175cfcb080c3; LG_LOGIN_USER_ID=911d6e241321d624916b73f87f04b5edd9d49117442d456a; _putrc=C6B36599E0947B4D; login=true; unick=^%^E7^%^A8^%^8B^%^E5^%^BA^%^8F^%^E8^%^AE^%^BE^%^E8^%^AE^%^A1^%^E5^%^A4^%^A7^%^E7^%^A5^%^9E; showExpriedIndex=1; showExpriedCompanyHome=1; showExpriedMyPublish=1; hasDeliver=23; __SAFETY_CLOSE_TIME__6450984=1; TG-TRACK-CODE=index_zhaopin; __lg_stoken__=37a7c24cd1874978bffc975d898d749028f9e5c3a0bdbca77087876857a9521e44600a3ad2fb4b4204b2c64fc9879640533b584bc4b4759057fa010837776d54882ef9162b5b; _gat=1; sensorsdata2015jssdkcross=^%^7B^%^22distinct_id^%^22^%^3A^%^226450984^%^22^%^2C^%^22first_id^%^22^%^3A^%^2217958f0ee0783d-0af05473790d7d-5b1e341a-2073600-17958f0ee08827^%^22^%^2C^%^22props^%^22^%^3A^%^7B^%^22^%^24latest_traffic_source_type^%^22^%^3A^%^22^%^E7^%^9B^%^B4^%^E6^%^8E^%^A5^%^E6^%^B5^%^81^%^E9^%^87^%^8F^%^22^%^2C^%^22^%^24latest_search_keyword^%^22^%^3A^%^22^%^E6^%^9C^%^AA^%^E5^%^8F^%^96^%^E5^%^88^%^B0^%^E5^%^80^%^BC_^%^E7^%^9B^%^B4^%^E6^%^8E^%^A5^%^E6^%^89^%^93^%^E5^%^BC^%^80^%^22^%^2C^%^22^%^24latest_referrer^%^22^%^3A^%^22^%^22^%^2C^%^22^%^24os^%^22^%^3A^%^22Windows^%^22^%^2C^%^22^%^24browser^%^22^%^3A^%^22Chrome^%^22^%^2C^%^22^%^24browser_version^%^22^%^3A^%^2291.0.4472.124^%^22^%^2C^%^22^%^24latest_referrer_host^%^22^%^3A^%^22^%^22^%^7D^%^2C^%^22^%^24device_id^%^22^%^3A^%^2217958f0ee0783d-0af05473790d7d-5b1e341a-2073600-17958f0ee08827^%^22^%^7D; SEARCH_ID=33822bfd83af4044bbdf5e739c357f57; index_location_city=^%^E5^%^8C^%^97^%^E4^%^BA^%^AC; X_HTTP_TOKEN=67bffba17c882ded9023076261e4738d3a918ca7a3; Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1626703316; LGRID=20210719220010-d8e08ce7-371c-4b2c-b461-35942f573b00',
        'cookie': 'user_trace_token=20210106160617-423191fc-458b-44fe-a8d4-6bcccac0c5ba; LGUID=20210106160617-9b12ac55-f84c-4195-b691-b39f94389871; _ga=GA1.2.1294001968.1609920726; LG_HAS_LOGIN=1; JSESSIONID=ABAAAECAAEBABII7F996E3B4F20C4F3E00655E9AF9448B2; WEBTJ-ID=20220301205653-17f458e15994ee-0bcd280e7a1733-45410429-2073600-17f458e159a5af; RECOMMEND_TIP=true; PRE_UTM=; PRE_HOST=; PRE_LAND=https%3A%2F%2Fwww.lagou.com%2F; LGSID=20220301205610-fa925193-4544-4c10-992d-f341af133457; PRE_SITE=https%3A%2F%2Fwww.lagou.com; _gat=1; Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1646139415; privacyPolicyPopup=false; Hm_lvt_6cda846867fd596b2ca7a177bac2039f=1646139415; _gid=GA1.2.130392468.1646139415; sensorsdata2015session=%7B%7D; gate_login_token=501d64411543dc5180384441ec6f2b3c0971ea98b71302fb; LG_LOGIN_USER_ID=31069c23673a9ef3bab6d9eb9a66df67806003fa990368de; _putrc=C6B36599E0947B4D; login=true; unick=%E7%A8%8B%E5%BA%8F%E8%AE%BE%E8%AE%A1%E5%A4%A7%E7%A5%9E; showExpriedIndex=1; showExpriedCompanyHome=1; showExpriedMyPublish=1; hasDeliver=23; index_location_city=%E5%8C%97%E4%BA%AC; X_HTTP_TOKEN=198c720d9261a486104931646181c7768a0355c84d; Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1646139446; __SAFETY_CLOSE_TIME__6450984=1; TG-TRACK-CODE=index_navigation; LGRID=20220301210234-5378f82f-1a00-45e8-9c72-2a4f56c38703; __lg_stoken__=2d2f31fbf63b61c798174fca433249e77f60c563743eaf7e2c99bfdafe67ec84923b11fe1ef3827e3e39accee822d731c33d57f64cae4d7c7c6b0b599307cc3167ce2f0908e4; Hm_lpvt_6cda846867fd596b2ca7a177bac2039f=1646139801; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%226450984%22%2C%22first_id%22%3A%22176d6c0c3fb310-0442506695abbf-45410429-2073600-176d6c0c3fc21c%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%2C%22%24os%22%3A%22Windows%22%2C%22%24browser%22%3A%22Chrome%22%2C%22%24browser_version%22%3A%2286.0.4240.198%22%7D%2C%22%24device_id%22%3A%22176d6c0c3fb310-0442506695abbf-45410429-2073600-176d6c0c3fc21c%22%7D',
    }

    data = {
        'first': 'true',
        'needAddtionalResult': 'false',
        'city': '^%^E5^%^85^%^A8^%^E5^%^9B^%^BD',
        'px': 'new',
        'pn': '2',
        'cl': 'false',
        'fromSearch': 'true',
        'kd': 'vue'
    }
    # s = requests.Session()
    # print('建立session：', s, '\n\n')
    # s.get(url=url1, headers=headers, timeout=3)
    # cookie = s.cookies
    # print('获取cookie：', cookie, '\n\n')
    #res = requests.post(url, headers=headers, data=data, timeout=3)
    res = requests.post('https://www.lagou.com/jobs/v2/positionAjax.json', headers=headers, data=data)
    res.raise_for_status()
    res.encoding = 'utf-8'
    page_data = res.json()
    print('请求响应结果：', page_data, '\n\n')
    return page_data


def get_page_num(count):
    """
    计算要抓取的页数，通过在拉勾网输入关键字信息，可以发现最多显示30页信息,每页最多显示15个职位信息
    :return:
    """
    page_num = math.ceil(count / 15)
    if page_num > 100:
        return 100
    else:
        return page_num


def get_page_info(jobs_list):
    """
    获取职位
    :param jobs_list:
    :return:
    """
    page_info_list = []
    for i in jobs_list:  # 循环每一页所有职位信息
        job_info = []
        job_info.append(i['companyFullName'])
        job_info.append(i['companyShortName'])
        job_info.append(i['companySize'])
        job_info.append(i['financeStage'])
        job_info.append(i['district'])
        job_info.append(i['positionName'])
        job_info.append(i['workYear'])
        job_info.append(i['education'])
        job_info.append(i['salary'])
        job_info.append(i['positionAdvantage'])
        job_info.append(i['industryField'])
        job_info.append(i['firstType'])
        job_info.append(",".join(i['companyLabelList']))
        job_info.append(i['secondType'])
        job_info.append(i['city'])
        page_info_list.append(job_info)
    return page_info_list

def unique(old_list):
    newList = []
    for x in old_list:
        if x not in newList :
            newList.append(x)
    return newList

def main():
    connect_info = 'mysql+pymysql://{}:{}@{}:{}/{}?charset=utf8'.format("root", "123456", "localhost", "3366",


**自我介绍一下，小编13年上海交大毕业，曾经在小公司待过，也去过华为、OPPO等大厂，18年进入阿里一直到现在。**

**深知大多数大数据工程师，想要提升技能，往往是自己摸索成长或者是报班学习，但对于培训机构动则几千的学费，着实压力不小。自己不成体系的自学效果低效又漫长，而且极易碰到天花板技术停滞不前！**

**因此收集整理了一份《2024年大数据全套学习资料》，初衷也很简单，就是希望能够帮助到想自学提升又不知道该从何学起的朋友。**
![img](https://img-blog.csdnimg.cn/img_convert/d339c40ffa27b1d86919d91bc0d78d9c.png)
![img](https://img-blog.csdnimg.cn/img_convert/862c9d55869ba13dfda751adf4e82916.png)
![img](https://img-blog.csdnimg.cn/img_convert/17a1ab57e3ccbac16b4252b5015ea471.png)
![img](https://img-blog.csdnimg.cn/img_convert/a7516e296e00cb09b7abc12f712c66e0.png)
![img](https://img-blog.csdnimg.cn/img_convert/56919144fffa46d3795f10f71b200393.png)

**既有适合小白学习的零基础资料，也有适合3年以上经验的小伙伴深入学习提升的进阶课程，基本涵盖了95%以上大数据开发知识点，真正体系化！**

**由于文件比较大，这里只是将部分目录大纲截图出来，每个节点里面都包含大厂面经、学习笔记、源码讲义、实战项目、讲解视频，并且后续会持续更新**

**如果你觉得这些内容对你有帮助，可以添加VX：vip204888 （备注大数据获取）**
![img](https://img-blog.csdnimg.cn/img_convert/1b7cfe7845451a7c9a9099af4ec5a273.png)

纲截图出来，每个节点里面都包含大厂面经、学习笔记、源码讲义、实战项目、讲解视频，并且后续会持续更新**

**如果你觉得这些内容对你有帮助，可以添加VX：vip204888 （备注大数据获取）**
[外链图片转存中...(img-i42hsf3i-1712891181983)]