行业类别数据爬取和uniapp多级数据联动（一）

最新推荐文章于 2024-06-01 10:35:00 发布

何忧非喜{傲娇的程序媛}

最新推荐文章于 2024-06-01 10:35:00 发布

阅读量625

点赞数

分类专栏：日常文章标签： python

本文链接：https://blog.csdn.net/u013252962/article/details/120702063

版权

原始类别数据参考51job的，如有侵权，告知即可删除
一级类目https://js.51jobcdn.com/in/js/2016/layer/layer_main_navigation.js?20210907
二级和三级泪目https://js.51jobcdn.com/in/js/2016/layer/funtype_array_c.js?20210907

数据采集-行业

# -*- coding: utf-8 -*-
import pymysql
"""
批量数据入库
"""

db = pymysql.connect(host='127.0.0.1', port=3306, user='root', password='root', db='test', charset='utf8mb4')

cursor = db.cursor()

# 行业一级数据
aITN = [
    {
   'c':'计算机/互联网/通信/电子','e':'Computer, Internet, Telecom, Electronics','nav':'01','category':['01','37','38','31','39','32','40','02','35']},
    {
   'c':'会计/金融/银行/保险','e':'Accounting, Finance, Banking, Insurance','nav':'41','category':['41','03','42','43','62']},
    {
   'c':'贸易/消费/制造/营运','e':'Trade,Sales, Manufacturing, Operations','nav':'04','category':['04','22','05','06','44','60','45','14','33','65']},
    {
   'c':'制药/医疗','e':'Pharmaceuticals, Healthcare','nav':'08','category':['08','46','47']},
    {
   'c':'广告/媒体','e':'Advertising, Media Related','nav':'12','category':['12','48','49','13','15']},
    {
   'c':'房地产/建筑','e':'Real Estates Related','nav':'26','category':['26','09','50','51','34','63']},
    {
   'c':'专业服务/教育/培训','e':'Professional Services, Education, Training','nav':'07','category':['07','59','52','18','23','24']},
    {
   'c':'服务业','e':'Customer Services','nav':'11','category':['11','53','17','54','27']},
    {
   'c':'物流/运输','e':'Logistics, Transportation','nav':'21','category':['21','55']},
    {
   'c':'能源/环保/化工','e':'Utilities, Environmental Protection, Chemical','nav':'19','category':['19','16','36','61','56','20']},
    {
   'c':'政府/非营利组织/其他','e':'Government, Non Profit, Others','nav':'28','category':['28','57','29','58']}
]
sql = "INSERT INTO category_industry(name, level, display,nav,pnav) VALUES (%s,%s,%s,%s,%s)"
datasql = []

"""
第一步：行业一级数据入库
"""
for i in aITN:
    tmp = (i['c'],1,1,i['nav'],0)
    datasql.append(tmp)
try:
    cursor.executemany(sql, tuple(datasql))
    db.commit()
except:
    db.rollback()
db.close()
sleep(2000)# 第一步和第二步切换使用
# 行业二级数据
secondeindus={
   
'01':'计算机软件',
'37':'计算机硬件',
'38':'计算机服务(系统、数据服务、维修)',
'31':'通信/电信/网络设备',
'39':'通信/电信运营、增值服务',
'32':'互联网/电子商务',
'40':'网络游戏',
'02':'电子技术/半导体/集成电路',
'35':'仪器仪表/工业自动化',
'41':'会计/审计',
'03':'金融/投资/证券',
'42':'银行',
'43':'保险',
'62':'信托/担保/拍卖/典当',
'04':'贸易/进出口',
'22':'批发/零售',
'05':'快速消费品(食品、饮料、化妆品)',
'06':'服装/纺织/皮革',
'44':'家具/家电/玩具/礼品',
'60':'奢侈品/收藏品/工艺品/珠宝',
'45':'办公用品及设备',
'14':'机械/设备/重工',
'33':'汽车',
'65':'汽车零配件',
'08':'制药/生物工程',
'46':'医疗/护理/卫生',
'47':'医疗设备/器械',
'12':'广告',
'48':'公关/市场推广/会展',
'49':'影视/媒体/艺术/文化传播',
'13':'文字媒体/出版',
'15':'印刷/包装/造纸',
'26':'房地产',
'09':'建筑/建材/工程',
'50':'家居/室内设计/装潢',
'51':'物业管理/商业中心',
'34':'中介服务',
'63':'租赁服务',
'07':'专业服务(咨询、人力资源、财会)',
'59':'外包服务',
'52':'检测，认证',
'18':'法律',
'23':'教育/培训/院校',
'24':'学术/科研',
'11':'餐饮业',
'53':'酒店/旅游',
'17':'娱乐/休闲/体育',
'54':'美容/保健',
'27':'生活服务',
'21':'交通/运输/物流',
'55':'航天/航空',
'19':'石油/化工/矿产/地质',
'16':'采掘业/冶炼',
'36':'电气/电力/水利',
'61':'新能源',
'56':'原材料和加工',
'28':'政府/公共事业',
'57':'非营利组织',
'20':'环保',
'29':'农/林/牧/渔',
'58':'多元化业务集团公司'
}
"""
第二步：行业一级和二级关系，二级数据入库
"""
# 行业一级和二级关系
datasql = []
sql = "INSERT INTO category_industry(name, level, display,nav,pnav) VALUES (%s,%s,%s,%s,%s)"
for i in aITN:
    print(i['category'])
    for j in i['category']:
        print(secondeindus[j])
        tmp = (secondeindus[j], 2, 1, j, i['nav'])
        datasql.append(tmp)
print(datasql)
try:
    cursor.executemany(sql, tuple(datasql))
    db.commit()
except:
    db.rollback()
db.close()

sql

CREATE TABLE `category_industry` (
  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
  `name` varchar(30) NOT NULL COMMENT '名称',
  `level` tinyint(1) NOT NULL COMMENT '级别',
  `display` tinyint(1) NOT NULL COMMENT '状态1显示2不显示',
  `created_at` datetime DEFAULT CURRENT_TIMESTAMP,
  `updated_at` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
  `nav` varchar(4) DEFAULT NULL,
  `pnav` varchar(11) DEFAULT NULL,
  PRIMARY KEY (`id`) USING BTREE,
  KEY `search` (`level`,`pnav`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='行业分类'

数据采集-职业

# -*- coding: utf-8 -*-
import pymysql
"""
批量数据入库


"""

db = pymysql.connect(host='127.0.0.1', port=3306, user='root', password='123456', db='social', charset='utf8mb4')

cursor = db.cursor()

# 行业一级数据入库
aFTN=[
{
   'c':'计算机/互联网/通信/电子','e':'Computer,Internet,Telecom,Electronics','nav':'0100','category':['0100','7700','7200','7300','7800','2700','7900','7500','6600','8000','2600','6100','6700','2900','2800']},
{
   'c':'销售/客服','e':'Sales,Customer Service,Technical Support','nav':'0200','category':['0200','3000','3100','3200']},
{
   'c':'会计/金融/银行/保险','e':'Accounting,Finance,Banking,Insurance','nav':'0400','category':['0400','3300','2200','3400']},
{
   'c':'生产/营运/采购/物流','e':'Manufacturing,Operation,Purchasing,Logistics','nav':'3500','category':['3500','3600','0500','6200','6300','7100','5400','5900','3700','3800','0800','3900','4000']},
{
   'c':'建筑/房地产','e':'Construction,Real Estate','nav':'6900','category':['6900','2100','4600','6000','4700']},
{
   'c':'设计/市场/媒体/广告','e':'Advertising, Marketing, Media, Design','nav':'7400','category':['7400','0900','0300','4200','4300','4400','4500']},
{
   'c':'人事/行政/高级管理','e':'HR,Admin.,Senior Management','nav':'0600','category':['0600','2300','0700']},
{
   'c':'教育/培训','e':'Education','nav':'1200','category':['1200','8100','8200','5700']},
{
   'c':'生物/制药/化工/医疗','e':'Biotechnology,Pharmaceuticals,Healthcare','nav':'4100','category':['4100','5500','1300']},
{
   'c':'咨询/法律/科研','e':'Consultant,Legal,Education','nav':'1400','category':['1400','1100','1000']},
{
   'c':'服务业','e':'Service','nav':'4800','category':['4800','4900','5000','5100','1800','5200','6400','6500']},
{
   'c':'政府机构/翻译/其他','e':'Official,Translator,Others','nav':'1500','category':['1500','2000','5600','5800','1600','1700','5300']}
]
sql = "INSERT INTO category_position(name, level, display,nav) VALUES (%s,%s,%s,%s)"
datasql = []
for i in aFTN:
    tmp = (i['c'],1,1,i['nav'])
    datasql.append(tmp)
try:
    cursor.executemany(sql, tuple(datasql))
    db.commit()
except:
    db.rollback()
db.close()

ft={
   
"0100":"后端开发",
"0121":"Java开发工程师",
"0120":"PHP开发工程师",
"0122":"C/C++开发工程师",
"0124":"Python开发工程师",
"0126":".NET开发工程师",
"0153":"C#开发工程师",
"0151":"Ruby开发工程师",
"0152":"Go开发工程师",
"0130":"大数据开发工程师",
"0129":"Hadoop工程师",
"0131":"爬虫开发工程师",
"0132":"脚本开发工程师",
"0133":"多媒体开发工程师",
"0155":"GIS工程师",
"0154":"全栈工程师",
"0117":"ERP技术开发",
"0128":"区块链开发",
"0106":"高级软件工程师",
"0107":"软件工程师",
"0143":"系统架构设计师",
"0123":"系统分析员",
"0149":"技术文员/助理",
"0150":"技术文档工程师",
"0142":"其他",
"7700":"移动开发",
"7701":"Android开发工程师",
"7702":"iOS开发工程师",
"7705":"小程序开发工程师",
"7703":"移动开发工程师",
"7704":"其他",
"7200":"前端开发",
"7201":"Web前端开发",
"7202":"HTML5开发工程师",
"7203":"其他",
"7300":"人工智能