2021-05-26

python爬虫爬取腾讯招聘网的职位招聘数据（个人学习记录）

很常规的爬取数据然后数据分析，重要部分是在url的分析上，no picture say a j8

下图为该url的分析：

 后期如果需要更快的爬取更多数据，可以加多线程或者修改参数pagesize

import requests,re,json
"""这里URL可以调参，index=页数，pagesize=一页能爬下来的份数，这里我默认两百份"""
url = "https://careers.tencent.com/tencentcareer/api/post/Query?timestamp=1621962191550&countryId=&cityId=&bgIds=&productId=&categoryId=&parentCategoryId=&attrId=&keyword=&pageIndex=1&pageSize=200&language=zh-cn&area=cn"

headers = {
    "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:65.0) Gecko/20100101 Firefox/65.0",
}
fp1 = open('data_json.json', 'w', encoding='utf-8')

def parse_json(url):
    """解析url，得到文本数据"""
    response = requests.get(url=url, headers=headers)
    response.close()
    return response.text



def data_analy(r):                  #对数据进行分析
    pat = re.compile(r'"PostId":"(?P<postid>.*?)","RecruitPostId":.*?,"RecruitPostName":"(?P<position>.*?)","CountryName":"(?P<country>.*?)","LocationName":"(?P<city>.*?)","BGName":".*?","ProductName":".*?","CategoryName":".*?","Responsibility":"(?P<introduction>.*?)","LastUpdateTime":"(?P<time>.*?)","PostURL":"(?P<posturl>.*?)","',re.S)
    data_tuple_value = re.findall(pattern=pat,string=r)
    # data_tuple_gzyq = re.
    data_list_keys = ['职业名称','国家','城市','工作介绍','工作要求','发布日期','提交申请url']
    for i in data_tuple_value:
        data_list_value = list(i)
        data_postid = data_list_value[0]
        del data_list_value[0]
        postid_url = f'https://careers.tencent.com/tencentcareer/api/post/ByPostId?timestamp=1622010434223&postId={data_postid}&language=zh-cn'
        r = requests.get(url = postid_url, headers=headers)
        r.encoding = r.apparent_encoding
        r1 = r.text.replace(r'},', '\n')
        pat1 = re.compile(r',"Requirement":"(?P<gzyq>.*?)",')
        data_require = re.findall(pat1, r1)
        data_require[0] = data_require[0].replace(r'\n','')
        data_list_value.insert(4,f'{data_require[0]}')
        data_list_value[0] = data_list_value[0].split('-')[1]
        data_list_value[3] = data_list_value[3].replace(r'\n','')
        data_dict = dict(zip(data_list_keys,data_list_value))
        print(data_dict)
        r.close()
        fp1.write(json.dumps(data_dict, ensure_ascii=False,indent=4,separators=(',',':')))
    fp1.close()


if __name__ == '__main__':
    r = parse_json(url).replace(r'},','\n')
    data_analy(r)
    print('爬取完成')

 

数据已经转为json文件，后期也可以数据可视化