python爬虫爬取腾讯招聘网的职位招聘数据(个人学习记录)
很常规的爬取数据然后数据分析,重要部分是在url的分析上,no picture say a j8
下图为该url的分析:
后期如果需要更快的爬取更多数据,可以加多线程或者修改参数pagesize
import requests,re,json
"""这里URL可以调参,index=页数,pagesize=一页能爬下来的份数,这里我默认两百份"""
url = "https://careers.tencent.com/tencentcareer/api/post/Query?timestamp=1621962191550&countryId=&cityId=&bgIds=&productId=&categoryId=&parentCategoryId=&attrId=&keyword=&pageIndex=1&pageSize=200&language=zh-cn&area=cn"
headers = {
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:65.0) Gecko/20100101 Firefox/65.0",
}
fp1 = open('data_json.json', 'w', encoding='utf-8')
def parse_json(url):
"""解析url,得到文本数据"""
response = requests.get(url=url, headers=headers)
response.close()
return response.text
def data_analy(r): #对数据进行分析
pat = re.compile(r'"PostId":"(?P<postid>.*?)","RecruitPostId":.*?,"RecruitPostName":"(?P<position>.*?)","CountryName":"(?P<country>.*?)","LocationName":"(?P<city>.*?)","BGName":".*?","ProductName":".*?","CategoryName":".*?","Responsibility":"(?P<introduction>.*?)","LastUpdateTime":"(?P<time>.*?)","PostURL":"(?P<posturl>.*?)","',re.S)
data_tuple_value = re.findall(pattern=pat,string=r)
# data_tuple_gzyq = re.
data_list_keys = ['职业名称','国家','城市','工作介绍','工作要求','发布日期','提交申请url']
for i in data_tuple_value:
data_list_value = list(i)
data_postid = data_list_value[0]
del data_list_value[0]
postid_url = f'https://careers.tencent.com/tencentcareer/api/post/ByPostId?timestamp=1622010434223&postId={data_postid}&language=zh-cn'
r = requests.get(url = postid_url, headers=headers)
r.encoding = r.apparent_encoding
r1 = r.text.replace(r'},', '\n')
pat1 = re.compile(r',"Requirement":"(?P<gzyq>.*?)",')
data_require = re.findall(pat1, r1)
data_require[0] = data_require[0].replace(r'\n','')
data_list_value.insert(4,f'{data_require[0]}')
data_list_value[0] = data_list_value[0].split('-')[1]
data_list_value[3] = data_list_value[3].replace(r'\n','')
data_dict = dict(zip(data_list_keys,data_list_value))
print(data_dict)
r.close()
fp1.write(json.dumps(data_dict, ensure_ascii=False,indent=4,separators=(',',':')))
fp1.close()
if __name__ == '__main__':
r = parse_json(url).replace(r'},','\n')
data_analy(r)
print('爬取完成')
数据已经转为json文件,后期也可以数据可视化