爬取腾讯招聘网-写入Excel

import requests
import jsonpath
import json
import xlwt

def saveData(data_list):
book=xlwt.Workbook(encoding=‘utf-8’,style_compression=0)
sheet=book.add_sheet(‘tengxun2’)
col=(“postid”,“职位名”,“工作地点”,“工作职责”,“工作要求”,“发布时间”,“url”)
for i in range(7):
sheet.write(0,i,col[i])
for i in range(len(data_list)):
data=data_list[i]
print(data)
for j in data:
# print(data)
print(j)
# for j in range(len(data)):C
sheet.write(i+1,0,j[‘id’])
sheet.write(i+1,1,j[‘职位’])
sheet.write(i+1,2,j[‘工作地点’])
sheet.write(i+1,3,j[‘工作职责’])
sheet.write(i+1,4,j[‘工作要求’])
sheet.write(i+1,5,j[‘发布时间’])
sheet.write(i+1,6,j[‘url’])

book.save('tengxun2.xls')

def spider_page(j):
url=‘https://careers.tencent.com/tencentcareer/api/post/ByPostId?timestamp=&postId=’+str(j)+’&language=zh-cn’
response = requests.get(url)
jsonobj = json.loads(response.text)
items = jsonpath.jsonpath(jsonobj, ‘$.Data’)

#print(items)
list1=[]

result={}
result['id']=items[0]['PostId']
result['职位']=items[0]['RecruitPostName']
result['工作地点']=items[0]['LocationName']
result['工作职责']=items[0]['Responsibility']
result['工作要求']=items[0]['Requirement']
result['发布时间']=items[0]['LastUpdateTime']
result['url']=items[0]['PostURL']
list1.append(result)
# print(list1)
#saveData(list1)
return  list1

def spider_url(post,num):

    url='https://careers.tencent.com/tencentcareer/api/post/Query?timestamp=1591689554351&countryId=&cityId=&bgIds=&productId=&categoryId=&parentCategoryId=&attrId=&keyword='+post+'&pageIndex='+str(num)+'&pageSize=10&language=zh-cn&area=cn'
    #url='https://careers.tencent.com/tencentcareer/api/post/Query?timestamp=1591681767428&countryId=&cityId=&bgIds=&productId=&categoryId=&parentCategoryId=40001&attrId=&keyword=&pageIndex='+str(num)+'&pageSize=10&language=zh-cn&area=cn'
    headers={'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36'}
    response=requests.get(url,headers=headers)

    html=response.text

    jsonobj=json.loads(html)

    items=jsonpath.jsonpath(jsonobj,'$.Data.Posts[*]')
    list=[]
    for i  in items:
        postid=i['PostId']
        list.append(postid)
    return  list

if name==‘main’:
data_list=[]
post=str(input(‘输入要查找的职位:’))
num=int(input(‘输入要爬取的页数:’))
for i in range(1,num+1):
for j in spider_url(post,i):
data_list.append(spider_page(j))

# for i in range(len(data_list)):
#     data=data_list[i]
#     for j in data:
#         print(j['id'])


saveData(data_list)
print('爬取完毕')