最近在爬取智联的简历信息,爬取简历详情页的时候,使用的是公司的分布式框架,获取的详情页的url不会立即爬取,导致出现链接已失效的问题,使用的是模拟ajax获取数据,分析和代码如下:
获取上面的request url 和headers,构造模拟请求:
cookies = "" #登录后获取
at = ''#从cookies中获取
resume_url = 'https://ihr.zhaopin.com/resume/details/?resumeNo=8W(iEsMnOnu1uhDK1PPPYQ_1_1&searchresume=1&resumeSource=1&t=1529983802935&k=83599B49CE67F59148411E9588BDA363'
headers = {
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
'Connection': 'keep-alive',
'Cache-Control': 'max-age=0',
'Host': 'ihr.zhaopin.com',
'Referer': resume_url,
'Cookie':cookies,
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0',
'X-Requested-With': 'XMLHttpRequest'
}
session.headers = headers
# resume_url = 'https://ihr.zhaopin.com/resumesearch/getresumedetial.do?access_token={}&'.format(at)+self.resume_url.split('https://ihr.zhaopin.com/resume/details/?&