python爬虫动态数据获取失败_知乎动态数据加载爬虫

base_url = 'https://www.zhihu.com/api/v4/topics/21239580/feeds/essence?'

#数据包的头部信息完整不容易被反爬

headers = {

'referer': 'https://www.zhihu.com/topic/21239580/top-answers',

'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',

'X-Requested-With': 'fetch',

'cookie':'_zap=dcef10ed-ac5b-47d7-be62-ce15422c243b; d_c0="AEAoweBBYw6PTtM5YdeXEGe9nkFjo1fukEs=|1539959215"; _xsrf=JfK6LMdyHsAQfznzKy9qqzVa1hGVudzX; __utmv=51854390.100--|2=registration_date=20171205=1^3=entry_date=20171205=1; _ga=GA1.2.231820689.1558959113; r_cap_id="MGNkZWRhYzBiMWEyNDUwYmI2ZmNhMWJlZTRmODJhYTY=|1584438632|0ffe2fcf055c73a9876c18c6f60dabbc0df9734b"; cap_id="YzhmN2M2NTdlOTFjNDJlZGFiYTFlNTQ4ZDVmOTRiOWI=|1584438632|ddc44638cf41bcea0634c664cfdc27f767ae79e7"; l_cap_id="ZWUwNTAzYWU1ZGExNDkyYmI4OWI3YzJmOGNlMTg3MzY=|1584438632|655548f26f77e6e254e52f4db91cbf340e699522"; tshl=; tst=r; q_c1=0ec4457e6f67420da14808f4ba874ab9|1586094208000|1539959216000; Hm_lvt_98beee57fd2ef70ccdd5ca52b9740c49=1584686926,1586092328,1586092679,1586309558; _gid=GA1.2.1489756283.1586309558; SESSIONID=vDYbvehGoVHeVUZQN9SUcsRPETOrwazMaLXnvmygBoV; JOID=U1AWAk1pmUkj11LRd2mXl5sPqTBlFtYSRqY-sjIBxgpEhgCCLdK5wX3ZUNhyUVORLtzYaDRq1-uGYkTGUyyTIJE=; osd=UlocAk1ok0Mj11PbfWmXlpEFqTBkHNwSRqc0uDIBxwBOhgCDJ9i5wXzTWthyUFmbLtzZYj5q1-qMaETGUiaZIJE=; capsion_ticket="2|1:0|10:1586309956|14:capsion_ticket|44:NTkzNDk3ZTU2MWU0NDM0ZGJhYmZkNjY5MGU3NDFiNTM=|aff74864db3420505f145e142e16a742ff658f23b7398b9c05d19ac57fa487a0"; z_c0="2|1:0|10:1586309963|4:z_c0|92:Mi4xWUVERUJnQUFBQUFBUUNqQjRFRmpEaVlBQUFCZ0FsVk5TM2w2WHdBd1RPd3p1bWRNTGRONmdNQXJWczlDZjVVVXR3|ebc6875dd9b03e62598afe37b2ee420f4d30578822a588ce82e93adb2b0732fe"; __utmc=51854390; __utma=51854390.231820689.1558959113.1586311624.1586315592.5; __utmz=51854390.1586315592.5.3.utmcsr=zhihu.com|utmccn=(referral)|utmcmd=referral|utmcct=/question/377935555; Hm_lpvt_98beee57fd2ef70ccdd5ca52b9740c49=1586329609; _gat_gtag_UA_149949619_1=1; KLBRSID=2177cbf908056c6654e972f5ddc96dc2|1586329618|1586319827',

}

#获取动态加载的json文件

def get_page(offset):

params = {

'include':'data[?(target.type=topic_sticky_module)].target.data[?(target.type=answer)].target.content,relationship.is_authorized,is_author,voting,is_thanked,is_nothelp;data[?(target.type=topic_sticky_module)].target.data[?(target.type=answer)].target.is_normal,comment_count,voteup_count,content,relevant_info,excerpt.author.badge[?(type=best_answerer)].topics;data[?(target.type=topic_sticky_module)].target.data[?(target.type=article)].target.content,voteup_count,comment_count,voting,author.badge[?(type=best_answerer)].topics;data[?(target.type=topic_sticky_module)].target.data[?(target.type=people)].target.answer_count,articles_count,gender,follower_count,is_followed,is_following,badge[?(type=best_answerer)].topics;data[?(target.type=answer)].target.annotation_detail,content,hermes_label,is_labeled,relationship.is_authorized,is_author,voting,is_thanked,is_nothelp;data[?(target.type=answer)].target.author.badge[?(type=best_answerer)].topics;data[?(target.type=article)].target.annotation_detail,content,hermes_label,is_labeled,author.badge[?(type=best_answerer)].topics;data[?(target.type=question)].target.annotation_detail,comment_count;',

'limit': 10,

'offset': offset #offset是一个变量

}

url = base_url + urlencode(params)

try:

response = requests.get(url, headers=headers)

if response.status_code == 200:

return response.json()

except requests.ConnectionError as e:

print('Error', e.args)