一段错误代买,因为ajax是局部刷新,这里暂时记:接送对应ajax。text对应一般python模式
page_text=requests.get(url=url,headers=headers).text
with open('.huazhuangpin.html','w',encoding='utf-8') as fp:
fp.write(page_text)
在这里,哪怕点击企业名,“进入新的页面也没有发起新的url,也只是ajax局部刷新,变化的只有id所以循环id即可”
import requests
import json
if __name__ == '__main__':
url='http://scxk.nmpa.gov.cn:81/xk/itownet/portalAction.do?method=getXkzsList'
headers={
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Edg/88.0.705.56'
}
data={
'on': 'true',
'page': '1',
'pageSize': '15',
'productName':'',
'conditionType': '1',
'applyname':'',
'applysn':''
}
id_list=[]
all_data_list=[]
json_ids=requests.post(url=url,headers=headers,data=data).json()
for dic in json_ids['list']:
id_list.append(dic['ID'])
print(dic['ID'])
post_url='http://scxk.nmpa.gov.cn:81/xk/itownet/portalAction.do?method=getXkzsById'
for id in id_list:
data={
'id':id
}
detail_json = requests.post(url=post_url,headers=headers,data=data).json()
all_data_list.append(detail_json)
fp=open('./allData.json','w',encoding='utf-8')
json.dump(all_data_list,fp=fp,ensure_ascii=False)
print("over!!!")
接下来实现,跨页面收集,收集五个页面的信息
这里只是对page进行了循环。
import requests
import json
if __name__ == '__main__':
url='http://scxk.nmpa.gov.cn:81/xk/itownet/portalAction.do?method=getXkzsList'
headers={
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Edg/88.0.705.56'
}
id_list = []
all_data_list = []
for page in range(1,6):
page=str(page)
data={
'on': 'true',
'page': page,
'pageSize': '15',
'productName':'',
'conditionType': '1',
'applyname':'',
'applysn':''
}
json_ids=requests.post(url=url,headers=headers,data=data).json()
for dic in json_ids['list']:
id_list.append(dic['ID'])
post_url='http://scxk.nmpa.gov.cn:81/xk/itownet/portalAction.do?method=getXkzsById'
for id in id_list:
data={
'id':id
}
detail_json = requests.post(url=post_url,headers=headers,data=data).json()
all_data_list.append(detail_json)
fp=open('./allData.json','w',encoding='utf-8')
json.dump(all_data_list,fp=fp,ensure_ascii=False)
print("over!!!")