爬虫第二节课——抓取药监网

一段错误代买，因为ajax是局部刷新，这里暂时记：接送对应ajax。text对应一般python模式

    page_text=requests.get(url=url,headers=headers).text
   	with open('.huazhuangpin.html','w',encoding='utf-8') as fp:
    fp.write(page_text)

在这里，哪怕点击企业名，“进入新的页面也没有发起新的url，也只是ajax局部刷新，变化的只有id所以循环id即可”

import requests
import json
if __name__ == '__main__':
   url='http://scxk.nmpa.gov.cn:81/xk/itownet/portalAction.do?method=getXkzsList'
   headers={
      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Edg/88.0.705.56'
   }
   data={
      'on': 'true',
      'page': '1',
      'pageSize': '15',
      'productName':'',
      'conditionType': '1',
      'applyname':'',
      'applysn':''
   }
   id_list=[]
   all_data_list=[]
   json_ids=requests.post(url=url,headers=headers,data=data).json()
   for dic in json_ids['list']:
      id_list.append(dic['ID'])
      print(dic['ID'])
   post_url='http://scxk.nmpa.gov.cn:81/xk/itownet/portalAction.do?method=getXkzsById'
   for id in id_list:
      data={
         'id':id
      }
      detail_json = requests.post(url=post_url,headers=headers,data=data).json()
      all_data_list.append(detail_json)
   fp=open('./allData.json','w',encoding='utf-8')
   json.dump(all_data_list,fp=fp,ensure_ascii=False)
   print("over!!!")

接下来实现，跨页面收集，收集五个页面的信息
这里只是对page进行了循环。

import requests
import json
if __name__ == '__main__':
   url='http://scxk.nmpa.gov.cn:81/xk/itownet/portalAction.do?method=getXkzsList'
   headers={
      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Edg/88.0.705.56'
   }
   id_list = []
   all_data_list = []
   for page in range(1,6):
      page=str(page)
      data={
         'on': 'true',
         'page': page,
         'pageSize': '15',
         'productName':'',
         'conditionType': '1',
         'applyname':'',
         'applysn':''
      }
      json_ids=requests.post(url=url,headers=headers,data=data).json()
      for dic in json_ids['list']:
         id_list.append(dic['ID'])
   post_url='http://scxk.nmpa.gov.cn:81/xk/itownet/portalAction.do?method=getXkzsById'
   for id in id_list:
      data={
         'id':id
      }
      detail_json = requests.post(url=post_url,headers=headers,data=data).json()
      all_data_list.append(detail_json)
   fp=open('./allData.json','w',encoding='utf-8')
   json.dump(all_data_list,fp=fp,ensure_ascii=False)
   print("over!!!")