爬取药监局的数据尝试

1.直接利用网址http://app1.nmpa.gov.cn/datasearchcnda/face3/search.jsp?tableId=121&State=1&bcId=152894035121716369704750131820&State=1&curstart=’+str(i)+’&State=1&tableName=TABLE121&State=1&viewtitleName=COLUMN1615&State=1&viewsubTitleName=COLUMN1618,COLUMN1616&State=1&tableView=%25E5%2585%25A8%25E5%259B%25BD%25E8%258D%25AF%25E5%2593%2581%25E6%258A%25BD%25E6%25A3%2580&State=1&cid=0&State=1&ytableId=0&State=1&searchType=search&State=1
访问，selenium爬取

from selenium import webdriver
from urllib.parse import urlencode
from bs4 import BeautifulSoup
from pybloom_live import BloomFilter
import re
import os
import datetime
import pymysql
import time

def get_data():
    options = webdriver.ChromeOptions()
    options.add_argument(argument='headless')  # 设置无界面，可选,使用无头模式会验证失败
    options.add_argument('user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"')
    browser  = webdriver.Chrome(chrome_options=options)
    browser.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
        "source": """
                        Object.defineProperty(navigator, 'webdriver', {
                          get: () => undefined
                        })
                      """
    })  # selenium的反爬虫策略
    #url的拼接
    for i in range(1):

        base_url = 'http://app1.nmpa.gov.cn/datasearchcnda/face3/search.jsp?tableId=121&State=1&bcId=152894035121716369704750131820&State=1&curstart='+str(i)+'&State=1&tableName=TABLE121&State=1&viewtitleName=COLUMN1615&State=1&viewsubTitleName=COLUMN1618,COLUMN1616&State=1&tableView=%25E5%2585%25A8%25E5%259B%25BD%25E8%258D%25AF%25E5%2593%2581%25E6%258A%25BD%25E6%25A3%2580&State=1&cid=0&State=1&ytableId=0&State=1&searchType=search&State=1'
        browser.get(base_url)
        content = browser.page_source
        print('content:',content)
        soup = BeautifulSoup(content, 'lxml')
        # 15个循环
        try:
            for j in range(15):  # 最后一页不足十五条肯定会报错,但实际运行是没问题的
                tags1 = soup.findAll('a')[j].get('href').split(',')[1][1:-1]
                url = 'http://app1.nmpa.gov.cn/datasearchcnda/face3/' + tags1
                print(url)
        except Exception as e:  # 不报错，继续运行
            print(e)
            time.sleep(5)


if __name__ == '__main__':
    get_data()

隔天又可以了
http://app1.nmpa.gov.cn/datasearchcnda/face3/search.jsp?tableId=132&bcId=154209313929078698414236686309&curstart=1

tags1=tags1.replace('医疗器械生产企业（许可）',p.quote('医疗器械生产企业（许可）',encoding='gbk')) #gbk的编码形式

用request直接访问也得不到结果

import requests
import json
from bs4 import BeautifulSoup
import pymssql
import time

def main():
    for i in range(1):
        # UA伪装
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3741.400 QQBrowser/10.5.3863.400',
            'Content-Type': 'application/x-www-form-urlencoded'

        }
        # 指定URL
        # 批量获取不同企业的ID值(ctrl+r正则匹配工具栏)
        url = 'http://app1.nmpa.gov.cn/datasearchcnda/face3/search.jsp?tableId=121&State=1&bcId=152894035121716369704750131820&State=1&curstart=%27%20str%28i%29%20%27&State=1&tableName=TABLE121&State=1&viewtitleName=COLUMN1615&State=1&viewsubTitleName=COLUMN1618,COLUMN1616&State=1&tableView=%25E5%2585%25A8%25E5%259B%25BD%25E8%258D%25AF%25E5%2593%2581%25E6%258A%25BD%25E6%25A3%2580&State=1&cid=0&State=1&ytableId=0&State=1&searchType=search&State=1'  #名字不同而已，还以为是一样的

        content = requests.get(url=url,
                                headers=headers)
        print('content: ',content.text)



if __name__ == '__main__':
    main()

发生错误
第一次能爬出来，刷新之后就没了?
之前拼接的地址也是如此。
http://app1.nmpa.gov.cn/datasearchcnda/face3/search.jsp?tableId=132&bcId=154209313929078698414236686309&curstart=1
关于详情页面，拼接后的地址，第一次直接访问能出来结果
http://app1.nmpa.gov.cn/datasearchcnda/face3/content.jsp?tableId=132&tableName=TABLE132&tableView=%D2%BD%C1%C6%C6%F7%D0%B5%C9%FA%B2%FA%C6%F3%D2%B5%A3%A8%D0%ED%BF%C9%A3%A9&Id=9720

最终还是没能爬取成功

尝试二：
参考： 爬取国家药品监督管理局数据
想用selenium模拟click,点击相应的页面，最后得到的结果却是服务器未返回数据。应该是被识破了。

from selenium.webdriver.common.action_chains import ActionChains
base_url='http://app1.nmpa.gov.cn/datasearchcnda/face3/base.jsp?tableId=132&tableName=TABLE132&title=%D2%BD%C1%C6%C6%F7%D0%B5%C9%FA%B2%FA%C6%F3%D2%B5%A3%A8%D0%ED%BF%C9%A3%A9&bcId=154209313929078698414236686309'
browser.get(base_url)
action = ActionChains(browser)
click_name = browser.find_element_by_link_text('1.苏食药监械生产许20110051号 (扬州富达医疗器械有限公司)')
click_name.click()
content = browser.page_source
print(content)

---

最后只爬出了二级目录页面

代码如下

from selenium import webdriver
from urllib.parse import urlencode
from bs4 import BeautifulSoup
from pybloom_live import BloomFilter
import re
import os
import datetime
import pymysql
import time
from selenium.webdriver.common.action_chains import ActionChains

def get_data():
    options = webdriver.ChromeOptions()
    options.add_argument(argument='headless')  # 设置无界面，可选,使用无头模式会验证失败
    options.add_argument('user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"')
    options.add_argument('Content-Type="text/html;encoding=gbk"')

    browser  = webdriver.Chrome(chrome_options=options)
    browser.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
        "source": """
                        Object.defineProperty(navigator, 'webdriver', {
                          get: () => undefined
                        })
                      """
    })  # selenium的反爬虫策略
    #url的拼接
    for i in range(2):

        base_url='http://app1.nmpa.gov.cn/datasearchcnda/face3/search.jsp?tableId=132&bcId=154209313929078698414236686309&curstart='+str(i+1)
        browser.get(base_url)

        content = browser.page_source
        soup = BeautifulSoup(content, 'lxml')
        # 15个循环
        try:
            for j in range(15):  # 最后一页不足十五条肯定会报错,但实际运行是没问题的
                tags1 = soup.findAll('a')[j].get('href').split(',')[1][1:-1]
                url = 'http://app1.nmpa.gov.cn/datasearchcnda/face3/' + tags1
                print(url)
        except Exception as e:  # 不报错，继续运行
            print(e)
            time.sleep(5)


if __name__ == '__main__':
    get_data()

---

尝试之后还是失败告终!!!
请问有人知道是什么原因吗?