from selenium import webdriver
from urllib.parse import urlencode
from bs4 import BeautifulSoup
from pybloom_live import BloomFilter
import re
import os
import datetime
import pymysql
import time
def get_data():
options = webdriver.ChromeOptions()
options.add_argument(argument='headless') # 设置无界面,可选,使用无头模式会验证失败
options.add_argument('user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"')
browser = webdriver.Chrome(chrome_options=options)
browser.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
"source": """
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined
})
"""
}) # selenium的反爬虫策略
#url的拼接
for i in range(1):
base_url = 'http://app1.nmpa.gov.cn/datasearchcnda/face3/search.jsp?tableId=121&State=1&bcId=152894035121716369704750131820&State=1&curstart='+str(i)+'&State=1&tableName=TABLE121&State=1&viewtitleName=COLUMN1615&State=1&viewsubTitleName=COLUMN1618,COLUMN1616&State=1&tableView=%25E5%2585%25A8%25E5%259B%25BD%25E8%258D%25AF%25E5%2593%2581%25E6%258A%25BD%25E6%25A3%2580&State=1&cid=0&State=1&ytableId=0&State=1&searchType=search&State=1'
browser.get(base_url)
content = browser.page_source
print('content:',content)
soup = BeautifulSoup(content, 'lxml')
# 15个循环
try:
for j in range(15): # 最后一页不足十五条肯定会报错,但实际运行是没问题的
tags1 = soup.findAll('a')[j].get('href').split(',')[1][1:-1]
url = 'http://app1.nmpa.gov.cn/datasearchcnda/face3/' + tags1
print(url)
except Exception as e: # 不报错,继续运行
print(e)
time.sleep(5)
if __name__ == '__main__':
get_data()
tags1=tags1.replace('医疗器械生产企业(许可)',p.quote('医疗器械生产企业(许可)',encoding='gbk')) #gbk的编码形式
用request直接访问也得不到结果
import requests
import json
from bs4 import BeautifulSoup
import pymssql
import time
def main():
for i in range(1):
# UA伪装
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3741.400 QQBrowser/10.5.3863.400',
'Content-Type': 'application/x-www-form-urlencoded'
}
# 指定URL
# 批量获取不同企业的ID值(ctrl+r正则匹配工具栏)
url = 'http://app1.nmpa.gov.cn/datasearchcnda/face3/search.jsp?tableId=121&State=1&bcId=152894035121716369704750131820&State=1&curstart=%27%20str%28i%29%20%27&State=1&tableName=TABLE121&State=1&viewtitleName=COLUMN1615&State=1&viewsubTitleName=COLUMN1618,COLUMN1616&State=1&tableView=%25E5%2585%25A8%25E5%259B%25BD%25E8%258D%25AF%25E5%2593%2581%25E6%258A%25BD%25E6%25A3%2580&State=1&cid=0&State=1&ytableId=0&State=1&searchType=search&State=1' #名字不同而已,还以为是一样的
content = requests.get(url=url,
headers=headers)
print('content: ',content.text)
if __name__ == '__main__':
main()
发生错误
第一次能爬出来,刷新之后就没了?
之前拼接的地址也是如此。
http://app1.nmpa.gov.cn/datasearchcnda/face3/search.jsp?tableId=132&bcId=154209313929078698414236686309&curstart=1
关于详情页面,拼接后的地址,第一次直接访问能出来结果
http://app1.nmpa.gov.cn/datasearchcnda/face3/content.jsp?tableId=132&tableName=TABLE132&tableView=%D2%BD%C1%C6%C6%F7%D0%B5%C9%FA%B2%FA%C6%F3%D2%B5%A3%A8%D0%ED%BF%C9%A3%A9&Id=9720
最终还是没能爬取成功
尝试二:
参考: 爬取国家药品监督管理局数据
想用selenium模拟click,点击相应的页面,最后得到的结果却是服务器未返回数据。应该是被识破了。
from selenium.webdriver.common.action_chains import ActionChains
base_url='http://app1.nmpa.gov.cn/datasearchcnda/face3/base.jsp?tableId=132&tableName=TABLE132&title=%D2%BD%C1%C6%C6%F7%D0%B5%C9%FA%B2%FA%C6%F3%D2%B5%A3%A8%D0%ED%BF%C9%A3%A9&bcId=154209313929078698414236686309'
browser.get(base_url)
action = ActionChains(browser)
click_name = browser.find_element_by_link_text('1.苏食药监械生产许20110051号 (扬州富达医疗器械有限公司)')
click_name.click()
content = browser.page_source
print(content)
最后只爬出了二级目录页面
代码如下
from selenium import webdriver
from urllib.parse import urlencode
from bs4 import BeautifulSoup
from pybloom_live import BloomFilter
import re
import os
import datetime
import pymysql
import time
from selenium.webdriver.common.action_chains import ActionChains
def get_data():
options = webdriver.ChromeOptions()
options.add_argument(argument='headless') # 设置无界面,可选,使用无头模式会验证失败
options.add_argument('user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"')
options.add_argument('Content-Type="text/html;encoding=gbk"')
browser = webdriver.Chrome(chrome_options=options)
browser.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
"source": """
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined
})
"""
}) # selenium的反爬虫策略
#url的拼接
for i in range(2):
base_url='http://app1.nmpa.gov.cn/datasearchcnda/face3/search.jsp?tableId=132&bcId=154209313929078698414236686309&curstart='+str(i+1)
browser.get(base_url)
content = browser.page_source
soup = BeautifulSoup(content, 'lxml')
# 15个循环
try:
for j in range(15): # 最后一页不足十五条肯定会报错,但实际运行是没问题的
tags1 = soup.findAll('a')[j].get('href').split(',')[1][1:-1]
url = 'http://app1.nmpa.gov.cn/datasearchcnda/face3/' + tags1
print(url)
except Exception as e: # 不报错,继续运行
print(e)
time.sleep(5)
if __name__ == '__main__':
get_data()
尝试之后还是失败告终!!!
请问有人知道是什么原因吗?