按照:https://dryscrape.readthedocs.io/en/latest/installation.html
安装dryscrape
以下是简单实现
import dryscrape
from bs4 import BeautifulSoup
# 使用dryscrape库 动态抓取页面
def get_url_dynamic(url):
dryscrape.start_xvfb()
session_req=dryscrape.Session()
session_req.visit(url) #请求页面
response=session_req.body() #网页的文本
soup = BeautifulSoup(response, "html.parser")
for k in soup.find_all('a'):
print(k)
# print(soup.prettify())
return response
get_url_dynamic('https://www.cnblogs.com/sweetsunnyflower/p/11156056.html')
使用 pymysql 操作数据库