【Python】静态和动态获取网页信息

静态网页信息

---

• 工具：requests

from bs4 import BeautifulSoup
from urllib.parse import urlparse
import requests

# 设置UA
headers = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36' + 
    '(KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36' 
}

# 给定URL路径 (http或https开头)
url = 'http://www.baidu.com'

# 获取域名
host = urlparse(url).netloc 

# 访问页面 
res = requests.get(url, headers = headers)
res.encoding = 'utf-8'
contents = res.text

# 写入文件中
f = open('test.html', 'w', encoding='utf-8')
f.writelines(contents)
f.close()

# 格式化
soup = BeautifulSoup(contents, "html.parser")

BeautifulSoup官网：
https://www.crummy.com/software/BeautifulSoup/bs4/doc.zh/
 
 

动态网页信息（JS运行后的网页）

---

• 工具：selenium

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from bs4 import BeautifulSoup

# 给定URL路径 (http或https开头)
url = 'http://www.baidu.com'

# 两种方式
# 1.使用模拟器方式
mobile_emulation = {'deviceName': 'Galaxy S5'}
options = webdriver.ChromeOptions()
options.add_experimental_option("mobileEmulation", mobile_emulation)

# 2.使用内置Chrome选项
chrome_options = Options()
chrome_options.add_argument('--headless')

# 填充选项，chrome_options选用两种方式之一
browser = webdriver.Chrome(executable_path='chromedriver.exe', chrome_options=chrome_options)

# 访问网页
browser.get(url)

# 写入文件中
f = open('test.html', 'w', encoding='utf-8')
res = browser.page_source
f.write(res)
f.close()

# 格式化
soup = BeautifulSoup(browser.page_source, 'html.parse')
browser.quit()