利用selenium获取接口数据

起因：页面展示的内容被隐藏了部分，无法获取完整内容

处理方案：
1.利用request模拟接口获取返参，模拟了请求头，但操作时一直无法获得数据，报错：org.apache.catalina.connector.ClientAbortException。未深究，大概率是服务器安全问题
selenium获取请求头可参考：https://blog.csdn.net/qq_31042199/article/details/119278315
但是跟selenium库有冲突好像，不知道是否需要把selenium卸载了再安装seleniumwire，待后续有时间验证一下。

2.通过webdriver提供的API查询，使用的函数是Network.getResponseBody
代码：

import json
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
import time
 
caps = {
    'browserName': 'chrome',
    'loggingPrefs': {
        'browser': 'ALL',
        'driver': 'ALL',
        'performance': 'ALL',
    },
    'goog:chromeOptions': {
        'perfLoggingPrefs': {
            'enableNetwork': True,
        },
        'w3c': False, 
    },
}
driver = webdriver.Chrome(desired_capabilities=caps)
 
driver.get('https://partner.oceanengine.com/union/media/login/')
# 必须等待一定的时间，不然会报错提示获取不到日志信息，因为絮叨等所有请求结束才能获取日志信息
time.sleep(3)
 
request_log = driver.get_log('performance')
print(request_log)
 
for i in range(len(request_log)):
    message = json.loads(request_log[i]['message'])
    message = message['message']['params']
    # .get() 方式获取是了避免字段不存在时报错
    request = message.get('request')
    if(request is None):
        continue
 
    url = request.get('url')
    if(url == "https://s3.pstatp.com/bytecom/resource/union_web2/media/manifest.json"):
        # 得到requestId
        print(message['requestId'])
        # 通过requestId获取接口内容
        content = driver.execute_cdp_cmd('Network.getResponseBody', {'requestId': message['requestId']})
        print(content)
        break

3.以上方法在操作中一直报错“no resource with given identifier found”，经排查，requestId是可以获取的，后来发现该接口是Ajax，selenium抓取Ajax接口的数据用如下demo（这个demo中包含了动态url获取过程）：

import os, time, json
from selenium.webdriver.common.by import By
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

if __name__ == '__main__':
    # 引入chromedriver
    chrome_options = webdriver.ChromeOptions()
    # chrome_options.add_argument('--headless')
    # chrome_options.add_argument('--no-sandbox')
    # chrome_options.add_argument("--disable-extensions")
    # chrome_options.add_argument("--disable-gpu")
    prefs = {
        'download.default_directory': os.getenv('OS_LOG_PATH')
    }
    chrome_options.add_experimental_option('prefs', prefs)
    # make chrome log requests
    capabilities = DesiredCapabilities.CHROME
    # caps['goog:loggingPrefs']
    capabilities["goog:loggingPrefs"] = {"performance": "ALL"}  # newer: goog:loggingPrefs
    # capabilities['acceptSslCerts'] = True
    #browser = webdriver.Chrome(executable_path=r'/opt/google/chrome/chromedriver', options=chrome_options)
    browser = webdriver.Chrome(executable_path=r'C:\Program Files (x86)\Google\Chrome\Application\chromedriver.exe', options=chrome_options,desired_capabilities=capabilities)
    browser.implicitly_wait(3)
    browser.get("https://#########")

    # 输入账号
    browser.implicitly_wait(3)
    username_inputbox = browser.find_elements(By.XPATH, '//*[@id="app"]/section/div[1]/div/div[1]/div/div[2]/form/div[1]/div/div/div/input')[0]
    username_inputbox.click()
    browser.implicitly_wait(3)
    username_inputbox.clear()
    username_inputbox.send_keys("#######")

    # 输入密码
    browser.implicitly_wait(3)
    password_inputbox = browser.find_elements(By.XPATH, '//*[@id="app"]/section/div[1]/div/div[1]/div/div[2]/form/div[2]/div/div/div/input')[0]
    password_inputbox.click()
    browser.implicitly_wait(3)
    password_inputbox.clear()
    password_inputbox.send_keys("#######")

    # 点击登录
    browser.implicitly_wait(3)
    submit_btn = browser.find_elements(By.XPATH, '//*[@id="app"]/section/div[1]/div/div[1]/div/div[2]/form/div[3]/div/button')[0]
    submit_btn.click()
    # 等待5秒
    time.sleep(5)
    #-------------------------------------------------
    base_url = "https://#########"
    browser.get(base_url)

    time.sleep(3)
    # extract requests from logs
    logs_raw = browser.get_log("performance")
    logs = [json.loads(lr["message"])["message"] for lr in logs_raw]

    def log_filter(log_):
        return (
            # is an actual response
                log_["method"] == "Network.responseReceived"
                # and json
                and "json" in log_["params"]["response"]["mimeType"]
        )

    for log in filter(log_filter, logs):
        request_id = log["params"]["requestId"]
        resp_url = log["params"]["response"]["url"]
        print(request_id)
        print(f"Caught {resp_url}")
        json_str = json.dumps(browser.execute_cdp_cmd("Network.getResponseBody", {"requestId": request_id}), indent=4)
        # 创建一个params.json文件
        with open(f'{request_id}.json', 'w') as f:
            f.write(json_str)  # 将json_str写到文件中


    browser.quit()

后续如遇见更多的获取接口数据的情况，再做补充