Selenium 获取请求响应

'''
Python 3.7
selenium==3.141.0
urllib3==1.26.2
Chromium 109.0.5405.0 （32 位） 
'''

import json
from selenium import webdriver
from selenium.common.exceptions import WebDriverException
import time

options = webdriver.ChromeOptions()
# 谷歌浏览器位置
chrome_location = r'D:\\Program Files (x86)\\Google\Chrome\\Application\\chrome.exe'
# 谷歌浏览器驱动地址
chromedriver_path = r'D:\\Program Files (x86)\\Google\\Chrome\\Application\\chromedriver_win32\\chromedriver.exe'

options.binary_location = chrome_location
###################################################################################
# 写法一
# (网上还有其他的方法，有的会报错 可能是版本问题，selenium.common.exceptions.InvalidArgumentException: Message: invalid argument: log type 'performance' not found，
# 下面两种测试正常)

caps = {
    'browserName': 'chrome',
    'version': '',
    'platform': 'ANY',
    'goog:loggingPrefs': {'performance': 'ALL'},
    'goog:chromeOptions': {'extensions': [], 'args': ['--headless']}
}

caps = {
    "browserName": "chrome",
    'goog:loggingPrefs': {'performance': 'ALL'}
}

driver = webdriver.Chrome(executable_path=chromedriver_path, options=options, desired_capabilities=caps)
###################################################################################

###################################################################################
# 写法二 (建议用这种，selenium 4 测试也行)
# options.set_capability("goog:loggingPrefs", {"performance": "ALL", "browser": "ALL"})
# driver = webdriver.Chrome(executable_path=chromedriver_path, options=options)
###################################################################################

# 查询的 IP
list_query = ['135.89.67.33', '34.66.45.22']

for query in list_query:
    driver.get(f'http://ip-api.com/json/{query}')
    # 等待所有请求完成，可以用等待界面元素方法
    time.sleep(10)
    logs = driver.get_log("performance")

    for item in logs:
        # print(item)
        log = json.loads(item["message"])["message"]
        # if "Network.response" in log["method"] or "Network.request" in log["method"] or "Network.webSocket" in log["method"]:
        # pprint(log)
        if log["method"] == 'Network.responseReceived':
            url = log['params']['response']['url']
            if url == 'data:,':  # 过滤掉初始data页面，后续可以根据 log['params']['response']['type']过滤请求类型
                continue
            print('请求', url)
            request_id = log['params']['requestId']
            response_headers = log['params']['response']['headers']
            status_code = log['params']['response']['status']
            try:
                request_data = driver.execute_cdp_cmd('Network.getRequestPostData', {'requestId': request_id})
            except WebDriverException:  # 没有后台数据获取时会有异常
                request_data = None

            response_body = driver.execute_cdp_cmd('Network.getResponseBody', {'requestId': request_id})['body']
            print('响应', response_body)


'''
输出：
请求 http://ip-api.com/json/135.89.67.33
响应 {"status":"success","country":"United States","countryCode":"US","region":"IN","regionName":"Indiana","city":"Indianapolis","zip":"46204","lat":39.7709,"lon":-86.1585,"timezone":"America/Indiana/Indianapolis","isp":"AT\u0026T Services","org":"AT\u0026T Services, Inc.","as":"","query":"135.89.67.33"}

请求 http://ip-api.com/json/34.66.45.22
响应 {"status":"success","country":"United States","countryCode":"US","region":"IA","regionName":"Iowa","city":"Council Bluffs","zip":"","lat":41.2619,"lon":-95.8608,"timezone":"America/Chicago","isp":"Google LLC","org":"Google Cloud (us-central1)","as":"AS396982 Google LLC","query":"34.66.45.22"}
'''

'''
Python 3.8
selenium==4.21.0
urllib3==2.2.2
Chromium 109.0.5405.0 （32 位） 
'''

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import WebDriverException
import json
import time

# 谷歌浏览器位置
chrome_location = r'D:\\Program Files (x86)\\Google\Chrome\\Application\\chrome.exe'
# 谷歌浏览器驱动地址
chromedriver_path = r'D:\\Program Files (x86)\\Google\\Chrome\\Application\\chromedriver_win32\\chromedriver.exe'

# 启用性能日志
options = Options()
options.set_capability("goog:loggingPrefs", {"performance": "ALL", "browser": "ALL"})
options.binary_location = chrome_location
# 启动WebDriver
service = Service(executable_path=chromedriver_path)
driver = webdriver.Chrome(service=service, options=options)

# 查询的 IP
list_query = ['135.89.67.33', '34.66.45.22']

for query in list_query:
    driver.get(f'http://ip-api.com/json/{query}')
    # 等待所有请求完成，可以用等待界面元素方法
    time.sleep(10)
    logs = driver.get_log("performance")

    for item in logs:
        # print(item)
        log = json.loads(item["message"])["message"]
        # if "Network.response" in log["method"] or "Network.request" in log["method"] or "Network.webSocket" in log["method"]:
        # pprint(log)
        if log["method"] == 'Network.responseReceived':
            url = log['params']['response']['url']
            if url == 'data:,':  # 过滤掉初始data页面，后续可以根据 log['params']['response']['type']过滤请求类型
                continue
            print('请求', url)
            request_id = log['params']['requestId']
            response_headers = log['params']['response']['headers']
            status_code = log['params']['response']['status']
            try:
                request_data = driver.execute_cdp_cmd('Network.getRequestPostData', {'requestId': request_id})
            except WebDriverException:  # 没有后台数据获取时会有异常
                request_data = None

            response_body = driver.execute_cdp_cmd('Network.getResponseBody', {'requestId': request_id})['body']
            print('响应', response_body)

'''
可以获取 每次 driver.get(url) 的日志信息，重新刷新后就没有之前的了。
'''

'''
参考：
https://blog.csdn.net/MXB_1220/article/details/131775148
https://blog.csdn.net/u014376732/article/details/133973141
https://www.cnblogs.com/szyicol/p/18093390
https://www.cnblogs.com/superhin/p/15023302.html
https://segmentfault.com/q/1010000043296964
https://www.yanue.net/post-176.html
'''