java selenium获取Network response的内容

这两天遇到一个棘手的异常，时不时页面会弹出：“系统繁忙，请稍候再试！”，这时候我们去看网络请求数据，结果状态码全部都是 200，没有其它信息，这压根没法定位不了问题。

这就说明：网络出现异常的时候，仅靠状态码是不够的。我们最好能拿到 http 所有数据，包括：请求头、响应头、请求体、响应体。其中请求头、响应头，可以通过 PERFORMANCE_LOG 拿到，问题都不大。但是请求体与响应体，我们可以拿到么？

这个问题困扰了我整整一天的时间，终于解决了。为什么这么困难？

我们先来看 selenium，它为什么不直接支持这个功能呢？因为开发人员觉得这不是他们目标：

we will not be adding this feature to the WebDriver API as it falls outside of our current scope (emulating user actions).

然后我继续翻网络，发现谷歌的devtools-protocol明确是支持的

那到底有没有办法解决这个问题？

有！且不止一种方法....

第一种方法：我们使用BrowserMob Proxy

首先获取WebDriver及BrowserProxy

import net.lightbody.bmp.BrowserMobProxy;
import net.lightbody.bmp.BrowserMobProxyServer;
import net.lightbody.bmp.client.ClientUtil;
import net.lightbody.bmp.proxy.CaptureType;
import org.openqa.selenium.Proxy;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.logging.LogType;
import org.openqa.selenium.logging.LoggingPreferences;
import org.openqa.selenium.remote.CapabilityType;

import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;

/**
 * BrowserProxy 单例
 *
 * @author ant
 * @date 2019-11-21 10:38:20
 */
public class BrowserProxy {
    /**
     * BrowserMobProxy
     */
    private BrowserMobProxy proxy;

    /**
     * WebDriver
     */
    private WebDriver driver;


    public BrowserMobProxy getProxy() {
        return proxy;
    }

    private void setProxy(BrowserMobProxy proxy) {
        this.proxy = proxy;
    }

    public WebDriver getDriver() {
        return driver;
    }

    private void setDriver(WebDriver driver) {
        this.driver = driver;
    }

    /**
     * 无参构造 初始化参数
     * 私有构造 禁止外部new
     */
    private BrowserProxy() {
        String webDriverPath = System.getProperty("user.dir") + "/resources/chromedriver.exe";
        System.out.println("webDriverPath:"+webDriverPath);
        //String webDriverPath = "E:\\IDEA\\common\\src\\test\\java\\com\\autotest\\tools\\chromedriver.exe";
        System.setProperty("webdriver.chrome.driver", webDriverPath);

        ChromeOptions options = new ChromeOptions();
        // 禁用阻止弹出窗口
        options.addArguments("--disable-popup-blocking");
        // 启动无沙盒模式运行
        options.addArguments("no-sandbox");
        // 禁用扩展
        options.addArguments("disable-extensions");
        // 默认浏览器检查
        options.addArguments("no-default-browser-check");

        Map<String, Object> prefs = new HashMap();
        prefs.put("credentials_enable_service", false);
        prefs.put("profile.password_manager_enabled", false);
        // 禁用保存密码提示框
        options.setExperimentalOption("prefs", prefs);

        // set performance logger
        // this sends Network.enable to chromedriver
        LoggingPreferences logPrefs = new LoggingPreferences();
        logPrefs.enable(LogType.PERFORMANCE, Level.ALL);
        options.setCapability(CapabilityType.LOGGING_PREFS, logPrefs);

        // start the proxy
        BrowserMobProxy proxy = new BrowserMobProxyServer();
        // 端口号
        proxy.start(4566);

        // get the Selenium proxy object
        Proxy seleniumProxy = ClientUtil.createSeleniumProxy(proxy);
        options.setCapability(CapabilityType.PROXY, seleniumProxy);
        // start the browser up
        WebDriver driver = new ChromeDriver(options);
        // 等待10秒
        driver.manage().timeouts().implicitlyWait(10, TimeUnit.SECONDS);

        // enable more detailed HAR capture, if desired (see CaptureType for the complete list)
        proxy.enableHarCaptureTypes(CaptureType.REQUEST_CONTENT, CaptureType.RESPONSE_CONTENT);
        // enable cookies  --> REQUEST_COOKIES, RESPONSE_COOKIES
        proxy.enableHarCaptureTypes(CaptureType.getCookieCaptureTypes());
        // enable headers --> REQUEST_HEADERS, RESPONSE_HEADERS
        //proxy.enableHarCaptureTypes(CaptureType.getHeaderCaptureTypes());
        // 向对象赋值
        this.setProxy(proxy);
        this.setDriver(driver);
    }

    /**
     * 类级的内部类，也就是静态的成员式内部类，该内部类的实例与外部类的实例没有绑定关系，而且只有被调用到才会装载，从而实现了延迟加载
     */
    private static class BrowserProxyHolder {
        private final static BrowserProxy instance = new BrowserProxy();
    }

    /**
     * 提供公有调用方法
     *
     * @return
     */
    public static BrowserProxy getInstance() {
        return BrowserProxyHolder.instance;
    }

}

在这里需要注意的是

proxy.enableHarCaptureTypes(CaptureType.REQUEST_CONTENT, CaptureType.RESPONSE_CONTENT);

如果要获取request content或者response content则必需加上这行参数,否则下面获取不到数据。

那么，如果是获取cookie呢？

proxy.enableHarCaptureTypes(CaptureType.getCookieCaptureTypes());

换了一种写法，其实和上面获取content效果是一样的，当然还有其他写法，很灵活。

准备工作做好了，下面干正事。

import com.example.cat.selenium.BrowserProxy;
import net.lightbody.bmp.BrowserMobProxy;
import net.lightbody.bmp.core.har.*;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import java.util.List;


public class BrowserMobProxyTest {

    public void tearDown(WebDriver driver,BrowserMobProxy proxy) {
        if(proxy.isStarted()){
            proxy.stop();
        }
        // 关闭当前窗口
        driver.close();
    }

    public static void main(String[] args) {
        BrowserMobProxyTest browserMobProxyTest =  new BrowserMobProxyTest();
        BrowserProxy browserProxy = BrowserProxy.getInstance();
        WebDriver driver = browserProxy.getDriver();
        BrowserMobProxy proxy = browserProxy.getProxy();
        browserMobProxyTest.testMethod(driver,proxy);
        //browserMobProxyTest.tearDown(driver,proxy);
    }
    public void testMethod(WebDriver driver,BrowserMobProxy proxy) {
        // 这里必需在driver.get(url)之前
        proxy.newHar("test_har");

        driver.get("https://translate.google.cn/");

        String text = "selenium";

        driver.findElement(By.xpath("//*[@id=\"source\"]")).sendKeys(text);

        Har har = proxy.getHar();
        List<HarEntry> list =  har.getLog().getEntries();
        for (HarEntry harEntry : list){
            HarResponse harResponse = harEntry.getResponse();
            String responseBody = harResponse.getContent().getText();
            List<HarCookie> cookies = harResponse.getCookies();
            // network response
            System.out.println("responseBody:"+responseBody);
            // 可获取httpOnly 类型的cookie
            System.out.println("cookies:"+cookies);
        }
    }
}

如此，想获取network response的内容或是获取httpOnly类型的cookie也都ok了。

但是有一个问题，仍未解决

AnnotatedConnectException: Connection timed out: no further information: accounts.google.com/172.217.160.109:443

为什么会连接connection 这个ip呢？有发现的小伙伴留言交流！

接下来 第二种方法 我们通过requtesId去实现

package com.example.cat.test.hacpai;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.example.cat.utils.Hclient;
import org.apache.http.HttpResponse;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeDriverService;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.logging.LogEntries;
import org.openqa.selenium.logging.LogEntry;
import org.openqa.selenium.logging.LogType;
import org.openqa.selenium.logging.Logs;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;


public class ChromeDriverProxy extends ChromeDriver {
    /**
     * method --> Network.responseReceived
     */
    public static final String NETWORK_RESPONSE_RECEIVED = "Network.responseReceived";

    /**
     * post 请求超时时间
     */
    private static final int COMMAND_TIMEOUT = 5000;

    /**
     * 必须固定端口，因为ChromeDriver没有实时获取端口的接口；
     */
    private static final int CHROME_DRIVER_PORT = 9102;

    private static ChromeDriverService driverService = new ChromeDriverService.Builder().usingPort(CHROME_DRIVER_PORT).build();

    public ChromeDriverProxy(ChromeOptions options) {
        super(driverService, options);
    }


    // 根据请求ID获取返回内容
    public String getResponseBody(String requestId) {
        try {
            // CHROME_DRIVER_PORT chromeDriver提供的端口
            String url = String.format("http://localhost:%s/session/%s/goog/cdp/execute",
                    CHROME_DRIVER_PORT, getSessionId());
            HttpPost httpPost = new HttpPost(url);
            JSONObject object = new JSONObject();
            JSONObject params = new JSONObject();
            params.put("requestId", requestId);
            object.put("cmd", "Network.getResponseBody");
            object.put("params", params);
            httpPost.setEntity(new StringEntity(object.toString()));

            RequestConfig requestConfig = RequestConfig
                    .custom()
                    .setSocketTimeout(COMMAND_TIMEOUT)
                    .setConnectTimeout(COMMAND_TIMEOUT).build();

            CloseableHttpClient httpClient = HttpClientBuilder.create()
                    .setDefaultRequestConfig(requestConfig).build();

            HttpResponse response = httpClient.execute(httpPost);
            String s = EntityUtils.toString(response.getEntity());
            return  this.getResponseValue(s);
        } catch (IOException e) {
            //logger.error("getResponseBody failed!", e);
        }
        return null;
    }



    // 根据请求ID获取返回内容
    // https://github.com/bayandin/chromedriver/blob/master/server/http_handler.cc
    public JSONArray getCookies(String requestId) {
        try {
            // CHROME_DRIVER_PORT chromeDriver提供的端口
            String url = String.format("http://localhost:%s/session/%s/cookie",
                    CHROME_DRIVER_PORT, getSessionId());
            String cookieStr = Hclient.doGet(url, null);
           return JSONArray.parseArray(getResponseValue(cookieStr));
        } catch (Exception e) {
            //logger.error("getResponseBody failed!", e);
        }
        return null;
    }
    /**
     * 获取响应结果{"sessionId": "","status": 0,"value": ""}的value
     *
     * @param data
     * @return
     */
    private String getResponseValue(String data){
        JSONObject json = JSONObject.parseObject(data);
        ResponseBodyVo responseBodyVo = JSONObject.toJavaObject(json,ResponseBodyVo.class);
        if(0 == responseBodyVo.getStatus()){
            return responseBodyVo.getValue();
        }else{
            System.out.println("status error:" + data);
            return "";
        }

    }

    public static void saveHttpTransferDataIfNecessary(ChromeDriverProxy driver) {
        Logs logs = driver.manage().logs();
        Set<String> availableLogTypes = logs.getAvailableLogTypes();

        if(availableLogTypes.contains(LogType.PERFORMANCE)) {
            LogEntries logEntries = logs.get(LogType.PERFORMANCE);
            List<ResponseReceivedEvent> responseReceivedEvents = new ArrayList<>();

            for(LogEntry entry : logEntries) {
                JSONObject jsonObj = JSON.parseObject(entry.getMessage()).getJSONObject("message");
                String method = jsonObj.getString("method");
                String params = jsonObj.getString("params");

                if (method.equals(NETWORK_RESPONSE_RECEIVED)) {
                    ResponseReceivedEvent response = JSON.parseObject(params, ResponseReceivedEvent.class);
                    responseReceivedEvents.add(response);
                }
            }
            doSaveHttpTransferDataIfNecessary(driver, responseReceivedEvents);
        }

    }

    // 保存网络请求
    private static void doSaveHttpTransferDataIfNecessary(ChromeDriverProxy driver, List<ResponseReceivedEvent> responses) {
        for(ResponseReceivedEvent responseReceivedEvent : responses) {
            String url = JSONObject.parseObject(responseReceivedEvent.getResponse()).getString("url");
            boolean staticFiles = url.endsWith(".png")
                    || url.endsWith(".jpg")
                    || url.endsWith(".css")
                    || url.endsWith(".ico")
                    || url.endsWith(".js")
                    || url.endsWith(".gif");

            if(!staticFiles && url.startsWith("http")) {
                // 使用上面开发的接口获取返回数据
                String  bodyx= driver.getResponseBody(responseReceivedEvent.getRequestId());

                System.out.println("url:"+url+" ,body-->"+bodyx);
                JSONArray cookies = driver.getCookies(responseReceivedEvent.getRequestId());
                System.out.println("url:"+url+" ,cookies-->"+cookies);
            }
        }
    }



}

附上几个实体类

ResponseBodyVo

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

@Data
@AllArgsConstructor
@NoArgsConstructor
public class ResponseBodyVo {
    private String sessionId;
    private String value;
    private int status;
}

ResponseReceivedEvent

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;



@Data
@AllArgsConstructor
@NoArgsConstructor
public class ResponseReceivedEvent {
    private String frameId;
    private String requestId;
    private String response;
    private String loaderId;
    private String type;
    private String timestamp;
}

测试类

import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.logging.LogType;
import org.openqa.selenium.logging.LoggingPreferences;
import org.openqa.selenium.remote.CapabilityType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.HashMap;
import java.util.Map;
import java.util.logging.Level;


public class Test {
    private final static Logger logger = LoggerFactory.getLogger(Test.class);

    public static void main(String[] args) {
        String url = "https://xxxx";
        test(url);
    }

    private static void test(String url) {
        String webDriverPath = "E:\\IDEA\\common\\src\\test\\java\\com\\autotest\\tools\\chromedriver.exe";
        System.setProperty("webdriver.chrome.driver", webDriverPath);
        ChromeDriverProxy driver = null;
        try {
            ChromeOptions options = new ChromeOptions();
            options.addArguments("--disable-popup-blocking"); // 禁用阻止弹出窗口
            options.addArguments("no-sandbox"); // 启动无沙盒模式运行
            options.addArguments("disable-extensions"); // 禁用扩展
            options.addArguments("no-default-browser-check"); // 默认浏览器检查
            Map<String, Object> prefs = new HashMap();
            prefs.put("credentials_enable_service", false);
            prefs.put("profile.password_manager_enabled", false);
            options.setExperimentalOption("prefs", prefs);// 禁用保存密码提示框

            // set performance logger
            // this sends Network.enable to chromedriver
            LoggingPreferences logPrefs = new LoggingPreferences();
            logPrefs.enable(LogType.PERFORMANCE, Level.ALL);

            options.setCapability(CapabilityType.LOGGING_PREFS, logPrefs);

            driver = new ChromeDriverProxy(options);
            // do something

            ChromeDriverProxy.saveHttpTransferDataIfNecessary(driver);
        } finally {
            driver.close();
        }
    }
}

同样 需要注意的是logPrefs.enable(LogType.PERFORMANCE, Level.ALL);

以上两种方法基本获取network response的内容了。

大家有好的方法可以一起留言交流。

参考：

java+selenium+chromedriver 有什么办法能拿到 network 返回的数据吗，求大佬指教

selenium 获取请求返回内容的解决方案