获取chrome的network内容并选择下载其中的资源

现在存在问题：要将网站反馈回来的get信号的内容获取到并下载资源
eg：
chrome+键F12可见的内容

解决方式与思路：
方法：

1. 使用webDriver或者selenium模拟浏览器传输信号，再截取返回回来的信号（这里使用的就是这个方法）。
2. 使用selenium与browsermob-proxy；browsermob-proxy可以获取http内容导出为HAR文件。

思路：
3. 用selenium的api获取其中里面的message
4. 其中存在将请求变成log的文件的api( LogEntries logs = driver.manage().logs().get(LogType.PERFORMANCE);
)，该文件是个json文件,可以使用json来解析获取想要的信息以及其的链接
5. 使用URL获取文件流，下载文件

代码如下：

package test;
import java.io.*;
import java.net.URL;
import java.util.Iterator;
import java.util.logging.Level;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.json.JSONException;
import org.json.JSONObject;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeDriverService;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.logging.LogEntries;
import org.openqa.selenium.logging.LogEntry;
import org.openqa.selenium.logging.LogType;
import org.openqa.selenium.logging.LoggingPreferences;
import org.openqa.selenium.remote.CapabilityType;
import org.openqa.selenium.remote.DesiredCapabilities;

/**
 * chromedriver
 * dev-tool
 */
public class ChromedriverNetwork {
    public static final String port = "1527";
    public static final String filePath = "C:\\Users\\28124\\Desktop\\测试自动化\\自动化项目\\driver\\chromedriver.exe";
    public static void main(String[] args) {

        String sessionId ;
        String url = "http://www.ting56.com/video/18743-0-3.html";
        //System.setProperty("webdriver.chrome.driver",filePath);
        ChromeDriver driver = null;
        try {
            ChromeOptions options = new ChromeOptions();
            DesiredCapabilities cap = new DesiredCapabilities();
            ChromeDriverService service = new ChromeDriverService.Builder().usingPort(Integer.valueOf(port))
                    .usingDriverExecutable(new File(filePath))
                    .build();
            options.addArguments("disable-infobars");
            cap.setCapability(ChromeOptions.CAPABILITY, options);
            //配置日志
            LoggingPreferences logPrefs = new LoggingPreferences();
            logPrefs.enable(LogType.PERFORMANCE, Level.ALL);
            /**flash 扩展*/
            cap.setCapability("profile.managed_default_content_settings.images",1);
            cap.setCapability("profile.content_settings.plugin_whitelist.adobe-flash-player",1);
            cap.setCapability("profile.content_settings.exceptions.plugins.*,*.per_resource.adobe-flash-player",1);
            cap.setCapability(CapabilityType.LOGGING_PREFS, logPrefs);
            driver = new ChromeDriver(service, cap);
            //or driver.get(url)
            driver.navigate().to(url);
//            System.out.println("session id :" + driver.getSessionId());
            sessionId = driver.getSessionId().toString();
            LogEntries logs = driver.manage().logs().get(LogType.PERFORMANCE);
            for (Iterator<LogEntry> it = logs.iterator(); it.hasNext();) {
                LogEntry entry = it.next();
                try {
                    //System.out.println(entry.toString());
                    JSONObject json = new JSONObject(entry.getMessage());
//                    System.out.println(json.toString());
                    JSONObject message = json.getJSONObject("message");
                    String method = message.getString("method");
                    //如果是响应
                    if (method != null && "Network.responseReceived".equals(method)) {
                        JSONObject params = message.getJSONObject("params");
                        JSONObject response = params.getJSONObject("response");
                        String messageUrl = response.getString("url");
//                        System.out.println("-----------------------------");
                        if (params.toString().contains("https://od.qingting.fm/m4a/")) {
                            InputStream in=new URL(messageUrl).openConnection().getInputStream(); //创建连接、输入copy流
                            FileOutputStream f = new FileOutputStream("C:\\Users\\28124\\Desktop\\测试自动化\\自动化项目\\file2\\a.m4a");//创建文百件度输出流
                            byte [] bb=new byte[1024]; //接收缓存
                            int len;
                            while( (len=in.read(bb))>0){ //接收
                                f.write(bb, 0, len); //写入问文件
                            }
                            f.close();
                            in.close();
                        }
//                        System.err.println("url:" + url + " params: " + params.toString() + " response: " + response.toString());
                        //打印调用chromedriver 调chrome httpapi 结果
//                        System.out.println(getResponseBody(params.getString("requestId"),port,sessionId));
//                        System.out.println("-----------------------------");
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }finally{
            if (driver != null)
            {
                driver.quit();
            }
        }
    }


    // 根据请求ID获取返回内容
    public static String getResponseBody(String requestId,String port,String sessionId) {
        try {
            // CHROME_DRIVER_PORT chromeDriver提供的端口
            String url = String.format("http://localhost:%s/session/%s/goog/cdp/execute",
                    port, sessionId);

            HttpPost httpPost = new HttpPost(url);
            JSONObject object = new JSONObject();
            JSONObject params = new JSONObject();
            params.put("requestId", requestId);
            //api https://chromedevtools.github.io/devtools-protocol/tot/Network
            object.put("cmd", "Network.getResponseBody");

            object.put("params", params);

            httpPost.setEntity(new StringEntity(object.toString()));

            RequestConfig requestConfig = RequestConfig
                    .custom()
                    .setSocketTimeout(60 * 1000)
                    .setConnectTimeout(60 * 1000).build();

            CloseableHttpClient httpClient = HttpClientBuilder.create()
                    .setDefaultRequestConfig(requestConfig).build();

            HttpResponse response = httpClient.execute(httpPost);
            return EntityUtils.toString(response.getEntity());
        } catch (Exception e) {
            e.printStackTrace();
        }
        return null;
    }
}

总结：

1. 如果使用思路2，可以参考链接
   https://blog.csdn.net/m0_37618247/article/details/85066272

参考链接：
https://blog.csdn.net/qq_36783371/article/details/100122599