获取xhr请求工具类

import json
from selenium import webdriver
from selenium.webdriver import DesiredCapabilities


def get_xhr_logs(chrome):
    log_xhr_array = []
    for typelog in chrome.log_types:
        perfs = chrome.get_log(typelog)
        for row in perfs:
            log_data = row
            message_ = log_data['message']
            try:
                log_json = json.loads(message_)
                log = log_json['message']
                if log['method'] == 'Network.responseReceived':
                    # 去掉静态js、css等，仅保留xhr请求
                    type_ = log['params']['type']
                    if type_ == "XHR":
                        log_xhr_array.append(log)
            except:
                pass
    return log_xhr_array


def get_log_options():
    option = webdriver.ChromeOptions()
    option.add_argument('--no-sandbox')
    option.add_argument('--headless')
    option.add_argument("--disable-extensions")
    option.add_argument("--allow-running-insecure-content")
    option.add_argument("--ignore-certificate-errors")
    option.add_argument("--disable-single-click-autofill")
    option.add_argument("--disable-autofill-keyboard-accessory-view[8]")
    option.add_argument("--disable-full-form-autofill-ios")
    option.add_argument('user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:55.0) Gecko/20100101 Firefox/55.0')
    option.add_experimental_option('w3c', False)
    option.add_experimental_option('perfLoggingPrefs', {
        'enableNetwork': True,
        'enablePage': False,
    })
    return option


def get_caps():
    caps = DesiredCapabilities.CHROME
    caps['loggingPrefs'] = {
        'browser': 'ALL',
        'performance': 'ALL',
    }
    caps['perfLoggingPrefs'] = {
        'enableNetwork': True,
        'enablePage': False,
        'enableTimeline': False
    }
    return caps

使用：

# 拦截进入详情页面，使用selenium绕过反爬
class SeleniumMiddlewareDetail:
    def process_response(self, request, response, spider):
        if request.meta['code'] == 1:
            chrome = spider.chrome
            chrome.get(request.url)
            # network日志
            logs = get_xhr_logs(chrome)
            return HtmlResponse(url=spider.chrome.current_url,
                                body=bytes(json.dumps(logs), encoding='utf8'),
                                encoding="utf8",
                                request=request)