middleware_demo

代理中间件

import random
import scrapy.conf import settings
class ProxyMiddleware(object):

    def process_request(self, request, spider):
        proxy = random.choice(settings['PROXIES'])
        request.meta['proxy'] = proxy # 设置代理

UA中间件

class UAMiddleware(object):

    def process_request(self, request, spider):
        ua = random.choice(settings['USER_AGENT_LIST'])
        request.headers['User-Agent'] = ua # 设置ua

COOKIES中间件

class CookiesMiddleware(object):
    # 从redis中获取cookies
    def __init__(self):
        self.client = redis.StrictRedis()

    def process_request(self, request, spider):
        if spider.name == 'login_spider':
            cookies = json.loads(self.client.lpop('cookies').decode())
            request.cookies = cookies # 设置cookies

Retry中间件

import scrapy
class RetryCustomMiddleware(RetryMiddleware):
    def __init__(self):
        RetryMiddleware.__init__(self, settings)

    def process_response(self, request, response, spider):
        if spider.name = 'retry_spider':
            if response.status == 404: # 响应码404
                origin_url = request.meta['redirect_urls'][0] # 404之前的url
                return request.replace( # 不返回response, 重新发送请求
                    url=origin_url,
                )
            # 正常返回response    
            return response

集成selenium

from scrapy.http import HtmlResponse
from selenium import webdriver
import time


class SeleniumMiddleware(object):

    def __init__(self):
        self.driver = webdriver.Chrome()
    
    def process_request(self, request, spider):
        if spider.name = 'selenium_spider':
            self.driver.get(request.url)
            time.sleep(2)
            body = self.driver.page_source # 获取网页源码

            return HtmlResponse( # 返回响应
                self.driver.current_url,
                body=body,
                encoding='utf8',
                request=request
            )