Scrapy-downloadmiddleware

Scrapy模拟登陆

1.携带cookie登陆

# -*- coding: utf-8 -*-
import scrapy
import re

class RrenSpider(scrapy.Spider):
    name = 'rren'
    allowed_domains = ['renren.com']
    start_urls = ['http://renren.com/']

    def start_requests(self):
        cookies = "anonymid=k2rajzy1-y5supb; _r01_=1; depovince=HEB; ick_login=030375d5-7a0e-4ef0-908b-263c62d38781; _de=22F1ED8872F1EFD1F1A1C43A51983E74; t=bc208101d62130babbb1abfccf0c42c79; societyguester=bc208101d62130babbb1abfccf0c42c79; id=972941119; xnsid=632932de; JSESSIONID=abcjfJ35iI5xg_uXS1C6w; ver=7.0; loginfrom=null; jebe_key=9b17c2a2-6cb2-48f3-ae2f-befbc18fcf70%7Cc251768bba4c0f28c358f6dcc014a91b%7C1574589355158%7C1%7C1574589359930; jebe_key=9b17c2a2-6cb2-48f3-ae2f-befbc18fcf70%7Cc251768bba4c0f28c358f6dcc014a91b%7C1574589355158%7C1%7C1574589359933; wp_fold=0; jebecookies=f3066e0a-1eff-46d7-bf61-4456597c2294|||||"
        cookies = {i.split("=")[0]: i.split("=")[1] for i in cookies.split(";")}
        yield scrapy.Request(
            self.start_urls[0],
            callback=self.parse,
            cookies=cookies
        )

    def parse(self, response):
        print(re.findall("三寸季", response.body.decode()))
        yield scrapy.Request(
            "http://www.renren.com/972941119/profile?v=info_timeline",
            callback=self.parse_detail
        )

    def parse_detail(self, response):
        print(re.findall("三寸季", response.body.decode()))

下载中间件

Downloader Middlewares默认的方法：
process_request(self,request,spider):
  当每个request通过下载中间件时，该方法被调用
proces_response(self,request,response,spider):
  当下载器完成http请求，传递响应给引擎的时候调用

# -*- coding: utf-8 -*-

# Define here the models for your spider middleware
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/spider-middleware.html
import random

class RandomUserAgentMiddleware:
    def process_request(self,request,spider):
        ua = random.choice(spider.settings.get("USER_AGENTS_LIST"))
        request.headers["User-Agent"] = ua


class CheckUserAgent:
    def process_response(self,request,response,spider):
        # print(dir(response.request))
        print(request.headers["User-Agent"])
        return response