scrapy—下载中间键

主要目的是为了使用代理，以及模拟的useragent

1：在settings.py 文件中设置

DOWNLOADER_MIDDLEWARES = {
    'xxx.middlewares.RandomUserAgent': 100,
    'xxx.middlewares.RandomProxy': 200,

}

模拟的USER_AGENTS

USER_AGENTS = [

    'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)',
    'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.2)',
    'Opera/9.27 (Windows NT 5.2; U; zh-cn)',
    'Opera/8.0 (Macintosh; PPC Mac OS X; U; en)',
    'Mozilla/5.0 (Macintosh; PPC Mac OS X; U; en) Opera 8.0',
    'Mozilla/5.0 (Linux; U; Android 4.0.3; zh-cn; M032 Build/IML74K) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30',
    'Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13'

]

代理设置：

PROXIES = [

        {"ip_port" :"xxxxx:xxx", "user_passwd" : "xxx:xxx"},
        #{"ip_prot" :"xxxx:xxxx", "user_passwd" : ""}

]

在爬虫目录下创建文件：middlewares.py

import random
import base64


from settings import USER_AGENTS
from settings import PROXIES


# 随机的User-Agent
class RandomUserAgent(object):
    def process_request(self, request, spider):
        useragent = random.choice(USER_AGENTS)
        #print useragent
        request.headers.setdefault("User-Agent", useragent)


class RandomProxy(object):
    def process_request(self, request, spider):
        proxy = random.choice(PROXIES)


        if proxy['user_passwd'] is None:
            # 没有代理账户验证的代理使用方式
            request.meta['proxy'] = "http://" + proxy['ip_port']


        else:
            # 对账户密码进行base64编码转换
            base64_userpasswd = base64.b64encode(proxy['user_passwd'])
            # 对应到代理服务器的信令格式里
            request.headers['Proxy-Authorization'] = 'Basic ' + base64_userpasswd


            request.meta['proxy'] = "http://" + proxy['ip_port']