--------代理使用

settings.py

PROXIES = [
    # {'host' : '124.239.236.127:9999'},
    # {'host' : '202.100.167.149:80'},
]

AUTH_PROXIES = [
    {'host' : '120.78.166.84:6666', 'auth' : 'alice:123456'}

]

middleware.py

class FreeRandomProxy(object):
    def process_request(self,request,spider):
        # 随机选出代理信息
        # proxy = random.choice()
        proxy = get_proxy.getproxy()
        request.meta['proxy'] = '%s://%s:%s' % (proxy[2],proxy[0],proxy[1])

class AuthRandomProxy(object):
    def process_request(self,request,spider):
        # 随机选出代理信息
        proxy = random.choice(settings.AUTH_PROXIES)
        # 设置代理的认证信息
        auth = base64.b64encode(bytes(proxy['auth'],'utf-8'))
        request.headers['Proxy-Authorization'] = b'Basic ' + auth
        # 设置代理ip
        request.meta['proxy'] = 'http://' + proxy['host']