此笔记只为本人复习为用,如有侵权请联系我,马上删除。
(一)首先在setting.py进行设置
IPPOOLS= ['http://183.207.95.45:80',
'http://202.85.213.220:3128'
]
DOWNLOADER_MIDDLEWARES = {P
'项目的名字.middlewares.ProxiesMiddleware': 490,
}
然后再middlewares.py中进行设置
class ProxiesMiddleware(object):
def __init__(self, ip):
self.ip = ip
@classmethod
def from_crawler(cls, crawler):
return cls(ip=crawler.settings.get('IPPOOLS'))
def process_request(self, request, spider):
ip = random.choice(self.ip)
request.meta['proxy'] = ip
(二)如何果在settings.py中IPPOOLS设置为
IPPOOLS=[
{“ip”:”192.168.0.3:8080”},
{“ip”:”192.168.0.1:9100”},
]
然后再middlewares.py中进行设置
import random
from scrapy import signals
from 项目.settings import IPPOOLS
class MyproxiesSpiderMiddleware(object):
def __init__(self,ip):
self.ip=ip
def process_request(self, request, spider):
thisip=random.choice(IPPOOLS)
request.meta["proxy"]="http://"+thisip["ip"]