scrapy设置user_agent及代理ip

在 middlewares.py 中添加如下代码，即可添加user_agent及代理ip

// middlewares.py
class UserAgentMiddleware(object):
    USER_AGENTS = [
        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36',
        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2',
        'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
        'Opera/8.0 (Windows NT 5.1; U; en)'
    ]

    def process_request(self,request,spider):
        user_agent = random.choice(self.USER_AGENTS)
        request.headers['User-Agent'] = user_agent


class ProxyMiddleware(object):
    PROXIES = ["http://116.252.39.176:53281","http://116.252.39.176:53281"]

    def process_request(self,request,spider):
        proxy = random.choice(self.PROXIES)
        request.meta['proxy'] = proxy

注意事项：需在settings.py中添加相应的中间件，如下

DOWNLOADER_MIDDLEWARES = {
   'Middle_ware.middlewares.UserAgentMiddleware': 543,
   'Middle_ware.middlewares.ProxyMiddleware': 100,
}