第一种 自定义middleware文件
爬虫文件
# -*- coding: utf-8 -*-
import scrapy
class TestSpider(scrapy.Spider):
name = 'test'
# allowed_domains = ['www.baidu.com']
start_urls = ['https://www.baidu.com/s?wd=ip']
# 本地爬虫配置文件
custom_settings = {
'DOWNLOADER_MIDDLEWARES' : {
'Test_C.middlewares.Random_UA': 1,
'Test_C.middlewares.Proxy_MD':2
},
}
def parse(self, response):
print('*_'*20)
print(response.css('.c-gap-right::text').extract())
print(response.css('.c-span21.c-span-last.op-ip-detail td::text').extract()[1].strip())
print(response.status)
# print(response.headers)
print('*_'*20)
middleware.py 文件