scrapy随机UserAgent

#middlewares.py
from fake_useragent import UserAgent  #使用在线UA列表


class UserAgentMiddleware(object):
    def __init__(self,user_agent=''):
        self.ua = UserAgent(verify_ssl=False)#生成ua,忽略ssl证书认证
        '''
        当请求对象交给下载器downloader处理之前，先要触发down middleware的process_request方法
        '''
    def process_request(self, request, spider):
        if self.ua:
            print("++++++++++++++Current UserAgent:%s++++++++++++++++++")
            custom_ua = self.ua.random
            print('custom_ua:', custom_ua)
            request.headers.setdefault(b'User-Agent',custom_ua)
#settings.py
DOWNLOADER_MIDDLEWARES = {
   'bilibili.middlewares.UserAgentMiddleware': 333,
}

pip install fake_useragent