爬虫笔记---构造随机User-Agent

1.构造随机User-Agent

import random

def get_ua():
    first_num = random.randint(55, 62)
    third_num = random.randint(0, 3200)
    fourth_num = random.randint(0, 140)
    os_type = [
        '(Windows NT 6.1; WOW64)', '(Windows NT 10.0; WOW64)', '(X11; Linux x86_64)',
        '(Macintosh; Intel Mac OS X 10_12_6)'
    ]
    chrome_version = 'Chrome/{}.0.{}.{}'.format(first_num, third_num, fourth_num)

    ua = ' '.join(['Mozilla/5.0', random.choice(os_type), 'AppleWebKit/537.36',
                   '(KHTML, like Gecko)', chrome_version, 'Safari/537.36']
                  )
    return ua

2.在scrapy框架的下载中间件中使用User-Agent，scrapy框架中不能把User-Agent写入headers中

<middlewares.py>  # 在下载中间件中写入自定义中间件
import random
def get_ua():
    first_num = random.randint(55, 62)
    third_num = random.randint(0, 3200)
    fourth_num = random.randint(0, 140)
    os_type = [
        '(Windows NT 6.1; WOW64)', '(Windows NT 10.0; WOW64)', '(X11; Linux x86_64)',
        '(Macintosh; Intel Mac OS X 10_12_6)'
    ]
    chrome_version = 'Chrome/{}.0.{}.{}'.format(first_num, third_num, fourth_num)

    ua = ' '.join(['Mozilla/5.0', random.choice(os_type), 'AppleWebKit/537.36',
                   '(KHTML, like Gecko)', chrome_version, 'Safari/537.36']
                  )
    return ua
    
class UserAgentMiddleware(object):
     def process_request(self,request,spider):)
         request.headers['User-Agent'] = get_ua()
<setting.py>  # 在setting文件中开启中间件
SPIDER_MIDDLEWARES = {
    'renren.middlewares.UserAgentMiddleware': 543,
}

3.定义实现随机使用代理的下载中间件，scrapy框架中不能把代理写入headers中

 class ProxyMiddleware(object):
     def process_request(self,request,spider):
         proxy = random.choice(proxies)
         request.meta['proxy'] = proxy