Python爬虫request批量代理设置

大量爬取为了必备封IP，需要大量代理IP，代码如下

from urllib import error
import random,requests
# 代理IP
proxies_list = [
    {'http':'39.137.77.66:8080'},
    { 'http':'124.42.7.103:80'}
]

url = 'http://www.baidu.com'
# 免费代理会失效，需要try
try:
    proxies = random.choice(proxies_list)

    rsp = requests.post(url,proxies=proxies)
    rsp.encoding='utf-8'
    html = rsp.text
    print(html)

except error.URLError as e:
    print(e)
except Exception as e:
    print(e)

私密代理

import requests

# 如果代理需要使用HTTP Basic Auth，可以使用下面这种格式：
proxy = { "http": "mr_mao_hacker:sffqry9r@61.158.163.130:16816" }

response = requests.get("http://www.baidu.com", proxies = proxy)

print response.text