urllib和requests爬虫模块使用代理

urllib

from urllib import request

# 没有使用代理的
url = 'http://httpbin.org/ip'
resp = request.urlopen(url)
print(resp.read())

# 使用代理的
url = 'http://httpbin.org/ip'
# 1. 使用ProxyHandler，传入代理构建一个handler
handler = request.ProxyHandler({"http":"223.241.78.43:8010"})
# 2. 使用上面创建的handler构建一个opener
opener = request.build_opener(handler)
# 3. 使用opener去发送一个请求
resp = opener.open(url)
print(resp.read())

requests

import requests

proxy = {
    'http': '123.54.194.96:38661'
}

response = requests.get("http://httpbin.org/ip",proxies=proxy)
print(response.text)