python爬虫之requests库与Selenium库代理设置详解
1. requests库代理的设置
1.1 不需要认证的代理
import requests
proxy='127.0.0.1:1943'
proxies={
'http':'http://'+proxy,
'http':'https://'+proxy,
}
html=requests.get(url,proxies=proxies)
print(html.text)
1.2 对于需要认证的代理,写法如下
proxy='username:password@127.0.0.1:1943'
1.3 如果需要使用SOCK5代理,则可以使用如下方式来设置
proxy='127.0.0.1:1943'
proxies={
'http':'sock5://'+proxy,
'http':'sock5://'+proxy,
}
1.4 另外一种设置代理的方法
pip3 install PySocks
import requests
import socks
import socketsocks.set_default_proxy(socks.SOCKS5, '127.0.0.1', 9742)
socket.socket = socks.socksocket
try:
response = requests.get('http://httpbin.org/get')
print(response.text)
except requests.exceptions.ConnectionError as e:
print('Error', e.args)
2. Selenium库代理设置
2.1 Chrome代理设置
2.1.1 不需要认证
from selenium import webdriver
proxy='127.0.0.1:9743'
chrome_options=webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=http://'+proxy)
browser = webdriver.Chrome(chrome_options=chrome_options)
browser.get('https://www.baidu.com')
2.1.2 若代理需要认证(相对比较复杂)
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import zipfile
ip = '127.0.0.1'
port = 9743
username = 'foo'
password = 'bar'
manifest_json = """
{
"version": "1.0.0",
"manifest_version": 2,
"name": "Chrome Proxy",
"permissions": [
"proxy",
"tabs",
"unlimitedStorage",
"storage",
"<all_urls>",
"webRequest",
"webRequestBlocking"
],
"background": {
"scripts": ["background.js"]
}
}
"""
background_js = """
var config = {
mode: "fixed_servers",
rules: {
singleProxy: {
scheme: "http",
host: "%(ip)s",
port: %(port)s
}
}
}
chrome.proxy.settings.set({value: config, scope: "regular"}, function() {});
function callbackFn(details) {
return {
authCredentials: {
username: "%(username)s",
password: "%(password)s"
}
}
}
chrome.webRequest.onAuthRequired.addListener(
callbackFn,
{urls: ["<all_urls>"]},
['blocking']
)
""" % {'ip': ip, 'port': port, 'username': username, 'password': password}
plugin_file = 'proxy_auth_plugin.zip'
with zipfile.ZipFile(plugin_file, 'w') as zp:
zp.writestr("manifest.json", manifest_json)
zp.writestr("background.js", background_js)
chrome_options = Options()
chrome_options.add_argument("--start-maximized")
chrome_options.add_extension(plugin_file)
browser = webdriver.Chrome(chrome_options=chrome_options)
browser.get('http://httpbin.org/get')
2.2 PhantomJS
2.2.1 不需要认证
from selenium import webdriver
service_args=[
'--proxy=127.0.0.1:9743',
'--proxy-type=http'
]
browser=webdriver.PhantomJS(service_args=service_args)
browser.get('http://httpbin.org/get')
2.2.2 需要认证,只需要加入–proxy–auth选项即可
service_args=[
'--proxy=127.0.0.1:9743',
'--proxy-type=http',
'--proxy-auth=username:password'
]