Python语言使用爬虫IP代码示例

我们在使用爬虫ip对网站进行请求时，经常性会有访问失败的情况，引起这种原因的方式有很多，如果首先排除爬虫IP的问题，那么接下来就要对爬虫程序进行系统的检查，下文就是有关Python语言使用爬虫IP的一些经验可以供大家参考。

Python requests

import requests
targetURL = "http://jshk.com.cn/ip"
proxyAddr = "219.151.125.106:31615"
authKey = "895314XY"
password = "24D6YB309ZCB"
# 账密模式
proxyUrl = "http://%(user)s:%(password)s@%(server)s" % {
    "user": authKey,
    "password": password,
    "server": proxyAddr,
}
proxies = {
    "http": proxyUrl,
    "https": proxyUrl,
}
resp = requests.get(targetURL, proxies=proxies)
print(resp.text)

Python aiohttp

import aiohttp,asyncio
targetURL = "http://jshk.com.cn/ip"
proxyAddr = "219.151.125.106:31615"
authKey = "895314XY"
password = "24D6YB309ZCB"
# 账密模式
proxyUrl = "http://%(user)s:%(password)s@%(server)s" % {
    "user": authKey,
  "password": password,
  "server": proxyAddr,
}
async def entry():
    conn = aiohttp.TCPConnector(ssl=False)
    async with aiohttp.ClientSession(connector=conn) as session:
        async with session.get(targetURL, proxy=proxyUrl) as resp:
            body = await resp.read()
            print(resp.status)
            print(body)
loop = asyncio.get_event_loop()
loop.run_until_complete(entry())
loop.run_forever()

Python urllib2、urllib

import urllib2
targetURL = "http://jshk.com.cn/ip"
proxyAddr = "219.151.125.106:31615"
authKey = "895314XY"
password = "24D6YB309ZCB"
proxyUrl = "http://%(user)s:%(password)s@%(server)s" % {
    "user": authKey,
    "password": password,
    "server": proxyAddr,
}
proxies = urllib2.ProxyHandler({
    "http": proxyUrl,
    "https": proxyUrl,
})
opener = urllib2.build_opener(proxies)
urllib2.install_opener(opener)
resp = urllib2.urlopen(targetURL).read()
print(resp)