代理--ip池的使用--阿布云的使用

代理

import urllib.request

proxy = {
    "http": "34.84.238.121"
}
url = "https://www.baidu.com/s?ie=UTF-8&wd=ip"

handler = urllib.request.ProxyHandler(proxies=proxy)
opener = urllib.request.build_opener(handler)
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'
}
request = urllib.request.Request(url=url, headers=headers)
response = opener.open(request)

with open("daili.html", "wb") as fp:
    fp.write(response.read())

ip池

# coding: utf-8
import urllib.request
import random
import urllib.error
import time

def get_ipool():
    return [
    '34.84.238.121',
    '218.60.8.98:3129',
    '113.200.56.13:8010',
    # '61.160.247.63:808',
    # '59.38.60.191:9797'
    ]
# 代理IP
ip_pool = get_ipool()
url = "https://www.baidu.com/s?ie=UTF-8&wd=ip"
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'
}
request = urllib.request.Request(url=url, headers=headers)
# 随机一个代理
while 1:
    ip = random.choice(ip_pool)
    print("打印随机代理ip为--%s--" %ip)
    ptoxy = {
        "http": ip
    }
    handler = urllib.request.ProxyHandler(proxies=ptoxy)
    opener = urllib.request.build_opener(handler)
    try:
        response = opener.open(request, timeout=5)
        print('代理--%s--使用成功' % ip)
        with open("daili2.html", "wb")as fp:
            fp.write(response.read())
        break
    except urllib.error.URLError as e:
        print('代理--%s--使用失败' % ip)
        ip_pool.remove(ip)
        if len(ip_pool) < 2:
            ip_pool = get_ipool()
        time.sleep(2)

阿布云（收费使用）

import urllib.request
import base64

# 将用户名和密码通过冒号拼接，然后base64一下
user = 'HEA6N8B46645BG6D'
pwd = '81D9834E399D2C79'
string = user + ':' + pwd
value = base64.b64encode(string.encode('utf-8')).decode('utf8')

url = 'http://www.baidu.com/s?ie=UTF-8&wd=ip'
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',
    'Proxy-Authorization': 'Basic ' + value
}
# 构建请求对象
request = urllib.request.Request(url=url, headers=headers)

# 创建handler和opener
handler = urllib.request.ProxyHandler(proxies={'http': 'http-dyn.abuyun.com:9020'})
opener = urllib.request.build_opener(handler)

response = opener.open(request)

with open('abu.html', 'wb') as fp:
    fp.write(response.read())