python 爬虫技术——代理池

"""
    headers: 是爬虫和反爬虫斗争的第一步

    代理 proxies:
        代理是反爬虫和爬虫斗争的第二步

    获取代理IP的网站：
        1.西刺代理
        2.快代理
        3.全网代理
    1.普通代理：
        proxies = {"协议":"IP 地址:端口号"}
        测试：117.91.232.53 9999  HTTP
"""
import requests
import random
L_ip = [
    {"http":"120.43.62.124:22909"},
    {"http":"123.54.54.174:22952"},
    {"http":"182.132.203.91:22941"}
]
url = "http://www.taobao.com/"
url1 = "http://httpbin.org/get"

headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36"}

# 这是使用的西刺免费代理---代理ip失效或者使用的人太多，所以访问出错
# proxy = {"http":'163.204.244.234:9999'}

proxy = {"http":'123.162.25.56:19100'}

res = requests.get(url1,proxies=proxy,headers=headers)

res.encoding = "utf-8"

print(res.text)
# for i in range(5):
#     try:
#         # 每次都随机从IP池中获取IP
#         proxies = random.choice(L_ip)
#
#         res = requests.get(url,proxies = proxies)
#         print(proxies)
#         res.encoding = "utf-8"
#         print(res.text)
#     except Exception as e:
#         print(i)