使用代理流程
爬取数据
最基本的爬取
import requestsurl = http://current.ip.16yun.cn:802response = requests.get(url)content = requests.get(url).contentprint("response headers:", response.headers)print("content:", content)
import requestsdata = {'wd':'nike', 'ie':'utf-8'}url='https://www.baidu.com'response = requests.get(url=url, params=data)
如何配置动态的代理ip
import requests import random # 要访问的目标页面 targetUrl = "http://httpbin.org/ip" # 要访问的目标HTTPS页面 # targetUrl = "https://httpbin.org/ip" # 代理服务器(产品官网 www.16yun.cn) proxyHost = "t.16yun.cn" proxyPort = "31111" # 代理隧道验证信息 proxyUser = "username" proxyPass = "password" proxyMeta = "http://%(user)s:%(pass)s@%(host)s:%(port)s" % { "host" : proxyHost, "port" : proxyPort, "user" : proxyUser, "pass" : proxyPass, } # 设置 http和https访问都是用HTTP代理 proxies = { "http" : proxyMeta, "https" : proxyMeta, } # 设置IP切换头 tunnel = random.randint(1,10000) headers = {"Proxy-Tunnel": str(tunnel)} resp = requests.get(targetUrl, proxies=proxies, headers=headers) print resp.status_code print resp.text