爬虫笔记-代理和代理池

1、代理

import urllib.request

url = 'http://www.baidu.com/s?ie=UTF-8&wd=ip'

headers = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36'
}
# 请求对象的定制
request = urllib.request.Request(url=url,headers=headers)

# 模拟服务器来访问服务器
# response = urllib.request.urlopen(request)

proxies = {
    'http':'118.24.219.151:16817'
}

# handler  build_opener  open
handler = urllib.request.ProxyHandler(proxies= proxies)

opener = urllib.request.build_opener(handler)

response = opener.open(request)
# 获取响应的信息
content = response.read().decode('utf-8')

# 保存在本地
with open('daili.html','w',encoding='utf-8') as fp:
    fp.write(content)

2、代理池

import urllib.request

proxies_pool = [
    {'http':'61.216.156.222:60808'},
    {'http':'47.106.105.236:80'},
    {'http':'106.15.197.250:8001'}
]

import random
proxies = random.choice(proxies_pool)

url = 'https://www.daidu.com/s?wd=ip'

headers = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36'
}

request = urllib.request.Request(url=url,headers=headers)

handler = urllib.request.ProxyHandler(proxies=proxies)

opener = urllib.request.build_opener(handler)

response = opener.open(request)

content = response.read().decode('utf-8')

with open ('daili_pool.html','w',encoding='utf-8') as fp:
    fp.write(content)