爬虫学习笔记-urllib_handler处理器、代理及代理池的基本使用

 hanlder处理器源码

import urllib.request

url = 'http://www.baidu.com'
headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
    }
request = urllib.request.Request(url=url,headers=headers)

# 1.获取hanlder对象
hanlder = urllib.request.HTTPHandler()
# 2.获取opener对象
opener = urllib.request.build_opener(hanlder)
# 3.调用open方法
response = opener.open(request)
# 4.接收响应数据
content = response.read().decode('utf-8')
print(content)

 代理源码

import urllib.request

url = 'http://www.baidu.com/s?wd=ip'
headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
    }
request = urllib.request.Request(url=url,headers=headers)

proxies = {
    'ip':'180.123.9.124:8888'
}
# 1.获取hanlder对象
hanlder = urllib.request.ProxyHandler(proxies=proxies)
# 2.获取opener对象
opener = urllib.request.build_opener(hanlder)
# 3.调用open方法
response = opener.open(request)
# 4.接收响应数据
content = response.read().decode('utf-8')
print(content)

 代理池源码

import random
import urllib.request

url = 'http://www.baidu.com/'
headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
    }
request = urllib.request.Request(url=url,headers=headers)

proxies_pool= [
    {'ip':'192.168.xxx.xxx:8885'},
    {'ip':'192.168.xxx.xxx:8886'},
    {'ip':'192.168.xxx.xxx:8887'},
    {'ip':'192.168.xxx.xxx:8888'}
]
#随机选择列表数据
proxies = random.choice(proxies_pool)
print(proxies)
# 1.获取hanlder对象
hanlder = urllib.request.ProxyHandler(proxies=proxies)
# 2.获取opener对象
opener = urllib.request.build_opener(hanlder)
# 3.调用open方法
response = opener.open(request)
# 4.接收响应数据
content = response.read().decode('utf-8')
print(content)