- 爬虫频繁更换User-Agent
# pip install fake_useragent
# 使用其中UserAgent模块
useragent = UserAgent()
response = requests.get(url,headers={
'User-Agent':useragent.random})
- 使用IP代理
proxies = {
'http': 'http://116.196.87.86:20183',
'https': 'https://106.37.195.199:8080'
}
response = requests.get(url, headers={
'User-Agent': useragent.random},
proxies=proxies)
- requests模块
Requests是一个专门用于HTTP请求的库
# 两种主要的HTTP请求方式
# GET请求
payload = {
'key1': 'value1', 'key2': 'value2'}
response = requests.get('https://api.github.com/events',params=payload)