1. requests
import requests
url = 'http://www.baidu.com'
response =requests.get(url)
print(response)
print(response.text)
print(response.reason)
print(response.links)
print(response.history)
print(response.apparent_encoding
response.encoding = response.apparent_encoding)
print(response.content)
print(response.cookies)
print(response.headers)
print(response.request.url)
2.0 ip代理
from urllib.request import urlopen, Request, build_opener, ProxyHandler
import random
user_agent_list = [
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0'
]
headers ={
'User-Agent': 'random.choice(user_agent_list)'
}
ip_list = [
'124.193.85.88:8080',