Python爬虫笔记（urllib模块）

# 测试地址：httpbin.org

import urllib.request

# 获取一个get请求
response = urllib.request.urlopen("http://baidu.com")
print(response.read().decode("utf-8"))

# 获取一个post请求
import urllib.parse
data = bytes(urllib.parse.urlencode({"user":"password"}),encoding="utf-8")
response = urllib.request.urlopen("http://httpbin.org/post",data= data)
print(response.read().decode("utf-8"))

# 超时处理
try:
    response = urllib.request.urlopen("http://httpbin.org/get",timeout=0.1)
    print(response.read().decode("utf-8"))
except urllib.error.URLError as e:
    print("time out!")
    
# 获取response信息
response = urllib.request.urlopen("http://baidu.com")
print(response.status)
print(response.getheaders())
print(response.getheader("Server"))

# 模拟header访问豆瓣
url = "http://douban.com"
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36"
}
# data = bytes(urllib.parse.urlencode({"user":"password"}),encoding="utf-8") # post参数
req = urllib.request.Request(url=url,headers=headers)
response = urllib.request.urlopen(req)
print(response.read().decode("utf-8"))