python爬虫 -01- urllib的简单实用

get请求

import urllib.request
import json

res = urllib.request.urlopen("http://httpbin.org/get")
# 读取response的内容
text = res.read()
# http返回状态码
print(res.status, res.reason)
obj = json.loads(text)
print(obj)

for k, v in res.headers._headers:
    print(f"({k}:{v})")

 输出

自定义头信息

import urllib.request
import json

# 添加自定义的头信息
ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"
req = urllib.request.Request("http://httpbin.org/user-agent")
req.add_header("User-Agent", ua)
res = urllib.request.urlopen(req)
resp = json.load(res)
print("user-agent", resp["user-agent"])

身份认证

import urllib.request
import json

auth_handler = urllib.request.HTTPBasicAuthHandler()
auth_handler.add_password(realm="httpbin auth", uri="/basic-auth/admin/123456",
                          user="admin", passwd="123456")
opener = urllib.request.build_opener(auth_handler)
urllib.request.install_opener(opener)
res = urllib.request.urlopen("http://httpbin.org")
print(res.read().decode("utf-8"))

带参数(get,post)

import urllib.request
import urllib.parse
import json

# get
params = urllib.parse.urlencode({"param": 1, "eggs": 2, "bacon": 2})
url = f"http://httpbin.org/get?{params}"
res = urllib.request.urlopen(url)
print(json.load(res))

# post
data = urllib.parse.urlencode({"name": "小明", "age": 20})
data = data.encode()
res = urllib.request.urlopen("http://httpbin.org/post", data)
print(json.load(res))