urlib 获取网页

将 url 对应的网页下载到本地

eg1

import urllib.request


def get_html(url):
    response = urllib.request.urlopen(url)
    buff = response.read()
    html = buff.decode('utf8')
    return html


if __name__ == '__main__':
    url = "http://www.baidu.com"
    html = get_html(url)
    print(html)

结果： 

---

 eg2（with header）

import urllib.request


def get_html(url, headers):
    req = urllib.request.Request(url)
    for key in headers:
        req.add_header(key, headers[key])
    response = urllib.request.urlopen(req)
    buff = response.read()
    html = buff.decode("utf8")
    return html


if __name__ == '__main__':
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36"
    }
    url = "http://www.baidu.com"
    html = get_html(url, headers)
    print(html)

---

eg3(post请求)

import urllib.request
import urllib.parse


def get_response(url, data):
    data = bytes(urllib.parse.urlencode(data), encoding='utf8')
    response = urllib.request.urlopen(url, data=data)
    buff = response.read()
    result = buff.decode("utf8")
    return result


if __name__ == '__main__':
    data = {
        "key1": "value1",
        "key2": "value2"
    }
    url = "http://httpbin.org/post"
    html = get_response(url, data)
    print(html)

结果：