将 url 对应的网页下载到本地
eg1
import urllib.request
def get_html(url):
response = urllib.request.urlopen(url)
buff = response.read()
html = buff.decode('utf8')
return html
if __name__ == '__main__':
url = "http://www.baidu.com"
html = get_html(url)
print(html)
结果:
eg2(with header)
import urllib.request
def get_html(url, headers):
req = urllib.request.Request(url)
for key in headers:
req.add_header(key, headers[key])
response = urllib.request.urlopen(req)
buff = response.read()
html = buff.decode("utf8")
return html
if __name__ == '__main__':
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36"
}
url = "http://www.baidu.com"
html = get_html(url, headers)
print(html)
eg3(post请求)
import urllib.request
import urllib.parse
def get_response(url, data):
data = bytes(urllib.parse.urlencode(data), encoding='utf8')
response = urllib.request.urlopen(url, data=data)
buff = response.read()
result = buff.decode("utf8")
return result
if __name__ == '__main__':
data = {
"key1": "value1",
"key2": "value2"
}
url = "http://httpbin.org/post"
html = get_response(url, data)
print(html)
结果: