1、爬取晋江网页
(1)报错:UnicodeDecodeError: 'utf-8' codec can't decode byte 0x8b in position 1: invalid start by...
(2)原因:网站发包是gzip格式,因此解码会是乱码:headers['Accept-Encoding'] = "gzip
(3)解决
用gzip包解压传回来的网页
正常
def askURL(url):
head = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36 Edg/87.0.664.75"
}
request = urllib.request.Request(url, headers=head)
html = ""
try:
response = urllib.request.urlopen(request)
html = response.read().decode('GBK')
# html = response.read()
# buff = BytesIO(html)
# f = gzip.GzipFile(fileobj=buff)
# html = f.read().decode("GBK")
# 这是之前解不开编码时尝试的其它方法
except urllib.error.URLError as e:
if hasattr(e, "code"):
print(e.code)
if hasattr(e, "reason"):
print(e.reason)
return html
gzip:
import gzip
from io import BytesIO
def askURL_novel(url):
head = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36 Edg/87.0.664.75",
"Accept-encoding": "gzip"