处理未知页面的编码问题

乱码演示

url = 'https://newhouse.fang.com/house/s/#'
a_li = '//div[@id="cityi010"]/a'
aa = './text()'
header = {'User-Agent': 'Mozilla/5.0 (compatible; Baiduspider/2.0;+http://www.baidu.com/search/spider.html)'}
res = requests.get(url=url, headers=header)
etr = etree.HTML(res.text)
a_lis = []
for k in etr.xpath(a_li):
    kk = k.xpath(aa)[0]
    print(kk)
    try:
        a = k.xpath(aa)[0].replace('\r','').replace('\n', '').replace(' ', '').en
        a_lis.append(a)
    except:
        a_lis.append(' ')
print(a_lis)
print(len(a_lis))

 

完成代码

url = "https://newhouse.fang.com/house/s/#"
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36'}
r = requests.get(url, headers=headers)

# 检测编码
raw_data = requests.get(url).content
charset = chardet.detect(raw_data)
encoding = charset['encoding']
# 更改编码方式
r.encoding = encoding
print(r.text)  # 未出现乱码