今天在爬数据,使用lxml.path的时候出现了异常“UnicodeDecodeError: 'utf8' codec can't decode byte 0x87 in position 0: invalid start byte”,在此记录一下解决方案
出错的代码:
# coding: utf-8
import sys
from lxml import etree, html
import urllib2
import traceback
if __name__ == '__main__':
curl = 'http://www.hydcd.com/cd/htm1/ci000825j.htm'
print curl
req = urllib2.Request(curl)
response = urllib2.urlopen(req)
s = response.read()
root = etree.HTML(s)
tmp_words = root.xpath('//table/tr/td/div/font/a/font/text()')
for w in tmp_words:
print w
出错的Traceback
Traceback (most recent call last):
File "test.py", line 36, in <module>
tmp_words = root.xpath('//table/tr/td/div/font/a/font/text()')
File "lxml.etree.pyx", line 1444, in lxml.etree._Element.xpath (src/lxml/lxml.etree.c:41726)
File "xpath.pxi", line 321, in lxml.etree.XPathElementEvaluator.__call__ (src/lxml/lxml.etree.c:117867)
File "xpath.pxi", line 242, in lxml.etree._XPathEvaluatorBase._handle_result (src/lxml/lxml.etree.c:117069)
File "extensions.pxi", line 546, in lxml.etree._unwrapXPathObject (src/lxml/lxml.etree.c:112529)
File "extensions.pxi", line 580, in lxml.etree._createNodeSetResult (src/lxml/lxml.etree.c:112902)
File "extensions.pxi", line 602, in lxml.etree._unpackNodeSetEntry (src/lxml/lxml.etree.c:113078)
File "extensions.pxi", line 719, in lxml.etree._buildElementStringResult (src/lxml/lxml.etree.c:114139)
File "apihelpers.pxi", line 1344, in lxml.etree.funicode (src/lxml/lxml.etree.c:21872)
UnicodeDecodeError: 'utf8' codec can't decode byte 0x9a in position 0: invalid start byte
改进的方案:将页面内容先以ignore的方式先decode,这样能避免decode失败,具体编码看编码方式;
# coding: utf-8
from lxml import etree, html
import urllib2
if __name__ == '__main__':
curl = 'http://www.hydcd.com/cd/htm1/ci000825j.htm'
print curl
req = urllib2.Request(curl)
response = urllib2.urlopen(req)
s = response.read()
root = html.fromstring(s.decode('gb18030', 'ignore'))
tmp_words = root.xpath('//table/tr/td/div/font/a/font/text()')
for w in tmp_words:
print w