解决xpath提取不规则网页的问题

在数据采集时，处理不规范的HTML页面是件令人头疼的事。因为不规范的HTML页面往往会引起xpath解析失败，造成得不到正确的数据。
这里引入tidylib,实例采用抓取慧聪网为例子,使用示例如下,

from tidylib import tidy_document
import requests
from lxml import etree

url = "https://s.hc360.com/?w=%BF%D5%B5%F7"
s = requests.Session()
# proxies={
   
#     "https":"https://192.168.43.1:1800"
# }
headers = {
    "Host": "s.hc360.com",
    "user-agent": "Mozilla/5.0 (X11; Linux x86_64) "
                  "AppleWebKit/537.36 (KHTML, like Gecko)"
                  " Chrome/67.0.3396.87 Safari/537.36",
    "cookies": "hcsearchurlport=1; visitid_time=2018-8-27%2011%3A0%3A1; hc360visitid=C81E205C97B000016A91177717E91DA7; hc360first_time=2018-08-27; hcbrowserid=C81E205CA0100001BD5AC09D4D99F260; hckIndex=C81E205CA0A00001613715F0773DE730; topmatchkey=; Hm_lvt_1437b8f613f9fcba581e33d8d178e1f5=1535338807; hccordet=00; hcpreurl=; hclastsearchkeyword=%u7A7A%u8C03; Hm_lvt_e1e386be074a459371b2832363c0d7e7=1535338809; hc360sessionid=C81E205EE6000001B74C508019802980; hc360sessionid=C81E205EE6000001B74C508019802980; hc360firstvisittime=1535338811032; hc360firstvisittime=1535338811032; _ga=GA1.2.1558106052.1535338807; _gid=GA1.2.1254508720.1535338856; hc360analyid=C81E24E7EC600001801D413B1900ED60; hc360analycopyid=C81E24E7EC6000016E7E7D601F0098B0; Hm_lpvt_1437b8f613f9fcba581e33d8d178e1f5=1535345882;"
               " Hm_lpvt_e1e386be074a459371b2832363c0d7e7=1535345884; hc5minbeat=1535346223259"

}
response = s.get(url, headers=headers).text
response, errors = tidy_document(response)
res = etree.HTML(response)
items = res.xpath('//div/ul/li[contains(@class,"grid-list")]')
print(response)
print(len(items))
for item in items:
    d = dict()
    d['name'] = item.xpath('.//div[2]/dl/dd/p/a/text()')
    d['title'] = item.xpath('.//div[@class="NewItem"]/div/a/@title')
    d['price'] = item.xpath('.//div[2]/dl/dt/span/text()')
    d['bcid'] = item.xpath('.//div[@class="NewItem"]/@data-bcid')
    d['username'] =item.xpath('.//div[@class="NewItem"]/@data-username')
    d['tel'] =item.xpath('./@data-telphone')
    d['businid'] =item.xpath('./@data-businid')
    d['sellerproviderid'] =item.xpath('./@data-sellerproviderid')
    d['supcatid'] =item.xpath('./@data-supcatid')
    d['main_product'] =item.xpath('.//div[@class="NewItem"]/@data-obj')
    print(d)

可以匹配导数据

{
  'businid': ['668378345'], 'username': ['yzhh'], 'sellerproviderid': ['100020178332'], 'bcid': ['668378345'], 'tel': ['13801459353'], 'price': ['2.36万/套'],