解决xpath提取不规则网页的问题

在数据采集时，处理不规范的HTML页面是件令人头疼的事。因为不规范的HTML页面往往会引起xpath解析失败，造成得不到正确的数据。
这里引入tidylib,实例采用抓取慧聪网为例子,使用示例如下,

from tidylib import tidy_document
import requests
from lxml import etree

url = "https://s.hc360.com/?w=%BF%D5%B5%F7"
s = requests.Session()
# proxies={
#     "https":"https://192.168.43.1:1800"
# }
headers = {
    "Host": "s.hc360.com",
    "user-agent": "Mozilla/5.0 (X11; Linux x86_64) "
                  "AppleWebKit/537.36 (KHTML, like Gecko)"
                  " Chrome/67.0.3396.87 Safari/537.36",
    "cookies": "hcsearchurlport=1; visitid_time=2018-8-27%2011%3A0%3A1; hc360visitid=C81E205C97B000016A91177717E91DA7; hc360first_time=2018-08-27; hcbrowserid=C81E205CA0100001BD5AC09D4D99F260; hckIndex=C81E205CA0A00001613715F0773DE730; topmatchkey=; Hm_lvt_1437b8f613f9fcba581e33d8d178e1f5=1535338807; hccordet=00; hcpreurl=; hclastsearchkeyword=%u7A7A%u8C03; Hm_lvt_e1e386be074a459371b2832363c0d7e7=1535338809; hc360sessionid=C81E205EE6000001B74C508019802980; hc360sessionid=C81E205EE6000001B74C508019802980; hc360firstvisittime=1535338811032; hc360firstvisittime=1535338811032; _ga=GA1.2.1558106052.1535338807; _gid=GA1.2.1254508720.1535338856; hc360analyid=C81E24E7EC600001801D413B1900ED60; hc360analycopyid=C81E24E7EC6000016E7E7D601F0098B0; Hm_lpvt_1437b8f613f9fcba581e33d8d178e1f5=1535345882;"
               " Hm_lpvt_e1e386be074a459371b2832363c0d7e7=1535345884; hc5minbeat=1535346223259"

}
response = s.get(url, headers=headers).text
response, errors = tidy_document(response)
res = etree.HTML(response)
items = res.xpath('//div/ul/li[contains(@class,"grid-list")]')
print(response)
print(len(items))
for item in items:
    d = dict()
    d['name'] = item.xpath('.//div[2]/dl/dd/p/a/text()')
    d['title'] = item.xpath('.//div[@class="NewItem"]/div/a/@title')
    d['price'] = item.xpath('.//div[2]/dl/dt/span/text()')
    d['bcid'] = item.xpath('.//div[@class="NewItem"]/@data-bcid')
    d['username'] =item.xpath('.//div[@class="NewItem"]/@data-username')
    d['tel'] =item.xpath('./@data-telphone')
    d['businid'] =item.xpath('./@data-businid')
    d['sellerproviderid'] =item.xpath('./@data-sellerproviderid')
    d['supcatid'] =item.xpath('./@data-supcatid')
    d['main_product'] =item.xpath('.//div[@class="NewItem"]/@data-obj')
    print(d)

可以匹配导数据

{'businid': ['668378345'], 'username': ['yzhh'], 'sellerproviderid': ['100020178332'], 'bcid': ['668378345'], 'tel': ['13801459353'], 'price': ['2.36万/套'], 'supcatid': ['051009003'], 'title': ['HAIER/海尔家用中央空调智尊·MXS系列一拖四套餐/室外机5匹125-4-A'], 'name': ['青岛海尔空调电子'], 'main_product': ['空调,:0']}
{'businid': ['416147806'], 'username': ['yhjddmc'], 'sellerproviderid': ['100018953096'], 'bcid': ['416147806'], 'tel': ['18562142856'], 'price': ['450.00/台'], 'supcatid': ['012034003001'], 'title': ['安徽优质供应风机盘管|2匹壁挂式远博牌风机盘管|德州远博空调'], 'name': ['德州远博空调有限'], 'main_product': ['风机盘管机组,水源热泵机组,地源热泵机组,恒温恒湿机组 ,风冷冷（热）水机组,水冷冷（热）水机组,卡式风机盘管,卧式明装风机盘管,卧室暗装风机盘管 ,立式明装风机盘管,空气处理机组,组合式空调机组 ,立式暗装风机盘管,风机盘管,嵌入式风机盘管:0']}
{'businid': ['680346169'], 'username': ['hisurp'], 'sellerproviderid': ['100017484813'], 'bcid': ['680346169'], 'tel': ['18968377151'], 'price': ['5000.00/台'], 'supcatid': ['051003005'], 'title': ['GLONTER/吉霖特射流机组 车间降温空调 工厂降空调 工业车间空调 高大空间空调'], 'name': ['浙江吉霖特制冷设'], 'main_product': ['精密机房空调,特种定制空调,高温空调,屋顶式空调机组,直膨式空调机组,工业空调,风冷热泵机组,集装箱空调,洁净式空调机组,恒温恒湿机组,全新风空调机组,帐篷空调,组合式空调机组,医用净化风机盘管,电气室专用空调,粮库空调,三联供空调,冷凝热回收排风机组,基站空调,电柜空调:0']}
{'businid': ['689674421'], 'username': ['scjfangchao'], 'sellerproviderid': ['100008027596'], 'bcid': ['689674421'], 'tel': ['13098893885'], 'price': ['3700.00/台'], 'supcatid': ['051003012'], 'title': ['TCL 轿厢制冷空调 观光电梯空调 客梯货梯专用空调 湖北武汉电梯空调'], 'name': ['武汉阿力格科技有'], 'main_product': ['电子防潮柜,除湿机,移动空调,工业安全柜,防火防磁柜:0']}
{'businid': ['657068596'], 'username': ['bjhydkt'], 'sellerproviderid': ['100022760823'], 'bcid': ['657068596'], 'tel': ['18910393401'], 'price': ['7100.00/台'], 'supcatid': ['051009003'], 'title': ['海尔 海尔嵌入式吸顶四处风空调 四面出风;吊顶安装,节省空间 海尔在中央空调批发销售安装'], 'name': ['北京华远东机电设'], 'main_product': ['大金中央空调,约克中央空调,麦克维尔中央空调,海尔中央空调,美的中央空调,格力中央空调,开利中央空调,新风通风设备,给排水暖气,中央空调清洗保养:0']}
{'businid': ['685074827'], 'username': ['pulicool'], 'sellerproviderid': ['100029874313'], 'bcid': ['685074827'], 'tel': ['18927358608'], 'price': ['7.12万/台'], 'supcatid': ['051003010'], 'title': ['意恒森Y42 精密空调高效能机房专用空调恒温恒湿空调小型节能基站空调正压一体机空调'], 'name': ['深圳市普立精密空'], 'main_product': ['机房空调,精密空调,恒温恒湿密码,工业特种空调,酒窖专用空调,试验室专用空调,列间空调,基站空调,洁净空调,除湿机:0']}
{'businid': ['627871625'], 'username': ['sdfx317'], 'sellerproviderid': ['100022280992'], 'bcid': ['627871625'], 'tel': ['13520363056'], 'price': ['3.16万/台'], 'supcatid': ['051003010'], 'title': ['英维克空调 CY512P 机房精密空调 英维克12.5KW恒温恒湿机房空调 上送风 英维克机房空调'], 'name': ['北京晟达飞翔科技'], 'main_product': ['UPS电源,蓄电池,机房精密空调,:0']}
{'businid': ['568871043'], 'username': ['xuxiangruite'], 'sellerproviderid': ['100010742933'], 'bcid': ['568871043'], 'tel': ['13699197591'], 'price': ['9.60万/台'], 'supcatid': ['051003010'], 'title': ['艾默生空调 大型机房专用精密空调PEX全系列 P1035F三相供电 单系统35KW'], 'name': ['北京旭祥瑞特科技'], 'main_product': ['UPS电源,梅兰日兰UPS电源,山特UPS,APCUPS电源,铅酸蓄电池,汤浅电池,松下电池,冠军电池,山特电池,大力神电池,艾默生UPS电源,山顿UPS电源,科华UPS,科华电池,不间断电源,UPS,废旧电池回收,:0']}
{'businid': ['692521377'], 'username': ['wanniukongtiao'], 'sellerproviderid': ['100030367971'], 'bcid': ['692521377'], 'tel': ['15900691268'], 'price': ['1.00/台'], 'supcatid': ['051009001'], 'title': ['吸顶机 空调 冷库 中央空调 厂家批发 皖牛冷暖设备工程'], 'name': ['上海皖牛冷暖设备'], 'main_product': ['二手空调,挂壁式空调,柜式空调,嵌入式空调,天花机,家用商用多联机,风管机,风冷模块热泵机组,水冷模块机组,吸顶机:0']}
{'businid': ['601637762'], 'username': ['sdgrdlst'], 'sellerproviderid': ['100024717625'], 'bcid': ['601637762'], 'tel': ['13053405000'], 'price': ['1000.00/台'], 'supcatid': ['051009016'], 'title': ['型号齐全的山东格瑞德组合式空调,价格低廉 格瑞德组合式空调机组'], 'name': ['山东格瑞德集团有'], 'main_product': ['轴流风机,离心风机,排烟风机,:0']}
{'businid': ['629725377'], 'username': ['dingshengjn'], 'sellerproviderid': ['100027390537'], 'bcid': ['629725377'], 'tel': ['13926892719'], 'price': ['880.00/台'], 'supcatid': ['051003019'], 'title': ['億兴达厂家DS-4500 批发移水空调蒸发式移动冷风机江西节能环空调'], 'name': ['东莞市鼎盛节能设'], 'main_product': ['空气能热水器,环保空调,水空调,冷风机,节能空调:0']}
{'businid': ['690727109'], 'username': ['qhdfxzl'], 'sellerproviderid': ['100030213987'], 'bcid': ['690727109'], 'tel': ['13643351986'], 'price': ['1900.00/台'], 'supcatid': ['051003009'], 'title': ['美博KFRD-35G/S350 空调'], 'name': ['秦皇岛市峰雪制冷'], 'main_product': [':0']}
{'businid': ['669800172'], 'username': ['htgxm'], 'sellerproviderid': ['100001224511'], 'bcid': ['669800172'], 'tel': ['15021052444'], 'price': ['3.22/台'], 'supcatid': ['051009001'], 'title': ['供应众有RF84W 恒温恒湿机空调'], 'name': ['上海众有实业有限'], 'main_product': ['除湿机,加湿器,工业除湿机,机房精密空调,风冷冷水机,转轮除湿机,恒温恒湿机,调温除湿机,除湿设备,防爆空调,管道除湿机,工业冷水机,酒窖空调,机房空调,直膨式空调机组:0']}
{'businid': ['522230399'], 'username': ['sscyscl'], 'sellerproviderid': ['100010410833'], 'bcid': ['522230399'], 'tel': ['18642654172'], 'price': ['1200.00/台'], 'supcatid': ['051009011'], 'title': ['志高 空调 壁挂式空调 工业暖风机 高温辐射器 暖风机'], 'name': ['大连圣星华越机电'], 'main_product': ['电热风幕机,贯流风幕机,圆弧风幕机,远红外高温辐射器,水汽/电风幕机,高温辐射器,风幕机,海尔空调,大金/志高空调,商用净水器及配件,太阳能热水器,新风换气机,净水器滤芯及滤料,空气净化器装修除笨等,商用家用加湿机,除湿机,风机盘管,风淋室,空气源热泵,:0']}
{'businid': ['661247394'], 'username': ['gdqe183'], 'sellerproviderid': ['100028842752'], 'bcid': ['661247394'], 'tel': ['13809277580'], 'price': ['1750.00/台'], 'supcatid': ['051003005'], 'title': ['企鹅QE-300 电气柜空调'], 'name': ['广东企鹅冷链技术'], 'main_product': ['冷库,保鲜库,冷库机组,电气柜空调,:0']}
{'businid': ['236561461'], 'username': ['kgdadn'], 'sellerproviderid': ['100014423128'], 'bcid': ['236561461'], 'tel': ['13827202698'], 'price': ['4099.00/台'], 'supcatid': ['051003022'], 'title': ['供应顺德乐从环保空调 顺德环保空调 顺德科瑞莱空调'], 'name': ['东莞市科永机电设'], 'main_product': ['科瑞莱环保空调,水冷空调,冷风机,冷风扇,蒸发式冷气机,环保空调配件,负压风机,降温湿帘,环保空调拆装,清洗环保空调,环保空调维修,环保空调销售,:0']}
{'businid': ['686522918'], 'username': ['rymlanbo'], 'sellerproviderid': ['100015076593'], 'bcid': ['686522918'], 'tel': ['13779998924'], 'price': ['2300.00/台'], 'supcatid': ['012034003007'], 'title': ['TCL1.5匹变频 KFRD-35GW/BW13BPA 壁挂式挂机冷暖定速静音空调'], 'name': ['厦门蓝博科技开发'], 'main_product': ['不锈钢水箱:0']}
{'businid': ['683996048'], 'username': ['shxc18'], 'sellerproviderid': ['100029220951'], 'bcid': ['683996048'], 'tel': ['18916043666'], 'price': ['3500.00/个'], 'supcatid': ['012034003007'], 'title': ['上海雪巢 厨房间空调(冷暖型)KFQ-20-(R22A)-LN-36Y-XCCW'], 'name': ['马德明'], 'main_product': [':0']}
{'businid': ['517446489'], 'username': ['huilingdq'], 'sellerproviderid': ['100021898049'], 'bcid': ['517446489'], 'tel': ['15932777171'], 'price': ['2000.00/台'], 'supcatid': ['012034003001'], 'title': ['惠菱壁挂式空调冷制热KFRD-36GWB 新款空调 花漾]惠菱空调壁挂式空调白色制可私人定制 空调厂家'], 'name': ['河北惠菱电器有限'], 'main_product': ['家用空调,柜式空调,壁挂空调,冰箱,洗衣机:0']}
{'businid': [], 'username': [], 'sellerproviderid': [], 'bcid': [], 'tel': [], 'price': ['250.00/台'], 'supcatid': [], 'title': [], 'name': [], 'main_product': []}