python+xpath爬取苏宁商品品类

学习爬虫实战二

#encoding=utf-8
from lxml import etree
import requests

"""
爬取苏宁上的家具
第一层：爬取大分类
1.总入口：https://list.suning.com/?safp=d488778a.46601.0.4786e76351
链接：/html/body/div[5]/div[2]/div[1]/div/div[@class='t-left fl clearfix']/a/@href   
大分类：/html/body/div[5]/div[2]/div[1]/div/div[@class='t-left fl clearfix']/a/@title
小分类：/html/body/div[5]/div[2]/div[1]/div/div[@class='t-right fl clearfix']/a/text()
 
"""

def get_html_by_etree(url,headers=None):
    headers = {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36", "Referer":"https://suning.com/"} 
    
    urlContent = ''
    req = requests.get(url, headers=headers)
    encodings = None
    encodings = requests.utils.get_encodings_from_content(req.text)
    if encodings:
        urlContent = req.content.decode(encodings[0])
    else:
        urlContent = req.content.decode("gb2312")
    return urlContent, etree.HTML(urlContent)

if __name__ == "__main__":
    level1Url= "https://list.suning.com/"
    entryClass = "/html/body/div[5]/div[2]/div/h2/text()"
    bigClasspath = "/html/body/div[5]/div[2]/div/div/div[@class='t-left fl clearfix']/a/@title"  #大分类
    subleftBigClassesNamePath= "/html/body/div[5]/div[2]/div[{}]/div/div[@class='t-left fl clearfix']/a/@title"  #小品类
    subrightClassesNamePath = "/html/body/div[5]/div[2]/div[{}]/div/div[@class='t-right fl clearfix']/a/text()"   #小品类对应的细分产品
    urlContent, etreeObj = get_html_by_etree(level1Url)
    bigClasses = etreeObj.xpath(bigClasspath)
    entryClasses = etreeObj.xpath(entryClass)
    classCount = len(bigClasses)
    dicData = {}
    for i in range(len(entryClasses[:5])):
        dicData[entryClasses[i]]={}
        
        print("一级分类：{}:{}".format(i, entryClasses[i]))
        secClasses = etreeObj.xpath(subleftBigClassesNamePath.format(i+1))
        secClassLen = len(secClasses)
        print(",".join(secClasses))
        for  j in range(secClassLen):
             
            print("第{}级的二级分类-------：{}".format(i,secClasses[j]))
            titles = etreeObj.xpath("/html/body/div[5]/div[2]/div[{}]/div[{}]/div[@class='t-right fl clearfix']/a/text()".format(i+1, j+1))
            newtitles = []
            for t in titles:
                newtitles.extend( t.split("/") )
            dicData[entryClasses[i]][secClasses[j]] = newtitles
            print("第三级细分：{}".format(",".join(newtitles)))   
          
    print("ok")