学习爬虫实战二
#encoding=utf-8
from lxml import etree
import requests
"""
爬取苏宁上的家具
第一层:爬取大分类
1.总入口:https://list.suning.com/?safp=d488778a.46601.0.4786e76351
链接:/html/body/div[5]/div[2]/div[1]/div/div[@class='t-left fl clearfix']/a/@href
大分类:/html/body/div[5]/div[2]/div[1]/div/div[@class='t-left fl clearfix']/a/@title
小分类:/html/body/div[5]/div[2]/div[1]/div/div[@class='t-right fl clearfix']/a/text()
"""
def get_html_by_etree(url,headers=None):
headers = {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36", "Referer":"https://suning.com/"}
urlContent = ''
req = requests.get(url, headers=headers)
encodings = None
encodings = requests.utils.get_encodings_from_content(req.text)
if encodings:
urlContent = req.content.decode(encodings[0])
else:
urlContent = req.content.decode("gb2312")
return urlContent, etree.HTML(urlContent)
if __name__ == "__main__":
level1Url= "https://list.suning.com/"
entryClass = "/html/body/div[5]/div[2]/div/h2/text()"
bigClasspath = "/html/body/div[5]/div[2]/div/div/div[@class='t-left fl clearfix']/a/@title" #大分类
subleftBigClassesNamePath= "/html/body/div[5]/div[2]/div[{}]/div/div[@class='t-left fl clearfix']/a/@title" #小品类
subrightClassesNamePath = "/html/body/div[5]/div[2]/div[{}]/div/div[@class='t-right fl clearfix']/a/text()" #小品类对应的细分产品
urlContent, etreeObj = get_html_by_etree(level1Url)
bigClasses = etreeObj.xpath(bigClasspath)
entryClasses = etreeObj.xpath(entryClass)
classCount = len(bigClasses)
dicData = {}
for i in range(len(entryClasses[:5])):
dicData[entryClasses[i]]={}
print("一级分类:{}:{}".format(i, entryClasses[i]))
secClasses = etreeObj.xpath(subleftBigClassesNamePath.format(i+1))
secClassLen = len(secClasses)
print(",".join(secClasses))
for j in range(secClassLen):
print("第{}级的二级分类-------:{}".format(i,secClasses[j]))
titles = etreeObj.xpath("/html/body/div[5]/div[2]/div[{}]/div[{}]/div[@class='t-right fl clearfix']/a/text()".format(i+1, j+1))
newtitles = []
for t in titles:
newtitles.extend( t.split("/") )
dicData[entryClasses[i]][secClasses[j]] = newtitles
print("第三级细分:{}".format(",".join(newtitles)))
print("ok")