scrapy爬取华为商城所有商品信息--科技快人一步

华为商城 https://www.vmall.com/index.html

目标:华为商城下的商品信息

• 按主页的左边手机,笔记本&平板,智能穿戴……分类
• 每一个分类下的小分类
  • 商品标题
  • 商品价格
• 规格参数
  • 主要参数
  • 主体
  • ……
  • 商品编码
• 写入excel
• 设置好excel数据表,分析数据

---

代码如下（scrapy）：

# -*- coding: utf-8 -*-
import os
import re
import urllib.request
from copy import deepcopy

import scrapy
import xlrd
import xlwt
from ..items import HuaweiItem


class HuaWei(scrapy.Spider):
    name = 'huawei'
    allowed_domains = ['vmall.com', 'vmallres.com']
    start_urls = ['http://vmall.com/']

    def parse(self, response):
        self.new_xls()
        # 主页
        print("分割线-----------------------主页------------------------分割线")
        classify_list_A = response.xpath('//div[@id="category-block"]/div/ol/li')
        print("大分类长度:", len(classify_list_A))
        for i in classify_list_A:
            # print("现在位置:", classify_list_A)
            item = HuaweiItem()
            item['classify_A'] = i.xpath('.//input[2]/@value').extract_first()
            classify_list = i.xpath('.//div[2]//li[not(@class="subcate-btn")]')
            # classify_list = i.xpath('.//div[2]//li[last()]')
            for i in classify_list:
                item['classify_B'] = i.xpath('.//input[1]/@value').extract_first()
                href = "https://www.vmall.com" + str(i.xpath('.//a/@href').extract_first()) + '-1-3-0'
                # print("href:", href)
                yield scrapy.Request(
                    href,
                    callback=self.parse_A,
                    meta={"item": deepcopy(item)}
                )
        rb = xlrd.open_workbook('华为商城.xls')
        # 通过sheet_by_index()获取的sheet
        rs = rb.sheet_by_index(0)
        print("已爬取的商品数量:", rs.nrows - 1)

    def parse_A(self, response):
        # 中间页
        print("分割线-----------------------中间页------------------------分割线")
        li_list = response.xpath('//div[@class="layout"]/div[@class="channel-list"]/div[@class="pro-list clearfix"]/ul/li')
        if li_list:
            print("正在爬取页面链接:", response.request.url)
            print("此页面商品数量:", len(li_list))
            for i in li_list:
                item = response.meta["item"]
                rb = xlrd.open_workbook('华为商城.xls')
                # 通过sheet_by_index()获取的sheet
                rs = rb.sheet_by_index(0)
                cods = rs.col_values(0, start_rowx=0, end_rowx=None)
                item['title'] = i.xpath('./div[1]/p[2]/a/span[1]/text()').extract_first()
                # print("+++++++++++++++++++++++++++++++++++++++++++", item['title'])
                item['price'] = round(float(i.xpath('./div[1]/p[3]/b/text()').extract_first().split("¥")[1]) if i.xpath('./div[1]/p[3]/b/text()') else 0, 2)
                item['comments'] = int(i.xpath('./div[1]/div[@class="p-button clearfix"]//label//text()').extract_first().split("人")[0])
                item['img'] = i.xpath('./div[1]/p[1]/a/img/@src').extract_first()
                item['href'] = "https://www.vmall.com" + i.xpath('./div[1]/p[1]/a/@href').extract_first()
                item['coding'] = re.findall('[(]\'(.*?)\'[)]', i.xpath('./div[1]/p[1]/a/@onclick').extract_first())
                # for s in cods:
                #     if s == item['coding']:
                #         break
                # print(cods)
                # print(item['coding'])
                if item['coding'][0] not in cods:
                    yield scrapy.Request(
                        item['href'],
                        callback=self.parse_B,
                        meta={"item": deepcopy(item)}
                    )
            next_url_len = len(response.xpath('//ul[@id="page_ul"]/a'))
            # print("::::::::::::::::::::::::;;", next_url_len)
            if int(response.request.url.split("-")[2]) < next_url_len:
                href = response.request.url.split("-")[0] + "-" + response.request.url.split("-")[1] + "-" + str(
                    int(response.request.url.split("-")[2]) + 1) + '-3-0'
                print("next_href:", href)
                yield scrapy.Request(
                    href,
                    callback=self.parse_A,
                    meta={"item": deepcopy(item)}
                )


    def parse_B(self, response):
        # 详情页
        print("分割线-----------------------详情页------------------------分割线")
        item = response.meta["item"]
        print("现在位置%s/%s" % (item["classify_A"], item["classify_B"]))
        print("正在爬取:", item['title'])
        content = response.xpath('//div[@id="product-property-recommand"]')
        if content:
            item['promotion'] = self.get_cx(response)
            item['coding'] = content.xpath(
                './div[@class="product-description clearfix"]/div[@class="fl"]/text()').extract_first().strip()
            item['explain'] = content.xpath('.//div[@id="skuPromWord"]//span/text()').extract_first()
            server_explain = content.xpath(
                './/div[@id="product-pulldown1"]/div[1]/div[@class="product-description-list clearfix"]/ul/li')
            item['server_explain'] = self.get_cm(server_explain)
            item['content'] = content.xpath('.//h1[@id="pro-name"]/text()').extract_first()
            cu_1 = re.findall(r'<script src="(.*?)" namespace="ec"></script>', response.text)[1]
            yield scrapy.Request(
                cu_1,
                callback=self.get_cu_1,
                meta={"item": deepcopy(item)},
                dont_filter=True
            )
        else:
            content = response.xpath('//div[@class="pro-meta-area"]')
            item['content'] = content.xpath('.//h1[@id="pro-name"]/text()').extract_first()
            item['explain'] = content.xpath('.//div[@id="skuPromWord"]//span/text()').extract_first()
            item['server_explain'] = content.xpath('.//div[@class="pro-service"]/text()').extract_first()
            item['promotion'] = "暂无活动"
            yield item

    def get_cx(self, response):
        print("获取促销")
        """获取促销数据"""
        str = ""
        cu = re.findall(r'_groupPhotoList.push[(]{name:.*?}[)]; (_promotionsList+.*?); _prolongLst.push', response.text)
        # print(cu)
        if cu:
            try:
                cs = re.findall(r'"(.*?)"', cu[1])
            except:
                cs = re.findall(r'"(.*?)"', cu[0])
                print(cu)
                print(len(cu))
            # print(cs)
            index = 0
            pop_list = []
            for i in cs:
                # 遍历促销，去掉没用的数据
                # print("开始", index)
                i = i.replace(r'&#x2f', "/")
                if i.find('&#x') != -1:
                    i = i.replace("&#x", "\\u")
                    i = i.replace(";", "")
                    i = i.replace("\n", "")
                    i = i.replace("\t", "")
                    i = i.replace(" ", "")
                    i = i.encode().decode('unicode-escape')
                    cs[index] = i
                else:
                    # print("添加了:", index)
                    pop_list.append(index)
                index += 1
                # print("结束")
            for i in pop_list[::-1]:
                cs.pop(i)
            ins = 0
            for i in cs:
                # print("index:", cs.index(i))
                str += i
                ins += 1
                if ins % 2 is 0:
                    str += ";"
                elif ins % 2 is 1:
                    str += ":"
        return str
        # cu_1 = re.findall(r'<script src="(.*?)" namespace="ec"></script>', response.text)[1]
        # print(cu_1)
        # yield scrapy.Request(
        #     cu_1,
        #     callback=self.get_cu_1,
        #     meta={"item": item, "str": str}
        # )
        # req = urllib.request.urlopen(cu_1)
        # req = req.read()
        # req = req.decode("utf-8")
        # str += self.get_cu_1(req)
        # print("str_s", str)
        # return str
        # item['promotion'] = str

        # print("cu_1", cu_1)
        # scrapy.Request(
        #     cu_1,
        #     callback=self.get_cu_1,
        #     meta={"item": item}
        # )
        # print(rs)
        #
        # return str
        # yield scrapy.Request(
        #     cu_1,
        #     callback=self.get_cu_1,
        #     meta={"item": item, "str": str}
        # )

    def get_cu_1(self, response):
        #获取促销（购买可的积分）
        print("进入GET_CU_1")
        item = response.meta["item"]
        print(item)
        cu1 = re.findall(r' \\x3e\'[)],a.push[(](.*?")[)],', response.text)[0]
        cul_1 = re.findall(r'\\x3e(.*?)\\x3c', cu1)[0].encode().decode('unicode-escape')
        cul_2 = re.findall(r'a.push[(]"(.*?)"', cu1)[0].encode().decode('unicode-escape')
        str = cul_1 + ":" + cul_2 + ";"
        print("--------------------------str----------------------------------")
        item['promotion'] += str
        if item['promotion'] is '':
            item['promotion'] = "暂无活动"
        yield item
        # self.get_cu_1(self)



    def new_xls(self):
        """创建表格"""
        if not os.path.exists("华为商城.xls"):
            print("正在创建。。。")
            wb = xlwt.Workbook(encoding='utf-8')
            # 括号内参数为表名
            ws = wb.add_sheet('商品数据')
            # 参数1：行数
            # 参数2：列数 从0开始计数
            # 参数3：值   即单元格的内容
            ws.write(0, 0, label='商品编码')
            ws.write(0, 1, label='祖分类')
            ws.write(0, 2, label='父分类')
            ws.write(0, 3, label='标题')
            ws.write(0, 4, label='图片')
            ws.write(0, 5, label='链接')
            ws.write(0, 6, label='价格')
            ws.write(0, 7, label='评价数量')
            ws.write(0, 8, label='内容')
            ws.write(0, 9, label='说明')
            ws.write(0, 10, label='服务说明')
            ws.write(0, 11, label="促销")
            wb.save('华为商城.xls')

    def get_cm(self, server_explain):
        # 获取服务说明
        cm = ""
        for i in server_explain:
            text = i.xpath('./text()')
            if len(text) > 1:
                mm = ""
                str_1 = re.findall(r'data=\'(.+?)\'>', str(text))
                if i.xpath('./span/text()'):
                    for k in str_1:
                        mm += k
                        if str_1.index(k) == 0:
                            mm += i.xpath('./span/text()').extract_first()
                cm += mm
            else:
                cm += str(text.extract_first()) + ';'

        return cm

git地址