Python Scrapy 爬取易车品牌 车系 车款（在售 停售）及车辆参数

基于Scrapy  爬取易车品牌 车系 车款（在售  停售）及车辆参数

源码地址：https://download.csdn.net/download/wpaycn/11548531

 

# -*- coding: utf-8 -*-
import scrapy
import re
import json
import logging
from copy import deepcopy
from yicar.items import YicarItem

logger = logging.getLogger(__name__)


# json替换key
def replacea(matched):
    return '\"' + matched.group('value') + '\":'


class YcSpider(scrapy.Spider):
    name = 'yc'
    allowed_domains = ['bitauto.com']
    start_urls = [
        'http://api.car.bitauto.com/CarInfo/getlefttreejson.ashx?tagtype=chexing&pagetype=masterbrand&objid=0']

    def parse(self, response):
        result = re.sub('(?P<value>\w+):', replacea,
                        response.text[response.text.find('{'):response.text.rfind('}') + 1].replace('https:', ''))
        data = json.loads(result)
        for char in data['char']:
            try:
                for brand in data['brand']['%s' % char]:
                    item = {}
                    item["params"] = {}
                    item['id'] = brand['id']
                    item['name'] = brand['name']
                    item['initial'] = char

                    url = 'http://car.bitauto.com/tree_chexing/mb_{}/'.format(item['id'])
                    item['url'] = url

                    yield scrapy.Request(
                        url,
                        callback=self.parse_serial,
                        meta={"item": deepcopy(item)}
                    )
                    return
            except KeyError:
                pass

    def parse_serial(self, response):
        item = response.meta.get("item")

        brands_div = response.xpath("//div[@id='divCsLevel_0']")

        car_cates = brands_div.xpath('./h5')

        # 循环遍历得到子品牌
        brand_item_id = 0
        for i in car_cates:
            brand_item_id = brand_item_id + 1
            brand_cate = brands_div.xpath('./h5[' + str(brand_item_id) + ']/a/text()').extract_first()
            #    logger.warning(brand_cate)
            item["brand_item"] = brand_cate
            brand_item = brands_div.xpath("./div[" + str(brand_item_id) + "]")
            brand_item_col = brand_item.xpath("./div[@class='col-xs-3']")
            for j in brand_item_col:
                brand_name = j.xpath("./div/ul/li[contains(@class,'name')]/a/text()").extract_first()
                brand_price = j.xpath("./div/ul/li[@class='price']/a/text()").extract_first()
                brand_url = j.xpath("./div/ul/li[contains(@class,'name')]/a/@href").extract_first()
                brand_url = "http://car.bitauto.com" + str(brand_url)
                #     logger.warning(str(brand_name) + "---" + str(brand_price) + "---" + brand_url)
                item["model"] = str(brand_name)
                item["price"] = str(brand_price)
                if str(brand_price) == "未上市":
                    pass
                else:
                    logger.warning("##### " + item["model"] + " #######")
                    yield scrapy.Request(
                        brand_url,
                        callback=self.parse_vehicle,
                        meta={"item": deepcopy(item)}
                    )
                    return

    # 获取在售车款
    def parse_vehicle(self, response):
        item = response.meta.get("item")

        # 在售车款
        tr_list = response.xpath("//tr[contains(@id,'car_filter_id')]/td[1]")
        for tr in tr_list:
            vehicle_name = tr.xpath("./a/text()").extract_first()
            vehicle_url = tr.xpath("./a/@href").extract_first()
            vehicle_url = "http://car.bitauto.com" + str(vehicle_url)
            logger.warning(vehicle_name + "  ---  " + vehicle_url)
            #  logger.warning(item["model"] + "--" + str(vehicle_name) + "---" + str(vehicle_url))
            item["vehicle"] = vehicle_name
            item["issell"] = "在售"
            yield scrapy.Request(
                vehicle_url,
                callback=self.parse_vehicle_params,
                meta={"item": deepcopy(item)}
            )

        # 停售车款
        old_vehicle_url_id = response.xpath("//input[@id='csHid']/@value").extract_first()
        # 得到停售车辆年份
        drop_a = response.xpath("//div[@class='drop-layer']/a/text()").extract()
        for i in drop_a:
            brand_url = "http://car.bitauto.com/AjaxNew/GetNoSaleSerailListByYear.ashx?csID=" + str(
                str(old_vehicle_url_id)) + "&year=" + str(i)[0:4]
            # logger.warning(brand_url)
            yield scrapy.Request(
                brand_url,
                callback=self.parse_old_vehicle,
                meta={"item": deepcopy(item)}
            )

    # 解析非在售车款
    def parse_old_vehicle(self, response):
        item_model = response.meta.get("item")
        data = json.loads(response.body.decode())

        for item in data:
            for itemj in item["carList"]:
                vehicle_name = itemj["Name"]
                vehicle_id = itemj["CarID"]
                vehicle_year_type = itemj["YearType"]
                vehicle_spell = itemj["Spell"]
                brand_url = "http://car.bitauto.com/" + str(vehicle_spell) + "/m" + str(vehicle_id)
                item_model["vehicle"] = str(vehicle_year_type) + " " + str(vehicle_name)
                item_model["issell"] = "停产"
                logger.warning(item_model["vehicle"] + "  *** " + brand_url)

                yield scrapy.Request(
                    brand_url,
                    callback=self.parse_vehicle_params,
                    meta={"item": deepcopy(item_model)}
                )

    # 解析车辆参数
    def parse_vehicle_params(self, response):
        item_model = response.meta.get("item")
        # 存放参数
        params = []
        item_model["params"] = params

        cate = response.xpath("//div[@class='caption-1']")
        cate_layout = response.xpath("//div[@class='special-layout-18 layout-1']")
        cate_item_index = 0
        for i in cate_layout:
            param = {}
            item_model["params"].append(param)
            cate_ll = cate[cate_item_index]
            big_cate = cate_ll.xpath("./h6/text()").extract_first()
            # logger.warning(str(big_cate))
            param["param_cate"] = str(big_cate)
            # 获取所有的参数
            pars_tr = i.xpath("./table/tbody/tr")
            # 获取到所有的参数
            index = 0
            param["param_cate_value"] = []
            # 每一行
            for j in pars_tr:
                pars_td = j.xpath("./td")
                for k in pars_td:

                    if index % 2 == 0:
                        param_item = {}
                        p = k.xpath("./span/text()").extract_first()
                        param_item["key"] = p
                    else:

                        # 颜色
                        if param_item["key"] == "外观颜色：":
                            a_list = k.xpath("./div/ul/li/a/@title").extract()
                            param_item["value"] = a_list
                        # 可选配置  判断
                        elif len(k.xpath("./div")) != 0:

                            p_list = []
                            div_list = k.xpath("./div/div/div[@class='l']")
                            for xi in div_list:
                                p_obj = {}
                                cricle = xi.xpath("./i/text()").extract_first()
                                text = xi.xpath("./text()").extract_first()
                                p_obj["isSelect"] = cricle
                                p_obj["value"] = text
                                p_list.append(p_obj)
                            param_item["value"] = p_list
                        # 字
                        else:
                            p = k.xpath("./span/text()").extract_first()
                            param_item["value"] = p
                        param["param_cate_value"].append(param_item)

                    index = index + 1

            cate_item_index = cate_item_index + 1
        logger.warning(json.dumps(item_model, ensure_ascii=False))
       # yield item_model