基于Django的电商购物项目通过网络爬虫技术以及ORM框架获取项目商品数据

基于Django的电商购物项目通过网络爬虫技术以及ORM框架获取项目商品数据

这里先提供源码 , 具体思路下一期再进行解释

if __name__ == '__main__':
    import os
    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'JiXuShopSystem.settings')
    import django
    django.setup()
    from goods.models import *
    import json
    import random
    import requests
    import re
    from pypinyin import pinyin, Style

    class Spider():
        def __init__(self):

            self.headers = {
                "accept": "application/json, text/javascript, */*; q=0.01",
                "accept-language": "zh-CN,zh;q=0.9",
                "authorization": "OAuth api_sign=d463028caef4688f248d1d05cac958d0153afe12",
                "cache-control": "no-cache",
                "content-type": "application/x-www-form-urlencoded; charset=UTF-8",
                "origin": "https://detail.vip.com",
                "pragma": "no-cache",
                "priority": "u=1, i",
                "referer": "https://detail.vip.com/",
                "sec-ch-ua": "\"Chromium\";v=\"124\", \"Google Chrome\";v=\"124\", \"Not-A.Brand\";v=\"99\"",
                "sec-ch-ua-mobile": "?0",
                "sec-ch-ua-platform": "\"Windows\"",
                "sec-fetch-dest": "empty",
                "sec-fetch-mode": "cors",
                "sec-fetch-site": "same-site",
                "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
                "x-requested-with": "XMLHttpRequest"
            }
            self.cookies = {
                "vip_cps_cid": "1703946155095_f8aebf721aa4d69f55487762e3ca4c21",
                "PAPVisitorId": "58460c7a22e31f6b4acb2a1ed741f921",
                "vip_new_old_user": "1",
                "mars_cid": "1703946163504_4eebec221de3364e0da3bbe4a2182454",
                "mars_pid": "0",
                "vip_cps_cuid": "CU1715695090775e67fbb574e7ca3b54",
                "vip_city_name": "%E5%B9%BF%E5%B7%9E%E5%B8%82",
                "VipUINFO": "luc%3Aa%7Csuc%3Aa%7Cbct%3Ac_new%7Chct%3Ac_new%7Cbdts%3A0%7Cbcts%3A0%7Ckfts%3A0%7Cc10%3A0%7Crcabt%3A0%7Cp2%3A0%7Cp3%3A1%7Cp4%3A0%7Cp5%3A0%7Cul%3A3105",
                "vip_address": "%257B%2522pname%2522%253A%2522%255Cu5b89%255Cu5fbd%255Cu7701%2522%252C%2522pid%2522%253A%2522103104%2522%252C%2522cname%2522%253A%2522%255Cu5e7f%255Cu5dde%255Cu5e02%2522%252C%2522cid%2522%253A%2522103104105%2522%257D",
                "vip_province": "103104",
                "vip_province_name": "%E5%AE%89%E5%BE%BD%E7%9C%81",
                "vip_city_code": "103104105",
                "vip_wh": "VIP_HZ",
                "vip_ipver": "31",
                "user_class": "a",
                "visit_id": "DBEBA17A04F9893A4CBF22BF2C37ADB4",
                "mars_sid": "f9ea9bcbdad7fcd940397690e20183a7",
                "VIP_QR_FIRST": "1",
                "vip_tracker_source_from": "",
                "vipshop_passport_src": "https%3A%2F%2Fdetail.vip.com%2Fdetail-1711573035-6920320379434192651.html",
                "pg_session_no": "11",
                "VipDFT": "-1",
                "vip_access_times": "%7B%22list%22%3A3%2C%22detail%22%3A0%7D"
            }
            self.url = "https://mapi.vip.com/vips-mobile/rest/shopping/pc/detail/main/v6"



        def brand(self,response):
            # 品牌
            brandStoreInfo = response['data']['brandStoreInfo']['brandStoreName']
            # Logo
            brandStoreLogo = response['data']['brandStoreInfo']['brandStoreLogo']
            # 首字母
            if brandStoreInfo:
                first_char_pinyin_list = pinyin(brandStoreInfo[0], style=Style.FIRST_LETTER, strict=False)
                # 取第一个拼音的第一个字母（转换为大写）
                first_letter =  first_char_pinyin_list[0][0].upper()
            else:
                first_letter = None
            brand_data = {
                'brandStoreInfo':brandStoreInfo,
                'brandStoreLogo':brandStoreLogo,
                'first_letter':first_letter,
            }
            return brand_data



        def spu(self,response):
            # spu
            spu_name = response['data']['base']['title']
            spu_sales = 0
            spu_comments = 0

            images_detls = response['data']['images']['groups'].values()
            detail = ''
            for detil in images_detls:
                # 介绍
                detailImages = detil['detailImages']
                for imageUrls in detailImages[1:]:
                    imageUrl = imageUrls['imageUrl']
                    detail += f'<p><img src="">{imageUrl}</p>'
                break


            url = "https://mapi.vip.com/vips-mobile/rest/shopping/pc/detail/vendorqa/v1"
            params = {
                "callback": "getVendorQaCb",
                "app_name": "shop_pc",
                "app_version": "4.0",
                "warehouse": "VIP_HZ",
                "fdc_area_id": "103104105",
                "client": "pc",
                "mobile_platform": "1",
                "province_id": "103104",
                "api_key": "70f71280d5d547b2a7bb370a529aeea1",
                "user_id": "",
                "mars_cid": "1703946163504_4eebec221de3364e0da3bbe4a2182454",
                "wap_consumer": "a",
                "productId": "6920549355391848472"
            }
            response_pack = requests.get(url, headers=self.headers, cookies=self.cookies, params=params).text.replace('getVendorQaCb(','').replace(')','')
            response_pack_dict = json.loads(response_pack)
            decs_pack = response_pack_dict['data']['list']



            # 包装清单
            decs_data = ''
            for decs in decs_pack:
                decs_data += f"""
                <dl class="q-tit-item">
                        <dt class="qti-title">
                            <i class="qti-question"></i>
                            <p class="qti-txt" style="color: red">{decs['question']}</p>
                        </dt>
                        <dd class="qti-content">
                            <i class="qti-answer"></i>
                            <p class="qti-txt">
                                {decs['answer']}
                            </p>
                        </dd>
                    </dl>
                """


            # 售后服务
            tsaleService_data = ''
            tsaleServiceListitle = response['data']['saleServiceList']
            for tsaleService in tsaleServiceListitle:
                tsaleService_data += f"""
                    <dl class="q-tit-item">
                            <dt class="qti-title">
                                <i class="qti-question"></i>
                                <p class="qti-txt" style="color: red">{tsaleService['name']}</p>
                            </dt>
                            <dd class="qti-content">
                                <i class="qti-answer"></i>
                                <p class="qti-txt">
                                    {tsaleService['value']}
                                </p>
                            </dd>
                        </dl>
                    """

            spu_data = {
                'name' : spu_name,
                'sales' : spu_sales,
                'comments' : spu_comments,
                'desc_detail' : detail,
                'desc_pack' : decs_data,
                'desc_service' : tsaleService_data

            }
            return spu_data


        def sku(self,response):
            # sku

            # 名称
            title = response['data']['base']['title']


            saleProps_list = []
            products = response['data']['products'].values()

            for product in products:
                # 标题
                longTitle = product['merchandiseSn']
                # 名称
                merchandiseSn = product['merchandiseSn']
                # 市场价
                saleMarketPrice = product['priceView']['salePrice']['saleMarketPrice']
                # 售价
                salePrice = product['priceView']['salePrice']['salePrice']
                # 折扣
                cost_price = int(saleMarketPrice) - int(salePrice)
                # 库存
                stock = 1000
                # 销量
                sales = random.randint(1, 800)
                # 评价
                comments = 0
                # 是否上架
                is_launched = True
                # 默认图片
                default_image = product['smallImage'].replace('http://h2.appsimg.com/a.appsimg.com/upload/merchandise/','').replace('.jpg','')
                saleProps_dict = {
                    'name' : merchandiseSn,
                    'price' : salePrice,
                    'caption' : title,
                    'cost_price' : cost_price ,
                    'market_price' : saleMarketPrice,
                    'stock' : stock,
                    'sales' : sales,
                    'comments' : comments,
                    'is_launched':is_launched,
                    'default_image':default_image
                }
                saleProps_list.append(saleProps_dict)

            return saleProps_list



        def sku_image(self,response):
            images_detls = response['data']['images']['groups'].values()
            SKUImage_detail = []
            # 商品详情图
            for detil in images_detls:
                image_lsit = detil['previewImages']
                for image_urls in image_lsit:
                    image_url = image_urls['imageUrl'].replace('http://h2.appsimg.com/a.appsimg.com/upload/merchandise/','').replace('.jpg','')
                    SKUImage_detail.append(image_url)
                break

            return SKUImage_detail


        def spu_pecification(self,response):
            # 规格
            saleProps_list = []
            saleProps = response['data']['saleProps']
            for saleProp in saleProps:
                name = saleProp['name']
                saleProps_list.append(name)
            return saleProps_list


        def spu_optiuons(self,response):
            saleProps = response['data']['saleProps']
            sption_list = []
            for saleProp in saleProps:
                values = saleProp['values']
                for value in values:
                    sub_value = value['name']
                    sption_list.append(sub_value)
            return sption_list



        def main(self):
            for page in range(0, 2400, 120):
                url = "https://mapi.vip.com/vips-mobile/rest/shopping/pc/search/product/rank"
                params = {
                    "callback": "getMerchandiseIds",
                    "app_name": "shop_pc",
                    "app_version": "4.0",
                    "warehouse": "VIP_HZ",
                    "fdc_area_id": "103104105",
                    "client": "pc",
                    "mobile_platform": "1",
                    "province_id": "103104",
                    "api_key": "70f71280d5d547b2a7bb370a529aeea1",
                    "user_id": "",
                    "mars_cid": "1703946163504_4eebec221de3364e0da3bbe4a2182454",
                    "wap_consumer": "a",
                    "standby_id": "nature",
                    "keyword": "手机",
                    "lv3CatIds": "",
                    "lv2CatIds": "",
                    "lv1CatIds": "",
                    "brandStoreSns": "",
                    "props": "",
                    "priceMin": "",
                    "priceMax": "",
                    "vipService": "",
                    "sort": "0",
                    "pageOffset": f"{page}",
                    "channelId": "1",
                    "gPlatform": "PC",
                    "batchSize": "120",
                    "_": "1715836699310"
                }
                response = requests.get(url, headers=self.headers, cookies=self.cookies, params=params).text

                # 获得产品编码
                pid_code = re.findall('{"pid":"(.*?)"}', response)

                for pid in pid_code:
                    self.spider_data(pid)



        def spider_data(self,pid):
            self.data = {
                "app_name": "shop_pc",
                "app_version": "4.0",
                "warehouse": "VIP_HZ",
                "fdc_area_id": "103104105",
                "client": "pc",
                "mobile_platform": "1",
                "province_id": "103104",
                "api_key": "70f71280d5d547b2a7bb370a529aeea1",
                "user_id": "",
                "mars_cid": "1703946163504_4eebec221de3364e0da3bbe4a2182454",
                "wap_consumer": "a",
                "scene": "detail",
                "productId": f"{pid}",
                "opts": "priceView:13;quotaInfo:1;restrictTips:1;panelView:3;foreShowActive:1;invisible:1;floatingView:1;announcement:1;svipView:2;showSingleColor:1;svipPriceMode:1;promotionTips:6;foldTips:3;formula:2;extraDetailImages:1;shortVideo:1;countryFlagStyle:1;saleServiceList:1;storeInfo:2;brandCountry:1;freightTips:3;priceBannerView:1;bannerTagsView:1;buyMoreFormula:1;mergeGiftTips:0;kf:1;priceIcon:1;tuv:3;promotionTags:7;mergeGiftTips:3;topDetailImage:2;deliveryInfo:1;relatedProdSpu:1"
            }
            response = requests.post(self.url, headers=self.headers, cookies=self.cookies, data=self.data).json()

            brand_data = self.brand(response)
            print(brand_data)

            brand_object = Brand.objects.create(
                name=brand_data['brandStoreInfo'],
                logo=brand_data['brandStoreLogo'],
                first_letter=brand_data['first_letter']
            )

            good_category1 = GoodsCategory.objects.get(id=1)
            good_category2 = GoodsCategory.objects.get(id=38)
            good_category3 = GoodsCategory.objects.get(id=115)

            spu_data = self.spu(response)
            print(spu_data)

            spu_object = SPU.objects.create(
                name=spu_data['name'],
                sales=spu_data['sales'],
                comments=spu_data['comments'],
                desc_detail=spu_data['desc_detail'],
                desc_pack=spu_data['desc_pack'],
                desc_service=spu_data['desc_service'],
                brand=brand_object,
                category1=good_category1,
                category2=good_category2,
                category3=good_category3,
            )

            sku_data_list = self.sku(response)
            for sku_data in sku_data_list:
                sku_object = SKU.objects.create(
                    name=sku_data['name'],
                    price=sku_data['price'],
                    caption=sku_data['caption'],
                    cost_price=sku_data['cost_price'],
                    market_price=sku_data['market_price'],
                    stock=sku_data['stock'],
                    sales=sku_data['sales'],
                    comments=sku_data['comments'],
                    is_launched=sku_data['is_launched'],
                    default_image=sku_data['default_image'],
                    category=good_category3,
                    spu=spu_object
                )

                sku_image_list = self.sku_image(response)
                for sku_image_data in sku_image_list:
                    sku_image_object = SKUImage.objects.create(
                        image=sku_image_data,
                        sku= sku_object
                    )

                spu_pecification_list = self.spu_pecification(response)
                for spu_pecification_data in spu_pecification_list:
                    spu_pecification_object = SPUSpecification.objects.create(
                        name=spu_pecification_data,
                        spu=spu_object
                    )

                    spu_optiuons_list = self.spu_optiuons(response)
                    for spu_optiuons in spu_optiuons_list:
                        spu_optiuons_object = SpecificationOption.objects.create(
                            value=spu_optiuons,
                            spec=spu_pecification_object
                        )

                        SKUSpecification_object = SKUSpecification.objects.create(
                            option=spu_optiuons_object,
                            spec=spu_pecification_object,
                            sku=sku_object

                        )








    spider = Spider()
    spider.main()