爬虫笔记3-获取美团智库的行业、品类、单店数据用于分析

最新推荐文章于 2024-09-26 17:36:35 发布
Forrest King
最新推荐文章于 2024-09-26 17:36:35 发布
阅读量632
点赞数 5
文章标签：爬虫笔记 python
本文链接：https://blog.csdn.net/weixin_42978994/article/details/140826034
版权
背景：
1、都是以脚本的形式做的，方便后面做后端接口；
2、请求需要账号，建议是已经有了账号的才能获取哈；
3、还没做完，基本完成了对具体某个商圈内的某个店铺从行业对比、品类占比、单店对比和菜品分析几个维度的数据，还没有做深入分析。
4、本来要生成ppt模版报告的，还没来得及写。后面有时间完善。
import requests
import json

# 《《《说明：本爬虫用于获取“美团数智”上的不同数据用于对某个门店的周边、竞争、菜品、顾客等多维度的分析，并生成报告》》》

        
headers={
# headers根据自己登录账号获取填写。
}
# -----解析层：针对地址、餐饮类别、商圈名称，根据输入的文字，匹配对应的code，方便后续生产payload提交请求。
# 《《《解析-地址
def parseAddress(input_address):
    # 省份列表，比如安徽
    url_province='https://pos.meituan.com/web/api/v1/insight/province/list?yodaReady=h5&csecplatform=4&csecversion=2.4.0'
    # 获取省市列表
    res=requests.get(url=url_province,headers=headers).text
    data_provinces=json.loads(res)['data']

    # -》省、市代码获取。  获取输入地址对应的code,首先遍历所有省份信息，跟输入的一致确定下标,城市级别一样的操作
    for data_province in data_provinces:
        if data_province['label']==input_address[0]:
            province_id=data_province['value']
            for city in data_province['children']:
                if city['label']==input_address[1]:
                    city_id=city['value']
    # print(province_id,city_id)
    # 获取类别列表
    return province_id,city_id
# 《《《解析-类别
def parseCategory(input_category):
    # 类别列表，比如火锅
    url_category='https://pos.meituan.com/web/api/v1/insight/category/list?yodaReady=h5&csecplatform=4&csecversion=2.4.0'
    # 获取餐饮分类列表
    res=requests.get(url=url_category,headers=headers).text
    data_categorys=json.loads(res)['data']
    # -》类别代码获取。   
    for data_category in data_categorys:
        if data_category['categoryName']==input_category[0]:
            category2_id=data_category['categoryId']
            # print(data_category)
            for sub_category in data_category['subCategory']:
                if sub_category['categoryName']==input_category[1]:
                    category3_id=sub_category['categoryId']
    # print(category2_id,category3_id)
    return category2_id,category3_id
# 《《《解析-商圈
def parseBussinessArea(input_address,input_bussiness_area):
    # 商圈列表，比如观音桥
    url_bussiness_area='https://pos.meituan.com/web/api/v1/insight/topSearch/businessAreaInfo?yodaReady=h5&csecplatform=4&csecversion=2.4.0'
    # 通过城市获取当地商圈参数
    province_id,city_id=parseAddress(input_address)
    city_id_json={"cityId":city_id}
    # 获取商圈分类列表
    res=requests.post(url=url_bussiness_area,data=city_id_json,headers=headers).text
    data_bussiness_areas=json.loads(res)['data']['businessAreaInfo']
    # print(data_bussiness_areas)
    # -》类别代码获取。 
    for data_bussiness_area in data_bussiness_areas:
        if data_bussiness_area['bareaName']==input_bussiness_area[0]:
            businessArea_id=data_bussiness_area['bareaId']
    # print(businessArea_id)
    return businessArea_id

# -----参数生成层：基于不同的请求地址的payload不同，通过flag做区分生产payload参数
# 《《《生成-详细信息请求的payload参数
def gen_payload(input_address,input_category,input_bussiness_area,input_date,flag):
    # 1-分别调用地址、类别、商圈的解析函数，获取对应的code
    province_id,city_id=parseAddress(input_address)
    category2_id,category3_id=parseCategory(input_category)
    businessArea_id=parseBussinessArea(input_address,input_bussiness_area)
    if flag=='shop':
    # 构造payload
        pay_load={
            "provinceId": province_id,
            "cityId": city_id,
            "mdcPoiId": 1793793547,
            "category2Id": category2_id,
            "category3Id": category3_id,
            "businessAreaId": businessArea_id,
            "dataReadyDate": input_date[0]
        }
    elif flag=='industry':
        month=input_date[0].split('-')[0]+input_date[0].split('-')[1]
        pay_load={
            "month": month,
            "category2Id": category2_id,
            "category3Id": category3_id,
            "cityId": city_id,
            "provinceId": province_id,
            "bareaId": businessArea_id
        }
    elif flag=='dishes':
        pay_load={
        "cityId": city_id,
        "category2Id": category2_id,
        "category3Id": category3_id,
        "statDateType": 9,             #8：近7日，9：近30日
        "orderBy": "saleCntPoiAvg",    #按照店均销量
        "bareaId": businessArea_id,
        "orderType": 10,
        "startDate": None
        }
    elif flag=='consumer':
        pay_load={
        "timeType": 9,                 #8：近7日，9：近30日
        "isWm": 10,
        "cityId": city_id,
        "bareaId": businessArea_id,
        "category2Id": category2_id,
        "category3Id": category3_id
        }
    elif flag=='category':
        month=input_date[0].split('-')[0]+input_date[0].split('-')[1]+'01'
        pay_load={
        "statMonth": "20240701",
        "cityId": city_id,
        "categoryType": 0,        #是否展示子品类，0：展示，1:不展示
        "category2IdList": [
            category2_id
        ],
        "order": "bizAmount",     #根据什么排序，bizAmount表示消费金额
        "orderType": "desc",      #排序方式，desc降序由高到低
        "page": 1,
        "pageSize": 10,
        "ratioType": "-1",
        "bareaId": businessArea_id
        }
    return pay_load

# -----信息获取层：获取自己的店铺、菜品，以及所属行业的门店、菜品信息，以便后期分析
# 《《《获取-店铺信息
def get_shop_info(input_address,input_category,input_bussiness_area,input_date):
    # 1-构造参数
    pay_load=gen_payload(input_address,input_category,input_bussiness_area,input_date,flag='shop')
    # 2-发起请求获取信息
    url_overview='https://pos.meituan.com/web/api/v1/insight/shopAnalysis/overview?yodaReady=h5&csecplatform=4&csecversion=2.4.0'
    res_overview=requests.post(url=url_overview,data=pay_load,headers=headers).text
    # print(res_overview)
    return json.loads(res_overview)
# 《《《获取-菜品信息
def get_dish_info(input_address,input_category,input_bussiness_area,input_date):
    # 1-构造参数
    pay_load=gen_payload(input_address,input_category,input_bussiness_area,input_date,flag='shop')
    # 2-发起请求获取信息
    url_dishes='https://pos.meituan.com/web/api/v1/insight/shopAnalysis/dishes?yodaReady=h5&csecplatform=4&csecversion=2.4.0'
    res_dish=requests.post(url=url_dishes,data=pay_load,headers=headers).text
    # print(res_dish)
    return json.loads(res_dish)
# 《《《获取-商圈周边门店信息
def get_industy_info(input_address,input_category,input_bussiness_area,input_date):
    # 1-构造参数
    url_industry='https://pos.meituan.com/web/api/v1/insight/market-new/industry-area?yodaReady=h5&csecplatform=4&csecversion=2.4.0'
    pay_load=gen_payload(input_address,input_category,input_bussiness_area,input_date,flag='industry')
    # 2-发起请求获取信息
    res_industry=requests.post(url=url_industry,data=pay_load,headers=headers).text
    return json.loads(res_industry)
# 《《《获取-周边菜品信息，城市热榜-》菜品榜，看在这个商圈内相同赛道的菜品哪些最受欢迎。
def get_dishes_info(input_address,input_category,input_bussiness_area,input_date):
    # 1-构造请求参数
    url_dishes='https://pos.meituan.com/web/api/v1/insight/market-new/hotdishes?yodaReady=h5&csecplatform=4&csecversion=2.4.0'
    pay_load=gen_payload(input_address,input_category,input_bussiness_area,input_date,flag='dishes')
    # 2-发起请求获取信息
    res_dishes=requests.post(url=url_dishes,data=pay_load,headers=headers).text
    return json.loads(res_dishes)
# 《《《获取-品类信息，用于分析其细分类目占比和增速
def get_catetory_info(input_address,input_category,input_bussiness_area,input_date):
    # 1-构造请求参数
    url_dishes='https://pos.meituan.com/web/api/v1/insight/hotArea/categoryRank?yodaReady=h5&csecplatform=4&csecversion=2.4.0'
    pay_load=gen_payload(input_address,input_category,input_bussiness_area,input_date,flag='category')
    # 2-发起请求获取信息
    res_dishes=requests.post(url=url_dishes,data=pay_load,headers=headers).text
    return json.loads(res_dishes)
# 《《《获取-客群画像
def get_customer_feature(input_address,input_category,input_bussiness_area,input_date):
    # 1-构造请求参数
    url_customer_feature='https://pos.meituan.com/web/api/v1/insight/insightUserGroup/userGroupFeature?yodaReady=h5&csecplatform=4&csecversion=2.4.0'
    pay_load=gen_payload(input_address,input_category,input_bussiness_area,input_date,flag='consumer')
    # 2-发起请求获取信息
    res_dishes=requests.post(url=url_customer_feature,data=pay_load,headers=headers).text
    return json.loads(res_dishes)
# 《《《获取-客群消费价格
def get_customer_money(input_address,input_category,input_bussiness_area,input_date):
    # 1-构造请求参数
    url_customer_money='https://pos.meituan.com/web/api/v1/insight/insightUserGroup/priceDistribution?yodaReady=h5&csecplatform=4&csecversion=2.4.0'
    pay_load=gen_payload(input_address,input_category,input_bussiness_area,input_date,flag='consumer')
    # 2-发起请求获取信息
    res_dishes=requests.post(url=url_customer_money,data=pay_load,headers=headers).text
    return json.loads(res_dishes)

# -----对比分析层：分析店铺、菜品、食客并得出结论
# 《《《分析-店铺分析
def shop_analysis(input_address,input_category,input_bussiness_area,input_date): 
    # 所选二级细分类目在一级类目中的占比,如火锅中的重庆火锅占比情况
    data_categorys=get_catetory_info(input_address,input_category,input_bussiness_area,input_date)
    # print(data_categorys)
    # 做具体店铺分析前，需要对店铺所在的区域的行业情况作收集
    data_industry=get_industy_info(input_address,input_category,input_bussiness_area,input_date)
    # --------------------市场分析-------------------------------
    industry_total_money=data_industry['data']['overview']['orderAmtArea']        #餐饮消费金额
    print('你所在商圈跟你一个赛道%s,本月合计消费%s元'%(input_category[1],industry_total_money))
    industry_total_people=data_industry['data']['overview']['consumeUserCntArea']  #餐饮消费人口
    print('当月总共有%s人吃了%s'%(industry_total_people,input_category[1]))
    industry_average_price=data_industry['data']['overview']['atvPriceArea']        #人均消费金额
    print('这些人平均下来，人均消费%s元'%industry_average_price)
    industry_day_money=data_industry['data']['overview']['orderAmtDavgArea']    #日均流水
    industry_day_order=data_industry['data']['overview']['orderCntDavgArea']    #日均订单
    print('日均订单量为%s单'%industry_day_order)
    # --------------------竞争分析-------------------------------
    industry_total_brand=data_industry['data']['overview']['brandCntArea']        #入驻品牌数
    industry_total_shop=data_industry['data']['overview']['dobizPoiCntArea']    #营业门店数
    print('你所在商圈有运营%s的门店数量为%s个'%(input_category[1],industry_total_shop))
    # --------------------食客分析-------------------------------
    industry_total_rest=data_industry['data']['overview']['restConsumeUserCntArea']        #堂食人数
    industry_total_wm=data_industry['data']['overview']['wmConsumeUserCntArea']    #外卖人数



    # 1-获取店铺信息
    data_shop=get_shop_info(input_address,input_category,input_bussiness_area,input_date)['data']
    # 2-详细信息解析
    # 2-1均价，跟周边门店比较差异
    price_average=data_shop['priceAverage']       #均价
    # 2-1-1、如果店铺价格高于行业价格5元，说明需要做菜单价格优化
    if price_average-5>industry_average_price:
        print('你店的价格跟整个商圈周边看，太贵了，可以适当降价，需要做菜单优化')
    elif price_average+5<industry_average_price:
        print('你店的价格跟整个商圈周边看，价格很实惠，可以适当提价，需要做菜单优化')
    else:
        print('菜品价格合适')
    # 2-2、店铺分
    shop_score=data_shop['shopScore']             #店铺分
    # 2-3、当月评价数
    evaluation_amount=data_shop['evaluationAmount']  #评价数
    # 2-4、针对差评的数据
    consumer_impressions=data_shop['consumerImpression']     #差评
    for consumer_impression in consumer_impressions:
        if consumer_impression['label']==-1:
            bad_impression=consumer_impression['aspect']
            bad_impression_count=consumer_impression['reviewCount']
# 《《《分析-菜品分析
def dish_analysis(input_address,input_category,input_bussiness_area,input_date):
    # 1-获取所处行业菜品信息
    data_dishes=get_dishes_info(input_address,input_category,input_bussiness_area,input_date)['data']['dishedList']
    # 遍历获取前10个畅销菜品推荐
    for index,data_dish in enumerate(data_dishes):
        if index==10:
            break
        dish_name=data_dish['spuName']
        dish_price=data_dish['priceAvg']/100
        print(dish_name,dish_price)

    print('-----------------------------------------------')
    # 2-获取自己门店菜品信息
    dishes=get_dish_info(input_address,input_category,input_bussiness_area,input_date)['data']['line']
    for dish in dishes:
        name=dish['name']
        price=dish['price']
        print(name,price)
# 《《《分析-顾客分析
def customer_analysis(input_address,input_category,input_bussiness_area,input_date):
    data_customer_features=get_customer_feature(input_address,input_category,input_bussiness_area,input_date)
    data_customer_money=get_customer_money(input_address,input_category,input_bussiness_area,input_date)
    print(data_customer_money)


input_address=['重庆市','重庆市']
input_category=['火锅','重庆火锅']
input_bussiness_area=['观音桥']
input_date=['2024-07-30']     #写当天时间   
shop_analysis(input_address,input_category,input_bussiness_area,input_date)
dish_analysis(input_address,input_category,input_bussiness_area,input_date)
customer_analysis(input_address,input_category,input_bussiness_area,input_date)
最后几个是analysis结尾的是分析函数，主要是对shop、dish、customer的分析，数据都获取到了还没有做深入分析。