新零售项目总结02
DWB层
DWB层存在的意义:基于业务,对上一层也就是DWD层所得到的事实表和维度表,进行降维操作;通俗地讲,就是一个个的join操作,最终得到三个明细宽表:
- 订单明细宽表 dwb_order_detail
- 店铺明细宽表 dwb_shop_detail
- 商品明细宽表 dwb_goods_detail
DWD到DWB层的处理方式
这一层的处理没有太多的细节可言,就是纯join,关键在于字段的把握,从哪个表中获取
- 例子
with so as (
select
-- 订单事实表
id ,
order_num,
buyer_id,
store_id,
order_from,
order_state,
create_date,
finnshed_time,
is_settlement,
is_delete,
evaluation_state,
way,
is_stock_up
from yp_dwd.fact_shop_order
where end_date = '9999-99-99' and is_valid = 1
),
soad as (
select
-- 订单副表
id,
order_amount,
discount_amount,
goods_amount,
is_delivery,
buyer_notes,
pay_time,
receive_time,
delivery_begin_time,
arrive_store_time,
arrive_time,
create_user,
create_time,
update_user,
update_time,
is_valid
from yp_dwd.fact_shop_order_address_detail
where end_date = '9999-99-99' and is_valid = 1
),
sog as (
select
order_id,
group_id,
is_pay
from yp_dwd.fact_shop_order_group
where end_date = '9999-99-99' and is_valid = 1
)
select
-- 订单事实表
so.id as order_id,
so.order_num,
so.buyer_id,
so.store_id,
so.order_from,
so.order_state,
so.create_date,
so.finnshed_time,
so.is_settlement,
so.is_delete,
so.evaluation_state,
so.way,
so.is_stock_up,
-- 订单副表
soad.order_amount,
soad.discount_amount,
soad.goods_amount,
soad.is_delivery,
soad.buyer_notes,
soad.pay_time,
soad.receive_time,
soad.delivery_begin_time,
soad.arrive_store_time,
soad.arrive_time,
soad.create_user,
soad.create_time,
soad.update_user,
soad.update_time,
soad.is_valid,
-- 订单组表
sog.group_id,
sog.is_pay
from so
left join soad on so.id = soad.id
left join sog on so.id = sog.order_id;
DWS层
DWS层作用:用于最细粒度的统计操作,是基于主题分析存在的
统计的维度和指标
- 维度
日期
日期+城市
日期+城市+商圈
日期+城市+商圈+店铺
日期+品牌
日期+大类
日期+大类+中类
日期+大类+中列+小类
- 指标
销售收入、平台收入、配送成交额、小程序成交额、安卓APP成交额、苹果APP成交额、PC商城成交额、订单量、参 评单量、差评单量、配送单量、退款单量、小程序订单量、安卓APP订单量、苹果APP订单量、PC商城订单量
DWB到DWS层处理的方式
针对维度,以及获取当前是哪个维度的,可以使用grouping set
结合grouping
的方式进行处理。针对指标,有些细节需要处理,比如同个订单下的不同商品,可以使用row_number
的方式进行脏数据的过滤。
- 例子(presto写法)
insert into yp_dws.dws_sale_daycount
with t0 as (
select
-- 列裁剪
-- 维度字段
od.dt,
city_id,
city_name,
trade_area_id,
trade_area_name,
store_name,
brand_id,
brand_name,
max_class_name,
max_class_id,
mid_class_name,
mid_class_id,
min_class_name,
min_class_id,
-- 指标字段
order_id,
order_amount,
total_price,
plat_fee,
delivery_fee,
order_from,
evaluation_id,
geval_scores,
delievery_id,
refund_id,
od.store_id,
row_number() over (partition by order_id,goods_id ) as rk1, -- 过滤脏数据
row_number() over (partition by order_id ) as rk2
from yp_dwb.dwb_order_detail od
left join yp_dwb.dwb_shop_detail sd on od.store_id = sd.id
left join yp_dwb.dwb_goods_detail gd on od.goods_id = gd.id
)
select
city_id,
city_name,
trade_area_id,
trade_area_name,
store_id,
store_name,
brand_id,
brand_name,
max_class_id,
max_class_name,
mid_class_id,
mid_class_name,
min_class_id,
min_class_name,
case when grouping(store_id) = 0 -- if
then 'store' -- 日期 + 城市 + 商圈 + 店铺
when grouping(trade_area_id) = 0 -- else if
then 'trade_area' --日期 + 城市 + 商圈
when grouping(city_id) = 0 -- else if
then 'city ' --日期 + 城市
when grouping(brand_id) = 0 -- else if
then 'brand' -- 日期 = 品牌
when grouping(min_class_id) = 0 -- else if
then 'min_class' -- 日期 + 大类 + 中类 + 小类
when grouping(mid_class_id) = 0 -- else if
then 'mid_class' -- 日期 + 大类 + 中类
when grouping(max_class_id) = 0
then 'max_clas' -- 日期 + 大类
else
'all' -- 日期
end as group_type,
-- 总销售额
case when grouping(store_id) = 0
then sum(if(store_id is not null,total_price,0))
when grouping(trade_area_id) = 0
then sum(if(trade_area_id is not null,total_price,0))
when grouping(city_id) = 0
then sum(if(city_id is not null,total_price,0))
when grouping(brand_id) = 0
then sum(if(brand_id is not null,total_price,0))
when grouping(min_class_id) = 0
then sum(if(min_class_id is not null,total_price,0))
when grouping(mid_class_id) = 0
then sum(if(mid_class_id is not null,total_price,0))
when grouping(mid_class_id) = 0
then sum(if(mid_class_id is not null,total_price,0)) -- 聚合定制
when grouping(max_class_id) = 0
then sum(if(max_class_id is not null,total_price,0))
else
sum(if(dt is not null,total_price,0)) -- 日期
end as sale_amt,
-- 平台收入
case when grouping(store_id) = 0
then sum(if(store_id is not null,plat_fee,0))
when grouping(trade_area_id) = 0
then sum(if(trade_area_id is not null,plat_fee,0))
when grouping(city_id) = 0
then sum(if(city_id is not null,plat_fee,0))
when grouping(brand_id) = 0
then sum(if(brand_id is not null,plat_fee,0))
when grouping(min_class_id) = 0
then sum(if(min_class_id is not null,plat_fee,0))
when grouping(mid_class_id) = 0
then sum(if(mid_class_id is not null,plat_fee,0))
when grouping(mid_class_id) = 0
then sum(if(mid_class_id is not null,plat_fee,0)) -- 聚合定制
when grouping(max_class_id) = 0
then sum(if(max_class_id is not null,plat_fee,0))
else
sum(if(dt is not null,plat_fee,0)) -- 日期
end as plat_amt,
-- 配送成交额
case when grouping(store_id) = 0
then sum(if(store_id is not null and delievery_id is not null,total_price,0))
when grouping(trade_area_id) = 0
then sum(if(trade_area_id is not null and delievery_id is not null,total_price,0))
when grouping(city_id) = 0
then sum(if(city_id is not null and delievery_id is not null,total_price,0))
when grouping(brand_id) = 0
then sum(if(brand_id is not null and delievery_id is not null,total_price,0))
when grouping(min_class_id) = 0
then sum(if(min_class_id is not null and delievery_id is not null,total_price,0))
when grouping(mid_class_id) = 0
then sum(if(mid_class_id is not null and delievery_id is not null,total_price,0))
when grouping(mid_class_id) = 0
then sum(if(mid_class_id is not null and delievery_id is not null,total_price,0)) -- 聚合定制
when grouping(max_class_id) = 0
then sum(if(max_class_id is not null and delievery_id is not null,total_price,0))
else
sum(if(dt is not null and delievery_id is not null,total_price,0)) -- 日期
end as deliver_sale_amt,
-- 小程序成交额
case when grouping(store_id) = 0
then sum(if(store_id is not null and order_from = 'miniapp',total_price,0))
when grouping(trade_area_id) = 0
then sum(if(trade_area_id is not null and order_from = 'miniapp',total_price,0))
when grouping(city_id) = 0
then sum(if(city_id is not null and order_from = 'miniapp',total_price,0))
when grouping(brand_id) = 0
then sum(if(brand_id is not null and order_from = 'miniapp',total_price,0))
when grouping(min_class_id) = 0
then sum(if(min_class_id is not null and order_from = 'miniapp',total_price,0))
when grouping(mid_class_id) = 0
then sum(if(mid_class_id is not null and order_from = 'miniapp',total_price,0))
when grouping(mid_class_id) = 0
then sum(if(mid_class_id is not null and order_from = 'miniapp',total_price,0)) -- 聚合定制
when grouping(max_class_id) = 0
then sum(if(max_class_id is not null and order_from = 'miniapp',total_price,0))
else
sum(if(dt is not null and order_from = 'miniapp',total_price,0)) -- 日期
end as mini_app_sale_amt,
-- android成交额
case when grouping(store_id) = 0
then sum(if(store_id is not null and order_from = 'android',total_price,0))
when grouping(trade_area_id) = 0
then sum(if(trade_area_id is not null and order_from = 'android',total_price,0))
when grouping(city_id) = 0
then sum(if(city_id is not null and order_from = 'android',total_price,0))
when grouping(brand_id) = 0
then sum(if(brand_id is not null and order_from = 'android',total_price,0))
when grouping(min_class_id) = 0
then sum(if(min_class_id is not null and order_from = 'android',total_price,0))
when grouping(mid_class_id) = 0
then sum(if(mid_class_id is not null and order_from = 'android',total_price,0))
when grouping(mid_class_id) = 0
then sum(if(mid_class_id is not null and order_from = 'android',total_price,0)) -- 聚合定制
when grouping(max_class_id) = 0
then sum(if(max_class_id is not null and order_from = 'android',total_price,0))
else
sum(if(dt is not null and order_from = 'android',total_price,0)) -- 日期
end as android_sale_amt,
-- ios成交额
case when grouping(store_id) = 0
then sum(if(store_id is not null and order_from = 'ios',total_price,0))
when grouping(trade_area_id) = 0
then sum(if(trade_area_id is not null and order_from = 'ios',total_price,0))
when grouping(city_id) = 0
then sum(if(city_id is not null and order_from = 'ios',total_price,0))
when grouping(brand_id) = 0
then sum(if(brand_id is not null and order_from = 'ios',total_price,0))
when grouping(min_class_id) = 0
then sum(if(min_class_id is not null and order_from = 'ios',total_price,0))
when grouping(mid_class_id) = 0
then sum(if(mid_class_id is not null and order_from = 'ios',total_price,0))
when grouping(mid_class_id) = 0
then sum(if(mid_class_id is not null and order_from = 'ios',total_price,0)) -- 聚合定制
when grouping(max_class_id) = 0
then sum(if(max_class_id is not null and order_from = 'ios',total_price,0))
else
sum(if(dt is not null and order_from = 'ios',total_price,0)) -- 日期
end as ios_sale_amt,
-- pcweb成交额
case when grouping(store_id) = 0
then sum(if(store_id is not null and order_from = 'pcweb',total_price,0))
when grouping(trade_area_id) = 0
then sum(if(trade_area_id is not null and order_from = 'pcweb',total_price,0))
when grouping(city_id) = 0
then sum(if(city_id is not null and order_from = 'pcweb',total_price,0))
when grouping(brand_id) = 0
then sum(if(brand_id is not null and order_from = 'pcweb',total_price,0))
when grouping(min_class_id) = 0
then sum(if(min_class_id is not null and order_from = 'pcweb',total_price,0))
when grouping(mid_class_id) = 0
then sum(if(mid_class_id is not null and order_from = 'pcweb',total_price,0))
when grouping(mid_class_id) = 0
then sum(if(mid_class_id is not null and order_from = 'pcweb',total_price,0)) -- 聚合定制
when grouping(max_class_id) = 0
then sum(if(max_class_id is not null and order_from = 'pcweb',total_price,0))
else
sum(if(dt is not null and order_from = 'pcweb',total_price,0)) -- 日期
end as pcweb_sale_amt,
-- 成交单量
case when grouping(store_id) = 0
then count(if(store_id is not null and rk2 = 1,order_id,null))
when grouping(trade_area_id) = 0
then count(if(trade_area_id is not null and rk2 = 1,order_id,null))
when grouping(city_id) = 0
then count(if(city_id is not null and rk2=1,order_id,null))
when grouping(brand_id) = 0
then count(if(brand_id is not null and rk2=1,order_id,null))
when grouping(min_class_id) = 0
then count(if(min_class_id is not null and rk2=1,order_id,null))
when grouping(mid_class_id) = 0
then count(if(mid_class_id is not null and rk2=1,order_id,null))
when grouping(mid_class_id) = 0
then count(if(mid_class_id is not null and rk2=1,order_id,null)) -- 聚合定制
when grouping(max_class_id) = 0
then count(if(max_class_id is not null and rk2=1,order_id,null))
else
count(if(dt is not null and rk2=1,order_id,null)) -- 日期
end as order_cnt,
-- 参评单量
case when grouping(store_id) = 0
then count(if(store_id is not null and rk2=1 and evaluation_id is not null and evaluation_id is not null,order_id,null))
when grouping(trade_area_id) = 0
then count(if(trade_area_id is not null and rk2=1 and evaluation_id is not null,order_id,null))
when grouping(city_id) = 0
then count(if(city_id is not null and rk2=1 and evaluation_id is not null,order_id,null))
when grouping(brand_id) = 0
then count(if(brand_id is not null and rk2=1 and evaluation_id is not null,order_id,null))
when grouping(min_class_id) = 0
then count(if(min_class_id is not null and rk2=1 and evaluation_id is not null,order_id,null))
when grouping(mid_class_id) = 0
then count(if(mid_class_id is not null and rk2=1 and evaluation_id is not null,order_id,null))
when grouping(mid_class_id) = 0
then count(if(mid_class_id is not null and rk2=1 and evaluation_id is not null,order_id,null)) -- 聚合定制
when grouping(max_class_id) = 0
then count(if(max_class_id is not null and rk2=1 and evaluation_id is not null,order_id,null))
else
count(if(dt is not null and rk2=1 and evaluation_id is not null,order_id,null)) -- 日期
end as eva_order_cnt,
-- 差评单量
case when grouping(store_id) = 0
then count(if(store_id is not null and rk2=1 and evaluation_id is not null and geval_scores <= 6,order_id,null))
when grouping(trade_area_id) = 0
then count(if(trade_area_id is not null and rk2=1 and evaluation_id is not null and geval_scores <= 6,order_id,null))
when grouping(city_id) = 0
then count(if(city_id is not null and rk2=1 and evaluation_id is not null and geval_scores <= 6,order_id,null))
when grouping(brand_id) = 0
then count(if(brand_id is not null and rk2=1 and evaluation_id is not null and geval_scores <= 6,order_id,null))
when grouping(min_class_id) = 0
then count(if(min_class_id is not null and rk2=1 and evaluation_id is not null and geval_scores <= 6,order_id,null))
when grouping(mid_class_id) = 0
then count(if(mid_class_id is not null and rk2=1 and evaluation_id is not null and geval_scores <= 6,order_id,null))
when grouping(mid_class_id) = 0
then count(if(mid_class_id is not null and rk2=1 and evaluation_id is not null and geval_scores <= 6,order_id,null)) -- 聚合定制
when grouping(max_class_id) = 0
then count(if(max_class_id is not null and rk2=1 and evaluation_id is not null and geval_scores <= 6,order_id,null))
else
count(if(dt is not null and rk2=1 and evaluation_id is not null and geval_scores <= 6,order_id,null)) -- 日期
end as bad_eva_order_cnt,
-- 配送单量
case when grouping(store_id) = 0
then count(if(store_id is not null and rk2=1 and delievery_id is not null,order_id,null))
when grouping(trade_area_id) = 0
then count(if(trade_area_id is not null and rk2=1 and delievery_id is not null,order_id,null))
when grouping(city_id) = 0
then count(if(city_id is not null and rk2=1 and delievery_id is not null,order_id,null))
when grouping(brand_id) = 0
then count(if(brand_id is not null and rk2=1 and delievery_id is not null,order_id,null))
when grouping(min_class_id) = 0
then count(if(min_class_id is not null and rk2=1 and delievery_id is not null,order_id,null))
when grouping(mid_class_id) = 0
then count(if(mid_class_id is not null and rk2=1 and delievery_id is not null,order_id,null))
when grouping(mid_class_id) = 0
then count(if(mid_class_id is not null and rk2=1 and delievery_id is not null,order_id,null)) -- 聚合定制
when grouping(max_class_id) = 0
then count(if(max_class_id is not null and rk2=1 and delievery_id is not null,order_id,null))
else
count(if(dt is not null and rk2=1 and delievery_id is not null,order_id,null)) -- 日期
end as deliver_order_cnt,
-- 退款单量
case when grouping(store_id) = 0
then count(if(store_id is not null and rk2=1 and refund_id is not null,order_id,null))
when grouping(trade_area_id) = 0
then count(if(trade_area_id is not null and rk2=1 and refund_id is not null,order_id,null))
when grouping(city_id) = 0
then count(if(city_id is not null and rk2=1 and refund_id is not null,order_id,null))
when grouping(brand_id) = 0
then count(if(brand_id is not null and rk2=1 and refund_id is not null,order_id,null))
when grouping(min_class_id) = 0
then count(if(min_class_id is not null and rk2=1 and refund_id is not null,order_id,null))
when grouping(mid_class_id) = 0
then count(if(mid_class_id is not null and rk2=1 and refund_id is not null,order_id,null))
when grouping(mid_class_id) = 0
then count(if(mid_class_id is not null and rk2=1 and refund_id is not null,order_id,null)) -- 聚合定制
when grouping(max_class_id) = 0
then count(if(max_class_id is not null and rk2=1 and refund_id is not null,order_id,null))
else
count(if(dt is not null and rk2=1 and refund_id is not null,order_id,null)) -- 日期
end as refund_order_cnt,
-- 小程序成交单量
case when grouping(store_id) = 0
then count(if(store_id is not null and rk2=1 and order_from = 'miniapp',order_id,null))
when grouping(trade_area_id) = 0
then count(if(trade_area_id is not null and rk2=1 and order_from = 'miniapp',order_id,null))
when grouping(city_id) = 0
then count(if(city_id is not null and rk2=1 and order_from = 'miniapp',order_id,null))
when grouping(brand_id) = 0
then count(if(brand_id is not null and rk2=1 and order_from = 'miniapp',order_id,null))
when grouping(min_class_id) = 0
then count(if(min_class_id is not null and rk2=1 and order_from = 'miniapp',order_id,null))
when grouping(mid_class_id) = 0
then count(if(mid_class_id is not null and rk2=1 and order_from = 'miniapp',order_id,null))
when grouping(mid_class_id) = 0
then count(if(mid_class_id is not null and rk2=1 and order_from = 'miniapp',order_id,null)) -- 聚合定制
when grouping(max_class_id) = 0
then count(if(max_class_id is not null and rk2=1 and order_from = 'miniapp',order_id,null))
else
count(if(dt is not null and rk2=1 and order_from = 'miniapp',order_id,null)) -- 日期
end as miniapp_order_cnt,
-- android成交单量
case when grouping(store_id) = 0
then count(if(store_id is not null and rk2=1 and order_from = 'android',order_id,null))
when grouping(trade_area_id) = 0
then count(if(trade_area_id is not null and rk2=1 and order_from = 'android',order_id,null))
when grouping(city_id) = 0
then count(if(city_id is not null and rk2=1 and order_from = 'android',order_id,null))
when grouping(brand_id) = 0
then count(if(brand_id is not null and rk2=1 and order_from = 'android',order_id,null))
when grouping(min_class_id) = 0
then count(if(min_class_id is not null and rk2=1 and order_from = 'android',order_id,null))
when grouping(mid_class_id) = 0
then count(if(mid_class_id is not null and rk2=1 and order_from = 'android',order_id,null))
when grouping(mid_class_id) = 0
then count(if(mid_class_id is not null and rk2=1 and order_from = 'android',order_id,null)) -- 聚合定制
when grouping(max_class_id) = 0
then count(if(max_class_id is not null and rk2=1 and order_from = 'android',order_id,null))
else
count(if(dt is not null and rk2=1 and order_from = 'android',order_id,null)) -- 日期
end as android_order_cnt,
-- ios成交单量
case when grouping(store_id) = 0
then count(if(store_id is not null and rk2=1 and order_from = 'ios',order_id,null))
when grouping(trade_area_id) = 0
then count(if(trade_area_id is not null and rk2=1 and order_from = 'ios',order_id,null))
when grouping(city_id) = 0
then count(if(city_id is not null and rk2=1 and order_from = 'ios',order_id,null))
when grouping(brand_id) = 0
then count(if(brand_id is not null and rk2=1 and order_from = 'ios',order_id,null))
when grouping(min_class_id) = 0
then count(if(min_class_id is not null and rk2=1 and order_from = 'ios',order_id,null))
when grouping(mid_class_id) = 0
then count(if(mid_class_id is not null and rk2=1 and order_from = 'ios',order_id,null))
when grouping(mid_class_id) = 0
then count(if(mid_class_id is not null and rk2=1 and order_from = 'ios',order_id,null)) -- 聚合定制
when grouping(max_class_id) = 0
then count(if(max_class_id is not null and rk2=1 and order_from = 'ios',order_id,null))
else
count(if(dt is not null and rk2=1 and order_from = 'ios',order_id,null)) -- 日期
end as ios_order_cnt,
-- pcweb成交单量
case when grouping(store_id) = 0
then count(if(store_id is not null and rk2=1 and order_from = 'pcweb',order_id,null))
when grouping(trade_area_id) = 0
then count(if(trade_area_id is not null and rk2=1 and order_from = 'pcweb',order_id,null))
when grouping(city_id) = 0
then count(if(city_id is not null and rk2=1 and order_from = 'pcweb',order_id,null))
when grouping(brand_id) = 0
then count(if(brand_id is not null and rk2=1 and order_from = 'pcweb',order_id,null))
when grouping(min_class_id) = 0
then count(if(min_class_id is not null and rk2=1 and order_from = 'pcweb',order_id,null))
when grouping(mid_class_id) = 0
then count(if(mid_class_id is not null and rk2=1 and order_from = 'pcweb',order_id,null))
when grouping(mid_class_id) = 0
then count(if(mid_class_id is not null and rk2=1 and order_from = 'pcweb',order_id,null)) -- 聚合定制
when grouping(max_class_id) = 0
then count(if(max_class_id is not null and rk2=1 and order_from = 'pcweb',order_id,null))
else
count(if(dt is not null and order_from = 'pcweb',order_id,null)) -- 日期
end as pcweb_order_cnt,
dt
from t0
where rk1 = 1
group by
grouping sets (
dt,
(dt,city_id,city_name),
(dt,city_id,city_name,trade_area_id,trade_area_name),
(dt,city_id,city_name,trade_area_id,trade_area_name,store_id,store_name),
(dt,brand_id,brand_name),
(dt,max_class_id,max_class_name),
(dt,max_class_id,max_class_name,mid_class_id,mid_class_name),
(dt,max_class_id,max_class_name,mid_class_id,mid_class_name,min_class_id,min_class_name)
);
对于维度日期 + 品牌
对应指标的一点理解:
品牌这块,针对同个订单而言,可能存在不同商品且同个品牌的情况,针对这点,可以使用row_number() over (partition by order_id,goods_id ) as rk1
的方式,对同个订单同个商品的数据进行过滤,而total_price
字段指的就是购买商品的价格,所以可以对应得取得;在求订单量的时候,由于只计算订单存在的数量,所以可以使用row_number() over (partition by order_id ) as rk2
的方式进行过滤。