SparkCore练习
2019-07-17_38_6502cdc9-cf95-4b08-8854-f03a25baa917_47_2019-07-17 00:00:54_null_14_79_null_null_null_null_2
2019-07-17_38_6502cdc9-cf95-4b08-8854-f03a25baa917_27_2019-07-17 00:00:59_null_3_50_null_null_null_null_26
2019-07-17_38_6502cdc9-cf95-4b08-8854-f03a25baa917_27_2019-07-17 00:01:05_i7_-1_-1_null_null_null_null_17
2019-07-17_38_6502cdc9-cf95-4b08-8854-f03a25baa917_24_2019-07-17 00:01:07_null_5_39_null_null_null_null_10
2019-07-17_38_6502cdc9-cf95-4b08-8854-f03a25baa917_25_2019-07-17 00:01:13_i7_-1_-1_null_null_null_null_24
2019-07-17_38_6502cdc9-cf95-4b08-8854-f03a25baa917_22_2019-07-17 00:01:21_null_19_62_null_null_null_null_20
2019-07-17_38_6502cdc9-cf95-4b08-8854-f03a25baa917_41_2019-07-17 00:01:27_null_4_58_null_null_null_null_9
2019-07-17_38_6502cdc9-cf95-4b08-8854-f03a25baa917_2_2019-07-17 00:01:33_苹果_-1_-1_null_null_null_null_21
2019-07-17_38_6502cdc9-cf95-4b08-8854-f03a25baa917_8_2019-07-17 00:01:39_笔记本_-1_-1_null_null_null_null_21
按下划线从左到右切割依次为
时间戳,用户,session_id,页面id,搜索关键字,点击品Id,点击类Id,下单类id,下单品id,支付类id,支付品id,城市id
再次基础上制作相应得bean对象
package cn.lpc.spark.bean
case class UserVisitAction(date: String,
user_id: Long,
session_id: String,
page_id: Long,
action_time: String,
search_keyword: String,
click_category_id: Long,
click_product_id: Long,
order_category_ids: String,
order_product_ids: String,
pay_category_ids: String,
pay_product_ids: String,
city_id: Long)
case class CategoryCountInfo(categoryId: String,
clickCount: Long,
orderCount: Long,
payCount: Long)
取top10得热门商品
// 处理逻辑得主程序
object CategoryTopApp {
def statCategoryTo10(sc: SparkContext,UserVisitActionAdd: RDD[UserVisitAction]): Array[CategoryCountInfo] ={
// 自定义累加器
val acc = new accAction
// 注册累加器
sc.register(acc,"accAction")
// 类加
UserVisitActionAdd.foreach(
UserVisitAction=>{
acc.add(UserVisitAction)
}
)
// 对累加得值进行分过滤分组等操作就能完成
val cidToCateInfo: Map[String, Map[(String, String), Long]] = acc.value.filter(x => x._1 != ("wrongData", "wrongData")).groupBy(_._1._1)
val categoryCountInfos: Array[CategoryCountInfo] = cidToCateInfo.map {
case (cid, map) => {
CategoryCountInfo(cid,
map.getOrElse((cid, "click"), 0),
map.getOrElse((cid, "order"), 0),
map.getOrElse((cid, "pay"), 0))
}
}.toArray
categoryCountInfos.sortBy(info => (info.clickCount, info.orderCount, info.clickCount))(Ordering.Tuple3(Ordering.Long.reverse, Ordering.Long.reverse, Ordering.Long.reverse))
.take(10)
}
}
// 主要代码在累加器之中
class accAction extends AccumulatorV2[UserVisitAction,Map[(String,String),Long]]{
private var map: Map[(String, String), Long] = Map[(String, String), Long]()
override def isZero: Boolean = map.isEmpty
override def copy(): AccumulatorV2[UserVisitAction, Map[(String, String), Long]] = {
val acc = new accAction
acc.map = map
acc
}
override def reset(): Unit = map =Map[(String, String), Long]()
// 根据不同得cid和action组成一个key进行类加
override def add(v: UserVisitAction): Unit = {
// 有些action有多个cid
val cidAndActions: (String, String) = ParseUtil.parserAction(v)
val cids: Array[String] = cidAndActions._1.split(",")
cids.foreach{
cid =>{
val key: (String, String) = (cid,cidAndActions._2)
map += (key->(map.getOrElse(key,0L)+1L))
}
}
}
// 多个分区之间的元素进行合并
override def merge(other: AccumulatorV2[UserVisitAction, Map[(String, String), Long]]): Unit = other match {
case o:accAction =>{
o.map.foreach {
case (cidAction, count) =>
this.map += cidAction -> (this.map.getOrElse(cidAction, 0L) + count)
}
}
}
override def value: Map[(String, String), Long] = map
}
// 自定义的解析器
package cn.lpc.spark.Util
import cn.lpc.spark.bean.{CategoryCountInfo, UserVisitAction}
case class ParseUtil(){
}
object ParseUtil{
def parseLine(line:String): UserVisitAction ={
val splits: Array[String] = line.split("_")
UserVisitAction(
splits(0),
splits(1).toLong,
splits(2),
splits(3).toLong,
splits(4),
splits(5),
splits(6).toLong,
splits(7).toLong,
splits(8),
splits(9),
splits(10),
splits(11),
splits(12).toLong
)
}
def parserAction(action:UserVisitAction):(String,String)={
if(action.click_category_id != -1){
(action.click_category_id.toString,"click")
}else if (action.order_category_ids != "null"){
(action.order_category_ids,"order")
}else if (action.pay_category_ids !="null"){
(action.pay_category_ids,"pay")
}else{
("wrongData","wrongData")
}
}
def getCidFormCategoryCountInfo(CategoryInfos: Array[CategoryCountInfo])={
CategoryInfos.map(x=>x.categoryId)
}
}
Top10热门品类中每个品类的 Top10 活跃 Session 统计
活跃session
def statCategoryTo10(sc: SparkContext,UserVisitActionAdd: RDD[UserVisitAction]): Array[CategoryCountInfo] ={
val acc = new accAction
sc.register(acc,"accAction")
UserVisitActionAdd.foreach(
UserVisitAction=>{
acc.add(UserVisitAction)
}
)
val cidToCateInfo: Map[String, Map[(String, String), Long]] = acc.value.filter(x => x._1 != ("wrongData", "wrongData")).groupBy(_._1._1)
val categoryCountInfos: Array[CategoryCountInfo] = cidToCateInfo.map {
case (cid, map) => {
CategoryCountInfo(cid,
map.getOrElse((cid, "click"), 0),
map.getOrElse((cid, "order"), 0),
map.getOrElse((cid, "pay"), 0))
}
}.toArray
categoryCountInfos.sortBy(info => (info.clickCount, info.orderCount, info.clickCount))(Ordering.Tuple3(Ordering.Long.reverse, Ordering.Long.reverse, Ordering.Long.reverse))
.take(10)
}
- 过滤出来 category Top10的日志
- 需要用到需求1的结果, 然后只需要得到categoryId就可以了
- 转换结果为 RDD[(categoryId, sessionId), 1] 然后统计数量 => RDD[(categoryId, sessionId), count]
- 统计每个品类 top10. => RDD[categoryId, (sessionId, count)] => RDD[categoryId, Iterable[(sessionId, count)]]
- 对每个 Iterable[(sessionId, count)]进行排序, 并取每个Iterable的前10
- 把数据封装到 CategorySession 中
页面单跳转化率统计
// 代码总体上不难
// 做漏斗分析需要注意的就是单Session在连续时间下相邻的转换才是合法的跳转
// 需要注意的是灵活的利用zip,构造转换
object PageConversion {
def statPageConversion(sc: SparkContext, UserVisitActionAdd: RDD[UserVisitAction],pages:String)={
// pages ->1,2,3,4,5,6,7
val splits: Array[String] = pages.split(",")
// 将页面的跳转行为进行表示
val fromPages: Array[String] = splits.slice(0, splits.length - 1)
val toPages: Array[String] = splits.slice(1, splits.length)
val fromToPages: Array[String] = fromPages.zip(toPages).map {
case (fromPage, toPage) => fromPage + "->" + toPage
}
// 计算分母
val UserVisitActionFiltered: RDD[UserVisitAction] = UserVisitActionAdd.filter(action => splits.contains(action.page_id.toString))
// 将包含所需pageId的页面过滤出来
val pageToInt: collection.Map[Long, Long] =UserVisitActionFiltered.map(
action => (action.page_id, 1L)
).countByKey()
println(pageToInt)
// 计算分子
// 一个session一个action集合
val sessionWithAction: RDD[(String, Iterable[UserVisitAction])] = UserVisitActionFiltered.groupBy(_.session_id)
val SessionToTrains: RDD[(String, List[Long])] = sessionWithAction.mapValues(t => t.toList.sortBy(_.action_time).map(sessionWithAction =>
sessionWithAction.page_id))
val AllConversion: RDD[String] = SessionToTrains.flatMap(
x => {
val fromPages: List[Long] = x._2.slice(0, x._2.size - 1)
val toPages: List[Long] = x._2.slice(1, x._2.size)
fromPages.zip(toPages).map {
case (fromPage, toPage) => fromPage + "->" + toPage
}
}
)
val conversionAndCount: collection.Map[String, Long] = AllConversion.filter(x=>fromToPages.contains(x)).map(x => (x, 1)).countByKey()
val format = new DecimalFormat(".00%")
val changAndRate: collection.Map[String, String] = conversionAndCount.map {
k => (k._1,format.format( k._2.toDouble / pageToInt(k._1.split("->")(0).toLong)))
}
changAndRate
}
}