/**
* 测试数据:
* 1,001,10,2015-10-29
* 1,001,8.9,2015-10-28
* 2,002,5,2015-10-27
* 2,002,3,2015-10-28
* 3,003,5.9,2015-11-03
*
*/
object RemoveMultiRecordTest extends App{
private val sep = "\t"
val sc = ContextInit.getSparkContext();
val filepath = "order/multi_record.txt"
val multiRecord = sc.textFile(filepath).map(_.split(",",-1)).cache()
/**
* 组装key_value键-值对,groupByKey,对同一Key下的列表求最大值
*/
multiRecord.map(line=>(line(0)+sep+line(1),line(2)+sep+line(3))).groupByKey().map(r=>{
val dateIter = r._2.toIterator
var maxRecord = dateIter.next
var maxDateVal = maxRecord.split(sep)(1)
while(dateIter.hasNext){
val tempRecord = dateIter.next()
val tempDate = tempRecord.split(sep)(1)
if(DateUtil.isBefore(maxDateVal,tempDate)){
maxDateVal = tempDate
maxRecord = tempRecord
}
}
Joiner.on(sep).join(r._1,maxRecord)
}).foreach(println)
/**
* 组装key_value键-值对,groupByKey,通过自定义排序,按日期字段排序,获取key下第一个记录
*/
implicit val KeyOrdering = new Ordering[String] {
override def compare(x: String, y: String): Int = {
val date_x = x.split(sep)(1)
val date_y = y.split(sep)(1)
date_y.compareTo(date_x)
}
}
multiRecord.map(line=>(line(0)+sep+line(1),line(2)+sep+line(3))).groupByKey().map(r=>{
val list = r._2.toArray.sorted(KeyOrdering)
(r._1,list(0))
}).foreach(println)
/**
* 简洁版
*/
val b = multiRecord.groupBy(a=>(a(0),a(1))).map(r=>{
val t = r._2.toArray.maxBy(m=>m(3))
t.mkString(sep)
}).foreach(println)
}
Spark求数据集中同一主键记录中的最新数据
最新推荐文章于 2024-07-07 22:47:13 发布