importorg.apache.spark.SparkConf;importorg.apache.spark.api.java.JavaPairRDD;importorg.apache.spark.api.java.JavaRDD;importorg.apache.spark.api.java.JavaSparkContext;importorg.apache.spark.api.java.function.FlatMapFunction;importorg.apache.spark.api.java.function.Function;importorg.apache.spark.api.java.function.Function2;importorg.apache.spark.api.java.function.PairFunction;importorg.apache.spark.sql.sources.In;importscala.Tuple2;import java.util.*;/*** # _*_ coding:utf-8 _*_
* # Author:xiaoshubiao
* # Time : 2020/5/17 9:13
**/
public classanli_test {public static voidmain(String[] args) {
SparkConf conf= new SparkConf().setMaster("local[*]").setAppName("test");
JavaSparkContext sc= newJavaSparkContext(conf);
JavaRDD stringJavaRDD = sc.textFile("D:\\tmp\\rizhi.txt");
JavaPairRDD stringIntegerJavaPairRDD =stringJavaRDD.//转换成键值对的形式 如(省份-广告,点击数)
mapToPair(new PairFunction() {
@Overridepublic Tuple2 call(String s) throwsException {
String[] s1= s.split(" ");return new Tuple2(s1[1] + "-" + s1[4], 1);
}
}
)//计算相同key的点击数和:(省份-广告,点击数和)
.reduceByKey(new Function2() {
@Overridepublic Integer call(Integer integer, Integer integer2) throwsException {return integer +integer2;
}
});
stringIntegerJavaPairRDD.//转换key的结构 (省份-广告,点击数和)=》(省份,(广告,点击数和))
mapToPair(new PairFunction, String, Tuple2>() {
@Overridepublic Tuple2 call(Tuple2 stringIntegerTuple2) throwsException {
String[] s= stringIntegerTuple2._1().split("-");
Tuple2 stringIntegerTuple21 = new Tuple2<>(s[1], stringIntegerTuple2._2());return new Tuple2<>(s[0],stringIntegerTuple21);
}
}
)//按照k聚合
.groupByKey()//对值进行排序
.mapValues(new Function, Iterable>() {
@Overridepublic Iterable call(Iterable tuple2s) throwsException {
ArrayList> tuple2s1 = new ArrayList<>();
Iterator iterator =tuple2s.iterator();while(iterator.hasNext()){
Tuple2 next=iterator.next();
tuple2s1.add(next);
}
tuple2s1.sort(new Comparator>() {
@Overridepublic int compare(Tuple2 o1, Tuple2o2) {return o2._2() -o1._2();
}
}
);
ArrayList> t = new ArrayList<>();
Iterator> iterator1 =tuple2s1.iterator();
Integer i= 0;
Integer n= 2;while (iterator1.hasNext() & i
t.add(iterator1.next());
i++;
}returnt;
}
}
)
.collect().forEach(x->System.out.println(x));
}
}