Java计算广告_java spark 计算各个省份广告点击数的top3

importorg.apache.spark.SparkConf;importorg.apache.spark.api.java.JavaPairRDD;importorg.apache.spark.api.java.JavaRDD;importorg.apache.spark.api.java.JavaSparkContext;importorg.apache.spark.api.java.function.FlatMapFunction;importorg.apache.spark.api.java.function.Function;importorg.apache.spark.api.java.function.Function2;importorg.apache.spark.api.java.function.PairFunction;importorg.apache.spark.sql.sources.In;importscala.Tuple2;import java.util.*;/*** # _*_ coding:utf-8 _*_

* # Author:xiaoshubiao

* # Time : 2020/5/17 9:13

**/

public classanli_test {public static voidmain(String[] args) {

SparkConf conf= new SparkConf().setMaster("local[*]").setAppName("test");

JavaSparkContext sc= newJavaSparkContext(conf);

JavaRDD stringJavaRDD = sc.textFile("D:\\tmp\\rizhi.txt");

JavaPairRDD stringIntegerJavaPairRDD =stringJavaRDD.//转换成键值对的形式 如(省份-广告,点击数)

mapToPair(new PairFunction() {

@Overridepublic Tuple2 call(String s) throwsException {

String[] s1= s.split(" ");return new Tuple2(s1[1] + "-" + s1[4], 1);

}

}

)//计算相同key的点击数和：(省份-广告,点击数和)

.reduceByKey(new Function2() {

@Overridepublic Integer call(Integer integer, Integer integer2) throwsException {return integer +integer2;

}

});

stringIntegerJavaPairRDD.//转换key的结构 (省份-广告,点击数和)=》(省份,(广告，点击数和))

mapToPair(new PairFunction, String, Tuple2>() {

@Overridepublic Tuple2 call(Tuple2 stringIntegerTuple2) throwsException {

String[] s= stringIntegerTuple2._1().split("-");

Tuple2 stringIntegerTuple21 = new Tuple2<>(s[1], stringIntegerTuple2._2());return new Tuple2<>(s[0],stringIntegerTuple21);

}

}

)//按照k聚合

.groupByKey()//对值进行排序

.mapValues(new Function, Iterable>() {

@Overridepublic Iterable call(Iterable tuple2s) throwsException {

ArrayList> tuple2s1 = new ArrayList<>();

Iterator iterator =tuple2s.iterator();while(iterator.hasNext()){

Tuple2 next=iterator.next();

tuple2s1.add(next);

}

tuple2s1.sort(new Comparator>() {

@Overridepublic int compare(Tuple2 o1, Tuple2o2) {return o2._2() -o1._2();

}

}

);

ArrayList> t = new ArrayList<>();

Iterator> iterator1 =tuple2s1.iterator();

Integer i= 0;

Integer n= 2;while (iterator1.hasNext() & i

t.add(iterator1.next());

i++;

}returnt;

}

}

)

.collect().forEach(x->System.out.println(x));

}

}