问题描述:
原始数据
2017-01-01 a
2017-01-01 b
2017-01-01 c
2017-01-02 a
2017-01-02 b
2017-01-02 d
2017-01-03 b
2017-01-03 e
2017-01-03 f
根据数据可以看出我们要求的结果为: 2017-01-01 新增三个用户(a,b,c) 2017-01-02 新增一个用户(d) 2017-01-03 新增两个用户(e,f)求:统计列一中每个日期出现的次数,即为对应日期新增用户数。
package homeworkRDD;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import java.text.SimpleDateFormat;
import java.util.Date;
import scala.Tuple2;
public class newuser {
public static void main(String[] args) {
// TODO Auto-generated method stub
SparkConf sparkConf = new SparkConf().setAppName("PeopleInfoCalculator").setMaster("local");
JavaSparkContext sc = new JavaSparkContext(sparkConf);
JavaRDD<String> dataFile = sc.textFile
("file:///home/gyq/下载/spark-2.3.2-bin-hadoop2.7/data/newuser.txt");
JavaRDD<String> rdd_1=dataFile.map(f->{
Date format2 = null;
format2 = new SimpleDateFormat("yyyy-MM-dd").parse(f.split(" ")[0]);
return new SimpleDateFormat("yyyyMMdd").format(format2)+" "+f. split(" ")[1];
});
JavaPairRDD<String,Long> rdd2=rdd_1.mapToPair(f->{
return new Tuple2<String,Long>(f.split(" ")[1],new Long(f.split(" ")[0]));
});
JavaPairRDD<String,Long> rdd3=rdd2.reduceByKey((x,y)->{
return x<y?x:y;
});
JavaPairRDD<Long,String> rdd4=rdd3.mapToPair(f->{
return new Tuple2<>(f._2,f._1);
});
JavaPairRDD<Long,Integer> rdd5=rdd4.mapToPair(f->{
return new Tuple2<Long,Integer>(f._1,1);
});
JavaPairRDD<Long,Integer> rdd6=rdd5.reduceByKey((x,y)->x+y);
JavaPairRDD<String,Integer> rdd7=rdd6.mapToPair(f->{
return new Tuple2<String,Integer>(String.valueOf(f._1),f._2);
});
JavaPairRDD<String,Integer> rdd8=rdd7.mapToPair(f->{
Date format1 = null;
format1 = new SimpleDateFormat("yyyyMMdd").parse(f._1);
String longDate = new SimpleDateFormat("yyyy-MM-dd").format(format1);
return new Tuple2<>(longDate,f._2);
});
rdd8.foreach(f->System.out.println(f._1()+"新增用户为: "+f._2));
// JavaPairRDD<String, Iterable<String>> rdd2=rdd1.groupByKey();
// rdd2.foreach(f->System.out.println(f));
}
}
结果如下: