spark实现word count程序

5.1 使用scala 实现wordcount程序

object WordCount {
    def main(args:Array[String]): Unit ={
      //1.创建sparkConf对象，设置appName和master地址，local[2]表示本地采用2个线程去执行
      val  sparkConf:SparkConf=new SparkConf().setAppName("wordCount").setMaster("local[2]")
     //2. 创建sparkContxt对象，它是所有spark程序的执行入口，它内部会构建DAGScheduler和TaskScheduler
      val sc=new SparkContext(sparkConf)
      //3.读取本地数据文件
      val data:RDD[String]=sc.textFile("e:\\words.txt")
      //4.切分每一行，获取所有的单词
      val words：RDD[String]=data.flatMap(x=>x.split(" "))
      //5.每个单词计为1
      val wordAndOne:RDD[(String,Int)]=words.map(x=>(x,1))
      //6.相同单词出现的1累加
      val result:RDD[(String,Int)]=wordAndOne.reduceByKey((x,y)=x+y)
        //按照单词出现的次数降序排列，默认是第二个参数是true是升序，改为false就是降序
      val  sortedRDD:RDD[(String,Int)]=result.sortBy(x=>x._2,false)
      //7.收集打印
      val finalResult:Array[(String,Int)]=result.collect()
      finalResult.foreach(x=>println(x))
      //关闭sc
      sc.stop()
    }

}
通过maven打成jar包，放到集群中执行
Spark-submit --master spark：//node3:7077 -class cn.itcat.wordcount.WordCount_online --executor-memory 1g --total-executor-cores 4 original-spark_class15-1.0-SNAPSHOT.jar /words.txt /out_spark

5.2 使用Java 实现wordcount程序



public class WowdCount_Java {
    public static void main(String args[]){
         //1.创建sparkConf对象
        SparkConf sparkConf=new SparkConf().setAppName("WordCount_java").setMaster("local[2]");
        //2.创建javasparkContext
        JavaSparkContext jsc=new JavaSparkContext(sparkConf);
        //3.读取数据文件
        JavaRDD<String> data=jsc.textFile("e:\\words.txt");
        //4.切分每一行，获取所有的单词
        JavaRDD<String> words=data.flatMap(new FlatMapFunction<String,String>(){
         String [] words=line.split(" ");
         return Arrays.asList(words).iterator();
        });
        //5.每个单词计为1
        JavaPairRDD<String,Integer> wordsAndone=words.mapToPair(new PairFunction<String,String,Integer>(){
            public Tuple2<String,Integer> call(String word) throws Exception{
                return new Tuple2<String,Integer>(word,_2:1);
            }
        });
        //6.相同单词出现的1累加
      JavaPairRDD<String,Integer> result=   wordAndOne.reduceByKey(new Function2<Integer,Integer,Integer>(){
             public Integer call(Integer v1,Integer v2) throws Exception{
                 return v1+v2;
             }
         });
        /**
         * 按照单词出现的次数降序排列 （单词，次数）----》（次数，单词）。sortbykey---》（单词，次数）
         * JavaPairRDD<Integer,String> reverseRDD=result.mapTopair(new PairFunction<Tuple2<String,Integer>>){
         * public Tuple2<Integer,String> call(Tuple2<String,Integer> t) throws Exception{
         *  return new Tuple2<Integer,String>(t._2,t._1);
         * }
         * });
         * JavaPairRDD<String,Integer> sortedRDD=reverseRDD.sortByKey(false).mapToPair(new PairFunction<Tuple2<Integer,String>,String,Integer>(){
         * public Tuple2<String,Integer> call(Tuple2<Integer,String> ) throws Exception{
         *return new Tuple2<String,Integer>(t._2,t._1);
         * }
         *
         * })
         */
        //7.收集打印
        List<Tuple2<String,Integer>> finalResult=result.collect();
        for(Tuple2<String,Integer> tuple:finalResult){
            System.out.println("单词:"+tuple._1+"次数:"+tuple._2);
        }
        //8关闭sc
        jsc.stop()
    }
}