作者水平低,如有错误,敬请指正!欢迎讨论指导!
目录
第一种
def main(args: Array[String]): Unit = { //建立spark连接 val sparkConf = new SparkConf().setMaster("local[*]").setAppName("WordCountTwo") val sc = new SparkContext(sparkConf) //执行操作 val lines: RDD[String] = sc.textFile("datas") val words: RDD[String] = lines.flatMap(_.split(" ")) val group: RDD[(String, Iterable[String])] = words.groupBy(word => word) val wordcount = group.map { case (word, list) => { (word, list.size) } } val array: Array[(String,Int)] = wordcount.collect() array.foreach(println) //关闭 sc.stop() }
第二种
def main(args: Array[String]): Unit = { val sparkConf = new SparkConf().setMaster("local[*]").setAppName("WordCountThree") val sc = new SparkContext(sparkConf) val lines: RDD[String] = sc.textFile("datas") val words: RDD[String] = lines.flatMap(_.split(" ")) val wordToOne = words.map(word => (word, 1)) val group: RDD[(String, Iterable[(String, Int)])] = wordToOne.groupBy(t => t._1) val wordcount = group.map { case (word, list) => { list.reduce( (t1, t2) => { (t1._1, t1._2 + t2._2) } ) } } val array: Array[(String, Int)] = wordcount.collect() array.foreach(println) sc.stop() }
第三种
def main(args: Array[String]): Unit = { //建立spark运行配置对象 val sparkConf = new SparkConf().setMaster("local[*]").setAppName("WordCount") //建立连接对象 val sc:SparkContext = new SparkContext(sparkConf) //读取文件 val fileRDD: RDD[String] = sc.textFile("datas") //分词 val wordRDD: RDD[String] = fileRDD.flatMap(_.split(" ")) //数据结构转换 val word2RDD: RDD[(String, Int)] = wordRDD.map((_,1)) //分组聚合 val word2count: RDD[(String, Int)] = word2RDD.reduceByKey(_+_) //采集结果到内存 val wordend: Array[(String, Int)] = word2count.collect() //打印 word2count.foreach(println) //关闭spark sc.stop() }