Spark练习综测

最新推荐文章于 2024-11-02 15:48:17 发布

ZaKka.

最新推荐文章于 2024-11-02 15:48:17 发布

阅读量407

点赞数 12

文章标签： ajax 前端 javascript scala

本文链接：https://blog.csdn.net/2301_77281478/article/details/137650489

版权

本文展示了如何使用ApacheSpark在Java和Python中执行三个不同的文本处理任务：从文件中去除重复单词、统计特定起始数字的流量总量以及计算由相同字母组成的单词。

摘要由CSDN通过智能技术生成

1.使用Spark完成单词去重

text02.txt的数据

java php hello word

phpp hi exe java

python hello kitty

php happy abc java


import org.apache.spark.{SparkConf, SparkContext}

object WordDistinct {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("WordDistinct").setMaster("local")
    val sc   = new SparkContext(conf)

    val lines = sc.textFile("file:///C:\\Users\\Administrator\\Desktop\\text02.txt")
    val words = lines.flatMap(line => line.split(" "))
    val uniqueWords = words.distinct()

    uniqueWords.collect().foreach(println)
  }
}

2.使用Spark统计133 136 139开头的总流量

13326293050 81

13626594101 50

13326435696 30

13926265119 40

13326564191 2106

13626544191 1432

13919199499 300

import org.apache.spark.{SparkConf, SparkContext}
object TrafficCount {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("TrafficCount").setMaster("local")
    val sc = new SparkContext(conf)

    val data = sc.parallelize(List(
      ("13326293050", 81),
      ("13626594101", 50),
      ("13326435696", 30),
      ("13926265119", 40),
      ("13326564191", 2106),
      ("13626544191", 1432),
      ("13919199499", 300)
    ))

    val traffic = data.filter { case (number, _) => number.startsWith("133") || number.startsWith("136") || number.startsWith("139") }
      .map { case (_, value) => value }
      .sum()

    println(traffic)
  }
}

3.完成统计相同字母组成的单词

text01.txt的数数据

abc acb java

avaj bac

cba abc

jvaa php hpp

pph python thonpy

import org.apache.spark.{SparkConf, SparkContext}

object AnagramCount {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("AnagramCount").setMaster("local")
    val sc = new SparkContext(conf)

    val lines = sc.textFile("file:///C:\\Users\\Administrator\\Desktop\\text01.txt")
    val words = lines.flatMap(line => line.split(" "))
    val anagrams = words.map(word => (word.toLowerCase.sorted, 1))
      .reduceByKey(_ + _)

    anagrams.collect().foreach(println)
  }
}