林子雨大数据应用实验七部分代码

最新推荐文章于 2024-07-25 09:48:00 发布

凡我喵族，喵喵喵喵

最新推荐文章于 2024-07-25 09:48:00 发布

阅读量533

点赞数 1

文章标签：大数据 spark scala

本文链接：https://blog.csdn.net/weixin_64604651/article/details/128045956

版权

1.SimpleApp.scala

/* SimpleApp.scala */
import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import org.apache.spark.SparkConf

object SimpleApp {
    def main(args: Array[String]) {
        val logFile = " hdfs://localhost:9000/user/hadoop/test.txt"
    val conf = new SparkConf().setAppName("Simple Application")
    val sc = new SparkContext(conf)
    val logData = sc.textFile(logFile, 2)
    val num = logData.count()
    printf ("The num of this file is %d", num)
    }
}

2.simple.sbt

name := "Simple Project"
version := "1.0"
scalaVersion := "2.11.12"
libraryDependencies += "org.apache.spark" %% "spark-core" % "2.4.0"

3.数据去重remdup.scala

import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import org.apache.spark.SparkConf
import org.apache.spark.HashPartitioner

object RemDup {
    def main(args: Array[String]) {
        val conf = new SparkConf().setAppName("RemDup")
        val sc = new SparkContext(conf)
        val dataFile = "file:///home/charles/data"
        val data = sc.textFile(dataFile,2)
        val    res    =    data.filter(_.trim().length>0).map(line=>(line.trim,"")).partitionBy(new HashPartitioner(1)).groupByKey().sortByKey().keys
        res.saveAsTextFile("result")
    }
}

3.simple.sbt

name := "Simple Project"
version := "1.0"
scalaVersion := "2. 11. 12"
libraryDependencies += "org.apache.spark" %% "spark-core" % "2.4.0"

4.求平均值avgscore.scala

import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import org.apache.spark.SparkConf
import org.apache.spark.HashPartitioner

object AvgScore {
  def main(args: Array[String]) {
    val conf = new SparkConf().setAppName("AvgScore")
    val sc = new SparkContext(conf)
    val dataFile = "file:///home/hadoop/data"
    val data = sc.textFile(dataFile,3)

    val  res  =  data.filter(_.trim().length>0).map(line=>(line.split("  ")(0).trim(),line.split(" ")(1).trim().toInt)).partitionBy(new HashPartitioner(1)).groupByKey().map(x => {
    var n = 0
    var sum = 0.0
    for(i <- x._2){
    sum = sum + i
    n = n +1
    }
    val avg = sum/n
    val format = f"$avg%1.2f".toDouble
    (x._1,format)
	})
        res.saveAsTextFile("result")
    }
}