spark的pv+uv

最新推荐文章于 2022-07-22 14:16:13 发布

慢慢活成讨厌的样子

最新推荐文章于 2022-07-22 14:16:13 发布

阅读量88

点赞数

本文链接：https://blog.csdn.net/qq_41458071/article/details/106364569

版权

object PV {
  System.setProperty("hadoop.home.dir","D:\\hadoop\\hadoop-2.9.2")

  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setMaster("local").setAppName("TopN")
    val sc = new SparkContext(conf)

    val lines = sc.textFile(args(0))
    //方式一
    /*lines.map(x => ("pv",1))
      .reduceByKey(_+_)
      .foreach(println(_))*/
    //方式二
    lines.map(x => (s"${Random.nextInt(lines.getNumPartitions)}",1))
      .reduceByKey(_+_)
      .foreach(println(_))
    sc.stop()
  }
}

object UV {
  System.setProperty("hadoop.home.dir","D:\\hadoop\\hadoop-2.9.2")

  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setMaster("local").setAppName("TopN")
    val sc = new SparkContext(conf)

    val lines = sc.textFile(args(0))
    lines.map(_.split(" "))
      .map(x => x(0))
        .distinct()
      .map(x=>("UV",1))
      .reduceByKey(_+_)
      .foreach(println(_))
    sc.stop()
  }
}