Spark 读取mysql表统计

添加maven依赖

      <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.6</version>
        </dependency>

读取数据库

package com.spark

import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.functions._

object StatsProductVersion {

  def main(args: Array[String]): Unit = {

    val spark = SparkSession.builder().master("local[4]").getOrCreate();

    import spark.implicits._

    val otaDeviceVersion = spark.read.format("jdbc").option("url", "jdbc:mysql://xxx.xx.xx.xxx:33061/iot")
      .option("dbtable", "ota_device_version").option("user", "xxxx").option("password", "xxxx").load();

    otaDeviceVersion.show();

    val result = otaDeviceVersion.groupBy($"product_id", $"version").agg(countDistinct($"device_id") as "num");

    result.show()

  }

}