SparkSQL之读取数据

最新推荐文章于 2024-02-21 21:06:34 发布

维维weiwei

最新推荐文章于 2024-02-21 21:06:34 发布

阅读量1.2k

点赞数

分类专栏： Spark生态系统

本文链接：https://blog.csdn.net/tangshiweibbs/article/details/70239386

版权

Spark生态系统专栏收录该内容

24 篇文章 0 订阅

订阅专栏

def main(args: Array[String]): Unit = {
  val conf = new SparkConf().setMaster("local").setAppName("ScalaSparkSQL")
  val sc = new SparkContext(conf)
  val sqlContext = new SQLContext(sc)

  //1.读取文件格式为json

  val personDF = sqlContext.read.format("json").load("hdfs://master:9000/input/spark/sql/people.json")
  val personDF = sqlContext.read.json("hdfs://master:9000/input/spark/sql/people.json")
  personDF.show()

  //2.读取文件格式为jdbc
  val url = "jdbc:mysql://localhost:3306/bigdata_db"
  val table = "person"
  val partitionCol = "pid"
  val lowerBound = 2
  val upperBound = 4
  val numPartitions = 2
  val properties = new Properties()
  properties.setProperty("user", "root")
  properties.setProperty("password", "root")
  val jdbcDF = sqlContext.read.jdbc(url, table, partitionCol, lowerBound, upperBound, numPartitions, properties)
  jdbcDF.show()
  sc.stop()
}