spark-mongoDB的Api

导入依赖

<dependency>
            <groupId>org.mongodb.spark</groupId>
            <artifactId>mongo-spark-connector_2.11</artifactId>
            <version>2.4.0</version>
        </dependency>

读取mongodb中的数据

package mongo

import com.mongodb.spark.MongoSpark
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.types.{StringType, StructField, StructType}


object ReadMongo {

  def main(args: Array[String]): Unit = {

    val spark=SparkSession.builder()
      .appName(this.getClass.getSimpleName)
      .master("local[*]")
    //设置Mongodb的数据库连接字符串
      .config("spark.mongodb.input.uri","mongodb://kunyi:123456@hadoop01/mobike.users")
      .getOrCreate()

    /*//加载Mongodb的数据,放回DataFrame
    val dataFrame = MongoSpark.load(spark)

    //打印表结构信息
    dataFrame.printSchema()
    dataFrame.show()*/

    //自定义获取schema字段
    val schema = StructType(
      List(
        StructField("name",StringType),
        StructField("age",StringType)
      )
    )
    val dataFrame = spark.read.format("com.mongodb.spark.sql").schema(schema).load()

   // dataFrame.show()
    dataFrame.createTempView("account")

    spark.sql("SELECT name,age FROM account").show()
    spark.close()
  }
}

将数据添加到mongoDB中

package mongo

import com.mongodb.spark.MongoSpark
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.SparkSession
import org.bson.Document



object WriteMongo {
  def main(args: Array[String]): Unit = {
    val spark =SparkSession.builder()
      .appName(this.getClass.getSimpleName)
      .master("local[*]")
      //配置参数
      .config("spark.mongodb.input.uri","mongodb://kunyi:123456@hadoop01/mobike.users")
      .config("spark.mongodb.output.uri","mongodb://kunyi:123456@hadoop01/mobike.users")
      .getOrCreate()

    val document1=new Document
    document1.append("name","lisi").append("age",19).append("set","F")

    val document2=new Document
    document2.append("name","zhaoliu").append("age",25)

    val seq = Seq(document1,document2)
    val rdd: RDD[Document] = spark.sparkContext.makeRDD(seq)

    //保存到数据库mongodb中
    MongoSpark.save(rdd)

    //资源释放
    spark.close()

  }
}