导入依赖
<dependency>
<groupId>org.mongodb.spark</groupId>
<artifactId>mongo-spark-connector_2.11</artifactId>
<version>2.4.0</version>
</dependency>
读取mongodb中的数据
package mongo
import com.mongodb.spark.MongoSpark
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.types.{StringType, StructField, StructType}
object ReadMongo {
def main(args: Array[String]): Unit = {
val spark=SparkSession.builder()
.appName(this.getClass.getSimpleName)
.master("local[*]")
//设置Mongodb的数据库连接字符串
.config("spark.mongodb.input.uri","mongodb://kunyi:123456@hadoop01/mobike.users")
.getOrCreate()
/*//加载Mongodb的数据,放回DataFrame
val dataFrame = MongoSpark.load(spark)
//打印表结构信息
dataFrame.printSchema()
dataFrame.show()*/
//自定义获取schema字段
val schema = StructType(
List(
StructField("name",StringType),
StructField("age",StringType)
)
)
val dataFrame = spark.read.format("com.mongodb.spark.sql").schema(schema).load()
// dataFrame.show()
dataFrame.createTempView("account")
spark.sql("SELECT name,age FROM account").show()
spark.close()
}
}
将数据添加到mongoDB中
package mongo
import com.mongodb.spark.MongoSpark
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.SparkSession
import org.bson.Document
object WriteMongo {
def main(args: Array[String]): Unit = {
val spark =SparkSession.builder()
.appName(this.getClass.getSimpleName)
.master("local[*]")
//配置参数
.config("spark.mongodb.input.uri","mongodb://kunyi:123456@hadoop01/mobike.users")
.config("spark.mongodb.output.uri","mongodb://kunyi:123456@hadoop01/mobike.users")
.getOrCreate()
val document1=new Document
document1.append("name","lisi").append("age",19).append("set","F")
val document2=new Document
document2.append("name","zhaoliu").append("age",25)
val seq = Seq(document1,document2)
val rdd: RDD[Document] = spark.sparkContext.makeRDD(seq)
//保存到数据库mongodb中
MongoSpark.save(rdd)
//资源释放
spark.close()
}
}