maven pom文件配置依赖
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<version>2.1.0</version>
</dependency>
<dependency>
<groupId>org.mongodb.spark</groupId>
<artifactId>mongo-spark-connector_2.11</artifactId>
<version>2.1.1</version>
</dependency>
连接mongodb 的router server 服务器就行。需要router server 的用户和密码
package mongodbUtils
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
/**
* Created by Administrator on 2019/6/6.
*/
object ReadMongoSchema {
def main(args: Array[String]): Unit = {
val spark = SparkSession.builder()
.master("local")
.appName("MyApp")
.config("spark.mongodb.input.uri", "mongodb://lx:123456@172.16.18.26:20000/test1.test?authSource=test1")
.config("spark.mongodb.output.uri","mongodb://lx:123456@172.16.18.26:20000/test1.test3?authSource=test1")
.getOrCreate()
// 设置log级别
spark.sparkContext.setLogLevel("WARN")
val schema = StructType(
List(
StructField("name", StringType),
StructField("age", IntegerType),
StructField("sex", StringType)
)
)
// 通过schema约束,直接获取需要的字段
val df = spark.read.format("com.mongodb.spark.sql").schema(schema).load()
df.show()
df.createOrReplaceTempView("user")
val resDf = spark.sql("select * from user")
resDf.show()
//write to mongodb 方法1
//val uriStr = "mongodb://lx:123456@172.16.18.26:20000/test1.test2?authSource=test1"
//df.write.options(Map("spark.mongodb.output.uri"-> uriStr)).mode("overwrite").format("com.mongodb.spark.sql").save()
//write to mongodb 方法2,配置对应的是spark.mongodb.output.uri
resDf.write.mode("append").format("com.mongodb.spark.sql").save()
spark.stop()
System.exit(0)
}
}