spark on es 多索引查询

最新推荐文章于 2022-12-27 18:01:39 发布

weixin_34342578

最新推荐文章于 2022-12-27 18:01:39 发布

阅读量435

点赞数

文章标签：大数据

原文链接：http://www.cnblogs.com/wq920/p/8640009.html

版权

核心接口

trait SparkOnEsService {
  val conf = new SparkConf
//  conf.setMaster("local[10]")
  val file = new File("lib").getCanonicalFile.listFiles()
  val fs = file.toSeq.map(f => {
    f.getAbsolutePath
  })
  conf.setJars(fs)
  conf.setAppName("spark_sql_on_es")
  val sqlContext = SparkSession.builder().config(conf).getOrCreate()
  val options = Map("pushdown" -> "true", "es.nodes" -> "ip", "es.port" -> "prot")
  valt1 = sqlContext.read.format("org.elasticsearch.spark.sql").options(options).load("index/index_type")
  val t2 = sqlContext.read.format("org.elasticsearch.spark.sql").options(options).load("index/index_type")
  t1.createTempView("v1")
  t2.createTempView("v2")
  def excutor(sql: String): DataResult
}