elasticsearch-hadoop提供ElasticSearch与Apache Spark的本地集成。从ElasticSearch读取的数据是以Spark里RDD的形式进行操作,同时Spark的RDD的内容也可以被转换为文档,保存进ElasticSearch中进行查询。下面是两个交互的简单实例:
相关的依赖jar包
elasticsearch-spark_2.10-2.3.2.jar
elasticsearch-spark_2.11-2.3.2.jar
elasticsearch-spark-1.2_2.10-2.3.2.jar
elasticsearch-spark-1.2_2.11-2.3.2.jar
ElasticSearch -> Spark
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.elasticsearch.spark.rdd.EsSpark
import org.apache.spark.sql.SQLContext
object E2SExample {
def main(args: Array[String]) {
var conf = new SparkConf()
conf.setAppName("S2EExample")
conf.setMaster("spark://master:7077")
conf.set("es.nodes", "master:9200")
val sc = new SparkContext(conf)
val rdd = EsSpark.esRDD(sc, "data/t1", "?q=王*")
println("rdd count: " + rdd.count())
rdd.collect().foreach(record => {
print(record._1 + ":")
for ((k, v) <- record._2) {
print(k + ":" + v)
}
println()
})
val sqlContext = new SQLContext(sc)
val df = sqlContext.read.format("org.elasticsearch.spark.sql").load("data/t1")
df.printSchema()
df.collect().foreach(println)
sc.stop()
}
}
Spark -> ElasticSearch
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.elasticsearch.spark.rdd.EsSpark
case class Job(jobName:String, jobUrl:String, companyName:String, salary:String)
object S2EExample {
def main(args:Array[String]) {
var conf = new SparkConf()
conf.setAppName("S2EExample")
conf.setMaster("spark://master:7077")
conf.set("es.index.auto.create", "true")
conf.set("es.nodes", "master:9200")
val sc = new SparkContext(conf)
val job1 = Job("C开发工程师","http://job.c.com","c公司","10000")
val job2 = Job("C++开发工程师","http://job.c++.com","c++公司","10000")
val job3 = Job("C#开发工程师","http://job.c#.com","c#公司","10000")
val job4 = Job("Java开发工程师","http://job.java.com","java公司","10000")
val job5 = Job("Scala开发工程师","http://job.scala.com","java公司","10000")
val rdd = sc.makeRDD(Seq(job1, job2, job3, job4, job5))
EsSpark.saveToEs(rdd, "data/job")
sc.stop()
}
}
$bin/spark-submit --class org.esspark.examples.S2EExample --master
spark://master:7077
--jars ../elasticsearch-hadoop-2.3.2/dist/elasticsearch-spark_2.10-2.3.2.jar
../temp/esspark-0.0.1-SNAPSHOT.jar