Spark 2.0 读写Elasticsearch 6

本地启动ES,我的版本是ES 6.3.2

• maven配置

---

 <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <spark.version>2.1.2</spark.version>
        <hadoop.version>2.6.5</hadoop.version>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-hive_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>org.elasticsearch</groupId>
            <artifactId>elasticsearch-hadoop</artifactId>
            <version>6.3.2</version>
        </dependency>
     
    </dependencies>

---

Java 代码如下:

import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import org.elasticsearch.spark.sql.api.java.JavaEsSparkSQL;

/**
 * @author allen
 * @date 2018/5/9.
 */
public class SparkSqlToEs {

	public static void main(String[] args) {
		SparkSession spark=SparkSession.builder().appName("Sql2Es").config("es.nodes","localhost")
				.config("es.port","9200").master("local[3]").getOrCreate();

		Dataset<Row> person=spark.read().json("/opt/spark/examples/src/main/resources/people.json");

		person.show();

		JavaEsSparkSQL.saveToEs(person,"test/person");

	}
}

curl访问命令或者chrome打开

curl http://localhost:9200/test/person/_search?pretty

就可以看到新增加的数据document.

---

读es表

        Dataset<Row> table= JavaEsSparkSQL.esDF(spark,"trafficwisdom.hot_section/route");
        table.show();
        table.printSchema();