Flink 1.12.1 ElasticSearch连接 Sink

Flink 1.12.1 ElasticSearch连接 Sink

参考Flink官网地址：https://ci.apache.org/projects/flink/flink-docs-release-1.12/zh/dev/connectors/elasticsearch.html

• Flink版本：1.12.1
• scala版本：2.12
• ElasticSearch版本：7.10.2

pom依赖

<!-- https://mvnrepository.com/artifact/org.apache.flink/flink-connector-elasticsearch7 -->
<dependency>
    <groupId>org.apache.flink</groupId>
    <artifactId>flink-connector-elasticsearch7_2.12</artifactId>
    <version>1.12.1</version>
</dependency>

代码

def main(args: Array[String]): Unit = {
    val env = StreamExecutionEnvironment.getExecutionEnvironment
    env.setParallelism(1)

    val value: DataStream[SensorReading] = env.addSource(new SensorSource)
      .filter(_.sensor.equals("sensor_1"))
        .map(t => new SensorReading(t.sensor,t.curTime,t.curTemp))

    val hosts = new java.util.ArrayList[HttpHost]

    val host = new HttpHost("localhost", 9200)

    hosts.add(host)


    val esSinkBuilder: ElasticsearchSink.Builder[SensorReading] = new ElasticsearchSink.Builder[SensorReading](
      hosts,
      new ElasticsearchSinkFunction[SensorReading] {
        override def process(t: SensorReading, runtimeContext: RuntimeContext, requestIndexer: RequestIndexer): Unit = {
          val json: util.HashMap[String, String] = new java.util.HashMap[String, String]

          json.put("id", t.sensor)
          json.put("temperature", t.curTemp.toString)
          json.put("time", t.curTime.toString)

          val request: IndexRequest = Requests.indexRequest()
            .index("sensor_record")
            .source(json)


          requestIndexer.add(request)
        }
      }
    )


    esSinkBuilder.setBulkFlushMaxActions(1) //无界流使用此方法可以来一条进行一次写，否则会进入缓冲区。

    /*// provide a RestClientFactory for custom configuration on the internally created REST client
    esSinkBuilder.setRestClientFactory(new RestClientFactory {
      override def configureRestClientBuilder(restClientBuilder: RestClientBuilder): Unit = {
        restClientBuilder.setDefaultHeaders()
        restClientBuilder.setMaxRetryTimeoutMillis(...)
        restClientBuilder.setPathPrefix(...)
        restClientBuilder.setHttpClientConfigCallback(...)
      }
    })*/

    value.addSink(esSinkBuilder.build())

    env.execute()
  }

执行结果

GET _cat/indices?v

GET /sensor_record/_search