2020-12-05

Flink ElasticSearch Sink 密码 安全校验

如何编写ES Sink(需要输入用户名和密码)

Flink官网没有给出详细的如果，ES带有权限校验，如何连接下面就是详细的ES Sink

object EsSink {
  def esBuilder(params:ParameterTool): ElasticsearchSink.Builder[TopicAndValue] = {
	//添加ES相关的信息
    val httpHosts = new java.util.ArrayList[HttpHost]
    httpHosts.add(new HttpHost("192.168.10.204", 9200, "http"))
    val esSinkBuilder = new ElasticsearchSink.Builder[TopicAndValue](
      httpHosts,
      new ElasticsearchSinkFunction[TopicAndValue] {
        def process(element: TopicAndValue, ctx: RuntimeContext, indexer: RequestIndexer) {
          //这里输入ES相关的信息
          val rqst: IndexRequest = Requests.indexRequest
            .index("test_duration_v2")
            .`type`("_doc")
            .source(element.value, XContentType.JSON)
          indexer.add(rqst)
        }
      }
    )

    esSinkBuilder.setBulkFlushMaxActions(100000)
    esSinkBuilder.setBulkFlushBackoffRetries(1)
    esSinkBuilder.setBulkFlushInterval(TimeUnit.SECONDS.toSeconds(10))
    esSinkBuilder.setBulkFlushMaxSizeMb(100)

    esSinkBuilder.setRestClientFactory(new RestClientFactory{
      override def configureRestClientBuilder(restClientBuilder: RestClientBuilder): Unit = {
        restClientBuilder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
          override def customizeHttpClient(httpClientBuilder: HttpAsyncClientBuilder): HttpAsyncClientBuilder = {
            val credentialsProvider = new BasicCredentialsProvider
            //在这里添加你的账号和密码
            credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "123456"))
            httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)
          }
        })
        restClientBuilder.setMaxRetryTimeoutMillis(100)
      }
    })
    esSinkBuilder
  }
}

主程序代码

object ComputeTimeDifferences {

  def main(args: Array[String]): Unit = {
    val env = StreamExecutionEnvironment.getExecutionEnvironment
    val params = ParameterTool.fromArgs(args)
    val kafkaDStream = KafkaSourceDao
      .getKafkaSource(env, params)

    val sideOutKafkaTag = new OutputTag[TopicAndValue]("side-tag")
    val splitStream = kafkaDStream
      .process(processFunction = new ProcessFunction[TopicAndValue, TopicAndValue] {
        override def processElement(value: TopicAndValue, ctx: ProcessFunction[TopicAndValue, TopicAndValue]#Context, out: Collector[TopicAndValue]): Unit = {
          val specialDataType = getSpecialDataType(value.value)
          if (specialDataType._2.isEmpty || specialDataType._2.size() == 0) {
            ctx.output(sideOutKafkaTag, value)
          } else {
            out.collect(value)
          }
        }
      })

    splitStream
      .getSideOutput(sideOutKafkaTag)
      .addSink(EsSink.esBuilder(params).build())
      .name("side-stream")

    splitStream
      .keyBy(data => {
        val specialDataType = getSpecialDataType(data.value)
        val tag = specialDataType._2.getString("tag")
        val dataJSONObject = ParseJsonDataUtils.getJsonData(data.value)
        val uid = dataJSONObject.getString("uid")
        tag + uid
      })
      .process(new AppendDurationProcessFunction(1000 * 30))
      //在这里引用刚才的累即可
      .addSink(EsSink.esBuilder(params).build())
      .name("es-stream")
    env.execute("job")
  }

}