flink写入elasticsearch（esSink）

最新推荐文章于 2024-02-16 22:58:32 发布

婲落ヽ紅顏誶

最新推荐文章于 2024-02-16 22:58:32 发布

阅读量931

点赞数 1

分类专栏： flink学习

本文链接：https://blog.csdn.net/qq_37195258/article/details/106426005

版权

flink学习专栏收录该内容

12 篇文章 0 订阅

订阅专栏

1.所需依赖

 <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-connector-elasticsearch6_2.11</artifactId>
      <version>1.9.0</version>
    </dependency>
<!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
<dependency>
    <groupId>org.apache.httpcomponents</groupId>
    <artifactId>httpclient</artifactId>
    <version>4.5.5</version>
</dependency>

2.获取esSink

 /**
    * 生成ElasticsearchSink
    */
   private static ElasticsearchSink<Tuple2<String, Long>> generateESSink() {
       // 配置HttpHost
       List<HttpHost> httpHosts = Collections.singletonList(
               //es的连接参数
               new HttpHost("192.168.19.10", 9200,"http")
       );
       ElasticsearchSinkFunction<Tuple2<String, Long>> sinkFunction = new ElasticsearchSinkFunction<Tuple2<String, Long>>() {
           @Override
           public void process(Tuple2<String, Long> tuple2, RuntimeContext runtimeContext, RequestIndexer requestIndexer) {
               // 封装数据
               HashMap<String, String> map = new HashMap<>();
               map.put("content", tuple2.f0);
               map.put("eventTime", tuple2.f1.toString());
               map.put("processTime", String.valueOf(System.currentTimeMillis()));
               // 封装Request
               IndexRequest request = Requests.indexRequest()
                       .index("my_index")//这个索引必须已经存在
                       .type("my_data")
                       .source(map);

               // 发送request
               requestIndexer.add(request);
           }
       };
       ElasticsearchSink.Builder<Tuple2<String, Long>> esSinkBuilder = new ElasticsearchSink.Builder<>(httpHosts, sinkFunction);
       //设置缓存区的大小 （一次存入的数据条数）  如果不设置  默认一次写入
       esSinkBuilder.setBulkFlushMaxActions(50);
       esSinkBuilder.setRestClientFactory(
   restClientBuilder -> {
       restClientBuilder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
           @Override
           public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {

               // elasticsearch username and password
               CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
               credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("$USERNAME", "$PASSWORD"));

               return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
           }
       });
   }
);
       //构建essink
       return esSinkBuilder.build();
   }