java flink 读取ES

2运行环境flink standalone模式

1、main 入口

   
   

     
     

      
      

     
     
     
     

      
      

       
       package es;
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       import org.apache.flink.api.java.DataSet;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       import org.apache.flink.api.java.ExecutionEnvironment;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       import org.apache.flink.api.java.tuple.Tuple3;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       import org.apache.flink.util.FileUtils;
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       import java.io.File;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       import java.util.List;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       import java.util.Map;
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       public 
       
       class 
       
       EsReadTest {
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      
    
       
       private 
       
       static 
       
       EsRestClientService 
       
       esRestClientService 
       
       = 
       
       new 
       
       EsRestClientService();
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      
    
       
       public 
       
       static 
       
       void 
       
       main
       
       (String[] args) 
       
       throws Exception {
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      
        
       
       // set up the execution environment
      
      
     
     

     
     

      
      

     
     
     
     

      
      
        
       
       final 
       
       ExecutionEnvironment 
       
       env 
       
       = ExecutionEnvironment.getExecutionEnvironment();
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      
        
       
       // 查询数据searchResponse
      
      
     
     

     
     

      
      

     
     
     
     

      
      
        
       
       String 
       
       scrollId 
       
       = 
       
       null;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
               DataSet<Tuple3<String, String, Integer>> dataSet = 
       
       null;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
               List<Tuple3<String, String, Integer>> dataList = 
       
       null;
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      
        
       
       int 
       
       count 
       
       = 
       
       0;
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      
        
       
       while (!
       
       "none".equals(scrollId)) {
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                   Map<String, Object> map = esRestClientService.queryDeviceListPage(scrollId);
      
      
     
     

     
     

      
      

     
     
     
     

      
      
            
       
       if (map.get(
       
       "tupleList") 
       
       instanceof List)
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                       dataList = (List<Tuple3<String, String, Integer>>) map.get(
       
       "tupleList");
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                   scrollId = map.get(
       
       "scrollId").toString();
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      
            
       
       if (dataList == 
       
       null || dataList.size() < 
       
       10000 || count > 
       
       3)
      
      
     
     

     
     

      
      

     
     
     
     

      
      
                
       
       break;
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      
            
       
       // 导入数据
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                   DataSet<Tuple3<String, String, Integer>> dataSetTemp = env.fromCollection(dataList);
      
      
     
     

     
     

      
      

     
     
     
     

      
      
            
       
       if (dataSet == 
       
       null) {
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                       dataSet = dataSetTemp;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                   } 
       
       else {
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                       dataSet = dataSet.union(dataSetTemp);
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                   }
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                   ++count;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
               }
      
      
     
     

     
     

      
      

     
     
     
     

      
      
        
       
       // 分组计算规则
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
               dataSet = dataSet.groupBy(
       
       0).sum(
       
       2);
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      
        
       
       String 
       
       output 
       
       = 
       
       "/opt/flink-data/esoutput2.txt";
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
               FileUtils.deleteFileOrDirectory(
       
       new 
       
       File(output));
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
               dataSet.writeAsText(output);
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
               env.execute(
       
       "read es");
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
           }
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       }
      
      
     
     
   
   

2、游标方式读取es

   
   

     
     

      
      

     
     
     
     

      
      

       
       package es;
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       import com.alibaba.fastjson.JSONObject;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       import org.apache.flink.api.java.tuple.Tuple3;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       import org.apache.http.HttpHost;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       import org.elasticsearch.action.search.*;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       import org.elasticsearch.client.RestClient;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       import org.elasticsearch.client.RestClientBuilder;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       import org.elasticsearch.client.RestHighLevelClient;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       import org.elasticsearch.index.query.BoolQueryBuilder;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       import org.elasticsearch.index.query.QueryBuilders;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       import org.elasticsearch.rest.RestStatus;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       import org.elasticsearch.search.SearchHit;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       import org.elasticsearch.search.builder.SearchSourceBuilder;
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       import java.io.IOException;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       import java.util.ArrayList;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       import java.util.HashMap;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       import java.util.List;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       import java.util.Map;
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       /**
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
        * 阿里云服务器搭建的ES服务
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
        *
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
        * @author lizixian
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
        * @date 2020/3/16 10:41
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
        */
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       public 
       
       class 
       
       EsRestClientService {
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      
    
       
       private 
       
       String 
       
       host 
       
       = 
       
       "172.168.0.138:9200";
      
      
     
     

     
     

      
      

     
     
     
     

      
      
    
       
       private 
       
       String 
       
       scheme 
       
       = 
       
       "http";
      
      
     
     

     
     

      
      

     
     
     
     

      
      
    
       
       private 
       
       String 
       
       index 
       
       = 
       
       "es_index";
      
      
     
     

     
     

      
      

     
     
     
     

      
      
    
       
       private 
       
       String 
       
       type 
       
       = 
       
       "es_type";
      
      
     
     

     
     

      
      

     
     
     
     

      
      
    
       
       private 
       
       RestClientBuilder 
       
       builder 
       
       = 
       
       null;
      
      
     
     

     
     

      
      

     
     
     
     

      
      
    
       
       private 
       
       RestHighLevelClient 
       
       client 
       
       = 
       
       null;
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      
    
       
       public 
       
       void 
       
       init
       
       () {
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
               String[] nodeIpInfos = host.split(
       
       ":");
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
               builder = RestClient.builder(
       
       new 
       
       HttpHost(nodeIpInfos[
       
       0], Integer.parseInt(nodeIpInfos[
       
       1]), scheme))
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                       .setRequestConfigCallback(requestConfigBuilder -> {
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                           requestConfigBuilder.setConnectTimeout(
       
       10 * 
       
       60 * 
       
       1000);
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                           requestConfigBuilder.setSocketTimeout(
       
       10 * 
       
       60 * 
       
       1000);
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                           requestConfigBuilder.setConnectionRequestTimeout(
       
       10 * 
       
       60 * 
       
       1000);
      
      
     
     

     
     

      
      

     
     
     
     

      
      
                    
       
       return requestConfigBuilder;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                       }).setMaxRetryTimeoutMillis(
       
       10 * 
       
       60 * 
       
       1000);
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
               client = 
       
       new 
       
       RestHighLevelClient(builder);
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
           }
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      
    
       
       /**
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
            * 分页查询应设备应用安装列表-使用游标
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
            *
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
            * @author lizixian
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
            * @date 2020/5/10 18:01
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
            */
      
      
     
     

     
     

      
      

     
     
     
     

      
      
    
       
       public Map<String, Object> 
       
       queryDeviceListPage
       
       (String scrollId) {
      
      
     
     

     
     

      
      

     
     
     
     

      
      
        
       
       String 
       
       brand 
       
       = 
       
       "CH";
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      
        
       
       //设置查询数量
      
      
     
     

     
     

      
      

     
     
     
     

      
      
        
       
       SearchSourceBuilder 
       
       sourceBuilder 
       
       = 
       
       new 
       
       SearchSourceBuilder();
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
               sourceBuilder.size(
       
       10000);
      
      
     
     

     
     

      
      

     
     
     
     

      
      
        
       
       BoolQueryBuilder 
       
       bool 
       
       = QueryBuilders.boolQuery();
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      
        
       
       // 平台
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       //        bool.must(QueryBuilders.termQuery("brand", brand));
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
               sourceBuilder.query(bool);
       
       //查询条件
      
      
     
     

     
     

      
      

     
     
     
     

      
      
        
       
       return queryDeviceListPageResult(sourceBuilder, scrollId);
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
           }
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      
    
       
       private Map<String, Object> 
       
       queryDeviceListPageResult
       
       (SearchSourceBuilder sourceBuilder, String scrollId) {
      
      
     
     

     
     

      
      

     
     
     
     

      
      
        
       
       SearchRequest 
       
       searchRequest 
       
       = 
       
       new 
       
       SearchRequest(index)
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                       .types(type)
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                       .scroll(
       
       "2m")
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                       .source(sourceBuilder);
      
      
     
     

     
     

      
      

     
     
     
     

      
      
        
       
       if (client == 
       
       null) {
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                   init();
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
               }
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
               Map<String, Object> resultMap = 
       
       new 
       
       HashMap<>(
       
       5);
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
               List<Tuple3<String, String, Integer>> tupleList = 
       
       new 
       
       ArrayList<>();
      
      
     
     

     
     

      
      

     
     
     
     

      
      
        
       
       try {
      
      
     
     

     
     

      
      

     
     
     
     

      
      
            
       
       SearchResponse 
       
       response 
       
       = 
       
       null;
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      
            
       
       if (scrollId != 
       
       null) {
      
      
     
     

     
     

      
      

     
     
     
     

      
      
                
       
       SearchScrollRequest 
       
       scrollRequest 
       
       = 
       
       new 
       
       SearchScrollRequest(scrollId).scroll(
       
       "2m");
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                       response = client.searchScroll(scrollRequest);
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                   } 
       
       else {
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                       response = client.search(searchRequest);
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                   }
      
      
     
     

     
     

      
      

     
     
     
     

      
      
 
      
      
     
     

     
     

      
      

     
     
     
     

      
      
            
       
       int 
       
       s 
       
       = response.status().getStatus();
      
      
     
     

     
     

      
      

     
     
     
     

      
      
            
       
       if (s == RestStatus.OK.getStatus()) {
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                       SearchHit[] hits = response.getHits().getHits();
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                       scrollId = response.getScrollId();
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                       System.out.println(
       
       "*********************查询es结果");
      
      
     
     

     
     

      
      

     
     
     
     

      
      
                
       
       if (hits != 
       
       null) {
      
      
     
     

     
     

      
      

     
     
     
     

      
      
                    
       
       for (SearchHit hit : hits) {
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                               System.out.println(
       
       "*********************查询es结果：" + hit.getSourceAsString());
      
      
     
     

     
     

      
      

     
     
     
     

      
      
                        
       
       JSONObject 
       
       json 
       
       = JSONObject.parseObject(hit.getSourceAsString());
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                               tupleList.add(
       
       new 
       
       Tuple3<>(json.getString(
       
       "name"), json.getString(
       
       "city"), 
       
       1));
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                           }
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                       }
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                   } 
       
       else {
      
      
     
     

     
     

      
      

     
     
     
     

      
      
                
       
       //清除滚屏
      
      
     
     

     
     

      
      

     
     
     
     

      
      
                
       
       ClearScrollRequest 
       
       clearScrollRequest 
       
       = 
       
       new 
       
       ClearScrollRequest();
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                       clearScrollRequest.addScrollId(scrollId);
       
       //也可以选择setScrollIds()将多个scrollId一起使用
      
      
     
     

     
     

      
      

     
     
     
     

      
      
                
       
       ClearScrollResponse 
       
       clearScrollResponse 
       
       = client.clearScroll(clearScrollRequest);
      
      
     
     

     
     

      
      

     
     
     
     

      
      
                
       
       boolean 
       
       succeeded 
       
       = clearScrollResponse.isSucceeded();
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                   }
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                   resultMap.put(
       
       "scrollId", scrollId);
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                   resultMap.put(
       
       "tupleList", tupleList);
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
               } 
       
       catch (IOException e) {
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
                   e.printStackTrace();
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
               }
      
      
     
     

     
     

      
      

     
     
     
     

      
      
        
       
       return resultMap;
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
           }
      
      
     
     

     
     

      
      

     
     
     
     

      
      

       
       }