SpringBoot整合ES（异步HttpClient和Http连接池）

知识点

• RestHighLevelClient使用
• 异步HttpClient创建
• HTTP连接池
• ES 游标使用

---

一、引入pom

        <!-- httpclient -->
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
            <version>4.5.10</version>
        </dependency>

        <!--为了使用@ConfigurationProperties,还需要这个依赖-->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-configuration-processor</artifactId>
            <optional>true</optional>
            <version>2.3.7.RELEASE</version>
        </dependency>

        <!-- ES Java Api -->
        <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>elasticsearch-rest-high-level-client</artifactId>
            <version>${elasticsearch.version}</version>
            <exclusions>
                <exclusion>
                    <groupId>org.elasticsearch.client</groupId>
                    <artifactId>elasticsearch-rest-client</artifactId>
                </exclusion>
            </exclusions>
        </dependency>

        <dependency>
            <groupId>org.elasticsearch</groupId>
            <artifactId>elasticsearch</artifactId>
            <version>6.6.0</version>
        </dependency>

        <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>elasticsearch-rest-client</artifactId>
            <version>6.6.0</version>
        </dependency>
        <!-- commons-pool2 -->
        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-pool2</artifactId>
            <version>2.6.0</version>
        </dependency>

二、配置文件 application-local.yml

es:
  index: vehaior-2021
  outPath: D:\\usr\\esData.txt

spring:
  # yml配置的优先级高于java配置；如果yml配置和java配置同时存在，则yml配置会覆盖java配置
  http-client:
    pool:
      #连接池的最大连接数，0代表不限；如果取0，需要考虑连接泄露导致系统崩溃的后果
      maxTotalConnect: 500
      #每个路由的最大连接数,如果只调用一个地址,可以将其设置为最大连接数
      maxConnectPerRoute: 400
      # 指客户端和服务器建立连接的超时时间,ms , 最大约21秒,因为内部tcp在进行三次握手建立连接时,默认tcp超时时间是20秒
      connectTimeout: 3000
      # 指客户端从服务器读取数据包的间隔超时时间,不是总读取时间,也就是socket timeout,ms
      readTimeout: 5000
      # 从连接池获取连接的timeout,不宜过大,ms
      connectionRequestTimout: 200
      # 重试次数
      retryTimes: 3
      charset: UTF-8
      # 长连接保持时间 单位s,不宜过长
      keepAliveTime: 15
      # 针对不同的网址,长连接保持的存活时间,单位s,如果是频繁而持续的请求,可以设置小一点,不建议设置过大,避免大量无用连接占用内存资源
#      keepAliveTargetHost:
#        www.baidu.com: 5


# elasticSearch 配置
# 连接多个逗号分隔,格式(IP:端口)
config:
  elasticsearch:
    address: 1xx.xxx.xxx.xxx:9012
    #未加验证可以不配置账户
    #    userName: elastic
    #    password: 1234567
    connectTimeout: 10000
    socketTimeout: 30000
    connectionRequestTimeout: 15000

三、config包

1、ElasticSearchProperty

/**
 * @author: zmm
 * @date: 2021/7/1 15:39
 */

@Data
@Configuration
@ConfigurationProperties(prefix = "config.elasticsearch")
public class ElasticSearchProperty {
    /**
     * 连接地址,格式：IP:端口
     * 多个逗号分隔
     * 示例：127.0.0.1:9200,127.0.0.1:9200,127.0.0.1:9200
     */
    private String address;
    /**
     * 用户名
     */
    private String userName;
    /**
     * 密码
     */
    private String password;
    /**
     * 连接超时时间
     * 默认10s
     */
    private int connectTimeout = 10000;
    /**
     * socket超时时间
     * 默认10s
     */
    private int socketTimeout = 10000;
    /**
     * 请求连接超时时间
     * 默认10s
     */
    private int connectionRequestTimeout = 10000;
}

2、ElasticSearchConfig

• Http连接池
• 异步HttpClient

/**
 * @author: zmm
 * @date: 2021/7/1 20:42
 */
@Configuration
@Slf4j
public class ElasticSearchConfig {
    @Resource
    private ElasticSearchProperty elasticSearchProperty;
    @Resource
    private HttpClientPoolConfig httpClientPoolConfig;

    @Bean
    public RestClientBuilder restClientBuilder() {
        Assert.notNull(elasticSearchProperty, "elasticSearchProperty cannot null ");
        Assert.notNull(elasticSearchProperty.getAddress(), "address hosts cannot null ");
        //ElasticSearch 连接地址地址
        HttpHost[] httpHosts = this.getElasticSearchHttpHosts();

        return RestClient.builder(httpHosts).setRequestConfigCallback(requestConfigBuilder -> {
            //设置连接超时时间
            requestConfigBuilder.setConnectTimeout(elasticSearchProperty.getConnectTimeout());
            requestConfigBuilder.setSocketTimeout(elasticSearchProperty.getSocketTimeout());
            requestConfigBuilder.setConnectionRequestTimeout(elasticSearchProperty.getConnectionRequestTimeout());
            return requestConfigBuilder;
        }).setFailureListener(new RestClient.FailureListener() {
            //某节点失败
            @Override
            public void onFailure(Node node) {
                log.error("[ ElasticSearchClient ] >>  node :{}, host:{},  fail !", node.getName(),
                        node.getHost());
            }
        }).setHttpClientConfigCallback(httpSyncClientBuilder -> {
            try {
                //设置信任ssl访问
                SSLContext sslContext = SSLContexts.createDefault();
                // 设置协议http和https对应的处理socket链接工厂的对象
                Registry<SchemeIOSessionStrategy> sessionStrategyRegistry = RegistryBuilder
                        .<SchemeIOSessionStrategy>create()
                        .register("http", NoopIOSessionStrategy.INSTANCE)
                        .register("https", new SSLIOSessionStrategy(sslContext))
                        .build();

                // 配置io线程
                IOReactorConfig ioReactorConfig = IOReactorConfig.custom()
                     .setIoThreadCount(Runtime.getRuntime().availableProcessors())
                     // .setIoThreadCount(2)
                        .build();

                ConnectingIOReactor ioReactor = new DefaultConnectingIOReactor(ioReactorConfig);
                Assert.notNull(ioReactor, "ioReactor init error");

                //使用Httpclient连接池的方式配置(推荐)，同时支持netty，okHttp以及其他http框架
                PoolingNHttpClientConnectionManager poolConnManager = new PoolingNHttpClientConnectionManager(ioReactor,
                        null, sessionStrategyRegistry, (DnsResolver) null);
                // 最大连接数
                poolConnManager.setMaxTotal(httpClientPoolConfig.getMaxTotalConnect());
                // 同路由并发数
                poolConnManager.setDefaultMaxPerRoute(httpClientPoolConfig.getMaxConnectPerRoute());
                //配置连接池
                httpSyncClientBuilder.setConnectionManager(poolConnManager);
                //设置默认请求头
                List<Header> headers = getDefaultHeaders();
                httpSyncClientBuilder.setDefaultHeaders(headers);
                // 设置长连接策略
                httpSyncClientBuilder.setKeepAliveStrategy(connectionKeepAliveStrategy());
                httpSyncClientBuilder.disableAuthCaching();
            } catch (IOReactorException e) {
                log.error("ES的Http异步连接池配置错误", e);
            }
            return getHttpAsyncClientBuilder(httpSyncClientBuilder);
        });
    }

    /**
     * ElasticSearch Rest client 配置
     *
     * @return RestHighLevelClient
     */
    @Bean
    public RestHighLevelClient restHighLevelClient(@Qualifier("restClientBuilder") RestClientBuilder restClientBuilder) {
        return new RestHighLevelClient(restClientBuilder);
    }

    /**
     * ElasticSearch 连接地址
     * 多个逗号分隔
     * 示例：127.0.0.1:9201,127.0.0.1:9202,127.0.0.1:9203
     */
    private HttpHost[] getElasticSearchHttpHosts() {
        String[] hosts = elasticSearchProperty.getAddress().split(",");
        HttpHost[] httpHosts = new HttpHost[hosts.length];
        for (int i = 0; i < httpHosts.length; i++) {
            String host = hosts[i];
            host = host.replaceAll("http://", "").replaceAll("https://", "");
            Assert.isTrue(host.contains(":"), String.format("your host %s format error , Please refer to [ 127.0.0" +
                    ".1:9200 ] ", host));
            httpHosts[i] = new HttpHost(host.split(":")[0], Integer.parseInt(host.split(":")[1]), "http");
        }
        return httpHosts;
    }

    /**
     * HttpAsyncClientBuilder
     *
     * @param httpAsyncClientBuilder
     * @return
     */
    private HttpAsyncClientBuilder getHttpAsyncClientBuilder(HttpAsyncClientBuilder httpAsyncClientBuilder) {
        if (ObjectUtils.isEmpty(elasticSearchProperty.getUserName()) || ObjectUtils.isEmpty(elasticSearchProperty.getPassword())) {
            return httpAsyncClientBuilder;
        }
        //账密设置
        CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        //es账号密码（一般使用,用户elastic）
        credentialsProvider.setCredentials(AuthScope.ANY,
                new UsernamePasswordCredentials(elasticSearchProperty.getUserName(),
                        elasticSearchProperty.getPassword()));
        httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
        return httpAsyncClientBuilder;
    }

    /**
     * 配置长连接保持策略
     *
     * @return
     */
    public ConnectionKeepAliveStrategy connectionKeepAliveStrategy() {
        return (response, context) -> {
            // Honor 'keep-alive' header
            HeaderElementIterator it = new BasicHeaderElementIterator(
                    response.headerIterator(HTTP.CONN_KEEP_ALIVE));
            while (it.hasNext()) {
                HeaderElement he = it.nextElement();
                log.info("HeaderElement:{}", JSON.toJSONString(he));
                String param = he.getName();
                String value = he.getValue();
                if (value != null && "timeout".equalsIgnoreCase(param)) {
                    try {
                        return Long.parseLong(value) * 1000;
                    } catch (NumberFormatException ignore) {
                        log.error("解析长连接过期时间异常", ignore);
                    }
                }
            }
            HttpHost target = (HttpHost) context.getAttribute(
                    HttpClientContext.HTTP_TARGET_HOST);
            //如果请求目标地址,单独配置了长连接保持时间,使用该配置
            Optional<Map.Entry<String, Integer>> any =
                    Optional.ofNullable(httpClientPoolConfig.getKeepAliveTargetHost()).orElseGet(HashMap::new)
                            .entrySet().stream().filter(
                            e -> e.getKey().equalsIgnoreCase(target.getHostName())).findAny();
            //否则使用默认长连接保持时间
            return any.map(en -> en.getValue() * 1000L).orElse(httpClientPoolConfig.getKeepAliveTime() * 1000L);
        };
    }

    /**
     * 设置请求头
     *
     * @return
     */
    private List<Header> getDefaultHeaders() {
        List<Header> headers = new ArrayList<>();
/*        headers.add(new BasicHeader("User-Agent",
                "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.16 Safari/537" +
                        ".36"));
        headers.add(new BasicHeader("Accept-Encoding", "gzip,deflate"));
        headers.add(new BasicHeader("Accept-Language", "zh-CN"));*/
        headers.add(new BasicHeader("Connection", "Keep-Alive"));
        return headers;
    }
}

3、HttpClientPoolConfig

/**
 * @author: zmm
 * @date: 2021/7/1 23:53
 */

@Component
@ConfigurationProperties(prefix = "spring.http-client.pool")
@Data
public class HttpClientPoolConfig {
    /**
     * java配置的优先级低于yml配置；如果yml配置不存在，会采用java配置
     */
    /**
     * 连接池的最大连接数
     */
    private int maxTotalConnect;
    /**
     * 同路由的并发数
     */
    private int maxConnectPerRoute;
    /**
     * 客户端和服务器建立连接超时，默认2s
     */
    private int connectTimeout = 2 * 1000;
    /**
     * 指客户端从服务器读取数据包的间隔超时时间,不是总读取时间，默认30s
     */
    private int readTimeout = 30 * 1000;

    private String charset = "UTF-8";
    /**
     * 重试次数,默认2次
     */
    private int retryTimes = 2;
    /**
     * 从连接池获取连接的超时时间,不宜过长,单位ms
     */
    private int connectionRequestTimout = 200;
    /**
     * 针对不同的地址,特别设置不同的长连接保持时间
     */
    private Map<String, Integer> keepAliveTargetHost;
    /**
     * 针对不同的地址,特别设置不同的长连接保持时间,单位 s
     */
    private int keepAliveTime = 10;
}

4、WebServerConfiguration

/**
 * @author: zmm
 * @date: 2021/7/1 23:24
 */

@Component
public class WebServerConfiguration implements WebServerFactoryCustomizer<ConfigurableWebServerFactory> {
    @Override
    public void customize(ConfigurableWebServerFactory factory) {
        ((TomcatServletWebServerFactory) factory).addConnectorCustomizers(new TomcatConnectorCustomizer() {
            @Override
            public void customize(Connector connector) {
                Http11NioProtocol protocol = (Http11NioProtocol) connector.getProtocolHandler();
                /**
                 * 定制化keepalivetimeout。设置30秒内没有请求，则服务端自动断开keepalive链接
                 */
                protocol.setKeepAliveTimeout(10000);
                /**
                 * 当客户端发送超过60000个请求则自动断开keepalive链接
                 */
                protocol.setMaxKeepAliveRequests(8000);
            }
        });
    }
}

三、utils包

1、EsUtils

/**
 * @author: zmm
 * @date: 2021/7/2 10:20
 */
@Slf4j
@Component
public class EsUtils {
    private EsUtils() {
    }

    @Autowired
    private RestHighLevelClient restHighLevelClient;

    /**
     * 从es中获取全量的道路事件数据
     *
     * @param index
     * @return
     */
    public List<Map<String, Object>> getAllData(String index) {
        List<Map<String, Object>> mapList = new ArrayList<>();

        try {
            // 1、构建搜索请求对象
            SearchRequest searchRequest = new SearchRequest(index);
            // 2、构建搜索源对象,获取需要的字段信息
            SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
            BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
            boolQuery.must(QueryBuilders.matchAllQuery());

            // 4、向搜索请求对象中设置搜索源
            sourceBuilder.size(5000); //设定每次返回多少条数据
            sourceBuilder.query(boolQuery);

            final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L)); //设定滚动时间间隔
            searchRequest.scroll(scroll);
            // 4、向搜索请求对象中设置搜索源
            searchRequest.source(sourceBuilder);
            // 5、执行搜索，向ES发起http请求，获取结果对象
            SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
            //取回这条响应的scroll id,在后续的scroll调用中会用到
            String scrollId = searchResponse.getScrollId();

            //得到文档数组
            SearchHit[] searchHits = searchResponse.getHits().getHits();
            for (SearchHit searchHit : searchHits) {
                mapList.add(searchHit.getSourceAsMap());
            }
            //遍历搜索命中的数据，直到没有数据
            while (searchHits != null && searchHits.length > 0) {
                SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
                scrollRequest.scroll(scroll);
                searchResponse = restHighLevelClient.scroll(scrollRequest, RequestOptions.DEFAULT);

                scrollId = searchResponse.getScrollId();
                searchHits = searchResponse.getHits().getHits();
                if (searchHits != null && searchHits.length > 0) {
                    for (SearchHit searchHit : searchHits) {
                        mapList.add(searchHit.getSourceAsMap());
                    }
                }
            }

            //清除滚屏
            ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
            clearScrollRequest.addScrollId(scrollId);//也可以选择setScrollIds()将多个scrollId一起使用
            ClearScrollResponse clearScrollResponse = restHighLevelClient.clearScroll(clearScrollRequest,
                    RequestOptions.DEFAULT);
            boolean succeeded = clearScrollResponse.isSucceeded();
            //  System.out.println("succeeded:" + succeeded);
        } catch (IOException e) {
            log.error("请求ES出错", e);
        }
        return mapList;
    }


    public void batchAdd(List<String> lineList, String index) {
        BulkProcessor bulkProcessor = null;
        try {
            bulkProcessor = BulkProcessor.builder(
                    (request, bulkListener) ->
                            restHighLevelClient.bulkAsync(request, RequestOptions.DEFAULT, bulkListener), listener
            )
                    .setBulkActions(10000)
                    .setBulkSize(new ByteSizeValue(30, ByteSizeUnit.MB))
                    .setFlushInterval(TimeValue.timeValueSeconds(5))
                    //并发线程数
                    .setConcurrentRequests(1)
                    // 重试补偿策略
                    .setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100), 3))
                    .build();

            for (String line : lineList) {
                IndexRequest indexRequest = new IndexRequest(index);
                System.out.println(line);
                indexRequest.source(line, XContentType.JSON);
                indexRequest.type("_doc");
                //字符串类型的操作类型参数: 可以是 create 或 update (默认值),可以指定文档id
                // indexRequest.opType(DocWriteRequest.OpType.INDEX);

                bulkProcessor.add(indexRequest);
            }
            bulkProcessor.close();
        } catch (Exception e) {
            log.error("bulk-processor-error:", e);
        } finally {
            bulkProcessor.close();
        }
    }

    BulkProcessor.Listener listener = new BulkProcessor.Listener() {
        @Override
        public void beforeBulk(long executionId, BulkRequest request) {
            // todo do something
            int i = request.numberOfActions();
            log.error("ES 同步数量{}", i);
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
            // todo do something
            Iterator<BulkItemResponse> iterator = response.iterator();
            while (iterator.hasNext()) {
                System.out.println(JSON.toJSONString(iterator.next()));
            }
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
            // todo do something
            log.error("写入ES 重新消费");
        }
    };
}

四、批量更新和下载左右

1、Service

/**
 * @author: zmm
 * @date: 2021/7/2 11:54
 */
@Service
@Slf4j
public class AllDataServiceImpl implements AllDataService {
    @Autowired
    private EsUtils esUtils;
    @Autowired
    private Environment environment;

    @Override
    public String getAllData() {
        String index = environment.getProperty("es.index");
        String outPath = environment.getProperty("es.outPath");
        List<Map<String, Object>> allData = esUtils.getAllData(index);
        BufferedWriter bufferedWriter = null;
        try {
            bufferedWriter = new BufferedWriter(new FileWriter(outPath));
            for (Map<String, Object> sourceAsMap : allData) {
                String vehicleVin = sourceAsMap.get("vehicleVin").toString();
                double drivingDistance = Double.parseDouble(sourceAsMap.get("drivingDistance").toString());
                int tripTimes = Integer.parseInt(sourceAsMap.get("tripTimes").toString());
                int breakTimes = Integer.parseInt(sourceAsMap.get("breakTimes").toString());
                int accelerateTimes = Integer.parseInt(sourceAsMap.get("accelerateTimes").toString());
                int speedOverTimes = Integer.parseInt(sourceAsMap.get("speedOverTimes").toString());
                double drivingTime = Double.parseDouble(sourceAsMap.get("drivingTime").toString());
                LocalDate currentDate = LocalDate.parse(sourceAsMap.get("currentDate").toString(),
                        DateTimeFormatter.ofPattern("yyyy-MM-dd"));

                Object scoreObj = sourceAsMap.get("score");
                Long score = null;
                if (Optional.ofNullable(scoreObj).isPresent()) {
                    score = Long.parseLong(scoreObj.toString());
                }

                String vehicle_state = sourceAsMap.get("vehicle_state").toString();

                Vehiclebehavior vehiclebehavior = new Vehiclebehavior(vehicleVin, drivingDistance, tripTimes,
                        breakTimes, accelerateTimes, speedOverTimes,
                        drivingTime, currentDate, score, vehicle_state);

                String beanString = JSON.toJSON(vehiclebehavior).toString();

                bufferedWriter.write(beanString);
                bufferedWriter.newLine();
                bufferedWriter.flush();
            }
        } catch (IOException e) {
            log.error("ES数据写出失败");
        } finally {
            if (bufferedWriter != null) {
                try {
                    bufferedWriter.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
        return "写出完成";
    }

    @Override
    public String putAllData() {
        String index = environment.getProperty("es.index");
        BufferedReader br = null;
        List<String> lineList = new ArrayList<>();

        try {
            br = new BufferedReader(new FileReader("C:\\Users\\zmm\\Desktop\\esdata.txt"));
            String line;
            while ((line = br.readLine()) != null) {
                lineList.add(line);
            }
            esUtils.batchAdd(lineList, index);

        } catch (FileNotFoundException e) {
            log.error("读取本地ES数据时出错", e);
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            if (br != null) {
                try {
                    br.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
        return "批量更新成功";
    }
}