ES报错:Connection reset by peer解决经历

项目场景：

Spring boot集成spring-data-elasticsearch4.4.10, 使用ElasticsearchRestTemplate. es为个人在容器内拉去镜像搭建，不需要用户名密码

---

问题描述

每隔20分钟报错：Connection reset by peer; nested exception is java.lang.RuntimeException: Connection reset by peer

---

原因分析：

1.查看es容器是否有报错，排查结果无报错

$ curl 'localhost:9200/_cat/health'  # 查询节点健康状况
$ curl 'localhost:9200/_cat/indices?v'  #查看集群中所有索引的详细信息。包括索引的健康度、状态、分片数和副本数、包含的文档数等
$ curl 'localhost:9200/_cluster/health'  #查询集群的健康状态
$ curl 'localhost:9200/_cluster/allocation/explain' #定位未分配问题

2.查看服务端keepalive，为两小时，与观察到的20分钟现象不符合（当前情况应属于服务端断开连接，不需要查看客户端keepalive）

root@es7-5ffdd5b7fc-zfqft:/usr/share/elasticsearch# cat /proc/sys/net/ipv4/tcp_keepalive_time
7200

3.由于项目部署在阿里云k8s容器内，检查容器内是否开启tcp健康检查，tcp健康检查的默认频率很快，会频繁扫描端口，网络不好时会导致断连

4.可能是防火墙的原因，查询资料：防火墙上的各会话缺省保持时间都相对较短，例如：缺省情况下，TCP的保持时间为1200s，UDP的保持时间为120s

此时基本锁定防火墙

---

解决方案：

方案一：自行编写心跳代码
该方法尝试了时间间隔5分钟，2分钟，1分钟，10秒还是会偶尔出现报错

    @Scheduled(cron = "*/10 * * * * * ")
    @Async
    public void heartbeatTask() {
        //todo 执行es查询
    }

方案二：设置保活机制
主要关注httpClient设置保活策略，这里设置为5分钟。该方案可以彻底解决问题

import org.apache.http.HttpHost;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
//import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories;

import java.time.Duration;

@Configuration
@EnableElasticsearchRepositories(basePackages = "com.example.test")
//@ComponentScan(basePackages = { "com.example.test" })
public class ElasticRestClientConfig {

    @Value("${spring.elasticsearch.rest.uris}")
    private String url;
//    @Value("${spring.elasticsearch.rest.username}")
//    private String username;
//    @Value("${spring.elasticsearch.rest.password}")
//    private String password;

    @Bean
    public RestHighLevelClient client() {
        url = url.replace("http://","");
        String[] urlArr = url.split(",");
        HttpHost[] httpPostArr = new HttpHost[urlArr.length];
        for (int i = 0; i < urlArr.length; i++) {
            HttpHost httpHost = new HttpHost(urlArr[i].split(":")[0].trim(),
                    Integer.parseInt(urlArr[i].split(":")[1].trim()), "http");
            httpPostArr[i] = httpHost;
        }
        /** 提供密码登录代码相关代码 本项目不需要密码登录
        final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(AuthScope.ANY,new UsernamePasswordCredentials(username,password));*/
        RestClientBuilder builder = RestClient.builder(httpPostArr)
                // 异步httpclient配置
                .setHttpClientConfigCallback(httpClientBuilder -> {
                    // 账号密码登录
//                    httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
                    // httpclient连接数配置
                    httpClientBuilder.setMaxConnTotal(30);
                    httpClientBuilder.setMaxConnPerRoute(10);
                    // httpclient保活策略
                    httpClientBuilder.setKeepAliveStrategy(((response, context) -> Duration.ofMinutes(5).toMillis()));
                    return httpClientBuilder;
                });
        return new RestHighLevelClient(builder);
    }

    @Bean
    public ElasticsearchRestTemplate elasticsearchRestTemplate() {
        return new ElasticsearchRestTemplate(client());
    }
}