elasticsearch restclient bulk批量索引数据，可回滚返回失败数据及原因（ES5 仅供参考）

自用代码，不喜勿喷，感谢指正。（仅供参考，仅供参考，仅供参考。）

maven：

<!-- 引入es依赖包 -->
        <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>elasticsearch-rest-high-level-client</artifactId>
            <!-- web项目可以不排除下面的依赖 -->
            <exclusions>
                <exclusion>
                    <groupId>com.google.guava</groupId>
                    <artifactId>guava</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>com.fasterxml.jackson.core</groupId>
                    <artifactId>jackson-core</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>com.fasterxml.jackson.dataformat</groupId>
                    <artifactId>jackson-dataformat-smile</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>com.fasterxml.jackson.dataformat</groupId>
                    <artifactId>jackson-dataformat-yaml</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>com.fasterxml.jackson.core</groupId>
                    <artifactId>jackson-databind</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>com.fasterxml.jackson.dataformat</groupId>
                    <artifactId>jackson-dataformat-cbor</artifactId>
                </exclusion>
            </exclusions>
            <version>${es.version}</version>
        </dependency>
        <!--es sniffer 开启集群嗅探功能-->
        <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>elasticsearch-rest-client-sniffer</artifactId>
            <version>${es.version}</version>
        </dependency>

java：

import javafx.util.Pair;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpHost;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager;
import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor;
import org.apache.http.impl.nio.reactor.IOReactorConfig;
import org.apache.http.nio.reactor.IOReactorException;
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.sniff.ElasticsearchHostsSniffer;
import org.elasticsearch.client.sniff.HostsSniffer;
import org.elasticsearch.client.sniff.SniffOnFailureListener;
import org.elasticsearch.client.sniff.Sniffer;
import org.elasticsearch.common.xcontent.XContentType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.*;

/**
 * elasticsearch rest client operation class
 * 方法：canRollbackBulkDocIndex。可回调的批量插入方法。
 * 注意：使用前需先行调用 initialize() 方法修改变量 hostAndPorts 中的 es 集群地址信息并初始化客户端，client 在使用完毕后需要调用该工具类的 close() 方法。
 */
public class ESRestClientTools {
    private static Logger log = LoggerFactory.getLogger(ESRestClientTools.class);
    private static RestClient restClient = null;
    private static final ElasticsearchHostsSniffer.Scheme SCHEMA = ElasticsearchHostsSniffer.Scheme.HTTP;
    private static String hostAndPorts;
    private static Sniffer sniffer = null;

    /**
     * obtain elasticsearch high level rest client.
     *
     * @return HighLevelRestClient
     */
    public static RestHighLevelClient getHighLevelRestClient() {
        if (null != restClient && ping() == 200) {
            return new RestHighLevelClient(restClient);
        } else {
            if (StringUtils.isNotEmpty(hostAndPorts)) {
                initialize(hostAndPorts);
                if (null != restClient && ping() == 200) {
                    return new RestHighLevelClient(restClient);
                } else {
                    log.info("can not obtain useful ESRestClient, please have a check for ip or network status.");
                    return null;
                }
            } else {
                log.info("the connect param is enpty or null!");
                return null;
            }
        }
    }

    /**
     * test server network state.
     *
     * @return status code.
     */
    private static int ping() {
        Map<String, String> params = Collections.singletonMap("pretty", "true");
        int code = 0;
        try {
            if (restClient != null) {
                Response response = restClient.performRequest("GET", "/", params);
                code = response.getStatusLine().getStatusCode();
                log.info("code:{},elasticsearch connected success!", code);
            } else {
                log.info("the rest client is null!");
            }
        } catch (IOException e) {
            log.info("elasticsearch connect error：{}。", e.getLocalizedMessage());
            e.printStackTrace();
        }
        return code;
    }

    /**
     * close elasticsearch rest client.
     */
    public static void close() {
        try {
            if (null != sniffer) {
                sniffer.close();
                log.info("ESRestClient sniffer closed。。。。。。。。。");
            }
            if (null != restClient) {
                restClient.close();
                log.info("ESRestClient closed。。。。。。。。。");

            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * initialize logic.
     */
    public static void initialize(String firstInputAddressOfCluster) {
        if (null != restClient){
            create(firstInputAddressOfCluster);
        } else {
            if (ping() != 200) {
                close();
                create(firstInputAddressOfCluster);
            } else {
                log.info("the client is OK please dong't initialize repeat!");
            }
        }
    }
    /**
     * create elasticsearch rest client.
     */
    private static void create(String firstInputAddressOfCluster) {
        hostAndPorts = firstInputAddressOfCluster;
        if (StringUtils.isNotEmpty(hostAndPorts)) {
            String[] splites = hostAndPorts.split(",");
            HttpHost[] hosts = new HttpHost[splites.length];
            for (int i = 0; i < splites.length; i++) {
                String host = splites[i].split(":")[0].trim();
                int port = Integer.parseInt(splites[i].split(":")[1].trim());
                hosts[i] = new HttpHost(host, port, SCHEMA.toString());
            }
            try {
                RestClientBuilder builder = RestClient.builder(hosts);
                builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
                    @Override
                    public RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
                        requestConfigBuilder.setConnectTimeout(1000 * 10);
                        requestConfigBuilder.setSocketTimeout(1000 * 60);
                        requestConfigBuilder.setConnectionRequestTimeout(1000 * 5);
                        return requestConfigBuilder;
                    }
                });
                builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
                    @Override
                    public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
                        try {
                            final IOReactorConfig ioReactorConfig = IOReactorConfig.custom()
                                    .setConnectTimeout(1000 * 20)
                                    .setSoTimeout(1000 * 60 * 2)
                                    .setRcvBufSize(4096)
                                    .setSndBufSize(8192)
                                    .setSoKeepAlive(true)
                                    .build();
                            final PoolingNHttpClientConnectionManager connManager = new PoolingNHttpClientConnectionManager(new
                                    DefaultConnectingIOReactor(ioReactorConfig));
                            connManager.setMaxTotal(100);//多线程访问时最大并发量
                            connManager.setDefaultMaxPerRoute(20); //单次路由线程上限

                            httpClientBuilder.disableAuthCaching();
                            httpClientBuilder.setDefaultIOReactorConfig(ioReactorConfig);
                            httpClientBuilder.setConnectionManager(connManager);
                        } catch (IOReactorException e) {
                            e.printStackTrace();
                        }
                        return httpClientBuilder;
                    }
                });

                //star cluster sniffe function, it will be ok for provide partial IP address of es cluster.
                SniffOnFailureListener sniffOnFailureListener = new SniffOnFailureListener();

                builder.setFailureListener(sniffOnFailureListener);
                builder.setMaxRetryTimeoutMillis(1000 * 60 * 5); //the time(ms) of max retry timeout
                restClient = builder.build();

                HostsSniffer hostsSniffer = new ElasticsearchHostsSniffer(
                        restClient,
                        ElasticsearchHostsSniffer.DEFAULT_SNIFF_REQUEST_TIMEOUT * 5,
                        ElasticsearchHostsSniffer.Scheme.HTTP);

                sniffer = Sniffer.builder(restClient)
                        .setHostsSniffer(hostsSniffer)
                        .setSniffIntervalMillis(1000 * 60 * 3)
                        .setSniffAfterFailureDelayMillis(1000 * 10)
                        .build();

                sniffOnFailureListener.setSniffer(sniffer);

                log.info("ESRestClient already initialized。。。。。。。。。");
            } catch (Exception ex) {
                log.info("ESRestClient initialize error: {}. Please have a check for ip or network status.", ex.getMessage());
                ex.printStackTrace();
            }
        }
    }

    /**
     * es bulk index api
     *
     * @param client    RestHighLevelClient
     * @param indexName index
     * @param typeName  type
     * @param batch     batch data
     * @param batchSize the size of every batch
     * @return failure doc collect
     */
    public static ArrayList<Pair<String,String>> canRollbackBulkDocIndex(RestHighLevelClient client, String indexName, String typeName, ArrayList<String> batch, int batchSize) {
        ArrayList<Pair<String,String>> failure = new ArrayList<>();
        if (batchSize <= 0 || batchSize > 500) {
            batchSize = 500;
        }
        Iterator<String> batchIter = batch.iterator();
        BulkRequest request = new BulkRequest();
        int cursor = 0;
        int count = 0;
        try {
            while (batchIter.hasNext()) {
                String json = batchIter.next();
                if (StringUtils.isNotEmpty(json)) {
                    if (cursor < batchSize) {
                        request.add(new IndexRequest(indexName, typeName).create(false).source(json, XContentType.JSON));
                    } else {
                        everyBatchWrapper(client, request, failure);
                        cursor = 0;
                        request = new BulkRequest();
                        request.add(new IndexRequest(indexName, typeName).create(false).source(json, XContentType.JSON));
                    }
                    cursor++;
                } else {
                    log.info("index of {} is null or enpty.", count);
                }
                count++;
            }
            everyBatchWrapper(client, request, failure);
        } catch (Exception e) {
            log.info("the failed json element index range is {} - {}.", count - cursor, count);
            log.info("Cause by: {}.", e.getMessage());
            e.printStackTrace();
        }
        if (failure.size() > 0) {
            log.info("the number of failed doc index about this target index collention is {}.", failure.size());
        }
        return failure;
    }

    /**
     * every batch operation
     *
     * @param client  RestHighLevelClient
     * @param request BulkRequest
     * @param failure failure collection
     * @throws IOException
     */
    private static void everyBatchWrapper(RestHighLevelClient client, BulkRequest request, ArrayList<Pair<String,String>> failure) throws IOException {
        BulkResponse response = client.bulk(request);
        if (response.hasFailures()) {
            List<DocWriteRequest> list = request.requests();
            Iterator<BulkItemResponse> iter = response.iterator();
            while (iter.hasNext()) {
                BulkItemResponse bir = iter.next();
                if (bir.isFailed()) {
                    String reason = "HTTP status: [" +  bir.getFailure().getStatus().getStatus() + "], Cause by: " + bir.getFailureMessage() + ".";
                    int docIndex = bir.getItemId();
                    IndexRequest ir = (IndexRequest) list.get(docIndex);
                    failure.add(new Pair<>(ir.source().utf8ToString(), reason));
                }
            }
        }
    }
}

scala（bigdata 使用）：

import scala.collection.JavaConverters._
import scala.collection.JavaConversions._
import org.apache.commons.lang3.StringUtils
import org.apache.http.HttpHost
import org.apache.http.client.config.RequestConfig
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
import org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager
import org.apache.http.impl.nio.reactor.{DefaultConnectingIOReactor, IOReactorConfig}
import org.apache.http.nio.reactor.IOReactorException
import org.elasticsearch.action.DocWriteRequest
import org.elasticsearch.action.bulk.{BulkItemResponse, BulkRequest, BulkResponse}
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.client.sniff.{ElasticsearchHostsSniffer, HostsSniffer, SniffOnFailureListener, Sniffer}
import org.elasticsearch.client.{Response, RestClient, RestClientBuilder, RestHighLevelClient}
import org.elasticsearch.common.xcontent.XContentType
import org.slf4j.LoggerFactory

import scala.collection.mutable
import scala.collection.mutable.ListBuffer

/**
  * elasticsearch rest client operation class
  * 方法：canRollbackBulkDocIndex。可回调的批量插入方法。
  * 注意：使用前需先行调用 initialize() 方法修改变量 hostAndPorts 中的 es 集群地址信息并初始化客户端，client 在使用完毕后需要调用该工具类的 close() 方法。
  */
object ESRestClientUtils {
  private val log = LoggerFactory.getLogger(ESRestClientUtils.getClass.getSimpleName)

  private var restClient: Option[RestClient] = None
  private val SCHEMA: ElasticsearchHostsSniffer.Scheme = ElasticsearchHostsSniffer.Scheme.HTTP
  private var hostAndPorts: String = _
  private var sniffer: Option[Sniffer] = None

  /**
    * obtain elasticsearch high level rest client.
    * @return HighLevelRestClient
    */
  def getHighLevelRestClient: Option[RestHighLevelClient] = {
    if (restClient.isDefined && ping() == 200) {
      Some(new RestHighLevelClient(restClient.get))
    } else {
      if (StringUtils.isNotEmpty(hostAndPorts)) {
        initialize(hostAndPorts)
        if (null != restClient && ping() == 200) {
          Some(new RestHighLevelClient(restClient.get))
        } else {
          log.info("can not obtain useful ESRestClient, please have a check for ip or network status.")
          None
        }
      } else {
        log.info("the connect param is enpty or null!")
        None
      }
    }
  }

  /**
    * test server network state.
    *
    * @return status code.
    */
  private def ping(): Int = {
    val params: Map[String, String] = Map("pretty" -> "true")
    var code = 0
    try {
      if (restClient.isDefined) {
        val response: Response = restClient.get.performRequest("GET", "/", params)
        code = response.getStatusLine.getStatusCode
        log.info("code:{},elasticsearch connected success!", code)
      } else {
        log.info("the rest client is null!")
      }
    } catch {
      case ex: Exception =>
        log.info("elasticsearch connect error：{}。", ex.getLocalizedMessage)
        ex.printStackTrace()
    }
    code
  }

  /**
    * close elasticsearch rest client.
    */
  def close(): Unit = {
    try {
      if (sniffer.isDefined) {
        sniffer.get.close()
        log.info("ESRestClient sniffer closed。。。。。。。。。")
      }
      if (restClient.isDefined) {
        restClient.get.close()
        log.info("ESRestClient closed。。。。。。。。。")
      }
    } catch {
      case ex: Exception => ex.printStackTrace()
    }
  }

  /**
    * initialize logic.
    */
  def initialize(firstInputAddressOfCluster: String): Unit = {
    if (restClient.isEmpty) {
      create(firstInputAddressOfCluster)
    } else {
      if (ping() != 200) {
        close()
        create(firstInputAddressOfCluster)
      } else {
        log.info("the client is OK please dong't initialize repeat!")
      }
    }
  }

  /**
    * initialize elasticsearch rest client.
    */
  private def create(firstInputAddressOfCluster: String): Unit = {
    hostAndPorts = firstInputAddressOfCluster
    if (StringUtils.isNotEmpty(hostAndPorts)) {
      val splites = hostAndPorts.split(",")
      val hosts = new Array[HttpHost](splites.length)

      for (i <- 0 until splites.length) {
        val host = splites(i).split(":")(0).trim()
        val port = Integer.parseInt(splites(i).split(":")(1).trim())
        hosts(i) = new HttpHost(host, port, SCHEMA.toString)
      }
      try {
        val builder: RestClientBuilder = RestClient.builder(hosts: _*)
        builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
          @Override
          def customizeRequestConfig(requestConfigBuilder: RequestConfig.Builder): RequestConfig.Builder = {
            requestConfigBuilder.setConnectTimeout(1000 * 10)
            requestConfigBuilder.setSocketTimeout(1000 * 60)
            requestConfigBuilder.setConnectionRequestTimeout(1000 * 5)
            requestConfigBuilder
          }
        })
        builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
          @Override
          def customizeHttpClient(httpClientBuilder: HttpAsyncClientBuilder): HttpAsyncClientBuilder = {
            try {
              val ioReactorConfig: IOReactorConfig = IOReactorConfig.custom()
                .setConnectTimeout(1000 * 20)
                .setSoTimeout(1000 * 60 * 2)
                .setRcvBufSize(4096)
                .setSndBufSize(8192)
                .setSoKeepAlive(true)
                .build()
              val connManager: PoolingNHttpClientConnectionManager = new PoolingNHttpClientConnectionManager(new
                  DefaultConnectingIOReactor(ioReactorConfig))
              connManager.setMaxTotal(100); //多线程访问时最大并发量
              connManager.setDefaultMaxPerRoute(20); //单次路由线程上限

              httpClientBuilder.disableAuthCaching()
              httpClientBuilder.setDefaultIOReactorConfig(ioReactorConfig)
              httpClientBuilder.setConnectionManager(connManager);
            } catch {
              case ex: IOReactorException => ex.printStackTrace()
              case _: Exception =>
            }
            httpClientBuilder
          }
        })

        //star cluster sniffe function, it will be ok for provide partial IP address of es cluster.
        val sniffOnFailureListener: SniffOnFailureListener = new SniffOnFailureListener()

        builder.setFailureListener(sniffOnFailureListener)
        builder.setMaxRetryTimeoutMillis(1000 * 60 * 5); //the time(ms) of max retry timeout
        restClient = Some(builder.build())

        if (restClient.isDefined) {
          val hostsSniffer: HostsSniffer = new ElasticsearchHostsSniffer(
            restClient.get,
            ElasticsearchHostsSniffer.DEFAULT_SNIFF_REQUEST_TIMEOUT * 5,
            ElasticsearchHostsSniffer.Scheme.HTTP)

          sniffer = Some(Sniffer.builder(restClient.get)
            .setHostsSniffer(hostsSniffer)
            .setSniffIntervalMillis(1000 * 60 * 3)
            .setSniffAfterFailureDelayMillis(1000 * 10)
            .build())

          if (sniffer.isDefined) {
            sniffOnFailureListener.setSniffer(sniffer.get)
          }
        }
        log.info("ESRestClient already initialized。。。。。。。。。");
      } catch {
        case ex: Exception =>
          log.info("ESRestClient initialize error: {}. Please have a check for ip or network status.", ex.getMessage)
          ex.printStackTrace();
      }
    }
  }

  /**
    * es bulk index api
    *
    * @param client      RestHighLevelClient
    * @param indexName   index
    * @param typeName    type
    * @param batch       batch data
    * @param sizeOfBatch the size of every batch
    * @return failure doc collect
    */
  def canRollbackBulkDocIndex(client: RestHighLevelClient, indexName: String, typeName: String, batch: List[String], sizeOfBatch: Int = 10): List[(String, String)] = {
    val failure = new ListBuffer[(String, String)]()
    var batchSize = sizeOfBatch
    if (sizeOfBatch <= 0 || sizeOfBatch > 1000) {
      batchSize = 1000
    }
    val batchIter: Iterator[String] = batch.iterator
    var request: BulkRequest = new BulkRequest()
    var cursor = 0
    var count = 0
    try {
      while (batchIter.hasNext) {
        val json = batchIter.next()
        if (StringUtils.isNotEmpty(json)) {
          if (cursor < batchSize) {
            request.add(new IndexRequest(indexName, typeName).create(false).source(json, XContentType.JSON))
          } else {
            everyBatchWrapper(client, request, failure)
            cursor = 0
            request = new BulkRequest()
            request.add(new IndexRequest(indexName, typeName).create(false).source(json, XContentType.JSON))
          }
          cursor += 1
        } else {
          log.info("index of {} is null or enpty.", count)
        }
        count += 1
      }
      everyBatchWrapper(client, request, failure);
    } catch {
      case e: Exception =>
        log.info("the failed json element index range is {} - {}, Cause by: {}.", count - cursor, count)
        log.info("Cause by: {}.", e.getMessage)
        e.printStackTrace();
    }
    if (failure.size() > 0) {
      log.info("the number of failed doc index about this target index collention is {}.", failure.size())
    }
    failure.toList
  }

  /**
    * every batch operation
    *
    * @param client  RestHighLevelClient
    * @param request BulkRequest
    * @param failure failure collection
    */
  private def everyBatchWrapper(client: RestHighLevelClient, request: BulkRequest, failure: ListBuffer[(String, String)]): Unit = {
    val response: BulkResponse = client.bulk(request)
    if (response.hasFailures) {
      val list: mutable.Buffer[DocWriteRequest[_]] = request.requests().asScala
      val iter: Iterator[BulkItemResponse] = response.iterator()
      while (iter.hasNext) {
        val bir: BulkItemResponse = iter.next()
        if (bir.isFailed) {
          val reason = s"""HTTP status: [${bir.getFailure.getStatus.getStatus}], Cause by: ${bir.getFailureMessage}."""
          val docIndex = bir.getItemId
          val ir: IndexRequest = list.get(docIndex).asInstanceOf[IndexRequest]
          failure.+=((ir.source().utf8ToString(), reason))
        }
      }
    }
  }
}