ESBulkManager批量工具类

9300方式
ESBulkManager批量工具类

符合条件（100条、1000条自己设置）自动提交

工具类

package com.yb.esBulk;

import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.elasticsearch.action.bulk.BulkProcessor;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;

public class ESBulkManager {
	static final Logger logger = LogManager.getLogger("ESBulkManager");

	Client client = null;
	BulkProcessor bulkProcessor = null;

	/**
	 * 依据HBase表名进行初始化工作
	 * 
	 * @param tableName
	 */
	public synchronized void init() {
		initESClient();
		initESBulkProcessor();
	}

	/**
	 * 初始化ES客户端
	 */
	private synchronized void initESClient() {
		logger.info("开始初始化Elastic Search客户端");
		String hosts = "127.0.0.1,9300";
		String clusterName = "localhost";

		// 集群名称
		Settings settings = ImmutableSettings.settingsBuilder().put("cluster.name", clusterName).build();
		TransportClient tc = new TransportClient(settings);

		String[] clients = StringUtils.split(hosts, ";");
		for (String client : clients) {
			tc.addTransportAddress(new InetSocketTransportAddress(client.substring(0, client.indexOf(",")),
					Integer.parseInt(client.substring(client.indexOf(",") + 1).trim())));
		}
		client = tc;
	}

	/**
	 * 初始化ES批量处理器
	 */
	private synchronized void initESBulkProcessor() {
		logger.info("开始初始化批量操作处理器 BulkProcessor:bulkAsync");
		BulkProcessor.Listener bulkListener = new BulkProcessor.Listener() {
			public void beforeBulk(long executionId, BulkRequest request) {
				int numberOfActions = request.numberOfActions();
				logger.debug("executionId=" + executionId + ", numberOfActions=" + numberOfActions);
			}

			public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
				// System.out.println("after bulk - - - -- -todo something");
			}

			public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
				logger.error("BulkProcessor.Listener.afterBulk :: " + executionId);
				logger.error(failure);
			}
		};

		/*
		 * setBulkActions(1000):每添加1000个request，执行一次bulk操作 setBulkSize(new
		 * ByteSizeValue(5, ByteSizeUnit.MB)):每达到5M的请求size时，执行一次bulk操作
		 * setFlushInterval(TimeValue.timeValueSeconds(5)):每5s执行一次bulk操作
		 * setConcurrentRequests(1):默认是1，表示积累bulk
		 * requests和发送bulk是异步的，其数值表示发送bulk的并发线程数，设置为0表示二者同步的
		 * setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.
		 * timeValueMillis(100),3)):当ES由于资源不足发生异常
		 * EsRejectedExecutionException重試策略：默认（50ms, 8）, 策略算法：start + 10 *
		 * ((int) Math.exp(0.8d * (currentlyConsumed)) - 1)
		 */
		BulkProcessor.Builder builder = BulkProcessor.builder(client, bulkListener);
		builder.setBulkActions(1000);
		builder.setBulkSize(new ByteSizeValue(5L, ByteSizeUnit.MB));
		builder.setConcurrentRequests(1);
		builder.setFlushInterval(TimeValue.timeValueSeconds(10L));
		bulkProcessor = builder.build();
		logger.info("BulkProcessor:bulkAsync 初始化完毕");
	}

	/**
	 * 新增
	 * 
	 * @param ir
	 */
	public void addRequest(IndexRequest ir) {
		bulkProcessor.add(ir);
	}

	/**
	 * 更新
	 * 
	 * @param ur
	 */
	public void addRequest(UpdateRequest ur) {
		bulkProcessor.add(ur);
	}

	/**
	 * 删除
	 * 
	 * @param dr
	 */
	public void addRequest(DeleteRequest dr) {
		bulkProcessor.add(dr);
	}

	/**
	 * 关闭批量操作，关闭客户端
	 */
	public synchronized void closeBulk() {
		logger.info("开始开始关闭  ES BulkProcessor");
		try {
			if (bulkProcessor != null)
				bulkProcessor.close();
		} catch (Exception e) {
			logger.error("ESManager.closeClient:bulkProcessor", e);
		} finally {
			try {
				if (client != null)
					client.close();
			} catch (Exception e) {
				logger.error("ESManager.closeClient:client", e);
			}
		}
	}

}

测试类

package com.yb.esBulk;

import org.elasticsearch.action.index.IndexRequest;

import com.alibaba.fastjson.JSONObject;

 
public class TestEs {

	public static void main(String[] args) {
		new TestEs().doWork();
	}

	private void doWork() {
		ESBulkManager esBulk = new ESBulkManager();
		esBulk.init();
		int ic=0;
		for (int i = 6000; i < 20000; i++) {
			JSONObject json = new JSONObject();
			json.put("dataType", "test");
			json.put("user", "u"+i);
			IndexRequest request = new IndexRequest("test", "test", String.valueOf(i)).source(json.toString());
			ic++;
			if (ic==999||ic==1999||ic==2999||ic==3999) {
				System.out.println();
			}
			esBulk.addRequest(request);
		}

	}

}