mongodb数据库同步到elasticsearch中

由于公司需要，搭建elasticsearch检索服务但是同步mongodb数据是个问题，在百度资料找到使用mongo-connector同步，经过自己测试这款工具是python写的，使用时不稳定，会中断同步问题，即使加上守护线程同步并没有达到预期效果，当然这可能是我的测试环境的问题~~~

所以自己实现一个实时同步的工具，有两个月的测试记录了，完美同步数据~~~

测试的版本mongodb3.x，elasticsearch5.x

package io.realtime.mongo_els5;

import java.io.FileInputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.util.Arrays;
import java.util.Date;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import org.elasticsearch.client.Client;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.kohsuke.args4j.CmdLineException;
import org.kohsuke.args4j.CmdLineParser;
import org.kohsuke.args4j.Option;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.mongodb.MongoClient;
import com.mongodb.MongoCredential;
import com.mongodb.ServerAddress;


public class MongoElsRetm {

	private static Logger LOGGER = LoggerFactory.getLogger(MongoElsRetm.class);

	@Option(name = "-m", usage = "Processing mode", required = true, metaVar = "<bulk | realtime>")
	private String mode;

	@Option(name = "-c", usage = "Configuration file", required = true, metaVar = "<file name>")
	private String configFileName;

	private MongoClient mongoClient;

	private Client elsClient;

	public static void main(final String[] args) {
		new MongoElsRetm().doMain(args);
	}

	private void doMain(final String[] args) {
		final CmdLineParser parser = new CmdLineParser(this);

		try {
			parser.parseArgument(args);
			final Config cfg = Config.get();
			cfg.load(new FileInputStream(configFileName));

			LOGGER.info("Config : {}", cfg);

			ConnectMongo();
			ConnectEls();

			LOGGER.info("Start : {}", new Date());

			if ("realtime".equalsIgnoreCase(mode)) {
				final RealtimeIndexing indexing = new RealtimeIndexing(elsClient);
				final OplogTailer tailer = new OplogTailer(mongoClient);
				tailer.addListener(indexing);
				final ExecutorService exec = Executors.newSingleThreadExecutor();
				exec.submit(tailer);
			} else {
				System.err.println("Bad mode : " + mode);
				parser.printUsage(System.err);
			}

		} catch (final CmdLineException e) {
			System.err.println(e.getMessage());
			parser.printUsage(System.err);
			LOGGER.error(System.err+"\r\n"+e.getClass(), e.getMessage());

		} catch (final Exception e) {
			LOGGER.error("Problem during processing", e.getMessage());
		}
		LOGGER.info("End : {}", new Date());

	}

	private void ConnectMongo() {
		final Config cfg = Config.get();
		// 用户名 数据库 密码
		MongoCredential credential = MongoCredential.createCredential(cfg.get(Config.MONGO_USER), "admin",
				cfg.get(Config.MONGO_PWD).toCharArray());
		// IP port
		ServerAddress addr = new ServerAddress(cfg.get(Config.MONGO_HOST),
				Integer.parseInt(cfg.get(Config.MONGO_PORT)));
		this.mongoClient = new MongoClient(addr, Arrays.asList(credential));
	}

	@SuppressWarnings("resource")
	private void ConnectEls() throws IOException {
		final Config cfg = Config.get();
		final String host = cfg.get(Config.ELS_HOST);
		final String cluster_name = cfg.get(Config.ELS_CLUSTERNAME);
		final int port = cfg.getInt(Config.ELS_PORT);
		final Settings settings = Settings.builder().put("cluster.name", cluster_name).build();
		this.elsClient = new PreBuiltTransportClient(settings)
				.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(host), port));
	}

}

package io.realtime.mongo_els5;

import org.bson.Document;

public interface OplogListener {
    
    void onOplog(Document doc);
}

package io.realtime.mongo_els5;

import com.mongodb.CursorType;
import com.mongodb.MongoClient;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoCursor;
import org.bson.Document;

import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.Callable;

public class OplogTailer implements Callable<Void>{

	private MongoClient client;
	private final MongoCollection<Document> oplog;
	private final String elsIndex;
	private final String elsType;
	private final List<OplogListener> listeners = new LinkedList<>();
	private final Config cfg = Config.get();

	public OplogTailer(final MongoClient client) {
		this.client = client;
		this.elsType = cfg.get(Config.MONGO_COLLECTION);
		this.elsIndex = cfg.get(Config.MONGO_DB);
		this.oplog = this.client.getDatabase("local").getCollection("oplog.rs");
	}

	public void addListener(RealtimeIndexing listener) {
		listeners.add(listener);
	}

	@Override
	public Void call() throws Exception {
		/*final Date now = new Date();
		final Document query = new Document("ns", ns).append("ts",
				new Document("$gt", new BSONTimestamp((int) (now.getTime() / 1000), 0)));
*/
		final MongoCursor<Document> cursor = oplog.find().cursorType(CursorType.TailableAwait).iterator();

		while (cursor.hasNext()) {
			final Document doc = cursor.next();
			String index = doc.getString("ns").split("[.]")[0];
			if(index.equals(elsIndex))
			for (final OplogListener listener : listeners) {
				listener.onOplog(doc);
			}
		}

		return null;
	}
}

未待完结。。。