由于公司需要,搭建elasticsearch检索服务但是同步mongodb数据是个问题,在百度资料找到使用mongo-connector同步,经过自己测试这款工具是python写的,使用时不稳定,会中断同步问题,即使加上守护线程同步并没有达到预期效果,当然这可能是我的测试环境的问题~~~
所以自己实现一个实时同步的工具,有两个月的测试记录了,完美同步数据~~~
测试的版本mongodb3.x,elasticsearch5.x
package io.realtime.mongo_els5;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.util.Arrays;
import java.util.Date;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.kohsuke.args4j.CmdLineException;
import org.kohsuke.args4j.CmdLineParser;
import org.kohsuke.args4j.Option;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.mongodb.MongoClient;
import com.mongodb.MongoCredential;
import com.mongodb.ServerAddress;
public class MongoElsRetm {
private static Logger LOGGER = LoggerFactory.getLogger(MongoElsRetm.class);
@Option(name = "-m", usage = "Processing mode", required = true, metaVar = "<bulk | realtime>")
private String mode;
@Option(name = "-c", usage = "Configuration file", required = true, metaVar = "<file name>")
private String configFileName;
private MongoClient mongoClient;
private Client elsClient;
public static void main(final String[] args) {
new MongoElsRetm().doMain(args);
}
private void doMain(final String[] args) {
final CmdLineParser parser = new CmdLineParser(this);
try {
parser.parseArgument(args);
final Config cfg = Config.get();
cfg.load(new FileInputStream(configFileName));
LOGGER.info("Config : {}", cfg);
ConnectMongo();
ConnectEls();
LOGGER.info("Start : {}", new Date());
if ("realtime".equalsIgnoreCase(mode)) {
final RealtimeIndexing indexing = new RealtimeIndexing(elsClient);
final OplogTailer tailer = new OplogTailer(mongoClient);
tailer.addListener(indexing);
final ExecutorService exec = Executors.newSingleThreadExecutor();
exec.submit(tailer);
} else {
System.err.println("Bad mode : " + mode);
parser.printUsage(System.err);
}
} catch (final CmdLineException e) {
System.err.println(e.getMessage());
parser.printUsage(System.err);
LOGGER.error(System.err+"\r\n"+e.getClass(), e.getMessage());
} catch (final Exception e) {
LOGGER.error("Problem during processing", e.getMessage());
}
LOGGER.info("End : {}", new Date());
}
private void ConnectMongo() {
final Config cfg = Config.get();
// 用户名 数据库 密码
MongoCredential credential = MongoCredential.createCredential(cfg.get(Config.MONGO_USER), "admin",
cfg.get(Config.MONGO_PWD).toCharArray());
// IP port
ServerAddress addr = new ServerAddress(cfg.get(Config.MONGO_HOST),
Integer.parseInt(cfg.get(Config.MONGO_PORT)));
this.mongoClient = new MongoClient(addr, Arrays.asList(credential));
}
@SuppressWarnings("resource")
private void ConnectEls() throws IOException {
final Config cfg = Config.get();
final String host = cfg.get(Config.ELS_HOST);
final String cluster_name = cfg.get(Config.ELS_CLUSTERNAME);
final int port = cfg.getInt(Config.ELS_PORT);
final Settings settings = Settings.builder().put("cluster.name", cluster_name).build();
this.elsClient = new PreBuiltTransportClient(settings)
.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(host), port));
}
}
package io.realtime.mongo_els5;
import org.bson.Document;
public interface OplogListener {
void onOplog(Document doc);
}
package io.realtime.mongo_els5;
import com.mongodb.CursorType;
import com.mongodb.MongoClient;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoCursor;
import org.bson.Document;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.Callable;
public class OplogTailer implements Callable<Void>{
private MongoClient client;
private final MongoCollection<Document> oplog;
private final String elsIndex;
private final String elsType;
private final List<OplogListener> listeners = new LinkedList<>();
private final Config cfg = Config.get();
public OplogTailer(final MongoClient client) {
this.client = client;
this.elsType = cfg.get(Config.MONGO_COLLECTION);
this.elsIndex = cfg.get(Config.MONGO_DB);
this.oplog = this.client.getDatabase("local").getCollection("oplog.rs");
}
public void addListener(RealtimeIndexing listener) {
listeners.add(listener);
}
@Override
public Void call() throws Exception {
/*final Date now = new Date();
final Document query = new Document("ns", ns).append("ts",
new Document("$gt", new BSONTimestamp((int) (now.getTime() / 1000), 0)));
*/
final MongoCursor<Document> cursor = oplog.find().cursorType(CursorType.TailableAwait).iterator();
while (cursor.hasNext()) {
final Document doc = cursor.next();
String index = doc.getString("ns").split("[.]")[0];
if(index.equals(elsIndex))
for (final OplogListener listener : listeners) {
listener.onOplog(doc);
}
}
return null;
}
}
未待完结。。。