package com.company.listener;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.mongodb.BasicDBObject;
import com.mongodb.client.MongoCursor;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.Aggregates;
import com.mongodb.client.model.Filters;
import com.mongodb.client.model.changestream.ChangeStreamDocument;
import com.yunsom.sink.KakfaSink;
import com.yunsom.utils.MongoDbUtil;
import org.bson.Document;
import org.bson.conversions.Bson;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
import static java.util.Collections.singletonList;
public class MongoDbListener {
private volatile MongoDatabase database;
private String DBHOST;
private String DBUSER;
private String DBPWD;
private String DBNAME;
private String DBPORT;
private volatile MongoDbUtil mongoDbUtil;
public MongoDbListener(String DBHOST, String DBPORT, String DBUSER, String DBPWD, String DBNAME) {
this.DBHOST = DBHOST;
this.DBUSER = DBUSER;
this.DBPWD = DBPWD;
this.DBNAME = DBNAME;
this.DBPORT = DBPORT;
mongoDbUtil = MongoDbUtil.getInstance(DBHOST, DBPORT, DBUSER, DBPWD, DBNAME);
database = mongoDbUtil.getDataBase();
}
public void watchDB(){
List<Bson> pipeline = singletonList(Aggregates.match(Filters.or(
Document.parse("{'fullDocument.username': 'process_engine'}"),
Filters.in("operationType", Arrays.asList("insert", "update", "delete")))));
MongoCursor<ChangeStreamDocument<Document>> cursor = database.getCollection("tablename").watch(pipeline).iterator();
JsonParser jsonParser = new JsonParser();
String message = null;
while (cursor.hasNext()){
ChangeStreamDocument<Document> next = cursor.next();
String Operation = next.getOperationType().getValue();
String tableName = next.getNamespace().getCollectionName();
//获取主键id的值
assert next.getDocumentKey() != null;
String pk_id = jsonParser.parse(next.getDocumentKey().toJson()).getAsJsonObject().get("_id").getAsString();
//同步修改数据的操作
if (next.getUpdateDescription() != null) {
assert next.getUpdateDescription().getUpdatedFields() != null;
JsonObject jsonObject = jsonParser.parse(next.getUpdateDescription().getUpdatedFields().toJson()).getAsJsonObject();
message = parseJson(jsonObject, tableName, pk_id, Operation);
}
//同步插入数据的操作
if (next.getFullDocument() != null) {
JsonObject jsonObject = jsonParser.parse(next.getFullDocument().toJson()).getAsJsonObject();
message = parseJson(jsonObject, tableName, pk_id, Operation);
}
//同步删除数据的操作
if (next.getUpdateDescription() == null && Operation.matches("delete")) {
JsonObject jsonObject = jsonParser.parse(pk_id).getAsJsonObject();
message = parseJson(jsonObject, tableName, pk_id, Operation);
}
KakfaSink.sinkToKafka("topic",message);
// System.out.println(message);
}
}
private String parseJson(JsonObject object, String tableName, String pkid, String Operation) {
String eventType = object.toString(); //更改的事件名称,具体的事件
StringBuilder sb = new StringBuilder();
/**
* 将主键Id插入到pk_Id中去
*/
if (Operation.matches("insert")) {
sb.append(eventType);
} else if (Operation.matches("update")) {
BasicDBObject param = new BasicDBObject();
param.put("_id", pkid);
MongoCursor<Document> iterator = mongoDbUtil.findByquery(tableName, param).iterator();
if (iterator.hasNext()){
sb.append(iterator.next().toJson());
}
} else if (Operation.matches("delete")) {
}
return sb.toString();
}
public static void main(String[] args) {
Properties properties = new Properties();
try {
properties.load(MongoDbListener.class.getClassLoader().getResourceAsStream("env.properties"));
} catch (IOException e) {
e.printStackTrace();
}
String MONGO_DBHOST = properties.getProperty("MONGO_DBHOST");
String MONGO_DBPORT =properties.getProperty("MONGO_DBPORT");
String MONGO_DBUSER =properties.getProperty("MONGO_DBUSER");
String MONGO_DBPWD =properties.getProperty("MONGO_DBPWD");
String MONGO_DBNAME =properties.getProperty("MONGO_DBNAME");
MongoDbListener listener = new MongoDbListener(MONGO_DBHOST, MONGO_DBPORT, MONGO_DBUSER, MONGO_DBPWD, MONGO_DBNAME);
listener.watchDB();
}
}
依赖版本如下
<dependencies>
<!-- https://mvnrepository.com/artifact/org.mongodb/mongo-java-driver -->
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
<version>3.12.2</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>2.3.0</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.2.4</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>org.apache.spark</groupId>-->
<!-- <artifactId>spark-streaming-kafka-0-8-assembly_2.11</artifactId>-->
<!-- <version>2.4.6</version>-->
<!-- </dependency>-->
</dependencies>