Spring Boot整合MongoDB监听oplog数据实时写入MySQL数据库

1. 环境准备

• Spring Boot 2.x
• MongoDB 4.x
• MySQL 5.x
• JDK 8+

2. 依赖引入

在 pom.xml 文件中添加以下依赖：

<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-data-mongodb</artifactId>
</dependency>

<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>

<dependency>
    <groupId>org.mongodb</groupId>
    <artifactId>mongodb-driver</artifactId>
    <version>3.12.7</version>
</dependency>

<dependency>
    <groupId>org.mongodb</groupId>
    <artifactId>mongodb-driver-sync</artifactId>
    <version>3.12.7</version>
</dependency>

3. 实现MongoDB的Oplog监听器

@Component
public class OplogListener implements ApplicationListener<ContextRefreshedEvent> {

    private MongoTemplate mongoTemplate;

    private EntityManager entityManager;

    @Autowired
    public OplogListener(MongoTemplate mongoTemplate, EntityManager entityManager) {
        this.mongoTemplate = mongoTemplate;
        this.entityManager = entityManager;
    }

    @Override
    public void onApplicationEvent(ContextRefreshedEvent event) {
        MongoDatabase db = mongoTemplate.getDb();
        MongoCollection<Document> oplog = db.getCollection("oplog.rs");

        BsonTimestamp startTS = getStartTimestamp();
        BsonTimestamp endTS = getEndTimestamp();

        Bson filter = Filters.and(Filters.gt("ts", startTS), Filters.lte("ts", endTS));

        MongoCursor<Document> cursor = oplog.find(filter).cursorType(CursorType.TailableAwait).iterator();

        while (true) {
            if (cursor.hasNext()) {
                Document doc = cursor.next();
                String operation = doc.getString("op");

                if (!"n".equals(operation)) {
                    String namespace = doc.getString("ns");
                    String[] nsParts = StringUtils.split(namespace, ".");
                    String collectionName = nsParts[1];
                    String databaseName = nsParts[0];

                    if ("i".equals(operation)) {
                        Document object = (Document) doc.get("o");
                        insert(object, databaseName, collectionName);
                    } else if ("u".equals(operation)) {
                        Document object = (Document) doc.get("o");
                        Document update = (Document) doc.get("o2");
                        update(object, update, databaseName, collectionName);
                    } else if ("d".equals(operation)) {
                        Document object = (Document) doc.get("o");
                        delete(object, databaseName, collectionName);
                    }
                }
            }
        }
    }

    private BsonTimestamp getStartTimestamp() {
        long currentSeconds = System.currentTimeMillis() / 1000;
        return new BsonTimestamp((int) currentSeconds, 1);
    }

    private BsonTimestamp getEndTimestamp() {
        return new BsonTimestamp(0, 1);
    }

    private void insert(Document object, String databaseName, String collectionName) {
        entityManager.getTransaction().begin();
        try {
            String json = JSON.serialize(object);
            Query query = entityManager.createNativeQuery("INSERT INTO " + collectionName + " (json) VALUES (:json)");
            query.setParameter("json", json);
            query.executeUpdate();
            entityManager.getTransaction().commit();
        } catch (Exception e) {
            entityManager.getTransaction().rollback();
            throw new RuntimeException(e);
        }
    }

    private void update(Document object, Document update, String databaseName, String collectionName) {
        entityManager.getTransaction().begin();
        try {
            String json = JSON.serialize(object);
            String updateJson = JSON.serialize(update);
            Query query = entityManager.createNativeQuery("UPDATE " + collectionName + " SET json = :json WHERE json = :updateJson");
            query.setParameter("json", json);
            query.setParameter("updateJson", updateJson);
            query.executeUpdate();
            entityManager.getTransaction().commit();
        } catch (Exception e) {
            entityManager.getTransaction().rollback();
            throw new RuntimeException(e);
        }
    }

    private void delete(Document object, String databaseName, String collectionName) {
        entityManager.getTransaction().begin();
        try {
            String json = JSON.serialize(object);
            Query query = entityManager.createNativeQuery("DELETE FROM " + collectionName + " WHERE json = :json");
            query.setParameter("json", json);
            query.executeUpdate();
            entityManager.getTransaction().commit();
        } catch (Exception e) {
            entityManager.getTransaction().rollback();
            throw new RuntimeException(e);
        }
    }
}

4. 配置文件

在 application.yml 文件中添加以下配置：

spring:
  data:
    mongodb:
      uri: mongodb://localhost:27017/test
  datasource:
    driver-class-name: com.mysql.jdbc.Driver
    url: jdbc:mysql://localhost:3306/test
    username: root
    password: root
  jpa:
    show-sql: true
    properties:
      hibernate:
        dialect: org.hibernate.dialect.MySQL5Dialect

5. 测试

在MongoDB中插入一条记录：

db.users.insert({
    "name" : "Tom",
    "age" : 20
})

在MySQL中查询该条记录：

SELECT * FROM users;

6. 总结

本文介绍了如何使用Spring Boot整合MongoDB监听oplog数据实时写入MySQL数据库，实现了MongoDB和MySQL之间的数据同步。