使用canal+rabbitmq实现数据实时同步

使用canal+rabbitmq实现数据实时同步

本文使用canal加rabbtmq实现同步交易数据，这里只贴出客户端实现代码，canal服务端部署篇后期会更新

package com.canal;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
import org.springframework.scheduling.annotation.EnableScheduling;


/**
 * @description:
 * @author: kevin
 * @time: 2020/12/10 18:23
 */
@SpringBootApplication
@EnableDiscoveryClient
@EnableScheduling
public class CanalApp {

    public static void main(String[] args) {
        SpringApplication.run(com.canal.CanalApp.class, args);
    }
    }

这里采用的canal版本1.1.4，版本要与canal服务端采用一致


     <dependency>
            <groupId>com.alibaba.otter</groupId>
            <artifactId>canal.client</artifactId>
            <version>1.1.4</version>
        </dependency>
        <dependency>
            <groupId>com.alibaba.otter</groupId>
            <artifactId>canal.common</artifactId>
            <version>1.1.4</version>
        </dependency>
        <dependency>
            <groupId>com.alibaba.otter</groupId>
            <artifactId>canal.protocol</artifactId>
            <version>1.1.4</version>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-amqp</artifactId>
        </dependency>

canal客户端连接服务端代码，这里业务需求是只需要订阅channel库中的两张表，那么同步数据只会对订阅的两张表的binlog日志做同步

package com.canal.config;
import com.alibaba.otter.canal.client.CanalConnector;
import com.alibaba.otter.canal.client.CanalConnectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;
import java.net.InetSocketAddress;

/**
 * @description:
 * @author: kevin
 * @time: 2020/12/11 17:51
 */
@Component
@Slf4j
public class CanalConfig implements DisposableBean {

    private CanalConnector canalConnector;


    @Bean
    public CanalConnector getCanalConnector() {
       //这个是单机版本写法
       // canalConnector = CanalConnectors.newSingleConnector(new //InetSocketAddress("8.129.108.000",
//                11111), "aaa", "canal", "canal");
      // 这个是集群连zk版本
        canalConnector = CanalConnectors.newClusterConnector("ip:2181,ip:2181,ip:2181", "aaa", "canal", "canal");

        try {
            canalConnector.connect();
            canalConnector.subscribe("canal\\\\..*,channel.channel_deposit_order,channel.pos_channel_order");
        } catch (Throwable t) {
            log.error("failed to connect to canal server", t);
            canalConnector.disconnect();
            throw t;
        }

        canalConnector.rollback();
        log.info("canal connect success");
        return canalConnector;
    }

    @Override
    public void destroy() {
        if (canalConnector != null) {
            canalConnector.disconnect();
        }
    }


}

canal客户端定时从canal服务端的mq中获取binlog日志信息(包含insert，update)，提取表字段数据，并对数据做清洗放入rabbitmq

package com.canal.config;
import cn.hutool.core.collection.CollectionUtil;
import com.alibaba.otter.canal.client.CanalConnector;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.Message;
import com.google.protobuf.InvalidProtocolBufferException;
import com.sdmgr.canal.constant.Constant;
import com.sdmgr.canal.mq.Produce;
import com.sdmgr.common.core.exception.GlobalException;
import com.sdmgr.common.core.result.ResultEnum;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * @description:
 * @author: kevin
 * @time: 2020/12/11 17:47
 */
@Component
@Slf4j
public class CanalJob {


    @Autowired
    private CanalConnector canalConnector;

    @Autowired
    private Produce produce;


    @Scheduled(fixedDelay = 1000)
    public void handle() {

        long batchId = -1;
        try {
            int batchSize = 1000;
            Message message = canalConnector.getWithoutAck(batchSize);
            batchId = message.getId();
            List<CanalEntry.Entry> entries = message.getEntries();
            if (batchId != -1 && entries.size() > 0) {
                entries.forEach(entry -> {
                    if (entry.getEntryType() == CanalEntry.EntryType.ROWDATA) {
                        String name = entry.getHeader().getTableName();
                        CanalEntry.RowChange change = null;
                        try {
                            change = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
                        } catch (InvalidProtocolBufferException e) {
                            log.error("parseFrom error ", e);
                            return;
                        }
                        change.getRowDatasList().forEach(rowData -> {
                            List<CanalEntry.Column> columns = rowData.getAfterColumnsList();
                            try {

                                Map<String, Object> jsonMap = new HashMap<>();
                                if (name.equals(Constant.SURFACE_ONE)) {
                                    columns.forEach(column -> {
                                        if (column == null) {
                                            return;
                                        }
                                        if (column.getName().equals("mer_no")) {
                                            jsonMap.put(column.getName(), column.getValue());
                                        }
                                        if (column.getName().equals("term_no")) {
                                            jsonMap.put(column.getName(), column.getValue());
                                        }
                                        if (column.getName().equals("amount")) {
                                            jsonMap.put(column.getName(), column.getValue());
                                        }
                                        if (column.getName().equals("fee")) {
                                            jsonMap.put(column.getName(), column.getValue());
                                        }
                                        if (column.getName().equals("tradeStatus")) {
                                            jsonMap.put(column.getName(), column.getValue());
                                        }
                                        if (column.getName().equals("create_time")) {
                                            jsonMap.put(column.getName(), column.getValue());
                                        }
                                        if (column.getName().equals("finish_time")) {
                                            jsonMap.put(column.getName(), column.getValue());
                                        }
                                        if (column.getName().equals("settle_merno")) {
                                            jsonMap.put(column.getName(), column.getValue());
                                        }
                                        if (column.getName().equals("settle_term")) {
                                            jsonMap.put(column.getName(), column.getValue());
                                        }
                                        if (column.getName().equals("transaction_id")) {
                                            jsonMap.put(column.getName(), column.getValue());
                                        }
                                        if (column.getName().equals("trade_type")) {
                                            jsonMap.put(column.getName(), column.getValue());
                                        }
                                        if (column.getName().equals("settle_flag")) {
                                            jsonMap.put(column.getName(), column.getValue());
                                        }
                                        if (column.getName().equals("d0_flag")) {
                                            jsonMap.put(column.getName(), column.getValue());
                                        }
                                        if (column.getName().equals("card_Code")) {
                                            jsonMap.put(column.getName(), column.getValue());
                                        }
                                        if (column.getName().equals("sn")) {
                                            jsonMap.put(column.getName(), column.getValue());
                                        }
                                        if (column.getName().equals("card_type")) {
                                            jsonMap.put(column.getName(), column.getValue());
                                        }
                                        if (column.getName().equals("trans_categary")) {
                                            jsonMap.put(column.getName(), column.getValue());
                                        }
                                        if (column.getName().equals("charge_fee")) {
                                            jsonMap.put(column.getName(), column.getValue());
                                        }
                                        if (column.getName().equals("order_id")) {
                                            jsonMap.put(column.getName(), column.getValue());
                                        }
                                    });
                                    if (CollectionUtil.isEmpty(jsonMap)) {
                                        throw new GlobalException(ResultEnum.DATA_HAVE_EMPTY);
                                    }
                                    produce.sendMessage1(jsonMap);
                                    log.info("发送消息111成功");

                                } else {
                                    columns.forEach(column -> {
                                        if (column == null) {
                                            return;
                                        }
                                        jsonMap.put(column.getName(), column.getValue());
                                    });
                                    if (CollectionUtil.isEmpty(jsonMap)) {
                                        throw new GlobalException(ResultEnum.DATA_HAVE_EMPTY);
                                    }
                                    produce.sendMessage2(jsonMap);
                                    log.info("发送消息222成功");

                                }

                            } catch (Exception e) {
                                log.error("parseFrom error ", e);
                            }

                        });
                    }
                });
            }
            canalConnector.ack(batchId);
        } catch (Exception e) {
            log.error("data parse error ", e);
            canalConnector.rollback(batchId);
        }

    }


}

配置rabbitmq的topic模式配置类

package com.canal.config;
import com.sdmgr.canal.constant.Constant;
import org.springframework.amqp.core.Binding;
import org.springframework.amqp.core.BindingBuilder;
import org.springframework.amqp.core.Queue;
import org.springframework.amqp.core.TopicExchange;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;

/**
 * @description:
 * @author: kevin
 * @time: 2020/12/10 17:51
 */
@Component
public class TopicConfig {


    @Bean
    public Queue topicOpenQueue() {
        return new Queue(Constant.TOPIC_CANAL_ORDER_QUEUE);
    }

    @Bean
    public Queue topicFlumeQueue() {
        return new Queue(Constant.TOPIC_CANAL_FREZE_QUEUE);
    }


    @Bean
    TopicExchange topicExchange() {
        return new TopicExchange(Constant.EXCHANGE_NAME);
    }


    @Bean
    Binding bindingExchangeOpen(Queue topicOpenQueue, TopicExchange topicExchange) {
        return BindingBuilder.bind(topicOpenQueue).to(topicExchange).with(Constant.CANAL_ROUTING_KEY);

    }

    @Bean
    Binding bindingExchangeFlume(Queue topicFlumeQueue, TopicExchange topicExchange) {
        return BindingBuilder.bind(topicFlumeQueue).to(topicExchange).with(Constant.CANAL_FLUME_ROUTING_KEY);

    }


}

package com.canal.constant;

/**
 * @description:
 * @author: kevin
 * @time: 2020/12/11 15:02
 */
public class Constant {

    public final static String TOPIC_CANAL_ORDER_QUEUE = "sd.order.queue";

    public final static String EXCHANGE_NAME = "sd.order.exchange";

    public final static String CANAL_ROUTING_KEY = "sd.order.trans";

    public final static String TOPIC_CANAL_FREZE_QUEUE = "sd.flume.queue";

    public final static String CANAL_FLUME_ROUTING_KEY = "sd.flume.trans";

    public static String SURFACE_ONE = "pos_channel_order";


}

mq消息生产者的代码，将清洗后的数据放入mq中

package com.canal.mq;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.sdmgr.canal.constant.Constant;
import lombok.extern.slf4j.Slf4j;
import org.springframework.amqp.core.AmqpTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.Map;

/**
 * @description:
 * @author: kevin
 * @time: 2020/12/11 16:30
 */
@Component
@Slf4j
public class Produce {


    @Autowired
    private AmqpTemplate amqpTemplate;



    /**
     * @description: 发送交易订单
     */
    public void sendMessage1(Map<String,Object> data) throws JsonProcessingException {
        ObjectMapper objectMapper = new ObjectMapper();
        String result = objectMapper.writeValueAsString(data);
        amqpTemplate.convertAndSend(Constant.EXCHANGE_NAME, Constant.CANAL_ROUTING_KEY, result);
        log.info("发送订单成功 {}",result);

    }


    /**
     * @description: 发送附加订单
     */
    public void sendMessage2(Map<String,Object> data) throws JsonProcessingException {
        ObjectMapper objectMapper = new ObjectMapper();
        String result = objectMapper.writeValueAsString(data);
        amqpTemplate.convertAndSend(Constant.EXCHANGE_NAME, Constant.CANAL_FLUME_ROUTING_KEY, result);
        log.info("发送附加订单成功：{}",result);
    }

}