canal基本使用

1、pom.xml文件中加入jar
·

com.alibaba.otter
canal.client
1.1.2


2、创建CanalClient.java --用来解析binlog文件变化信息
·import com.alibaba.otter.canal.client.CanalConnector;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.CanalEntry.*;
import com.alibaba.otter.canal.protocol.Message;
import com.google.protobuf.InvalidProtocolBufferException;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.SystemUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import org.springframework.util.Assert;
import org.springframework.util.CollectionUtils;

import java.io.UnsupportedEncodingException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;

/**

  • 测试基类

  • @author jianghang 2013-4-15 下午04:17:12

  • @version 1.0.4
    */
    public class AbstractCanalClientTest {

    protected final static Logger logger = LoggerFactory.getLogger(AbstractCanalClientTest.class);
    protected static final String SEP = SystemUtils.LINE_SEPARATOR;
    protected static final String DATE_FORMAT = “yyyy-MM-dd HH:mm:ss”;
    protected volatile boolean running = false;
    protected Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {

                                                                  public void uncaughtException(Thread t, Throwable e) {
                                                                      logger.error("parse events has an error", e);
                                                                  }
                                                              };
    

    protected Thread thread = null;
    protected CanalConnector connector;
    protected static String context_format = null;
    protected static String row_format = null;
    protected static String transaction_format = null;
    protected String destination;

    static {
    context_format = SEP + “" + SEP;
    context_format += "
    Batch Id: [{}] ,count : [{}] , memsize : [{}] , Time : {}" + SEP;
    context_format += "
    Start : [{}] " + SEP;
    context_format += "
    End : [{}] " + SEP;
    context_format += "
    ***” + SEP;

     row_format = SEP
                  + "----------------> binlog[{}:{}] , name[{},{}] , eventType : {} , executeTime : {}({}) , gtid : ({}) , delay : {} ms"
                  + SEP;
    
     transaction_format = SEP
                          + "================> binlog[{}:{}] , executeTime : {}({}) , gtid : ({}) , delay : {}ms"
                          + SEP;
    

    }

    public AbstractCanalClientTest(String destination){
    this(destination, null);
    }

    public AbstractCanalClientTest(String destination, CanalConnector connector){
    this.destination = destination;
    this.connector = connector;
    }

    protected void start() {
    Assert.notNull(connector, “connector is null”);
    thread = new Thread(new Runnable() {

         public void run() {
             process();
         }
     });
    
     thread.setUncaughtExceptionHandler(handler);
     running = true;
     thread.start();
    

    }

    protected void stop() {
    if (!running) {
    return;
    }
    running = false;
    if (thread != null) {
    try {
    thread.join();
    } catch (InterruptedException e) {
    // ignore
    }
    }

     MDC.remove("destination");
    

    }

    protected void process() {
    int batchSize = 5 * 1024;
    while (running) {
    try {
    MDC.put(“destination”, destination);
    connector.connect();
    connector.subscribe();
    while (running) {
    Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据
    long batchId = message.getId();
    int size = message.getEntries().size();
    if (batchId == -1 || size == 0) {
    // try {
    // Thread.sleep(1000);
    // } catch (InterruptedException e) {
    // }
    } else {
    printSummary(message, batchId, size);
    printEntry(message.getEntries());
    }

                 connector.ack(batchId); // 提交确认
                 // connector.rollback(batchId); // 处理失败, 回滚数据
             }
         } catch (Exception e) {
             logger.error("process error!", e);
         } finally {
             connector.disconnect();
             MDC.remove("destination");
         }
     }
    

    }

    private void printSummary(Message message, long batchId, int size) {
    long memsize = 0;
    for (Entry entry : message.getEntries()) {
    memsize += entry.getHeader().getEventLength();
    }

     String startPosition = null;
     String endPosition = null;
     if (!CollectionUtils.isEmpty(message.getEntries())) {
         startPosition = buildPositionForDump(message.getEntries().get(0));
         endPosition = buildPositionForDump(message.getEntries().get(message.getEntries().size() - 1));
     }
    
     SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
     logger.info(context_format, new Object[] { batchId, size, memsize, format.format(new Date()), startPosition,
             endPosition });
    

    }

    protected String buildPositionForDump(Entry entry) {
    long time = entry.getHeader().getExecuteTime();
    Date date = new Date(time);
    SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
    String position = entry.getHeader().getLogfileName() + “:” + entry.getHeader().getLogfileOffset() + “:”
    + entry.getHeader().getExecuteTime() + “(” + format.format(date) + “)”;
    if (StringUtils.isNotEmpty(entry.getHeader().getGtid())) {
    position += " gtid(" + entry.getHeader().getGtid() + “)”;
    }
    return position;
    }

    protected void printEntry(List entrys) {
    for (Entry entry : entrys) {
    long executeTime = entry.getHeader().getExecuteTime();
    long delayTime = new Date().getTime() - executeTime;
    Date date = new Date(entry.getHeader().getExecuteTime());
    SimpleDateFormat simpleDateFormat = new SimpleDateFormat(“yyyy-MM-dd HH:mm:ss”);

         if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND) {
             if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN) {
                 TransactionBegin begin = null;
                 try {
                     begin = TransactionBegin.parseFrom(entry.getStoreValue());
                 } catch (InvalidProtocolBufferException e) {
                     throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
                 }
                 // 打印事务头信息,执行的线程id,事务耗时
                 logger.info(transaction_format,
                     new Object[] { entry.getHeader().getLogfileName(),
                             String.valueOf(entry.getHeader().getLogfileOffset()),
                             String.valueOf(entry.getHeader().getExecuteTime()), simpleDateFormat.format(date),
                             entry.getHeader().getGtid(), String.valueOf(delayTime) });
                 logger.info(" BEGIN ----> Thread id: {}", begin.getThreadId());
                 printXAInfo(begin.getPropsList());
             } else if (entry.getEntryType() == EntryType.TRANSACTIONEND) {
                 TransactionEnd end = null;
                 try {
                     end = TransactionEnd.parseFrom(entry.getStoreValue());
                 } catch (InvalidProtocolBufferException e) {
                     throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
                 }
                 // 打印事务提交信息,事务id
                 logger.info("----------------\n");
                 logger.info(" END ----> transaction id: {}", end.getTransactionId());
                 printXAInfo(end.getPropsList());
                 logger.info(transaction_format,
                     new Object[] { entry.getHeader().getLogfileName(),
                             String.valueOf(entry.getHeader().getLogfileOffset()),
                             String.valueOf(entry.getHeader().getExecuteTime()), simpleDateFormat.format(date),
                             entry.getHeader().getGtid(), String.valueOf(delayTime) });
             }
    
             continue;
         }
    
         if (entry.getEntryType() == EntryType.ROWDATA) {
             RowChange rowChage = null;
             try {
                 rowChage = RowChange.parseFrom(entry.getStoreValue());
             } catch (Exception e) {
                 throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
             }
    
             EventType eventType = rowChage.getEventType();
    
             logger.info(row_format,
                 new Object[] { entry.getHeader().getLogfileName(),
                         String.valueOf(entry.getHeader().getLogfileOffset()), entry.getHeader().getSchemaName(),
                         entry.getHeader().getTableName(), eventType,
                         String.valueOf(entry.getHeader().getExecuteTime()), simpleDateFormat.format(date),
                         entry.getHeader().getGtid(), String.valueOf(delayTime) });
    
             if (eventType == EventType.QUERY || rowChage.getIsDdl()) {
                 logger.info(" sql ----> " + rowChage.getSql() + SEP);
                 continue;
             }
    
             printXAInfo(rowChage.getPropsList());
             for (RowData rowData : rowChage.getRowDatasList()) {
                 if (eventType == EventType.DELETE) {
                     printColumn(rowData.getBeforeColumnsList());
                 } else if (eventType == EventType.INSERT) {
                     printColumn(rowData.getAfterColumnsList());
                 } else {
                     printColumn(rowData.getAfterColumnsList());
                 }
             }
         }
     }
    

    }

    protected void printColumn(List columns) {
    for (Column column : columns) {
    StringBuilder builder = new StringBuilder();
    try {
    if (StringUtils.containsIgnoreCase(column.getMysqlType(), “BLOB”)
    || StringUtils.containsIgnoreCase(column.getMysqlType(), “BINARY”)) {
    // get value bytes
    builder.append(column.getName() + " : "
    + new String(column.getValue().getBytes(“ISO-8859-1”), “UTF-8”));
    } else {
    builder.append(column.getName() + " : " + column.getValue());
    }
    } catch (UnsupportedEncodingException e) {
    }
    builder.append(" type=" + column.getMysqlType());
    if (column.getUpdated()) {
    builder.append(" update=" + column.getUpdated());
    }
    builder.append(SEP);
    logger.info(builder.toString());
    }
    }

    protected void printXAInfo(List pairs) {
    if (pairs == null) {
    return;
    }

     String xaType = null;
     String xaXid = null;
     for (Pair pair : pairs) {
         String key = pair.getKey();
         if (StringUtils.endsWithIgnoreCase(key, "XA_TYPE")) {
             xaType = pair.getValue();
         } else if (StringUtils.endsWithIgnoreCase(key, "XA_XID")) {
             xaXid = pair.getValue();
         }
     }
    
     if (xaType != null && xaXid != null) {
         logger.info(" ------> " + xaType + " " + xaXid);
     }
    

    }

    public void setConnector(CanalConnector connector) {
    this.connector = connector;
    }

    /**

    • 获取当前Entry的 GTID信息示例
    • @param header
    • @return
      */
      public static String getCurrentGtid(CanalEntry.Header header) {
      List props = header.getPropsList();
      if (props != null && props.size() > 0) {
      for (CanalEntry.Pair pair : props) {
      if (“curtGtid”.equals(pair.getKey())) {
      return pair.getValue();
      }
      }
      }
      return “”;
      }

    /**

    • 获取当前Entry的 GTID Sequence No信息示例
    • @param header
    • @return
      */
      public static String getCurrentGtidSn(CanalEntry.Header header) {
      List props = header.getPropsList();
      if (props != null && props.size() > 0) {
      for (CanalEntry.Pair pair : props) {
      if (“curtGtidSn”.equals(pair.getKey())) {
      return pair.getValue();
      }
      }
      }
      return “”;
      }

    /**

    • 获取当前Entry的 GTID Last Committed信息示例
    • @param header
    • @return
      */
      public static String getCurrentGtidLct(CanalEntry.Header header) {
      List props = header.getPropsList();
      if (props != null && props.size() > 0) {
      for (CanalEntry.Pair pair : props) {
      if (“curtGtidLct”.equals(pair.getKey())) {
      return pair.getValue();
      }
      }
      }
      return “”;
      }

}
3、创建连接
·import com.alibaba.otter.canal.client.CanalConnector;
import com.alibaba.otter.canal.client.CanalConnectors;

import java.net.InetSocketAddress;

/**

  • 单机模式的测试例子

  • @author jianghang 2013-4-15 下午04:19:20

  • @version 1.0.4
    */
    public class SimpleCanalClientTest extends AbstractCanalClientTest {

    public SimpleCanalClientTest(String destination){
    super(destination);
    }

    public static void main(String args[]) {
    // 根据ip,直接创建链接,无HA的功能
    String destination = “example”;
    CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress(“172.26.xx.xx”, 11111),
    destination,
    “”,
    “”);

     final SimpleCanalClientTest clientTest = new SimpleCanalClientTest(destination);
     clientTest.setConnector(connector);
     clientTest.start();
     Runtime.getRuntime().addShutdownHook(new Thread() {
    
         public void run() {
             try {
                 logger.info("## stop the canal client");
                 clientTest.stop();
             } catch (Throwable e) {
                 logger.warn("##something goes wrong when stopping canal:", e);
             } finally {
                 logger.info("## canal client is down.");
             }
         }
    
     });
    

    }

}

评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值