1,找到TiDB的官方文档中binlog的说明:
https://docs.pingcap.com/zh/tidb/stable/binlog-slave-client#binlog-slave-client-%E7%94%A8%E6%88%B7%E6%96%87%E6%A1%A3
官方的文档说明 TiDB的binlog主要是采用protobuf对日志文件序列化并发送到kafka中;
2,将文档中protobuf文件的定义的格式copy到一个 binlog.proto文件中;
3,maven项目pom中依赖
<!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java -->
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>3.9.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java-util -->
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java-util</artifactId>
<version>3.9.1</version>
</dependency>
<build>
<extensions>
<extension>
<groupId>kr.motd.maven</groupId>
<artifactId>os-maven-plugin</artifactId>
<version>1.5.0.Final</version>
</extension>
</extensions>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<groupId>org.xolstice.maven.plugins</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
<version>0.5.1</version>
<extensions>true</extensions>
<configuration>
<protoSourceRoot>${project.basedir}/src/main/java/com/zhangna/tidb/binlog/proto</protoSourceRoot>
<outputDirectory>${project.basedir}/src/main/java/com/zhangna/tidb/binlog/proto</outputDirectory>
<!--设置是否在生成java文件之前清空outputDirectory的文件,默认值为true,设置为false时也会覆盖同名文件-->
<clearOutputDirectory>false</clearOutputDirectory>
</configuration>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>test-compile</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
4, 将binlog.proto,descriptor.proto,gogo.proto文件放在我们的项目模块中
5, maven clean,compile之后,就会生成解析出来的binlog对应的java文件
6,编代码进一步解析,示例
这里要注意这几点,在kafka消费中,序列化必须是ByteArrayDeserializer,否则解析不出来。
Properties props = new Properties();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:9092");
props.put(ConsumerConfig.GROUP_ID_CONFIG ,"binlog_consumer") ;
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000");
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class);
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class);
package com.zhangna.tidb.binlog.service;
import com.zhangna.tidb.binlog.util.BaseException;
import java.util.List;
import java.util.Map;
/**
* @Author zhangna
* @Date 2019-12-03 20:30
* @Description
*/
public interface ParseBinLogService {
List<Map< String, Object >> parseBinLog2Data(byte[] record) throws BaseException;
}
package com.zhangna.tidb.binlog.service;
import com.google.common.collect.Maps;
import com.google.protobuf.InvalidProtocolBufferException;
import com.zhangna.tidb.binlog.enums.TableEnums;
import com.zhangna.tidb.binlog.proto.BinLogInfo;
import com.zhangna.tidb.binlog.util.BaseException;
import com.zhangna.tidb.binlog.util.ServiceUtil;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.stereotype.Service;
import java.util.*;
/**
* @Author zhangna
* @Date 2019-12-03 20:30
* @Description
*/
@Service
@Slf4j
public class ParseBinLogServiceImpl implements ParseBinLogService {
@Override
public List<Map< String, Object >> parseBinLog2Data(byte[] record) throws BaseException {
return ServiceUtil.execute(() -> parseTable(record),
throwable -> log.error("parse binlog is failed.;t:{}",throwable));
}
private List<Map< String, Object >> parseTable(byte[] record){
if(Objects.isNull(record)){
return null;
}
BinLogInfo.Binlog binlog = null;
try {
binlog = BinLogInfo.Binlog.parseFrom(record);
} catch (InvalidProtocolBufferException e) {
throw new BaseException("parse binlog is failed");
}
if(Objects.isNull(binlog)){
return null;
}
BinLogInfo.DMLData dmlData = binlog.getDmlData();
if(Objects.isNull(dmlData)){
return null;
}
List< BinLogInfo.Table > tablesList = dmlData.getTablesList();
List< String > tableNames = TableEnums.getTableNames();
List<Map< String, Object >> rs= new ArrayList<>();
for( BinLogInfo.Table table : tablesList){
String tableName = table.getTableName();
log.info("tableName:{}",tableName);
if(!tableNames.contains(tableName)){
continue;
}
List< Map< String, Object > > result = this.getResult(table);
rs.addAll(result);
}
return rs;
}
private List<String> parseColumn(BinLogInfo.Table table){
List< BinLogInfo.ColumnInfo > columnInfoList = table.getColumnInfoList();
if(CollectionUtils.isEmpty(columnInfoList)){
return Collections.EMPTY_LIST;
}
List<String> columnKeys = new ArrayList<>();
columnInfoList.forEach(columnInfo -> {
columnKeys.add(columnInfo.getName());
});
return columnKeys;
}
private List< BinLogInfo.TableMutation > getTableMutation(BinLogInfo.Table table){
if(Objects.isNull(table)){
return null;
}
return table.getMutationsList();
}
private List<List<Object>> parseColumnValue(BinLogInfo.Table table) {
List<List<Object>> rs = new ArrayList<>();
List< BinLogInfo.TableMutation > tableMutations = getTableMutation(table);
if (CollectionUtils.isEmpty(tableMutations)) {
return Collections.EMPTY_LIST;
}
for (BinLogInfo.TableMutation tableMutation : tableMutations) {
BinLogInfo.Row rowList = tableMutation.getRow();
if (Objects.isNull(rowList)) {
continue;
}
List< BinLogInfo.Column > columnsList = rowList.getColumnsList();
if (CollectionUtils.isEmpty(columnsList)) {
continue;
}
List< Object > columnValues = new ArrayList<>();
columnsList.forEach(column -> {
String stringValue = column.getStringValue();
long int64Value = column.getInt64Value();
if (int64Value != 0) {
columnValues.add(int64Value);
}
if (null != stringValue && int64Value == 0) {
columnValues.add(stringValue);
}
});
rs.add(columnValues);
}
return rs;
}
private String getOperatorType(BinLogInfo.Table table){
List< BinLogInfo.TableMutation > tableMutations = getTableMutation(table);
if (CollectionUtils.isEmpty(tableMutations)){
return null;
}
BinLogInfo.TableMutation tableMutation = tableMutations.get(0);
BinLogInfo.MutationType type = tableMutation.getType();
if (Objects.isNull(type)){
return null;
}
return type.toString();
}
private List<Map< String, Object >> getResult(BinLogInfo.Table table) {
List<Map< String, Object >> rs = new ArrayList<>();
List< String > columnKeys = parseColumn(table);
List< List< Object > > columnValues = parseColumnValue(table);
if (CollectionUtils.isEmpty(columnKeys) || CollectionUtils.isEmpty(columnValues) ) {
return null;
}
Map< Integer, String > tempFeildMap = Maps.newHashMapWithExpectedSize(columnKeys.size());
String operatorType = getOperatorType(table);
for (String columnKey : columnKeys) {
tempFeildMap.put(columnKeys.indexOf(columnKey), columnKey);
}
Map< String, Object > resultMap = new HashMap<>();
for(List< Object > objects : columnValues){
for(int i= 0; i < objects.size(); i++ ){
resultMap.put(tempFeildMap.get(i), objects.get(i));
}
resultMap.put("operatorType",operatorType);
rs.add(resultMap);
}
return rs;
}
}
github代码示例:https://github.com/zhangna-java/sunshine.git