如何使用java解析TiDB的binlog文件

1，找到TiDB的官方文档中binlog的说明：
https://docs.pingcap.com/zh/tidb/stable/binlog-slave-client#binlog-slave-client-%E7%94%A8%E6%88%B7%E6%96%87%E6%A1%A3

官方的文档说明 TiDB的binlog主要是采用protobuf对日志文件序列化并发送到kafka中；

2，将文档中protobuf文件的定义的格式copy到一个 binlog.proto文件中；
3，maven项目pom中依赖

        <!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java -->
        <dependency>
            <groupId>com.google.protobuf</groupId>
            <artifactId>protobuf-java</artifactId>
            <version>3.9.0</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java-util -->
        <dependency>
            <groupId>com.google.protobuf</groupId>
            <artifactId>protobuf-java-util</artifactId>
            <version>3.9.1</version>
        </dependency>

      <build>
        <extensions>
            <extension>
                <groupId>kr.motd.maven</groupId>
                <artifactId>os-maven-plugin</artifactId>
                <version>1.5.0.Final</version>
            </extension>
        </extensions>

        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-assembly-plugin</artifactId>
                <configuration>
                    <descriptorRefs>
                        <descriptorRef>jar-with-dependencies</descriptorRef>
                    </descriptorRefs>
                </configuration>

                <executions>
                    <execution>
                        <id>make-assembly</id>
                        <phase>package</phase>
                        <goals>
                            <goal>single</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                </configuration>
            </plugin>

            <plugin>
                <groupId>org.xolstice.maven.plugins</groupId>
                <artifactId>protobuf-maven-plugin</artifactId>
                <version>0.5.1</version>
                <extensions>true</extensions>
                <configuration>
		                    <protoSourceRoot>${project.basedir}/src/main/java/com/zhangna/tidb/binlog/proto</protoSourceRoot>
		                    <outputDirectory>${project.basedir}/src/main/java/com/zhangna/tidb/binlog/proto</outputDirectory>
                            <!--设置是否在生成java文件之前清空outputDirectory的文件，默认值为true，设置为false时也会覆盖同名文件-->
		                    <clearOutputDirectory>false</clearOutputDirectory>
		                </configuration>
		                <executions>
		                    <execution>
		                        <goals>
		                            <goal>compile</goal>
		                            <goal>test-compile</goal>
		                        </goals>
		                    </execution>
		                </executions>
		            </plugin>
		
		        </plugins>
		    </build>

4， 将binlog.proto,descriptor.proto,gogo.proto文件放在我们的项目模块中

5, maven clean，compile之后，就会生成解析出来的binlog对应的java文件

6，编代码进一步解析，示例

这里要注意这几点，在kafka消费中，序列化必须是ByteArrayDeserializer，否则解析不出来。

Properties props = new Properties();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,                   "127.0.0.1:9092");
props.put(ConsumerConfig.GROUP_ID_CONFIG ,"binlog_consumer") ;
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000");
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,                 ByteArrayDeserializer.class);
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class);


package com.zhangna.tidb.binlog.service;

import com.zhangna.tidb.binlog.util.BaseException;

import java.util.List;
import java.util.Map;

/**
 * @Author zhangna
 * @Date 2019-12-03 20:30
 * @Description
 */
public interface ParseBinLogService {

	List<Map< String, Object >> parseBinLog2Data(byte[] record) throws BaseException;

}

package com.zhangna.tidb.binlog.service;

import com.google.common.collect.Maps;
import com.google.protobuf.InvalidProtocolBufferException;
import com.zhangna.tidb.binlog.enums.TableEnums;
import com.zhangna.tidb.binlog.proto.BinLogInfo;
import com.zhangna.tidb.binlog.util.BaseException;
import com.zhangna.tidb.binlog.util.ServiceUtil;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.stereotype.Service;

import java.util.*;

/**
 * @Author zhangna
 * @Date 2019-12-03 20:30
 * @Description
 */
@Service
@Slf4j
public class ParseBinLogServiceImpl implements ParseBinLogService {

	@Override
	public List<Map< String, Object >> parseBinLog2Data(byte[] record) throws BaseException {

		return ServiceUtil.execute(() -> parseTable(record),
				throwable -> log.error("parse binlog is failed.;t:{}",throwable));

	}


	private List<Map< String, Object >> parseTable(byte[] record){

		if(Objects.isNull(record)){
			return null;
		}


		BinLogInfo.Binlog binlog = null;

		try {
			binlog = BinLogInfo.Binlog.parseFrom(record);
		} catch (InvalidProtocolBufferException e) {
			throw new BaseException("parse binlog is failed");
		}
		if(Objects.isNull(binlog)){
			return null;
		}
		BinLogInfo.DMLData dmlData = binlog.getDmlData();
		if(Objects.isNull(dmlData)){
			return null;
		}

		List< BinLogInfo.Table > tablesList = dmlData.getTablesList();

		List< String > tableNames = TableEnums.getTableNames();

		List<Map< String, Object >> rs= new ArrayList<>();
		for( BinLogInfo.Table table : tablesList){
			String tableName = table.getTableName();
			log.info("tableName:{}",tableName);
			if(!tableNames.contains(tableName)){
				continue;
			}

			List< Map< String, Object > > result = this.getResult(table);
			rs.addAll(result);
		}

		return rs;

	}

	private List<String> parseColumn(BinLogInfo.Table table){

		List< BinLogInfo.ColumnInfo > columnInfoList = table.getColumnInfoList();
		if(CollectionUtils.isEmpty(columnInfoList)){
			return Collections.EMPTY_LIST;
		}
		List<String> columnKeys = new ArrayList<>();
		columnInfoList.forEach(columnInfo -> {
			columnKeys.add(columnInfo.getName());
		});

		return columnKeys;

	}

	private List< BinLogInfo.TableMutation > getTableMutation(BinLogInfo.Table table){

		if(Objects.isNull(table)){
			return null;
		}
		return table.getMutationsList();
	}

	private List<List<Object>> parseColumnValue(BinLogInfo.Table table) {

		List<List<Object>> rs = new ArrayList<>();

		List< BinLogInfo.TableMutation > tableMutations = getTableMutation(table);
		if (CollectionUtils.isEmpty(tableMutations)) {
			return Collections.EMPTY_LIST;
		}

		for (BinLogInfo.TableMutation tableMutation : tableMutations) {

			BinLogInfo.Row rowList = tableMutation.getRow();
			if (Objects.isNull(rowList)) {
				continue;
			}


			List< BinLogInfo.Column > columnsList = rowList.getColumnsList();
			if (CollectionUtils.isEmpty(columnsList)) {
				continue;
			}

			List< Object > columnValues = new ArrayList<>();
			columnsList.forEach(column -> {
				String stringValue = column.getStringValue();
				long int64Value = column.getInt64Value();
				if (int64Value != 0) {
					columnValues.add(int64Value);
				}
				if (null != stringValue && int64Value == 0) {
					columnValues.add(stringValue);
				}
			});

			rs.add(columnValues);

		}
		return rs;
	}


	private String getOperatorType(BinLogInfo.Table table){
		List< BinLogInfo.TableMutation > tableMutations = getTableMutation(table);

		if (CollectionUtils.isEmpty(tableMutations)){
			return null;
		}

		BinLogInfo.TableMutation tableMutation = tableMutations.get(0);
		BinLogInfo.MutationType type = tableMutation.getType();
		if (Objects.isNull(type)){
			return null;
		}

		return type.toString();
	}

	private List<Map< String, Object >> getResult(BinLogInfo.Table table) {

		List<Map< String, Object >> rs = new ArrayList<>();

		List< String > columnKeys = parseColumn(table);
		List< List< Object > > columnValues = parseColumnValue(table);


		if (CollectionUtils.isEmpty(columnKeys) || CollectionUtils.isEmpty(columnValues) ) {
			return null;
		}

		Map< Integer, String > tempFeildMap = Maps.newHashMapWithExpectedSize(columnKeys.size());

		String operatorType = getOperatorType(table);

		for (String columnKey : columnKeys) {
			tempFeildMap.put(columnKeys.indexOf(columnKey), columnKey);
		}

		Map< String, Object > resultMap = new HashMap<>();

		for(List< Object > objects : columnValues){
			for(int i= 0; i < objects.size(); i++ ){
				resultMap.put(tempFeildMap.get(i), objects.get(i));
			}
			resultMap.put("operatorType",operatorType);
			rs.add(resultMap);

		}

		return rs;
	}



}

github代码示例：https://github.com/zhangna-java/sunshine.git