大数据实时传输组件Maxwell-时区问题源码修改

本文链接：https://blog.csdn.net/weixin_45943866/article/details/123569615

本文档详细介绍了在大数据实时平台搭建中，Maxwell在处理timestamp字段时默认使用UTC时间导致8小时时差的问题。通过修改Maxwell源码，新增`TimeStampColumnDef`类并调整配置，成功解决了timestamp字段的时区问题。同时，也提到尽管解决了timestamp问题，但由于Maxwell在使用过程中出现心跳错误导致宕机，最终选择了Canal作为替代方案。

摘要由CSDN通过智能技术生成

项目场景：

大数据实时平台搭建，数据采集部分…

问题描述

Maxwell在处理timestamp字段类型时，默认使用utc时间，会差8个小时。设置配置文件中的参数jdbc_options和replication_jdbc_options，发现只对datetime字段生效，通过修改源码解决了这个问题。

各版本源码下载地址：https://github.com/zendesk/maxwell/tags
下载*.zip包就可以，本地解压后用idea打开修改编译

1.修改

类名：com.zendesk.maxwell.schema.columndef.ColumnDef.java

修改前：
case "datetime":
	case "timestamp":
	return new DateTimeColumnDef(name, type, pos, columnLength);
 
修改后：
case "datetime":
	return new DateTimeColumnDef(name, type, pos, columnLength);
case "timestamp":
	return new TimeStampColumnDef(name, type, pos, columnLength);

2.新增：

类名：com.zendesk.maxwell.schema.columndef.TimeStampColumnDef.java

package com.zendesk.maxwell.schema.columndef;
 
import com.zendesk.maxwell.producer.MaxwellOutputConfig;
 
import java.sql.Timestamp;
 
public class TimeStampColumnDef extends ColumnDefWithLength {
	public TimeStampColumnDef(String name, String type, short pos, Long columnLength) {
		super(name, type, pos, columnLength);
	}
 
	@Override
	protected String formatValue(Object value, MaxwellOutputConfig config) {
		Timestamp ts = TimeStampFormatter.extractTimestamp(value);
		String dateString = TimeStampFormatter.formatDateTime(value, ts);
 
		if ( dateString == null )
			return null;
		else
			return appendFractionalSeconds(dateString, ts.getNanos(), columnLength);
	}
 
}

类名：com.zendesk.maxwell.schema.columndef.TimeStampFormatter.java

package com.zendesk.maxwell.schema.columndef;
 
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.TimeZone;
 
public class TimeStampFormatter {
	private static SimpleDateFormat makeFormatter(String format, boolean utc) {
		SimpleDateFormat dateFormatter = new SimpleDateFormat(format);
		dateFormatter.setTimeZone(TimeZone.getTimeZone("Asia/Shanghai"));
		return dateFormatter;
	}
 
	private static SimpleDateFormat dateFormatter           = makeFormatter("yyyy-MM-dd", false);
	private static SimpleDateFormat dateUTCFormatter        = makeFormatter("yyyy-MM-dd", true);
	private static SimpleDateFormat dateTimeFormatter       = makeFormatter("yyyy-MM-dd HH:mm:ss", false);
	private static SimpleDateFormat dateTimeUTCFormatter    = makeFormatter("yyyy-MM-dd HH:mm:ss", true);
 
	public static Timestamp extractTimestamp(Object value) {
		if (value instanceof Long) {
			Long micros = (Long) value;
			long millis = floorDiv(micros, 1000L);
			Timestamp t = new Timestamp(millis);
			long microsOnly = floorMod(micros, (long) 1000000);
			t.setNanos((int) microsOnly * 1000);
			return t;
		} else if (value instanceof Timestamp) {
			return (Timestamp) value;
		} else if ( value instanceof Date ) {
			Long time = ((Date) value).getTime();
			return new Timestamp(time);
		} else
			throw new RuntimeException("couldn't extract date/time out of " + value);
	}
 
	private static Timestamp MIN_DATE = Timestamp.valueOf("1000-01-01 00:00:00");
 
	private static String format(SimpleDateFormat formatter, Timestamp ts) {
		if ( ts.before(MIN_DATE) ) {
			return null;
		} else {
			synchronized(formatter) {
					return formatter.format(ts);
			}
		}
	}
 
	public static String formatDate(Object value) {
		SimpleDateFormat formatter;
		if ( value instanceof Long )
			formatter = dateUTCFormatter;
		else
			formatter = dateFormatter;
 
		return format(formatter, extractTimestamp(value));
	}
 
	public static String formatDateTime(Object value, Timestamp ts) {
		SimpleDateFormat formatter;
 
		if ( value instanceof Long )
			formatter = dateTimeUTCFormatter;
		else
			formatter = dateTimeFormatter;
 
		return format(formatter, ts);
	}
 
	private static long floorDiv(long a, long b) {
		return ((a < 0)?(a - (b - 1)):a) / b;
	}
 
	private static long floorMod(long x, long y) {
		return x - floorDiv(x, y) * y;
	}
}