docker发布maxwell并集成rocketmq

 从github上clone maxwell源码

git clone https://github.com/zendesk/maxwell.git

对maxwell/pom.xml增加rocketmq支持

<profile>
    <id>rocketmq</id>
    <activation>
        <activeByDefault>true</activeByDefault>
    </activation>
    <dependencies>
	    <dependency>
            <groupId>org.apache.rocketmq</groupId>
            <artifactId>rocketmq-client</artifactId>
            <version>4.7.1</version>
        </dependency>
    </dependencies>
</profile>

参照rabbitmq在MaxwellConfig类增加rocketmq配置项

......
public String rocketmqNamesrvAddr;
public String rocketmqProducerGroup;
public String rocketmqSendTopic;
public String rocketmqTags;
public Integer rocketmqMaxsize;
public Integer rocketmqComprsize;
......


private void setup(OptionSet options, Properties properties) {
    ......
    this.rocketmqNamesrvAddr = fetchStringOption("rocketmq_namesrv_addr", options, properties, "localhost:9876");
	this.rocketmqProducerGroup = fetchStringOption("rocketmq_producer_group", options, properties, "rocketmq_producer_group");
	this.rocketmqSendTopic = fetchStringOption("rocketmq_send_topic", options, properties, "maxwell");
	this.rocketmqTags = fetchStringOption("rocketmq_tags", options, properties, "maxwell");
	this.rocketmqMaxsize = fetchIntegerOption("rocketmq_maxsize", options, properties, 0);
	this.rocketmqComprsize = fetchIntegerOption("rocketmq_comprsize", options, properties, 0);
    ......
}

在com.zendesk.maxwell.producer包下增加RocketmqProducer

package com.zendesk.maxwell.producer;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.remoting.common.RemotingHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.zendesk.maxwell.MaxwellContext;
import com.zendesk.maxwell.row.RowMap;

public class RocketmqProducer extends AbstractProducer {

	private static final Logger LOGGER = LoggerFactory.getLogger(RocketmqProducer.class);
	// 实例化消息生产者Producer
	private DefaultMQProducer producer;

	private String rocketmqSendTopic;
	private String rocketmqTags;
	private Integer rocketmqMaxsize;
	private Integer rocketmqComprsize;

	public RocketmqProducer(MaxwellContext context) {
		super(context);
		try {
			producer = new DefaultMQProducer(context.getConfig().rocketmqProducerGroup);
			// 设置NameServer的地址
			producer.setNamesrvAddr(context.getConfig().rocketmqNamesrvAddr);

			rocketmqSendTopic = context.getConfig().rocketmqSendTopic;
			rocketmqTags = context.getConfig().rocketmqTags;
			rocketmqMaxsize = context.getConfig().rocketmqMaxsize;
			rocketmqComprsize = context.getConfig().rocketmqComprsize;
			// 启动Producer实例
			producer.start();
		} catch (MQClientException e) {
			throw new RuntimeException(e);
		}

	}

	@Override
	public void push(RowMap r) throws Exception {
		if (!r.shouldOutput(outputConfig)) {
			context.setPosition(r.getNextPosition());
			return;
		}
		String value = r.toJSON(outputConfig);
		Message msg = null;
		// 创建消息，并指定Topic，Tag和消息体
		if (rocketmqMaxsize != null && rocketmqMaxsize > 0 && value.length() > rocketmqMaxsize) {
			r.getOldData().clear();

			List<String> pkNames = r.getPrimaryKeyColumns();
			Set<String> fieldKeys = new HashSet<>();
			fieldKeys.addAll(r.getData().keySet());
			for (String field : fieldKeys) {
				if (!pkNames.contains(field)) {
					r.getData().remove(field);
				}
			}
			value = r.toJSON(outputConfig);
			msg = new Message(rocketmqSendTopic /* Topic */, rocketmqTags /* Tag */, rocketmqTags /* key */, value.getBytes(RemotingHelper.DEFAULT_CHARSET));
		} else if (rocketmqComprsize != null && rocketmqComprsize > 0 && value.length() > rocketmqComprsize) {
			byte[] compressVal = compress(r.toJSON(outputConfig));
			msg = new Message(rocketmqSendTopic /* Topic */, rocketmqTags /* Tag */, (rocketmqTags + "_compress") /* key */, compressVal);
		} else {
			msg = new Message(rocketmqSendTopic /* Topic */, rocketmqTags /* Tag */, value.getBytes(RemotingHelper.DEFAULT_CHARSET));
		}
		producer.send(msg);
		if (r.isTXCommit()) {
			context.setPosition(r.getNextPosition());
		}
		if (LOGGER.isDebugEnabled()) {
			LOGGER.debug("->  topic:" + rocketmqSendTopic + ", tags:" + rocketmqTags);
		}
	}

	/**
	 * 字符串压缩
	 *
	 * @param source 需要压缩的源字符串
	 * @return 压缩后的字符串
	 */
	private static byte[] compress(String source) {
		if (null == source || source.length() <= 0) {
			return null;
		}
		try (ByteArrayOutputStream out = new ByteArrayOutputStream(); GZIPOutputStream gzip = new GZIPOutputStream(out)) { // 将字符串以字节的形式写入到 GZIP 压缩输出流中
			gzip.write(source.getBytes(RemotingHelper.DEFAULT_CHARSET));
			gzip.close();
			return out.toByteArray();
		} catch (IOException e) {
			LOGGER.warn("gzip压缩失败", e);
		}
		return null;
	}

	/**
	 * 字符串解压缩
	 *
	 * @param source 待解压的字符串
	 * @return 解压后的字符串
	 */
	private static String decompress(byte[] source) {
		if (source == null || source.length == 0) {
			return null;
		}
		try (ByteArrayOutputStream out = new ByteArrayOutputStream();
				ByteArrayInputStream in = new ByteArrayInputStream(source);
				GZIPInputStream gunzip = new GZIPInputStream(in)) {
			byte[] buffer = new byte[256];
			int n;
			// 从 GZIP 压缩输入流读取字节数据到 buffer 数组中
			while ((n = gunzip.read(buffer)) >= 0) {
				out.write(buffer, 0, n);
			}
			return out.toString(RemotingHelper.DEFAULT_CHARSET);
		} catch (IOException e) {
			LOGGER.warn("gzip解压失败", e);
		}
		return null;
	}
}

docker编译maxwell
1、上传本地maven的settings.xml文件到maxwell目录下。
2、修改maxwell源码下的dockerfile，设置本地maven仓库

# prime so we can have a cached image of the maven deps
COPY pom.xml /tmp
#增加本地maven配置,加快编译
COPY settings.xml /usr/share/maven/conf/settings.xml
RUN cd /tmp && mvn dependency:resolve

3、执行docker编译命令
docker build -t me/maxwell:1.0.0 .
4、将编译后的镜像打包并推送到harbo私服
docker tag h55aaj1o9h1 172.10.20.11/me/maxwell:1.0.0
docker push 172.10.20.11/me/maxwell:1.0.0

5、在master数据库创建maxwell用户并授权
create database maxwell;
CREATE USER 'maxwell'@'%' IDENTIFIED BY 'xxxxxx';
GRANT ALL ON maxwell.* TO 'maxwell'@'%';
GRANT SELECT, REPLICATION CLIENT, REPLICATION SLAVE on *.* to 'maxwell'@'%';

6、docker运行maxwell

docker run -it --rm me/maxwell:1.0.0 bin/maxwell --user='maxwell' --password='xxxxxx' --host='172.10.20.12' --port='3306' --filter 'exclude: *.*, include: me.t_order_info,include: me.t_product_info,include: me.member_info' --producer=rocketmq   --rocketmq_namesrv_addr='172.10.20.201:9876' --rocketmq_send_topic=me_maxwell --rocketmq_producer_group=me_maxwell_producer_group

-- 进入docker容器执行全量索引
./bin/maxwell-bootstrap --user maxwell --password xxxxxxxx --host 172.10.20.11 --port 3306 --schema_database maxwell --database me  --table t_order_info,t_product_info,member_info --replication_host  172.10.10.30  --replication_port 3306 --log_level DEBUG --client_id maxwell --where create_time between '2023-01-01 00:00:00' and '2023-07-04 23:59:59'