kafka 两台单机实现 数据同步

kafka数据同步

zookeeper配置

[root@node1 zookeeper-3.4.9]# grep '^[a-Z]' conf/zoo.cfg 
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/opt/iotplatform/zookeeper-3.4.9/data
dataLogDir=/opt/iotplatform/zookeeper-3.4.9/datalog
clientPort=2181

kafka配置

# 修改配置文件如下
[root@node1 kafka_2.11-2.3.0]# grep '^[a-Z]' config/server.properties
broker.id=1
listeners=PLAINTEXT://node1.iotplatform.com:9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/opt/iotplatform/kafka_2.11-2.3.0/datalog
num.partitions=5
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=24
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=node1.iotplatform.com:2181
zookeeper.connection.timeout.ms=60000
group.initial.rebalance.delay.ms=0


#node1.iotplatform.com:添加以下配置
[root@node1 kafka_2.11-2.3.0]# cat sourceClusterConsumer.config 
zookeeper.connect=node1.iotplatform.com:2181
bootstrap.servers=node1.iotplatform.com:9092
group.id=1
partition.assignment.strategy=org.apache.kafka.clients.consumer.RoundRobinAssignor

[root@node1 kafka_2.11-2.3.0]# cat targetClusterProducer.config 
zookeeper.connect=node2.iotplatform.com:2181
bootstrap.servers=node2.iotplatform.com:9092


nohup bin/kafka-mirror-maker.sh  --consumer.config sourceClusterConsumer.config --num.streams 2 --producer.config targetClusterProducer.config -whitelist=".*" &

如果node1节点挂掉之后,启用node2

# node2.iotplatform.com:添加以下配置
[root@node1 kafka_2.11-2.3.0]# cat sourceClusterConsumer.config 
zookeeper.connect=node2.iotplatform.com:2181
bootstrap.servers=node2.iotplatform.com:9092
group.id=1
partition.assignment.strategy=org.apache.kafka.clients.consumer.RoundRobinAssignor

[root@node1 kafka_2.11-2.3.0]# cat targetClusterProducer.config 
zookeeper.connect=node1.iotplatform.com:2181
bootstrap.servers=node1.iotplatform.com:9092

# 启动mirror-maker
nohup bin/kafka-mirror-maker.sh  --consumer.config sourceClusterConsumer.config --num.streams 2 --producer.config targetClusterProducer.config -whitelist=".*" &
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值