基于 HBase &amp； Phoenix 构建实时数仓（5）—— 用 Kafka Connect 做实时数据同步_phoenix 数据同步

2301_79099434

于 2024-04-27 10:51:25 发布

阅读量690

点赞数 14

文章标签： hbase kafka android

本文链接：https://blog.csdn.net/2301_79099434/article/details/138244511

版权

insert into test.t1 (remark) values (‘第一行：row1’),(‘第二行：row2’),(‘第三行：row3’);


        输出：

mysql> show master status;
±-----------------±---------±-------------±-----------------±-----------------------------------------+
| File | Position | Binlog_Do_DB | Binlog_Ignore_DB | Executed_Gtid_Set |
±-----------------±---------±-------------±-----------------±-----------------------------------------+
| mysql-bin.000001 | 977 | | | ba615057-e11c-11ee-b80e-246e961c91f8:1-3 |
±-----------------±---------±-------------±-----------------±-----------------------------------------+
1 row in set (0.00 sec)

mysql> create user ‘repl’@‘%’ identified with mysql_native_password by ‘123456’;
Query OK, 0 rows affected (0.01 sec)

mysql> grant replication client,replication slave on . to ‘repl’@‘%’;
Query OK, 0 rows affected (0.00 sec)

mysql> create database test;
Query OK, 1 row affected (0.00 sec)

mysql> create table test.t1 (
-> id bigint(20) not null auto_increment,
-> remark varchar(32) default null comment ‘备注’,
-> createtime timestamp not null default current_timestamp comment ‘创建时间’,
-> primary key (id));
Query OK, 0 rows affected (0.01 sec)

mysql> insert into test.t1 (remark) values (‘第一行：row1’),(‘第二行：row2’),(‘第三行：row3’);
Query OK, 3 rows affected (0.00 sec)
Records: 3 Duplicates: 0 Warnings: 0


        3307 从库实例执行：

change master to
master_host=‘172.18.16.156’,
master_port=3306,
master_user=‘repl’,
master_password=‘123456’,
master_log_file=‘mysql-bin.000001’,
master_log_pos=977;

start slave;
show slave status\G
select user,host from mysql.user;
select * from test.t1;

输出：
mysql> change master to
-> master_host=‘172.18.16.156’,
-> master_port=3306,
-> master_user=‘repl’,
-> master_password=‘123456’,
-> master_log_file=‘mysql-bin.000001’,
-> master_log_pos=977;
Query OK, 0 rows affected, 2 warnings (0.00 sec)

mysql> start slave;
Query OK, 0 rows affected (0.01 sec)

mysql> show slave status\G
*************************** 1. row ***************************
Slave_IO_State: Waiting for master to send event
Master_Host: 172.18.16.156
Master_User: repl
Master_Port: 3306
Connect_Retry: 60
Master_Log_File: mysql-bin.000001
Read_Master_Log_Pos: 2431
Relay_Log_File: vvgg-z2-music-mysqld-relay-bin.000002
Relay_Log_Pos: 1776
Relay_Master_Log_File: mysql-bin.000001
Slave_IO_Running: Yes
Slave_SQL_Running: Yes
Replicate_Do_DB:
Replicate_Ignore_DB:
Replicate_Do_Table:
Replicate_Ignore_Table:
Replicate_Wild_Do_Table:
Replicate_Wild_Ignore_Table:
Last_Errno: 0
Last_Error:
Skip_Counter: 0
Exec_Master_Log_Pos: 2431
Relay_Log_Space: 1999
Until_Condition: None
Until_Log_File:
Until_Log_Pos: 0
Master_SSL_Allowed: No
Master_SSL_CA_File:
Master_SSL_CA_Path:
Master_SSL_Cert:
Master_SSL_Cipher:
Master_SSL_Key:
Seconds_Behind_Master: 0
Master_SSL_Verify_Server_Cert: No
Last_IO_Errno: 0
Last_IO_Error:
Last_SQL_Errno: 0
Last_SQL_Error:
Replicate_Ignore_Server_Ids:
Master_Server_Id: 1563306
Master_UUID: ba615057-e11c-11ee-b80e-246e961c91f8
Master_Info_File: mysql.slave_master_info
SQL_Delay: 0
SQL_Remaining_Delay: NULL
Slave_SQL_Running_State: Slave has read all relay log; waiting for more updates
Master_Retry_Count: 86400
Master_Bind:
Last_IO_Error_Timestamp:
Last_SQL_Error_Timestamp:
Master_SSL_Crl:
Master_SSL_Crlpath:
Retrieved_Gtid_Set: ba615057-e11c-11ee-b80e-246e961c91f8:4-8
Executed_Gtid_Set: ba615057-e11c-11ee-b80e-246e961c91f8:4-8,
c2df1946-e11c-11ee-8026-246e961c91f8:1-3
Auto_Position: 0
Replicate_Rewrite_DB:
Channel_Name:
Master_TLS_Version:
Master_public_key_path:
Get_master_public_key: 0
Network_Namespace:
1 row in set (0.00 sec)

mysql> select * from test.t1;
±—±-----------------±--------------------+
| id | remark | createtime |
±—±-----------------±--------------------+
| 1 | 第一行：row1 | 2024-03-20 10:25:32 |
| 2 | 第二行：row2 | 2024-03-20 10:25:32 |
| 3 | 第三行：row3 | 2024-03-20 10:25:32 |
±—±-----------------±--------------------+
3 rows in set (0.00 sec)


        MySQL主从复制相关配置参见“[配置异步复制]( )”。


## 四、安装部署 Kafka Connector


        在 node2 上执行以下步骤。


### 1. 创建插件目录

mkdir $KAFKA_HOME/plugins


### 2. 解压文件到插件目录

debezium-connector-mysql

unzip debezium-debezium-connector-mysql-2.4.2.zip -d $KAFKA_HOME/plugins/

kafka-connect-hbase

unzip confluentinc-kafka-connect-hbase-2.0.13.zip -d $KAFKA_HOME/plugins/


### 3. 配置 Kafka Connector


#### （1）配置属性文件

编辑 connect-distributed.properties 文件

vim $KAFKA_HOME/config/connect-distributed.properties


        内容如下：

bootstrap.servers=node2:9092,node3:9092,node4:9092
group.id=connect-cluster
key.converter=org.apache.kafka.connect.json.JsonConverter
value.converter=org.apache.kafka.connect.json.JsonConverter
key.converter.schemas.enable=false
value.converter.schemas.enable=false
offset.storage.topic=connect-offsets
offset.storage.replication.factor=3
offset.storage.partitions=3
config.storage.topic=connect-configs
config.storage.replication.factor=3
status.storage.topic=connect-status
status.storage.replication.factor=3
status.storage.partitions=3
offset.flush.interval.ms=10000
plugin.path=/root/kafka_2.13-3.7.0/plugins