Flink SQL CDC Connectors
Flink SQL CDC
一、安装flink
将flink-sql-connector-mysql-cdc-1.2.0.jar 拷贝至flink lib目录下
启动flink
./start-cluster.sh
二、配置数据库
创建用户和授权
CREATE USER 'flink'@'%' IDENTIFIED BY 'flink';
GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'flink' IDENTIFIED BY 'flink';
FLUSH PRIVILEGES;
查询binlog是否开启
set global show_compatibility_56=on;
SELECT variable_value as "BINARY LOGGING STATUS (log-bin) ::"
FROM information_schema.global_variables WHERE variable_name='log_bin';
通过mysql客户端连接数据库是交互式连接,通过jdbc连接数据库是非交互式连接。
查看mysql客户端超时参数
show global variables like 'wait_timeout';
show global variables like 'interactive_timeout';
三、flink客户端操纵
1.启动flink客户端
2.查看Mysql CDC 同步
CREATE TABLE orders (
order_id INT,
order_date TIMESTAMP(0),
customer_name STRING,
price DECIMAL(10, 5),
product_id INT,
order_status BOOLEAN
) WITH (
'connector' = 'mysql-cdc',
'hostname' = 'localhost',
'port' = '3306',
'username' = 'flink',
'password' = 'flink',
'database-name' = 'flink',
'table-name' = 'orders'
);
SELECT * FROM orders;
四、通过flink-connector-mysql-cdc(JAVA代码)监听数据变化
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.types.Row;
public class FlinkConnectorMysqlCDCDemo {
public static void main(String[] args) throws Exception {
final ParameterTool params = ParameterTool.fromArgs(args);
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1); // source only supports parallelism of 1
final StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
// register a table in the catalog
tEnv.executeSql(
"CREATE TABLE orders (\n" +
" order_id INT,\n" +
" order_date TIMESTAMP(0),\n" +
" customer_name STRING,\n" +
" price DECIMAL(10, 5),\n" +
" product_id INT,\n" +
" order_status BOOLEAN\n" +
") WITH (\n" +
" 'connector' = 'mysql-cdc',\n" +
" 'hostname' = 'localhost',\n" +
" 'port' = '3306',\n" +
" 'username' = 'flink',\n" +
" 'password' = 'flink',\n" +
" 'database-name' = 'flink',\n" +
" 'table-name' = 'orders'\n" +
")");
// define a dynamic aggregating query
final Table result = tEnv.sqlQuery("SELECT * FROM orders");
// print the result to the console
tEnv.toRetractStream(result, Row.class).print();
env.execute();
}
}