数据湖仓一体(六)安装flink

上传安装包到/opt/software目录并解压

[bigdata@node106 software]$ tar -zxvf flink-1.17.2-bin-scala_2.12.tgz -C /opt/services/

重命名文件

[bigdata@node106 services]$ mv flink-1.17.2-bin-scala_2.12 flink-1.17.2

配置环境变量

[bigdata@node106 ~]$ sudo vim /etc/profile.d/bigdata_env.sh

export FLINK_HOME=/opt/services/flink-1.17.2
export $PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$ZK_HOME/bin:$KAFKA_HOME/bin:$SEA_HOME/bin:$HIVE_HOME/bin:$SPARK_HOME/bin:$FLINK_HOME/bin

分发环境变量

[bigdata@node106 bin]$ sudo ./bin/xsync /etc/profile.d/bigdata_env.sh

刷新环境变量，5台机器上执行

[bigdata@node106 ~]$ source /etc/profile

配置flink-conf.yaml

taskmanager.numberOfTaskSlots: 4
execution.checkpointing.interval: 1min
execution.checkpointing.externalized-checkpoint-retention: RETAIN_ON_CANCELLATION
execution.checkpointing.max-concurrent-checkpoints: 1
execution.checkpointing.min-pause: 0
state.backend: rocksdb
state.checkpoints.dir: hdfs://mycluster:8020/flink-checkpoints
state.savepoints.dir: hdfs://mycluster:8020/flink-savepoints
state.backend.incremental: true
jobmanager.archive.fs.dir: hdfs://mycluster:8020/flink-history
historyserver.web.address: node107
historyserver.web.port: 8082
historyserver.archive.fs.dir: hdfs://mycluster:8020/flink-history
historyserver.archive.fs.refresh-interval: 10000

配置sql-client-init.sql

CREATE CATALOG hoodie_catalog with (
        'type' = 'hudi',
        'default-database' = 'default',
        'catalog.path' = '/user/hudi/warehouse',
        'hive.conf.dir' = '/opt/services/spark-3.3.1/conf',
        'mode' = 'hms' 
);
use catalog hoodie_catalog;
load module hive with ('hive-version'='3.1.3');
set sql-client.execution.result-mode=tableau;
set table.dynamic-table-options.enabled=true;
set table.exec.sink.not-null-enforcer=drop;

上传jar包到lib目录下

[bigdata@node106 software]$ cp /opt/services/hadoop-3.3.5/share/hadoop/mapreduce/hadoop-mapreduce-client-core-3.3.5.jar /opt/services/flink-1.17.2/lib/
[bigdata@node106 software]$ cp /opt/services/hadoop-3.3.5/share/hadoop/common/lib/guava-27.0-jre.jar /opt/services/flink-1.17.2/lib/
[bigdata@node106 software]$ cp mysql-connector-java-8.0.18.jar /opt/services/flink-1.17.2/lib/
[bigdata@node106 software]$ cp flink-sql-connector-hive-3.1.3_2.12-1.17.1.jar /opt/services/flink-1.17.2/lib/
[bigdata@node106 software]$ cp hudi-flink1.17-bundle-0.14.1.jar /opt/services/flink-1.17.2/lib/

替换jar包

[bigdata@node106 conf]$ mv /opt/services/flink-1.17.2/opt/flink-table-planner_2.12-1.17.2.jar  /opt/services/flink-1.17.2/lib/flink-table-planner_2.12-1.17.2.jar  
[bigdata@node106 conf]$ mv /opt/services/flink-1.17.2/lib/flink-table-planner-loader-1.17.2.jar  /opt/services/flink-1.17.2/opt/flink-table-planner-loader-1.17.2.jar

hdfs上创建文件夹

[bigdata@node106 conf]$ hdfs dfs -mkdir /flink-history 
[bigdata@node106 conf]$ hdfs dfs -mkdir /flink-checkpoint
[bigdata@node106 conf]$ hdfs dfs -mkdir /flink-savepoints

编写flink.sh脚本

[bigdata@node106 bin]$ vim flink.sh 

#!/bin/bash

if [ $# -lt 1 ]
then
    echo "No Args Input...[start,stop]"
    exit ;
fi
case $1 in
"start")
        echo ==================== 启动flink history服务 =========================
        ssh node106 "$FLINK_HOME/bin/historyserver.sh start"

;;
"stop")
        echo ==================== 关闭flink history服务 =========================
        ssh node106 "$FLINK_HOME/bin/historyserver.sh stop"
;;
*)
    echo "Input Args Error...[start,stop]"
;;
esac

授权

[bigdata@node106 bin]$ chmod +x flink.sh

分发到其他机器

[bigdata@node106 bin]$ xsync  flink.sh 

copy到其他机器

[bigdata@node107 bin]$ scp -r bigdata@node106:/opt/services/flink-1.17.2/ /opt/services/flink-1.17.2/   
[bigdata@node108 bin]$ scp -r bigdata@node106:/opt/services/flink-1.17.2/ /opt/services/flink-1.17.2/

启动历史服务

[bigdata@node106 bin]$ flink.sh start 

启动yarn-session

[bigdata@node106 flink-1.17.2]$ bin/yarn-session.sh -d

启动flink-sql

[bigdata@node106 flink-1.17.2]$ bin/sql-client.sh embedded  -i  /opt/services/flink-1.17.2/conf/sql-client-init.sql   -s yarn-session