使用flink的standalone模式同步Kafka的数据到clickhouse

longlongbreak

已于 2024-03-25 14:24:21 修改

阅读量539

点赞数 8

文章标签： flink kafka clickhouse

于 2024-03-15 14:28:09 首次发布

本文链接：https://blog.csdn.net/longlongbreak/article/details/136738104

版权

1.使用flink官网的jar包

flink官网1.16.3版本的下载地址

flink1.16.3的安装包

2.修改配置

vim conf/flink-conf.yaml

修改如下内容：

# JobManager节点地址.

jobmanager.rpc.address: localhost #或者直接写ip地址

jobmanager.bind-host: 0.0.0.0

rest.address: localhost #或者直接写ip地址

rest.bind-address: 0.0.0.0

# TaskManager节点地址.需要配置为当前机器名

taskmanager.bind-host: 0.0.0.0

taskmanager.host: localhost #或者直接写ip地址

因为是单机模式，只用了一台服务器，所以vim workers是localhost，vim masters是localhost:8081

3.准备

把写好的jar包（胖包）放在flink的lib目录下，flink启动时会自动扫描lib目录，注意log4j依赖冲突问题，改屏蔽的依赖在打包时要屏蔽

standalone的应用模式下不会提前创建集群，所以不能调用 start-cluster.sh 脚本。我们可以使用同样在 bin 目录下的 standalone-job.sh 来创建一个 JobManager

4.启动 JobManager。（不用指定jar包，启动脚本会自动扫描 lib 目录下所有的jar 包，这里直接指定作业入口类）

./bin/standalone-job.sh   start   --job-classname  zhilong.com.dw.dwd.FlinkSinkClickhouse

./bin/standalone-job.sh   start   --job-classname zhilong.com.dw.dwd.KafkaToClickhouse
./bin/taskmanager.sh start

5.启动 TaskManager。

./bin/taskmanager.sh start

6.如果希望停掉集群，同样可以使用脚本，命令如下。（这里未测试成功别停止脚本）

./bin/standalone-job.sh stop
./bin/taskmanager.sh stop


    
./bin/jobmanager.sh stop
./bin/taskmanager.sh stop

7.根据代码里设置的Kafka数据属性在clickhouse里建表

CREATE TABLE
  ods_countlyV2 (
    appKey String,
    appVersion String,
    deviceId String,
    phone_no String
  ) ENGINE = MergeTree ()
ORDER BY
  (appKey, appVersion, deviceId, phone_no);

8.起一个Kafka生产者发送一条消息，然后观察clickhouse对应表里的情况

9.观察clickhouse表里数据的情况

#代码

1.主程序类

package com.kszx;

import com.alibaba.fastjson.JSON;
import com.kszx.Mail;
import com.kszx.MyClickHouseUtil;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;

import java.util.HashMap;
import java.util.Properties;


public class FlinkSinkClickhouse {

    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.enableCheckpointing(5000);
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

        // source
        String topic = "topic_test2";

        Properties props = new Properties();
        // 设置连接kafka集群的参数
        props.setProperty("bootstrap.servers", "172.xx.xxx.x:9092,172.xx.xxx.x:9092,172.xx.xxx.x:9092");

        // 定义Flink Kafka Consumer
        FlinkKafkaConsumer<String> consumer = new FlinkKafkaConsumer<String>(topic, new SimpleStringSchema(), props);

        consumer.setStartFromGroupOffsets();
        consumer.setStartFromEarliest();    // 设置每次都从头消费

        // 添加source数据流
        DataStreamSource<String> source = env.addSource(consumer);
        source.print("111");
        System.out.println(source);
        SingleOutputStreamOperator<Mail> dataStream = source.map(new MapFunction<String, Mail>() {
            @Override
            public Mail map(String value) throws Exception {
                HashMap<String, String> hashMap = JSON.parseObject(value, HashMap.class);
                // System.out.println(hashMap);
                String appKey = hashMap.get("appKey");
                String appVersion = hashMap.get("appVersion");
                String deviceId = hashMap.get("deviceId");
                String phone_no = hashMap.get("phone_no");
                Mail mail = new Mail(appKey, appVersion, deviceId, phone_no);
                // System.out.println(mail);
                return mail;
            }
        });
        dataStream.print();

        // sink
        String sql = "INSERT INTO testmaxwell1.ods_countlyV2 (appKey, appVersion, deviceId, phone_no) " +
                "VALUES (?, ?, ?, ?)";
        MyClickHouseUtil ckSink = new MyClickHouseUtil(sql);
        dataStream.addSink(ckSink);


        env.execute();
    }
}

2.工具类写入clickhouse

package com.kszx;

import com.kszx.Mail;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import ru.yandex.clickhouse.ClickHouseConnection;
import ru.yandex.clickhouse.ClickHouseDataSource;
import ru.yandex.clickhouse.settings.ClickHouseProperties;
import ru.yandex.clickhouse.settings.ClickHouseQueryParam;


import java.sql.PreparedStatement;
import java.util.HashMap;
import java.util.Map;

public class MyClickHouseUtil extends RichSinkFunction<Mail> {
    private ClickHouseConnection conn = null;

    String sql;

    public MyClickHouseUtil(String sql) {
        this.sql = sql;
    }

    @Override
    public void open(Configuration parameters) throws Exception {
        super.open(parameters);
        return ;
    }

    @Override
    public void close() throws Exception {
        super.close();
        if (conn != null)
        {
            conn.close();
        }
    }

    @Override
    public void invoke(Mail mail, Context context) throws Exception {

        String url = "jdbc:clickhouse://172.xx.xxx.xxx:8123/testmaxwell1";
        ClickHouseProperties properties = new ClickHouseProperties();
        properties.setUser("default");
        properties.setPassword("xxxxxx");
        properties.setSessionId("default-session-id2");

        ClickHouseDataSource dataSource = new ClickHouseDataSource(url, properties);
        Map<ClickHouseQueryParam, String> additionalDBParams = new HashMap<>();

        additionalDBParams.put(ClickHouseQueryParam.SESSION_ID, "new-session-id2");

        try {
            conn = dataSource.getConnection();
            PreparedStatement preparedStatement = conn.prepareStatement(sql);
            preparedStatement.setString(1,mail.getAppKey());
            preparedStatement.setString(2, mail.getAppVersion());
            preparedStatement.setString(3, mail.getDeviceId());
            preparedStatement.setString(4, mail.getPhone_no());

            preparedStatement.execute();
        }
        catch (Exception e){
            e.printStackTrace();
        }
    }
}

3.表属性类

package com.kszx;

//package com.demo.flink.pojo;

public class Mail {
    private String appKey;
    private String appVersion;
    private String deviceId;
    private String phone_no;

    public Mail(String appKey, String appVersion, String deviceId, String phone_no) {
        this.appKey = appKey;
        this.appVersion = appVersion;
        this.deviceId = deviceId;
        this.phone_no = phone_no;
    }

    public String getAppKey() {
        return appKey;
    }

    public void setAppKey(String appKey) {
        this.appKey = appKey;
    }

    public String getAppVersion() {
        return appVersion;
    }

    public void setAppVersion(String appVersion) {
        this.appVersion = appVersion;
    }

    public String getDeviceId() {
        return deviceId;
    }

    public void setDeviceId(String deviceId) {
        this.deviceId = deviceId;
    }

    public String getPhone_no() {
        return phone_no;
    }

    public void setPhone_no(String phone_no) {
        this.phone_no = phone_no;
    }

    @Override
    public String toString() {
        return "Mail{" +
                "appKey='" + appKey + '\'' +
                ", appVersion='" + appVersion + '\'' +
                ", deviceId='" + deviceId + '\'' +
                ", phone_no='" + phone_no + '\'' +
                '}';
    }

    public Mail of(String appKey, String appVersion, String deviceId, String phone_no)
    {
        return new Mail(appKey, appVersion, deviceId, phone_no);
    }
}

4.pom依赖(注意打包在服务器上运行时会和flink的lib目录下的log4j依赖冲突的问题，如果在服务器上执行jar包时依赖冲突报错的话，最好屏幕代码里的依赖，保留flink原版lib下的依赖)

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.kszx</groupId>
    <artifactId>flink1kc</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-java</artifactId>
            <version>1.11.1</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients_2.11</artifactId>
            <version>1.11.1</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-kafka_2.11</artifactId>
            <version>1.11.1</version>
        </dependency>



        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-api-java-bridge_2.11</artifactId>
            <version>1.11.1</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java_2.10</artifactId>
            <version>1.3.2</version>
        </dependency>

        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.59</version>
        </dependency>


        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka_2.11</artifactId>
            <version>1.0.2</version>
        </dependency>

        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka-clients</artifactId>
            <version>1.0.2</version>
        </dependency>

        <!-- 写入数据到clickhouse -->
        <dependency>
            <groupId>ru.yandex.clickhouse</groupId>
            <artifactId>clickhouse-jdbc</artifactId>
            <version>0.1.54</version>
        </dependency>




    </dependencies>




    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.8.1</version> <!-- 这里指定了插件版本 -->
                <configuration>
                    <!-- 在这里可以配置编译器参数 -->
                    <source>8</source>
                    <target>8</target>
                </configuration>
            </plugin>
            <!-- 可能还有其他插件配置 -->
        </plugins>
    </build>

</project>

#小技巧

查看flink正在执行的任务，如果提交作业后，再查看flink任务时，没有job正在运行，说明作业未成功运行，看报错或看日志。具体在哪里出问题就在那个范围内解决

./flink list

如果配置了flink的环境变量的话，可以直接执行下面的命令进入flink目录

cd $FLINK_HOME

查看flink版本，能及时查看flink环境变量是否生效

flink -v

longlongbreak

关注

8
点赞
踩
5

收藏

觉得还不错? 一键收藏
1
评论
使用flink的standalone模式同步Kafka的数据到clickhouse

standalone的应用模式下不会提前创建集群，所以不能调用 start-cluster.sh 脚本。我们可以使用同样在 bin 目录下的 standalone-job.sh 来创建一个 JobManager。把写好的jar包（胖包）放在flink的lib目录下，flink启动时会自动扫描lib目录，注意log4j依赖冲突问题，改屏蔽的依赖在打包时要屏蔽。（不用指定jar包，启动脚本会自动扫描 lib 目录下所有的jar 包，这里直接指定作业入口类）6.如果希望停掉集群，同样可以使用脚本，命令如下。
复制链接

扫一扫