2024年最新使用flink的standalone模式同步Kafka的数据到clickhouse，2024年最新【大牛系列教学

疯狂的石头。

于 2024-05-14 11:16:51 发布

阅读量283

点赞数 3

分类专栏：程序员文章标签：大数据面试学习

本文链接：https://blog.csdn.net/pasehelpme/article/details/138846765

版权

程序员专栏收录该内容

186 篇文章 1 订阅

订阅专栏

网上学习资料一大堆，但如果学到的知识不成体系，遇到问题时只是浅尝辄止，不再深入研究，那么很难做到真正的技术提升。

需要这份系统化资料的朋友，可以戳这里获取

一个人可以走的很快，但一群人才能走的更远！不论你是正从事IT行业的老鸟或是对IT行业感兴趣的新人，都欢迎加入我们的的圈子（技术交流、学习资源、职场吐槽、大厂内推、面试辅导），让我们一起学习成长！

    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.enableCheckpointing(5000);
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

    // source
    String topic = "topic_test2";

    Properties props = new Properties();
    // 设置连接kafka集群的参数
    props.setProperty("bootstrap.servers", "172.xx.xxx.x:9092,172.xx.xxx.x:9092,172.xx.xxx.x:9092");

    // 定义Flink Kafka Consumer
    FlinkKafkaConsumer<String> consumer = new FlinkKafkaConsumer<String>(topic, new SimpleStringSchema(), props);

    consumer.setStartFromGroupOffsets();
    consumer.setStartFromEarliest();    // 设置每次都从头消费

    // 添加source数据流
    DataStreamSource<String> source = env.addSource(consumer);
    source.print("111");
    System.out.println(source);
    SingleOutputStreamOperator<Mail> dataStream = source.map(new MapFunction<String, Mail>() {
        @Override
        public Mail map(String value) throws Exception {
            HashMap<String, String> hashMap = JSON.parseObject(value, HashMap.class);
            // System.out.println(hashMap);
            String appKey = hashMap.get("appKey");
            String appVersion = hashMap.get("appVersion");
            String deviceId = hashMap.get("deviceId");
            String phone_no = hashMap.get("phone_no");
            Mail mail = new Mail(appKey, appVersion, deviceId, phone_no);
            // System.out.println(mail);
            return mail;
        }
    });
    dataStream.print();

    // sink
    String sql = "INSERT INTO testmaxwell1.ods_countlyV2 (appKey, appVersion, deviceId, phone_no) " +
            "VALUES (?, ?, ?, ?)";
    MyClickHouseUtil ckSink = new MyClickHouseUtil(sql);
    dataStream.addSink(ckSink);


    env.execute();
}

}


2.工具类写入clickhouse

package com.kszx;

import com.kszx.Mail;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import ru.yandex.clickhouse.ClickHouseConnection;
import ru.yandex.clickhouse.ClickHouseDataSource;
import ru.yandex.clickhouse.settings.ClickHouseProperties;
import ru.yandex.clickhouse.settings.ClickHouseQueryParam;

import java.sql.PreparedStatement;
import java.util.HashMap;
import java.util.Map;

public class MyClickHouseUtil extends RichSinkFunction {
private ClickHouseConnection conn = null;

String sql;

public MyClickHouseUtil(String sql) {
    this.sql = sql;
}

@Override
public void open(Configuration parameters) throws Exception {
    super.open(parameters);
    return ;
}

@Override
public void close() throws Exception {
    super.close();
    if (conn != null)
    {
        conn.close();
    }
}

@Override
public void invoke(Mail mail, Context context) throws Exception {

    String url = "jdbc:clickhouse://172.xx.xxx.xxx:8123/testmaxwell1";
    ClickHouseProperties properties = new ClickHouseProperties();
    properties.setUser("default");
    properties.setPassword("xxxxxx");
    properties.setSessionId("default-session-id2");

    ClickHouseDataSource dataSource = new ClickHouseDataSource(url, properties);
    Map<ClickHouseQueryParam, String> additionalDBParams = new HashMap<>();

    additionalDBParams.put(ClickHouseQueryParam.SESSION_ID, "new-session-id2");

    try {
        conn = dataSource.getConnection();
        PreparedStatement preparedStatement = conn.prepareStatement(sql);
        preparedStatement.setString(1,mail.getAppKey());
        preparedStatement.setString(2, mail.getAppVersion());
        preparedStatement.setString(3, mail.getDeviceId());
        preparedStatement.setString(4, mail.getPhone_no());

        preparedStatement.execute();
    }
    catch (Exception e){
        e.printStackTrace();
    }
}

}


3.表属性类

package com.kszx;

//package com.demo.flink.pojo;

public class Mail {
private String appKey;
private String appVersion;
private String deviceId;
private String phone_no;

public Mail(String appKey, String appVersion, String deviceId, String phone_no) {
    this.appKey = appKey;
    this.appVersion = appVersion;
    this.deviceId = deviceId;
    this.phone_no = phone_no;
}

public String getAppKey() {
    return appKey;
}

public void setAppKey(String appKey) {
    this.appKey = appKey;
}

public String getAppVersion() {
    return appVersion;
}

public void setAppVersion(String appVersion) {
    this.appVersion = appVersion;
}

public String getDeviceId() {
    return deviceId;
}

public void setDeviceId(String deviceId) {
    this.deviceId = deviceId;
}

public String getPhone_no() {
    return phone_no;
}

public void setPhone_no(String phone_no) {
    this.phone_no = phone_no;
}

@Override
public String toString() {
    return "Mail{" +
            "appKey='" + appKey + '\'' +
            ", appVersion='" + appVersion + '\'' +
            ", deviceId='" + deviceId + '\'' +
            ", phone_no='" + phone_no + '\'' +
            '}';
}

public Mail of(String appKey, String appVersion, String deviceId, String phone_no)
{
    return new Mail(appKey, appVersion, deviceId, phone_no);
}

}


4.pom依赖(注意打包在服务器上运行时会和flink的lib目录下的log4j依赖冲突的问题，如果在服务器上执行jar包时依赖冲突报错的话，最好屏幕代码里的依赖，保留flink原版lib下的依赖)

<?xml version="1.0" encoding="UTF-8"?>

4.0.0

<groupId>com.kszx</groupId>
<artifactId>flink1kc</artifactId>
<version>1.0-SNAPSHOT</version>

<properties>
    <maven.compiler.source>8</maven.compiler.source>
    <maven.compiler.target>8</maven.compiler.target>
</properties>

<dependencies>
    <dependency>
        <groupId>org.apache.flink</groupId>
        <artifactId>flink-java</artifactId>
        <version>1.11.1</version>
    </dependency>

    <dependency>
        <groupId>org.apache.flink</groupId>
        <artifactId>flink-clients_2.11</artifactId>
        <version>1.11.1</version>
    </dependency>

    <dependency>
        <groupId>org.apache.flink</groupId>
        <artifactId>flink-connector-kafka_2.11</artifactId>
        <version>1.11.1</version>
    </dependency>



    <dependency>
        <groupId>org.apache.flink</groupId>
        <artifactId>flink-table-api-java-bridge_2.11</artifactId>
        <version>1.11.1</version>
    </dependency>

    <dependency>
        <groupId>org.apache.flink</groupId>
        <artifactId>flink-streaming-java_2.10</artifactId>
        <version>1.3.2</version>
    </dependency>

    <dependency>
        <groupId>com.alibaba</groupId>
        <artifactId>fastjson</artifactId>
        <version>1.2.59</version>
    </dependency>


    <dependency>
        <groupId>org.apache.kafka</groupId>
        <artifactId>kafka_2.11</artifactId>
        <version>1.0.2</version>
    </dependency>

    <dependency>
        <groupId>org.apache.kafka</groupId>
        <artifactId>kafka-clients</artifactId>
        <version>1.0.2</version>
    </dependency>

    <!-- 写入数据到clickhouse -->
    <dependency>
        <groupId>ru.yandex.clickhouse</groupId>
        <artifactId>clickhouse-jdbc</artifactId>
        <version>0.1.54</version>
    </dependency>




</dependencies>




<build>
    <plugins>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>

网上学习资料一大堆，但如果学到的知识不成体系，遇到问题时只是浅尝辄止，不再深入研究，那么很难做到真正的技术提升。

需要这份系统化资料的朋友，可以戳这里获取

-z58reVb9-1715656589357)]
[外链图片转存中…(img-twUj61Pl-1715656589357)]

网上学习资料一大堆，但如果学到的知识不成体系，遇到问题时只是浅尝辄止，不再深入研究，那么很难做到真正的技术提升。

需要这份系统化资料的朋友，可以戳这里获取

疯狂的石头。

关注

3
点赞
踩
3

收藏

觉得还不错? 一键收藏
0
评论
2024年最新使用flink的standalone模式同步Kafka的数据到clickhouse，2024年最新【大牛系列教学

网上学习资料一大堆，但如果学到的知识不成体系，遇到问题时只是浅尝辄止，不再深入研究，那么很难做到真正的技术提升。需要这份系统化资料的朋友，可以戳这里获取一个人可以走的很快，但一群人才能走的更远！不论你是正从事IT行业的老鸟或是对IT行业感兴趣的新人，都欢迎加入我们的的圈子（技术交流、学习资源、职场吐槽、大厂内推、面试辅导），让我们一起学习成长！}package com.kszx;import com.kszx.Mail;import org.apache.flink.configuration.Con
复制链接

扫一扫