flink read 带Kerberos的hbase

最新推荐文章于 2023-12-30 13:49:30 发布
最新推荐文章于 2023-12-30 13:49:30 发布
阅读量538 收藏 3
点赞数
分类专栏： flink hbase 文章标签： flink hbase big data
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/qq_34009542/article/details/120676298
版权

package com.hx.test;

import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.security.UserGroupInformation;

import java.io.IOException;
import java.util.Iterator;


public class FlinkReadHbase {

    public static void main(String[] args) throws Exception{

        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(3,1000));

        System.setProperty("java.security.auth.login.config","D:\\workspace\\flink2doris\\src\\main\\resources\\kafka_client_jaas.conf");
        System.setProperty("java.security.krb5.conf","D:\\workspace\\flink2doris\\src\\main\\resources\\krb5.ini");
        System.setProperty("javax.security.auth.useSubjectCredsOnly", "false");

        DataStream<Tuple2<String, String>> result = env.addSource(new HBaseReader());

        result.print();
        env.execute();
    }

    public static class HBaseReader extends RichSourceFunction<Tuple2<String, String>> {

        private Connection conn = null;
        private Table table = null;
        private Scan scan = null;

        @Override
        public void open(Configuration parameters) throws Exception {
            super.open(parameters);
            org.apache.hadoop.conf.Configuration conf = HBaseConfiguration.create();
            conf.set("hbase.zookeeper.quorum", "cdp-kfk01.hx.tc,cdp-kfk02.hx.tc,cdp-kfk03.hx.tc");
            conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, "2181"); //2181
            conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/hbase"); //hbase

            //        认证需要的参数
//        hbase authentication is kerberos ,rpc is privacy

            //
            conf.set("hadoop.security.authentication", "kerberos");
            conf.set("hbase.security.authentication","kerberos");
            conf.set("hbase.rpc.protection", "privacy");

            // 指定kerberos配置参数
            conf.set("keytab.file", "D:/soft/kerbros/hbase.keytab");
            conf.set("hbase.master.kerberos.principal", "hbase/_HOST@hx.TC");
            conf.set("hbase.regionserver.kerberos.principal", "hbase/_HOST@hx.TC");
            conf.set("kerberos.principal", "hbase/cdp-master01.hx.tc@hx.TC");

            // 设置配置文件信息
            UserGroupInformation.setConfiguration(conf);
            // 通过keytab登录用户
            System.out.println("--------------> 开始进行keytab认证 <--------------");
            UserGroupInformation.loginUserFromKeytab("hbase/cdp-master01.hx.tc@hx.TC", "D:/soft/kerbros/hbase.keytab");
            System.out.println("--------------> keytab认证结束 <--------------");


            conn = ConnectionFactory.createConnection(conf);
            table = conn.getTable(TableName.valueOf("ods_sr_prd:tbl_store"));
            scan = new Scan();

            scan.addFamily(Bytes.toBytes("cf"));
            scan.withStartRow(Bytes.toBytes("1"));
            scan.withStopRow(Bytes.toBytes("1"),true);


        }

        @Override
        public void run(SourceContext<Tuple2<String, String>> ctx) throws Exception {
            ResultScanner rs = table.getScanner(scan);
            Iterator<Result> iterator = rs.iterator();
            while (iterator.hasNext()) {
                Result result = iterator.next();
                String rowkey = Bytes.toString(result.getRow());
                StringBuffer sb = new StringBuffer();
                for (Cell cell : result.listCells()) {
                    String value = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
                    sb.append(value).append(",");
                }
                String valueString = sb.replace(sb.length() - 1, sb.length(), "").toString();
                Tuple2<String, String> tuple2 = new Tuple2<>();
                tuple2.setFields(rowkey, valueString);
                ctx.collect(tuple2);
            }

        }

        @Override
        public void cancel() {
            try {
                if (table != null) {
                    table.close();
                }
                if (conn != null) {
                    conn.close();
                }
            } catch (IOException e) {
                System.out.println("Close HBase Exception666:"+ e.toString());
            }

        }
    }


}


-------------------------------------------------pom 文件---------------------------------------------

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.com.hx</groupId>
    <artifactId>flink2doris</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
        <CodeCacheSize>512m</CodeCacheSize>
        <flink.version>1.13.2</flink.version>
<!--        <flink.version>1.12.1</flink.version>-->
        <encoding>UTF-8</encoding>
        <java.version>1.8</java.version>
        <scala.binary.version>2.11</scala.binary.version>
        <maven.compiler.target>${java.version}</maven.compiler.target>
        <maven.compiler.source>${java.version}</maven.compiler.source>
        <log4j.version>1.2.17</log4j.version>
<!--        <hadoop.version>2.7.3</hadoop.version>-->
        <hbase.version>2.1.0-cdh6.3.2</hbase.version>
        <hadoop.version>3.0.0-cdh6.3.2</hadoop.version>
    </properties>


    <dependencies>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-java</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java_2.11</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients_2.11</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <!-- Using Flink Table & SQL -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-planner_2.11</artifactId>
            <version>${flink.version}</version>
        </dependency>


        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-planner-blink_2.11</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <!-- Either... -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-api-java-bridge_2.11</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-common</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table</artifactId>
            <version>${flink.version}</version>
            <type>pom</type>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-json</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-csv</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-kafka_2.11</artifactId>
            <version>1.13.2</version>
        </dependency>

<!--        sink to redis -->
        <dependency>
            <groupId>org.apache.bahir</groupId>
            <artifactId>flink-connector-redis_2.11</artifactId>
            <version>1.0</version>
        </dependency>

        <!--        sink to es -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-elasticsearch6_2.11</artifactId>
            <version>1.13.2</version>
        </dependency>

        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.47</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>${hadoop.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>${hadoop.version}</version>
            <!--将netty包排除-->
            <exclusions>
                <exclusion>
                    <groupId>io.netty</groupId>
                    <artifactId>netty</artifactId>
                </exclusion>
            </exclusions>
        </dependency>

        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.62</version>
        </dependency>

       <!-- hbase -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-hbase-base_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
            <exclusions>
                <exclusion>
                    <groupId>hadoop-hdfs</groupId>
                    <artifactId>org.apache.hadoop</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>hadoop-common</groupId>
                    <artifactId>org.apache.hadoop</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>org.apache.hadoop</groupId>
                    <artifactId>hadoop-auth</artifactId>
                </exclusion>
                <exclusion>
                    <artifactId>hbase-protocol</artifactId>
                    <groupId>org.apache.hbase</groupId>
                </exclusion>
                <exclusion>
                    <artifactId>hbase-prefix-tree</artifactId>
                    <groupId>org.apache.hbase</groupId>
                </exclusion>
                <exclusion>
                    <artifactId>hbase-server</artifactId>
                    <groupId>org.apache.hbase</groupId>
                </exclusion>
            </exclusions>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-hbase_2.11</artifactId>
            <version>1.10.3</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hbase</groupId>
            <artifactId>hbase-client</artifactId>
            <version>${hbase.version}</version>
            <exclusions>
                <exclusion>
                    <groupId>hadoop-common</groupId>
                    <artifactId>org.apache.hadoop</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>hadoop-auth</groupId>
                    <artifactId>org.apache.hadoop</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <version>RELEASE</version>
            <scope>compile</scope>
        </dependency>

    </dependencies>

    <repositories>
        <repository>
            <id>cloudera</id>
            <url>https://mvnrepository.com/artifact/org.apache.flink/flink-connector-kafka-0.11</url>
            <releases>
                <enabled>true</enabled>
            </releases>
            <snapshots>
                <enabled>false</enabled>
            </snapshots>
        </repository>
<!--        <repository>-->
<!--            <id>cloudera</id>-->
<!--            <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>-->
<!--            <releases>-->
<!--                <enabled>true</enabled>-->
<!--            </releases>-->
<!--            <snapshots>-->
<!--                <enabled>false</enabled>-->
<!--            </snapshots>-->
<!--        </repository>-->
    </repositories>

    <build>
        <sourceDirectory>src/main/java</sourceDirectory>

        <plugins>

            <plugin>
                <groupId>net.alchim31.maven</groupId>
                <artifactId>scala-maven-plugin</artifactId>
                <version>3.2.2</version>
                <executions>
                    <execution>
                        <goals>
                            <goal>compile</goal>
                            <goal>testCompile</goal>
                        </goals>
                        <configuration>
                            <args>
                                <arg>-dependencyfile</arg>
                                <arg>${project.build.directory}/.scala_dependencies</arg>
                            </args>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
            <plugin>

                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-shade-plugin</artifactId>
                <version>3.1.0</version>
                <executions>

                    <execution>

                        <phase>package</phase>
                        <goals>
                            <goal>shade</goal>
                        </goals>
                        <configuration>
                            <transformers>
                                <transformer
                                        implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
                            </transformers>
                            <relocations>
                                <relocation>
                                    <pattern>org.apache.http</pattern>
                                    <shadedPattern>org.apache.myhttp</shadedPattern>
                                </relocation>
                            </relocations>
                            <filters>
                                <filter>
                                    <artifact>*:*</artifact>
                                    <excludes>
                                        <exclude>META-INF/maven/**</exclude>
                                        <exclude>META-INF/*.SF</exclude>
                                        <exclude>META-INF/*.DSA</exclude>
                                        <exclude>META-INF/*.RSA</exclude>
                                    </excludes>
                                </filter>
                            </filters>
                        </configuration>
                    </execution>
                </executions>
            </plugin>

            <plugin>
                <groupId>org.codehaus.mojo</groupId>
                <artifactId>exec-maven-plugin</artifactId>
                <version>1.2.1</version>
                <executions>
                    <execution>
                        <goals>
                            <goal>exec</goal>
                        </goals>
                    </execution>
                </executions>
                <configuration>
                    <executable>java</executable>
                    <includeProjectDependencies>true</includeProjectDependencies>
                    <includePluginDependencies>false</includePluginDependencies>
                    <classpathScope>compile</classpathScope>
                    <mainClass></mainClass>
                </configuration>
            </plugin>

            <!--指定各种jdk等级的插件-->
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                </configuration>
            </plugin>
        </plugins>
    </build>

</project>

辉哥之吞并BAT
关注
  • 0
    点赞
  • 踩
  • 3
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
专栏目录
flink写入带kerberos认证的kudu connector
03-24
flink写入带kerberos认证的kudu connector
Flink在开启kerberos认证环境中，使用hbase进行维表关联
wpf_ysztqx的博客
08-06 2298
Flink在开启kerberos认证环境中，使用hbase进行维表关联 1.简介 ​ 近期在项目中用到了使用hbase做为维表，进行信息补充的一个需求，同时本公司的环境均开启了kerberos，遇到如下问题进行记录，同时分享完整kerberos认证步骤 2.环境介绍 组件 版本 flink 1.7.0 hdp 3.1.0.0 3.连接hbase的RichMap方法 连接具有kerberos环境的hbase核心代码如下： public class HbaseRichMap
Java 板 flink插入带Kerberos认证的hbase
qq_34009542的博客
10-09 782
----------直接上代码----------------- package com.hx.test; import com.hx.conf.HBaseConfig; import com.hx.test.model.IdTimeTemperature; import com.hx.utils.HBaseUtil; import org.apache.flink.api.common.functions.RichMapFunction; import org.apache.flink.api.com
Spark on Yarn与Flink on Yarn连接kerberos认证的Hbase
jorondo的博客
03-19 1653
Spark和Flink均为分布式计算引擎，在使用yarn作为资源调度器提交任务并且连接拥有Kerberos认证的Hbase时，同时面临着认证文件分发与获取的问题。两者的解决方案也是类似的，现在driver端加载认证文件，存储到分布式缓存，然后再Executor端获取文件目录进行认证. Spark on Yarn driver端 sparkSession.sparkContext.addFi...
Flink异步IO : 访问hbase遇到的kerberos验证问题
雾岛与鲸的博客
12-21 1123
问题描述: 使用异步IO 访问hbase, hbase需要kerberos验证，kerberos验证的时候，需要把kerberos验证文件加载到分布式缓存中，但是flink异步IO不支持访问分布式缓存，报错信息如下: 好了，不说废话，直接上解决方案: 在执行异步io操作之前，使用map方法构建对hbase的连接，使用静态变量，创建的hbase连接存储在内存中，后续使用hbase客户端的操作可以直接从taskManager的内存中去获取对应的连接即可 主要测试代码: import java.util.Pro
FlinkSQL实战：HBase的结合应用
02-23
而当下FlinkSQL的火热程度不用多说，FlinkSQL也为HBase提供了connector，因此HBase与FlinkSQL的结合非常有必要实践实践。当然，本文假设用户有一定的HBase知识基础，不会详细去介绍HBase的架构和原理，本文着重介绍...
基于Flink+SpringBoot+Hbase的商品实时推荐系统源码+全部资料齐全
最新发布
05-13
基于Flink+SpringBoot+Hbase的商品实时推荐系统源码+全部资料齐全 flink统计商品热度，放入redis缓存，分析日志信息，将画像标签和实时记录放入Hbase在用户发起推荐请求后，根据用户画像重排序热度榜，并结合协同...
flink-kafka-hbase:功能：实现kafka消息实时落地hbase，支持csvjson字符串两种格式的消息，支持自定义组合rowkey,列簇和列名，支持按照kafka消息流中不同字段join不同的hbase表，并自定义写入列簇和列(join时需评估一下性能)， 支持at least once语义 外部依赖：apollo配置中心，本项目依靠配置驱动，配置存储在apollo配置中心
05-26
flink-kafka-hbase功能：实现kafka消息实时落地hbase，支持csv/json字符串两种格式的消息，支持自定义组合rowkey,列簇和列名，支持按照kafka消息流中不同字段join不同的hbase表，并自定义写入列簇和列(join时需评估...
Flink读写Kerberos认证Kafka环境。
peidezhi的专栏
04-07 5926
Flink比较完美的支持kerberos读写Kafka. 1flink-conf.yaml , 将如下涉及kerberos的配置注解去除 2 代码读写kafka示例如下： def main(args: Array[String]): Unit = { val params: ParameterTool = ParameterTool.fromArgs(args)...
Flink---hbase-sink支持kerberos认证1
旺旺的博客
06-16 2640
1.介绍 项目中需要用到Flink往hbase写数据，但是默认的FlinkUpsertSinkFunction不支持kerberos认证，不支持就无法往hbase插入数据。 在原有的HBaseUpsertSinkFunction的基础上修改 增加kerberos认证，以下是修改好之后的代码，主要是修改了open方法，判断是否需要kerberos认证，如果需要认证则进行kerberos认证，如果不需要则不认证。jsonObject是kerberos的配置信息，keytab文件，kerberos账号这些信息
flink kerberos
dian张
11-13 1615
security.kerberos.login.keytab: /usr/local/keytab/flink/flink.keytab security.kerberos.login.principal: flink/master@AQ.COM security.kerberos.login.contexts: Client,KafkaClient security.kerberos.login...
《中级》Flink 读写hbase
huzechen的博客
08-29 1229
ReadFromHBase import lombok.extern.slf4j.Slf4j; import org.apache.flink.addons.hbase.TableInputFormat; import org.apache.flink.api.java.DataSet; import org.apache.flink.api.java.ExecutionEnvironment...
flink-sql-1.15.4配置hive catalog和hbase基于CDH6.3.2和kerberos
h952520296的博客
02-02 1268
使用flink源码重新编译生成flink-sql-connector-hive-2.2.0_2.11，编译之前需要修改flink-sql-connector-hive-2.2.0中的pom文件，将hive-exec的版本改成2.1.1-cdh6.3.2。位置在flink-connectors/flink-sql-connector-hive-2.2.0/flink-sql-connector-hive-2.2.0_2.12-1.15.3.jar。有大佬能解决望赐教。java 我的版本和cdh使用的一致。
Flink连接Hbase
lck_csdn的博客
05-10 2056
这里写自定义目录标题1.添加依赖2.Hbase连接工具类3.执行读取Hbase数据4.错误日志，及解决方式4.1AccessDeniedException4.2NoSuchColumnFamilyException 1.添加依赖 <properties> <flink.version>1.13.6</flink.version> <java.version>1.8</java.version> &lt
flink kerberos认证源码剖析
阿甘兄
09-07 559
超详细讲解flink kerberos认证
flink实战（一） flink-sql关联hbase维度数据处理
zhaocuit的博客
07-04 4999
概述 最近项目中用到flink进行实时计算，流程为从kafka读取数据，如果是维度数据，则插入到hbase中，如果是需要实时计算的数据， 则进行实时计算，并将计算结果保存到MySQL中。在实时计算过程中，可能会用到hbase中的维度数据，为了开发的效率，使用flink-sql的方式实现。 flink-sql是在flink流式计算的基础上进行了高度抽象，使开发过程更简单，更有效率，但要理解sql执行背后的原理还是需要 仔细学习flink流式计算的相关内容。本文主要以flink-sql实现相关功能。 假设需求如
Flink 内容分享(七)：Flink 读写 HBase 总结
之乎者也·的博客
12-30 1856
总结 Flink 读写 HBase本文总结了Flink SQL 读写 HBase 的参数配置，解决了在kerberos环境下因 hudi 包 hbase-site.xml 配置冲突引起的异常，学习总结了 Flink SQL 读写 HBase 时加载 HBase 配置的优先级，但是没有详细的分析源码中的逻辑，可能会在后面的文章中补充相关的源码分析~
【Flink】Flink跨集群访问开启Kerberos认证的Kafka
九师兄
07-30 2704
Flink提供了三个模块来对集群进行安全验证，分别是HadoopModule、JaasModule、ZooKeeperModule。安全认证相关参数对应的类SecurityOptions。HadoopModule用来对使用UserGroupInformation进行身份验证的框架（kudu、hbase同步框架、hdfs等）进行认证配置。JaasModule用来对使用JaasConfig进行身份验证的框架（kafka、zk、hbase异步框架等）进行认证配置。......
flink程序设置kerberos认证——专业实用版
ilyomind的博客
02-19 4547
flink程序提交命令 flink run -d -m yarn-cluster \ -yD security.kerberos.login.keytab=/path/your/user.keytab \ -yD security.kerberos.login.principal=your@HADOOP.COM \ -c com.your.class.Name flink on yarn认证原理 将keytab文件自动上传hdfs，由yarn管理，分发给每个executor缓存token，定
flink读取kafka写到kafka
09-11
可以使用Apache Flink来读取Kafka消息并将其写回Kafka。以下是基本的代码示例： 首先，您需要在项目中添加Flink和Kafka的依赖包。这里假设您已经设置好了Maven项目。 ```xml <dependencies> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-streaming-java_2.12</artifactId> <version>${flink.version}</version> </dependency> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-connector-kafka_2.12</artifactId> <version>${flink.version}</version> </dependency> </dependencies> ``` 确保`${flink.version}`是您要使用的Flink版本。 接下来，您可以使用以下代码片段来实现从Kafka读取消息并将其写回Kafka： ```java import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer; import org.apache.flink.streaming.util.serialization.SimpleStringSchema; import java.util.Properties; public class FlinkKafkaExample { public static void main(String[] args) throws Exception { // 设置执行环境 StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); // 设置Kafka消费者的配置 Properties consumerProps = new Properties(); consumerProps.setProperty("bootstrap.servers", "localhost:9092"); consumerProps.setProperty("group.id", "flink-consumer"); // 从Kafka读取消息 FlinkKafkaConsumer<String> kafkaConsumer = new FlinkKafkaConsumer<>("input-topic", new SimpleStringSchema(), consumerProps); DataStream<String> inputDataStream = env.addSource(kafkaConsumer); // 设置Kafka生产者的配置 Properties producerProps = new Properties(); producerProps.setProperty("bootstrap.servers", "localhost:9092"); // 将消息写回Kafka FlinkKafkaProducer<String> kafkaProducer = new FlinkKafkaProducer<>("output-topic", new SimpleStringSchema(), producerProps); inputDataStream.addSink(kafkaProducer); // 执行任务 env.execute("Flink Kafka Example"); } } ``` 请注意，上述代码中的`input-topic`和`output-topic`需要替换为您实际使用的Kafka主题名称。您还可以根据需要调整其他Kafka和Flink配置。 希望这能帮助您开始在Flink中读取Kafka消息并将其写回Kafka。如有任何问题，请随时提问！

“相关推荐”对你有帮助么？

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值