flink,pulsar异步访问redis,并将数据写入Pg

最新推荐文章于 2024-08-27 08:46:13 发布
最新推荐文章于 2024-08-27 08:46:13 发布
阅读量525 收藏 2
点赞数
文章标签： flink
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/xiaoyan5686670/article/details/124171686
版权 
package Pulsar;

import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.connector.jdbc.JdbcConnectionOptions;
import org.apache.flink.connector.jdbc.JdbcSink;
import org.apache.flink.connector.pulsar.source.PulsarSource;
import org.apache.flink.connector.pulsar.source.enumerator.cursor.StartCursor;
import org.apache.flink.connector.pulsar.source.reader.deserializer.PulsarDeserializationSchema;
import org.apache.flink.streaming.api.CheckpointingMode;
import org.apache.flink.streaming.api.datastream.AsyncDataStream;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.CheckpointConfig;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.pulsar.client.api.SubscriptionType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.time.Duration;
import java.util.concurrent.TimeUnit;

/*********************************
 @Author:xiaoyan.qin
 @Description:
 @Date:Created in 18:05 2022/3/28
 @Modified By:
 **********************************/
public class test {
    private static Logger LOGGER = LoggerFactory.getLogger(test.class);

    public static void main(String[] args) throws Exception {

        StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(new Configuration());
        env.enableCheckpointing(300000);
        // 设置模式为精确一次 (这是默认值)
        env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
        // 确认 checkpoints 之间的时间会进行 500 ms
        env.getCheckpointConfig().setMinPauseBetweenCheckpoints(500);

        // Checkpoint 必须在一分钟内完成，否则就会被抛弃
        env.getCheckpointConfig().setCheckpointTimeout(60000);

        // 允许两个连续的 checkpoint 错误
        env.getCheckpointConfig().setTolerableCheckpointFailureNumber(2);

        // 同一时间只允许一个 checkpoint 进行
        env.getCheckpointConfig().setMaxConcurrentCheckpoints(1);

        env.getCheckpointConfig().setCheckpointStorage("file:///d/temp/checkpoint");


        // 使用 externalized checkpoints，这样 checkpoint 在作业取消后仍就会被保留
        env.getCheckpointConfig().setExternalizedCheckpointCleanup(
                CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);

        env.setRestartStrategy(RestartStrategies.fixedDelayRestart(
                3,
                Time.of(10, TimeUnit.SECONDS)
        ));
        StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);

        PulsarSource<String> pulsarSource = PulsarSource.builder()
                .setServiceUrl("pulsar://localhost:6650")
                .setAdminUrl("http://localhost:8080")
                .setStartCursor(StartCursor.latest())
                .setTopics("test")
                .setDeserializationSchema(PulsarDeserializationSchema.flinkSchema(new SimpleStringSchema()))
                .setSubscriptionName("my-subscription")
                .setSubscriptionType(SubscriptionType.Exclusive)
                .build();



        DataStream<String> stream = env.fromSource(pulsarSource, WatermarkStrategy.forMonotonousTimestamps(), "Pulsar Source");


        SingleOutputStreamOperator<String> resultStream=  AsyncDataStream.unorderedWait(stream, new AsyncRedis2("redis://localhost"), 500L, TimeUnit.MILLISECONDS)
                .setParallelism(1);

        resultStream.addSink(JdbcSink.sink(
                "insert into test (id) values (?)",
                (ps, t) -> {
                    ps.setInt(1,Integer.valueOf(t));

                },
                new JdbcConnectionOptions.JdbcConnectionOptionsBuilder()
                        .withUrl("jdbc:postgresql://localhost/postgres")
                        .withDriverName( "org.postgresql.Driver").withUsername("postgres").withPassword("mysecretpassword")
                        .build()));
//        Table inputTable = tEnv.fromDataStream(resultStream);
//        tEnv.createTemporaryView("InputTable", inputTable);
//        tEnv.sqlQuery("SELECT count(*) FROM InputTable").execute().print();
        //tEnv.from("InputTable").execute().print();
        resultStream.print();
       // stream.print();
        env.execute("qxy");
    }
}




package Pulsar;

import com.alibaba.fastjson.JSON;
import io.lettuce.core.RedisClient;
import io.lettuce.core.RedisFuture;
import io.lettuce.core.api.StatefulRedisConnection;
import io.lettuce.core.api.async.RedisAsyncCommands;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.async.ResultFuture;
import org.apache.flink.streaming.api.functions.async.RichAsyncFunction;

import java.util.Collections;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;


/*********************************
 @Author:xiaoyan.qin
 @Description:
 @Date:Created in 10:58 2022/4/1
 @Modified By:
 **********************************/
public class AsyncRedis2 extends RichAsyncFunction<String, String> {
    private RedisAsyncCommands<String,String> async;
    private String url;
    private StatefulRedisConnection<String,String> connection;
    private RedisClient redisClient;
    public AsyncRedis2(String url) {
        this.url = url;
    }
    @Override
    public void open(Configuration parameters) throws Exception{
        redisClient = RedisClient.create(url);
        connection = redisClient.connect();
        async  = connection.async();
    }

    @Override
    public void asyncInvoke(String input, ResultFuture<String> resultFuture) throws Exception {
        String imei = JSON.parseObject(input).get("eventId").toString();
        RedisFuture<String> redisFuture = async.hget("userlogin:"+imei,"context");
        CompletableFuture.supplyAsync(() -> {
            try {
                return redisFuture.get();
            } catch (InterruptedException e) {
                e.printStackTrace();
            } catch (ExecutionException e) {
                e.printStackTrace();
            }
            return "exception";
        }).thenAccept(result -> {
            if (result == null) {
                result = "nothing";
            }
            // return result
            resultFuture.complete(Collections.singleton( JSON.parseObject(result).get("userId").toString()));

        });
    }

    @Override
    public void close() throws Exception {
        super.close();
        if (connection != null) {
            connection.close();
        }
        if (redisClient != null) {
            redisClient.shutdown();
        }
    }

    @Override
    public void timeout(String input, ResultFuture<String> resultFuture) throws Exception {
        //超时了。丢弃即可
        System.out.println("超时了：" + input);
        //重新调用处理方法
      // asyncInvoke(input,resultFuture);

    }
}



<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
		 xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>

	<groupId>frauddetection</groupId>
	<artifactId>frauddetection</artifactId>
	<version>0.1</version>
	<packaging>jar</packaging>

	<name>Flink Walkthrough DataStream Java</name>
	<url>https://flink.apache.org</url>

	<properties>
		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
		<flink.version>1.14.3</flink.version>
		<target.java.version>1.8</target.java.version>
		<scala-version>2.11</scala-version>
		<java.version>1.8</java.version>
		<scala.bin.version>2.11</scala.bin.version>
		<scala.binary.version>2.11</scala.binary.version>
		<maven.compiler.source>${target.java.version}</maven.compiler.source>
		<maven.compiler.target>${target.java.version}</maven.compiler.target>
		<log4j.version>2.17.1</log4j.version>
	</properties>



	<dependencies>
		<!-- https://mvnrepository.com/artifact/com.alibaba/fastjson -->
		<dependency>
			<groupId>com.alibaba</groupId>
			<artifactId>fastjson</artifactId>
			<version>1.2.79</version>
		</dependency>
		<!-- https://mvnrepository.com/artifact/redis.clients/jedis -->
		<dependency>
			<groupId>redis.clients</groupId>
			<artifactId>jedis</artifactId>
			<version>3.8.0</version>
		</dependency>
		<dependency>
			<groupId>org.apache.flink</groupId>
			<artifactId>flink-connector-kafka_2.11</artifactId>
			<version>1.14.3</version>
		</dependency>

		<dependency>
			<groupId>org.apache.flink</groupId>
			<artifactId>flink-walkthrough-common_${scala.binary.version}</artifactId>
			<version>${flink.version}</version>
		</dependency>


		<!-- This dependency is provided, because it should not be packaged into the JAR file. -->
		<dependency>
			<groupId>org.apache.flink</groupId>
			<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
			<version>${flink.version}</version>
			<scope>compile</scope>
		</dependency>
		<!-- https://mvnrepository.com/artifact/org.apache.flink/flink-streaming-scala -->
		<dependency>
			<groupId>org.apache.flink</groupId>
			<artifactId>flink-streaming-scala_2.11</artifactId>
			<version>${flink.version}</version>
			<scope>compile</scope>

		</dependency>
		<!-- https://mvnrepository.com/artifact/org.apache.flink/flink-scala -->
		<dependency>
			<groupId>org.apache.flink</groupId>
			<artifactId>flink-scala_2.11</artifactId>
			<version>${flink.version}</version>
			<scope>compile</scope>
		</dependency>
		<dependency>
			<groupId>org.apache.flink</groupId>
			<artifactId>flink-table-planner_2.11</artifactId>
			<version>1.14.4</version>
			<scope>provided</scope>
		</dependency>
		<dependency>
			<groupId>org.apache.flink</groupId>
			<artifactId>flink-streaming-scala_2.11</artifactId>
			<version>1.14.4</version>
			<scope>provided</scope>
		</dependency>
		<dependency>
			<groupId>org.apache.flink</groupId>
			<artifactId>flink-table-api-java-bridge_2.11</artifactId>
			<version>1.14.4</version>
			<scope>provided</scope>
		</dependency>

		<dependency>
			<groupId>org.apache.flink</groupId>
			<artifactId>flink-clients_${scala.binary.version}</artifactId>
			<version>${flink.version}</version>
			<scope>compile</scope>

		</dependency>
		<!-- https://mvnrepository.com/artifact/org.apache.flink/flink-connector-pulsar -->
		<dependency>
			<groupId>org.apache.flink</groupId>
			<artifactId>flink-connector-pulsar_2.11</artifactId>
			<version>1.14.4</version>
		</dependency>

		<dependency>
			<groupId>org.apache.flink</groupId>
			<artifactId>flink-connector-jdbc_2.11</artifactId>
			<version>1.14.4</version>
		</dependency>

		<!-- https://mvnrepository.com/artifact/org.postgresql/postgresql -->
		<dependency>
			<groupId>org.postgresql</groupId>
			<artifactId>postgresql</artifactId>
			<version>42.3.3</version>
		</dependency>

		<!-- https://mvnrepository.com/artifact/org.apache.flink/flink-runtime-web-->
		<dependency>
			<groupId>org.apache.flink</groupId>
			<artifactId>flink-runtime-web_2.12</artifactId>
			<version>1.14.3</version>
			<scope>compile</scope>
		</dependency>

		<dependency>
			<groupId>io.lettuce</groupId>
			<artifactId>lettuce-core</artifactId>
			<version>5.0.5.RELEASE</version>
		</dependency>

		<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core 	-->
		<dependency>
			<groupId>org.apache.spark</groupId>
			<artifactId>spark-core_2.11</artifactId>
			<version>2.4.8</version>
		</dependency>

		<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql-->
		<dependency>
			<groupId>org.apache.spark</groupId>
			<artifactId>spark-sql_2.11</artifactId>
			<version>2.4.8</version>
			<scope>provided</scope>
		</dependency>


			<dependency>
				<groupId>com.redislabs</groupId>
				<artifactId>spark-redis_2.11</artifactId>
				<version>2.4.2</version>
			</dependency>



		<!-- Add connector dependencies here. They must be in the default scope (compile). -->

		<!-- Example:

		<dependency>
		    <groupId>org.apache.flink</groupId>
		    <artifactId>flink-connector-kafka_${scala.binary.version}</artifactId>
		    <version>${flink.version}</version>
		</dependency>
		-->

		<!-- Add logging framework, to produce console output when running in the IDE. -->
		<!-- These dependencies are excluded from the application JAR by default. -->
		<dependency>
			<groupId>org.apache.logging.log4j</groupId>
			<artifactId>log4j-slf4j-impl</artifactId>
			<version>${log4j.version}</version>
			<scope>compile</scope>
		</dependency>
		<dependency>
			<groupId>org.apache.logging.log4j</groupId>
			<artifactId>log4j-api</artifactId>
			<version>${log4j.version}</version>
			<scope>compile</scope>
		</dependency>
		<dependency>
			<groupId>org.slf4j</groupId>
			<artifactId>slf4j-log4j12</artifactId>
			<version>1.7.7</version>
			<scope>compile</scope>
		</dependency>
		<dependency>
			<groupId>org.apache.logging.log4j</groupId>
			<artifactId>log4j-core</artifactId>
			<version>${log4j.version}</version>
			<scope>compile</scope>
		</dependency>
		<dependency>
			<groupId>org.projectlombok</groupId>
			<artifactId>lombok</artifactId>
			<version>RELEASE</version>
			<scope>compile</scope>
		</dependency>
		<dependency>
			<groupId>junit</groupId>
			<artifactId>junit</artifactId>
			<version>RELEASE</version>
			<scope>test</scope>
		</dependency>

		<dependency>
			<groupId>org.apache.flink</groupId>
			<artifactId>flink-state-processor-api_2.11</artifactId>
			<version>1.14.4</version>
		</dependency>
		<!--
		<dependency>
			<groupId>org.apache.flink</groupId>
			<artifactId>flink-test-utils-junit</artifactId>
			<version>1.14.4</version>
			<scope>test</scope>
		</dependency>
		-->
	</dependencies>

	<build>
		<plugins>
			<plugin>
				<groupId>org.apache.maven.plugins</groupId>
				<artifactId>maven-shade-plugin</artifactId>
				<version>3.0.0</version>
				<executions>
					<!-- Run shade goal on package phase -->
					<execution>
						<phase>package</phase>
						<goals>
							<goal>shade</goal>
						</goals>
						<configuration>
							<artifactSet>
								<excludes>
									<exclude>org.apache.flink:flink-shaded-force-shading</exclude>
									<exclude>com.google.code.findbugs:jsr305</exclude>
									<exclude>org.slf4j:*</exclude>
									<exclude>org.apache.logging.log4j:*</exclude>
								</excludes>
							</artifactSet>
							<filters>
								<filter>
									<!-- Do not copy the signatures in the META-INF folder.
									Otherwise, this might cause SecurityExceptions when using the JAR. -->
									<artifact>*:*</artifact>
									<excludes>
										<exclude>META-INF/*.SF</exclude>
										<exclude>META-INF/*.DSA</exclude>
										<exclude>META-INF/*.RSA</exclude>
									</excludes>
								</filter>
							</filters>
							<transformers>
								<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
									<mainClass>spendreport.FraudDetectionJob</mainClass>
								</transformer>
							</transformers>
						</configuration>
					</execution>
				</executions>
			</plugin>


			<!-- Java Compiler -->
			<plugin>
				<groupId>org.apache.maven.plugins</groupId>
				<artifactId>maven-compiler-plugin</artifactId>
				<version>3.1</version>
				<configuration>
					<source>${target.java.version}</source>
					<target>${target.java.version}</target>
				</configuration>
			</plugin>
			<!-- Scala Compiler -->
			<plugin>
				<groupId>net.alchim31.maven</groupId>
				<artifactId>scala-maven-plugin</artifactId>
				<version>3.2.2</version>
				<executions>
					<execution>
						<goals>
							<goal>compile</goal>
							<goal>testCompile</goal>
						</goals>
					</execution>
				</executions>
				<configuration>
					<args>
						<arg>-nobootcp</arg>
					</args>
				</configuration>
			</plugin>
			<!-- We use the maven-shade plugin to create a fat jar that contains all necessary dependencies. -->
			<!-- Change the value of <mainClass>...</mainClass> if your program entry point changes. -->

		</plugins>

		<pluginManagement>
			<plugins>

				<!-- This improves the out-of-the-box experience in Eclipse by resolving some warnings. -->
				<plugin>
					<groupId>org.eclipse.m2e</groupId>
					<artifactId>lifecycle-mapping</artifactId>
					<version>1.0.0</version>
					<configuration>
						<lifecycleMappingMetadata>
							<pluginExecutions>
								<pluginExecution>
									<pluginExecutionFilter>
										<groupId>org.apache.maven.plugins</groupId>
										<artifactId>maven-shade-plugin</artifactId>
										<versionRange>[3.0.0,)</versionRange>
										<goals>
											<goal>shade</goal>
										</goals>
									</pluginExecutionFilter>
									<action>
										<ignore/>
									</action>
								</pluginExecution>
								<pluginExecution>
									<pluginExecutionFilter>
										<groupId>org.apache.maven.plugins</groupId>
										<artifactId>maven-compiler-plugin</artifactId>
										<versionRange>[3.1,)</versionRange>
										<goals>
											<goal>testCompile</goal>
											<goal>compile</goal>
										</goals>
									</pluginExecutionFilter>
									<action>
										<ignore/>
									</action>
								</pluginExecution>
							</pluginExecutions>
						</lifecycleMappingMetadata>
					</configuration>
				</plugin>
			</plugins>
		</pluginManagement>
	</build>
</project>

xiaoyan5686670
关注
flink将数据写入redis
congge_study的博客
01-13 5377
flink将数据写入redis
Flink 高级数据处理：打造定制化的反序列化器
击水三千里的专栏
09-08 1070
Flink自定义反序列化器
Flink通过异步IO实现redis维表join
u013939918的博客
06-30 1663
使用flink做实时数仓的公司越来越多了，浪尖这边也是很早就开发了一个flink 全sql平台来实现实时数仓的功能。说到实时数仓，两个表的概念大家一定会知道的：事实表和维表。 在实时数仓中，事实表就是flink消费的kafka的topic数据流，而维表和离线数仓一样，就是mysql等外部存储的维表。 当flink 事实表需要 使用维表来进行染色的时候，就需要flink 与维表进行join，这是需要注意与外部系统的通信延迟不会影响流应用程序的整体工作。 直接访问外部数据库中的数据，例如在MapFunct
记flink sql cdc消费kafka数据，sink到pg表数据漏数原因排查
a123147abc的博客
12-16 2007
记flink sql cdc消费kafka数据，sink到pg表数据漏数排查[原因分析] flink sql insert into t3 select cast(t1.a as varchar(100)) || cast(t1.b as varchar(100)) as id, -- 组合主键id t1.a ,t1.b ,t2.c ,sum(d) from t1 join t2 on t2.id = t1.id group by t1.a,t1.b,t2.c 现象与环境描述 1、flink
Flink教程-将流式数据写入redis
公众号[大数据技术与应用实战]，分享大数据实战案例
08-19 1756
redis作为一个高吞吐的存储系统，在生产中有着广泛的应用，今天我们主要讲一下如何将流式数据写入redis，以及遇到的一些问题 解决。官方并没有提供写入redis的connector，所以我们采用apache的另一个项目bahir-flink [1]中提供的连接器来实现。
Flinkx实时和离线同步Postgresql数据到Kafka
yrw1310305589的博客
12-02 1964
Flinkx实时和离线同步Postgresql数据到Kafka 一、环境部署 1.选择一台服务器，安装了Flink的更好，使用git工具把项目clone到本地 git clone https://github.com/DTStack/flinkx.git cd flinkx 或者直接下载源码 wget https://github.com/DTStack/flinkx/archive/1.8.5.zip unzip flinkx-1.8.5.zip cd flink-1.8.5 2.编译插件 mvn c
flink写入redis（redisSink）
婲落ヽ红颜谇的博客
05-30 617
1.所需依赖 <dependency> <groupId>org.apache.bahir</groupId> <artifactId>flink-connector-redis_2.11</artifactId> <version>1.0</version> </dependency> 2.redisSink /** * 生成RedisSi.
flink-connector-redis-2.10-1.1.5-API文档-中文版.zip
07-14
赠送jar包：flink-connector-redis_2.10-1.1.5.jar； 赠送原API文档：flink-connector-redis_2.10-1.1.5-javadoc.jar； 赠送源代码：flink-connector-redis_2.10-1.1.5-sources.jar； 赠送Maven依赖信息文件：flink-...
flink-connector-redis_2.10-1.1.5-API文档-中英对照版.zip
07-14
赠送jar包：flink-connector-redis_2.10-1.1.5.jar； 赠送原API文档：flink-connector-redis_2.10-1.1.5-javadoc.jar； 赠送源代码：flink-connector-redis_2.10-1.1.5-sources.jar； 赠送Maven依赖信息文件：flink-...
flink-connector-redis_2.11-1.1-SNAPSHOT
10-14
这个连接器允许Flink流处理应用程序将数据写入Redis数据库，以便进行实时数据存储和检索。这里的“2.11”代表依赖的Scala版本，“1.1-SNAPSHOT”则表示这是一个开发中的版本，可能包含最新的特性和改进。 描述中...
flink-connector-redis_2.11-1.1.5.jar
08-09
flink连接redis设置过期时间和密码
Flink Kafka读取数据并写入Redis实现Exactly-Once
我的祖传代码
06-02 2201
pom.xml <!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to yo
大数据_Flink_Java版_容错机制(6)-检查点和重启策略配置---Flink工作笔记0075
u011250186的博客
02-07 560
https://blog.csdn.net/lidew521/article/details/123666066
flink: 从pulsar中读取数据
amadeus_liu2的博客
03-27 608
【代码】flink: 从pulsar中读取数据。
Flink实战--异步IO实战入门(和Redis交互)
zg_hover的专栏
03-09 3815
本文通过一个例子来说明Flink异步IO的使用。在该例子中还说明了和Redis的交互。
Flink将本地数据写入Redis
weixin_53299145的博客
11-10 873
redis的启动文件默认安装在了/usr/local/bin/中
Redis Flink 连接器实战指南
gitblog_00582的博客
08-27 668
Redis Flink 连接器实战指南 项目地址:https://gitcode.com/gh_mirrors/fl/flink-connector-redis 1. 项目介绍 本项目是由 Jeff Zou 开发的一个异步 Flink 连接器，它基于 Lettuce，特别适用于处理与 Redis 的数据交互。该连接器不仅提供了写入 Redis 的 Sink 功能，还能发布数据到 Redis Pu...
flink-pulsar-connector示例
weixin_44284056的博客
05-13 1948
版本： pulsar：2.7.1 flink：1.12.2 maven依赖： <!-- flink-pulsar依赖--> <dependency> <groupId>io.streamnative.connectors</groupId> <artifactId>pulsar-flink-connector_2.11</artifactId> <version>1.12
打造全新批流融合：详解 Apache Flink 1.14.0 发布的 Pulsar Flink Connector
Apache Pulsar
11-24 2182
作者简介盛宇帆，StreamNative 开发工程师，Apache Pulsar 与 Apache Flink 贡献者。加入 StreamNative 之前，他曾就职于阿里大数据平台和腾...
Java版本Flink1.17怎么将数据流写入redis
最新发布
09-21
在Apache Flink 1.17版本中，要将数据流写入Redis，你需要使用Flink的连接器API——DataStream API。首先，确保已经添加了Flink Redis connector的依赖。以下是基本步骤： 1. 添加依赖：如果你使用的是Maven，添加以下依赖到pom.xml文件中： ```xml <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-connector-redis_2.11</artifactId> <version>${flink.version}</version> </dependency> ``` 替换`${flink.version}`为实际的Flink版本。 2. 创建RedisSinkFunction： ```java import org.apache.flink.api.common.serialization.SimpleStringSchema; import org.apache.flink.streaming.connectors.redis.RedisConnectionConfig; import org.apache.flink.streaming.connectors.redis.RedisSink; import org.apache.flink.streaming.connectors.redis.RedisSinkFunction; public class RedisDataWriter implements RedisSinkFunction<String> { private static final String KEY_FORMAT = "%s"; @Override public void invoke(String value, Context context) throws Exception { String key = String.format(KEY_FORMAT, generateKey(value)); // 生成Redis键 context.getCheckpointedStateContext().getStateBackend().getRedisStateBackend() .put(key, new SimpleStringSchema().toBytes(value)); } // 根据实际业务定制key生成逻辑 protected String generateKey(String value) { return value; // 示例：这里只是一个简单的字符串拼接，你可以替换为复杂逻辑 } // 初始化配置 @Override public void open(Configuration parameters) throws Exception { super.open(parameters); RedisConnectionConfig connectionConfig = new RedisConnectionConfig.Builder() .setHost("localhost") // 设置Redis服务器地址 .setPort(6379) // 设置端口 .build(); this.setConnectionConfig(connectionConfig); } } ``` 3. 在StreamExecutionEnvironment中设置和应用Sink： ```java StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<String> input = ... // 获取输入数据流 input.addSink(new RedisDataWriter()); // 将数据流写入Redis env.execute("Write to Redis"); // 启动流处理任务 ``` 4. 确保Redis服务器正在运行并且接收Flink的数据。Flink Redis connector支持多种数据序列化方式，如SimpleStringSchema，你可以根据需要选择适合的。 **相关问题：** 1. Flink如何处理Redis的分布式部署？ 2. Flink写入Redis时如何保证数据的一致性和可靠性？ 3. Flink如何处理Redis连接异常？
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值