Sink-redisSink的使用

最新推荐文章于 2024-03-05 14:36:15 发布

江湖侠客

最新推荐文章于 2024-03-05 14:36:15 发布

阅读量556

点赞数

文章标签： flink redis

本文链接：https://blog.csdn.net/weixin_39868387/article/details/118437244

版权

上篇：Sink-kafkaSink的使用

需求：输入数据，往redis写数据

直接代码：是一个无限流程序

创建nc -lk 8888，输入数据，就可以往redis写数据

package cn._51doit.flink.day01;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.redis.RedisSink;
import org.apache.flink.streaming.connectors.redis.common.config.FlinkJedisPoolConfig;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisCommand;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisCommandDescription;
import org.apache.flink.streaming.connectors.redis.common.mapper.RedisMapper;
import org.apache.flink.util.Collector;

/**
 * Sink-redisSink的使用
 * 需求：从指定的socket读取数据，对单词进行计算，将结果写入到redis中
 *
 * 步骤：
 *    （1）pom文件引入redis依赖
 *
 */
public class RedisSinkDemo {
    public static void main(String[] args) throws Exception {
        //创建Flink流式计算的执行环境（ExecutionEnvironment）
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        //创建DataStream
        //Source
        DataStream<String> lines = env.socketTextStream("Master",8888);
        //调用Transformation开始
        //调用Transformation(优化代码)
        SingleOutputStreamOperator<Tuple2<String, Integer>> wordAndOne = lines.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
            @Override
            public void flatMap(String line, Collector<Tuple2<String, Integer>> collector) throws Exception {
                String[] words = line.split(" ");
                for (String word : words) {
                    //new Tuple2<String,Integer>(word,1);
                    collector.collect(Tuple2.of(word, 2));

                }
            }
        });

        //分组
        KeyedStream<Tuple2<String, Integer>, String> keyed = wordAndOne.keyBy(new KeySelector<Tuple2<String, Integer>, String>() {
            @Override
            public String getKey(Tuple2<String, Integer> tp) throws Exception {
                return tp.f0;
            }
        });

        //聚合
        SingleOutputStreamOperator<Tuple2<String, Integer>> summed = keyed.sum(1);

        //Transformation结束

        //调用Sinnk
        //创建redis的连接池
        FlinkJedisPoolConfig conf = new FlinkJedisPoolConfig.Builder()
                .setHost("Master")
                .setDatabase(8).build();

        summed.addSink(new RedisSink<Tuple2<String, Integer>>(conf,new RedisWordCountMapper()));


        //启动执行
        env.execute("StreamingWordCount");

    }

    /**
     * 自定义一个类，通过进行对字段绑定【映射】
     */
    public static class RedisWordCountMapper implements RedisMapper<Tuple2<String,Integer>> {


        @Override
        public RedisCommandDescription getCommandDescription() {
            //把数据写到reis【在flink中没有做redis累计（累加的话采用flink的状态，如keyby），做覆盖】
            return new RedisCommandDescription(RedisCommand.HSET,"WORD_COUNT");

        }

        @Override
        public String getKeyFromData(Tuple2<String, Integer> data) {
            return data.f0;
        }

        @Override
        public String getValueFromData(Tuple2<String, Integer> data) {
            return data.f1.toString();
        }
    }
}

查看job：http://localhost:58014/#/job/b31166df92d7ba97b0cbb1e70f7c3b7a/overview