flink读取kafka的数据处理完毕写入redis

最新推荐文章于 2024-06-24 08:59:02 发布

JinVijay

最新推荐文章于 2024-06-24 08:59:02 发布

阅读量608

点赞数

分类专栏： flink 文章标签： kafka redis flink

本文链接：https://blog.csdn.net/JinVijay/article/details/123097953

版权

flink 专栏收录该内容

11 篇文章 1 订阅

订阅专栏

/* 
 * 从Kafka读取数据处理完毕写入Redis
 */
public class KafkaToRedis {

    public static void main(String[] args) throws Exception {

        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        //开启checkpointing
        env.enableCheckpointing(1000);
        //设置StateBackEnd 存储在HDFS中
        env.setStateBackend(new FsStateBackend("hdfs://mydfs/checkpoint"));
        //设置cancel任务checkpoint数据的策略  cancel任务保留checkpoint的数据
        env.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
        Properties properties = new Properties();
        //设置Broker地址
        properties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,"linux01:9092,linux02:9092,linux03:9092");
        //设置没有偏移量的话从头开始读取数据
        properties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_DOC,"earliest");
        //设置不自动提交偏移量
        properties.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG,"false");
        //设置GroupID
        properties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, UUID.randomUUID().toString());
        FlinkKafkaConsumer<String> kafkaConsumer = new FlinkKafkaConsumer<>("wordcount", new SimpleStringSchema(), properties);
        //设置在checkpoint后不提交偏移量到kafka特殊的topic中
        kafkaConsumer.setCommitOffsetsOnCheckpoints(false);
        DataStreamSource<String> kafkaSource = env.addSource(kafkaConsumer);

        SingleOutputStreamOperator<Tuple2<String, Integer>> wordAndOne = kafkaSource.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
            @Override
            public void flatMap(String line, Collector<Tuple2<String, Integer>> out) throws Exception {
                String[] words = line.split(" ");
                for (String word : words) {
                    out.collect(Tuple2.of(word,1));
                }
            }
        });

        SingleOutputStreamOperator<Tuple2<String, Integer>> result = wordAndOne.keyBy(tp -> tp.f0).sum(1);
        FlinkJedisPoolConfig config = new FlinkJedisPoolConfig.Builder().setHost("linux01").build();
        //将数据写入redis
        result.addSink(new RedisSink<Tuple2<String, Integer>>(config,new KafkaToRedisMapper()));
        //启动程序
        env.execute();
    }
    private static class KafkaToRedisMapper implements RedisMapper<Tuple2<String,Integer>>{
        //设置Redis中的key
        @Override
        public RedisCommandDescription getCommandDescription() {
            return new RedisCommandDescription(RedisCommand.HSET,"wordcount");
        }
        //设置Value -> Hash类型的key
        @Override
        public String getKeyFromData(Tuple2<String, Integer> data) {
            return data.f0;
        }
        //设置Value -> Hash类型的value
        @Override
        public String getValueFromData(Tuple2<String, Integer> data) {
            return data.f1.toString();
        }
    }
}