Flume操作kafka工具类

最新推荐文章于 2023-11-21 22:07:29 发布

小总结

最新推荐文章于 2023-11-21 22:07:29 发布

阅读量338

点赞数

文章标签： kafka flume java

本文链接：https://blog.csdn.net/qq_50408152/article/details/126024807

版权

import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;
import org.apache.flink.streaming.connectors.kafka.KafkaDeserializationSchema;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;

import java.util.Properties;

public class MyKafkaUtil {

   private static Properties properties = new Properties();
   private static final String KAFKA_SERVER = "hadoop102:9092";

   //一个并行度做一次
   static {
       properties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,KAFKA_SERVER);
   }

    public static FlinkKafkaConsumer<String> getFlinkKafkaConsumer(String topic,String groupId){

        properties.setProperty(ConsumerConfig.GROUP_ID_CONFIG,groupId);

        return new FlinkKafkaConsumer<String>(
                topic,
                new KafkaDeserializationSchema<String>() {
                    @Override
                    public boolean isEndOfStream(String nextElement) {
                        return false;
                    }

                    @Override
                    public String deserialize(ConsumerRecord<byte[], byte[]> record) throws Exception {
                        if (record == null || record.value() == null) {
                            //可以返回null,因为到下游会过滤，只要不出现空指针即可
                            return null;
                        } else {
                            return new String(record.value());
                        }
                    }

                    @Override
                    public TypeInformation<String> getProducedType() {
                        return BasicTypeInfo.STRING_TYPE_INFO;
                    }
                },properties);

    }

    public static FlinkKafkaProducer<String> getFlinkKafkaProducer(String topic){
        return new FlinkKafkaProducer<String>(topic,
                new SimpleStringSchema(),
                properties);
    }

    public static String getKafkaDDL(String topic, String groupId) {
        return " WITH ( " +
                "  'connector' = 'kafka', " +
                "  'topic' = '" + topic + "', " +
                "  'properties.bootstrap.servers' = '" + KAFKA_SERVER + "', " +
                "  'properties.group.id' = '" + groupId + "', " +
                "  'scan.startup.mode' = 'latest-offset', " +
                "  'format' = 'json' " +
                ")";
    }

    public static String getInsertKafkaDDL(String topic) {
        return " WITH ( " +
                "  'connector' = 'kafka', " +
                "  'topic' = '" + topic + "', " +
                "  'properties.bootstrap.servers' = '" + KAFKA_SERVER + "', " +
                "  'format' = 'json' " +
                ")";
    }

    public static String getUpsertKafkaDDL(String topic) {
        return " WITH ( " +
                "'connector' = 'upsert-kafka', " +
                "'topic' = '" + topic + "', " +
                "'properties.bootstrap.servers' = 'hadoop102:9092', " +
                "'key.format' = 'json', " +
                "'value.format' = 'json' " +
                ")";
    }

    public static String getTopicDb(String groupId) {
        return "CREATE TABLE topic_db ( " +
                "  `database` STRING, " +
                "  `table` STRING, " +
                "  `type` STRING, " +
                "  `ts` BIGINT, " +
                "  `data` Map<STRING,STRING>, " +
                "  `old` Map<STRING,STRING>, " +
                "  `pt` AS PROCTIME() " +
                ")" + getKafkaDDL("topic_db", groupId);
    }



}