KafkaUtil
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;
import org.apache.flink.streaming.connectors.kafka.KafkaSerializationSchema;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import javax.annotation.Nullable;
import java.util.Properties;
/**
* @author zyj
* @Date 2022/1/6 15:29
* 操作 kafka 的工具类
*/
public class MyKafkaUtil {
private static final String KAFKA_SERVER = "hadoop101:9092,hadoop102:9092,hadoop103:9092";
private static final String DEFAULT_TOPIC = "default_topic";
// 获取kafka的消费者
public static FlinkKafkaConsumer<String> getKafkaSource(String topic, String groupId) {
Properties props = new Properties();
props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, KAFKA_SERVER);
props.setProperty(ConsumerConfig.GROUP_ID_CONFIG, groupId);
return new FlinkKafkaConsumer<String>(topic, new SimpleStringSchema(), props);
}
// 获取kafka的生产者(String数据类型)
/*
注意:下面这中实现只能保证数据不丢,不能保证精准一次性
public static FlinkKafkaProducer<String> getKafkaSink(String topic) {
return new FlinkKafkaProducer<String>(
KAFKA_SERVER,
topic,
new SimpleStringSchema()
);
}*/
public static FlinkKafkaProducer<String> getKafkaSink(String topic) {
Properties props = new Properties();
// 指定kafka服务端ip地址
props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, KAFKA_SERVER);
// 设置生产超时时间
props.setProperty(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, 1000 * 60 * 15 + "");
// 创建kafka生产者,自定义序列化器 KafkaSerializationSchema<String> 这里是指定String类型
return new FlinkKafkaProducer<String>(DEFAULT_TOPIC, new KafkaSerializationSchema<String>() {
@Override
public ProducerRecord<byte[], byte[]> serialize(String str, @Nullable Long timestamp) {
return new ProducerRecord<byte[], byte[]>(topic, str.getBytes());
}
}, props, FlinkKafkaProducer.Semantic.EXACTLY_ONCE);
}
// 获取kafka的生产者(通用数据类型)
public static <T> FlinkKafkaProducer<T> getKafkaSinkSchema(KafkaSerializationSchema<T> kafkaSerializationSchema) {
Properties props = new Properties();
// 指定kafka服务端ip地址
props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, KAFKA_SERVER);
// 设置生产超时时间
props.setProperty(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, 1000 * 60 * 15 + "");
return new FlinkKafkaProducer<T>(DEFAULT_TOPIC, kafkaSerializationSchema, props, FlinkKafkaProducer.Semantic.EXACTLY_ONCE);
}
}