一、安装
解压kafka安装文件.tar.gz
在/opt/module/kafka 目录下创建logs文件夹 mkdir logs
修改配置文件 cd config/ vi server.properties
server.properties中
- 启动删除topic功能 delete.topic.enable=true
- kafka存放路径(数据与日志共同存放)log.dirs=/opt/module/kafka/logs
- 配置zookeeper集群地址 例:zookeeper.connect=hadoop102:2181(中间以逗号分隔)
环境变量配置
vi /etc/profile
#KAFKA_HOME
export KAFKA_HOME=opt/module/kafka
export PATH= P A T H : PATH: PATH:KAFKA_HOME/bin
source /etc/profile
分发安装包 xsync kafka/
二、启动
bin/kafka-server-start.sh -daemon config/server.properties
三、增删查topic
#查询
bin/kafka-topics.sh --list --zookeeper hadoop102:8181
#增加
bin/kakfka-topics.sh --create --zookeeper hadoop102:8181 --topic first --partitions 2 --replication-factor 2
#删除
bin/kafka-topics.sh --delete --zookeeper hadoop102:8181 --topic first
#查询详情
bin/kafka-topics.sh --describe --topic first --zookeeper hadoop102:2181
四、生产者Api
private void getKafkaProducer(){
Properties properties = new Properties();
//指定连接的kafka集群
properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,"hadoop102:9002");
//ack 应答级别
properties.put(ProducerConfig.ACKS_CONFIG,"all");
//重试次数
properties.put(ProducerConfig.RETRIES_CONFIG,1);
//批次大小 16k
properties.put(ProducerConfig.BATCH_SIZE_CONFIG,16384);
//等待时间
properties.put(ProducerConfig.LINGER_MS_CONFIG,1);
//recordAccumulator缓冲区大小
properties.put(ProducerConfig.BUFFER_MEMORY_CONFIG,33554432);
//key序列化
properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
//value序列化
properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringSerializer");
//创建生产者对象
KafkaProducer producer = new KafkaProducer<String,String>(properties);
//发送数据
for (int i = 0; i < 10; i++) {
producer.send(new ProducerRecord("number",100+i));
}
//关闭资源
producer.close();
}
五、消费者Api
public void getKafkaConsumer(){
Properties properties = new Properties();
//连接集群
properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,"hadoop102:9002");
//开启自动提交
properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG,true);
//自动提交延迟
properties.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG,"1000");
//反序列化
properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringDeserializer");
properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringDeserializer");
//消费组
properties.put(ConsumerConfig.GROUP_ID_CONFIG,"data");
//重置消费组的offset
properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,"earliest");
KafkaConsumer consumer = new KafkaConsumer<String,String>(properties);
//订阅主题
consumer.subscribe(Arrays.asList("f","s"));
while (true){
ConsumerRecords<String,String> poll = consumer.poll(100);
poll.forEach(p->{
System.out.println(p.key()+p.value());
});
}
}