常见命令
启动集群
[123] kafka-server-start.sh /opt/yjx/kafka_2.12-0.11.0.3/config/server.properties
//创建主题
[root@node01 ~]# kafka-topics.sh --zookeeper node01:2181 --create --replication-factor 2 --partitions 3 --topic userlog
[root@node01 ~]# kafka-topics.sh --zookeeper node01:2181 --create --replication-factor 2 --partitions 6 --topic studentlog
[root@node01 ~]# kafka-topics.sh --zookeeper node01:2181 --create --replication-factor 2 --partitions 6 --topic baidu
//查看所有主题
[root@node01 ~]# kafka-topics.sh --zookeeper node01:2181 --list
//查看主题
[root@node01 ~]# kafka-topics.sh --zookeeper node01:2181,node02:2181,node03:2181 --describe --topic userlog
//创建生产者
[root@node01 ~]# kafka-console-producer.sh --broker-list node01:9092,node02:9092,node03:9092 --topic userlog
//创建消费者
[root@node01 ~]# kafka-console-consumer.sh --zookeeper node01:2181,node02:2181,node03:2181 --from-beginning --topic userlog
生产者
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import java.util.Properties;
/**
* 创建生产者线程类
*/
public class Hello01Producer extends Thread {
//创建Kafka的生产者
private Producer<String, String> producer;
/**
* 创建构造器
*/
public Hello01Producer(String producerName) {
//设置线程的名字
super.setName(producerName);
//创建配置文件列表
```Properties properties = new Properties();
// kafka地址,多个地址用逗号分割
properties.put("bootstrap.servers", "192.168.88.101:9092,192.168.88.102:9092,192.168.88.103:9092");
//设置写出数据的格式
properties.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
properties.put("value.serializer","org.apache.kafka.common.serialization.StringSerializer");
//写出的应答方式
properties.put("acks", "all");
//错误重试
properties.put("retries", 1);
//批量写出
properties.put("batch.size", 16384);
//创建生产者对象
producer = new KafkaProducer<String, String>(properties);
}
@Override
public void run() {
//初始化一个计数器
int count = 0;
System.out.println("Hello01Producer.run--开始发送数据");
//迭代發送消息
while (count < 100) {
count++;
String key = "yjx01-" + count;
String value = Thread.currentThread().getName() + "--" +
count;
//封装消息对象
ProducerRecord<String, String> producerRecord = new
ProducerRecord<>("baidu", key, value);
//发送消息到服务器
producer.send(producerRecord);
//打印消息
//System.out.println("Producer.run--" + key + "--" + value);
//每个1秒发送1条
//try {
// Thread.sleep(1000);
//} catch (InterruptedException e) {
// e.printStackTrace();
//}
}
//强行将数据写出
producer.close();
System.out.println("Hello01Producer.run--发送数据完毕");
}
public static void main(String[] args) {
Hello01Producer producer = new Hello01Producer("优极限01");
producer.start();
}
}
消费者
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.TopicPartition;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
public class Hello01Consumer extends Thread {
//创建消费者对象
private KafkaConsumer<String, String> consumer;
/**
* 创建构造器
*/
public Hello01Consumer(String cname) {
super.setName(cname);
//读取配置文件
Properties properties = new Properties();
//ZK地址
properties.put("bootstrap.servers", "192.168.88.101:9092,192.168.88.102:9092,192.168.88.103:9092");
//消费者所在组的名称
properties.put("group.id", "yjx-bigdata");
//ZK超时时间
properties.put("zookeeper.session.timeout.ms", "1000");
//反序列化
properties.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
properties.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
//当消费者第一次消费时,从最低的偏移量开始消费
properties.put("auto.offset.reset", "earliest");
//自动提交偏移量
properties.put("auto.commit.enable", "true");
//消费者自动提交偏移量的时间间隔
properties.put("auto.commit.interval.ms", "1000");
//创建消费者对象
consumer = new KafkaConsumer<>(properties);
}
@Override
public void run() {
try {
consumer.subscribe(Arrays.asList("baidu"));
boolean flag = true;
while (flag) {
ConsumerRecords<String, String> records =
consumer.poll(100);
for (TopicPartition partition : records.partitions()) {
List<ConsumerRecord<String, String>> partitionRecords = records.records(partition);
for (ConsumerRecord<String, String> record :partitionRecords) {
System.out.println(record.topic() + ":" +
record.partition() + ":" + record.offset() + ":" + record.key() + ":" +
record.value());
}
consumer.commitSync();//同步
}
}
} finally {
consumer.close();
}
}
public static void main(String[] args) {
Hello01Consumer consumer01 = new Hello01Consumer("马龙YYDS");
consumer01.start();
}
}