1.生产者代码
import kafka.producer.KeyedMessage;
import kafka.serializer.StringEncoder;
import org.apache.kafka.clients.producer.*;
import org.jetbrains.annotations.NotNull;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Properties;
import java.util.concurrent.ExecutionException;
/**
* Created by huangjinchong on 2018/5/11.
*/
public class TestProducer {
private String topic;
public TestProducer(String topic) {
super();
this.topic = topic;
}
public void sendProducer(){
KafkaProducer producer = createProducer();
int i=0;
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
System.out.println(producer.toString());
for (int j = 0; j < 20; j++) {
// try {
// Object o=producer.send(new ProducerRecord(topic,"时间是:"+sdf.format(new Date())+"message:"+i)).get();
// System.out.println(o);
// } catch (InterruptedException e) {
// e.printStackTrace();
// } catch (ExecutionException e) {
// e.printStackTrace();
// }
producer.send(new ProducerRecord(topic,"时间是:"+sdf.format(new Date())+"message:"+i));
}
// producer.flush();
// System.out.println("时间是:"+sdf.format(new Date())+"message:"+i);
}
private KafkaProducer<Integer,String> createProducer(){
Properties prop = new Properties();
prop.put("value.serializer","org.apache.kafka.common.serialization.StringSerializer");
prop.put("key.serializer","org.apache.kafka.common.serialization.StringSerializer");
prop.put("bootstrap.servers","192.168.1.131:9092,192.168.1.132:9092");
prop.put("acks", "all");
prop.put("retries", 0);
prop.put("batch.size", 10);
prop.put("linger.ms", 0);
prop.put("buffer.memory", 60*1024);
return new KafkaProducer<Integer, String>(prop);
}
public static void main(String[] args) {
new TestProducer("test").sendProducer();
try {
Thread.sleep(10000);
} catch (InterruptedException e) {
e.printStackTrace();
}
System.out.println("程序运行完毕!");
}
}
经测试如果没有Thread.sleep(10000);需要producer.flush()才能将缓存中的数据push到kafka中
2.消费者代码
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.TopicPartition;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
/**
* Created by huangjinchong on 2018/5/11.
*/
public class TestConsumer {
private String topic;
public TestConsumer(String topic) {
super();
this.topic = topic;
}
Runnable run1= new Runnable() {
@Override
public void run() {
KafkaConsumer<String, String> consumer = createConsumer();
List l = new ArrayList();
l.add(topic);
consumer.subscribe(l);
while(true){
ConsumerRecords<String, String> records = consumer.poll(500);
for (ConsumerRecord<String, String> record : records){
System.out.println(record.value());
System.out.println(record.partition());
//手动提交已消费数据的offset
}
}
}
};
private KafkaConsumer<String, String> createConsumer(){
Properties props = new Properties();
props.put("bootstrap.servers", "192.168.1.131:9092,192.168.1.132:9092");
props.put("group.id", "hjc1");
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
props.put("auto.commit.interval.ms", "1000");
props.put("max.poll.records", 1);
// props.put("auto.offset.reset", "latest");//对新的消费者而言,是从头消费还是从其他消费者已经消费过的地方开始,不同的消费者群组
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
return consumer;
}
public static void main(String[] args) {
TestConsumer tc = new TestConsumer("test");
Thread th = new Thread(tc.run1);
th.start();
}
}
要以无限循环线程方式消费才能取得所有分区的数据,一次执行只能取得某一分区的数据,只消费某一个分区代码为:
List l1 = new ArrayList();
TopicPartition partitons = new TopicPartition(topic,0);
l1.add(partitons);
consumer.assign(l1);