java版kafka消费者取不到消息

本案例使用的当前kafka版本：kafka_2.12-0.10.2.0  

zookeeper版本：zookeeper-3.5.2-alpha

现在kafka的版本更新到0.10.2.0了，老的版本生产者和消费者实现起来有点麻烦，使用新的KafkaProducer、KafkaConsumer简化多了。

在生成和消费时一定要启动zookeeper、kafka服务，不然无法进行生成和消费，具体启动命令详见：http://blog.csdn.net/lilinoscar/article/details/64124338

Java项目如何引用jar包：

该jar包在你下载的kafka_2.12-0.10.2.0 文件夹libs里边，有很多包。

1.生产者

package examples;

public class KafkaProperties {
    public static final String TOPIC = "topic1";
    public static final String KAFKA_SERVER_URL = "localhost";
    public static final int KAFKA_SERVER_PORT = 9092;
    public static final int KAFKA_PRODUCER_BUFFER_SIZE = 64 * 1024;
    public static final int CONNECTION_TIMEOUT = 100000;
    public static final String TOPIC2 = "topic2";
    public static final String TOPIC3 = "topic3";
    public static final String CLIENT_ID = "SimpleConsumerDemoClient";

    private KafkaProperties() {}
}

package examples;

import org.apache.kafka.clients.producer.Callback;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;

import java.util.Properties;
import java.util.concurrent.ExecutionException;

public class Producer extends Thread {
    private final KafkaProducer<Integer, String> producer;
    private final String topic;
    private final Boolean isAsync;

    public Producer(String topic, Boolean isAsync) {
        Properties props = new Properties();
        props.put("bootstrap.servers", KafkaProperties.KAFKA_SERVER_URL + ":" + KafkaProperties.KAFKA_SERVER_PORT);
        props.put("client.id", "DemoProducer");
        
        //开始的时候下面5个参数未设置，导致消费时取不到数据，需要注意
        props.put("acks", "all");
        props.put("retries", 0);
        props.put("batch.size", 16384);
        props.put("linger.ms", 1);
        props.put("buffer.memory", 33554432);

        props.put("key.serializer", "org.apache.kafka.common.serialization.IntegerSerializer");
        props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        producer = new KafkaProducer<Integer, String>(props);
        this.topic = topic;
        this.isAsync = isAsync;
    }

    public void run() {
        int messageNo = 1;
        while (true) {
            String messageStr = "Message_" + messageNo;
            long startTime = System.currentTimeMillis();
            if (isAsync) { // Send asynchronously
                producer.send(new ProducerRecord<Integer, String>(topic,
                    messageNo,
                    messageStr), new DemoCallBack(startTime, messageNo, messageStr));
            } else { // Send synchronously
              
                    try {
						producer.send(new ProducerRecord<Integer, String>(topic,messageNo,messageStr)).get();
					} catch (InterruptedException e) {
						// TODO Auto-generated catch block
						e.printStackTrace();
					} catch (ExecutionException e) {
						// TODO Auto-generated catch block
						e.printStackTrace();
					}
                    System.out.println("Sent message: (" + messageNo + ", " + messageStr + ")");
               
            }
            ++messageNo;
            System.out.println("Sent message: (" + messageNo + ", " + messageStr + ")");
            if(messageNo==100){
            	
            	//break;
            }
        }
    }
}

class DemoCallBack implements Callback {

    private final long startTime;
    private final int key;
    private final String message;

    public DemoCallBack(long startTime, int key, String message) {
        this.startTime = startTime;
        this.key = key;
        this.message = message;
    }

    /**
     * A callback method the user can implement to provide asynchronous handling of request completion. This method will
     * be called when the record sent to the server has been acknowledged. Exactly one of the arguments will be
     * non-null.
     *
     * @param metadata  The metadata for the record that was sent (i.e. the partition and offset). Null if an error
     *                  occurred.
     * @param exception The exception thrown during processing of this record. Null if no error occurred.
     */
    public void onCompletion(RecordMetadata metadata, Exception exception) {
        long elapsedTime = System.currentTimeMillis() - startTime;
        if (metadata != null) {
            System.out.println(
                "message(" + key + ", " + message + ") sent to partition(" + metadata.partition() +
                    "), " +
                    "offset(" + metadata.offset() + ") in " + elapsedTime + " ms");
        } else {
            exception.printStackTrace();
        }
    }
}

2.消费者

package examples;

import java.util.Arrays;


import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;

import org.slf4j.Logger;  
import org.slf4j.LoggerFactory;  
import java.util.Arrays;
import java.util.Properties;

public class ConsumerThread extends Thread {
	    private final KafkaConsumer<Integer, String> consumer;
	
	    private final String topic;
	    private static final Logger LOG = LoggerFactory.getLogger(ConsumerThread.class);  
	    public ConsumerThread(String topic) {
	       
	        Properties props = new Properties();
	        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, KafkaProperties.KAFKA_SERVER_URL + ":" + KafkaProperties.KAFKA_SERVER_PORT);
	        props.put(ConsumerConfig.GROUP_ID_CONFIG, "test-consumer-group");
	        props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
	        props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000");
	        props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "30000");
	        props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.IntegerDeserializer");
	        props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");

	        this.consumer = new KafkaConsumer<Integer, String>(props);
	        this.topic = topic;
	    }
	public void run() {
		    System.out.println("ConsumerThread--run");
		    
		    consumer.subscribe(Arrays.asList("topic1"));
		    
		  // consumer.subscribe(Collections.singletonList(this.topic));  
	        while (true) {
	        	 ConsumerRecords<Integer, String> records = consumer.poll(200);
	        	 
	 	       for (ConsumerRecord<Integer, String> record : records) {
	 	            System.out.println("Received message: (" + record.key() + ", " + record.value() + ") at offset " + record.offset());
	 	        }
	        }
    }
}

3.Man方法

package examples;

public class KafkaConsumerProducerDemo {
    public static void main(String[] args) {
        boolean isAsync = args.length == 0 || !args[0].trim().equalsIgnoreCase("sync");
       Producer producerThread = new Producer(KafkaProperties.TOPIC, isAsync);
       producerThread.start();
        
       ConsumerThread consumerThread = new ConsumerThread(KafkaProperties.TOPIC);
       consumerThread.start();
    }
}

遇到问题：

1.可以生成，但是无法消费。

  kafka的config配置文件：consumer.properties文件里的group.id=test-consumer-group需要改成一致。

  确保生产者实例化Properties配置文件设置正确，少了几个也是无法消费。

2.不同的版本以及引用jar包也是不一样，需要一致。

3.注意配置文件ip地址、端口等。

以下是生产者和消费者案例：

http://www.henryxi.com/kafka-java-example

http://www.cnblogs.com/huxi2b/p/6124937.html

https://kafka.apache.org/0100/javadoc/index.html

同时java生产，C#消费都是可以进行实现，开始的时候遇到java可以生产，但是不能消费，C#可以消费，折腾了很久，原来是group.id以及生产的时候参数少了。