kafka java 编程

kafka java编程

1.引入kafka

<dependency>
        <groupId>org.apache.kafka</groupId>
        <artifactId>kafka_2.11</artifactId>
        <version>0.9.0.0</version>
</dependency>

2. 生产者

import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;
import java.util.Properties;
 
public class KafkaProducerDemo extends Thread{

    private final static String BROKER_LIST = "192.168.126.31:9092,192.168.126.32:9092,192.168.126.33:9092";
    private final static String TPOPIC = "test11";
    private Producer<Integer, String> producer;

    public   KafkaProducerDemo() {
        Properties properties = new Properties();
        properties.put("metadata.broker.list", BROKER_LIST);
        properties.put("serializer.class", "kafka.serializer.StringEncoder");
        properties.put("request.required.acks","1");
        producer = new Producer<>(new ProducerConfig(properties));
    }

    @Override
    public void run() {
        int messageNo = 1;
        while(messageNo < 100) {
            String message = "message_" + messageNo;
            producer.send(new KeyedMessage<Integer, String>(TPOPIC, message));
            System.out.println("Sent: " + message);
            messageNo ++ ;
            try{
                Thread.sleep(2000);
            } catch (Exception e){
                e.printStackTrace();
            }
        }
        producer.close();
    }
}

3. 消费者

import kafka.consumer.Consumer;
import kafka.consumer.ConsumerConfig;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;

 
public class KafkaConsumerDemo extends Thread {

    private final static String ZOOKEEPER_LIST = "192.168.126.31:2181,192.168.126.32:2181,192.168.126.33:2181";
    private final static String GROUP_ID = "test_group10";
    private final static String TPOPIC = "test10";
    private ConsumerConnector consumerConnector;

    public  KafkaConsumerDemo() {
        Properties properties = new Properties();
        properties.put("zookeeper.connect", ZOOKEEPER_LIST);
        properties.put("group.id", GROUP_ID);
        ConsumerConfig consumerConfig = new ConsumerConfig(properties);
        consumerConnector = Consumer.createJavaConsumerConnector(consumerConfig);
}

    @Override
    public void run() {
        Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
        topicCountMap.put(TPOPIC, new Integer(1));
        Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumerConnector.createMessageStreams(topicCountMap);
        KafkaStream<byte[], byte[]> stream = consumerMap.get(TPOPIC).get(0);
        ConsumerIterator<byte[], byte[]> it = stream.iterator();
        while (it.hasNext()) {
            System.out.println("Receive->[" + new String(it.next().message()) + "]");
            try {
                Thread.sleep(2000);
            } catch (Exception ex) {
                ex.printStackTrace();
            }
        }
    }
}

4.启动测试

1).启动zkServer
zkServer.sh start

2).启动kafka
kafka-server-start.sh -daemon ${KAFKA_HOME}/config/server.properties

3). 测试

public class TestProducerAndConsumer {

    public static void main(String[] args) {
        new KafkaProducerDemo().start();
        new KafkaConsumerDemo().start();
    }
}

5.遇到的坑

kafka.common.FailedToSendMessageException: Failed to send messages after 3 tries.

解决方案：
1）.找到$KAFKA_HOME/config/中的配置文件:
配置advertised.host.name=“ip”,advertised.port=“port”.具体解释可以参考配置文件中的注释
advertised.host.name。官方文档里的备注信息表明，该字段的值是生产者和消费者使用的。如果没有设置，则会取host.name的值，默认情况下，该值为localhost。思考一下，如果生产者拿到localhost这个值，只往本地发消息，必然会报错

2). 在本地telnet ip 9092，观察是否可以连接，如不能连接，关闭服务器防火墙，或者打开9092端口。

源码地址