kafka的java实例

1、依赖的jar包

 <properties>
        <kafka.version>0.10.2.0</kafka.version>
    </properties>
 <dependency>
        <groupId>org.apache.kafka</groupId>
        <artifactId>kafka-clients</artifactId>
        <version>${kafka.version}</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.kafka/kafka_2.11 -->
    <dependency>
        <groupId>org.apache.kafka</groupId>
        <artifactId>kafka_2.11</artifactId>
        <version>${kafka.version}</version>
    </dependency>

2、生产者

package com.kafka.demo;

import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;

import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Properties;

/**
 * Created by liyongguan on 2017/11/22.
 */
public class KafkaProducerDemo {

    /***  broker所在的机器 **/
    private final  String BROKER_LIST="172.16.36.112:9092,172.163.6.120:9092,172.16.36.121:9092";
    private final  String TOPIC = "test";
    public Properties getProperties(){
        Properties proper = new Properties();
        /* 定义kakfa 服务的地址 */
        proper.put("bootstrap.servers",BROKER_LIST);
        proper.put("ack","all");
        proper.put("retries",0);
        proper.put("batch.size",16384);
        proper.put("linger.ms",1);
        proper.put("buffer.memory",33554432);
        proper.put("key.serializer","org.apache.kafka.common.serialization.StringSerializer");
        proper.put("value.serializer","org.apache.kafka.common.serialization.StringSerializer");
        return  proper;
    }

    public void  produceMessage(){
        Properties properties = getProperties();
        Producer<String,String> producer = new KafkaProducer<String, String>(properties);
        String key,value;
        for(int i = 1;i<=20;i++){
            key = "key-"+i;
            value = "value-"+i+"-"+new SimpleDateFormat("HH:mm:ss").format(new Date());
            System.out.println("TOPIC: "+TOPIC+",key: "+key+",value: "+value);
            producer.send(new ProducerRecord<String, String>(TOPIC,key,value));
            try {
                Thread.sleep(1000);
            }catch (InterruptedException e){
                e.printStackTrace();
            }
        }
        producer.close();
    }

    public static void main(String[] args){
        new KafkaProducerDemo().produceMessage();
    }
}

3、消费者

package com.kafka.demo;


import kafka.common.Topic;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.errors.WakeupException;


import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;

/**
 * Created by liyongguan on 2017/11/22.
 */
public class KafkaConsumerDemo {
    //private  final static String ZOOKEEPER_LIST = "172.16.36.112:2181,172.16.36.120:2181,172.16.36.121:2181";
    private  final static String BROKER_LIST="172.16.36.112:9092";
    private  final  String TOPIC = "test";

    /**
     * 连接kafka的配置
     * @return
     */
    public static Properties getProperties(){
        Properties properties  = new Properties();
        /* 定义kakfa 服务的地址，不需要将所有broker指定上 */
        properties.put("bootstrap.servers",BROKER_LIST);
        /* 制定consumer group */
        properties.put("group.id","test-group");
        /* 是否自动确认选项 ,设置不自动提交，采用手动方式*/
        properties.put("enable.auto.commit","false");
        /* key的序列化类 */
        properties.put("key.deserializer","org.apache.kafka.common.serialization.StringDeserializer");
        /* value的序列化类 */
        properties.put("value.deserializer","org.apache.kafka.common.serialization.StringDeserializer");
        return properties;
    }


    public void consumerMessage(){
        /***使用线程池处理***/
        int consumerNums = 1;
        final ExecutorService executor =Executors.newFixedThreadPool(consumerNums);
        final List<KafkaConsumerRunner> kafkaConsumerRunnerList = new ArrayList<KafkaConsumerRunner>();
        for(int i = 0;i<consumerNums;i++){
            KafkaConsumerRunner kafkaConsumerRunner = new KafkaConsumerRunner(TOPIC);
            kafkaConsumerRunnerList.add(kafkaConsumerRunner);
            executor.submit(kafkaConsumerRunner);
            System.out.println(1234);
        }

        /*Runtime.getRuntime().addShutdownHook(new Thread(){
            public void run(){
                for(KafkaConsumerRunner kafkaConsumerRunner:kafkaConsumerRunnerList){
                    kafkaConsumerRunner.shutdown();
                }
                executor.shutdown();
                try{
                    executor.awaitTermination(1000,TimeUnit.MILLISECONDS);
                }catch (InterruptedException e){
                    e.printStackTrace();
                }
            }
        });*/
    }


    /**
     * 线程处理--消费者
     */
    public static  class KafkaConsumerRunner implements Runnable{

        private final  String topic;
        private final AtomicBoolean  closed = new AtomicBoolean(false);
        private  final KafkaConsumer<String,String> consumer;

        public KafkaConsumerRunner(String topic){
            this.topic = topic;
            Properties props = getProperties();
            consumer = new KafkaConsumer<String,String>(props);
        }

        public void handleRecord(ConsumerRecord record){
            System.out.println(/*"Thread name: "+Thread.currentThread().getName()+*/"topic:"+record.topic()+" , offset:" + record.offset() + ", key:" + record.key() + ", value:" + record.value());
        }
        public void run(){
            // 批量提交数量
            final int minBatchSize = 1;
            try{
                /* 消费者订阅的topic, 可同时订阅多个 */
                consumer.subscribe(Arrays.asList(topic));
                List<ConsumerRecord<String, String>> buffer = new ArrayList<ConsumerRecord<String, String>>();
                /* 读取数据，读取超时时间为1000ms */
                while (!closed.get()){
                    ConsumerRecords<String,String> records = consumer.poll(1000);
                    for (ConsumerRecord record:records){
                        handleRecord(record);
                        buffer.add(record);
                    }
                    if (buffer.size() >= minBatchSize) {
                        //这里就是处理成功了然后自己手动提交
                        consumer.commitSync();
                        buffer.clear();
                    }

                }
            }catch (WakeupException e){
                e.printStackTrace();
                if(!closed.get())
                    throw e;
            }
            finally {
                consumer.close();
            }
        }


        public void shutdown()
        {
            closed.set(true);
            consumer.wakeup();
        }
    }
    public static void main(String[] args){
        new KafkaConsumerDemo().consumerMessage();
    }
}

说明：关于kafka集群的搭建和配置，自行脑补。