2.kafka 操作

1、kafka脚本操作

1.1、kafka-topics命令

l 查看当前服务器中的所有topic

kafka-topics --zookeeper zk01:2181/kafka --list 

l 创建topic(分区、副本)

kafka-topics --zookeeper zk01:2181/kafka  --create --topic test --partitions 1  --replication-factor 1

l 查看某个Topic的详情

kafka-topics --zookeeper zk01:2181/kafka --describe --topic test 

l 对分区数进行修改

kafka-topics --zookeeper zk01:2181/kafka --alter --topic   utopic --partitions 15 

l 删除topic

kafka-topics --zookeeper zk01:2181/kafka --delete  --topic test

需要server.properties中设置delete.topic.enable=true否则只是标记删除或者直接重启。

1.2、kafka-console-producer命令

l 发送消息（手动一行行发送）

kafka-console-producer --broker-list kafka01:9092 --topic test

l 发送消息（管道，批量上传一个文件内容）

tail -F /root/log.log | kafka-console-producer --broker-list kafka01:9092 --topic test

1.3、kafka-console-consumer命令

l 消费消息

kafka-console-consumer --zookeeper zk01:2181/kafka --topic test --from-beginning

1.4、kafka-run-class/kafka-consumer-offset-checker命令（查看消费情况）

l 查看消费位置

kafka-run-class kafka.tools.ConsumerOffsetChecker --zookeeper zk01:2181/kafka --group testgroup

具体到topic

kafka-run-class kafka.tools.ConsumerOffsetChecker --zookeeper zk01:2181/kafka --group testgroup --topic test

或者

kafka-consumer-offset-checker --zookeeper zk01/kafka --group testgroup --topic test

2、通过Java的api操作：

maven依赖

<dependency>
   <groupId>org.apache.kafka</groupId>
   <artifactId>kafka_2.10</artifactId>
   <version>0.9.0.0</version>
   <exclusions>
      <exclusion>
         <groupId>junit</groupId>
         <artifactId>junit</artifactId>
      </exclusion>
   </exclusions>
</dependency>

生产者API

import kafka.javaapi.producer.Producer;  //注意：一定是Javaapi
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;

import java.util.Properties;

/**
 * @Company 任子行网络技术股份有限公司
 * @CreateTime 2017/8/21  13:59
 */
public class KafkaProducer {
    public static void main(String[] args) {
        Properties param = new Properties();
        param.put("metadata.broker.list", "rzx168:9092,rzx169:9092,rzx177:9092");
        param.put("request.required.acks", "-1");
        param.put("batch.num.messages", "200");
        //默认的分区器，可以自定义，看下面
        param.put("partitioner.class", "kafka.producer.DefaultPartitioner");

        Producer<byte[], byte[]> producer = new Producer<>(new ProducerConfig(param));

        for (int i = 0; i < 10000; i++){
            producer.send(new KeyedMessage<byte[],
            byte[]>("testWc1", ("我是第{ " + i + " }条数据").getBytes()));
        }
    }
}

自定义分区器

import kafka.producer.DefaultPartitioner;
import kafka.utils.VerifiableProperties;

/**
 * @Company 任子行网络技术股份有限公司
 * @CreateTime 2017/8/21  14:47
 */
public class KafkaPartitioner extends DefaultPartitioner {
    public KafkaPartitioner(VerifiableProperties props) {
        super(props);
    }

    public int partition(Object obj, int numPartitions) {
//        return Math.abs(obj.hashCode()%numPartitions);
        return 2;
    }
}

消费者API

import kafka.consumer.Consumer;
import kafka.consumer.ConsumerConfig;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
import kafka.message.MessageAndMetadata;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * @Company 任子行网络技术股份有限公司
 * @CreateTime 2017/8/21  14:17
 */
public class KafkaConsumer {
    public static void main(String[] args) {
        String topic = "testWc1";

        Properties param = new Properties();
        param.put("group.id", "test_wl_10");
        param.put("zookeeper.connect", "rzx168:2181,rzx169:2181,rzx177:2181/kafka");
        param.put("auto.offset.reset", "largest");
        param.put("auto.commit.interval.ms", "1000");
        param.put("partition.assignment.strategy", "roundrobin");

        ConsumerConfig config = new ConsumerConfig(param);
        ConsumerConnector consumerConn = Consumer.createJavaConsumerConnector(config);

        //创建topicCountMap
        Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
        topicCountMap.put(topic,2);

        Map<String, List<KafkaStream<byte[], byte[]>>> topicStreamsMap 
            = consumerConn.createMessageStreams(topicCountMap);
        //取出 `kafkaTest` 对应的 所有partition
        List<KafkaStream<byte[], byte[]>> everyPartition = topicStreamsMap.get(topic);
        //创建一个容量为20的线程池
        ExecutorService executor = Executors.newFixedThreadPool(9);
        //创建20个consumer threads
        for (int i = 0; i < everyPartition.size(); i++){
            ConsumerIterator<byte[], byte[]> iterator = everyPartition.get(i).iterator();
            executor.execute(new KafkaSimpleConsumer("消费者" + (i + 1), iterator));
        }
    }

    static class KafkaSimpleConsumer implements Runnable{
        String title;
        ConsumerIterator<byte[], byte[]> iterator;

        KafkaSimpleConsumer(String title, ConsumerIterator<byte[], byte[]> iterator){
            this.title = title;
            this.iterator = iterator;
        }

        @Override
        public void run() {
            while (iterator.hasNext()){
                MessageAndMetadata<byte[], byte[]> msgMetadata = iterator.next();
                String topic = msgMetadata.topic();
                int partition = msgMetadata.partition();
                byte[] message = msgMetadata.message();

                System.out.println("topic:" + topic + 
                    " | partition:" + partition + 
                    " | message:" + new String(message));
            }
        }
    }
}

注意问题：

如果需要在一个类中用相同的group.id多次连接kafka，需要每次都关闭上一次连接，否则的话，新创建的线程不会partition消费。

connector = Consumer.createJavaConsumerConnector(config);
messageStreams = connector.createMessageStreams(topicAndPartitionMap);
threadPool = Executors.newFixedThreadPool(threadPoolSize);

connector.shutdown();     //一定要关闭这三个对象，下次再重新创建
messageStreams.clear();
threadPool.shutdownNow();

kafka设置如下配置，那么producer生产数据的时候，会自动生成topic