Kafka使用入门

Kafka使用入门

kafka安装

参照http://kafka.apache.org/documentation.html#quickstart中的方法下载安装。

运行中kafka的JAVA进程（jps）

23210 Kafka
22566 QuorumPeerMain

kafka console api

基本命令有如下几个：

启动、关闭zookeeper

kafka依赖于zookeeper，而zookeeper集成在kafka中的，如果系统中的zookeeper正在运行，请直接忽略本部分内容。不要随意开启或关闭正在运行的zookeeper。

> bin/zookeeper-server-start.sh config/zookeeper.properties

zookeeper默认运行于localhost:2181端口，配置项在zookeeper.properties中。

启动、关闭kafka-server

启动命令为

> bin/kafka-server-start.sh config/server.properties

默认端口为9092，配置项在server.properties中。

topic相关命令

命令为

> bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test

> bin/kafka-topics.sh --list --zookeeper localhost:2181

> bin/kafka-topics.sh --delete --zookeeper localhsot:2181 --topic test

删除topic命令如果生效需要修改server.properties，使得

delete.topic.enable=true

consumer和producer命令

这两条是最常用的命令

> bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic test --from-beginning

最后的–from-beginning是从该topic的起始点开始消费数据，否则从最新处消费数据。

> bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test

console api 运行时的进程名

21009 ConsoleConsumer
24888 ConsoleProducer

kafka java api

使用maven构建项目最为方便，可以轻松的解决依赖项的问题。假定我们使用的依赖项如下所示：

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>kafka0.0.0</groupId>
  <artifactId>kafka0.0.0</artifactId>
  <version>0.0.1-SNAPSHOT</version>
  <description>kafka0.0.0</description>
  <properties>
        <kafka.version>0.8.2.1</kafka.version>
        <slf4j.version>1.7.12</slf4j.version>
        <junit.version>4.12</junit.version>
        <json.version>20160212</json.version>
  </properties> 
  <dependencies>   
        <dependency>
            <groupId>org.json</groupId>
            <artifactId>json</artifactId>
            <version>${json.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka_2.10</artifactId>
            <version>${kafka.version}</version>
            <scope>compile</scope>
        </dependency>

        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka-clients</artifactId>
            <version>${kafka.version}</version>
            <scope>compile</scope>
        </dependency>

        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>${junit.version}</version>
            <scope>test</scope>
        </dependency>
    </dependencies>
</project>

java producer

简单的kafka生产者java代码为：

package main.app1;

import java.util.Properties;

import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.ProducerConfig;

import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;

public class test1 
{
    private final KafkaProducer<String,String> producer;
    public final static String TOPIC = "test";

    private test1(){
        Properties props = new Properties();
        //配置kafka的端口
        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "host1:port1,host2:port2");

        //配置value的序列化类
        props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");
        //配置key的序列化类
        props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");

        //request.required.acks
        //0, which means that the producer never waits for an acknowledgement from the broker (the same behavior as 0.7). This option provides the lowest latency but the weakest durability guarantees (some data will be lost when a server fails).
        //1, which means that the producer gets an acknowledgement after the leader replica has received the data. This option provides better durability as the client waits until the server acknowledges the request as successful (only messages that were written to the now-dead leader but not yet replicated will be lost).
        //-1, which means that the producer gets an acknowledgement after all in-sync replicas have received the data. This option provides the best durability, we guarantee that no messages will be lost as long as at least one in sync replica remains.
        props.put(ProducerConfig.ACKS_CONFIG,"-1");

        producer = new KafkaProducer<String,String>(props);
    }

    void produce() {
        int messageNo = 0;
        final int COUNT = 100;

        while (messageNo < COUNT) {
            String key = String.valueOf(messageNo);
            String data = "{\"time\":\"2015-09-01T01:35:19Z\",\"url\":\"/foo\",\"user\":\"bob\",\"latencyMs\":66}";
            producer.send(new ProducerRecord<String,String>(TOPIC, data));
            System.out.println(data);
            messageNo ++;
        }

        producer.close();
    }

    public static void main( String[] args )
    {
        new test1().produce();
    }
}

执行时如果需要在eclipse或idea中远程执行，需要在server.properties中修改配置为：

advertised.host.name=host

java consumer

package main.app1;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import kafka.consumer.Consumer;
import kafka.consumer.ConsumerConfig;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;

/*
 * 详细可以参考：https://cwiki.apache.org/confluence/display/KAFKA/Consumer+Group+Example
 */
public class ConsumerDemo {
    private final ConsumerConnector consumer;
    private final String topic;
    private ExecutorService executor;

    public ConsumerDemo(String a_zookeeper, String a_groupId, String a_topic) {
        consumer = Consumer.createJavaConsumerConnector(createConsumerConfig(a_zookeeper,a_groupId));
        this.topic = a_topic;
    }

    public void shutdown() {
        if (consumer != null)
            consumer.shutdown();
        if (executor != null)
            executor.shutdown();
    }

    public void run(int numThreads) {
        Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
        topicCountMap.put(topic, new Integer(numThreads));
        Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer
                .createMessageStreams(topicCountMap);
        List<KafkaStream<byte[], byte[]>> streams = consumerMap.get(topic);

        // now launch all the threads
        executor = Executors.newFixedThreadPool(numThreads);

        // now create an object to consume the messages
        //
        int threadNumber = 0;
        for (final KafkaStream stream : streams) {
            executor.submit(new ConsumerMsgTask(stream, threadNumber));
            threadNumber++;
        }
    }

    private static ConsumerConfig createConsumerConfig(String a_zookeeper,
            String a_groupId) {
        Properties props = new Properties();
        props.put("zookeeper.connect", a_zookeeper);
        props.put("group.id", a_groupId);
        props.put("zookeeper.session.timeout.ms", "400");
        props.put("zookeeper.sync.time.ms", "200");
        props.put("auto.commit.interval.ms", "1000");

        return new ConsumerConfig(props);
    }

    public static void main(String[] arg) {
        String[] args = { "10.10.8.236:2181", "group-1", "pageviews", "1" };
        String zooKeeper = args[0];
        String groupId = args[1];
        String topic = args[2];
        int threads = Integer.parseInt(args[3]);

        ConsumerDemo demo = new ConsumerDemo(zooKeeper, groupId, topic);
        demo.run(threads);

        try {
            Thread.sleep(600000);
        } catch (InterruptedException ie) {

        }
        demo.shutdown();
    }
}
------------------------------------------------------------------------------
package main.app1;

import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;

public class ConsumerMsgTask implements Runnable {
    private KafkaStream m_stream;
    private int m_threadNumber;

    public ConsumerMsgTask(KafkaStream stream, int threadNumber) {
        m_threadNumber = threadNumber;
        m_stream = stream;
    }

    public void run() {
        ConsumerIterator<byte[], byte[]> it = m_stream.iterator();
        while (it.hasNext())
            System.out.println("Thread " + m_threadNumber + ": "
                    + new String(it.next().message()));
        System.out.println("Shutting down Thread: " + m_threadNumber);
    }
}