Kafka编程实践

Kafka 编程实践

搭建了一个Kafka集群之后，看怎么用Java程序编写去利用它。

参考Kafka文档（ http://kafka.apache.org/documentation/），Kafka可以用在三个场景：

1. Messaging System
2. Storage System
3. Stream Processing

我用Java实现了一个“ 生产者-消费者”的场景。打算用一个工程，通过参数的形式来分别启动“生产者”和“消费者”进程。

建一个Java+Maven工程，在pom.xml中加个包：

<dependency>     <groupId>org.apache.kafka</groupId>     <artifactId>kafka-clients</artifactId>     <version>1.0.0</version> </dependency>

新建三个类：Main、Producer、Consumer

• Main，负责启动程序并解析参数，选择启动Producer还是Consumer；
• Producer，负责生产一批量的消息到一个topic
• Consumer，循环监听一个topic，有消息就打印

Producer类

消息：我设定消息是一个Json字符串，模拟得像样一点：

private   String genJsonObject() {              return   "{\"id\":\""   + java.util.UUID. randomUUID ().toString().substring(0, 13) +   "\""                         +   ", \"value\":\""   +   random .nextFloat() +   "\""                         +   ", \"timestamp\":\""   +   sdformat .format( new   java.util.Date()) +   "\""                         +   "}" ;       }

生产它们：

public   void   produce(String   topic , Long   number ) {                           random   =   new   java.util.Random(System. currentTimeMillis ());                           for   ( int   i   = 0;   i   <   number ;   i ++) {                   String   message   = genJsonObject();                   String   key   = java.util.UUID. randomUUID ().toString().substring(0, 13);                    producer .send( new   ProducerRecord<String, String>( topic ,   key ,   message ));                                      System. out .println( "Sent: "   +   message );                                       // 随机延时                    try   {                         Thread. sleep (1000 +   random .nextInt(5000) );                   }   catch   (InterruptedException   e ) {                          e .printStackTrace();                   }             }                          System. out .println( "finish producing." );       }

Consumer类

获取：

public   void   consume(String   topics ) {             java.util.List<String>   topicList   =   new   java.util.ArrayList<String>();                           for   (String   t   :   topics .split( "," )) {                    topicList .add( t );             }              consumer .subscribe( topicList );              while   ( true ) {                   ConsumerRecords<String, String>   records   =   consumer .poll(1);                    for   (ConsumerRecord<String, String>   record   :   records )                         System. out .printf( "offset = %d, key = %s, value = %s%n" ,   record .offset(),   record .key(),   record .value());             }       }

Main类

package   kafkademo; public   class   Main {        static   public   void   main(String   args []) {              switch   ( args [0]) {              case   "consumer" :                    runAsConsumer ( args );                    break ;              case   "producer" :                    runAsProducer ( args );                    break ;                                 default :                    usage ();                    return ;             }       }        private   static   void   usage() {             System. out .println( "Usage:" );             System. out .println( "    Program type:" );             System. out .println( "        consumer    run as a consumer." );             System. out .println( "        producer    run as a producer." );             System. out .println();             System. out .println( "    Options:" );             System. out .println( "--server=host:port  kafka entry point." );             System. out .println( "--number=number     number of message to produce." );             System. out .println( " --topic=topic      topic of message to produce." );             System. out .println( "--topics=topics     topics of message to fetch from as a consumers." );       }        private   static   void   runAsProducer(String   args []) {                          String   host   =   "localhost:9092" ;             Long   number   = 1L;             String   topic   =   "test-demo" ;             java.util.Properties   props   =   new   java.util.Properties();                           for (String   arg   :   args ) {                    if   ( arg .startsWith( "--server=" )) {                          host   =   arg .split( "=" )[1];   // "49.4.67.131:9092");                   }                                       if   ( arg .startsWith( "--number=" )) {                         String   numString   =   arg .split( "=" )[1];                          if   ( "INFINITE" .equals( numString )) {                                number   = Long. MAX_VALUE ;                         }   else   {                                number   = Long. parseLong ( numString );                         }                   }                    if   ( arg .startsWith( "--topic=" )) {                          topic   =   arg .split( "=" )[1];                   }             }                          Producer   producer   =   new   Producer();                           props .put( "bootstrap.servers" ,   host );              producer .init( props );              producer .produce( topic ,   number );                           producer .tearDown();                    }        private   static   void   runAsConsumer(String   args []) {             String   host   =   "localhost:9092" ;             String   topics   =   "test-demo" ;             java.util.Properties   props   =   new   java.util.Properties();                           for (String   arg   :   args ) {                    if   ( arg .startsWith( "--server=" )) {                          host   =   arg .split( "=" )[1];   // "49.4.67.131:9092");                   }                                       if   ( arg .startsWith( "--topics=" )) {                          topics   =   arg .split( "=" )[1];                   }             }                          Consumer   consumer   =   new   Consumer();                           props .put( "bootstrap.servers" ,   host );              consumer .init( props );              consumer . consume ( topics );       } }

---

运行起来

生产：

消费：