大数据系列教程_分布式消息队列kafka

分布式消息队列kafka

1、启动kafka，每个server节点需要 ./kafka-server-start.sh /home/node6/kafka2800811/config/server.properties & 1、  创建主题： ./kafka-topics.sh --create  --topic mytopic --replication-factor  3  --partitions 1  --zookeeper zookeeper1:2181,zookeeper2:2181,zookeeper3:2181 2、  生产者代码 package com.yth.test;   import java.util.Properties;   import kafka.javaapi.producer.Producer; import kafka.producer.KeyedMessage; import kafka.producer.ProducerConfig; /**  *  * @author ASh  * @date 2014年9月10日下午10:47:01  */ public class Producers {   public static void main(String[] args) {                                  Properties props = new Properties();         props.put("metadata.broker.list","node6:9092,node7:9093,node8:9094");         props.put("serializer.class", "kafka.serializer.StringEncoder");         // key.serializer.class默认为serializer.class         props.put("key.serializer.class", "kafka.serializer.StringEncoder");         // 可选配置，如果不配置，则使用默认的partitioner              // 触发acknowledgement机制，否则是fire and forget，可能会引起数据丢失         // 值为0,1,-1,可以参考         // http://kafka.apache.org/08/configuration.html         props.put("request.required.acks", "1");            ProducerConfig config =  new  ProducerConfig(props);             Producer<String, String> producer =  new  Producer<String, String>(config);                           // The message is sent to a randomly selected partition registered in ZK              KeyedMessage<String, String> km=new KeyedMessage<String, String>("mytopic", "11111111");            producer.send(km);                              producer.close(); } }

 

3、  消费者代码   package com.yth.test; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors;   import kafka.consumer.Consumer; import kafka.consumer.ConsumerConfig; import kafka.consumer.KafkaStream; import kafka.javaapi.consumer.ConsumerConnector;   public class ConsumerDemo {     private final ConsumerConnector consumer;     private final String topic;     private ExecutorService executor;       public ConsumerDemo(String a_zookeeper, String a_groupId, String a_topic) {         consumer = Consumer.createJavaConsumerConnector(createConsumerConfig(a_zookeeper,a_groupId));         this.topic = a_topic;     }       public void shutdown() {         if (consumer != null)             consumer.shutdown();         if (executor != null)             executor.shutdown();     }       public void run(int numThreads) {         Map<String, Integer> topicCountMap = new HashMap<String, Integer>();         topicCountMap.put(topic, new Integer(numThreads));         Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer                 .createMessageStreams(topicCountMap);         List<KafkaStream<byte[], byte[]>> streams = consumerMap.get(topic);           // now launch all the threads         executor = Executors.newFixedThreadPool(numThreads);           // now create an object to consume the messages         //         int threadNumber = 0;         for (final KafkaStream stream : streams) {             executor.submit(new ConsumerMsgTask(stream, threadNumber));             threadNumber++;         }     }       private static ConsumerConfig createConsumerConfig(String a_zookeeper,             String a_groupId) {         Properties props = new Properties();         props.put("zookeeper.connect", a_zookeeper);         props.put("group.id", a_groupId);         props.put("zookeeper.session.timeout.ms", "400");         props.put("zookeeper.sync.time.ms", "200");         props.put("auto.commit.interval.ms", "1000");           return new ConsumerConfig(props);     }       public static void main(String[] arg) {         String[] args = { "zookeeper1:2181,zookeeper2:2181,zookeeper3:2181", "group-1", "mytopic", "12" };         String zooKeeper = args[0];         String groupId = args[1];         String topic = args[2];         int threads = Integer.parseInt(args[3]);           ConsumerDemo demo = new ConsumerDemo(zooKeeper, groupId, topic);         demo.run(threads);           try {             Thread.sleep(10000);         } catch (InterruptedException ie) {           }         demo.shutdown();     } } // package com.yth.test;   import kafka.consumer.ConsumerIterator; import kafka.consumer.KafkaStream;   public class ConsumerMsgTask implements Runnable {     private KafkaStream m_stream;     private int m_threadNumber;       public ConsumerMsgTask(KafkaStream stream, int threadNumber) {         m_threadNumber = threadNumber;         m_stream = stream;     }       public void run() {         ConsumerIterator<byte[], byte[]> it = m_stream.iterator();         while (it.hasNext())             System.out.println("Thread " + m_threadNumber + ": "                     + new String(it.next().message()));         System.out.println("Shutting down Thread: " + m_threadNumber);     } }