kafka 实例

1、produce

import java.util.{Date, Properties}

import kafka.producer.{KeyedMessage, Producer, ProducerConfig}

import scala.util.Random

object ScalaProducerExample extends App {
  val events = args(0).toInt
  val topic = args(1)
  val brokers = args(2)
  val rnd = new Random()
  val props = new Properties()
  props.put("metadata.broker.list", brokers)
  props.put("serializer.class", "kafka.serializer.StringEncoder")
  props.put("producer.type", "async")

  val config = new ProducerConfig(props)
  val producer = new Producer[String, String](config)
  val t = System.currentTimeMillis()
  for (nEvents <- Range(0, events)) {
    val runtime = new Date().getTime();
    val ip = "192.168.210.16" + rnd.nextInt(255);
    val msg = runtime + "," + nEvents + ",www.example.com," + ip;
    val data = new KeyedMessage[String, String](topic, ip, msg);
    producer.send(data);
  }

  System.out.println("sent per second: " + events * 1000 / (System.currentTimeMillis() - t));
  producer.close();
}

2、消费者

import java.util.Properties
import java.util.concurrent._
import kafka.utils.Logging

import kafka.consumer.{Consumer, ConsumerConfig, KafkaStream}

class ScalaConsumerExample(val zookeeper: String,
                           val groupId: String,
                           val topic: String,
                           val delay: Long) extends Logging {

  val config = createConsumerConfig(zookeeper, groupId)
  val consumer = Consumer.create(config)
  var executor: ExecutorService = null

  def shutdown() = {
    if (consumer != null)
      consumer.shutdown();
    if (executor != null)
      executor.shutdown();
  }

  def createConsumerConfig(zookeeper: String, groupId: String): ConsumerConfig = {
    val props = new Properties()
    props.put("zookeeper.connect", zookeeper);
    props.put("group.id", groupId);
    props.put("auto.offset.reset", "largest");
    props.put("zookeeper.session.timeout.ms", "400");
    props.put("zookeeper.sync.time.ms", "200");
    props.put("auto.commit.interval.ms", "1000");
    val config = new ConsumerConfig(props)
    config
  }

  def run(numThreads: Int) = {
    val topicCountMap = Map(topic -> numThreads)
    val consumerMap = consumer.createMessageStreams(topicCountMap);
    val streams = consumerMap.get(topic).get;

    executor = Executors.newFixedThreadPool(numThreads);
    var threadNumber = 0;
    for (stream <- streams) {
      executor.submit(new ScalaConsumerTest(stream, threadNumber, delay))
      threadNumber += 1
    }
  }
}

object ScalaConsumerExample extends App {
  val example = new ScalaConsumerExample(args(0), args(1), args(2),args(4).toLong)
  example.run(args(3).toInt)
}

class ScalaConsumerTest(val stream: KafkaStream[Array[Byte], Array[Byte]], val threadNumber: Int, val delay: Long) extends Logging with Runnable {
  def run {
    val it = stream.iterator()

    while (it.hasNext()) {
      val msg = new String(it.next().message());
      System.out.println(System.currentTimeMillis() + ",Thread " + threadNumber + ": " + msg);
    }

    System.out.println("Shutting down Thread: " + threadNumber);
  }
}

3、运行方式

（1）使用maven打成jar包

1）消费者

java -cp /opt/app/kafka_2.10-0.8.2.1/test/KafkaDemo-1.0-SNAPSHOT.jar:/opt/app/kafka_2.10-0.8.2.1/libs/kafka_2.10-0.8.2.1.jar:/opt/app/scala-2.10.5/lib/scala-library.jar:/opt/app/kafka_2.10-0.8.2.1/libs/log4j-1.2.16.jar:/opt/app/kafka_2.10-0.8.2.1/libs/kafka-clients-0.8.2.1.jar:/opt/app/kafka_2.10-0.8.2.1/libs/zkclient-0.3.jar:/opt/app/kafka_2.10-0.8.2.1/libs/metrics-core-2.2.0.jar:/opt/app/kafka_2.10-0.8.2.1/libs/slf4j-log4j12-1.6.1.jar:/opt/app/kafka_2.10-0.8.2.1/libs/slf4j-api-1.7.6.jar:/opt/app/kafka_2.10-0.8.2.1/libs/zookeeper-3.4.6.jar com.colobu.kafka.ScalaProducerExample 1 mykafka1 192.168.210.16:9092
2）生产者
java -cp /opt/app/kafka_2.10-0.8.2.1/test/KafkaDemo-1.0-SNAPSHOT.jar:/opt/app/kafka_2.10-0.8.2.1/libs/kafka_2.10-0.8.2.1.jar:/opt/app/scala-2.10.5/lib/scala-library.jar:/opt/app/kafka_2.10-0.8.2.1/libs/log4j-1.2.16.jar:/opt/app/kafka_2.10-0.8.2.1/libs/kafka-clients-0.8.2.1.jar:/opt/app/kafka_2.10-0.8.2.1/libs/zkclient-0.3.jar:/opt/app/kafka_2.10-0.8.2.1/libs/metrics-core-2.2.0.jar:/opt/app/kafka_2.10-0.8.2.1/libs/slf4j-log4j12-1.6.1.jar:/opt/app/kafka_2.10-0.8.2.1/libs/slf4j-api-1.7.6.jar:/opt/app/kafka_2.10-0.8.2.1/libs/zookeeper-3.4.6.jar com.colobu.kafka.ScalaConsumerExample 192.168.210.16:2181 group1 mykafka1 1 0

一定要加载第三方包，否则报类找不到错误