spark读写数据到kafka

集群环境：CDH5.8.0 / spark1.6.0 / scala2.10.4

在使用时，我们需要添加相应的依赖包：

    <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-streaming-kafka_2.10</artifactId>
        <version>${spark.version}</version>
        <scope>${dependency.scope}</scope>
    </dependency>

基于Scala的基本使用方式如下：

package com.egridcloud.kafka

import java.util.Properties

import kafka.consumer._
import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord}
import org.apache.kafka.common.serialization.StringSerializer

import scala.collection.mutable.HashMap
/**
  * Created by LHX on 2018/3/16 0016 上午 11:00.
  * 测试集群kafka发送、接收数据
  */
class KafkaTestConsumer(val topic: String) extends Thread {
  var consumer: ConsumerConnector = _
  def init: KafkaTestConsumer = {
    val pro = new Properties()
    pro.put("zookeeper.connect", "10.122.17.129:2181")
    pro.put("group.id", "group1")
    pro.put("zookeeper.session.timeout.ms", "60000")
    this.consumer = Consumer.create(new ConsumerConfig(pro))
    this
  }
  override def run(): Unit = {
    val topicConfig = new HashMap[String, Int]()
    topicConfig += topic -> 1
    val message: collection.Map[String, List[KafkaStream[Array[Byte], Array[Byte]]]] = consumer.createMessageStreams(topicConfig)
    val kafkaStream: KafkaStream[Array[Byte], Array[Byte]] = message.get(topic).get(0)
    //循环接收kafka数据 TODO 思考
    val iter: ConsumerIterator[Array[Byte], Array[Byte]] = kafkaStream.iterator()
    while (iter.hasNext()) {
      val bytes: Array[Byte] = iter.next().message()
      println(s"receives:${new String(bytes)}")
      Thread.sleep(1000)
    }
  }
}
// 伴生对象
object KafkaTestConsumer {
  def apply(topic: String): KafkaTestConsumer = new KafkaTestConsumer(topic).init
}

class KafkaTestProducer(val topic: String) extends Thread {
  var producer: KafkaProducer[String, String] = _

  def init: KafkaTestProducer = {
    val props = new Properties()
    props.put("bootstrap.servers", "10.122.17.129:9095")
    props.put("key.serializer", classOf[StringSerializer].getName)
    props.put("value.serializer", classOf[StringSerializer].getName)
    this.producer = new KafkaProducer[String, String](props)
    this
  }
  override def run(): Unit = {
    var num = 1
    while (true) {
      //要发送的消息
      val messageStr = new String(s"test_${num}")
      println(s"send:${messageStr}")
      producer.send(new ProducerRecord[String, String](this.topic, messageStr))
      num += 1
      if (num > 10) num = 0
      Thread.sleep(3000)
    }
  }
}
// 伴生对象
object KafkaTestProducer {
  def apply(topic: String): KafkaTestProducer = new KafkaTestProducer(topic).init
}

object KafkaTest {
  def main(args: Array[String]): Unit = {
    val consumer = KafkaTestConsumer("wordcount01")
    val producer = KafkaTestProducer("wordcount01")
    consumer.start()
    producer.start()
  }
}

运行结果：

注意事项：

(1)kafaka中server.properties设置advertised.host.name= IP还是hostname

props.put("bootstrap.servers", "IP:9095")

(2)kafaka中server.properties设置advertised.port=端口号

10.122.17.129:9095

完整项目参考：GitHub