scala操作kafka

1. 创建sbt eclipse 项目
   详情请参考：https://blog.csdn.net/weixin_44122028/article/details/103881508
2. 加入kafka依赖

name := "FirstKafka4Sbt"
scalaVersion := "2.11.11"
autoScalaLibrary := false
organization := "com.dd"
libraryDependencies ++= Seq(
   "org.apache.kafka" %% "kafka" % "0.10.0.0" % "provided"
)

3. 生产者 push message代码实现

import org.apache.kafka.clients.producer.KafkaProducer
import java.util.Properties
import org.apache.kafka.clients.producer.ProducerRecord
import org.apache.kafka.clients.producer.ProducerConfig
import org.apache.kafka.common.serialization.StringSerializer
/**
 1. scala实现kafka producer工具类
 */
object KafkaProducerUtil {
  //将生产者对象的获取封装到方法中
  def getKafkaProducer(brokerList: String): KafkaProducer[String, String] = {
    val properties = new Properties()
    properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList)
    properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, classOf[StringSerializer].getName) //key的序列化;
    properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, classOf[StringSerializer].getName) //value的序列化;
    var producer4Kafka = new KafkaProducer[String, String](properties)
    return producer4Kafka
  }
  def main(args: Array[String]): Unit = {
    //定义broker list,topic
    val brokersList = "sc-slave7:6667,sc-slave8:6667"
    val topic:String = "TestKafka_scala"
    //获取生产者对象
    var producer4Kafka = KafkaProducerUtil.getKafkaProducer(brokersList)
    //发送实际的message
    producer4Kafka.send(new ProducerRecord(topic,"this is a message"))
    //发送完成后关闭链接
    producer4Kafka.close;
    println("done!")
  }
}

4. 消费者 pull message代码实现

import org.apache.kafka.clients.producer.KafkaProducer
import java.util.Properties
import org.apache.kafka.clients.producer.ProducerRecord
import org.apache.kafka.clients.producer.ProducerConfig
import org.apache.kafka.common.serialization.StringSerializer
import org.apache.kafka.clients.consumer.KafkaConsumer
import java.util.Collections
import org.apache.kafka.clients.consumer.ConsumerConfig
import org.apache.kafka.common.serialization.StringDeserializer
/**
 * scala实现kafka consumer工具类
 */
object KafkaConsumerUtil {
  //将消费者对象的获取封装到方法中，注意groupid是必选项，此为与java api不相同之处
  def getKafkaConsumer(brokerList: String, topic: String, consumerGroupId: String): KafkaConsumer[String, String] = {
    val properties = new Properties()
    properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList)
    properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, classOf[StringDeserializer].getName) //key的序列化;
    properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, classOf[StringDeserializer].getName) //value的序列化;
    properties.put(ConsumerConfig.GROUP_ID_CONFIG, consumerGroupId) //指定groupid
    var consumer4Kafka = new KafkaConsumer[String, String](properties)
    consumer4Kafka.subscribe(Collections.singletonList(topic))
    return consumer4Kafka
  }
  def main(args: Array[String]): Unit = {
    //指定broker list列表
    val brokersList = "sc-slave7:6667"
    //必须指定消费者组id
    var consumerGroupId = "TestConsumerID"
    val topic: String = "TestKafka_scala"
    var consumer4Kafka = KafkaConsumerUtil.getKafkaConsumer(brokersList, topic, consumerGroupId)
    //注意用标志位做循环判断
    var runnable = true
    while (runnable) {
      val records = consumer4Kafka.poll(100)
      var iter = records.iterator()
      while (iter.hasNext()) {
        val record = iter.next()
        println(record.offset() + "--" + record.key() + "--" + record.value())
      }
    }
    consumer4Kafka.close()
    println("done!")
  }
}