Kafka生产者和消费者，并保证数据有序

生产者

package com.msb.bigdata.spark.streaming

import java.util.Properties
import java.util.concurrent.Future

import org.apache.kafka.clients.producer.{KafkaProducer, ProducerConfig, ProducerRecord, RecordMetadata}
import org.apache.kafka.common.serialization.StringSerializer

object KafkaProducer {
  def main(args: Array[String]): Unit = {
    val pros = new Properties()
    // 配置broker-list
    pros.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.100.123:9092,192.168.100.123:9093,192.168.100.123:9094")
    // 配置key的编码类
    pros.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, classOf[StringSerializer])
    // 配置value的编码类
    pros.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, classOf[StringSerializer])
    // 获取kafka producer
    val producer = new KafkaProducer[String, String](pros)
    while (true) {
      for (i <- 1 to 4; j <- 1 to 2) {
        val record = new ProducerRecord[String, String]("test", s"item$j", s"action$i")
        val records: Future[RecordMetadata] = producer.send(record)
        val metadata: RecordMetadata = records.get()
        val partition: Int = metadata.partition()
        val offset: Long = metadata.offset()
        println(s"[key:item$j, value:action$i, partition:$partition, offset:$offset]")
      }
      Thread.sleep(1000)
    }
    producer.close()
  }
}

消费者

package com.zjw.spark.streaming

import java.util
import java.util.Properties
import java.util.regex.Pattern

import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRebalanceListener, ConsumerRecord, ConsumerRecords, KafkaConsumer, OffsetAndMetadata}
import org.apache.kafka.common.TopicPartition
import org.apache.kafka.common.serialization.StringDeserializer

object KafkaConsumer {
  def main(args: Array[String]): Unit = {
    val pros = new Properties()
    // 配置kafka broker-list
    pros.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.100.123:9092,192.168.100.123:9093,192.168.100.123:9094")
    // 配置key的反序列化类
    pros.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, classOf[StringDeserializer])
    // 配置value的反序列化类
    pros.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, classOf[StringDeserializer])
    // 配置消费者的消费方式 earliest:按最近未消费的开始消费 latest:按最新位置消费，历史数据不关心
    pros.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest")
    // 是否自动提交offset，默认为true，表示消费者会在拉取数据之后立即提交offset到kafka自己维护的一个topic中，
    pros.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false")
    // 配置消费者组ID
    pros.put(ConsumerConfig.GROUP_ID_CONFIG, "bula5")
    // 得到一个KafkaConsumer对象
    val consumer = new KafkaConsumer[String, String](pros)
    // consumer消费数据分为两种方式，一种是subscribe，另一种是assign
    consumer.subscribe(Pattern.compile("test"), new ConsumerRebalanceListener {
      // 当消费者消费的partition变化时被调起
      override def onPartitionsRevoked(partitions: util.Collection[TopicPartition]): Unit = {
        // 打印当前消费者之前获得的分区情况（以前的分区）
        println(s"onPartitionsRevoked.........")
        val iter = partitions.iterator()
        while (iter.hasNext) {
          println(iter.next())
        }
      }

      // 当消费者消费的partition变化时被调起
      override def onPartitionsAssigned(partitions: util.Collection[TopicPartition]): Unit = {
        // 打印当前消费者重新分配之后分分区情况（分配给当前消费者的分区）
        println(s"onPartitionsAssigned.........")
        val iter = partitions.iterator()
        while (iter.hasNext) {
          println(iter.next())
        }
        //这里可以调用数据库或者第三方存储来获取offset并使用seek函数调整offset
        consumer.seek(new TopicPartition("test",1),220)
      }
    })
    val offMap = new util.HashMap[TopicPartition, OffsetAndMetadata]()
    var record: ConsumerRecord[String, String] = null;

    while (true) {
      // 拉取数据进行消费
      val records: ConsumerRecords[String, String] = consumer.poll(100)

      if (!records.isEmpty) {
        println(s"----${records.count()}----")
        // 获取拉回数据的一个迭代器
        val iter = records.iterator()
        while (iter.hasNext) {
          record = iter.next()
          val topic: String = record.topic()
          val partition: Int = record.partition()
          val offset = record.offset()

          val key: String = record.key()
          val value: String = record.value()
          println(s"[key:$key, value:$value, topic:$topic, partition:$partition, offset:$offset]")
        }

        //手动维护offset有两类地方：
        // 1)手动维护到kafka
        // 2)维护到第三方存储引擎如zk、mysql、redis等
        val partition = new TopicPartition("test", record.partition())
        val offset = new OffsetAndMetadata(record.offset())

        offMap.put(partition, offset)
        consumer.commitSync(offMap)
      }
    }
  }
}