精髓 spark kafka producer和Consumer

 Spark 消费kafka消息，并且写入Producer中作用域懒加载（foreachPartition）级别（部分转载-部分原创:QQ438509676）

===============================version===========================================

<scala.version>2.11.8</scala.version>
<spark.version>2.2.0</spark.version>
<kafka.version>0.10.2.1</kafka.version>

===============================Pom xml===========================================

<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-streaming_2.11</artifactId>
    <version>2.2.0</version>
</dependency>

<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-streaming-kafka-0-10_2.11</artifactId>
    <version>2.2.0</version>
    <exclusions>
        <exclusion> <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId></exclusion>
        <exclusion> <groupId>log4j</groupId> <artifactId>log4j</artifactId> </exclusion>
    </exclusions>
</dependency>

===============================Spark kafka========================================

import java.util.Properties
import com.mobike.dmp.utils.KafkaSink
import org.apache.kafka.clients.producer.{ProducerRecord, KafkaProducer}
import org.apache.kafka.common.serialization.{StringSerializer, StringDeserializer}
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.streaming.{StreamingContext, Seconds}
import org.apache.spark.{SparkContext, SparkConf}
import org.apache.spark.streaming.kafka010._
import org.apache.spark.streaming.kafka010.LocationStrategies.PreferConsistent
import org.apache.spark.streaming.kafka010.ConsumerStrategies.Subscribe

/**
  * Created by JohnVictor on 2018/9/11.
  */
object BrushOrderPushTj {
  def main(args: Array[String]) {
    val conf = new SparkConf()//.setMaster("local[*]").
     // .setAppName("T10LOCATION")
    val ssc = new StreamingContext(conf, Seconds(1800))
    val kafkaParams = Map[String, Object](
      "bootstrap.servers" -> "localhost:9092,localhost:9093,localhost:9094",
      "key.deserializer" -> classOf[StringDeserializer],
      "value.deserializer" -> classOf[StringDeserializer],
      "group.id" -> "group_id_bikeinfo",
      "auto.offset.reset" -> "latest",
      "enable.auto.commit" -> (false: java.lang.Boolean)
    )
    val kafkaProducer: Broadcast[KafkaSink[String, String]] = {
      val kafkaProducerConfig = {
        val p = new Properties()
        p.setProperty("bootstrap.servers","localhost1:9092,localhost1:9093,localhost1:9094")
        p.setProperty("key.serializer", classOf[StringSerializer].getName)
        p.setProperty("value.serializer", classOf[StringSerializer].getName)
        p
      }
      println("kafka producer init done!")
      ssc.sparkContext.broadcast(KafkaSink[String, String](kafkaProducerConfig))
    }
    val topics = Array("T10LOCATION")
    val stream = KafkaUtils.createDirectStream[String, String](
      ssc,
      PreferConsistent,
      Subscribe[String, String](topics, kafkaParams)
    )//.repartition(10)

    stream.foreachRDD { recordBikeInfos =>
      val offsetRanges = recordBikeInfos.asInstanceOf[HasOffsetRanges].offsetRanges
      if (!recordBikeInfos.isEmpty) {
        recordBikeInfos.foreach(record => {
          kafkaProducer.value.send("T10LOCATION", record.value())
          println("info :"+record.value())
          println("successed send kafka data")
          // do something else
        })
      }
      stream.asInstanceOf[CanCommitOffsets].commitAsync(offsetRanges)
    }
    ssc.start()
    ssc.awaitTermination()
  }
}

===============================KafkaSink 懒加载========================================

 

/**
  *
  *对于每个partition的每条记录，我们都需要创建KafkaProducer，
  *然后利用producer进行输出操作，注意这里我们并不能将KafkaProducer的新建任务放在foreachPartition外边，
  * 因为KafkaProducer是不可序列化的（not serializable）。
  * 显然这种做法是不灵活且低效的，因为每条记录都需要建立一次连接
  *
  */

import java.util.concurrent.Future
import org.apache.kafka.clients.producer.{ KafkaProducer, ProducerRecord, RecordMetadata }
class KafkaSink[K, V](createProducer: () => KafkaProducer[K, V]) extends Serializable {
  /* This is the key idea that allows us to work around running into
     NotSerializableExceptions. */
  lazy val producer = createProducer()
  def send(topic: String, key: K, value: V): Future[RecordMetadata] =
    producer.send(new ProducerRecord[K, V](topic, key, value))
  def send(topic: String, value: V): Future[RecordMetadata] =
    producer.send(new ProducerRecord[K, V](topic, value))
}

object KafkaSink {

  import scala.collection.JavaConversions._

  def apply[K, V](config: Map[String, Object]): KafkaSink[K, V] = {
    val createProducerFunc = () => {
      val producer = new KafkaProducer[K, V](config)
      sys.addShutdownHook {
        // Ensure that, on executor JVM shutdown, the Kafka producer sends
        // any buffered messages to Kafka before shutting down.
        producer.close()
      }
      producer
    }
    new KafkaSink(createProducerFunc)
  }

  def apply[K, V](config: java.util.Properties): KafkaSink[K, V] = apply(config.toMap)
}