智慧出行-当我们想使spark集成kafka的时候,需要创建一个kafkaParams,这个map集合一般使用IDEA的args传参,文章将会演示,实现步骤

package com.cartravel.programApp

import org.codehaus.jackson.map.deser.std.StringDeserializer

object StreamApp {
  def main(args: Array[String]): Unit = {
    //传入5个参数,如果不足5个,直接打印异常,退出
    if (args.length<5){
      System.err.println("Usage:\n"+
        "[batch-duration-in-seconds] \n"+
        "[kafka-bootstrap-servers] \n"+
        "[kafka-topics] \n"+
        "[kafka-consumer-group-id] \n"+
        "[kafka-zoopeeper-quorum] "
      )
      System.exit(1)
    }
    val batchDuration = args(0) //流处理的批次间隔
    val bootstrapServers = args(1).toString //指定kafka是那他机器
    val topicsSet = args(2).toString.split(",").toSet
    val consumerGroupID = args(3)//消费组
    val zkQuorum = args(4) //zk的地址

    val topics = topicsSet.toArray
    val kafkaParams = Map[String,Object](
      "bootstrap.servers"->bootstrapServers,
      "key.deserializer"->classOf[StringDeserializer],
      "value.deserializer"->classOf[StringDeserializer],
      "group.id"->consumerGroupID,
      "auto.offset.reset"->"latest",
      "enable.auto.commiy"->"false"
    )
  }

}