spark Streaming 直接消费Kafka数据，保存到 HDFS 实战编程实践

最近在学习spark streaming 相关知识，现在总结一下

主要代码如下

def createStreamingContext():StreamingContext ={
  val sparkConf = new SparkConf().setAppName("myStreamingText").setMaster(ConfigInfo.MasterConfig)
  sparkConf.set("spark.streaming.kafka.maxRetries", "100")
  sparkConf.set("spark.streaming.kafka.maxRatePerParititon", "1000")
  val ssc = new StreamingContext(sparkConf,Seconds(ConfigInfo.durationConfig))
  ssc.checkpoint(ConfigInfo.checkpointConfig)
  ssc
}

def createKafkaDstream(ssc : StreamingContext, kafkaParams : Map[String,String], topics : Set[String] ): InputDStream[(String,String)] = {

  val kafkaDstream = KafkaUtils.createDirectStream[String,String,StringDecoder, StringDecoder](ssc, kafkaParams, topics)
  kafkaDstream
}

val ssc = StreamingContext.getOrCreate(ConfigInfo.checkpointConfig, createStreamingContext)
val kafkaParams= Map[String,String]("me