package com.ultimate.music.streaming.tango
import java.sql.PreparedStatement
import com.ultimate.music.util.PropertyUtil._
import com.ultimate.music.util.{PropertyUtil}
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.spark.rdd.RDD
import org.apache.spark.streaming.kafka010.LocationStrategies._
import org.apache.spark.streaming.kafka010.{CanCommitOffsets, ConsumerStrategies, HasOffsetRanges, KafkaUtils}
import org.apache.spark.streaming.{Seconds, StreamingContext}
object Test {
val streamingInterval = PropertyUtil.getKey(STREAMING_INTERVAL).toLong
val spark = SparkSession
.builder()
.appName("streamingApp")
.config(sparkConf)
.getOrCreate()
//lazy val sc = new SparkContext(sparkConf)
val sc = spark.sparkContext
lazy val ssc = new StreamingContext(sc, Seconds(streamingInterval))
def main(args: Array[String]): Unit = {
val messages1 = KafkaUtils.createDirectStream[String, String](ssc, PreferConsistent, ConsumerStrategies.Subscribe[String, String]( PropertyUtil.getKey("topic.t1.api"):: Nil, kafkaParams))
messages1.foreachRDD(rdd => {
val offsetRanges = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
if (!rdd.isEmpty()) {
//do somthing else ...
// km.persistOffsets[String, String](rdd)
messages1.asInstanceOf[CanCommitOffsets].commitAsync(offsetRanges)
}
})
ssc.start()
//等待结束
ssc.awaitTermination()
}
}