sparkStreaming接收kafka消息,写入数据库
package com.ruijie.spark.stream
import org.apache.kafka.clients.consumer.ConsumerConfig
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.spark.streaming.kafka010.{ConsumerStrategies, KafkaUtils, LocationStrategies}
import com.alibaba.fastjson.JSON
import com.ruijie.bean.DeviceRscUsg
import com.ruijie.dao.DeviceRscUsgDao
import org.apache.spark.streaming.dstream.DStream
/**
* @ ClassName DevicePortConsumer
* @ Description TODO
* @ Author LXW
* @ Date 2020-06-06 13:57
**/
object DevicePortConsumer {
def main(args: Array[String]): Unit = {
// offset保存路径
val checkpointPath = "D:\\hadoop\\checkpoint\\kafka-direct"
val conf = new SparkConf()
.setAppName("devicePortConsumer")
.setMaster("local[2]")
val sc = new SparkContext(conf)
sc.setLogLevel("WARN")
val ssc = new StreamingContext(sc, Seconds(5))
ssc.checkpoint(checkpointPath)
val bootstrapServers = "172.18.130.210:9092"
val groupId = "lxw"
val topicName = "dev-port"
val maxPoll = 500
val kafkaParams = Map(
ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG -> bootstrapServers,
ConsumerConfig.GROUP_ID_CONFIG -> groupId,
ConsumerConfig.MAX_POLL_RECORDS_CONFIG -> maxPoll.toString,
ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG -> classOf[StringDeserializer],
ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG -> classOf[StringDeserializer]
)
val kafkaTopicDS = KafkaUtils.createDirectStream(ssc, LocationStrategies.PreferConsistent,
ConsumerStrategies.Subscribe[String, String](Set(topicName), kafkaParams))
val value: DStream[String] = kafkaTopicDS.map(_.value())
value.map(devPortMessage => {
print(devPortMessage)
val deviceRscUsg: DeviceRscUsg = JSON.parseObject(devPortMessage, classOf[DeviceRscUsg])
deviceRscUsg.setTs(new java.util.Date(deviceRscUsg.getTimestamp))
DeviceRscUsgDao.insert(deviceRscUsg)
}).print()
ssc.start()
ssc.awaitTermination()
}
}