一、概述
工业物联网数据通过mqtt协议发送到emqtt,kafka订阅emqtt数据,sparkstreaming消费kafka数据和原始留存在oracle的信息表关联计算。
二、demo示例
package streamTest
import java.util.concurrent.Future
import java.util.{Date, Properties}
import com.google.gson.Gson
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord, RecordMetadata}
import org.apache.kafka.common.serialization.{StringDeserializer, StringSerializer}
import org.apache.spark._
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.sql.types._
import org.apache.spark.sql.{Row, SparkSession}
import org.apache.spark.streaming.dstream.InputDStream
import org.apache.spark.streaming.{Durations, Seconds, StreamingContext}
import org.apache.spark.streaming.kafka010._
object KafkaStreamTest {
def main(args: Array[String]): Unit = {
val property = new Properties()
val url = "jdbc:oracle:thin:@//xxx:1634/GPS"
property.put("user","GPS2")
property.put("password","123456")
val conf = new SparkConf().setAppName("kafkaStreamTest").set("spark.driver.allowMultipleContexts","true")
val ssc = new StreamingContext(conf, Durations.seconds(1))
val sparkSession = SparkSession.builder().appName("kafkaStreamTest").enableHiveSupport().getOrCreate()
val rfrunDF=sparkSession.read.jdbc(url,"t_rfrun",property)
val driverDF=sparkSession.read.jdbc(url,"t_driver",property)
val msgDF=rfrunDF.join(driverDF,rfrunDF("WORKNUM")===driverDF("WORKNUM"),"inner")
.select(rfrunDF("equipnum"),rfrunDF("worknum"),rfrunDF("worktime"),rfrunDF("unworktime"),driverDF("name"),driverDF("remark2"))
msgDF.persist()
val topics = Array("pocGPS01")
val kafkaParams = Map[String, Object](
"bootstrap.servers" -> "xxx:9092",
"key.deserializer" -> classOf[StringDeserializer],
"value.deserializer" -> classOf[StringDeserializer],
"group.id" -> "stream",
"auto.offset.reset" -> "latest",
"enable.auto.commit" -> (false: java.lang.Boolean)
)
val kafkaStream: InputDStream[ConsumerRecord[String, String]] = KafkaUtils.createDirectStream[String, String](
ssc,
LocationStrategies.PreferConsistent,
ConsumerStrategies.Subscribe[String, String](topics, kafkaParams)
)
val recieveStream=kafkaStream
.filter(_.value().length>0)
.filter(_.value().nonEmpty)
.map(x=>handleJson2CaseClass(x.value()))
.mapPartitions(iter=>{
iter.map(x=>{
var distance=getDistance(x.jd1,x.wd1,x.jd2,x.wd2)
(x.num1,x.num2,distance,x.alertTime)