mysql mtq_Maxwell：实时同步MySQL数据到消息队列Kafka

def main(args: Array[String]): Unit = {

val sparkConf: SparkConf = new SparkConf().setAppName("ods_gmall_maxwell_app").setMaster("local[4]")

val ssc = new StreamingContext(sparkConf, Seconds(5))

val topic = "ODS_DB_GMALL0317_M"

val groupId = "ods_gmall_maxwell_group"

var inputDstream: InputDStream[ConsumerRecord[String, String]]=null

val offsetMap: Map[TopicPartition, Long] = OffsetManager.getOffset(topic,groupId )

//2、 把偏移量交给kafka ，让kafka按照偏移量的位置读取数据流

if(offsetMap!=null){

inputDstream = MyKafkaUtil.getKafkaStream(topic, ssc,offsetMap, groupId)

}else{

inputDstream = MyKafkaUtil.getKafkaStream(topic, ssc,  groupId)

}

//3、  获得偏移量的结束位置

//从流中rdd 获得偏移量的结束点 数组

var offsetRanges: Array[OffsetRange]=null

val inputWithOffsetDstream: DStream[ConsumerRecord[String, String]] = inputDstream.transform { rdd =>

offsetRanges = rdd.asInstanceOf[HasOffsetRanges].offsetRanges

rdd

}

val jsonObjDstream: DStream[JSONObject] = inputWithOffsetDstream.map { record =>

val jsonString: String = record.value()

JSON.parseObject(jsonString)

}

// 数据筛选 ， 分流

jsonObjDstream.foreachRDD{rdd=>

rdd.foreach{jsonObj=>

val dataJson: String = jsonObj.getString("data")

val table: String = jsonObj.getString("table")

val topic="ODS_T_"+table.toUpperCase

println(topic+"::"+dataJson)

if(dataJson!=null&&dataJson.size>2&&jsonObj.getString("type")!=null&&(

(table=="order_info"&&jsonObj.getString("type")=="insert")

|| (table=="order_detail"&&jsonObj.getString("type")=="insert")

|| (table=="base_province"&&(jsonObj.getString("type")=="insert"||jsonObj.getString("type")=="update")||jsonObj.getString("type")=="bootstrap-insert")

|| (table=="user_info"&&(jsonObj.getString("type")=="insert"||jsonObj.getString("type")=="update")||jsonObj.getString("type")=="bootstrap-insert")

)){

// Thread.sleep(200)

MyKafkaSender.send(topic,dataJson)

}

}

OffsetManager.saveOffset(topic,groupId,offsetRanges)

}

ssc.start()

ssc.awaitTermination()

}