KafkaSparkDemoMain

最新推荐文章于 2019-07-02 08:31:58 发布
tantieshuai
最新推荐文章于 2019-07-02 08:31:58 发布
阅读量247
点赞数
本文链接：https://blog.csdn.net/tantieshuai/article/details/70313571
版权
package kafka

import java.sql.DriverManager
import java.util.Properties

import DAO.{ScalaHbase, ScalaConn}
import kafka.kafkaToHbase.market_analysis
import kafka.serializer.StringDecoder
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hbase.TableName
import org.apache.hadoop.hbase.client.{Table, ConnectionFactory, Connection}
import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord}
import org.apache.spark.SparkConf
import org.apache.spark.streaming.dstream.{DStream, InputDStream}
import org.apache.spark.streaming.{Seconds, Minutes, Duration, StreamingContext}
import org.apache.spark.streaming.kafka.KafkaUtils

import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.streaming.kafka.KafkaUtils
/**
 * author 谭志坚
 * date   2007-03-16
 */
object KafkaSparkDemoMain {
  case class SP2PLOG(YMD: String, MOBILE: String, CHANNEL: String, REMARK: String, STATUS: String)
  def main(args: Array[String]): Unit = {

    val conf = new SparkConf().setAppName("kafkaDataTest").setMaster("local[2]")
    val sc = new SparkContext(conf)
    val ssc = new StreamingContext(sc, Seconds(10))
    ssc.checkpoint("D:\\BigData\\spark-warehouse")
    val topics = Set("SP2PLOG")
    val brokers = "192.168.100.110:9092,192.168.100.111:9092,192.168.100.112:9092"
    val kafkaParams = Map[String, String]("metadata.broker.list" -> brokers, "serializer.class" -> "kafka.serializer.StringEncoder")

    val kafkaStream = KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder](ssc, kafkaParams, topics)



//    val lines = kafkaStream.map(x => x._2)
    val events = kafkaStream.flatMap(line => {
      Some(line._2.toString())
    })
    try {
//      println("events========"+events.toString)
//      println("events========"+events.map(_.split(",").toString))
      val tmpdf = events.map(_.split(",")).map(x => SP2PLOG(x(0), x(1), x(2), x(3), x(4)))
      tmpdf.foreachRDD { rdd =>
        val spark =  ScalaConn.spark
        import spark.implicits._
        val dataFrame = rdd.toDF()
        dataFrame.createOrReplaceTempView("SP2PLOG")
        val bak_sql = spark.sql("select YMD,MOBILE,CHANNEL,REMARK,STATUS from SP2PLOG")

        if (bak_sql.collect().length > 0) {
          try {
            bak_sql.collect().foreach { userRow => {
              val YMD: String = String.valueOf(userRow.getString(0))
              val MOBILE: String = String.valueOf(userRow.getString(1))
              val CHANNEL: String = String.valueOf(userRow.getString(2))
              val REMARK: String = String.valueOf(userRow.getString(3))
              val STATUS: String = String.valueOf(userRow.getString(4))
              println("MOBILE ==" + MOBILE + "===========REMARK=" + REMARK + "  STATUS=" + STATUS)

            }
            }
          } catch {
            case e: Exception => {
              println("完成数据保存到Hbase后："+e.toString)
              e.printStackTrace
            }
          }

        }
      }
    }catch {
      case e: Exception =>
    }
    //                   将一行一行数据映射成对象,RT_SMS 获取申请手机号码注册，重置密码，绑定汇付
//    try {
//      //     将一行一行数据映射成对象,RT_SMS 获取申请手机号码注册，重置密码，绑定汇付
//      lines.filter(x => (!x.isEmpty
//        )).map(x => {
//        SP2PLOG_parse(x)
//      }).foreachRDD { rdd =>
//        val spark = ScalaConn.spark
//        import spark.implicits._
//        val dataFrame = rdd.toDF()
//        dataFrame.createOrReplaceTempView("RT_REG")
//        val RT_REG_sql = spark.sql("select YMD, MOBILE, CHANNEL,REMARK,STATUS from RT_REG")
//        println("RT_REG_sql.collect().length = " + RT_REG_sql.collect().length)
//        if (RT_REG_sql.collect().length > 0) {
//          try {
//            RT_REG_sql.collect().foreach { userRow => {
//              val YMD: String = String.valueOf(userRow.getString(0))
//              val MOBILE: String = String.valueOf(userRow.getString(1))
//              val CHANNEL: String = String.valueOf(userRow.getString(2))
//              val REMARK: String = String.valueOf(userRow.getString(3))
//              val STATUS: String = String.valueOf(userRow.getString(4))
//              //              println("MOBILE ==" + MOBILE + "===========REMARK=" + REMARK + "  STATUS=" + STATUS)
//              if (REMARK.contains("手机号注册") || STATUS.contains("注册成功")) {
//               println("再次获取展现: ================REMARK======"+REMARK+"+STATUS==========="+STATUS)
//              }
//              if (!YMD.isEmpty && !REMARK.isEmpty) {
//                println("再次获取展现: YMD========="+YMD+" =========================REMARK======"+REMARK)
//              }
//            }
//            }
//          } catch {
//            case e: Exception => {
//              println("完成数据保存到Hbase后："+e.toString)
//              e.printStackTrace
//            }
//          }
//        }
//      }
//    } catch {
//      case e: Exception => {
//        println("完成数据分析和运行后："+e.toString)
//        e.printStackTrace
//      }
//    }


    ssc.start()
    ssc.awaitTermination()
  }
}