UserClickCountAnalytics kafka

package kafka

import java.sql.{DriverManager}
import java.text.SimpleDateFormat
import java.util.{Properties, Calendar, Date}

import DAO.{ScalaHbase, ScalaConn}
import kafka.serializer.StringDecoder
import org.apache.commons.lang.time.DateFormatUtils
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hbase.client._
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.hbase.{TableName, HBaseConfiguration}
import org.apache.kafka.clients.producer.{KafkaProducer, ProducerConfig}
import org.apache.spark.streaming.kafka.KafkaUtils
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql.{DataFrame, SQLContext}
import org.apache.hadoop.hbase.client.{Table, Connection,ConnectionFactory}
import java.util.HashMap;
import util.DateUtil

import scala.util.Random
import org.slf4j.LoggerFactory



object UserClickCountAnalytics {
 case class AdClicked ( timestamp:String, ip:String,  userID:String, adID:String, province:String,city:String)
  val updateFunc = (currentValues: Seq[Int], preValue: Option[Int]) => {
    val curr = currentValues.sum
    val pre = preValue.getOrElse(0)
    Some(curr + pre)
  }

  def main(args: Array[String]): Unit = {
    var masterUrl = "local[3]"
    if (args.length > 0) {
      masterUrl = args(0)
    }
    // Create a StreamingContext with the given master URL
    val conf = new SparkConf().setMaster(masterUrl).setAppName("UserClickCountAnalytics")
    val ssc = new StreamingContext(conf, Seconds(20))
    ssc.checkpoint("D:\\BigData\\spark-warehouse")


    /**
     * 创建Kafka元数据,来让Spark Streaming这个Kafka Consumer利用
     */
    val topics = Set("AdClicked")
    val brokers = "192.168.100.110:9092,192.168.100.111:9092,192.168.100.112:9092"
    val kafkaParameters = Map[String, String]("metadata.broker.list" -> brokers, "serializer.class" -> "org.apache.kafka.common.serialization.ByteArraySerializer")

    // Create a direct stream
    val adClickedStreaming = KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder](ssc, kafkaParameters, topics)
    adClickedStreaming.map(_._2)      // 取出value
      .flatMap(_.split("\t")) // 将字符串使 用空格分隔
      .map(r => (r, 1))      // 每个单词映射成一个pair
      .updateStateByKey[Int](updateFunc)  // 用当前batch的数据区更新已有的数据
      .print() // 打印前10个数据

    val events = adClickedStreaming.flatMap(line => {
      Some(line._2.toString())
//      val data = JSONObject.fromObject(line._2)
//      Some(data)
    })



    try {
      // Compute user click times
      val userClicks = events.map(_.split("\t")).map(x => (x(2).toString, x(5).toString)).reduceByKey(_ + _)
      userClicks.foreachRDD(rdd => {
        rdd.foreachPartition(partitionOfRecords => {
          partitionOfRecords.foreach(pair => {
            val uid = pair._1
            val clickCount = pair._2
            println(" uid = "+uid+" clickCount = "+clickCount)
          })
        })
      })

      val tmpdf = events.map(_.split("\t")).map(x => AdClicked(x(0).toString, x(1).toString, x(2).toString, x(3).toString,x(4).toString,x(5).toString))


      tmpdf.foreachRDD { rdd =>
        val spark =  ScalaConn.spark
        import spark.implicits._
        val dataFrame = rdd.toDF()

//        println("dataFrame ===== "+dataFrame.collectAsList().toString)
        //注册为 dataFrame.createOrReplaceTempView("user_events")
        dataFrame.createOrReplaceTempView("AdClicked")
        //备份数据的sql
        val bak_DataFrame = spark.sql("select timestamp,ip,userID,adID,province,city from AdClicked")
        bak_DataFrame.show()
        //获取当前批次数据的最大时间和最小时间
//        val time_DataFrame = spark.sql("select min(timestamp),max(timestamp) from user_events")
//        time_DataFrame.show()
//        if (bak_sql.collect().length > 0) {
//           println("bak_sql.collect().length  ==  "+bak_sql.collect().length)
          //#############################新增hbase或者修改hbase################################
//          var myConf:Configuration  = ScalaConn.getHbaseConf
//          var hbaseconn:Connection    =  ConnectionFactory.createConnection(myConf)

          //            val familyColumn: Array[String] = Array[String]("USERS")
//          ScalaHbase.createTable(myConf, "user_events",null)
//          val hTable: Table = hbaseconn.getTable(TableName.valueOf("user_events"))

//          }

      }
    }catch {
      case e: Exception =>
    }
    ssc.start()
    ssc.awaitTermination()
  }

}