package kafka import java.sql.{DriverManager} import java.text.SimpleDateFormat import java.util.{Properties, Calendar, Date} import DAO.{ScalaHbase, ScalaConn} import kafka.serializer.StringDecoder import org.apache.commons.lang.time.DateFormatUtils import org.apache.hadoop.conf.Configuration import org.apache.hadoop.hbase.client._ import org.apache.hadoop.hbase.util.Bytes import org.apache.hadoop.hbase.{TableName, HBaseConfiguration} import org.apache.kafka.clients.producer.{KafkaProducer, ProducerConfig} import org.apache.spark.streaming.kafka.KafkaUtils import org.apache.spark.streaming.{Seconds, StreamingContext} import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.sql.{DataFrame, SQLContext} import org.apache.hadoop.hbase.client.{Table, Connection,ConnectionFactory} import java.util.HashMap; import util.DateUtil import scala.util.Random import org.slf4j.LoggerFactory object UserClickCountAnalytics { case class AdClicked ( timestamp:String, ip:String, userID:String, adID:String, province:String,city:String) val updateFunc = (currentValues: Seq[Int], preValue: Option[Int]) => { val curr = currentValues.sum val pre = preValue.getOrElse(0) Some(curr + pre) } def main(args: Array[String]): Unit = { var masterUrl = "local[3]" if (args.length > 0) { masterUrl = args(0) } // Create a StreamingContext with the given master URL val conf = new SparkConf().setMaster(masterUrl).setAppName("UserClickCountAnalytics") val ssc = new StreamingContext(conf, Seconds(20)) ssc.checkpoint("D:\\BigData\\spark-warehouse") /** * 创建Kafka元数据,来让Spark Streaming这个Kafka Consumer利用 */ val topics = Set("AdClicked") val brokers = "192.168.100.110:9092,192.168.100.111:9092,192.168.100.112:9092" val kafkaParameters = Map[String, String]("metadata.broker.list" -> brokers, "serializer.class" -> "org.apache.kafka.common.serialization.ByteArraySerializer") // Create a direct stream val adClickedStreaming = KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder](ssc, kafkaParameters, topics) adClickedStreaming.map(_._2) // 取出value .flatMap(_.split("\t")) // 将字符串使 用空格分隔 .map(r => (r, 1)) // 每个单词映射成一个pair .updateStateByKey[Int](updateFunc) // 用当前batch的数据区更新已有的数据 .print() // 打印前10个数据 val events = adClickedStreaming.flatMap(line => { Some(line._2.toString()) // val data = JSONObject.fromObject(line._2) // Some(data) }) try { // Compute user click times val userClicks = events.map(_.split("\t")).map(x => (x(2).toString, x(5).toString)).reduceByKey(_ + _) userClicks.foreachRDD(rdd => { rdd.foreachPartition(partitionOfRecords => { partitionOfRecords.foreach(pair => { val uid = pair._1 val clickCount = pair._2 println(" uid = "+uid+" clickCount = "+clickCount) }) }) }) val tmpdf = events.map(_.split("\t")).map(x => AdClicked(x(0).toString, x(1).toString, x(2).toString, x(3).toString,x(4).toString,x(5).toString)) tmpdf.foreachRDD { rdd => val spark = ScalaConn.spark import spark.implicits._ val dataFrame = rdd.toDF() // println("dataFrame ===== "+dataFrame.collectAsList().toString) //注册为 dataFrame.createOrReplaceTempView("user_events") dataFrame.createOrReplaceTempView("AdClicked") //备份数据的sql val bak_DataFrame = spark.sql("select timestamp,ip,userID,adID,province,city from AdClicked") bak_DataFrame.show() //获取当前批次数据的最大时间和最小时间 // val time_DataFrame = spark.sql("select min(timestamp),max(timestamp) from user_events") // time_DataFrame.show() // if (bak_sql.collect().length > 0) { // println("bak_sql.collect().length == "+bak_sql.collect().length) //#############################新增hbase或者修改hbase################################ // var myConf:Configuration = ScalaConn.getHbaseConf // var hbaseconn:Connection = ConnectionFactory.createConnection(myConf) // val familyColumn: Array[String] = Array[String]("USERS") // ScalaHbase.createTable(myConf, "user_events",null) // val hTable: Table = hbaseconn.getTable(TableName.valueOf("user_events")) // } } }catch { case e: Exception => } ssc.start() ssc.awaitTermination() } }
09-05
09-05
09-05