kafka+Structured Streaming+s3+dynamodb

本文主要介绍从kafka消费数据，并通过两个业务需求(统计PV,UV)，然后分别输出到dynamodb和S3的demo,demo仅做演示逻辑，无法直接使用

def main(args: Array[String]) {
    val bucket:String = _
    val pvCheckLocation:String = _
    val uvCheckLocation:String = _
    val uvPath:String = _
    //create spark session
    val spark = SparkSession.builder()
      .master("local[*]")
      .config("spark.eventLog.enabled", "false")
      .config("spark.driver.memory", "2g")
      .config("spark.executor.memory", "2g")
      .config("spark.sql.shuffle.partitions","4")
      .appName("kafkaDemoToS3AndDynamoDB")
      .getOrCreate()
    import spark.implicits._
    //read stream from kafka
    val kafkaStream = spark
      .readStream
      .format("kafka")
      .option("kafka.bootstrap.servers", "host1:port1,host2:port2")
      .option("subscribe", "topic1,topic2")
      .option("startingOffsets", "latest")
      .option("failOnDataLoss", false)
      .option("maxOffsetsPerTrigger", 10L)
      .load()
      .selectExpr("CAST(value AS STRING)",
        "CAST(topic as STRING)",
        "CAST(partition as INTEGER)")
      .as[(String, String, Integer)]
    val cols = List("domain", "ip", "timestamp")
    //parse data to entity
    val df = kafkaStream.map { line =>
    val columns = line._1.split(" ")
      (columns(0), columns(1), columns(2))
    }.toDF(cols:_*)

    //all logs
    val ds = df.distinct().select($"domain", $"ip", $"timestamp").as[ClickStream]

    //compute pv
    val dsPV = ds.map(x => (x.domain,(x.ip,x.timestamp))).groupBy(x.domain)....count()
    //compute uv
    val dsUV = ds.map(x => (x.domain,x.ip)).groupBy(x.domain).....count()

    //save pv to dynamodb
    val queryPVCount = dsPV.writeStream
      .outputMode("update")
      .trigger(Trigger.ProcessingTime("10 seconds"))
      .option("checkpointLocation", pvCheckLocation)
      .foreach(new ForeachWriter[Row] {
      var ddbClient: AmazonDynamoDBClient = _

      override def open(partitionId: Long, version: Long) = {
        ddbClient = new dynamodbv2.AmazonDynamoDBClient()
        true
      }

      override def process(value: Row) = {
        val put = new PutItemRequest()
        put.setTableName("table")
        var attr = new AttributeValue()
        attr.setS(value.getAs[String](0))
        put.addItemEntry("domain", attr)
        attr = new AttributeValue()
        attr.setN(value.getAs[Long](1).toString)
        put.addItemEntry("pv", attr)
        ddbClient.putItem(put)
      }

      override def close(errorOrNull: Throwable) = {
      }
    }
    ).start()

    //save uv to s3
    val queryUVCount = dsUV.writeStream
      .outputMode("update")
      .trigger(Trigger.ProcessingTime("10 seconds"))
      .option("checkpointLocation", uvCheckLocation)
      .start("s3://" + bucket+ "/" +  uvPath)

    queryPVCount.awaitTermination()
    queryUVCount.awaitTermination()
  }
  //entity
  case class ClickStream(domain:String, ip:String,timestamp:Long)