spark streaming python实例_实时 Spark Streaming 消费示例

## StructuredLoghubWordCount.Scala

object StructuredLoghubSample {

def main(args: Array[String]) {

if (args.length < 7) {

System.err.println("Usage: StructuredLoghubSample " +

" " +

" []")

System.exit(1)

}

val Array(project, logStore, accessKeyId, accessKeySecret, endpoint, startingOffsets, maxOffsetsPerTrigger, outputPath, _*) = args

val checkpointLocation =

if (args.length > 8) args(8) else "/tmp/temporary-" + UUID.randomUUID.toString

val spark = SparkSession

.builder

.appName("StructuredLoghubSample")

.master("local[5]")

.getOrCreate()

import spark.implicits._

// Create DataSet representing the stream of input lines from loghub

val lines = spark

.readStream

.format("loghub")

.option("sls.project", project)

.option("sls.store", logStore)

.option("access.key.id", accessKeyId)

.option("access.key.secret", accessKeySecret)

.option("endpoint", endpoint)

.option("startingoffsets", startingOffsets)

.option("zookeeper.connect.address", "localhost:2181")

.option("maxOffsetsPerTrigger", maxOffsetsPerTrigger)

.load()

.selectExpr("CAST(content AS STRING)")

.as[String]

val query = lines.writeStream

.format("parquet")

.option("checkpointLocation", checkpointLocation)

.option("path", outputPath)

.outputMode("append")

.trigger(Trigger.ProcessingTime(30000))

.start()

query.awaitTermination()

}

}