案例
kafka中读取数据
package com.xyf.demo2 import org.apache.spark.sql.{SaveMode, SparkSession} import org.apache.spark.streaming.kafka010.{ConsumerStrategies, KafkaUtils, LocationStrategies} import org.apache.spark.streaming.{Seconds, StreamingContext} import java.util.Properties import com.xyf.utils.NumberUtils object SparkStreamingDemo02_读取kafka { def main(args: Array[String]): Unit = { val spark = SparkSession.builder() .master("local[2]") .appName("streaming") .getOrCreate() val sc = spark.sparkContext import spark.implicits._ // 设置批的持续时间 val batchDuration = Seconds(2) val ssc = new StreamingContext(sc, batchDuration) ssc.checkpoint("C:\\Users\\xinyaofeng\\Desktop\\checkpoint") // kafka中消息的kv类型 // def createDirectStream[K, V]( // ssc: StreamingContext, // locationStrategy: LocationStrategy, // consumerStrategy: ConsumerStrategy[K, V] // ): InputDStream[ConsumerRecord[K, V]] = val topics = Iterable("topic-file-6") val consumerProps = Map( "bootstrap.servers" -> "bd1471:9092,bd1472:9092,bd1473:9092", "key.deserializer" -> "org.apache.kafka.common.serialization.StringDeserializer", "value.deserializer" -> "org.apache.kafka.common.serialization.StringDeserializer", "group.id" -> "test22", "auto.offset.reset" -> "earliest" ) val dStream = KafkaUtils.createDirectStream[String, String]( ssc, // 设置消费的位置策略,让所有执行器轮询的读取所有的分区 LocationStrategies.PreferConsistent, // 设置消费者配置 ConsumerStrategies.Subscribe[String, String]( topics, consumerProps ) ) val props = new Properties() props.setProperty("user", "root") props.setProperty("password", "123456") val part1 = dStream .filter(_.value.split(" ").length > 9) // part1.map(_.value()).print() part1 .map(r => { val fields = r.value().split(" ") // ip 访问量 成功访问次数 失败访问次数 上传流量 下载流量 val count200 = if ("200" == fields(8)) 1 else 0 val count404 = if ("404" == fields(8)) 1 else 0 val ip = fields(0) (ip, AccessLog( ip, 1, count200, count404, NumberUtils.toLong(fields(9)), NumberUtils.toLong(fields.last))) }) .foreachRDD(rdd => { val accessLog = rdd.map(_._2) .toDS() accessLog.show() }) ssc.start() ssc.awaitTermination() } } case class AccessLog( ip: String, var uv: Long, var count200: Long, var count404: Long, var up: Long, var down: Long ) extends Serializable { }
向数据库追加
package com.xyf.demo2 import org.apache.spark.sql.{SaveMode, SparkSession} import org.apache.spark.streaming.kafka010.{ConsumerStrategies, KafkaUtils, LocationStrategies} import org.apache.spark.streaming.{Seconds, StreamingContext} import java.util.Properties import com.xyf.utils.NumberUtils object SparkStreamingDemo02_读取kafka { def main(args: Array[String]): Unit = { val spark = SparkSession.builder() .master("local[2]") .appName("streaming") .getOrCreate() val sc = spark.sparkContext import spark.implicits._ // 设置批的持续时间 val batchDuration = Seconds(2) val ssc = new StreamingContext(sc, batchDuration) ssc.checkpoint("C:\\Users\\xinyaofeng\\Desktop\\checkpoint") // kafka中消息的kv类型 // def createDirectStream[K, V]( // ssc: StreamingContext, // locationStrategy: LocationStrategy, // consumerStrategy: ConsumerStrategy[K, V] // ): InputDStream[ConsumerRecord[K, V]] = val topics = Iterable("topic-file-6") val consumerProps = Map( "bootstrap.servers" -> "bd1471:9092,bd1472:9092,bd1473:9092", "key.deserializer" -> "org.apache.kafka.common.serialization.StringDeserializer", "value.deserializer" -> "org.apache.kafka.common.serialization.StringDeserializer", "group.id" -> "test22", "auto.offset.reset" -> "earliest" ) val dStream = KafkaUtils.createDirectStream[String, String]( ssc, // 设置消费的位置策略,让所有执行器轮询的读取所有的分区 LocationStrategies.PreferConsistent, // 设置消费者配置 ConsumerStrategies.Subscribe[String, String]( topics, consumerProps ) ) val props = new Properties() props.setProperty("user", "root") props.setProperty("password", "123456") val part1 = dStream .filter(_.value.split(" ").length > 9) // part1.map(_.value()).print() part1 .map(r => { val fields = r.value().split(" ") // ip 访问量 成功访问次数 失败访问次数 上传流量 下载流量 val count200 = if ("200" == fields(8)) 1 else 0 val count404 = if ("404" == fields(8)) 1 else 0 val ip = fields(0) (ip, AccessLog( ip, 1, count200, count404, NumberUtils.toLong(fields(9)), NumberUtils.toLong(fields.last))) }) .foreachRDD(rdd => { val accessLog = rdd.map(_._2) .toDS() accessLog.show() accessLog .write .mode(SaveMode.Overwrite) .jdbc( "jdbc:mysql://bd1471:3306/spark_test", "accessLog", props ) }) ssc.start() ssc.awaitTermination() } } case class AccessLog( ip: String, var uv: Long, var count200: Long, var count404: Long, var up: Long, var down: Long ) extends Serializable { }
进行状态合并
package com.xyf.demo2 import org.apache.spark.SparkConf import org.apache.spark.streaming.{Seconds, StreamingContext} object SparkStreamingDemo03_流的状态更新 { def main(args: Array[String]): Unit = { // 程序入口 StreamingContext // 数据抽象 DStream val conf = new SparkConf() .setAppName("streaming") .setMaster("local[2]") // 设置批的持续时间 val batchDuration = Seconds(5) val ssc = new StreamingContext(conf, batchDuration) ssc.checkpoint("C:\\Users\\Amos\\Desktop\\checkpoint") // ssc加载数据创建流 val dStream = ssc.socketTextStream( "bd0701", 23456) // 打印dStream中的数据 dStream .flatMap(x => { x.split("\\s+") }) .map((_, 1)) // DStream[(K,V)] // def updateStateByKey[S: ClassTag]( // updateFunc: (Seq[V], Option[S]) => Option[S] // ): DStream[(K, S)] = .updateStateByKey[Long]( (values: Seq[Int], state: Option[Long]) => { Some(values.sum + state.getOrElse(0L)) } ) .print() // SparkStreaming中的流默认不带状态 // 不带状态:每个批次之间的数据相互独立 // 如果希望跨批进行数据累加,则需要让上游RDD将数据传递到下游RDD // 传递称之为状态的更新 // updateStateByKey // updateFunc: ( // Seq[V] 当前批中 相同key对应的所有value // , Option[S] 上一个批 传递下来的状态 //) => Option[S] 当前批 更新后的状态 // 启动流 ssc.start() // 设置流的状态为 等待关闭 ssc.awaitTermination() } }
最终代码
package com.xyf.demo2 import org.apache.spark.sql.{SaveMode, SparkSession} import org.apache.spark.streaming.kafka010.{ConsumerStrategies, KafkaUtils, LocationStrategies} import org.apache.spark.streaming.{Seconds, StreamingContext} import java.util.Properties import com.xyf.utils.NumberUtils object SparkStreamingDemo02_读取kafka { def main(args: Array[String]): Unit = { val spark = SparkSession.builder() .master("local[2]") .appName("streaming") .getOrCreate() val sc = spark.sparkContext import spark.implicits._ // 设置批的持续时间 val batchDuration = Seconds(2) val ssc = new StreamingContext(sc, batchDuration) ssc.checkpoint("C:\\Users\\xinyaofeng\\Desktop\\checkpoint") // kafka中消息的kv类型 // def createDirectStream[K, V]( // ssc: StreamingContext, // locationStrategy: LocationStrategy, // consumerStrategy: ConsumerStrategy[K, V] // ): InputDStream[ConsumerRecord[K, V]] = val topics = Iterable("topic-file-6") val consumerProps = Map( "bootstrap.servers" -> "bd1471:9092,bd1472:9092,bd1473:9092", "key.deserializer" -> "org.apache.kafka.common.serialization.StringDeserializer", "value.deserializer" -> "org.apache.kafka.common.serialization.StringDeserializer", "group.id" -> "test1", "auto.offset.reset" -> "earliest" ) val dStream = KafkaUtils.createDirectStream[String, String]( ssc, // 设置消费的位置策略,让所有执行器轮询的读取所有的分区 LocationStrategies.PreferConsistent, // 设置消费者配置 ConsumerStrategies.Subscribe[String, String]( topics, consumerProps ) ) val props = new Properties() props.setProperty("user", "root") props.setProperty("password", "123456") val part1 = dStream .filter(_.value.split(" ").length > 9) // part1.map(_.value()).print() part1 .map(r => { val fields = r.value().split(" ") // ip 访问量 成功访问次数 失败访问次数 上传流量 下载流量 val count200 = if ("200" == fields(8)) 1 else 0 val count404 = if ("404" == fields(8)) 1 else 0 val ip = fields(0) (ip, AccessLog( ip, 1, count200, count404, NumberUtils.toLong(fields(9)), NumberUtils.toLong(fields.last))) }) .updateStateByKey[AccessLog]( (values: Seq[AccessLog], state: Option[AccessLog]) => { if (values.nonEmpty) { val accessLog = values.reduce((v1, v2) => v1.merge(v2)) state match { case Some(log) => Some(log.merge(accessLog)) case None => Some(accessLog) } } else { state } } ) .foreachRDD(rdd => { val accessLog = rdd.map(_._2) .toDS() accessLog.show() accessLog .write .mode(SaveMode.Overwrite) .jdbc( "jdbc:mysql://bd1471:3306/spark_test", "accessLog", props ) }) ssc.start() ssc.awaitTermination() } } case class AccessLog( ip: String, var uv: Long, var count200: Long, var count404: Long, var up: Long, var down: Long ) extends Serializable { def merge(o: AccessLog): AccessLog = { this.uv += o.uv this.count200 += o.count200 this.count404 += o.count404 this.up += o.up this.down += o.down this } }
工具类
package com.xyf.utils object NumberUtils { def toLong(s: String): Long = { try { s.toLong } catch { case e: Exception => 0L } } }