直接上代码,注意textFileStream数据源没有 receiver
import org.apache.spark.SparkConf import org.apache.spark.streaming.{Seconds, StreamingContext} object TextFileStreamWordCountApp { def main(args: Array[String]){ val sparkConf = new SparkConf().setAppName("TextFileStreamWordCountApp").setMaster("local[2]")
val ssc = new StreamingContext(sparkConf,Seconds(10)) val lines = ssc.textFileStream("C:\\wc") //val lines = ssc.socketTextStream("hadoop",9999) lines.flatMap(_.split(",")).map((_,1)).reduceByKey(_+_).print
ssc.start()
ssc.awaitTermination() }}
SparkStreaming关键性方法:
1、transform 将RDD转化为Dstream
2、updateStateByKey 对批处理结果进行累计
import java.sql.DriverManager
import org.apache.spark.SparkConf
import org.apache.spark.streaming.{Seconds, StreamingContext}
object ForeachRDDApp {
def main(args: Array[String]) {
val sc = new SparkConf()
//.setAppName("StatefulWordCountApp").setMaster("local[2]")
val ssc = new StreamingContext(sc, Seconds(10))
val lines = ssc.socketTextStream("hadoop", 9999)
val results = lines.flatMap(_.split(" ")).map((_, 1)).reduceByKey(_ + _)
//TODO.. 将results写入到mysql中
// results.foreachRDD(rdd =>{
// rdd.foreach(x =>{
// val connection = createConnection()
// val word = x._1
// val count = x._2.toInt
//
// val sql=s"insert into wc(word,c) values('$word','$count')"
// connection.createStatement().execute(sql)
// connection.close()
// })
// })
//TODO 最佳写法 rdd.foreachPartition
results.foreachRDD(rdd => {
rdd.foreachPartition(partition => {
val connection = createConnection()
partition.foreach(x => {
val word = x._1
val count = x._2.toInt
val sql = s"insert into wc(word,c) values('$word','$count');"
connection.createStatement().execute(sql)
})
connection.close()
})
})
ssc.start()
ssc.awaitTermination()
}
def createConnection() = {
Class.forName("com.mysql.jdbc.Driver")
DriverManager.getConnection("jdbc:mysql://hadoop:3306/ss2", "root", "123456")
}
}
Output Operations on DSstream
1、saveAsTextFiles 会导致小文件过多
2、saveAsHadoopFiles
3、foreachRDD(写入到关系型数据库中,必须用到的方法)