Spark Streaming-02

直接上代码，注意textFileStream数据源没有 receiver

import org.apache.spark.SparkConf
import org.apache.spark.streaming.{Seconds, StreamingContext}

object TextFileStreamWordCountApp {
  def main(args: Array[String]){
    val sparkConf = new SparkConf().setAppName("TextFileStreamWordCountApp").setMaster("local[2]")

        val ssc = new StreamingContext(sparkConf,Seconds(10)) val lines = ssc.textFileStream("C:\\wc") //val lines =                             ssc.socketTextStream("hadoop",9999) lines.flatMap(_.split(",")).map((_,1)).reduceByKey(_+_).print 

             ssc.start() 

             ssc.awaitTermination() }}

SparkStreaming关键性方法：

1、transform 将RDD转化为Dstream

2、updateStateByKey 对批处理结果进行累计

import java.sql.DriverManager
import org.apache.spark.SparkConf
import org.apache.spark.streaming.{Seconds, StreamingContext}

object ForeachRDDApp {
  def main(args: Array[String]) {
    val sc = new SparkConf()
    //.setAppName("StatefulWordCountApp").setMaster("local[2]")
    val ssc = new StreamingContext(sc, Seconds(10))

    val lines = ssc.socketTextStream("hadoop", 9999)
    val results = lines.flatMap(_.split(" ")).map((_, 1)).reduceByKey(_ + _)

    //TODO.. 将results写入到mysql中
//        results.foreachRDD(rdd =>{
//          rdd.foreach(x =>{
//            val connection = createConnection()
//            val word = x._1
//            val count = x._2.toInt
//
//            val sql=s"insert into wc(word,c) values('$word','$count')"
//            connection.createStatement().execute(sql)
//            connection.close()
//          })
//        })

    //TODO 最佳写法 rdd.foreachPartition
    results.foreachRDD(rdd => {
      rdd.foreachPartition(partition => {
        val connection = createConnection()
        partition.foreach(x => {
          val word = x._1
          val count = x._2.toInt
          val sql = s"insert into wc(word,c) values('$word','$count');"
          connection.createStatement().execute(sql)
        })
        connection.close()
      })
    })

    ssc.start()
    ssc.awaitTermination()
  }

  def createConnection() = {
    Class.forName("com.mysql.jdbc.Driver")
    DriverManager.getConnection("jdbc:mysql://hadoop:3306/ss2", "root", "123456")
  }
}

Output Operations on DSstream

1、saveAsTextFiles 会导致小文件过多

2、saveAsHadoopFiles 

3、foreachRDD（写入到关系型数据库中，必须用到的方法）