今天学习了一些比较简单的sparkstreaming原理和事例,终于明白spark为什么是实时的处理数据···
1.HdfsWordcount 对机器上/datatnt/text/文件夹实时监听。
package sprakstreaming
/**
* Created by sendoh on 2015/3/23.
*/
import org.apache.spark.SparkConf
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.streaming.StreamingContext._
//对监听的文件目录实时的显示其中单词的数量
object HdfsWordcount {
def main(args: Array[String]): Unit ={
val sparkConf = new SparkConf().setAppName("HdfsWordCount").setMaster("local[2]")
//val sparkConf = new SparkConf().setAppName("HdfsWordCount").setMaster("local[2]") 本地运行不放在集群上运行,sparkstreaming设置进程大于两个,一个用于监听,其它用于运行
val ssc = new StreamingContext(sparkConf, Seconds(20))//每隔20毫秒监听一次
val lines = ssc.textFileStream("/datatnt/text/")//要监听的问价目录
val words = lines.flatMap(_.split(" ")) //压扁切片
val wordCounts = words.map(x => (x, 1)).reduceByKey(_ + _) //单词统计
wordCounts.print()//显示结果
ssc.start()
ssc.awaitTermination()
}
}
2.销售模拟器:可以读取文件并将文件的每一行随机发出去
package sprakstreaming
/**
* Created by sendoh on 2015/3/23.
*/
import java.io.{PrintWriter}
import java.net.ServerSocket
import scala.io.Source
object SaleSimulation {
def index(length: Int) = {
import java.util.Random
val rdm = new Random
rdm.nextInt(length)
}
def main(args: Array[String]): Unit ={
if (args.length != 3){
System.err.println("Usage: <filename> <port> <millisecond>")//文件名:模拟器发送数据的端口:时间间隔
System.exit(1)
}
val filename = args(0)
val lines = Source.fromFile(filename).getLines.toList
val filerow = lines.length
val listener = new ServerSocket(args(1).toInt)//监听
while (true){
val socket = listener.accept()
new Thread(){
override def run = {
println("Got client connected from: " + socket.getInetAddress)
val out = new PrintWriter(socket.getOutputStream(), true)//发送数据
while(true){
Thread.sleep(args(2).toLong)//设定时间
val content = lines(index(filerow))
println(content)
out.write(content + '\n')
out.flush()
}
socket.close()
}
}.start()
}
}
}
将模拟器打包,在spark目录下运行文件:
:java -cp sparkstreamingtext.jar sparkstreamingtext.SaleSimulation /datatnt/text.txt 9999 1000 //发送datatnt文件夹下的text文件 9999:端口,1000:1秒。
3.网络数据的演示:
package sprakstreaming
/**
* Created by sendoh on 2015/3/23.
*/
import org.apache.spark.{SparkContext, SparkConf}
import org.apache.spark.streaming.{Milliseconds, Seconds, StreamingContext}
import org.apache.spark.streaming.StreamingContext._
import org.apache.spark.storage.StorageLevel
object NetworkWordCount {
def main(args: Array[String]): Unit ={
val conf = new SparkConf().setAppName("NetworkWordCount").setMaster("local[2]")
val sc = new SparkContext(conf)
val ssc = new StreamingContext(sc, Seconds(5))//五秒监听一次
val lines = ssc.socketTextStream(args(0), args(1).toInt, StorageLevel.MEMORY_AND_DISK_SER)//读取网络接口的流数据(服务器:端口9999)
val words = lines.flatMap(_.split(","))
val wordCounts = words.map(x => (x, 1)).reduceByKey(_ + _)
wordCounts.print()
ssc.start()
ssc.awaitTermination()
}
}
IDEA run configuration,对problem argument设置:服务器 + 端口 运行
以上是sparkstreaming关于无状态的操作
4.///有状态的操作
<pre name="code" class="plain">package sprakstreaming
/**
* Created by sendoh on 2015/3/23.
*/
import org.apache.spark.{SparkContext, SparkConf}
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.streaming.StreamingContext._
object StatefulWordCount {
def main(args: Array[String]): Unit ={
val updateFunc = (values: Seq[Int], state: Option[Int]) => { //新进来的值+状态(将被统计的值)
val currentCount = values.foldLeft(0)(_ + _)//新进来的值加上以前的值
val previousCount = state.getOrElse(0)//初始化为0
Some(currentCount + previousCount)//返回一个新的值
}
val conf = new SparkConf().setAppName("StatefulWordCount").setMaster("local[2]")
val sc = new SparkContext(conf)
//创建StreamingContext
val ssc = new StreamingContext(sc, Seconds(5))
ssc.checkpoint(".")
//获取数据
val lines = ssc.socketTextStream(args(0), args(1).toInt)
val words = lines.flatMap(_.split("."))
val wordCounts = words.map(x => (x, 1))
//使用updateStateByKey来更新状态
val stateDstream = wordCounts.updateStateByKey[Int](updateFunc)//调用updateFunc方法
stateDstream.print()
ssc.start()
ssc.awaitTermination()
}
}
IDEA run configuration,对problem argument设置:服务器 + 端口 运行
5.
package sprakstreaming
/**
* Created by sendoh on 2015/3/23.
*/
import org.apache.spark.{SparkContext, SparkConf}
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming._
import org.apache.spark.streaming.StreamingContext._
object WindowWordCount {
def main(args: Array[String]): Unit ={
val conf = new SparkConf().setAppName("WindowWordCount").setMaster("local[2]")
val sc = new SparkContext(conf)
//创建StreamingContext
val ssc = new StreamingContext(sc, Seconds(5))
ssc.checkpoint(".")
//获取数据
val lines = ssc.socketTextStream(args(0), args(1).toInt, StorageLevel.MEMORY_AND_DISK_SER)
val words = lines.flatMap(_.split(","))
//windows操作
val wordCounts = words.map(x => (x, 1)).reduceByKeyAndWindow((a:Int, b:Int) => (a + b), Seconds(args(2).toInt), Seconds(args(3).toInt))
//a+b:windows的大小,接下来是windows的时间间隔+windows所移动的时间间隔*(必须是streamingcontext的时间倍数)
wordCounts.print()
ssc.start()
ssc.awaitTermination()
}
}
IDEA run configuration,对problem argument设置:服务器 + 端口+windows时间间隔+窗口滑动时间间隔 运行
6.sale数据演示
//qryStockDetail.txt文件定义了订单明细
//订单号,行号,货品,数量,单价,金额
使用方法:
java -cp sparkstreamingtext.jar sparkstreamingtext.SaleSimulation /datatnt/qryStockDetail.txt 9999 1000
package sprakstreaming
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkContext, SparkConf}
import org.apache.spark.streaming.StreamingContext._
/**
* Created by sendoh on 2015/3/23.
*/
object SaleAmount {
def main(args: Array[String]): Unit ={
val conf = new SparkConf().setAppName("SaleAmount").setMaster("local[2]")
val sc = new SparkContext(conf)
val ssc = new StreamingContext(sc, Seconds(5))
val lines = ssc.socketTextStream(args(0), args(1).toInt, StorageLevel.MEMORY_AND_DISK_SER)
val words = lines.map(_.split(",")).filter(_.length == 6)//判定对6列队数据进行操作
val wordCounts = words.map(x => (1, x(5).toDouble)).reduceByKey(_ + _)
wordCounts.print()
ssc.start()
ssc.awaitTermination()
}
}
IDEA run configuration,对problem argument设置:服务器 + 端口+windows时间间隔+窗口滑动时间间隔 运行