- reduce
reduce是归并操作,它可以将KeyedStream 转变为 DataStream,实质是按照key做叠加计算。示例如下:
import org.apache.flink.api.common.functions.{RichFlatMapFunction, RichMapFunction, RichReduceFunction}
import org.apache.flink.api.java.tuple.{Tuple1, Tuple2}
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
import org.apache.flink.util.Collector
object ReduceOperator {
def main(args: Array[String]): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
env.fromElements(Tuple1.of("flink hadoop taskmanager")
,Tuple1.of("spark hadoop")
,Tuple1.of("hadoop hdfs")).flatMap(new RichFlatMapFunction[Tuple1[String],Tuple2[String,Long]] {
override def flatMap(value: Tuple1[String], out: Collector[Tuple2[String,Long]]): Unit = {
for(s:String <- value.f0.split(" ")){
out.collect(Tuple2.of(s,1L))}}}).keyBy(0)
.reduce(new RichReduceFunction[Tuple2[String, Long]] {
override def reduce(value1: Tuple2[String, Long], value2: Tuple2[String, Long]): Tuple2[String, Long] = {
return Tuple2.of(value1.f0,value1.f1+value2.f1)
}
}).print()
env.execute("flink reduce operator")
- union操作
union可以将多个流合并到一个流中,以便对合并的流进行集中统处理。是对多个流的水平拼接多个数据流必须是同类型。示例如下:
import org.apache.flink.api.java.tuple.Tuple1
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
object UnionOperator {
def main(args: Array[String]): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
val df1 = env.fromElements(Tuple1.of("flink")
,Tuple1.of("spark")
,Tuple1.of("hadoop"))
val df2 = env.fromElements(Tuple1.of("oracle")
,Tuple1.of("mysql")
,Tuple1.of("sqlserver"))
df1.union(df2).print()
env.execute("flink union operator")}}
- join操作
join是根据指定的key将两个流做关联。
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.api.windowing.triggers.CountTrigger
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import org.apache.flink.streaming.api.windowing.assigners.{ProcessingTimeSessionWindows}
import org.apache.flink.streaming.api.windowing.time.Time
import org.apache.flink.util.Collector
object JoinOperator {
def main(args: Array[String]): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
val df1 = env.fromElements(
Tuple2.apply("flink",1L)
,Tuple2.apply("spark",2L)
,Tuple2.apply("hadoop",3L))
val df2 = env.fromElements(Tuple2.apply("flink",1L)
,Tuple2.apply("mysql",1L)
,Tuple2.apply("spark",1L))
df1.join(df2)
.where(_._1)
.equalTo(_._1)
.window(ProcessingTimeSessionWindows.withGap(Time.seconds(10)))
.trigger(CountTrigger.of(1))
.apply((t1,t2,out:Collector[Tuple2[String,Long]]) =>{
out.collect(Tuple2.apply(t1._1,t1._2+t2._2))
})
.print()
env.execute("flink join operator")}}