Map算子: 类似spark的map算子,功能是转换数据类型,输入和输出的数量是1对1。下列例子是,将输入的String 转化为 string的长度
flatMap算子: 类似spark的flatMap, 功能是转换数据类型,输入和输出的数量可以是 1对多。 下列例子是分词, 将输入的整数数据 按照 " , " 分割,加上10后输出。
filter算子: 类似spark的filter,符合规则 就输出。 下列例子是 输入一组 整数数据,将偶数输出。
package com.flinkTest.transform;
import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;
import java.util.Arrays;
import java.util.List;
import java.util.Spliterator;
// base transform test, including map, richmap, flatmap, filter
public class transform01_base {
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
DataStreamSource<String> textInput = env.fromCollection(Arrays.asList("spark",
"flink",
"hive"
));
// map , 1对1, 转换算子, 本例是将 string 转换为 String的长度
SingleOutputStreamOperator<Integer> mapStream = textInput.map(new MapFunction<String, Integer>() {
@Override
public Integer map(String s) throws Exception {
return s.length();
}
});
// flatMap, 1对多, 转换算子,安找规则 将一条数据切分成多个, 本例是分词
DataStreamSource<List<Integer>> flatStream = env.fromCollection(Arrays.asList(
Arrays.asList(1, 2, 3, 4, 5),
Arrays.asList(6, 7, 8, 9, 10)
));
SingleOutputStreamOperator<Object> flatMapOut = flatStream.flatMap(new FlatMapFunction<List<Integer>, Object>() {
@Override
public void flatMap(List<Integer> integers, Collector<Object> out) throws Exception {
for (int num : integers) {
int newNum = num + 10;
out.collect(newNum);
}
}
});
// filter, 符合规则 就输出
DataStreamSource<Integer> filterStream = env.fromCollection(
Arrays.asList(6, 7, 8, 9, 10));
SingleOutputStreamOperator<Integer> filterOut = filterStream.filter(new FilterFunction<Integer>() {
@Override
public boolean filter(Integer num) throws Exception {
return num % 2 == 0;
}
});
mapStream.print("map");
flatMapOut.print("flatMap");
filterOut.print("filter");
env.execute();
}
}