上文介绍的是数据批处理,可以简单理解为就是一批固定的数据
数据流处理的意思是,有源源不断的数据,来一个数据进行以此处理。
还是一样的分词处理,这次用流处理方式,直接上代码(文件看上文)
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
public class StreamWordCount {
public static void main(String[] args) throws Exception{
//创建流处理环境
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
//设置并行度
//env.setParallelism(8);
//从文件中读取数据
String inputPath = "D:\\idle\\FlinkTest\\src\\main\\resources\\word.txt";
DataStream<String> inputDataStream = env.readTextFile(inputPath);
//基于数据流进行转换 new WordCount.MyflatMapper()上文写好的flatMap
DataStream<Tuple2<String,Integer>> resultStream = inputDataStream.flatMap(new WordCount.MyflatMapper()).keyBy(0)//按照当前key的hashcode进行重分区
.sum(1);
resultStream.print();
//执行任务 启动任务
env.execute();
}
}