source and sink
public class FileSystemConnector {
public static void main(String[] args) throws Exception {
// 创建流处理执行环境
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
// 从文本文件中数据并切分打散
DataStream<String> data = env.readTextFile("FlinkModule/src/main/resources/stream/word")
.flatMap(new FlatMapFunction<String, String>() {
@Override
public void flatMap(String s, Collector<String> collector) throws Exception {
String[] arr = s.split("\t");
for(String string:arr){
collector.collect(string);
}
}
});
// 创建新版本的hdfs sink
final StreamingFileSink<String> sink = StreamingFileSink
// 设置输出路径以及字符类型,输出的文件格式为/path/{date-time}/part-{parallel-task}-{count}
.forRowFormat(new Path("hdfs://master:9000/home/flink"), new SimpleStringEncoder<String>("UTF-8"))
.build();
// 添加沉槽
data.addSink(sink);
// 执行任务
env.execute("FileSystem connector");
}
}