Flink WordCount JavaAPI(从文件读取)
图中,data目录下是要读取的文件:words
内容:
hello java
hello spark
hello spark
hello spark
hello flink
hello flink
hello flink
hello flink
hello hadoop
hello hadoop
popx.xml文件:
代码:
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.operators.*;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.util.Collector;
public class Test1 {
public static void main(String[] args) throws Exception {
//final StreamExecutionEnvironment environment = StreamExecutionEnvironment .getExecutionEnvironment();
//DataStream<String> text = environment.readTextFile("./data/worlds");
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSource<String> dataSource = env.readTextFile("src/main/data/worlds");
FlatMapOperator<String, String> words = dataSource.flatMap(new FlatMapFunction<String, String>() {
@Override
public void flatMap(String line, Collector<String> collector) throws Exception {
String[] split = line.split(" ");
for (String word : split) {
collector.collect(word);
}
}
});
MapOperator<String, Tuple2<String, Integer>> pairWords = words.map(new MapFunction<String, Tuple2<String, Integer>>() {
@Override
public Tuple2<String, Integer> map(String word) throws Exception {
return new Tuple2<>(word, 1);
}
});
UnsortedGrouping<Tuple2<String, Integer>> tuple2UnsortedGrouping = pairWords.groupBy(0);
AggregateOperator<Tuple2<String, Integer>> sum = tuple2UnsortedGrouping.sum(1);
sum.print();
}
}
第一次运行报错:
Exception in thread “main” org.apache.flink.runtime.client.JobExecutionException: Could not retrieve JobResult.
at org.apache.flink.runtime.minicluster.MiniCluster.executeJobBlocking(MiniCluster.java:643)
at org.apache.flink.client.LocalExecutor.executePlan(LocalExecutor.java:223)
at org.apache.flink.api.java.LocalEnvironment.execute(LocalEnvironment.java:91)
at org.apache.flink.api.java.ExecutionEnvironment.execute(ExecutionEnvironment.java:817)
at org.apache.flink.api.java.DataSet.collect(DataSet.java:413)
at org.apache.flink.api.java.DataSet.print(DataSet.java:1652)
at Test1.main(Test1.java:34)
Caused by: org.apache.flink.runtime.client.JobSubmissionException: Failed to submit job.
at org.apache.flink.runtime.dispatcher.Dispatcher.lambda$submitJob 2 ( D i s p a t c h e r . j a v a : 267 ) a t j a v a . u t i l . c o n c u r r e n t . C o m p l e t a b l e F u t u r e . u n i E x c e p t i o n a l l y ( C o m p l e t a b l e F u t u r e . j a v a : 870 ) a t j a v a . u t i l . c o n c u r r e n t . C o m p l e t a b l e F u t u r e 2(Dispatcher.java:267) at java.util.concurrent.CompletableFuture.uniExceptionally(CompletableFuture.java:870) at java.util.concurrent.CompletableFuture 2(Dispatcher.java:267)atjava.util.concurrent.CompletableFuture.uniExceptionally(CompletableFuture.java:870)atjava.util.concurrent.CompletableFutureUniExceptionally.tryFire(CompletableFuture.java:852)
at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474)
at java.util.concurrent.CompletableFuture.postFire(CompletableFuture.java:561)
at java.util.concurrent.CompletableFuture U n i W h e n C o m p l e t e . t r y F i r e ( C o m p l e t a b l e F u t u r e . j a v a : 739 ) a t j a v a . u t i l . c o n c u r r e n t . C o m p l e t a b l e F u t u r e UniWhenComplete.tryFire(CompletableFuture.java:739) at java.util.concurrent.CompletableFuture UniWhenComplete.tryF