2 Flink 初体验

最新推荐文章于 2024-11-08 13:59:39 发布

一直奔跑的马

最新推荐文章于 2024-11-08 13:59:39 发布

阅读量293

点赞数

文章标签： flink 大数据

本文链接：https://blog.csdn.net/wangxingxingalulu/article/details/118112577

版权

单词统计(dataStream)

package wordcount;

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;

/**
 * @author wangkai 
 */
public class StreamWordCount {
    public static void main(String[] args) throws Exception{
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        ParameterTool tool = ParameterTool.fromArgs(args);
        String host = tool.get("host");
        Integer port = tool.getInt("port");
        DataStreamSource<String> source = env.socketTextStream(host, port);
        SingleOutputStreamOperator<Tuple2<String, Integer>> sum = source.flatMap(new MyFlatMapFunction())
                .keyBy(0)
                .sum(1);


        sum.print();
        env.execute("word count");
    }


    public static class MyFlatMapFunction implements FlatMapFunction<String,Tuple2<String,Integer>>{

        public void flatMap(String value, Collector<Tuple2<String, Integer>> out) throws Exception {
            String[] word = value.split(" ");
            for (String s:word
                 ) {
                out.collect(new Tuple2<String, Integer>(s,1));
            }

        }
    }
}

单词统计(批处理)

package wordcount;

import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.operators.AggregateOperator;
import org.apache.flink.api.java.operators.DataSource;
import org.apache.flink.api.java.tuple.Tuple2;

/**
 * @author wangkai 
 */
public class SetWordCount {
    public static void main(String[] args) throws Exception{
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);

        DataSource<String> stringDataSource = env.readTextFile("D:\\git\\csdn-flink\\csdn-flink-1\\src\\main\\resources\\word");
        AggregateOperator<Tuple2<String, Integer>> sum = stringDataSource.flatMap(new StreamWordCount.MyFlatMapFunction())
                .groupBy(0)
                .sum(1);

        sum.print();
    }

}