[flink]#11_应用开发quick_start

开发步骤

1. 获得一个执行环境
2. 加载/创建初始化数据
3. 指定操作数据的Transaction算子
4. 指定计算好的数据的存放位置
5. 调用execute（）触发执行程序

Flink流处理 开发

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.utils.ParameterTool;

import org.apache.flink.runtime.state.filesystem.FsStateBackend;
import org.apache.flink.runtime.state.memory.MemoryStateBackend;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.util.Collector;


public class SocketWindowWordCountJava {
    public static void main(String[] args) throws Exception{
        //获取需要的端口号
        int port;
        try {
            ParameterTool parameterTool = ParameterTool.fromArgs(args);
            port = parameterTool.getInt("port");
        }catch (Exception e){
            System.err.println("No port set. use default port 9000--Java");
            port = 9000;
        }

        //获取Flink的运行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        String hostname = "localhost";
        String delimiter = "\n";
        //连接Socket获取输入的数据
        DataStreamSource<String> text = env.socketTextStream(hostname, port, delimiter);

        // a a c

        // a 1
        // a 1
        // c 1
        DataStream<WordWithCount> windowCounts = text.flatMap(new FlatMapFunction
                <String, WordWithCount>() {
            public void flatMap(String value, Collector<WordWithCount> out) throws
                    Exception {
                String[] splits = value.split("\\s");
                for (String word : splits) {
                    out.collect(new WordWithCount(word, 1L));
                }
            }
        }).keyBy("word")
                .timeWindow(Time.seconds(2), Time.seconds(1))//指定时间窗口大小为2s，指定时间间隔为1s
                .sum("count");//在这里使用sum或者reduce都可以
                /*.reduce(new ReduceFunction<WordWithCount>() {
                                    public WordWithCount reduce(WordWithCount a,
WordWithCount b) throws Exception {

                                        return new WordWithCount(a.word,a.count+b.count);
                                    }
                                })*/
        //把数据打印到控制台并且设置并行度
        windowCounts.print().setParallelism(1);
        //这一行代码一定要实现，否则程序不执行
        env.execute("Socket window count");

    }

    public static class WordWithCount{
        public String word;
        public long count;
        public  WordWithCount(){}
        public WordWithCount(String word,long count){
            this.word = word;
            this.count = count;
        }
        @Override
        public String toString() {
            return "WordWithCount{" +
                    "word='" + word + '\'' +
                    ", count=" + count +
                    '}';
        }
    }
}

import org.apache.flink.api.java.utils.ParameterTool
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import org.apache.flink.streaming.api.windowing.time.Time

/**
 * 单词计数之滑动窗口计算
 *
 */
object SocketWindowWordCountScala {

    def main(args: Array[String]): Unit = {

        //获取Socket端口号
        val port: Int = try {
            ParameterTool.fromArgs(args).getInt("port")
        }catch {
            case e: Exception => {
                System.err.println("No port set. use default port 9000--Scala")
            }
                9000
        }

        //获取运行环境
        val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment

        //连接Socket获取输入数据
        val text = env.socketTextStream("localhost",port,'\n')

        //解析数据（把数据打平），分组，窗口计算，并且聚合求sum

        //注意：必须要添加这一行隐式转行，否则下面的FlatMap方法执行会报错
        import org.apache.flink.api.scala._

        val windowCounts = text.flatMap(line => line.split("\\s"))//打平，把每一行单词都切开
          .map(w => WordWithCount(w,1))//把单词转成word , 1这种形式
          .keyBy("word")//分组
          .timeWindow(Time.seconds(2),Time.seconds(1))//指定窗口大小，指定间隔时间
          .sum("count");// sum或者reduce都可以
        //.reduce((a,b)=>WordWithCount(a.word,a.count+b.count))

        //打印到控制台
        windowCounts.print().setParallelism(1);

        //执行任务
        env.execute("Socket window count");

    }

    case class WordWithCount(word: String,count: Long)

}

Flink批处理开发

package batch;

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.operators.DataSource;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.util.Collector;

/**
 *单词计数之离线计算
 *
 */
public class BatchWordCountJava {

    public static void main(String[] args) throws Exception{
        String inputPath = "/Users/eric/Desktop/flink-train/FlinkTech/src/main/resources/data/input";
        String outPath = "/Users/eric/Desktop/flink-train/FlinkTech/src/main/resources/data/result";

        //获取运行环境
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        //获取文件中的内容
        DataSource<String> text = env.readTextFile(inputPath);

        DataSet<Tuple2<String, Integer>> counts = text.flatMap(new Tokenizer()).groupBy(0).sum(1);
        counts.writeAsCsv(outPath,"\n"," ").setParallelism(1);
        env.execute("batch word count");

    }

    public static class Tokenizer implements FlatMapFunction<String,Tuple2<String,
            Integer>>{
        public void flatMap(String value, Collector<Tuple2<String, Integer>> out)
                throws Exception {
            String[] tokens = value.toLowerCase().split("\\W+");
            for (String token: tokens) {
                if(token.length()>0){
                    out.collect(new Tuple2<String, Integer>(token,1));
                }
            }
        }
    }
}

package batch

import org.apache.flink.api.scala.ExecutionEnvironment

/**
 * 单词计数之离线计算
 */
object BatchWordCountScala {

    def main(args: Array[String]): Unit = {
        val inputPath = "/Users/eric/Desktop/flink-train/FlinkTech/src/main/resources/data/input"
        val outPut = "/Users/eric/Desktop/flink-train/FlinkTech/src/main/resources/data/result"

        val env = ExecutionEnvironment.getExecutionEnvironment
        val text = env.readTextFile(inputPath)

        //引入隐式转换
        import org.apache.flink.api.scala._

        val counts = text.flatMap(_.toLowerCase.split("\\W+"))
          .filter(_.nonEmpty)
          .map((_,1))
          .groupBy(0)
          .sum(1)
        counts.writeAsCsv(outPut,"\n"," ").setParallelism(1)
        env.execute("batch word count")
    }

}