Flink Java 自定义数据源

实现SourceFunction接口

public class CustomNonParalleSourceFunc implements SourceFunction<Long> {
    boolean isRunning  = true;
    long count = 1;
    public void run(SourceContext<Long> ctx) throws Exception {
        while (true){
            ctx.collect(count);
            count++;
            Thread.sleep(2000);
        }
    }

    public void cancel() {
            isRunning = false;
    }
}

测试：

public static void main(String[] args) throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    nonParalleSourceFunc(env);
    env.execute("a");
}
private static void nonParalleSourceFunc(StreamExecutionEnvironment env) {
    DataStreamSource<Long> data = env.addSource(new CustomNonParalleSourceFunc()
    ).setParallelism(1);//只能为1
    data.print().setParallelism(2);
}

实现ParallelSourceFunction接口

public class CustomParalleSourceFunc implements ParallelSourceFunction<Long> {
    boolean isRunning  = true;
    long count = 1;
    public void run(SourceContext<Long> ctx) throws Exception {
        while (true){
            ctx.collect(count);
            count++;
            Thread.sleep(2000);
        }
    }

    public void cancel() {
            isRunning = false;
    }
}

测试

public static void main(String[] args) throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    paralleSourceFunc(env);
    env.execute("a");
}
private static void paralleSourceFunc(StreamExecutionEnvironment env) {
    DataStreamSource<Long> data = env.addSource(new CustomParalleSourceFunc()
    ).setParallelism(3);
    data.print().setParallelism(2);
}
/*
1> 1
2> 1
1> 1
2> 2
1> 2
2> 2
1> 3
1> 3
2> 3
2> 4
1> 4
2> 4
* */

继承RichParallelSourceFunction类
可以并发执行

public class CustomRichparalleSourceFunc extends RichParallelSourceFunction<Long> {
    boolean isRunning  = true;
    long count = 1;
    public void run(SourceContext<Long> ctx) throws Exception {
        while (true){
            ctx.collect(count);
            count++;
            Thread.sleep(2000);
        }
    }

    public void cancel() {
        isRunning = false;
    }

    @Override
    public void open(Configuration parameters) throws Exception {
        super.open(parameters);
    }

    @Override
    public void close() throws Exception {
        super.close();
    }
}

测试

public static void main(String[] args) throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    richParalleSourceFunc(env);
    env.execute("a");
}

private static void richParalleSourceFunc(StreamExecutionEnvironment env) {
    DataStreamSource<Long> data = env.addSource(new CustomRichparalleSourceFunc()
    ).setParallelism(3);
    data.print().setParallelism(3);
}

/*
3> 1
1> 1
2> 1
3> 2
2> 2
1> 2
3> 3
2> 3
1> 3
* */