Flink自定义数据源

Flink中已经支持了大部分常见的数据源,但是在某些特定的情况下的数据源Flink还没有支持,或者是自定义的数据类型,这个时候我们就需要使用到Flink的自定义数据源,废话不多说,上代码

import lombok.*;
import org.apache.commons.lang3.RandomUtils;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.SourceFunction;

import java.util.Random;

/**
 * @Author: J
 * @Version: 1.0
 * @CreateTime: 2023/6/13
 * @Description: 自定义数据源测试
 **/
public class FlinkCustomizeSource {
    public static void main(String[] args) throws Exception {
        // 创建流环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        // 添加自定义数据源
        DataStreamSource dataStreamSource = env.addSource(new customizeSource());
        // 打印内容
        dataStreamSource.print();
        env.execute();
    }
}

// 自定义数据源需要实现SourceFunction接口,注意这个接口是单机的数据源,如果是想自定义分布式的数据源需要集成RichParallelSourceFunction类
class customizeSource implements SourceFunction<CustomizeBean> {
    int flag;
    // Job执行的线程
    @Override
    public void run(SourceContext ctx) throws Exception {
        /*这个方法里就是具体的数据逻辑,实际内容要根据业务需求编写,这里只是为了演示方便*/
        CustomizeBean customizeBean = new CustomizeBean();
        String[] genders = {"M", "W"};
        while (flag != 100) {
            // 这里自定义的Bean作为数据源
            customizeBean.setAge(RandomUtils.nextInt(18, 80)); // 年龄
            customizeBean.setName("A-" + new Random().nextInt()); // 姓名
            customizeBean.setGender(genders[RandomUtils.nextInt(0, genders.length)]); // 性别
            // 将数据收集
            ctx.collect(customizeBean);
            // 这里加上睡眠时间是为了降低控制台打印的速度
            Thread.sleep(1000);
        }
    }

    // Job取消时就会调用cancel方法
    @Override
    public void cancel() {
        // flag为100时就会停止程序
        flag = 100;
    }
}

@Getter
@Setter
@ToString
@NoArgsConstructor
@AllArgsConstructor
class CustomizeBean{
    private String name; // 姓名
    private int age; // 年龄
    private String gender; // 性别
}

控制台打印内容如下

SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/Users/xxx/data/maven-repository/repository/org/apache/logging/log4j/log4j-slf4j-impl/2.17.2/log4j-slf4j-impl-2.17.2.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/Users/xxx/data/maven-repository/repository/org/slf4j/slf4j-log4j12/1.7.25/slf4j-log4j12-1.7.25.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory]
2> CustomizeBean(name=A-936337035, age=59, gender=M)
3> CustomizeBean(name=A--1083661159, age=46, gender=W)
4> CustomizeBean(name=A-1844226683, age=67, gender=M)
5> CustomizeBean(name=A--75267383, age=41, gender=M)
6> CustomizeBean(name=A-1372451924, age=67, gender=M)
7> CustomizeBean(name=A--1997466557, age=57, gender=W)
8> CustomizeBean(name=A-1024015616, age=55, gender=W)
9> CustomizeBean(name=A-1955095194, age=77, gender=M)
10> CustomizeBean(name=A--1661025692, age=51, gender=W)
1> CustomizeBean(name=A-961725897, age=52, gender=M)
2> CustomizeBean(name=A--1117628763, age=73, gender=M)
3> CustomizeBean(name=A-1171187615, age=29, gender=M)
4> CustomizeBean(name=A--838540662, age=60, gender=M)
5> CustomizeBean(name=A-355071472, age=45, gender=W)
6> CustomizeBean(name=A-1724444454, age=26, gender=W)
7> CustomizeBean(name=A-1889957428, age=69, gender=W)