public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(conf);
/**
* 自定义Source
* 可以实现 SourceFunction 或者 RichSourceFunction , 这两者都是非并行的 source 算子
* 也可实现 ParallelSourceFunction 或者 RichParallelSourceFunction , 这两者都是可并行的
* source 算子
* Rich-》带有open close getRuntimeContext方法
* parllel=》可并行
*/
//实现了SourceFunction的自定义Source并行度只能为1
DataStreamSource<Student> stream1 = env.addSource(new MySource_01());
DataStreamSource<Student> stream2 = env.addSource(new MySource_02()).setParallelism(2);
DataStreamSource<Student> stream3 = env.addSource(new MySource_03());
DataStreamSource<Student> stream4 = env.addSource(new MySource_04());
stream3.map(JSON::toJSONString).print();
env.execute();
}
class MySource_01 implements SourceFunction<Student> {
private Boolean flag = true;
@Override
public void run(SourceContext<Student> sourceContext) throws Exception {
Student st = new Student();
while (flag) {
Thread.sleep(1000);
st.setName(RandomStringUtils.randomAlphabetic(10).toUpperCase());
st.setAge(RandomUtils.nextInt(10, 1000));
st.setGender(RandomUtils.nextInt(10, 1000) > RandomUtils.nextInt(10, 1000) ? "male" : "female");
sourceContext.collect(st);
}
}
@Override
public void cancel() {
}
}
class MySource_02 implements ParallelSourceFunction<Student> {
private Boolean flag = true;
@Override
public void run(SourceContext<Student> sourceContext) throws Exception {
Student st = new Student();
while (flag) {
Thread.sleep(1000);
st.setName(RandomStringUtils.randomAlphabetic(10).toUpperCase());
st.setAge(RandomUtils.nextInt(10, 1000));
st.setGender(RandomUtils.nextInt(10, 1000) > RandomUtils.nextInt(10, 1000) ? "male" : "female");
sourceContext.collect(st);
}
}
@Override
public void cancel() {
}
}
class MySource_03 extends RichSourceFunction<Student> {
private Boolean flag = true;
@Override
public void open(Configuration parameters) throws Exception {
RuntimeContext runtimeContext = getRuntimeContext();
String taskName = runtimeContext.getTaskName();
int indexOfThisSubtask = runtimeContext.getIndexOfThisSubtask();
System.out.println(taskName + "-" + indexOfThisSubtask);
}
@Override
public void run(SourceContext<Student> sourceContext) throws Exception {
Student st = new Student();
while (flag) {
Thread.sleep(1000);
st.setName(RandomStringUtils.randomAlphabetic(10).toUpperCase());
st.setAge(RandomUtils.nextInt(10, 1000));
st.setGender(RandomUtils.nextInt(10, 1000) > RandomUtils.nextInt(10, 1000) ? "male" : "female");
sourceContext.collect(st);
}
}
@Override
public void cancel() {
}
@Override
public void close() throws Exception {
System.out.println("haha 关了");
}
}
class MySource_04 extends RichParallelSourceFunction<Student> {
private Boolean flag = true;
@Override
public void open(Configuration parameters) throws Exception {
RuntimeContext runtimeContext = getRuntimeContext();
String taskName = runtimeContext.getTaskName();
int indexOfThisSubtask = runtimeContext.getIndexOfThisSubtask();
System.out.println(taskName + "-" + indexOfThisSubtask);
}
@Override
public void run(SourceContext<Student> sourceContext) throws Exception {
Student st = new Student();
while (flag) {
Thread.sleep(1000);
st.setName(RandomStringUtils.randomAlphabetic(10).toUpperCase());
st.setAge(RandomUtils.nextInt(10, 1000));
st.setGender(RandomUtils.nextInt(10, 1000) > RandomUtils.nextInt(10, 1000) ? "male" : "female");
sourceContext.collect(st);
}
}
@Override
public void cancel() {
}
@Override
public void close() throws Exception {
System.out.println("haha 关了");
}
}
Flink自定义Source的四种实现方式
最新推荐文章于 2024-07-27 07:15:00 发布