Springboot接口提交flink程序到集群当中

一、问题描述

由于我们编写的flink程序无法实现动态的管理，并且无法和前端进行交互，所以在数据中台的环境下，可以通过springboot的接口程序提交flink程序到集群当中去，或者是直接springboot集成flink的程序，接下来演示springboot提交程序到flink集群当中，整体架构如下所示

二、实现 

controller层

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import springboot.flink.FlinkBean;
import springboot.service.FlinkService;

@RestController
@RequestMapping("/flink")
public class FlinkController {

    @Autowired
    private FlinkService flinkService;


    @GetMapping("/test2")
    public String test2(String arg1,String arg2,String arg3)
    {
        String args[]={arg1,arg2,arg3};
        String resultCommond = flinkService.buildCommond(args);
        String result = flinkService.exeCmd(resultCommond);
        return result;
    }
}

service层

package springboot.service;


import org.springframework.stereotype.Service;

import java.io.BufferedReader;
import java.io.InputStreamReader;

@Service
public class FlinkService {
    public String exeCmd(String commandStr) {

        String result = null;
        try {
            String[] cmd = new String[]{"/bin/sh", "-c",commandStr};
            Process ps = Runtime.getRuntime().exec(cmd);

            BufferedReader br = new BufferedReader(new InputStreamReader(ps.getInputStream()));
            StringBuffer sb = new StringBuffer();
            String line;
            while ((line = br.readLine()) != null) {
                sb.append(line).append("\n");
            }
            result = sb.toString();


        } catch (Exception e) {
            e.printStackTrace();
        }

        return result;

    }
    public String buildCommond(String... args)
    {
        StringBuffer commond=new StringBuffer();
        commond.append("/opt/modul/flink/bin/flink run -d -c com.studyflink.paramerTest.FlinkParamerTest /opt/software/flink/flink-study-book-1.0-SNAPSHOT-jar-with-dependencies.jar");
        for(int i=0;i<args.length;i++)
        {
            commond.append(" "+args[i]);
        }
        return commond.toString();
    }
}

测试的flink代码

import com.studyflink.bean.Event;
import com.studyflink.commentUtils.ClickSource;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import com.studyflink.bean.Event;
import org.apache.flink.streaming.api.functions.source.SourceFunction;

import java.util.Calendar;
import java.util.Random;
public class FlinkParamerTest {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(Integer.valueOf(args[0]));
        DataStreamSource<Event> dataStreamSource = env.addSource(new ClickSource());

        SingleOutputStreamOperator<String> mapStream = dataStreamSource.map(value -> {
            return value.getUser()+" 测试参数的flink程序";
        }).setParallelism(Integer.valueOf(args[1]));

        mapStream.print().setParallelism(Integer.valueOf(args[2]));
        env.execute();
    }
}



/**
 * @author ：family
 * @date ：2022/11/1 18:09
 */
class ClickSource implements SourceFunction<Event> {
    // 声明一个布尔变量，作为控制数据生成的标识位
    private Boolean running = true;
    @Override
    public void run(SourceContext<Event> ctx) throws Exception {
        Random random = new Random();    // 在指定的数据集中随机选取数据
        String[] users = {"Mary", "Alice", "Bob", "Cary"};
        String[] urls = {"./home", "./cart", "./fav", "./prod?id=1", "./prod?id=2"};

        while (running) {
            ctx.collect(new Event(
                    users[random.nextInt(users.length)],
                    urls[random.nextInt(urls.length)],
                    Calendar.getInstance().getTimeInMillis()
            ));
            // 隔1秒生成一个点击事件，方便观测
            Thread.sleep(1000);
        }
    }
    @Override
    public void cancel() {
        running = false;
    }
}

三、执行过程以及结果

前端调用接口，并且返回如下所示信息之后表示成功通过接口提交了flink程序到集群当中

 

四、总结 

通过前端api接口的形势来调用flink程序，可以随时修改flink的固定参数，例如，设置并行度，设置source端的数据源，设置sink的数据源，甚至设置transform端的转换规则都可以进行相应的设置