flink自定义数据源，写入数据到Mysql

最新推荐文章于 2024-10-07 16:13:47 发布

白驹过隙2189

最新推荐文章于 2024-10-07 16:13:47 发布

阅读量271

点赞数 3

文章标签： flink mysql 大数据

本文链接：https://blog.csdn.net/weixin_45928691/article/details/141183732

版权

flink自测案例代码，整合DataStream和table-api进行流式开发

1、定义待插入表类

package pojo;

public class Message {
    public String messageId;
    public Integer values;
    public String eventTIME;

    public void setMessageId(String messageId) {
        this.messageId = messageId;
    }

    public void setValues(Integer values) {
        this.values = values;
    }

    public void setEventTIME(String eventTIME) {
        this.eventTIME = eventTIME;
    }

    @Override
    public String toString() {
        return "Message{" +
                "messageId='" + messageId + '\'' +
                ", values=" + values +
                ", eventTIME='" + eventTIME + '\'' +
                '}';
    }
}

2、自定义Source端

package source;

import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
import pojo.Message;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Random;

public class MySourceCarTime extends RichSourceFunction<Message> {

    @Override
    public void run(SourceContext<Message> ctx) throws Exception {
        Random random = new Random();

        String messageId;
        SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        String format;
        Message message = new Message();
        /**
         针对10条数据，随机生成
        */
        while (true){
            Date date = new Date();
            messageId="Message-"+random.nextInt(10);
            long time = date.getTime();
            format = dateFormat.format(time);
            message.setMessageId(messageId);
            message.setValues(1);
            message.setEventTIME(format);
            //ctx上下游对象，用于将数据发送到下游
            ctx.collect(message);
            Thread.sleep(1000);
        }
    }

    @Override
    public void cancel() {

    }
}

3、读取Source端数据，并针对数据做简要汇总后写入Mysql

import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import pojo.Message;
import source.MySourceCarTime;

public class DemoDSToTable {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);

        DataStreamSource<Message> messageDS = env.addSource(new MySourceCarTime());

        Table messageTable = tableEnv.fromDataStream(messageDS);

        tableEnv.createTemporaryView("base", messageTable);

//        Table table = tableEnv.sqlQuery("select messageId,sum(`values`) num from base group by messageId");
        //在flink-sql中注册一张表，既待插入数据表，设置别名为MyUserTable
        tableEnv.executeSql(
                "CREATE TABLE MyUserTable (\n" +
                        "    messageId STRING not null primary key,\n" +
                        "    num Double\n" +
                        ") WITH (\n" +
                        "   'connector' = 'jdbc',\n" +
                        "   'url' = 'jdbc:mysql://localhost:3306/test_db?useUnicode=true&characterEncoding=UTF-8',\n" +
                        "   'table-name' = 'message',\n" +
                        "   'username' = 'root',\n" +
                        "   'password' = '123456'\n" +
                        ")"
        );

        //结果数据插入注册表
        tableEnv.executeSql("insert into MyUserTable select messageId,sum(`values`) num from base group by messageId");

        env.execute("Demo");
    }
}