flink自定义source插入或更新数据mysql数据

最新推荐文章于 2024-01-28 14:33:01 发布

hfx_2021

最新推荐文章于 2024-01-28 14:33:01 发布

阅读量528

点赞数

文章标签： flink mysql 大数据

本文链接：https://blog.csdn.net/hfx_2021/article/details/131622718

版权

文章展示了如何使用ApacheFlink进行数据流处理。首先定义了一个JavaBean类`Student`，接着创建了一个自定义的`SourceFunction`生成随机学生数据。然后在Flink的流执行环境中，将数据流转换为动态表并注册，以便在SQL中操作。最后，文章演示了如何连接到MySQL数据库，创建一个与Flink表结构匹配的表，并基于主键对学生数据进行更新操作。

摘要由CSDN通过智能技术生成

1.准备一个javabean

package com.atguigu.beans;


import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

/**
 * @author haungfx

 */

@Data
@NoArgsConstructor
@AllArgsConstructor

public class Student {
    private String student_id;
    private String student_name;
    private String birthday;
    private String sex;
}

2.自定义一个source

package com.atguigu.chapter05.source;

import com.atguigu.beans.Student;
import org.apache.flink.streaming.api.functions.source.SourceFunction;

import java.util.Random;

/**
 * @Author hfx@handsome.cn
 * @Date 2023/7/8 15:53
 */
public class RandowStudent implements SourceFunction<Student> {
    Boolean a=true;
    @Override
    public void run(SourceContext ctx) throws Exception {
        Random random = new Random();
     while (a){
         ctx.collect(new Student(random.nextInt(5)+"01",random.nextInt(100)+"赵雷",random.nextInt(5)+"19960809",random.nextInt(5)+"男"));
        Thread.sleep(500);
     }
    }

    @Override
    public void cancel() {
        a=false;
    }
}

3.将数据以mysql主键为准，更新mysql数据

package com.atguigu.chapter11;

import com.atguigu.beans.Student;
import com.atguigu.beans.WaterSensor;
import com.atguigu.chapter05.source.RandowStudent;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;

import static org.apache.flink.table.api.Expressions.$;

// 静态导入

/**
 * @Author huangfx
 * @Date 2021/11/19 10:26
 */
//直接打印动态表：result.execute().print();//直接打印表result
class Test02 {
    public static void main(String[] args) throws Exception {
//        获取流环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        DataStreamSource<Student> data = env.addSource(new RandowStudent());

        // 1. 创建表环境
        StreamTableEnvironment tenv = StreamTableEnvironment.create(env);
        // 2. 把流转成动态表
        Table table = tenv.fromDataStream(data,
                $("student_id"),$("student_name"),$("birthday"),$("sex"));
//        注册动态表让可以在sql里面查到
        tenv.createTemporaryView("student",table);
//        创建mysql的关联表
        tenv.executeSql("CREATE TABLE studentcp ( " +
                " student_id string, " +
                " student_name string, " +
                " birthday string, " +
                " sex string, " +
                " primary key (student_id) NOT ENFORCED "+ //可设置主键，插入时候根据这个字段进行数据的更新
                ")with(" +
                " 'connector' = 'jdbc', " +
                " 'url' = 'jdbc:mysql://127.0.0.1:3306/study?characterEncoding=utf8', " +
                " 'driver' = 'com.mysql.jdbc.Driver', " +
                " 'table-name' = 'studentcp', " +
                " 'username' = 'root', " +
                " 'password' = '123321' " +
                " )");
//插入数据
        tenv.executeSql("insert into studentcp select * from student");

//        env.execute();

    }
}