Flink 实时计算 - 进阶篇（如何自定义 Data Source 与 Data Sink）

第1关：Flink 实现自定义 Data Source (一)

import org.apache.flink.streaming.api.functions.source.RichSourceFunction;

public class CustomDataSource_1 extends RichSourceFunction<Integer> {

    @Override
    public void run(SourceContext ctx) throws Exception {
        /**********begin**********/
        for (int i = 0; i < 100; i++) {
            ctx.collect(i);
        }
        /**********end**********/
    }

    @Override
    public void cancel() {
    }
}

第2关：Flink 实现自定义 Data Source（

import org.apache.flink.streaming.api.functions.source.RichSourceFunction;

import java.io.BufferedReader;
import java.io.FileReader;

public class CustomDataSource_2 extends RichSourceFunction<String> {

    @Override
    public void run(SourceContext<String> ctx) throws Exception {
        // 指定文件路径
        String filePath = "/root/word.txt";
        
        // 使用 BufferedReader 读取文件内容
        try (BufferedReader br = new BufferedReader(new FileReader(filePath))) {
            String line;
            // 逐行读取文本内容并发送给 Flink 程序
            while ((line = br.readLine()) != null) {
                ctx.collect(line);
            }
        }
    }

    @Override
    public void cancel() {
        // 取消操作
    }
}

第3关

import org.apache.flink.configuration.Configuration;

import org.apache.flink.streaming.api.functions.source.RichSourceFunction;



import java.sql.Connection;

import java.sql.DriverManager;

import java.sql.PreparedStatement;

import java.sql.ResultSet;



public class CustomDataSource_3 extends RichSourceFunction<User> {

    private static final String URL = "jdbc:mysql://localhost:3306/mydb?useUnicode=true&characterEncoding=UTF-8";

    private static final String USERNAME = "root";

    private static final String PASSWORD = "123123";

    private static final String SQL = "SELECT id, username, sex, age FROM user";



    private transient Connection connection;

    private transient PreparedStatement statement;

    private volatile boolean isRunning = true;



    /**********begin**********/

    @Override

    public void open(Configuration parameters) throws Exception {

        Class.forName("com.mysql.jdbc.Driver");

        connection = DriverManager.getConnection(URL, USERNAME, PASSWORD);

        statement = connection.prepareStatement(SQL);

    }



    @Override

    public void run(SourceContext<User> ctx) throws Exception {

        ResultSet resultSet = statement.executeQuery();

        while (resultSet.next() && isRunning) {

            int id = resultSet.getInt("id");

            String userName = resultSet.getString("username");

            String sex = resultSet.getString("sex");

            int age = resultSet.getInt("age");

            ctx.collect(new User(id, userName, sex, age));

        }

    }



    @Override

    public void close() throws Exception {

        if (connection != null) {

            connection.close();

        }

        if (statement != null) {

            statement.close();

        }

    }

    /**********end**********/



    @Override

    public void cancel() {

        isRunning = false;

    }

}

第4关 

import org.apache.flink.configuration.Configuration;

import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;



import java.io.FileOutputStream;

import java.io.OutputStreamWriter;





public class CustomDataSink_1 extends RichSinkFunction<String> {

    

    private transient OutputStreamWriter writer;



    /**********begin**********/

    @Override

    public void open(Configuration parameters) throws Exception {

        FileOutputStream fos = new FileOutputStream("/root/dataSlink1.txt");

        writer = new OutputStreamWriter(fos);

    }



    @Override

    public void invoke(String value, Context context) throws Exception {

        writer.write(value);

        writer.write("\n"); // 换行

    }



    @Override

    public void close() throws Exception {

        if (writer != null) {

            writer.close();

        }

    }

    /**********end**********/



}

第5关 

import org.apache.flink.configuration.Configuration;

import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;



import java.sql.Connection;

import java.sql.DriverManager;

import java.sql.PreparedStatement;



public class CustomDataSink_2 extends RichSinkFunction<User> {

    private static final String URL = "jdbc:mysql://localhost:3306/mydb?useUnicode=true&characterEncoding=UTF-8";

    private static final String USERNAME = "root";

    private static final String PASSWORD = "123123";

    private static final String INSERT_SQL = "INSERT INTO user2 (id, username, sex, age) VALUES (?, ?, ?, ?)";



    private transient Connection connection;

    private transient PreparedStatement statement;



    /**********begin**********/

    @Override

    public void open(Configuration parameters) throws Exception {

        Class.forName("com.mysql.jdbc.Driver");

        connection = DriverManager.getConnection(URL, USERNAME, PASSWORD);

        statement = connection.prepareStatement(INSERT_SQL);

    }



    @Override

    public void invoke(User value, Context context) throws Exception {

        statement.setInt(1, value.getId());

        statement.setString(2, value.getUserName());

        statement.setString(3, value.getSex());

        statement.setInt(4, value.getAge());

        statement.executeUpdate();

    }



    @Override

    public void close() throws Exception {

        if (statement != null) {

            statement.close();

        }

        if (connection != null) {

            connection.close();

        }

    }

    /**********end**********/

}