JAVA实现Flink Table文件写入写出

最新推荐文章于 2023-09-20 21:47:36 发布

走到无路可退

最新推荐文章于 2023-09-20 21:47:36 发布

阅读量2k

点赞数

分类专栏：大数据文章标签： flink java 文件读写大数据 flink table

本文链接：https://blog.csdn.net/qq_33517844/article/details/121400458

版权

大数据专栏收录该内容

3 篇文章 0 订阅

订阅专栏

JAVA实现Flink Table读取文件内容，通过SQL进行查询，在将结果写入到文件

package org.fenghua.example.table.connector;

import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.TableEnvironment;
import org.apache.flink.table.api.java.StreamTableEnvironment;
import org.apache.flink.table.descriptors.Csv;
import org.apache.flink.table.descriptors.FileSystem;
import org.apache.flink.table.descriptors.Schema;
import org.apache.flink.types.Row;

/**
 * 文件
 *
 * @author: fenghua
 * @date: 2021/11/17
 */
public class FileConnector {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        StreamTableEnvironment tEnv = TableEnvironment.getTableEnvironment(env);
        tEnv.connect(
                new FileSystem()
                        .path("D:\\DevTool\\IdeaProjects\\HwFlinkKafka\\src\\main\\resources\\Sensor.txt")
        ).withFormat(
                new Csv()
                        .field("id", Types.STRING)
                        .field("name", Types.STRING)
                        .field("idCard", Types.STRING)
                        .field("addressCode", Types.STRING)
                        .field("age", Types.INT)
                        .fieldDelimiter(",")
                        .lineDelimiter("\n")
                        .ignoreParseErrors()
        ).withSchema(
                new Schema()
                        .field("id", Types.STRING)
                        .field("name", Types.STRING)
                        .field("idCard", Types.STRING)
                        .field("addressCode", Types.STRING)
                        .field("age", Types.INT)
        )
                .inAppendMode()
                .registerTableSource("sensor");
        Table table = tEnv.sqlQuery("select id,name,idCard,addressCode,age from sensor");
        tEnv.toAppendStream(table, Row.class).print();

        tEnv.connect(
                new FileSystem()
                        .path("D:\\DevTool\\IdeaProjects\\HwFlinkKafka\\src\\main\\resources\\SensorOut.txt")
        ).withFormat(
                new Csv()
                        .field("id", Types.STRING)
                        .field("name", Types.STRING)
                        .field("idCard", Types.STRING)
                        .field("addressCode", Types.STRING)
                        .field("age", Types.INT)
                        .fieldDelimiter(",")
        ).withSchema(
                new Schema()
                        .field("id", Types.STRING)
                        .field("name", Types.STRING)
                        .field("idCard", Types.STRING)
                        .field("addressCode", Types.STRING)
                        .field("age", Types.INT)
        )
                .inAppendMode()
                .registerTableSink("sensorOut");
        env.setParallelism(1);
        table.insertInto("sensorOut");
        env.execute(" test ");
    }
}