Flink的StreamingFileSink的行格式与列格式

最新推荐文章于 2024-05-02 20:48:43 发布

最后一瓶脉动

最新推荐文章于 2024-05-02 20:48:43 发布

阅读量147

点赞数

文章标签： flink java 服务器

本文链接：https://blog.csdn.net/weixin_43218601/article/details/133101094

版权

    public static void main(String[] args) throws Exception {

        Configuration conf = new Configuration();

        StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(conf);
        DataStreamSource<Student> stream1 = env.addSource(new MySource_01());

        //1.行格式
        DefaultRollingPolicy<String, String> policy = DefaultRollingPolicy.builder()
                .withInactivityInterval(10000)
                .withMaxPartSize(1024 * 1024)
                .withRolloverInterval(5000)
                .build();

        StreamingFileSink<String> sink1 = StreamingFileSink.forRowFormat(new Path("D://HAHA0920"), new SimpleStringEncoder<String>("UTF-8"))
                .withRollingPolicy(policy).build();

//        stream1.map(JSON::toJSONString).addSink(sink1);


        //2.列格式

        ParquetWriterFactory<Student> studentParquetWriterFactory = ParquetAvroWriters.forReflectRecord(Student.class);
        StreamingFileSink<Student> sink2 = StreamingFileSink.forBulkFormat(new Path("D://HAHA0920"), studentParquetWriterFactory)
                .withBucketAssigner(new DateTimeBucketAssigner<Student>("yyyy-MM-dd--HH"))
                .withRollingPolicy(OnCheckpointRollingPolicy.build())
                .build();
        stream1.addSink(sink2);
        env.execute();
    }
}