Flink实践代码-DataStream 增加 watermark，并做窗口统计

Will Cui

已于 2024-04-07 16:35:59 修改

阅读量1k

点赞数 26

文章标签：开发语言 flink 大数据

于 2024-04-07 16:13:32 首次发布

本文链接：https://blog.csdn.net/danevc/article/details/137467445

版权

DataStream 中增加 WaterMark

摘要由CSDN通过智能技术生成

1.代码与含义解释

1.1 思路

Flink 获取数据流后，需要做数据过滤那么首先就要有一下几个步骤：

构建运行环境
接入数据流
数据（DataStream）加入 WaterMark

1.2 直接上代码

package com.youtree.flink;

import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONObject;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.api.Schema;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.Tumble;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.types.Row;
import org.apache.flink.util.Collector;

import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.util.Objects;

import static org.apache.flink.table.api.Expressions.$;
import static org.apache.flink.table.api.Expressions.lit;

/******************************************************
 * @program        : com.youtree.flink
 * @ClassName      : Visit_Info_for_dataStreamWaterMark
 * @Author         : 111111
 * @Date           : 2024/4/7 10:12
 * @Description    :   
 * @Version        :  
/*******************************************************/
public class Visit_Info_for_dataStreamWaterMark {
    public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(10);
        final StreamTableEnvironment tabEnv = StreamTableEnvironment.create(env);


        DataStream<String> VisitInfo = env.readTextFile("/Users/willcui/Desktop/orgpath/visit.txt");

        DataStream<JSONObject> jsonObjectDataStream = VisitInfo.flatMap(new FlatMapFunction<String, JSONObject>() {
                    @Override
                    public void flatMap(String s, Collector<JSONObject> collector) throws Exception {
                        JSONObject jsonObject = JSON.parseObject(s);
                        jsonObject.remove("visit_info");
                        collector.collect(jsonObject);
                    }
                })
                .filter(value -> String.valueOf(value.get("is_valid")).equals("1"))
                .filter(value -> !String.valueOf(value.get("create_time")).isEmpty())
                .filter(Objects::nonNull);

        DataStream<Row> rowDataStream = jsonObjectDataStream.flatMap(new FlatMapFunction<JSONObject, Row>() {
            @Override
            public void flatMap(JSONObject jsonObject, Collector<Row> collector) throws Exception {
                Row row = new Row(jsonObject.size());
                Integer index = 0;
                for (String key : jsonObject.keySet()) {
                    if (key.equals("create_time")) {
                        String createTime= jsonObject.getString("create_time");
                        LocalDateTime formattedDate = Timestamp.valueOf(createTime).toLocalDateTime();
                        row.setField(index, formattedDate);
                    } else {
                        row.setField(index, jsonObject.get(key));
                    }
                    index++;
                }

                collector.collect(row);
            }
        }).returns(
                Types.ROW_NAMED(
                        new String[]{column infos},
                        Types.STRING,
                        Types.STRING,
                        Types.STRING,
                        Types.STRING,