Flink SQL实时计算案例一

最新推荐文章于 2024-01-25 19:48:15 发布
最新推荐文章于 2024-01-25 19:48:15 发布
阅读量4.3k 收藏 13
点赞数 1
分类专栏： 学习笔记 文章标签： flink
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/zhiwenshow/article/details/106310766
版权

消息发送器

import com.alibaba.fastjson.JSONObject;
import com.wzw.ali.bean.Message02;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.serialization.StringSerializer;

import java.util.Properties;
import java.util.Random;

/**
 * @description 视频直播解决方案之视频核心指标监控
 * @author: ZhiWen
 * @create: 2020-04-09 10:02
 **/
public class MessageGenerator02 {

    /*
     {
    "ip": "ip",
    "agent": "agent",
    "roomid": "123456789",
    "userid": "123456789",
    "abytes": "123456",
    "afcnt": "34",
    "adrop": "3",
    "afts": "1515922566",
    "alat": "123",
    "vbytes": "123456",
    "vfcnt": "34",
    "vdrop": "4",
    "vfts": "1515922566",
    "vlat": "123",
    "ublock": "1",
    "dblock": "2",
    "timestamp": "1515922566",
    "region": "hangzhou"
}
     */

    private static String[] regions = new String[]{"beijing","tianjin","nanjing","hangzhou","shanghai"};
    private static String[] agents = new String[]{"Android","IPhone OS","Windows","Mac"};
    /**房间号*/
    private static String[] roomId = new String[]{"10001","10002","10003"};//,"10004","10005","10006","10007","10008","10009","10010"};
    /**
     * 客户端IP
     */
    private static String[] clientIp = new String[]{"172.24.103.101","172.24.103.102","172.24.103.103","172.24.103.104","172.24.103.1","172.24.103.2","172.24.103.3","172.24.103.4","172.24.103.5","172.24.103.6"};

    private static int[] aBytes = new int[]{3345,2234,3443,4563,2343,3243,7342,6845,890,7688};
    private static int[] aFcnt = new int[]{45,34,43,63,43,43,42,45,23,88};
    private static int[] aDrop = new int[]{5,4,0,2,1,3,2,0,0,8};
    private static int[] aLate = new int[]{105,40,113,92,81,93,60,35,90,108};
    private static int[] block =new int[]{0,1,0,0,2,0,0};
    private static long[] userId =new long[]{100134556,20013456,30013456,40013456};


    public static void main(String[] args) {

        //配置信息
        Properties props = new Properties();
        //kafka服务器地址
        props.put("bootstrap.servers", "172.24.103.8:9092");
        //设置数据key和value的序列化处理类
        props.put("key.serializer", StringSerializer.class);
        props.put("value.serializer", StringSerializer.class);
        //创建生产者实例
        KafkaProducer<String,String> producer = new KafkaProducer<>(props);



        Random random = new Random();
        while(true){

            Message02 message = new Message02();
            message.setRoomid(roomId[random.nextInt(roomId.length)]);
            message.setUserid(userId[random.nextInt(userId.length)]+random.nextInt(100));
            message.setAgent(agents[random.nextInt(agents.length)]);
            message.setIp(clientIp[random.nextInt(clientIp.length)]);
            message.setAbytes(aBytes[random.nextInt(aBytes.length)]);
            message.setAfcnt(aFcnt[random.nextInt(aFcnt.length)]);
            message.setAlat(aLate[random.nextInt(aLate.length)]);
            message.setAdrop(aDrop[random.nextInt(aDrop.length)]);
            message.setAfts(DateUtils.getCurrentDate());
            message.setVbytes(aBytes[random.nextInt(aBytes.length)]);
            message.setVfcnt(aFcnt[random.nextInt(aFcnt.length)]);
            message.setVlat(aLate[random.nextInt(aLate.length)]);
            message.setVdrop(aDrop[random.nextInt(aDrop.length)]);
            message.setVfts(DateUtils.getCurrentDate());
            message.setUblock(block[random.nextInt(block.length)]);
            message.setDblock(block[random.nextInt(block.length)]);
            message.setStamp(DateUtils.getCurrentDate());
            message.setRegion(regions[random.nextInt(regions.length)]);


            String json = JSONObject.toJSONString(message);
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }

            ProducerRecord record = new ProducerRecord<String, String>("topic_live",json);
            //发送记录
            producer.send(record);
            System.out.println(json);


        }
    }


}

flink-1.10.0 FLINK SQL消费消息并计算

import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.EnvironmentSettings;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.java.StreamTableEnvironment;
import org.apache.flink.types.Row;


/**
 * @description 视频直播解决方案之 视频核心指标监控
 * @author: ZhiWen
 * @create: 2020-04-08 11:33
 **/
public class MainAppLive {
    /**源表建表语句*/
    public static final String KAFKA_SOURCE_SQL = "CREATE TABLE app_heartbeat_stream_source (\n" +
            "\t`ip`                   VARCHAR,\n" +
            "\tagent                   VARCHAR,\n" +
            "\troomid                  VARCHAR,\n" +
            "\tuserid                  VARCHAR,\n" +
            "\tabytes                  VARCHAR,\n" +
            "\tafcnt                   VARCHAR,\n" +
            "\tadrop                   VARCHAR,\n" +
            "\tafts                    VARCHAR,\n" +
            "\talat                    VARCHAR,\n" +
            "\tvbytes                  VARCHAR,\n" +
            "\tvfcnt                   VARCHAR,\n" +
            "\tvdrop                   VARCHAR,\n" +
            "\tvfts                    VARCHAR,\n" +
            "\tvlat                    VARCHAR,\n" +
            "\tublock                  VARCHAR,\n" +
            "\tdblock                  VARCHAR,\n" +
            "\tregion                  VARCHAR,\n" +
            "\tstamp             VARCHAR,\n" +
            " app_ts AS TO_TIMESTAMP(stamp), --定义生成WATERMARK的字段，\n" +
            " WATERMARK FOR app_ts AS app_ts - INTERVAL '10' SECOND --WATERMARK比数据时间线性增加10S\n" +
            ") WITH (\n" +
            "\t'connector.type' = 'kafka',\n" +
            "\t'connector.version' = 'universal',\n" +
            "\t'connector.topic' = 'topic_live',\n" +
            "\t'update-mode' = 'append',\n" +
            "\t'connector.properties.zookeeper.connect' = '172.24.103.8:2181',\n" +
            "\t'connector.properties.bootstrap.servers' = '172.24.103.8:9092',\n" +
            "\t'connector.startup-mode' = 'latest-offset',\n" +
            "\t'format.type' = 'json'\n" +
            ")";
    /**类型转换语句*/
    public static final String TYPE_CONVER_SQL = "SELECT\n" +
            "    ip,\n" +
            "    agent,\n" +
            "    CAST(roomid AS BIGINT) as roomid,\n" +
            "    CAST(userid AS BIGINT) as userid,\n" +
            "    CAST(abytes AS BIGINT) as abytes,\n" +
            "    CAST(afcnt AS BIGINT) as afcnt,\n" +
            "    CAST(adrop AS BIGINT) as adrop,\n" +
            "    unix_timestamp(afts) as afts,\n" +
            "    CAST(alat AS BIGINT) as alat,\n" +
            "    CAST(vbytes AS BIGINT) as vbytes,\n" +
            "    CAST(vfcnt AS BIGINT) as vfcnt,\n" +
            "    CAST(vdrop AS BIGINT) as vdrop,\n" +
            "    unix_timestamp(vfts) as vfts,\n" +
            "    CAST(vlat AS BIGINT) as vlat,\n" +
            "    CAST(ublock AS BIGINT) as ublock,\n" +
            "    CAST(dblock AS BIGINT) as dblock,\n" +
            "    app_ts,\n" +
            "\tregion\n" +
            "FROM\n" +
            " app_heartbeat_stream_source";
    /**案例一*/
    public static final String WINDOW_CASE01_SQL01 = "SELECT\n" +
            "    CAST(TUMBLE_START(app_ts, INTERVAL '1' MINUTE) as VARCHAR) as app_ts,\n" +
            "    roomid,\n" +
            "    SUM(ublock) as ublock, \n" +
            "    SUM(dblock) as dblock, \n" +
            "    SUM(adrop) as adrop, \n" +
            "    SUM(vdrop) as vdrop, \n" +
            "    SUM(alat) as alat, \n" +
            "    SUM(vlat) as vlat  \n" +
            "FROM\n" +
            " view_app_heartbeat_stream\n" +
            "GROUP BY\n" +
            " TUMBLE(app_ts,INTERVAL '1' MINUTE),roomid";
    public static final String WINDOW_CASE01_SQL02 = "CREATE TABLE output_1 (\n" +
            "\tapp_ts             VARCHAR,\n" +
            "\troomid             BIGINT,\n" +
            "\tublock             BIGINT,\n" +
            "\tdblock             BIGINT,\n" +
            "\tadrop              BIGINT,\n" +
            "\tvdrop              BIGINT,\n" +
            "\talat               BIGINT,\n" +
            "\tvlat               BIGINT\n" +
            "--不支持 \tPRIMARY KEY (roomid)\n" +
            ") WITH (\n" +
            "\t'connector.type' = 'jdbc',\n" +
            "\t'connector.url' = 'jdbc:mysql://172.24.103.3:3306/flink',\n" +
            "\t'connector.table' = 'live_output_1',\n" +
            "\t'connector.username' = 'root',\n" +
            "\t'connector.password' = '123456',\n" +
            "\t'connector.write.flush.max-rows' = '10',\n" +
            "\t'connector.write.flush.interval' = '5s'\n" +
            ")";
    public static final String WINDOW_CASE01_SQL03 = "INSERT INTO output_1\n" +
            "SELECT * FROM room_error_statistics_10min";

    /**案例二*/
    public static final String WINDOW_CASE02_SQL01 = "SELECT\n" +
            "    CAST(TUMBLE_START(app_ts, INTERVAL '1' MINUTE) as VARCHAR) as app_ts,\n" +
            "    region,\n" +
            "    SUM(alat)/COUNT(alat) as alat,\n" +
            "    SUM(vlat)/COUNT(vlat) as vlat\n" +
            "FROM\n" +
            "    view_app_heartbeat_stream\n" +
            "GROUP BY\n" +
            "    TUMBLE(app_ts, INTERVAL '1' MINUTE), region";
    public static final String WINDOW_CASE02_SQL02 = "CREATE TABLE output_2 (\n" +
            "    app_ts VARCHAR,\n" +
            "    region VARCHAR,\n" +
            "    alat   DOUBLE,\n" +
            "    vlat   DOUBLE\n" +
            ") WITH (\n" +
            "\t'connector.type' = 'jdbc',\n" +
            "\t'connector.url' = 'jdbc:mysql://172.24.103.3:3306/flink',\n" +
            "\t'connector.table' = 'live_output_2',\n" +
            "\t'connector.username' = 'root',\n" +
            "\t'connector.password' = '123456',\n" +
            "\t'connector.write.flush.max-rows' = '10',\n" +
            "\t'connector.write.flush.interval' = '5s'\n" +
            ")";
    public static final String WINDOW_CASE02_SQL03 = "INSERT INTO output_2\n" +
            "SELECT * FROM region_lat_statistics_10min";

    /**案例三*/
    public static final String WINDOW_CASE03_SQL01 = "SELECT\n" +
            "    CAST(TUMBLE_START(app_ts, INTERVAL '1' MINUTE) as VARCHAR) as app_ts,\n" +
            "    SUM(IF(ublock <> 0 OR dblock <> 0, 1, 0)) / CAST(COUNT(DISTINCT userid) AS DOUBLE) as block_rate\n" +
            "FROM\n" +
            "    view_app_heartbeat_stream\n" +
            "GROUP BY\n" +
            "    TUMBLE(app_ts, INTERVAL '1' MINUTE)";
    public static final String WINDOW_CASE03_SQL02 = "CREATE TABLE output_3 (\n" +
            "\tapp_ts                VARCHAR,\n" +
            "\tblock_rate            DOUBLE\n" +
            ") WITH (\n" +
            "\t'connector.type' = 'jdbc',\n" +
            "\t'connector.url' = 'jdbc:mysql://172.24.103.3:3306/flink',\n" +
            "\t'connector.table' = 'live_output_3',\n" +
            "\t'connector.username' = 'root',\n" +
            "\t'connector.password' = '123456',\n" +
            "\t'connector.write.flush.max-rows' = '10',\n" +
            "\t'connector.write.flush.interval' = '5s'\n" +
            ")";
    public static final String WINDOW_CASE03_SQL03 = "INSERT INTO output_3\n" +
            "SELECT * FROM block_total_statistics_10min";

    /**案例四*/
    public static final String WINDOW_CASE04_SQL01 = "SELECT\n" +
            "    CAST(TUMBLE_START(app_ts, INTERVAL '1' MINUTE) as VARCHAR) as app_ts,\n" +
            "    SUM(ublock+dblock) / CAST(COUNT(DISTINCT userid) AS DOUBLE) as block_peruser\n" +
            "FROM\n" +
            "    view_app_heartbeat_stream\n" +
            "GROUP BY\n" +
            "    TUMBLE(app_ts, INTERVAL '1' MINUTE)";
    public static final String WINDOW_CASE04_SQL02 = "CREATE TABLE output_4 (\n" +
            "\tapp_ts                   VARCHAR,\n" +
            "\tblock_peruser            DOUBLE\n" +
            ") WITH (\n" +
            "\t'connector.type' = 'jdbc',\n" +
            "\t'connector.url' = 'jdbc:mysql://172.24.103.3:3306/flink',\n" +
            "\t'connector.table' = 'live_output_4',\n" +
            "\t'connector.username' = 'root',\n" +
            "\t'connector.password' = '123456',\n" +
            "\t'connector.write.flush.max-rows' = '10',\n" +
            "\t'connector.write.flush.interval' = '5s'\n" +
            ")";
    public static final String WINDOW_CASE04_SQL03 = "INSERT INTO output_4\n" +
            "SELECT * FROM block_peruser_statistics_10min";

    public static void main(String[] args) throws Exception {//todo 已实现

        //获取流式环境
        StreamExecutionEnvironment bsEnv = StreamExecutionEnvironment.getExecutionEnvironment();
        EnvironmentSettings bsSettings = EnvironmentSettings.newInstance()
                .useBlinkPlanner()
                .inStreamingMode()
                .build();

        //设置时间类型 至关重要
        bsEnv.setParallelism(5).setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

        //todo 创建一个 TableEnvironment
        // or TableEnvironment bsTableEnv = TableEnvironment.create(bsSettings);
        StreamTableEnvironment bsTableEnv = StreamTableEnvironment.create(bsEnv, bsSettings);





/*--***********************建源表 转类型*******************--*/
        bsTableEnv.sqlUpdate(KAFKA_SOURCE_SQL);
        Table view01 = bsTableEnv.sqlQuery(TYPE_CONVER_SQL);
        bsTableEnv.createTemporaryView("view_app_heartbeat_stream",view01);

        bsTableEnv.toAppendStream(view01,Row.class).print();

 /*--***********************************1.统计房间故障，故障包括卡顿、丢帧、音视频不同步信息，使用10分钟一个窗口进行统计*********************************--*/
        Table view11 = bsTableEnv.sqlQuery(WINDOW_CASE01_SQL01);
        bsTableEnv.createTemporaryView("room_error_statistics_10min",view11);

        //bsTableEnv.toRetractStream(view11,Row.class).print();

        bsTableEnv.sqlUpdate(WINDOW_CASE01_SQL02);

        bsTableEnv.sqlUpdate(WINDOW_CASE01_SQL03);


/*-***********************************2.分地域统计数据端到端延迟平均情况，每10分钟统计音频、视频平均延迟情况***********************************-*/


        Table view21 = bsTableEnv.sqlQuery(WINDOW_CASE02_SQL01);
        bsTableEnv.createTemporaryView("region_lat_statistics_10min",view21);

        //bsTableEnv.toRetractStream(view21,Row.class).print();

        bsTableEnv.sqlUpdate(WINDOW_CASE02_SQL02);

        bsTableEnv.sqlUpdate(WINDOW_CASE02_SQL03);



/*-************************3.统计实时整体卡顿率，即出现卡顿的在线用户数/在线总用户数*100%，通过此指标可以衡量当前卡顿影响的人群范围。***********************-*/

        Table view31 = bsTableEnv.sqlQuery(WINDOW_CASE03_SQL01);
        bsTableEnv.createTemporaryView("block_total_statistics_10min",view31);

        //bsTableEnv.toRetractStream(view31,Row.class).print();

        bsTableEnv.sqlUpdate(WINDOW_CASE03_SQL02);

        bsTableEnv.sqlUpdate(WINDOW_CASE03_SQL03);

/*-***************************4.统计人均卡顿次数，即在线卡顿总次数/在线用户数，通过此指标可以从卡顿频次上衡量整体的卡顿严重程度。**************************-*/

        Table view41 = bsTableEnv.sqlQuery(WINDOW_CASE04_SQL01);
        bsTableEnv.createTemporaryView("block_peruser_statistics_10min",view41);
        //bsTableEnv.toRetractStream(view41,Row.class).print();


        bsTableEnv.sqlUpdate(WINDOW_CASE04_SQL02);

        bsTableEnv.sqlUpdate(WINDOW_CASE04_SQL03);

/*-***********************************开始执行**************************************-*/
        //输出并行度
        System.out.println(bsEnv.getParallelism());
        //执行query语句 1.10在execute之前会先执行update语句 1.11后更改
        bsTableEnv.execute("live");



    }
}

案例补充 topN

/**案例七*/
    public static final String WINDOW_CASE07_SQL01 = "SELECT\n" +
            "    CAST(TUMBLE_START(app_ts, INTERVAL '60' SECOND) as VARCHAR) as app_ts,\n" +
            "    roomid as room_id,\n" +
            "    COUNT(DISTINCT userid) as app_room_user_cnt\n" +
            "FROM\n" +
            "    view_app_heartbeat_stream\n" +
            "GROUP BY\n" +
            "    TUMBLE(app_ts, INTERVAL '60' SECOND), roomid";
    public static final String WINDOW_CASE07_SQL02 = "SELECT\n" +
            "  app_ts,\n" +
            "  room_id,\n" +
            "  app_room_user_cnt,\n" +
            "  ranking\n" +
            "  --PRIMARY KEY (app_ts,room_id,ranking)\n" +
            "FROM\n" +
            "(\n" +
            "    SELECT \n" +
            "        app_ts,\n" +
            "        room_id,\n" +
            "        app_room_user_cnt,\n" +
            "        ROW_NUMBER() OVER (PARTITION BY 1 ORDER BY app_room_user_cnt desc) AS ranking\n" +
            "    FROM\n" +
            "        view_app_room_visit_1min\n" +
            ") WHERE ranking <= 10";
    public static final String WINDOW_CASE07_SQL03 = "CREATE TABLE output_7 (\n" +
            "\tapp_ts                       VARCHAR,\n" +
            "\troomid                       BIGINT,\n" +
            "\tapp_room_user_cnt            BIGINT,\n" +
            "\tranking                      BIGINT\n" +
            ") WITH (\n" +
            "\t'connector.type' = 'jdbc',\n" +
            "\t'connector.url' = 'jdbc:mysql://172.24.103.3:3306/flink',\n" +
            "\t'connector.table' = 'live_output_7',\n" +
            "\t'connector.username' = 'root',\n" +
            "\t'connector.password' = '123456',\n" +
            "\t'connector.write.flush.max-rows' = '10',\n" +
            "\t'connector.write.flush.interval' = '5s'\n" +
            ")";
    public static final String WINDOW_CASE07_SQL04 = "INSERT INTO output_7\n" +
            "SELECT * FROM view_app_room_visit_top10";


/*-***********************************7.热门直播房间排行**************************************-*/
       /*Table view71 = bsTableEnv.sqlQuery(WINDOW_CASE07_SQL01);
        bsTableEnv.createTemporaryView("view_app_room_visit_1min",view71);
        Table view72 = bsTableEnv.sqlQuery(WINDOW_CASE07_SQL02);
        bsTableEnv.createTemporaryView("view_app_room_visit_top10",view72);
        bsTableEnv.toRetractStream(view72,Row.class).print();
        bsTableEnv.sqlUpdate(WINDOW_CASE07_SQL03);
        bsTableEnv.sqlUpdate(WINDOW_CASE07_SQL04);*/
橘子洲头无桔子
关注
  • 1
    点赞
  • 踩
  • 13
    收藏
    觉得还不错? 一键收藏
  • 15
    评论
专栏目录
阿里云流计算FlinkSQL核心功能解密.pptx
08-28
"阿里云技术专家伍翀（云邪）在2017广州云栖大会中做了题为《阿里云流计算 Flink SQL 核心功能解密》的分享,就Flink SQL 解密，StreamCompute 2.0 平台，应用案例等方面的内容做了深入的分析。
Flink1.15 SQL实现翻滚窗口实时计算
大数据研习社
05-07 1712
摘要： 随着Flink1.5的发布，FlinkSQL 流批一体更加成熟与完善。 Flink SQL可以替代Flink DataStream实现窗口计算。 1 Flink翻滚窗口适用场景 1.1定义 将数据依据固定的窗口度对无界数据流进行切片。 1.2特点 时间对、窗口长度固定、event无重叠。 1.3适用场景 BI统计(计算各个时间段的指标) 2 Flink SQL窗口编程模型 Table table = input .window([...
Flink SQL案例实践(1.11.0)
xiaohu21的博客
12-26 868
Flink SQL案例实践(1.11.0) 1. 背景 2020年随着阿里flink 批流一体大会开展,更多人和公司知道了flink的强大以及业务场景下的实际表现.关注和使用flink的公司以及个人会越来越多 在大数据领域,一个引擎可以同时支持处理结构化数据,图计算,机器学习,流计算目前主流还是spark和flink. 由于引擎设计理念,flink一开始就是为流计算设计,而spark则一开始就是以批处理设计的,不过是微批次的理念.这也导致了在一些方面,流数据处理领域flink会更加强大和灵活. 本文案例是
实时计算flinksql使用sum(),avg(),max(),min(),over(),WATERMARK示例
最新发布
weixin_44236758的博客
01-25 337
实时计算flinksql使用sum(),avg(),max(),min(),over(),WATERMARK示例
基于袋鼠云实时开发平台开发 FlinkSQL 任务的实践探索
a958014226的博客
07-05 138
每种开发模式没有绝对的好坏之分，需要根据不同企业的实时计算场景和阶段，采⽤不同的开发模式，才能真正达到降本增效的目的。· 当企业刚接触实时计算，数据开发⼈员对 FlinkSQL 熟悉度较低时，DEMO 模式是最好的选择；· 当企业已经上⼿实时计算，但是任务量还不⼤时，脚本模式或者向导模式是不错的选择；· 当企业实时计算达到⼀定规模，需要进⾏类似离线数仓的管理⽅式时，Catalog 模式是最优的选择。
利用Flink SQL实时统计单据状态（含历史数据）
王卫东 博客
08-01 7865
需求抽象下，有一种单据，整个生命周期中有很多种状态，现在要实时按照某个维度（例如用户id）统计其中某几个状态的数量，并按照该维度定时推送统计数据。
基于flinksql的实时计算案例
科学的N次方
10-25 1198
因公司发展需要，需半小时统计一次截止到当前时间的关键指标的当日累计值。 v1.0 链路采取的是 kafka-> clickhouse 在设计的时候，一开始本人是想一个flinksql从source到transform到sink完成. v2.0 链路采取的是 kafka->kafka-> clickhouse 奈何公司平台sink到ck的时候不支持upsert流。因此只能分解到先用一个flinksql加工出dwd.再起一个任务从dwd读数加工到ck. ...
Flink 应用案例——求网页访问量Top N 实时计算（附可执行代码）
m0_55685698的博客
03-17 1460
以下数据 为某网站的访问日志 现要求通过以下数据 统计出最近10s内最热门的N个页面（即url链接），并且每5s更新一次；即求出页面访问量中的TOP N
Flink+Kafka做有状态的实时计算实例
weixin_43424643的博客
11-27 1168
Flink消费Kafka数据，对消费的数据做实时计算是实时计算领域应用最为广泛的场景。下面介绍下如何使用Flink对接Kafka,并成功消费数据（考虑到计算程序复杂性本文以WordCount为例）做实时计算， 1.首先是pom文件，这里的Flink，Kafka版本要与集群中安装的一致。 <dependency> <groupId>org.apache.flink</groupId> <artifactId>
flink-sql大量使用案例
第一片心意的博客
08-02 6493
本文是 flink sql 的一些使用案例，同时也包括了一些特殊用法展示。
【大数据开发】FlinkSQL实战
白色风车
01-21 351
参考：https://cloud.tencent.com/developer/article/1450865 score.txt文件 17-18,詹姆斯-哈登,72,72,35.4,8.8,1.8,0.7,30.4 16-17,拉塞尔-威斯布鲁克,81,81,34.6,10.4,1.6,0.4,31.6 15-16,斯蒂芬-库里,79,79,34.2,6.7,2.1,0.2,30.1 14-15,拉塞尔-威斯布鲁克,67,67,34.4,8.6,2.1,0.2,28.1 13-14,凯文-杜兰特,81,.
阿里云流计算FlinkSQL核心功能解密.pdf
08-29
阿里云技术专家 伍翀（云邪）在2017云栖大会·北京峰会中做了题为《阿里云流计算 Flink SQL 核心功能解密》的分享,就Flink 2SQL 解密，StreamCompute 3 2.0 平台，应用案例等方面的内容做了深入的分析。
flink1.12入门到精通视频课程
07-15
共课程包含9个章节：Flink安装部署与快速入门、Flink批处理API、Flink流处理API、Flink高级API、Flink-Table与SQL、Flink-Action综合练习、Flink-高级特性和新特性、Flink多语言开发、Flink性能调优 课程目录： ...
BAT等关于Flink实际应用及优化案例PPT.rar
12-14
BAT等各大互联网巨头企业关于Flink实际应用及优化案例
flink入门到精通视频教程
07-15
11.Flink Table API与SQL 课程目录介绍 第一章 Flink简介 01.Flink的引入 02.什么是Flink 03.Flink流处理特性 04.Flink基石 05.批处理与流处理 第二章 Flink架构体系 01.Flink中重要角色 02.无界数据流与有界数据流 ...
Flink与Kafka的爱恨情仇
zhiwen
09-23 2686
使用 Flink-connector-kafka 可能会遇到的问题 如何消费多个 Kafka Topic 通常可能会有很多类型的数据全部发到 Kafka，但是发送的数据却不是在同一个 Topic 里面，然后在 Flink 处消费的时候，又要去同时消费这些多个 Topic，在 Flink 中除了支持可以消费单个 Topic 的数据，还支持传入多个 Topic，另外还支持 Topic 的正则表达式（因为有时候可能会事先不确定到底会有多少个 Topic，所以使用正则来处理会比较好，只要在 Kafka 建立的 To
FlinkKafkaConsumer
zhiwen
04-06 2219
offset什么时候提交 主要看是否开启了checkpoint 开启了checkpoint 情况1： 用户通过调用 consumer上的setCommitOffsetsOnCheckpoints(true) 方法来启用 offset 的提交(默认情况下为 true ) 那么当 checkpointing 完成时，Flink Kafka Consumer 将提交的 offset 存储在 checkpoint 状态中。 这确保 Kafka broker 中提交的 offset 与 checkpoint 状态中
Flink SQL语法检查
zhiwen
05-16 1891
可以直接调用flink自己的sql 解析包 /** * 解析 flink sql 语句 * * @param sql * @return */ public static void parseSql(String sql) throws Exception { SqlParser parser = SqlParser.create(sql, SqlParser.configBuilder() .setParserFactory(FlinkSqlParserImpl.
flink sql 案例
05-21
下面是一个简单的Flink SQL案例： 假设我们有一个名为"user_events"的表，其中包含用户ID和用户事件（如点击或购买）。我们希望每小时计算出每个用户的事件总数。 我们可以使用以下Flink SQL查询实现此目的： ```...

“相关推荐”对你有帮助么？

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论 15
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值