基于datahub源表和flink实时计算处理统计demo

 

// 流处理
public static void main(String[] args) throws Exception{

        //获取运行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        //获取datahub配置 连接并获取数据源
        Properties datahubProperties = DatahubConfigure.getDatahubProperties();
        //消费端点信息
        String endpoint = datahubProperties.getProperty("endpoint");
        String accessId = datahubProperties.getProperty("accessId");
        String accessKey = datahubProperties.getProperty("accessKey");
        // 项目名称
        String projectName = "projectName";
        //project下topic名称
        String topicName = "topicName";

         long now = Instant.now().toEpochMilli();
        /**
         * startInMs 启动位点的时间戳
         * maxRetryTimes 读取最大重试次数  2.2.7以下版本默认：3  以上版本默认20
         * retryIntervalMs  重试间隔     2.2.7以下版本：1000   以上版本：50
         * batchReadSize  单次读取数据的条数 默认值10，可设置最大值为1000
         */
        DatahubSourceFunction datahubSourceFunction = new DatahubSourceFunction(endpoint, projectName, topicName, accessId, accessKey, now,Long.MAX_VALUE, 1, 1, 1);

        env.addSource(datahubSourceFunction).flatMap(new FlatMapFunction<List<RecordEntry>, Object>() {
            @Override
            public void flatMap(List<RecordEntry> recordEntries, Collector<Object> collector) throws Exception {
                 for (RecordEntry recordEntry : recordEntries) {
                    TupleRecordData recordData = (TupleRecordData) (recordEntry.getRecordData());
                    String search_key = (String) recordData.getField("search_key");
                    if (search_key != null) {
                        Jedis jedis = new Jedis("localhost", 6379);
                        jedis.zincrby(start_time+"-"+end_time+"_sreach_log", 1, search_key);
                    }
                }

            }
        });


        //start
        env.execute("start.....");
}