flink的job日志管理

flink job 运行设置

1、pom文件依赖

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <groupId>com.exam.iot</groupId>
    <artifactId>flink-job</artifactId>
    <version>0.0.1</version>
    <name>flink-job</name>
    <packaging>jar</packaging>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
        <logback.version>1.2.3</logback.version>
        <log4j-over-slf4j.version>1.7.30</log4j-over-slf4j.version>

        <java.version>1.8</java.version>
        <mysql.version>8.0.28</mysql.version>
        <jackson.version>2.12.5</jackson.version>
        <flink.version>1.9.3</flink.version>
        <dubbo.version>2.7.14</dubbo.version>
        <guava.version>31.1-jre</guava.version>
        <influxdb-client.version>2.22</influxdb-client.version>
    </properties>

    <dependencies>
        <dependency>
            <groupId>ch.qos.logback</groupId>
            <artifactId>logback-core</artifactId>
            <version>${logback.version}</version>
        </dependency>
        <dependency>
            <groupId>ch.qos.logback</groupId>
            <artifactId>logback-access</artifactId>
            <version>${logback.version}</version>
        </dependency>
        <dependency>
            <groupId>ch.qos.logback</groupId>
            <artifactId>logback-classic</artifactId>
            <version>${logback.version}</version>
        </dependency>
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>log4j-over-slf4j</artifactId>
            <version>${log4j-over-slf4j.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-lang3</artifactId>
            <version>3.12.0</version>
        </dependency>

        <!-- redis依赖  -->
        <dependency>
            <groupId>org.redisson</groupId>
            <artifactId>redisson</artifactId>
            <version>3.16.8</version>
        </dependency>

        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.79</version>
        </dependency>

        <!-- MySQL的JDBC驱动包	-->
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>${mysql.version}</version>
        </dependency>

        <dependency>
            <groupId>com.zaxxer</groupId>
            <artifactId>HikariCP</artifactId>
            <version>4.0.3</version>
        </dependency>

        <!-- jackson:用于json和Object转换 -->
        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-annotations</artifactId>
            <version>${jackson.version}</version>
        </dependency>
        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-databind</artifactId>
            <version>${jackson.version}</version>
        </dependency>

        <!--flink依赖-->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-core</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-java</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients_2.11</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java_2.11</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-kafka_2.11</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-runtime-web_2.11</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-avro</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <!-- 提供 LoadingCache -->
        <dependency>
            <groupId>com.google.guava</groupId>
            <artifactId>guava</artifactId>
            <version>${guava.version}</version>
        </dependency>

        <!--JSON解析依赖-->
        <dependency>
            <groupId>com.google.code.gson</groupId>
            <artifactId>gson</artifactId>
            <version>2.9.0</version>
        </dependency>

        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <version>1.18.22</version>
            <optional>true</optional>
        </dependency>
        <!-- 提供过期移除map的集合ExpiringMap、LoadingCache -->
        <dependency>
            <groupId>net.jodah</groupId>
            <artifactId>expiringmap</artifactId>
            <version>0.5.10</version>
        </dependency>

        <!-- influxdb -->
        <dependency>
            <groupId>org.influxdb</groupId>
            <artifactId>influxdb-java</artifactId>
            <version>${influxdb-client.version}</version>
        </dependency>
        <!-- tdEngine -->
        <dependency>
            <groupId>com.taosdata.jdbc</groupId>
            <artifactId>taos-jdbcdriver</artifactId>
            <version>2.0.37</version>
        </dependency>

        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-pool2</artifactId>
            <version>2.11.1</version>
        </dependency>

        <dependency>
            <groupId>cn.hutool</groupId>
            <artifactId>hutool-all</artifactId>
            <version>5.7.22</version>
        </dependency>
    </dependencies>

    <build>
        <finalName>${project.name}</finalName>

        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.8.1</version>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                </configuration>
            </plugin>

            <!--打jar包-->
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-assembly-plugin</artifactId>
                <version>3.3.0</version>
                <configuration>
                    <archive>
                        <manifest>
                            <mainClass>com.exam.job.JobApplication</mainClass>
                        </manifest>
                    </archive>
                    <descriptorRefs>
                        <descriptorRef>jar-with-dependencies</descriptorRef>
                    </descriptorRefs>
                </configuration>
            </plugin>

            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-shade-plugin</artifactId>
                <version>3.2.4</version>
                <executions>
                    <execution>
                        <phase>package</phase>
                        <goals>
                            <goal>shade</goal>
                        </goals>
                        <configuration>
                            <artifactSet>
                                <excludes>
                                    <exclude>com.google.code.findbugs:jsr305</exclude>
                                    <exclude>org.slf4j:*</exclude>
                                    <exclude>log4j:*</exclude>
                                </excludes>
                            </artifactSet>
                            <filters>
                                <filter>
                                    <!-- Do not copy the signatures in the META-INF folder.
                                    Otherwise, this might cause SecurityExceptions when using the JAR. -->
                                    <artifact>*:*</artifact>
                                    <excludes>
                                        <exclude>META-INF/*.SF</exclude>
                                        <exclude>META-INF/*.DSA</exclude>
                                        <exclude>META-INF/*.RSA</exclude>
                                    </excludes>
                                </filter>
                            </filters>

                            <transformers combine.children="append">
                                <transformer
                                        implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                                    <mainClass>com.exam.iot.JobApplication</mainClass>
                                </transformer>

                                <transformer
                                        implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
                                    <resource>reference.conf</resource>
                                </transformer>

                                <transformer
                                        implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer">
                                </transformer>
                            </transformers>

                            <relocations>
                                <relocation>
                                    <pattern>org.codehaus.plexus.util</pattern>
                                    <shadedPattern>org.shaded.plexus.util</shadedPattern>
                                    <excludes>
                                        <exclude>org.codehaus.plexus.util.xml.Xpp3Dom</exclude>
                                        <exclude>org.codehaus.plexus.util.xml.pull.*</exclude>
                                    </excludes>
                                </relocation>
                            </relocations>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
        </plugins>

        <resources>
            <resource>
                <directory>src/main/java</directory>
                <includes>
                    <include>**/*.yml</include>
                    <include>**/*.properties</include>
                    <include>**/*.xml</include>
                </includes>
                <filtering>false</filtering>
            </resource>
            <resource>
                <directory>src/main/resources</directory>
                <includes>
                    <include>**/*.yml</include>
                    <include>**/*.properties</include>
                    <include>**/*.xml</include>
                </includes>
                <filtering>false</filtering>
            </resource>
        </resources>
    </build>
</project>

2、在src/main/resources创建logback.xml文件

<?xml version="1.0" encoding="UTF-8"?>
<configuration>

    <property name="LOGS" value="./logs/" />
    <appender name="Console" class="ch.qos.logback.core.ConsoleAppender">
        <layout class="ch.qos.logback.classic.PatternLayout">
         	 <!-- (1) -->
             <pattern>%black(%d{ISO8601}) %highlight(%-5level) [%blue(%t)] %yellow(%C{1.}): %msg%n%throwable</pattern>
              <!-- (2) -->
             <pattern>%date{yyyy-MM-dd HH:mm:ss} %highlight(%-5level) (%file:%line\)- %m%n</pattern>
            <!-- 控制台也要使用utf-8，不要使用gbk -->
            <charset>UTF-8</charset>
        </layout>
    </appender>

    <appender name="RollingFile" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <file>${LOGS}/electric-job.log</file>
        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
            <Pattern>%d %p %C{1.} [%t] %m%n</Pattern>
        </encoder>
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <!-- rollover daily and when the file reaches 10 MegaBytes -->
            <fileNamePattern>${LOGS}/archived/job-%d{yyyy-MM-dd}.%i.log
            </fileNamePattern>
            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
                <maxFileSize>100MB</maxFileSize>
            </timeBasedFileNamingAndTriggeringPolicy>
        </rollingPolicy>
    </appender>

   <!-- RollingFileAppender:滚动记录文件，先将日志记录到指定文件，当符合某个条件时，将日志记录到其他文件 -->
    <!-- 1.先按日期存日志，日期变了，将前一天的日志文件名重命名为xxx%日期%索引，新的日志仍然是sys.log -->
    <!-- 2.如果日期没有变化，但是当前日志文件的大小超过1kb时，对当前日志进行分割 重名名 -->
    <appender name="syslog" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <!--<File>${catalina.base}/mylog/sys.log</File>-->
        <File>./logs/%d/plug-flink-job.log</File>
        <!-- rollingPolicy:当发生滚动时，决定 RollingFileAppender 的行为，涉及文件移动和重命名。 -->
        <!-- TimeBasedRollingPolicy： 最常用的滚动策略，它根据时间来制定滚动策略，既负责滚动也负责出发滚动 -->
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <!-- 活动文件的名字会根据fileNamePattern的值，每隔一段时间改变一次 -->
            <!-- 文件名：mylog/sys.2017-12-05.0.log -->
            <!--<fileNamePattern>${catalina.base}/mylog/sys.%d.%i.log</fileNamePattern>-->
            <fileNamePattern>./logs/%d/plug-flink-job.%d.%i.log</fileNamePattern>
            <!-- 每产生一个日志文件，该日志文件的保存期限为30天 -->
            <maxHistory>30</maxHistory>
            <timeBasedFileNamingAndTriggeringPolicy  class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
                <!-- maxFileSize:这是活动文件的大小，默认值是10MB,本篇设置为1KB，只是为了演示 -->
                <maxFileSize>128MB</maxFileSize>
            </timeBasedFileNamingAndTriggeringPolicy>
        </rollingPolicy>
        <encoder>
            <!-- pattern节点，用来设置日志的输入格式 -->
            <pattern>
                %d %p (%file:%line\)- %m%n
            </pattern>
            <!-- 记录日志的编码 -->
            <charset>UTF-8</charset> <!-- 此处设置字符集 -->
        </encoder>
    </appender>

    <!-- LOG everything at INFO level -->
    <root level="INFO">
        <appender-ref ref="RollingFile" />
        <appender-ref ref="Console" />
    </root>

    <!-- LOG "com.exam.job*" at TRACE level -->
    <logger name="com.exam.iot" level="INFO" additivity="false">
        <appender-ref ref="RollingFile" />
        <appender-ref ref="Console" />
    </logger>
</configuration>

3、job示例

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.util.Collector;

import java.util.Properties;

public class FlinkKafkaWordCount {
    public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.enableCheckpointing(5000);
        env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);

        Properties properties = new Properties();
        properties.setProperty("bootstrap.servers", "10.170.224.246:9092,10.170.224.246:9093,10.170.224.246:9094");
        properties.setProperty("group.id", "word-count-group");

        FlinkKafkaConsumer<String> consumer = new FlinkKafkaConsumer<>("wc_topic_post", new SimpleStringSchema(), properties);

        DataStream<WordWithCount> counts = env.addSource(consumer)
                .flatMap(new FlatMapFunction<String, WordWithCount>() {
                    @Override
                    public void flatMap(String value, Collector<WordWithCount> collector) throws Exception {
                        for (String word : value.split("\t")) {
                            collector.collect(new WordWithCount(word, 1L));
                        }
                    }
                })
                .keyBy("word")
                .timeWindow(Time.seconds(5))
                .reduce(new ReduceFunction<WordWithCount>() {
                    @Override
                    public WordWithCount reduce(WordWithCount a, WordWithCount b) throws Exception {
                        return new WordWithCount(a.word, a.count + b.count);
                    }
                });

        counts.print().setParallelism(1);
        env.execute("flink-kafka-wordcount");
    }

    public static class WordWithCount {
        private String word;
        private long count;

        public WordWithCount() {
        }

        public WordWithCount(String word, long count) {
            this.word = word;
            this.count = count;
        }

        public String getWord() {
            return word;
        }

        public void setWord(String word) {
            this.word = word;
        }

        public long getCount() {
            return count;
        }

        public void setCount(long count) {
            this.count = count;
        }

        @Override
        public String toString() {
            return word + " : " + count;
        }
    }
}

4、设置checkpoint

       // 并行度优先级:算子>env>client>配置文件
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

        env.enableCheckpointing(10 * 60 * 1000, CheckpointingMode.EXACTLY_ONCE);
        // set checkpoint
        CheckpointConfig checkpointConfig = env.getCheckpointConfig();
        checkpointConfig.setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
        checkpointConfig.setCheckpointTimeout(60 * 1000);
        checkpointConfig.setMinPauseBetweenCheckpoints((60 * 1000));
        checkpointConfig.setMaxConcurrentCheckpoints(1);
        checkpointConfig.setTolerableCheckpointFailureNumber(3);
        // checkpointConfig.setPreferCheckpointForRecovery(true);
        // checkpointConfig.enableUnalignedCheckpoints(); // 非对齐的 Checkpoint
        // 可以配置 Checkpoint 定期持久化到从外部存储中。使用这种方式不会在任务正常停止的过程中清理 Checkpoint 数据，而是会一直保存在外部存储中，另外我们也可以通过从外部 Checkpoint 中对任务进行恢复：
        checkpointConfig.enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
        // 设置重启策略
        env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, 60 * 1000));

5、job设置

		try {
            env.addSource(AppConfig.getConsumer())
                    .setParallelism(8)
                    .map(new RichMapFunction<String, DeviceClientInfo>() {
                        @Override
                        public DeviceClientInfo map(String jsonLine) {
                            try {
                                log.info("jsonLine={}", jsonLine);
                                String line = jsonLine.replaceAll("socket_abb.", "");
                                return JSONObject.parseObject(line, DeviceClientInfo.class);
                            } catch (Exception e) {
                                log.error("job.line.map报错: {}", e.getMessage());
                            }
                            return null;
                        }
                    })
                    .assignTimestampsAndWatermarks(new AssignerWithPeriodicWatermarks<DeviceClientInfo>() {
                        // 观察到的最大时间戳
                        private long currentMaxTs = Long.MIN_VALUE;

                        @Nullable
                        @Override
                        public Watermark getCurrentWatermark() {
                            // 生成具有30秒容忍度的水位线
                            // 定义30秒的容忍间隔时间，即允许数据的最大乱序时间
                            long maxOutofOrderness = 30 * 1000;
                            return new Watermark(currentMaxTs - maxOutofOrderness);
                        }

                        @Override
                        public long extractTimestamp(DeviceClientInfo deviceClientInfo, long previousElementTimestamp) {
                            //获取当前记录的时间戳
                            long currentTs = deviceClientInfo.getTimestamp();
                            // 更新最大的时间戳
                            currentMaxTs = Math.max(currentMaxTs, currentTs);
                            // 返回记录的时间戳
                            return currentTs;
                        }
                    })
                    .filter(new RichFilterFunction<DeviceClientInfo>() {
                        @Override
                        public boolean filter(DeviceClientInfo deviceClientInfo) {
                            return deviceClientInfo != null
                                    && StringUtils.isNotEmpty(deviceClientInfo.getDeviceId())
                                    && deviceClientInfo.getPayload() != null
                                    && deviceClientInfo.getPayload().getProperties() != null;
                        }
                    })
                    .map(new RichMapFunction<DeviceClientInfo, Electric>() {
                        final int recordFixedTime = 15;
                        @Override
                        public Electric map(DeviceClientInfo deviceClientInfo) {
                            try {
                                return parseElectric(recordFixedTime, deviceClientInfo);
                            } catch (Exception e) {
                                log.error("job.dci.map报错: ", e);
                            }
                            return null;
                        }
                    }).filter(new RichFilterFunction<Electric>() {
                        @Override
                        public boolean filter(Electric electric) {
                            return electric != null;
                        }
                    })
                    .keyBy(Electric::getDeviceId)
                    .timeWindow(Time.minutes(1))
                    .reduce(new ReduceFunction<Electric>() {
                        @Override
                        public Electric reduce(Electric preE, Electric curE) {
                            log.info("***** Electric curE={}", JSONObject.toJSONString(curE));
                            return curE;
                        }
                    })
                    .addSink(new SinkMysqlWithOne())
                    .name("mysqlSink");

        } catch (Exception e) {
            e.printStackTrace();
        }

        env.execute("degree-sink");

6、在src/main/resources目录下创建log4j.properties文件

log4j.rootLogger=INFO,logfile,stdout

#log4j.logger.org.springframework.web.servlet=INFO,db
#log4j.logger.org.springframework.beans.factory.xml=INFO
#log4j.logger.com.exam.job=INFO,db
#log4j.appender.stdout=org.apache.log4j.ConsoleAppender
#log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
#log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH\:mm\:ss} %p [%c] %X{remoteAddr}  %X{remotePort}  %X{remoteHost}  %X{remoteUser} operator\:[\u59D3\u540D\:%X{userName} \u5DE5\u53F7\:%X{userId}] message\:<%m>%n

#write log into file
log4j.appender.logfile=org.apache.log4j.DailyRollingFileAppender
log4j.appender.logfile.Threshold=warn
log4j.appender.logfile.File=./logs/job.log
log4j.appender.logfile.DatePattern=.yyyy-MM-dd
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
log4j.appender.logfile.layout.ConversionPattern=[app]-[%d{yyyy-MM-dd HH:mm:ss}] %X{remoteAddr} %X{remotePort} %m %n

#display in console
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Threshold=info
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=[app]-[%d{yyyy-MM-dd HH:mm:ss}] %X{remoteAddr} %X{remotePort} %m %n

7、滚动窗口示例

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.util.Collector;

public class WindowTest {
    public static void main(String[] args) {
        try {
            // 创建流处理的执行环境
            StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

            //2.使用StreamExecutionEnvironment创建DataStream
            //Source(可以有多个Source)
            //Socket 监听本地端口8888
            // 接收一个socket文本流
            DataStreamSource<String> lines = env.socketTextStream("localhost", 8888);

            // Transformation(s) 对数据进行转换处理统计，先分词，再按照word进行分组，最后进行聚合统计
            DataStream<Tuple2<String, Integer>> windowCount = lines.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
                public void flatMap(String line, Collector<Tuple2<String, Integer>> collector) throws Exception {
                    String[] words = line.split(" ");
                    for (String word : words) {
                        //将每个单词与 1 组合，形成一个元组
                        Tuple2<String, Integer> tp = Tuple2.of(word, 1);
                        //将组成的Tuple放入到 Collector 集合，并输出
                        collector.collect(tp);
                    }
                }
            });

            // 1. 滚动窗口（Tumbling Windows）使用例子
            //进行分组聚合(keyBy：将key相同的分到一个组中) //定义一个1分钟的翻滚窗口,每分钟统计一次
            DataStream<Tuple2<String, Integer>> windowStream = windowCount.keyBy(0)
                    .timeWindow(Time.minutes(1))
                    .sum(1);

            // 调用Sink （Sink必须调用）
            windowStream.print("windows: ").setParallelism(1);
            //timePoint+=30;
            //启动(这个异常不建议try...catch... 捕获,因为它会抛给上层flink,flink根据异常来做相应的重启策略等处理)
            env.execute("StreamWordCount");
        }catch (Exception e) {
            e.printStackTrace();
        }
    }
}

8、kafka消费配置

  static FlinkKafkaConsumer<String> getConsumer() {
        Properties props = new Properties();
        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, KAFKA_BROKER_LIST);
        props.put(ConsumerConfig.GROUP_ID_CONFIG, KAFKA_GROUP_ID);
        props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
        props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
        props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true);
        props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "3000");
        // start rebanlance conf
        props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "10000");
        props.put(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, "3000");

        props.put(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG, "5000");
        props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest");
        // consumer ability
        props.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, "1000");
        props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, "2000");
        // idle time
        props.put(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG, "3000");
        props.put(ConsumerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG, "60000");

        props.put(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, "52428800");
        props.put(ConsumerConfig.SOCKET_CONNECTION_SETUP_TIMEOUT_MAX_MS_CONFIG, "30000");

        FlinkKafkaConsumer<String> consumer = new FlinkKafkaConsumer<>(KAFKA_TOPIC, new SimpleStringSchema(), props);
        consumer.assignTimestampsAndWatermarks(new AscendingTimestampExtractor<String>() {
            @Override
            public long extractAscendingTimestamp(String s) {
                return System.currentTimeMillis();
            }
        });
        return consumer;
    }