（21） Kafa-Flink-Mysql

最新推荐文章于 2023-07-05 14:59:15 发布

^果然好^

最新推荐文章于 2023-07-05 14:59:15 发布

阅读量206

点赞数

分类专栏： # Flink

本文链接：https://blog.csdn.net/allensandy/article/details/106062466

版权

Flink 专栏收录该内容

19 篇文章 4 订阅

订阅专栏

1、pom.xml

2、代码如下

3、运行效果

启动zookeeper、kafka、canal、flink

./zkServer.sh  start
./kafka-server-start.sh ../config/server.properties &
startup.bat  #
start-cluster.bat # windows启动

1、pom.xml

版本选择，参考：https://blog.csdn.net/yanghua_kobe/article/details/83210897

       <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-kafka_2.11</artifactId>
            <version>1.9.1</version>
        </dependency>

2、代码如下

package com.steven.flink;

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.aggregation.Aggregations;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.util.Collector;

import java.util.Properties;

/**
 * @Description: 第四个示例
 * @author: :  Steven
 * @Date: 2020/3/20 13:44
 */
public class Demo4 {
    public static void main(String[] args) throws Exception {
        // set up the streaming execution environment
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        //默认情况下，检查点被禁用。要启用检查点，请在StreamExecutionEnvironment上调用enableCheckpointing(n)方法，
        // 其中n是以毫秒为单位的检查点间隔。每隔5000 ms进行启动一个检查点,则下一个检查点将在上一个检查点完成后5秒钟内启动
        env.enableCheckpointing(5000);
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        Properties properties = new Properties();
        properties.setProperty("bootstrap.servers", "192.168.10.11:9092");//kafka的节点的IP或者hostName，多个使用逗号分隔
        properties.setProperty("zookeeper.connect", "192.168.10.11:2181");//zookeeper的节点的IP或者hostName，多个使用逗号进行分隔
        properties.setProperty("group.id", "test-consumer-group");//flink consumer flink的消费者的group.id
        FlinkKafkaConsumer<String> myConsumer = new FlinkKafkaConsumer<String>("example", new SimpleStringSchema(),
                properties);//test0是kafka中开启的topic
      //  myConsumer.setStartFromLatest();
     //  myConsumer.assignTimestampsAndWatermarks(new CustomWatermarkEmitter());
        DataStream<String> keyedStream = env.addSource(myConsumer);//将kafka生产者发来的数据进行处理，本例子我进任何处理
       // keyedStream.writeAsText("D:\\dev-tools\\apache-flink-1.9.2\\bin\\asdsss");
        keyedStream.print();//直接将从生产者接收到的数据在控制台上进行打印
        // execute program
        env.execute("Flink Streaming Java API Skeleton2");
    }
}