Flink将数据流写入Kafka,Redis,ES,Mysql

导入公共依赖

 		<dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-simple</artifactId>
            <version>1.7.25</version>
            <scope>compile</scope>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-java</artifactId>
            <version>1.10.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java_2.12</artifactId>
            <version>1.10.1</version>
        </dependency>

写入到 Mysql

导入依赖

 <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.44</version>
 </dependency>

/*
 * Copyright (C) 2021 Baidu, Inc. All Rights Reserved.
 */
package com.baidu.flink.function;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;

import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;

public class SinkToMysqlTest {
    //dataStream 写入mysql
    public static void main(String[] args) {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        DataStreamSource<String> data =
                env.readTextFile("data.txt 文件路径");
        data.addSink(new MyJdbcSink());

    }

    public static class MyJdbcSink extends RichSinkFunction<String> {
        private Connection connection=null;
        PreparedStatement insertStmt=null;//生成预编译语句
        PreparedStatement updateStmt=null;
        @Override
        public void open(Configuration parameters) throws Exception {
            connection = DriverManager.getConnection("jdbc:mysql://localhost:3306/test","root","root");
            insertStmt = connection.prepareStatement("insert into data (id) values(?)");
            updateStmt = connection.prepareStatement("update  data set id=? where  id = ?");
        }

        //每来一条数据，调用连接，执行sql
        @Override
        public void invoke(String value, Context context) throws Exception {
            //没有更新成功，就插入
            updateStmt.setString(1,value);
            updateStmt.execute();
            if(updateStmt.getUpdateCount()==0){
                insertStmt.setString(1,value);
                insertStmt.execute();
            }
        }

        @Override
        public void close() throws Exception {
            insertStmt.close();
            updateStmt.close();
            connection.close();
        }

        @Override
        public RuntimeContext getRuntimeContext() {
            return super.getRuntimeContext();
        }
    }
}

写入到ES

导入依赖

<dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-elasticsearch6_2.12</artifactId>
            <version>1.10.1</version>
</dependency>

//select 为经过转换得到的最终数据流
ArrayList<HttpHost> httpHosts = new ArrayList<>();
httpHosts.add(new HttpHost("localhost",9200));
select.addSink(new ElasticsearchSink.Builder<String>(httpHosts,new MyEsSinkFunction()).build());
        
env.execute();
  // 上面为 main 函数中的关键内容，下面为 main 函数外面的自定义函数，和 main 函数 定义在同一类中
 //自定义ES写入操作的MyEsSinkFunction
public static class MyEsSinkFunction implements ElasticsearchSinkFunction<String>{

        @Override
        public void process(String element, RuntimeContext ctx, RequestIndexer indexer) {
            //定义写入的数据
            //element就是来的每一条数据，这里假设数据都是String

            HashMap<String, String> dataSource = new HashMap<>();
            dataSource.put("id",element);

            //创建请求，作为向ES发起的写入命令

            IndexRequest indexRequest = Requests.indexRequest().index("sensor").type("readingData").source(dataSource);

            //用index 发送请求

            indexer.add(indexRequest);

        }
    }

向 Redis写入

导入依赖

 <dependency>
            <groupId>org.apache.bahir</groupId>
            <artifactId>flink-connector-redis_2.11</artifactId>
            <version>1.0</version>
</dependency>

// 该类写在 main 函数外面
 public static class MyRedis implements RedisMapper<String>{

        @Override
        public RedisCommandDescription getCommandDescription() {
            return new RedisCommandDescription(RedisCommand.HSET,"mytable");
        }

        @Override
        public String getKeyFromData(String s) {
            return s;
        }

        @Override
        public String getValueFromData(String s) {
            return s;
        }
    }
    
// 以下为main函数中的关键内容，select为经过转换得到的最终的数据流    
FlinkJedisPoolConfig jedis = new FlinkJedisPoolConfig.Builder().build();
        select.addSink(new RedisSink<>(jedis,new MyRedis())); //向Redis写入数据

向 kafka 写入

导入依赖

 <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-kafka-0.11_2.12</artifactId>
            <version>1.10.1</version>
</dependency>

// 以下为 main 函数中的关键内容，select 为最终得到的数据流
select.addSink(new FlinkKafkaProducer011<String>("localhost:9090","sinktest",new SimpleStringSchema()));
env.execute();