Flink-数据源(Source)&存储(Sink)

Source------------------------------------------------------------

从Kafka读取数据

添加相应的依赖

<dependency>
	    <groupId>org.apache.flink</groupId>
	    <artifactId>flink-connector-kafka_2.11</artifactId>
	    <version>1.12.0</version>
</dependency>

	import org.apache.flink.api.common.serialization.SimpleStringSchema;
	import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
	import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
	import java.util.Properties;
	
	public class Flink03_Source_Kafka {
	    public static void main(String[] args) throws Exception {
	
	        // 0.Kafka相关配置
	        Properties properties = new Properties();
	        properties.setProperty("bootstrap.servers", "hadoop102:9092,hadoop103:9092,hadoop104:9092");
	        properties.setProperty("group.id", "Flink01_Source_Kafka");
	        properties.setProperty("auto.offset.reset", "latest");
	
	        // 1. 创建执行环境
	        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	        env
	          .addSource(new FlinkKafkaConsumer<>("sensor", new SimpleStringSchema(), properties))
	          .print("kafka source");
	
	        env.execute();
	    }
}

开启kafka生产者,测试消费

kafka-console-producer.sh --broker-list hadoop102:9092 --topic sensor

自定义Source

import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.SourceFunction;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.Socket;
import java.nio.charset.StandardCharsets;

public class Flink04_Source_Custom {
    public static void main(String[] args) throws Exception {

        // 1. 创建执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env
          .addSource(new MySource("hadoop102", 9999))
          .print();

        env.execute();
    }

    public static class MySource implements SourceFunction<WaterSensor> {
        private String host;
        private int port;
        private volatile boolean isRunning = true;
        private Socket socket;

        public MySource(String host, int port) {
            this.host = host;
            this.port = port;
        }


        @Override
        public void run(SourceContext<WaterSensor> ctx) throws Exception {
            // 实现一个从socket读取数据的source
            socket = new Socket(host, port);
            BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8));
            String line = null;
            while (isRunning && (line = reader.readLine()) != null) {
                String[] split = line.split(",");
                ctx.collect(new WaterSensor(split[0], Long.valueOf(split[1]), Integer.valueOf(split[2])));
            }
        }

        /**
         * 大多数的source在run方法内部都会有一个while循环,
         * 当调用这个方法的时候, 应该可以让run方法中的while循环结束
         */

        @Override
        public void cancel() {
            isRunning = false;
            try {
                socket.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}

Sink---------------------------------------------------------------------------

KafkaSink

添加Kafka Connector依赖

	<dependency>
	    <groupId>org.apache.flink</groupId>
	    <artifactId>flink-connector-kafka_2.11</artifactId>
	    <version>1.12.0</version>
	</dependency>
	<dependency>
	    <groupId>com.alibaba</groupId>
	    <artifactId>fastjson</artifactId>
	    <version>1.2.75</version>
</dependency>

	import com.alibaba.fastjson.JSON;
	import com.atguigu.flink.java.chapter_5.WaterSensor;
	import org.apache.flink.api.common.serialization.SimpleStringSchema;
	import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
	import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;
	
	import java.util.ArrayList;
	
	public class Flink01_Sink_Kafka {
	    public static void main(String[] args) throws Exception {
	        ArrayList<WaterSensor> waterSensors = new ArrayList<>();
	        waterSensors.add(new WaterSensor("sensor_1", 1607527992000L, 20));
	        waterSensors.add(new WaterSensor("sensor_1", 1607527994000L, 50));
	        waterSensors.add(new WaterSensor("sensor_1", 1607527996000L, 50));
	        waterSensors.add(new WaterSensor("sensor_2", 1607527993000L, 10));
	        waterSensors.add(new WaterSensor("sensor_2", 1607527995000L, 30));
	
	        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment().setParallelism(1);
	        env
	          .fromCollection(waterSensors)
	          .map(JSON::toJSONString)
	          .addSink(new FlinkKafkaProducer<String>("hadoop102:9092", "topic_sensor", new SimpleStringSchema()));
	
	
	        env.execute();
	    }
}

在linux启动一个消费者, 查看是否收到数据

bin/kafka-console-consumer.sh --bootstrap-server hadoop102:9092 --topic topic_sensor

RedisSink

添加Redis Connector依赖

	<!-- https://mvnrepository.com/artifact/org.apache.flink/flink-connector-redis -->
	<dependency>
	    <groupId>org.apache.flink</groupId>
	    <artifactId>flink-connector-redis_2.11</artifactId>
	<version>1.1.5</version>
	</dependency>

	import com.alibaba.fastjson.JSON;
	import com.atguigu.flink.java.chapter_5.WaterSensor;
	import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
	import org.apache.flink.streaming.connectors.redis.RedisSink;
	import org.apache.flink.streaming.connectors.redis.common.config.FlinkJedisPoolConfig;
	import org.apache.flink.streaming.connectors.redis.common.mapper.RedisCommand;
	import org.apache.flink.streaming.connectors.redis.common.mapper.RedisCommandDescription;
	import org.apache.flink.streaming.connectors.redis.common.mapper.RedisMapper;
	
	import java.util.ArrayList;
	
	public class Flink02_Sink_Redis {
	    public static void main(String[] args) throws Exception {
	        ArrayList<WaterSensor> waterSensors = new ArrayList<>();
	        waterSensors.add(new WaterSensor("sensor_1", 1607527992000L, 20));
	        waterSensors.add(new WaterSensor("sensor_1", 1607527994000L, 50));
	        waterSensors.add(new WaterSensor("sensor_1", 1607527996000L, 50));
	        waterSensors.add(new WaterSensor("sensor_2", 1607527993000L, 10));
	        waterSensors.add(new WaterSensor("sensor_2", 1607527995000L, 30));
	
	        // 连接到Redis的配置
	        FlinkJedisPoolConfig redisConfig = new FlinkJedisPoolConfig.Builder()
	          .setHost("hadoop102")
	          .setPort(6379)
	          .setMaxTotal(100)
	          .setTimeout(1000 * 10)
	          .build();
	        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment().setParallelism(1);
	        env
	          .fromCollection(waterSensors)
	          .addSink(new RedisSink<>(redisConfig, new RedisMapper<WaterSensor>() {
	              /*
	                key                 value(hash)
	                "sensor"            field           value
	                                    sensor_1        {"id":"sensor_1","ts":1607527992000,"vc":20}
	                                    ...             ...
	               */
	
	              @Override
	              public RedisCommandDescription getCommandDescription() {
	                  // 返回存在Redis中的数据类型  存储的是Hash, 第二个参数是外面的key
	                  return new RedisCommandDescription(RedisCommand.HSET, "sensor");
	              }
	
	              @Override
	              public String getKeyFromData(WaterSensor data) {
	                  // 从数据中获取Key: Hash的Key
	                  return data.getId();
	              }
	
	              @Override
	              public String getValueFromData(WaterSensor data) {
	                  // 从数据中获取Value: Hash的value
	                  return JSON.toJSONString(data);
	              }
	          }));
	
	        env.execute();
	    }
}

ElasticsearchSink

	<!-- https://mvnrepository.com/artifact/org.apache.flink/flink-connector-elasticsearch6 -->
	<dependency>
	    <groupId>org.apache.flink</groupId>
	    <artifactId>flink-connector-elasticsearch6_2.11</artifactId>
	    <version>1.12.0</version>
</dependency>

	import com.alibaba.fastjson.JSON;
	import com.atguigu.flink.java.chapter_5.WaterSensor;
	import org.apache.flink.api.common.functions.RuntimeContext;
	import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
	import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkFunction;
	import org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer;
	import org.apache.flink.streaming.connectors.elasticsearch6.ElasticsearchSink;
	import org.apache.http.HttpHost;
	import org.elasticsearch.action.index.IndexRequest;
	import org.elasticsearch.client.Requests;
	import org.elasticsearch.common.xcontent.XContentType;
	
	import java.util.ArrayList;
	import java.util.Arrays;
	import java.util.List;
	
	public class Flink03_Sink_ES {
	    public static void main(String[] args) throws Exception {
	        ArrayList<WaterSensor> waterSensors = new ArrayList<>();
	        waterSensors.add(new WaterSensor("sensor_1", 1607527992000L, 20));
	        waterSensors.add(new WaterSensor("sensor_1", 1607527994000L, 50));
	        waterSensors.add(new WaterSensor("sensor_1", 1607527996000L, 50));
	        waterSensors.add(new WaterSensor("sensor_2", 1607527993000L, 10));
	        waterSensors.add(new WaterSensor("sensor_2", 1607527995000L, 30));
	
	        List<HttpHost> esHosts = Arrays.asList(
	          new HttpHost("hadoop102", 9200),
	          new HttpHost("hadoop103", 9200),
	          new HttpHost("hadoop104", 9200));
	        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment().setParallelism(1);
	        env
	          .fromCollection(waterSensors)
	          .addSink(new ElasticsearchSink.Builder<WaterSensor>(esHosts, new ElasticsearchSinkFunction<WaterSensor>() {
	
	              @Override
	              public void process(WaterSensor element, RuntimeContext ctx, RequestIndexer indexer) {
	                  // 1. 创建es写入请求
	                  IndexRequest request = Requests
	                    .indexRequest("sensor")
	                    .type("_doc")
	                    .id(element.getId())
	                    .source(JSON.toJSONString(element), XContentType.JSON);
	                  // 2. 写入到es
	                  indexer.add(request);
	              }
	          }).build());
	
	        env.execute();
	    }
}

自定义Sink -> mysql

	<dependency>
	    <groupId>mysql</groupId>
	    <artifactId>mysql-connector-java</artifactId>
	    <version>5.1.49</version>
</dependency>

	import com.atguigu.flink.java.chapter_5.WaterSensor;
	import org.apache.flink.configuration.Configuration;
	import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
	import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
	
	import java.sql.Connection;
	import java.sql.DriverManager;
	import java.sql.PreparedStatement;
	import java.util.ArrayList;
	
	public class Flink04_Sink_Custom {
	    public static void main(String[] args) throws Exception {
	        ArrayList<WaterSensor> waterSensors = new ArrayList<>();
	        waterSensors.add(new WaterSensor("sensor_1", 1607527992000L, 20));
	        waterSensors.add(new WaterSensor("sensor_1", 1607527994000L, 50));
	        waterSensors.add(new WaterSensor("sensor_1", 1607527996000L, 50));
	        waterSensors.add(new WaterSensor("sensor_2", 1607527993000L, 10));
	        waterSensors.add(new WaterSensor("sensor_2", 1607527995000L, 30));
	
	        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment().setParallelism(1);
	        env.fromCollection(waterSensors)
	          .addSink(new RichSinkFunction<WaterSensor>() {
	
	             private PreparedStatement ps;
	              private Connection conn;
	
	              @Override
	              public void open(Configuration parameters) throws Exception {
	                  conn = DriverManager.getConnection("jdbc:mysql://hadoop102:3306/test?useSSL=false", "root", "000000");
	                  ps = conn.prepareStatement("insert into sensor values(?, ?, ?)");
	              }
	
	              @Override
	              public void close() throws Exception {
	                ps.close();
	                  conn.close();
	              }
	
	              @Override
	              public void invoke(WaterSensor value, Context context) throws Exception {
	                ps.setString(1, value.getId());
	                  ps.setLong(2, value.getTs());
	                  ps.setInt(3, value.getVc());
	                  ps.execute();
	              }
	          });
	
	
	        env.execute();
	    }
}