Flink——Sink

Flink没有类似Spark中foreach方法，让用户进行迭代的操作。虽有对外输出操作都要利用SInk完成。最后通过类似如下方式完成整个任务最终输出操作。

 myDstream.addSink(new MySink(xxxx))

官方提供了一部分的框架的Sink，初次之外需要用户自定义实现Sink。

本项目所有代码上传到https://github.com/xtxxtxxtx/FlinkPractise，近期会对项目一直进行更新，希望能和大家一起交流。

Kafka

在pom中添加依赖：

		<dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-kafka-0.11_2.11</artifactId>
            <version>1.7.2</version>
       	</dependency>

创建MyKafkaUtil伴生对象：

package FlinkPractice.util

import java.util.Properties

import org.apache.flink.api.common.serialization.SimpleStringSchema
import org.apache.flink.streaming.connectors.kafka.{FlinkKafkaConsumer011, FlinkKafkaProducer011}

/**
  * Kafka既可以做Source也可以做Sink
  */
object MyKafkaUtil {

  val properties: Properties = new Properties()
  properties.setProperty("bootstarp.servers", "hadoop1:9092")
  properties.setProperty("group.id", "gmall")

  def getKafkaSource(topic : String): FlinkKafkaConsumer011[String] = {
    val kafkaConsumer: FlinkKafkaConsumer011[String] = new FlinkKafkaConsumer011[String](topic, new SimpleStringSchema(), properties)
    kafkaConsumer
  }

  def getKafkaSink(topic : String): FlinkKafkaProducer011[String] ={
    val kafkaSink: FlinkKafkaProducer011[String] = new FlinkKafkaProducer011[String]("hadoop1:9092", topic, new SimpleStringSchema())
    kafkaSink
  }
}

主函数中添加：

	//使用Kafka Sink
    val unionStream: DataStream[Startuplog] = appleStream.union(otherStream)
    unionStream.print("UNION")
    val kafkaSink: FlinkKafkaProducer011[String] = MyKafkaUtil.getKafkaSink("topic_apple")
    appleStream.map(startuplog => startuplog.ch).addSink(kafkaSink)

Redis

pom:

		<dependency>
            <groupId>org.apache.bahir</groupId>
            <artifactId>flink-connector-redis_2.11</artifactId>
            <version>1.0</version>
        </dependency>

RedisUtil：

package FlinkPractice.util

import org.apache.flink.streaming.connectors.redis.RedisSink
import org.apache.flink.streaming.connectors.redis.common.config.FlinkJedisPoolConfig
import org.apache.flink.streaming.connectors.redis.common.mapper.{RedisCommand, RedisCommandDescription, RedisMapper}

/**
  * RedisSink
  */
object RedisUtil {

  val conf: FlinkJedisPoolConfig = new FlinkJedisPoolConfig.Builder().setHost("hadoop1").setPort(6372).build()

  def getRedisSink(): RedisSink[(String, Int)] ={
    new RedisSink[(String, Int)](conf, new MyRedisMapper())
  }

  class MyRedisMapper extends RedisMapper[(String, Int)] {
    override def getCommandDescription: RedisCommandDescription = {
      new RedisCommandDescription(RedisCommand.HSET, "channel_sum")
    }

    override def getKeyFromData(data: (String, Int)): String = data._1

    override def getValueFromData(data: (String, Int)): String = data._2.toString
  }
}

主函数：

	//使用Redis Sink
    val chKeyedStream: KeyedStream[(String, Int), Tuple] = startuplogDstream.map(startuplog => (startuplog.ch, 1)).keyBy(0)
    val chSumDstream: DataStream[(String, Int)] = chKeyedStream.reduce((ch1, ch2) => (ch1._1, ch1._2 + ch2._2)).setParallelism(1)
    //结果存入redis中 hset：channel_sum field:channel value:count
    val redisSink: RedisSink[(String, Int)] = RedisUtil.getRedisSink()
    chSumDstream.addSink(redisSink)

ElasticSearch

pom：

		<dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-elasticsearch6_2.11</artifactId>
            <version>1.7.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
            <version>4.5.2</version>
        </dependency>

ElasticsearchUtil：

/**
  * Elasticsearch Sink
  */
object ElasticsearchUtil {

  //创建HttpHost
  private val httpHosts: util.ArrayList[HttpHost] = new util.ArrayList[HttpHost]()
  httpHosts.add(new HttpHost("hadoop1", 10000, "http"))
  httpHosts.add(new HttpHost("hadoop2", 10000, "http"))
  httpHosts.add(new HttpHost("hadoop3", 10000, "http"))


  //获取EsSink
  def getElasticsearchSink(indexName : String): ElasticsearchSink[String] ={
    val esFunc = new ElasticsearchSinkFunction[String] {
      override def process(element: String, ctx: RuntimeContext, indexer: RequestIndexer): Unit = {
        val jsonObj: JSONObject = JSON.parseObject(element)
        val indexRequest: IndexRequest = Requests.indexRequest().index(indexName).`type`("_doc").source(jsonObj)
        indexer.add(indexRequest)
      }
    }
    val EsSinkBuilder: ElasticsearchSink.Builder[String] = new ElasticsearchSink.Builder[String](httpHosts, esFunc)
    EsSinkBuilder.setBulkFlushMaxActions(10)
    EsSinkBuilder.build()
  }
}

主函数：

	//使用Elasticsearch Sink
    val esSink: ElasticsearchSink[String] = ElasticsearchUtil.getElasticsearchSink("")
    dstream.addSink(esSink)