Flink没有类似Spark中foreach方法,让用户进行迭代的操作。虽有对外输出操作都要利用SInk完成。最后通过类似如下方式完成整个任务最终输出操作。
myDstream.addSink(new MySink(xxxx))
官方提供了一部分的框架的Sink,初次之外需要用户自定义实现Sink。
本项目所有代码上传到https://github.com/xtxxtxxtx/FlinkPractise,近期会对项目一直进行更新,希望能和大家一起交流。
Kafka
在pom中添加依赖:
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-kafka-0.11_2.11</artifactId>
<version>1.7.2</version>
</dependency>
创建MyKafkaUtil伴生对象:
package FlinkPractice.util
import java.util.Properties
import org.apache.flink.api.common.serialization.SimpleStringSchema
import org.apache.flink.streaming.connectors.kafka.{FlinkKafkaConsumer011, FlinkKafkaProducer011}
/**
* Kafka既可以做Source也可以做Sink
*/
object MyKafkaUtil {
val properties: Properties = new Properties()
properties.setProperty("bootstarp.servers", "hadoop1:9092")
properties.setProperty("group.id", "gmall")
def getKafkaSource(topic : String): FlinkKafkaConsumer011[String] = {
val kafkaConsumer: FlinkKafkaConsumer011[String] = new FlinkKafkaConsumer011[String](topic, new SimpleStringSchema(), properties)
kafkaConsumer
}
def getKafkaSink(topic : String): FlinkKafkaProducer011[String] ={
val kafkaSink: FlinkKafkaProducer011[String] = new FlinkKafkaProducer011[String]("hadoop1:9092", topic, new SimpleStringSchema())
kafkaSink
}
}
主函数中添加:
//使用Kafka Sink
val unionStream: DataStream[Startuplog] = appleStream.union(otherStream)
unionStream.print("UNION")
val kafkaSink: FlinkKafkaProducer011[String] = MyKafkaUtil.getKafkaSink("topic_apple")
appleStream.map(startuplog => startuplog.ch).addSink(kafkaSink)
Redis
pom:
<dependency>
<groupId>org.apache.bahir</groupId>
<artifactId>flink-connector-redis_2.11</artifactId>
<version>1.0</version>
</dependency>
RedisUtil:
package FlinkPractice.util
import org.apache.flink.streaming.connectors.redis.RedisSink
import org.apache.flink.streaming.connectors.redis.common.config.FlinkJedisPoolConfig
import org.apache.flink.streaming.connectors.redis.common.mapper.{RedisCommand, RedisCommandDescription, RedisMapper}
/**
* RedisSink
*/
object RedisUtil {
val conf: FlinkJedisPoolConfig = new FlinkJedisPoolConfig.Builder().setHost("hadoop1").setPort(6372).build()
def getRedisSink(): RedisSink[(String, Int)] ={
new RedisSink[(String, Int)](conf, new MyRedisMapper())
}
class MyRedisMapper extends RedisMapper[(String, Int)] {
override def getCommandDescription: RedisCommandDescription = {
new RedisCommandDescription(RedisCommand.HSET, "channel_sum")
}
override def getKeyFromData(data: (String, Int)): String = data._1
override def getValueFromData(data: (String, Int)): String = data._2.toString
}
}
主函数:
//使用Redis Sink
val chKeyedStream: KeyedStream[(String, Int), Tuple] = startuplogDstream.map(startuplog => (startuplog.ch, 1)).keyBy(0)
val chSumDstream: DataStream[(String, Int)] = chKeyedStream.reduce((ch1, ch2) => (ch1._1, ch1._2 + ch2._2)).setParallelism(1)
//结果存入redis中 hset:channel_sum field:channel value:count
val redisSink: RedisSink[(String, Int)] = RedisUtil.getRedisSink()
chSumDstream.addSink(redisSink)
ElasticSearch
pom:
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-elasticsearch6_2.11</artifactId>
<version>1.7.2</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.2</version>
</dependency>
ElasticsearchUtil:
/**
* Elasticsearch Sink
*/
object ElasticsearchUtil {
//创建HttpHost
private val httpHosts: util.ArrayList[HttpHost] = new util.ArrayList[HttpHost]()
httpHosts.add(new HttpHost("hadoop1", 10000, "http"))
httpHosts.add(new HttpHost("hadoop2", 10000, "http"))
httpHosts.add(new HttpHost("hadoop3", 10000, "http"))
//获取EsSink
def getElasticsearchSink(indexName : String): ElasticsearchSink[String] ={
val esFunc = new ElasticsearchSinkFunction[String] {
override def process(element: String, ctx: RuntimeContext, indexer: RequestIndexer): Unit = {
val jsonObj: JSONObject = JSON.parseObject(element)
val indexRequest: IndexRequest = Requests.indexRequest().index(indexName).`type`("_doc").source(jsonObj)
indexer.add(indexRequest)
}
}
val EsSinkBuilder: ElasticsearchSink.Builder[String] = new ElasticsearchSink.Builder[String](httpHosts, esFunc)
EsSinkBuilder.setBulkFlushMaxActions(10)
EsSinkBuilder.build()
}
}
主函数:
//使用Elasticsearch Sink
val esSink: ElasticsearchSink[String] = ElasticsearchUtil.getElasticsearchSink("")
dstream.addSink(esSink)