Source读取kafka, 解析json数据
<properties>
<flink.version>1.11.2</flink.version>
<kafka.version>0.10.2.1</kafka.version>
<scope>compile</scope>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients_2.11</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-scala_2.11</artifactId>
<version>${flink.version}</version>
<scope>${scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-kafka_2.11</artifactId>
<version>${flink.version}</version>
<scope>${scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>${kafka.version}</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.47</version>
</dependency>
</dependencies>
package com.day
import java.util.Properties
import com.alibaba.fastjson.JSON
import org.apache.flink.api.common.serialization.SimpleStringSchema
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer
object SourceKafka {
def main(args: Array[String]): Unit = {
val env = StreamExecutionEnvironment.getExecutionEnvironment
val properties = new Properties()
properties.setProperty("bootstrap.servers", "jeff200:9092")
val kfkConsumer = new FlinkKafkaConsumer[String](
"test",
new SimpleStringSchema(),
properties
)
kfkConsumer.setStartFromEarliest()
val dataStream = env.addSource(kfkConsumer)
.map(x=>{
val obj = JSON.parseObject(x)
val ts = obj.getInteger("ts")
val uid = JSON.parseObject(obj.getString("common")).getInteger("uid")
(uid, ts)
})
dataStream.print()
env.execute("kafka Job")
}
}
- 程序运行结果