Flink读取kafka中的数据

pom文件需要添加的依赖

    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-scala -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-scala_2.11</artifactId>
      <version>${flink.version}</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-streaming-scala -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-streaming-scala_2.11</artifactId>
      <version>${flink.version}</version>
    </dependency>
      <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-clients -->
      <dependency>
        <groupId>org.apache.flink</groupId>
        <artifactId>flink-clients_2.11</artifactId>
        <version>${flink.version}</version>
      </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-shaded-hadoop-2-uber -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-shaded-hadoop-2-uber</artifactId>
      <version>2.4.1-9.0</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-connector-kafka-0.10 -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-connector-kafka_2.11</artifactId>
      <version>1.10.0</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.kafka/kafka -->
    <dependency>
      <groupId>org.apache.kafka</groupId>
      <artifactId>kafka_2.11</artifactId>
      <version>2.0.0</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.kafka/kafka-clients -->
    <dependency>
      <groupId>org.apache.kafka</groupId>
      <artifactId>kafka-clients</artifactId>
      <version>2.0.0</version>
    </dependency>

Flink的JAVA代码：

import java.util.Properties

import org.apache.flink.api.common.serialization.SimpleStringSchema
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.connectors.kafka.{FlinkKafkaConsumer}
import org.apache.kafka.clients.consumer.ConsumerConfig

object ReadKafka {
  def main(args: Array[String]): Unit = {
  	//创建执行环境
    val env = StreamExecutionEnvironment.getExecutionEnvironment
    //设置kafka的消费者的properties配置信息
    val prop = new Properties()
    prop.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,"192.168.181.132:9092")
    prop.put(ConsumerConfig.GROUP_ID_CONFIG,"uf1")
    prop.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringDeserializer")
    prop.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,"org.apache.kafka.common.serialization.StringDeserializer")

//创建flink的stream sources为kafka
    val ds = env.addSource(
      new FlinkKafkaConsumer[String](
      //指定topic
        "user_friends",
        new SimpleStringSchema(),
        prop
        //从头开始消费
      ).setStartFromEarliest()
    )
	//对数据集进行一系列的转换操作
    val dd = ds.filter(line => {
      var reg = ",$".r
      val iter = reg.findAllMatchIn(line)
      !iter.hasNext
    }).flatMap(line => {
      val info = line.split(",")
      info(1).split(" ").map((info(0), _))
    })
	//输出数据集
    dd.print()
    //触发程序执行
    env.execute("myfk")

  }
}