用spark Streamming 将kafka的数据 展示到控制台

第一步

依赖

<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-streaming-kafka-0-10_2.11</artifactId>
    <version>2.4.3</version>
</dependency>

drver编写

import org.apache.kafka.clients.consumer.ConsumerConfig
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.spark.SparkConf
import org.apache.spark.streaming.kafka010.{ConsumerStrategies, KafkaUtils, LocationStrategies}
import org.apache.spark.streaming.{Seconds, StreamingContext}

object DirectKafkaWordCount {
  def main(args: Array[String]): Unit = {
    val sparkConf = new SparkConf().setAppName("DirectKafkaWordCount").setMaster("local[6]")
    val ssc = new StreamingContext(sparkConf, Seconds(2))
    ssc.sparkContext.setLogLevel("FATAL")
    val kafkaParams = Map[String, Object](
      ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG -> "CentOS:9092",
      ConsumerConfig.GROUP_ID_CONFIG -> "g1",
      ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG -> classOf[StringDeserializer],
      ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG -> classOf[StringDeserializer])

    //直接读取Kafka中的数据 将kafka独立维护Kafka topic的offset偏移量 checkpoint
    val messages = KafkaUtils.createDirectStream[String, String](ssc,
      LocationStrategies.PreferConsistent,//设置读取策略，如果你的spark计算节点和kafka broker节点不在一台物理主机
      ConsumerStrategies.Subscribe[String, String](List("topic01"), kafkaParams))

    messages.map(record=>record.value)
    .flatMap(line=>line.split(" "))
    .map(word => (word, 1L))
    .reduceByKey(_ + _)
    .print()
    
    ssc.start()
    ssc.awaitTermination()
  }
}

启动 kafka 测试

 ./bin/kafka-server-start.sh  -daemon config/server.properties3628
  > Kafka

 ./bin/kafka-topics.sh --zookeeper spark:2181 --create --topic topic01 --partitions 1 --replication-factor 1              
 Created topic "topic01".

./bin/kafka-console-producer.sh --broker-list  spark:9092 --topic topic01
>d
>hellp oo oo
>

控制台输出

Time: 1565955684000 ms
-------------------------------------------
(oo,2)
(hellp,1)
---------------------