使用KafkaUtil以direct方式获取kafka的数据时遇到的问题
报的是nio.channels.closeException类似的错误
原因是我的server.properties配置文件里面
zookeeper的端口配错了
正确的代码是
package kafkaTest
import kafka.serializer.StringDecoder
import org.apache.spark.streaming.dstream.{DStream, InputDStream}
import org.apache.spark.streaming.kafka.KafkaUtils
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.{SparkConf, SparkContext}
/**
* Created by Administrator on 2018/5/13.
*/
object kafkaTest {
def main(args: Array[String]): Unit = {
val conf: SparkConf = new SparkConf().setAppName("kafkaTest").setMaster("local[2]")
val sc = new SparkContext(conf)
val scc = new StreamingContext(sc,Seconds(2))
//checkpoint目录
scc.checkpoint("hdfs://hadoop02:9000/sparkStreaming")
val TOPIC=Set("sparkstream")
//kafkaUtils创建direct dstream
var kafkaParams=Map("metadata.broker.list"->"hadoop02:9092")
val dstream: DStream[String] = KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder](scc, kafkaParams, TOPIC)
.map(_._2)
dstream.flatMap(line=>line.split(",")).map(x=>(x,1)).reduceByKey(_+_).print()
/**
* 偏移量 数据 解码器 解码器
* K: ClassTag,
V: ClassTag,
KD <: Decoder[K]: ClassTag,
VD <: Decoder[V]: ClassTag
* ssc: StreamingContext,
kafkaParams: Map[String, String],
topics: Set[String]
* */
scc.start()
scc.awaitTermination()
scc.stop()
}
}