kafka的安装参考我的这边博客(https://blog.csdn.net/qq_16563637/article/details/82776276)
1.安装zk集群
2.config/server.properties
添加zk地址:zookeeper.connect=node-1:2181,node-2:2181,node-3:2181
修改broker.id(唯一的):broker.id=0
3.启动
/bin/kafka-server-start.sh config/server.properties > /dev/null 2>&1 &
4.创建topic
bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 3 --partitions 3 --topic test
5.列出所有topic
bin/kafka-topics.sh --list --zookeeper localhost:2181
6.向topic中写入数据
bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test
7.消费数据
bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic test --from-beginning
8.查看指定topic的详情
bin/kafka-topics.sh --describe --zookeeper localhost:2181 --topic test
代码如下:
package cn.itcast.spark.day5
import org.apache.spark.storage.StorageLevel
import org.apache.spark.{HashPartitioner, SparkConf}
import org.apache.spark.streaming.kafka.KafkaUtils
import org.apache.spark.streaming.{Seconds, StreamingContext}
/**
* Created by root on 2016/5/21.
*/
//(多个zookeeper用,隔开)
//zookeper groupid topics numThreads
//参数:192.168.1.103:2181 g1 test 2
//向kafka中写数据
//bin/kafka-console-producer.sh --broker-list 192.168.1.103:9092 --topic test
object KafkaWordCount {
val updateFunc = (iter: Iterator[(String, Seq[Int], Option[Int])]) => {
//iter.flatMap(it=>Some(it._2.sum + it._3.getOrElse(0)).map(x=>(it._1,x)))
iter.flatMap { case (x, y, z) => Some(y.sum + z.getOrElse(0)).map(i => (x, i)) }
}
def main(args: Array[String]) {
LoggerLevels.setStreamingLogLevels()
val Array(zkQuorum, group, topics, numThreads) = args
val sparkConf = new SparkConf().setAppName("KafkaWordCount").setMaster("local[2]")
val ssc = new StreamingContext(sparkConf, Seconds(5))
ssc.checkpoint("c://ck2")
//"alog-2016-04-16,alog-2016-04-17,alog-2016-04-18"
//"Array((alog-2016-04-16, 2), (alog-2016-04-17, 2), (alog-2016-04-18, 2))"
val topicMap = topics.split(",").map((_, numThreads.toInt)).toMap
val data = KafkaUtils.createStream(ssc, zkQuorum, group, topicMap, StorageLevel.MEMORY_AND_DISK_SER)
val words = data.map(_._2).flatMap(_.split(" "))
val wordCounts = words.map((_, 1)).updateStateByKey(updateFunc, new HashPartitioner(ssc.sparkContext.defaultParallelism), true)
wordCounts.print()
ssc.start()
ssc.awaitTermination()
}
}