pom文件添加的依赖:
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.1.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<version>2.1.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-streaming -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.11</artifactId>
<version>2.1.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.kafka/kafka -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.11</artifactId>
<version>2.0.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.kafka/kafka-clients -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>2.0.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-streaming-kafka-0-10 -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming-kafka-0-10_2.11</artifactId>
<version>2.0.0</version>
</dependency>
</dependencies>
import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord}
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.spark.SparkConf
import org.apache.spark.streaming.dstream.InputDStream
import org.apache.spark.streaming.kafka010.{ConsumerStrategies, KafkaUtils, LocationStrategies}
import org.apache.spark.streaming.{Seconds, StreamingContext}
object ReadkafkaTopic {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setMaster("local[*]").
setAppName("readtest")
.set("spark.serializer","org.apache.spark.serializer.KryoSerializer")
val ssc = new StreamingContext(conf, Seconds(1))
ssc.checkpoint("d:/ck")
val kafkaParams = Map(
ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG -> "192.168.181.132:9092",
ConsumerConfig.GROUP_ID_CONFIG -> "zjy",
ConsumerConfig.MAX_POLL_RECORDS_CONFIG->"500",
ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG -> classOf[StringDeserializer],
ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG -> classOf[StringDeserializer],
ConsumerConfig.AUTO_OFFSET_RESET_CONFIG -> "earliest"
)
//读取user_friends的文件
// val ku: InputDStream[ConsumerRecord[String, String]] = KafkaUtils.createDirectStream(ssc,
// LocationStrategies.PreferConsistent,
// ConsumerStrategies.Subscribe[String, String](Set("user_friends"), kafkaParams)
// )
//过滤掉friend列为空的行数
// ku.filter(ln=>{
// val reg = ",$".r
// val it = reg.findAllMatchIn(ln.value())
// !it.hasNext
// }).flatMap(line=>{
// val info = line.value().split(",")
// info(1).split(" ").map(fid=>{
// (info(0),fid)
// })
// }).foreachRDD(rdd=>rdd.foreach(println))
//读取文件event_attendees
val kuea: InputDStream[ConsumerRecord[String, String]] = KafkaUtils.createDirectStream(ssc,
LocationStrategies.PreferConsistent,
ConsumerStrategies.Subscribe[String, String](Set("event_attendees"), kafkaParams))
//并对event_attendees文件进行处理后再控制台输出
val yes = kuea.map(line => {
val info = line.value().split(",", -1)
(info(0), info(1))
}).filter(_._2 != "").flatMap(x => {
x._2.split(" ").map((x._1, _, "yes"))
})
val maybe = kuea.map(line => {
val info = line.value().split(",", -1)
(info(0), info(2))
}).filter(_._2 != "").flatMap(x => {
x._2.split(" ").map((x._1, _, "maybe"))
})
val invited = kuea.map(line => {
val info = line.value().split(",", -1)
(info(0), info(3))
}).filter(_._2 != "").flatMap(x => {
x._2.split(" ").map((x._1, _, "invited"))
})
val no = kuea.map(line => {
val info = line.value().split(",", -1)
(info(0), info(4))
}).filter(_._2 != "").flatMap(x => {
x._2.split(" ").map((x._1, _, "no"))
})
yes.union(maybe).union(invited).union(no).foreachRDD(x=>x.foreach(println))
ssc.start()
ssc.awaitTermination()
}
}
split中加-1的作用:针对一行数据中后续全是以分隔符结尾的,加-1,是会取出每个分割符之间的空字符的