Spark streaming写入数据到hbase
import org.apache.kafka.common.serialization.{StringDeserializer,IntegerDeserializer}
import java.util.concurrent.atomic.AtomicInteger
import org.apache.spark.streaming.kafka010._
import org.apache.spark.streaming.kafka010.LocationStrategies.PreferConsistent
import org.apache.spark.streaming.kafka010.ConsumerStrategies.Subscribe
import org.apache.spark.streaming.StreamingContext._
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.SparkConf
import org.apache.spark.streaming.{Seconds, Minutes}
import org.apache.hadoop.hbase.mapreduce.TableOutputFormat
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hbase._
import org.apache.hadoop.hbase.client._
import org.apache.hadoop.mapreduce.Job
import org.apache.hadoop.hbase.io.ImmutableBytesWritable
import org.apache.hadoop.hbase.client.{Result,Put}
import org.apache.log4j.Logger
import org.apache.log4j.Level
object SparkStreamingToHbase{
def main(args:Array[String]){
Logger.getLogger("org.apache.spark").setLevel(Level.ERROR)
Logger.getLogger("org.eclipse.jetty.server").setLevel(Level.OFF)
val kafkaParams = Map[String, Object](
"bootstrap.servers" -> "localhost:9092",
"key.deserializer" -> classOf[StringDeserializer],
"value.deserializer" -> classOf[StringDeserializer],
"group.id" -> "use_a_separate_group_id_for_each_stream",
"auto.offset.reset" -> "latest",
"enable.auto.commit" -> (false: java.lang.Boolean))
val sc = new SparkConf().setAppName("ZkWordCount")
val ssc = new StreamingContext(sc, Seconds(10))
ssc.checkpoint("hdfs://nameservice1/test/checkpoint")
val topics = Array("zklog")
val stream = KafkaUtils.createDirectStream[String, String](
ssc,
PreferConsistent,
Subscribe[String, String](topics, kafkaParams)
)
val lines = stream.map(record => (record.value))
val words = lines.flatMap(_.split("org.apache.zookeeper.server.")).flatMap(_.split(","))
val pair = words.map(x => (x,1))
val wordCounts = pair.reduceByKeyAndWindow(_ + _,_ - _,Minutes(1),Seconds(10),3)
wordCounts.print()
wordCounts.foreachRDD(rdd => {
val hbaseConf =HBaseConfiguration.create()
hbaseConf.set(TableOutputFormat.OUTPUT_TABLE, "zklog")
hbaseConf.set("mapreduce.output.fileoutputformat.outputdir", "/tmp")
val job = Job.getInstance(hbaseConf)
job.setOutputKeyClass(classOf[ImmutableBytesWritable])
job.setOutputValueClass(classOf[Put])
job.setOutputFormatClass(classOf[TableOutputFormat[ImmutableBytesWritable]])
val save_rdd = rdd.map(x => {
val put=new Put(Bytes.toBytes(x._1.trim))
put.addImmutable(Bytes.toBytes("zk"),Bytes.toBytes("count"),Bytes.toBytes(x._2.toString))
(new ImmutableBytesWritable, put)
}).saveAsNewAPIHadoopDataset(job.getConfiguration())
})
ssc.start
ssc.awaitTermination
}
}