package com.xmgps.yfzx.sparkhbase
import org.apache.hadoop.hbase.{HBaseConfiguration, TableName}
import org.apache.hadoop.hbase.client.{ConnectionFactory, HTable, Put, Result}
import org.apache.hadoop.hbase.io.ImmutableBytesWritable
import org.apache.hadoop.hbase.mapreduce.TableInputFormat
import org.apache.hadoop.hbase.util.Bytes
import org.apache.spark.{SparkConf, SparkContext}
object SparkHbase extends Serializable{
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName("spark hbase").setMaster("local[1]")
.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
val hbaseconfig = HBaseConfiguration.create()
hbaseconfig.addResource("hbase-site.xml")
hbaseconfig.set(TableInputFormat.INPUT_TABLE,"LEASE")
val sc = SparkContext.getOrCreate(conf)
val connection = ConnectionFactory.createConnection(hbaseconfig)
val table = connection.getTable(TableName.valueOf("freight"))
for (i <- 1 to 10) {
val put = new Put(Bytes.toBytes("row0000" + i))
put.add(Bytes.toBytes("A"),Bytes.toBytes("name"),Bytes.toBytes("value" + i))
table.put(put)
}
val hrdd = sc.newAPIHadoopRDD(hbaseconfig,
classOf[TableInputFormat],
classOf[ImmutableBytesWritable],
classOf[Result]
)
val count = hrdd.count().toInt
println(count)
val res = hrdd.take(count)
for (j <- 1 until count) {
println("j: " + j)
val rs = res(j - 1)._2
val kvs = rs.raw
for (kv <- kvs)
println("rowkey:" + new String(kv.getRow()) +
" cf:" + new String(kv.getFamily()) +
" column:" + new String(kv.getQualifier()) +
" value:" + new String(kv.getValue()))
}
}
}
spark hbase
最新推荐文章于 2021-02-21 12:00:02 发布