package cn.jkjf.bigdata.utils.test
import cn.jkjf.bigdata.utils.mysql.Global
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.hadoop.hbase.client.{ConnectionFactory, HTable}
import org.apache.hadoop.hbase.io.ImmutableBytesWritable
import org.apache.hadoop.hbase.mapred.TableOutputFormat
import org.apache.hadoop.hbase.mapreduce.{HFileOutputFormat2, LoadIncrementalHFiles}
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.hbase.{HBaseConfiguration, KeyValue, TableName}
import org.apache.hadoop.mapreduce.Job
import org.apache.spark._
/**
* Created by maokm on 2018/9/20.
*/
object hbasetest {
def main(args: Array[String]) = {
val sc = new SparkContext("local",s"${this.getClass.getSimpleName}")
val hdfsRootPath = "hdfs://192.168.152.133:9000/data"
val hadoopConf = new Configuration()
hadoopConf.set("fs.defaultFS", hdfsRootPath)
val conf = HBaseConfiguration.create(hadoopConf)
conf.set("hbase.zookeeper.quorum", Global.ZKQUORUM)
conf.set("hbase.zookeeper.property.clientPort", "2181")
val hbaseConn = ConnectionFactory.createConnection(conf)
val tableName = "testgrace"
val table = new HTable(conf, tableName)
conf.set(TableOutputFormat.OUTPUT_TABLE, tableName)
val job = Job.getInstance(conf)
job.setMapOutputKeyClass (classOf[ImmutableBytesWritable])
job.setMapOutputValueClass (classOf[KeyValue])
val regionLocator = hbaseConn.getRegionLocator(TableName.valueOf(tableName))
println("regionLocator" + regionLocator)
HFileOutputFormat2.configureIncrementalLoad(job, table,regionLocator)
// Generate 10 sample data:
val num = sc.parallelize(1 to 10)
val rdd = num.map(x=>{
println("xxxxxxxxxxxxxxxxxxxxxxxx" + x)
val kv: KeyValue = new KeyValue(Bytes.toBytes("11"), "cf".getBytes(), "c1".getBytes(), "value_xxx".getBytes() )
println("-----kv------")
println(kv)
(new ImmutableBytesWritable(Bytes.toBytes("11")), kv)
})
rdd.foreach(println)
rdd.saveAsNewAPIHadoopFile(hdfsRootPath, classOf[ImmutableBytesWritable], classOf[KeyValue], classOf[HFileOutputFormat2], job.getConfiguration())
//将保存在临时文件夹的hfile数据保存到hbase中
val bulkLoader = new LoadIncrementalHFiles(conf)
bulkLoader.doBulkLoad(new Path(hdfsRootPath), table)
table.flushCommits()
table.close()
hbaseConn.close()
}
}