/**
* spark从hbase中读取数据
* @param hbaseConfig
* @param spark
*/
def getHbase(hbaseConfig:Configuration,spark:SparkSession): Unit ={
hbaseConfig.set(TableInputFormat.INPUT_TABLE,"stu_person") ;
val hbaseRdd = spark.sparkContext.newAPIHadoopRDD(hbaseConfig,classOf[TableInputFormat],
classOf[org.apache.hadoop.hbase.io.ImmutableBytesWritable],
classOf[org.apache.hadoop.hbase.client.Result]);
hbaseRdd.foreach(result=>{
val key = Bytes.toString(result._2.getRow);
val deptid = Bytes.toString(result._2.getValue("info".getBytes,"deptid".getBytes));
val name = Bytes.toString(result._2.getValue("info".getBytes,"name".getBytes));
val salary = Bytes.toString(result._2.getValue("info".getBytes,"salary".getBytes));
println("key:"+key+":info{name:"+name+"}");
});
}
/**
* spark写入数据到hbase
* @param hbaseConfig
* @param spark
*/
def writeHbase(hbaseConfig:Configuration,spark:SparkSession): Unit ={
val jobConf = new JobConf(hbaseConfig);
jobConf.setOutputFormat(classOf[TableOutputFormat]) ;
jobConf.set(TableOutputFormat.OUTPUT_TABLE,"stu_person");
val array = Array("002,dept-1,lee,2300",
"002,dept-1,lily,3300",
"003,dept-2,lucy,4300",
"003,dept-2,chen,2100",
"004,dept-3,sun,5300",
"004,dept-3,hello,9300");
val rdd = spark.sparkContext.makeRDD(array) ;
val resultRDD = rdd.map(line=>line.split(",")).map(x=>{
val put = new Put(Bytes.toBytes(x(0)));
put.addColumn(Bytes.toBytes("info"),Bytes.toBytes("deptid"),Bytes.toBytes(x(1))) ;
put.addColumn(Bytes.toBytes("info"),Bytes.toBytes("name"),Bytes.toBytes(x(2))) ;
put.addColumn(Bytes.toBytes("info"), Bytes.toBytes("salary"), Bytes.toBytes(x(3)));
(new ImmutableBytesWritable,put)
});
resultRDD.saveAsHadoopDataset(jobConf) ;
}