使用spark向hbase表添加数据(数据库、表、数据自己定), 使用spark将hbase表中的数据查询出来打入控制台。(在spark向hbase表添加数据需要:先开启zk、hbase,进入hbase shell窗口)
数据:
List((“1”,“apple”,“11”), (“2”,“banana”,“12”), (“3”,“pear”,“13”))
代码:
object hbase {
def main(args: Array[String]): Unit = {
//创建SparkContext
val config = new SparkConf().setAppName("habse").setMaster("local[*]")
val sc = new SparkContext(config)
//连接数据库
val conf = HBaseConfiguration.create()
conf.set("hbase.zookeeper.quorum", "hadoop01:2181,hadoop02:2181,hadoop03:2181")
//获取表名
val fruitTable = TableName.valueOf("fruit")
//设置表结构
val tableDescr = new HTableDescriptor(fruitTable)
//建立表与列族之间的关系
tableDescr.addFamily(new HColumnDescriptor("info".getBytes))
//实例admin管理员
val admin = new HBaseAdmin(conf)
//判断是否存在,是否损坏,是否存在删除
if (admin.tableExists(fruitTable)) {
admin.disableTable(fruitTable)
admin.deleteTable(fruitTable)
}
//创建一个新的表
admin.createTable(tableDescr)
def convert(triple: (String, String, String)) = {
//创建put对象,并指定rowkey
val put = new Put(Bytes.toBytes(triple._1))
//设置列族
put.addImmutable(Bytes.toBytes("info"), Bytes.toBytes("name"), Bytes.toBytes(triple._2))
put.addImmutable(Bytes.toBytes("info"), Bytes.toBytes("price"), Bytes.toBytes(triple._3))
(new ImmutableBytesWritable, put)
}
//插入数据
val dataRDD: RDD[(String, String, String)] = sc.parallelize(List(("1","apple","11"), ("2","banana","12"), ("3","pear","13")))
val targetRDD: RDD[(ImmutableBytesWritable, Put)] = dataRDD.map(convert)
//
val jobConf = new JobConf(conf)
jobConf.setOutputFormat(classOf[TableOutputFormat])
jobConf.set(TableOutputFormat.OUTPUT_TABLE, "fruit")
//写入数据
targetRDD.saveAsHadoopDataset(jobConf)
println("写入数据成功")
//读取数据
conf.set(TableInputFormat.INPUT_TABLE, "fruit")
//添加配置
val hbaseRDD: RDD[(ImmutableBytesWritable, Result)] = sc.newAPIHadoopRDD(conf, classOf[TableInputFormat],
classOf[org.apache.hadoop.hbase.io.ImmutableBytesWritable],
classOf[org.apache.hadoop.hbase.client.Result])
val count: Long = hbaseRDD.count()
println("hBaseRDD RDD Count:"+ count)
//遍历数据
hbaseRDD.foreach {
case (_, result) =>
val key = Bytes.toString(result.getRow)
val name = Bytes.toString(result.getValue("info".getBytes, "name".getBytes))
val color = Bytes.toString(result.getValue("info".getBytes, "price".getBytes))
println("Row key:" + key + " Name:" + name + " Color:" + color)
}
sc.stop()
}
}
结果: