由于 org.apache.hadoop.hbase.mapreduce.TableInputFormat 类的实现,Spark 可以通过Hadoop输入格式访问 HBase。
这个输入格式会返回键值对数据,其中键的类型为org. apache.hadoop.hbase.io.ImmutableBytesWritable,而值的类型为org.apache.hadoop.hbase.client.Result。
1. 导入依赖
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>2.1.1</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>1.3.1</version>
</dependency>
</dependencies>
2. Spark从HBase读取数据
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.hbase.client.Result
import org.apache.hadoop.hbase.io.ImmutableBytesWritable
import org.apache.hadoop.hbase.mapreduce.TableInputFormat
import org.apache.hadoop.hbase.util.Bytes
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
/**
* @author fczheng
*
*/
object Spark35_RDD_HBase {
def main(args: Array[String]): Unit = {
// TODO 1.创建Spark配置对象
val conf: SparkConf = new SparkConf().setAppName("Spark_HBase").setMaster("local[*]")
// TODO 2.创建Spark环境连接对象
val sc: SparkContext = new SparkContext(conf)
val hbaseConf: Configuration = HBaseConfiguration.create()
hbaseConf.set("hbase.zookeeper.quorum", "hadoop102,hadoop103,hadoop104")
hbaseConf.set(TableInputFormat.INPUT_TABLE,"student")
//TODO 3.从HBase中读取数据
val hbaseRDD: RDD[(ImmutableBytesWritable, Result)] = sc.newAPIHadoopRDD(
hbaseConf,
classOf[TableInputFormat],
classOf[ImmutableBytesWritable],
classOf[Result]
)
val rdd2: RDD[String] = hbaseRDD.map {
case (_, result) => Bytes.toString(result.getRow)
}
rdd2.collect().foreach(println)
// TODO 4. 释放连接
sc.stop()
}
}
3. Spark向HBase写入数据
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.hbase.client.Put
import org.apache.hadoop.hbase.io.ImmutableBytesWritable
import org.apache.hadoop.hbase.mapreduce.TableOutputFormat
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.mapreduce.Job
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
/**
* @author fczheng
*
*/
object Spark36_RDD_HBase1 {
def main(args: Array[String]): Unit = {
// TODO 1.创建Spark配置对象
val conf: SparkConf = new SparkConf().setAppName("Spark_HBase").setMaster("local[*]")
// TODO 2.创建Spark环境连接对象
val sc: SparkContext = new SparkContext(conf)
val hbaseConf: Configuration = HBaseConfiguration.create()
hbaseConf.set("hbase.zookeeper.quorum","hadoop102,hadoop103,hadoop104")
hbaseConf.set(TableOutputFormat.OUTPUT_TABLE,"student")
// TODO 3.通过job来设置输出的格式的类
val job: Job = Job.getInstance(hbaseConf)
job.setOutputFormatClass(classOf[TableOutputFormat[ImmutableBytesWritable]])
job.setOutputKeyClass(classOf[ImmutableBytesWritable])
job.setOutputValueClass(classOf[Put])
// TODO 4.向HBase插入数据
//Put
val dataRDD: RDD[(String, String, Int)] = sc.parallelize(List(("1003","male",22),("1004","female",88),("1005","female",66)))
//data => put
val hbaseRDD: RDD[(ImmutableBytesWritable, Put)] = dataRDD.map(x => {
val put: Put = new Put(Bytes.toBytes(x._1))
put.addColumn(Bytes.toBytes("info"), Bytes.toBytes("sex"), Bytes.toBytes(x._2))
put.addColumn(Bytes.toBytes("info"), Bytes.toBytes("age"), Bytes.toBytes(x._3))
(new ImmutableBytesWritable(), put)
})
hbaseRDD.saveAsNewAPIHadoopDataset(job.getConfiguration)
//TODO 5.释放连接
sc.stop()
}
}