关于HBase bulk loading原理的详细解析及完整案例可参考HBase Bulk Loading: What, Why and How
通过MapReduce进行bulk loading存在效率低下的问题, 如果数据预处理无法在一个Job中完成, 还需要分解为多个Job. 本文介绍使用Spark进行HBase bulk loading的两种方法.
基于Spark的bulk loading
本方法适用于HBase 1.4.x版本.
object SparkBulkLoad {
def main(args: Array[String]): Unit = {
if (args.length < 1) {
println("Usage: SparkBulkLoad + {stagingFolder}")
}
val tableName = "bulkload-table-test"
val columnFamily = "f"
val stagingFolder = args(0)
// get table info
val hConf = HBaseConfiguration.create()
val connection = ConnectionFactory.createConnection(hConf)
val table = connection.getTable(TableName.valueOf(tableName))
val regionLocator = connection.getRegionLocator(TableName.valueOf(tableName))
val sConf = new SparkConf().setAppName("bulkload").setMaster("local")
sConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
val sc = new SparkContext(sConf)
// simple data to bulk load to hbase
val rdd = sc.parallelize(Array(
(Bytes.toBytes("1"),
Array((Bytes.toBytes(columnFamily), Bytes.toBytes("4"), Bytes.toBytes("1")))),
(Bytes.toBytes("4"),
Array((Bytes.toBytes(columnFamily), Bytes.toBytes("2"), Bytes.toBytes("2")))),
(Bytes.toBytes("2"),
Array((Bytes.toBytes(columnFamily), Bytes.toBytes("8"), Bytes.toBytes("3")))),
(Bytes.toBytes("3"),
Array((Bytes.toBytes(columnFamily), Bytes.toBytes("9"), Bytes.toBytes("4")))),
(Bytes.toBytes("5"),
Array((Bytes.toBytes(columnFamily), Bytes.toBytes("1"), Bytes.toBytes("5"))))
)).repartition(2) // random partition to simulate a real scene
// parse the source data and repartition and sort by row key, family, qualify
val sortedRdd = rdd.map(line => {
val keyFamilyQualifier = new KeyFamilyQualifier(line._1, line._2(0)._1, line._2(0)._2)
val value = line._2(0)._3
(keyFamilyQualifier, value)
}).repartitionAndSortWithinPartitions(new BulkLoadPartitioner(regionLocator.getStartKeys))
// reformat the data so that we can save as HFileOutputFormat2
val hfileRdd = sortedRdd.map(line => {
val rowKey = new ImmutableBytesWritable(line._1.getRowKey)
val keyValue = new hbase.KeyValue(line._1.getRowKey, line._1.getFamily, line._1.getQualifier, line._2)
(rowKey, keyValue)
})
// save the rdd as hfile
hfileRdd.saveAsNewAPIHadoopFile(
stagingFolder,
classOf[ImmutableBytesWritable],
classOf[KeyValue],
classOf[HFileOutputFormat2], hConf)
// load the hfile from hdfs
val loader = new LoadIncrementalHFiles(hConf)
loader.doBulkLoad(new Path(stagingFolder), connection.getAdmin, table, regionLocator)
}
}
其中KeyFamilyQualifier
和BulkLoadPartitioner的定义如下:
public class KeyFamilyQualifier implements WritableComparable<KeyFamilyQualifier> {
private byte[] rowKey;
private byte[] family;
private byte[] qualifier;
public KeyFamilyQualifier(byte[] rowKey, byte[] family, byte[] qualifier) {
this.rowKey = rowKey;
this.family = family;
this.qualifier = qualifier;
}
@Override
public int compareTo(@Nonnull KeyFamilyQualifier other) {
int rowCmp = Bytes.compareTo(this.rowKey, other.rowKey);
if (rowCmp == 0) {
int familyCmp = Bytes.compareTo(this.family, other.family);
if (familyCmp == 0) {
return Bytes.compareTo(this.family, other.family);
}
return rowCmp;
}
return rowCmp;
}
@Override
public void write(DataOutput dataOutput) throws IOException {
dataOutput.writeInt(rowKey.length); dataOutput.write(rowKey);
dataOutput.writeInt(family.length); dataOutput.write(family);
dataOutput.writeInt(qualifier.length); dataOutput.write(qualifier);
}
@Override
public void readFields(DataInput dataInput) throws IOException {
int rowKeyLength = dataInput.readInt();
this.rowKey = new byte[rowKeyLength];
dataInput.readFully(this.rowKey, 0, rowKeyLength);
int familyLength = dataInput.readInt();
this.family = new byte[familyLength];
dataInput.readFully(this.family, 0, familyLength);
int qualifierLength = dataInput.readInt();
this.qualifier = new byte[qualifierLength];
dataInput.readFully(this.qualifier, 0, qualifierLength);
}
public byte[] getRowKey() {
return rowKey;
}
public byte[] getFamily() {
return family;
}
public byte[] getQualifier() {
return qualifier;
}
}
class BulkLoadPartitioner(startKeys:Array[Array[Byte]])
extends Partitioner {
// when table not exist, startKeys = Byte[0][]
override def numPartitions: Int = if (startKeys.length == 0) 1 else startKeys.length
override def getPartition(key: Any): Int = {
val comparator: Comparator[Array[Byte]] = new Comparator[Array[Byte]] {
override def compare(o1: Array[Byte], o2: Array[Byte]): Int = {
Bytes.compareTo(o1, o2)
}
}
val rowKey:Array[Byte] =
key match {
case qualifier: KeyFamilyQualifier =>
qualifier.getRowKey
case _ =>
key.asInstanceOf[Array[Byte]]
}
var partition = util.Arrays.binarySearch(startKeys, rowKey, comparator)
if (partition < 0)
partition = partition * -1 + -2
if (partition < 0)
partition = 0
partition
}
}
基于HBase Connectors的Bulk Loading
在HBase 2.2.x版本中HBase官方提供了HBase Connectors.可更方便地利用Spark进行bulk load. 详情见HBase Bulk Loading: What, Why and How. Java案例如下.
final public class JavaHBaseBulkLoadExample {
private JavaHBaseBulkLoadExample() {}
public static void main(String[] args) {
if (args.length < 1) {
System.out.println("JavaHBaseBulkLoadExample " + "{outputPath}");
return;
}
String tableName = "bulkload-table-test";
String columnFamily1 = "f1";
String columnFamily2 = "f2";
SparkConf sparkConf = new SparkConf().setAppName("JavaHBaseBulkLoadExample " + tableName);
JavaSparkContext jsc = new JavaSparkContext(sparkConf);
try {
List<String> list= new ArrayList<String>();
// row1
list.add("1," + columnFamily1 + ",b,1");
// row3
list.add("3," + columnFamily1 + ",a,2");
list.add("3," + columnFamily1 + ",b,1");
list.add("3," + columnFamily2 + ",a,1");
/* row2 */
list.add("2," + columnFamily2 + ",a,3");
list.add("2," + columnFamily2 + ",b,3");
JavaRDD<String> rdd = jsc.parallelize(list);
Configuration conf = HBaseConfiguration.create();
JavaHBaseContext hbaseContext = new JavaHBaseContext(jsc, conf);
hbaseContext.bulkLoad(rdd, TableName.valueOf(tableName),new BulkLoadFunction(), args[0],
new HashMap<byte[], FamilyHFileWriteOptions>(), false, HConstants.DEFAULT_MAX_FILE_SIZE);
} finally {
jsc.stop();
}
}
public static class BulkLoadFunction
implements Function<String, Pair<KeyFamilyQualifier, byte[]>> {
@Override
public Pair<KeyFamilyQualifier, byte[]> call(String v1) throws Exception {
if (v1 == null) {
return null;
}
String[] strs = v1.split(",");
if(strs.length != 4) {
return null;
}
KeyFamilyQualifier kfq = new KeyFamilyQualifier(Bytes.toBytes(strs[0]),
Bytes.toBytes(strs[1]), Bytes.toBytes(strs[2]));
return new Pair(kfq, Bytes.toBytes(strs[3]));
}
}
}