package com.lhjava.hbase;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import java.io.IOException;
/**
* 使用MapReduce完成HBase表数据的拷贝
* 把member表中的数据备份到member_bak表中
*/
public class CopyHBaseTableApp {
public static class MyMapper extends TableMapper<Text, Put> {
Text mapOutputKey = new Text();
/**
* key是hbase表中的rowkey
*/
public void map(ImmutableBytesWritable row, Result value, Context context) throws IOException, InterruptedException {
mapOutputKey.set(Bytes.toString(row.get()));
Put put = new Put(row.get());
for (Cell cell : value.rawCells()) {
put.add(cell);
}
context.write(mapOutputKey, put);
}
}
public static class MyTableReducer extends TableReducer<Text, Put, ImmutableBytesWritable> {
public void reduce(Text key, Iterable<Put> values, Context context) throws IOException, InterruptedException {
for (Put put : values){
context.write(null, put);
}
}
}
public static void main(String[] args)throws Exception {
Configuration config = HBaseConfiguration.create();
Job job = new Job(config,"CopyHBaseTableApp");
job.setJarByClass(CopyHBaseTableApp.class); // class that contains mapper and reducer
Scan scan = new Scan();
//scan.setCaching(500); // 1 is the default in Scan, which will be bad for MapReduce jobs
//scan.setCacheBlocks(false); // don't set to true for MR jobs
// set other scan attrs
TableMapReduceUtil.initTableMapperJob(
args[0], // input table
scan, // Scan instance to control CF and attribute selection
MyMapper.class, // mapper class
Text.class, // mapper output key
Put.class, // mapper output value
job);
TableMapReduceUtil.initTableReducerJob(
args[1], // output table
MyTableReducer.class, // reducer class
job);
job.setNumReduceTasks(1); // at least one, adjust as required
boolean b = job.waitForCompletion(true);
if (!b) {
throw new IOException("error with job!");
}
}
}
hadoop jar /home/fengqing/software/aa-1.0-SNAPSHOT.jar com.lhjava.hbase.CopyHBaseTableApp member member_bak