Hbase的MR实现
需求一:从mysuer表中读取数据,然后写入到Hbase的另一张表中
HBaseReadWrite:
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
public class HBaseReadWrite extends Configured implements Tool {
@Override
public int run(String[] strings) throws Exception {
Scan scan = new Scan();
Configuration conf = super.getConf();
Job job = Job.getInstance(super.getConf(), "hbaseReadWrite");
TableMapReduceUtil.initTableMapperJob("myuser",scan, HBaseMapper.class, Text.class, Put.class,job);
TableMapReduceUtil.initTableReducerJob("myuser2", HBaseReducer.class,job);
job.setNumReduceTasks(2);
boolean verbose = true;
boolean b = job.waitForCompletion(verbose);
return b?0:1 ;
}
public static void main(String[] args) throws Exception {
Configuration configuration = HBaseConfiguration.create();
configuration.set("hbase.zookeeper.quorum", "node01:2181,node02:2181,node03:2181");
int run = ToolRunner.run(configuration, new HBaseReadWrite(), args);
System.exit(run);
}
}
HBaseMapper
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
import java.util.List;
public class HBaseMapper extends TableMapper <Text, Put> {
@Override
protected void map(ImmutableBytesWritable key, Result result, Context context) throws IOException, InterruptedException {
byte[] rowKeyBytes = key.get();
String rowkey = Bytes.toString(rowKeyBytes);
List<Cell> cells = result.listCells();
for (Cell cell : cells) {
byte[] family = cell.getFamily();
byte[] qualifier = cell.getQualifier();
byte[] rowArray = cell.getRowArray();
Put put = new Put(rowKeyBytes);
if ("f1".equals(Bytes.toString(family))) {
if ("name".equals(Bytes.toString(qualifier)) && "age".equals(Bytes.toString(qualifier))) {
put.add(cell);
}
if (!put.isEmpty()) {
context.write(new Text(rowkey), put);
}
}
}
}
}
HBaseReducer
import</