HBase回顾四、HBase集成MapReduce
通过HBase的相关JavaAPI,我们可以实现伴随HBase操作的MapReduce过程,比如使用MapReduce将数据从本地文件系统导入到HBase的表中,比如我们从HBase中读取一些原始数据后使用MapReduce做数据分析。
案例1
要求:通过MapReduce对HBase中student表的数据复制到user表
流程图分析:
第一个案例很简单,只需要通过MR程序完成读写操作即可,而需要注意的时,因为数据的来源和去处都是HBase,所以需要利用HBase提供的MR工具类TableMapReduceUtil
来初始化对应的MapJob和ReduceJob,这样我们可以不用设置自定义的InputFormat和OutputFormat就可以从HBase中获取数据和存放数据
实际上可以通过编写协处理器来完成类似的操作
主程序
package com.chanzany.hbase;
import com.chanzany.hbase.tool.HBaseMapReduceTool;
import org.apache.hadoop.util.ToolRunner;
public class Table2TableApplication {
public static void main(String[] args) throws Exception {
//ToolRunner可以运行MR
ToolRunner.run(new HBaseMapReduceTool(),args);
}
}
HBaseMapReduceTool
package com.chanzany.hbase.tool;
import com.chanzany.hbase.mapper.ScanDataMapper;
import com.chanzany.hbase.reducer.InsertDataReducer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobStatus;
import org.apache.hadoop.util.Tool;
public class HBaseMapReduceTool implements Tool {
public int run(String[] strings) throws Exception {
Job job = Job.getInstance(new Configuration());
job.setJarByClass(HBaseMapReduceTool.class);
//mapper
//reducer
//input,output
TableMapReduceUtil.initTableMapperJob(
"chanzany:student",
new Scan(),
ScanDataMapper.class,
ImmutableBytesWritable.class,
Put.class,
job
);
TableMapReduceUtil.initTableReducerJob(
"chanzany:user",
InsertDataReducer.class,
job
);
//执行job
boolean flag = job.waitForCompletion(true);
return flag? JobStatus.State.SUCCEEDED.getValue():JobStatus.State.FAILED.getValue();
}
public void setConf(Configuration configuration) {
}
public Configuration getConf() {
return null;
}
}
mapper
package com.chanzany.hbase.mapper;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
public class ScanDataMapper extends TableMapper<ImmutableBytesWritable, Put> {
@Override
protected void map(ImmutableBytesWritable key, Result result, Context context) throws IOException, InterruptedException {
//运行mapper,查询数据
// scan result==>Put
Put put = new Put(key.get());
for (Cell cell : result.rawCells()) {
put.addColumn(
CellUtil.cloneFamily(cell),
CellUtil.cloneQualifier(cell),
CellUtil.cloneValue(cell));
}
context.write(key, put);
}
}
在这里提一嘴,因为是从HBase中读取数据,所以自定义的mapper类需要继承TableMapper
,而跟进源码发现,我们只需要在泛型中写入KEYOUT和VALUEOUT
reducer
package com.chanzany.hbase.reducer;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;
public class InsertDataReducer extends TableReducer<ImmutableBytesWritable,Put,NullWritable> {
@Override
protected void reduce(ImmutableBytesWritable key, Iterable<Put> values, Context context) throws IOException, InterruptedException {
//运行reducer,增加数据(put)
for (Put put : values) {
context.write(NullWritable.get(),put);
}
}
}
跟进Mutation的源码发现其有4个实现子类
刚好对应了我们在回顾三中所做的表操作所需类
比如Put的使用如下:
/**
* 增加数据
* @param rowkey rowkey
* @param family family
* @param column column
* @param value value
* @throws Exception
*/
public static void insertData(String tablename,String rowkey,String family,String column,String value)throws Exception{
Connection conn = connHolder.get();
Table hTable = conn.getTable(TableName.valueOf(tablename));
Put put = new Put(Bytes.toBytes(rowkey));
put.addColumn(Bytes.toBytes(family),Bytes.toBytes(column),Bytes.toBytes(value));
hTable.put(put);
hTable.close();
}
由于从Map端送过来的Result是包含rowKey的,通过shuffle转换为了reduce方法中的Iterable<Put> values
reduce(ImmutableBytesWritable key, Iterable<Put> values, Context context)
所以直接将values中各个Put对象写回HBase
for (Put put : values) {
context.write(NullWritable.get(),put);
}
案例2
将hdfs文件中的数据插入到Hbase中的student表中
流程图分析:
对于数据的输入,我们需要手动设置FileInputFormat
的输入路径,而对于数据的输出,我们还是采用TableMapReduceUtil
来指定表名即输出路径
主程序
package com.chanzany.hbase.mr;
import com.chanzany.hbase.mr.tool.File2TableTool;
import org.apache.hadoop.util.ToolRunner;
public class File2TableApplication {
public static void main(String[] args) throws Exception {
ToolRunner.run(new File2TableTool(),args);
}
}
File2TableTool
package com.chanzany.hbase.mr.tool;
import com.chanzany.hbase.mr.mapper.LoadHdfsMapper;
import com.chanzany.hbase.mr.reducer.InserDataReducer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobStatus;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.util.Tool;
public class File2TableTool implements Tool {
public int run(String[] strings) throws Exception {
Job job = Job.getInstance();
job.setJarByClass(File2TableTool.class);
/*hdfs:File-->hbase*/
//InputFormat
FileInputFormat.addInputPath(job,new Path("hdfs://hadoop102:9000/data/student.csv"));
//mapper
job.setMapperClass(LoadHdfsMapper.class);
job.setMapOutputKeyClass(ImmutableBytesWritable.class);
job.setMapOutputValueClass(Put.class);
//reducer
TableMapReduceUtil.initTableReducerJob(
"student",
InserDataReducer.class,
job
);
boolean flag = job.waitForCompletion(true);
return flag? JobStatus.State.SUCCEEDED.getValue():JobStatus.State.FAILED.getValue();
}
public void setConf(Configuration configuration) {
}
public Configuration getConf() {
return null;
}
}
需要注意的是
job.setMapOutputKeyClass(ImmutableBytesWritable.class);
job.setMapOutputValueClass(Put.class);
通过对案例1中Mapper的分析,我们可以得出对于HBase的rowKey,其数据类型是ImmutableBytesWritable
,而该rowKey对应的数据比如ColumnFamily、Column、Value,都会封装在Put
, Delet
等继承了Mutation
的类中
所以在这里设置Map的输出KV类型为<ImmutableBytesWritable,Put>
Put表示插入数据
Delete表示删除数据
Mapper
package com.chanzany.hbase.mr.mapper;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
public class LoadHdfsMapper extends Mapper<LongWritable,Text,ImmutableBytesWritable,Put>{
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String[] fields = value.toString().split(",");
String rk=fields[0]; //rowkey
String rv=fields[1]; //name-value
byte[] brk = Bytes.toBytes(rk);
ImmutableBytesWritable hTrowkey = new ImmutableBytesWritable(brk);
Put hTvalue = new Put(brk);
hTvalue.addColumn(
Bytes.toBytes("info"),
Bytes.toBytes("name"),
Bytes.toBytes(rv)
);
context.write(hTrowkey,hTvalue);
}
}
Reducer
package com.chanzany.hbase.mr.reducer;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;
public class InserDataReducer extends TableReducer<ImmutableBytesWritable,Put,NullWritable> {
@Override
protected void reduce(ImmutableBytesWritable key, Iterable<Put> values, Context context) throws IOException, InterruptedException {
//拿到从map传过来的数据,然后插入到HBase
for (Put put : values) {
context.write(NullWritable.get(),put);//put对象封装了每个row对应的数据
}
}
}
打包运行流程图
先将编写了代码的模块或项目打成jar包,根据需求可以打成依赖jar包(直接通过maven的package)或者是可以独立运行的jar包。在案例1、2中,我们编写了启动类所以采用第二种形式。
打开项目结构,编辑artifacts
然后点击build,完成Jar包的构建
最后将该jar包上传到Linux,通过hadoop jar或yarn jar命令运行。