HBase回顾四、HBase集成MapReduce

HBase回顾四、HBase集成MapReduce

通过HBase的相关JavaAPI，我们可以实现伴随HBase操作的MapReduce过程，比如使用MapReduce将数据从本地文件系统导入到HBase的表中，比如我们从HBase中读取一些原始数据后使用MapReduce做数据分析。

案例1

要求：通过MapReduce对HBase中student表的数据复制到user表

流程图分析：

第一个案例很简单，只需要通过MR程序完成读写操作即可，而需要注意的时，因为数据的来源和去处都是HBase,所以需要利用HBase提供的MR工具类TableMapReduceUtil来初始化对应的MapJob和ReduceJob,这样我们可以不用设置自定义的InputFormat和OutputFormat就可以从HBase中获取数据和存放数据

实际上可以通过编写协处理器来完成类似的操作

主程序

package com.chanzany.hbase;

import com.chanzany.hbase.tool.HBaseMapReduceTool;
import org.apache.hadoop.util.ToolRunner;

public class Table2TableApplication {
    public static void main(String[] args) throws Exception {
        //ToolRunner可以运行MR
        ToolRunner.run(new HBaseMapReduceTool(),args);

    }
}

HBaseMapReduceTool

package com.chanzany.hbase.tool;

import com.chanzany.hbase.mapper.ScanDataMapper;
import com.chanzany.hbase.reducer.InsertDataReducer;
import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobStatus;
import org.apache.hadoop.util.Tool;

public class HBaseMapReduceTool implements Tool {
    public int run(String[] strings) throws Exception {
        Job job = Job.getInstance(new Configuration());
        job.setJarByClass(HBaseMapReduceTool.class);

        //mapper
        //reducer
        //input,output
        TableMapReduceUtil.initTableMapperJob(
                 "chanzany:student",
                new Scan(),
                ScanDataMapper.class,
                ImmutableBytesWritable.class,
                Put.class,
                job
        );
        TableMapReduceUtil.initTableReducerJob(
                "chanzany:user",
                InsertDataReducer.class,
                job
        );

        //执行job
        boolean flag = job.waitForCompletion(true);
        return flag? JobStatus.State.SUCCEEDED.getValue():JobStatus.State.FAILED.getValue();
    }

    public void setConf(Configuration configuration) {

    }

    public Configuration getConf() {
        return null;
    }
}

mapper

package com.chanzany.hbase.mapper;

import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class ScanDataMapper extends TableMapper<ImmutableBytesWritable, Put> {
    @Override
    protected void map(ImmutableBytesWritable key, Result result, Context context) throws IOException, InterruptedException {
        //运行mapper,查询数据
        // scan result==>Put
        Put put = new Put(key.get());
        for (Cell cell : result.rawCells()) {
            put.addColumn(
                    CellUtil.cloneFamily(cell),
                    CellUtil.cloneQualifier(cell),
                    CellUtil.cloneValue(cell));
        }
        context.write(key, put);
    }
}

在这里提一嘴，因为是从HBase中读取数据，所以自定义的mapper类需要继承TableMapper,而跟进源码发现，我们只需要在泛型中写入KEYOUT和VALUEOUT

reducer

package com.chanzany.hbase.reducer;

import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class InsertDataReducer extends TableReducer<ImmutableBytesWritable,Put,NullWritable> {

    @Override
    protected void reduce(ImmutableBytesWritable key, Iterable<Put> values, Context context) throws IOException, InterruptedException {
        //运行reducer，增加数据(put)
        for (Put put : values) {
            context.write(NullWritable.get(),put);
        }
    }
}

跟进Mutation的源码发现其有4个实现子类

刚好对应了我们在回顾三中所做的表操作所需类

比如Put的使用如下：

/**
     * 增加数据
     * @param rowkey rowkey
     * @param family family
     * @param column column
     * @param value value
     * @throws Exception
     */
    public static void insertData(String tablename,String rowkey,String family,String column,String value)throws Exception{
        Connection conn = connHolder.get();
        Table hTable = conn.getTable(TableName.valueOf(tablename));
        Put put = new Put(Bytes.toBytes(rowkey));
        put.addColumn(Bytes.toBytes(family),Bytes.toBytes(column),Bytes.toBytes(value));
        hTable.put(put);
        hTable.close();
    }

由于从Map端送过来的Result是包含rowKey的，通过shuffle转换为了reduce方法中的Iterable<Put> values

reduce(ImmutableBytesWritable key, Iterable<Put> values, Context context)

所以直接将values中各个Put对象写回HBase

for (Put put : values) {
    context.write(NullWritable.get(),put);
}

案例2

将hdfs文件中的数据插入到Hbase中的student表中

流程图分析：

对于数据的输入，我们需要手动设置FileInputFormat的输入路径，而对于数据的输出，我们还是采用TableMapReduceUtil来指定表名即输出路径

主程序

package com.chanzany.hbase.mr;

import com.chanzany.hbase.mr.tool.File2TableTool;
import org.apache.hadoop.util.ToolRunner;

public class File2TableApplication {
    public static void main(String[] args) throws Exception {
        ToolRunner.run(new File2TableTool(),args);

    }
}

---

File2TableTool

package com.chanzany.hbase.mr.tool;

import com.chanzany.hbase.mr.mapper.LoadHdfsMapper;
import com.chanzany.hbase.mr.reducer.InserDataReducer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobStatus;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.util.Tool;

public class File2TableTool implements Tool {
    public int run(String[] strings) throws Exception {
        Job job = Job.getInstance();
        job.setJarByClass(File2TableTool.class);

        /*hdfs:File-->hbase*/
        //InputFormat
        FileInputFormat.addInputPath(job,new Path("hdfs://hadoop102:9000/data/student.csv"));

        //mapper
        job.setMapperClass(LoadHdfsMapper.class);
        job.setMapOutputKeyClass(ImmutableBytesWritable.class);
        job.setMapOutputValueClass(Put.class);

        //reducer
        TableMapReduceUtil.initTableReducerJob(
                "student",
                InserDataReducer.class,
                job
        );

        boolean flag = job.waitForCompletion(true);
        return flag? JobStatus.State.SUCCEEDED.getValue():JobStatus.State.FAILED.getValue();
    }

    public void setConf(Configuration configuration) {

    }

    public Configuration getConf() {
        return null;
    }
}

需要注意的是

job.setMapOutputKeyClass(ImmutableBytesWritable.class);
job.setMapOutputValueClass(Put.class);

通过对案例1中Mapper的分析，我们可以得出对于HBase的rowKey,其数据类型是ImmutableBytesWritable,而该rowKey对应的数据比如ColumnFamily、Column、Value,都会封装在Put, Delet等继承了Mutation的类中

所以在这里设置Map的输出KV类型为<ImmutableBytesWritable,Put>

Put表示插入数据

Delete表示删除数据

Mapper

package com.chanzany.hbase.mr.mapper;


import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class LoadHdfsMapper extends Mapper<LongWritable,Text,ImmutableBytesWritable,Put>{

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String[] fields = value.toString().split(",");
        String rk=fields[0]; //rowkey
        String rv=fields[1]; //name-value

        byte[] brk = Bytes.toBytes(rk);

        ImmutableBytesWritable hTrowkey = new ImmutableBytesWritable(brk);
        Put hTvalue = new Put(brk);
        hTvalue.addColumn(
                Bytes.toBytes("info"),
                Bytes.toBytes("name"),
                Bytes.toBytes(rv)
        );

        context.write(hTrowkey,hTvalue);

    }
}

Reducer

package com.chanzany.hbase.mr.reducer;

import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class InserDataReducer extends TableReducer<ImmutableBytesWritable,Put,NullWritable> {
    @Override
    protected void reduce(ImmutableBytesWritable key, Iterable<Put> values, Context context) throws IOException, InterruptedException {
        //拿到从map传过来的数据，然后插入到HBase
        for (Put put : values) {
            context.write(NullWritable.get(),put);//put对象封装了每个row对应的数据
        }
    }
}

打包运行流程图

先将编写了代码的模块或项目打成jar包，根据需求可以打成依赖jar包(直接通过maven的package)或者是可以独立运行的jar包。在案例1、2中，我们编写了启动类所以采用第二种形式。

打开项目结构，编辑artifacts

然后点击build，完成Jar包的构建

最后将该jar包上传到Linux,通过hadoop jar或yarn jar命令运行。