【HBase】- 使用MapReduce对HBase数据进行操作案例 ⭐️⭐️⭐️

案例 一

1. 需求

从 Person 表中读取name列的数据 写入到 Person_mr

2.代码

Map 阶段

package com.wangt.habse.mr01;

import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.util.Bytes;

import java.io.IOException;

/**
 * 从 Person 表中读取数据 写到 Person_mr 表中
 *
 * @author 王天赐
 * @create 2019-08-02 20:08
 */
public class PersonMapper extends TableMapper<ImmutableBytesWritable, Put> {
    //  输出类型是 ImmutableBytesWritable(序列化的Byte 数组) 和  Put

    /**
     * 从 Hbase 的 Person 表中读取数据
     *
     * @param key     rowkey
     * @param value   从 Hbase 表中读取数据的 Result 对象
     * @param context
     * @throws IOException
     * @throws InterruptedException
     */
    @Override
    protected void map(ImmutableBytesWritable key, Result value, Context context)
            throws IOException, InterruptedException {

        // 构建 Put 对象
        // 注意 : 这个key是序列化的 rowkey
        Put put = new Put(key.get());
        System.out.println(Bytes.toString(key.get()));
        // 遍历数组
        Cell[] cells = value.rawCells();
        for (Cell cell : cells) {
            // 获取 HBase 的列
           String qualifier = Bytes.toString(CellUtil.cloneQualifier(cell));
           //  列族
           String family =  Bytes.toString(CellUtil.cloneFamily(cell));
           switch (qualifier) {
                // 判断字段 是否是 name
                // 如果是 name 列 ,则将 name列的数据 添加到 Put 对象中 注意 一个 map 只读一条数据
                case "name":
                    put.add(cell);
                    break;
           }
        }

        if(!put.isEmpty()){
            // 输出
            context.write(key, put);
        }
    }
}

Reduce

package com.wangt.habse.mr01;

import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.io.NullWritable;

import java.io.IOException;

/**
 * 将数据写入到 Person_mr 表中
 *
 * @author 王天赐
 * @create 2019-08-02 20:12
 */
public class PersonReducer extends TableReducer<ImmutableBytesWritable, Put, ImmutableBytesWritable> {

    @Override
    protected void reduce(ImmutableBytesWritable key, Iterable<Put> values, Context context) throws IOException, InterruptedException {

        // 将 put 直接输出即可
        // put 中封装了 存入新表的数据
        for (Put value : values) {
            context.write(key, value);
        }
    }
}

Driver 类

package com.wangt.habse.mr01;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

/**
 * @author 王天赐
 * @create 2019-08-02 20:12
 */
public class PersonDriver extends Configuration implements Tool {
    // 官方建议的写法是 继承 Configuration 和 实现 Tool
    private Configuration conf = new Configuration();

    @Override
    public int run(String[] args) throws Exception {
        // 创建 Job 对象
        Job job = Job.getInstance(conf);
        // 设置 Driver class
        job.setJarByClass(PersonDriver.class);

        // 设置 Map 对象
        TableMapReduceUtil.initTableMapperJob(
                TableName.valueOf("Person"),
                new Scan(),
                PersonMapper.class,
                ImmutableBytesWritable.class,
                Put.class,
                job);

        // 设置 Reduce Class
        TableMapReduceUtil.initTableReducerJob(
                "Person_mr",
                PersonReducer.class,
                job);

        // 提交
        boolean result = job.waitForCompletion(true);
        return result ? 0 : 1;
    }

    @Override
    public void setConf(Configuration conf) {
        this.conf = conf;
    }

    @Override
    public Configuration getConf() {
        return conf;
    }

    public static void main(String[] args) throws Exception {

        Configuration configuration = HBaseConfiguration.create();
        ToolRunner.run(configuration, new PersonDriver(), args);
    }
}

案例 二

1.需求

从 hdfs 上读取 文件中的数据 写入到 Person 表

Person.csv

2001,info01,name,maomoa
2002,info01,name,maomao2
2003,info01,name,kakak
2004,info01,name,fafa

2.代码

Mapper

package com.wangt.habse.mr02;

import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

/**
 * @author 王天赐
 * @create 2019-08-04 16:20
 */
public class HDFSMapper extends Mapper<LongWritable, Text, ImmutableBytesWritable, Put> {


    @Override
    protected void map(LongWritable key, Text value, Context context)
            throws IOException, InterruptedException {
        // 2001,info01,name,maomoa
        // 对数据进行切分
        String[] fields = value.toString().split(",");
        // 获取各个字段信息 :
        byte[] rowkey = Bytes.toBytes(fields[0]);
        byte[] family = Bytes.toBytes(fields[1]);
        byte[] qualifier = Bytes.toBytes(fields[2]);
        byte[] value_ = Bytes.toBytes(fields[3]);

        // 创建 Put 对象
        Put put = new Put(rowkey);
        // 将要添加的信息封装到 Put 对象中
        put.addColumn(family, qualifier, value_);

        // 判断当 Put不为空时输出到 Reduce
        if(!put.isEmpty()){
            context.write(new ImmutableBytesWritable(rowkey), put);
        }
    }
}

Reduce

package com.wangt.habse.mr02;

import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.io.NullWritable;

import java.io.IOException;

/**
 * @author 王天赐
 * @create 2019-08-04 16:23
 */
public class HDFSReducer extends TableReducer <ImmutableBytesWritable, Put, NullWritable>{

    @Override
    protected void reduce(ImmutableBytesWritable key, Iterable<Put> values, Context context) throws IOException, InterruptedException {

        // 直接遍历输出即可
        for (Put value : values) {
            context.write(NullWritable.get(), value);
        }
    }
}

Driver

package com.wangt.habse.mr02;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

/**
 * @author 王天赐
 * @create 2019-08-04 16:24
 */
public class HDFSDriver extends Configuration implements Tool {
    // Hadoop 配置文件对象 创建 Job 对象时使用
    private Configuration conf = new Configuration();

    @Override
    public int run(String[] strings) throws Exception {

        // 创建 Job 对象
        Job job = Job.getInstance(conf);
        // 设置 Driver Class
        job.setJarByClass(HDFSDriver.class);

        // 设置 Map
        job.setMapperClass(HDFSMapper.class);
        job.setMapOutputKeyClass(ImmutableBytesWritable.class);
        job.setMapOutputValueClass(Put.class);

        // 设置 Reduce
        TableMapReduceUtil.initTableReducerJob(
                "Person",
                HDFSReducer.class,
                job);

        // 设置 输入路径
        FileInputFormat.addInputPath(job, new Path(strings[0]));
        // 提交
        boolean result = job.waitForCompletion(true);
        return result ? 0 : 1;
    }

    @Override
    public void setConf(Configuration configuration) {
        this.conf = configuration;
    }

    @Override
    public Configuration getConf() {
        return conf;
    }

    public static void main(String[] args) throws Exception {

        Configuration conf = HBaseConfiguration.create();
        int status = ToolRunner.run(conf, new HDFSDriver(), args);
        System.exit(status);
    }
}

Shell 命令

hadoop jar Day03-1.0-SNAPSHOT.jar com.wangt.habse.mr02.HDFSDriver /mr/in/person.csv