【HBase】自定义 HBase-MapReduce1

目标

实现将 HDFS 中的数据写入到 Hbase 表中。

分步实现

1．构建 FruitMapper 类，读取 HDFS 中fruit表的数据

package com.chris.hbase.mr1;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class FruitMapper extends Mapper<LongWritable, Text,LongWritable,Text> {
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

        context.write(key,value);
    }
}

2． 构建 FruitReducer 类，用于将读取到的 fruit 表中的数据写入到 fruit1 表中

package com.chris.hbase.mr1;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class FruitReducer extends TableReducer<LongWritable, Text, NullWritable> {

//    private String cf1 = null;
//
//    @Override
//    protected void setup(Reducer<LongWritable, Text, NullWritable, Mutation>.Context context) throws IOException, InterruptedException {
//
//        Configuration configuration = context.getConfiguration();
//        cf1 = configuration.get("cf1");
//    }

    @Override
    protected void reduce(LongWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
        // 1.遍历values
        for (Text value : values) {
            // 2.获取每一行数据
            String[] fields = value.toString().split("\t");
            // 3.构造put对象 (传入rowKey）
            Put put = new Put(Bytes.toBytes(fields[0]));
            // 4.给put对象赋值
            put.addColumn(Bytes.toBytes("info"),Bytes.toBytes("name"),Bytes.toBytes(fields[1]));
            put.addColumn(Bytes.toBytes("info"),Bytes.toBytes("color"),Bytes.toBytes(fields[2]));
            // 5.写出
            context.write(NullWritable.get(),put);
        }
    }
}

3.    构建 驱动类Driver

package com.chris.hbase.mr1;

import com.sun.tracing.dtrace.ArgsAttributes;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

public class FruitDriver implements Tool {
    // 定义一个configuration
    private Configuration configuration = null;
    @Override
    public int run(String[] args) throws Exception {
        // 1. 获取job对象
        Job job = Job.getInstance(configuration);
        // 2. 设置驱动类路径
        job.setJarByClass(FruitDriver.class);
        // 3.设置Mapper输出的KV类型
        job.setMapperClass(FruitMapper.class);
        job.setMapOutputKeyClass(LongWritable.class);
        job.setMapOutputValueClass(Text.class);
        // 4.设置Reducer类
        TableMapReduceUtil.initTableReducerJob(args[1],
                FruitReducer.class,job);
        // 5.设置输入参数
        FileInputFormat.setInputPaths(job,new Path(args[0]));
        // 6.提交任务
        boolean result = job.waitForCompletion(true);
        return result?0:1;
    }

    @Override
    public void setConf(Configuration conf) {
        configuration = conf;
    }

    @Override
    public Configuration getConf() {
        return configuration;
    }

    public static void main(String[] args) {
        try {
            Configuration configuration = new Configuration();
            int run = ToolRunner.run(configuration, new FruitDriver(), args);
            System.exit(run);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

4. 在hbase端口创建fruit1表：

5. 打jar包，将jar包拖到hbase目录下，运行以下命令：

# com.chris...是主类名，/fruit.tsv是HDFS上的表，fruit1是hbase中要传入数据的表
yarn jar hbase_demo-1.0-SNAPSHOT.jar com.chris.hbase.mr1.FruitDriver /fruit.tsv fruit1

 6. 查询结果