MR--HBase

1.配置环境

1.配置环境变量

sudo vim /etc/profile

在其中添加：

export HADOOP_CLASSPATH=`$HBASE_HOME/bin/hbase mapredcp`

2.刷新环境变量文件

source /etc/profile

3.尝试运行官方案例
官方案例在lib文件夹中

yarn jar hbase-server-1.3.1.jar rowcounter fiudd

统计有多少行数据
4.尝试运行官方案例2
首先将数据上传到hdfs上
在hbase上建立一个表
create ‘t1’,‘info’
yarn jar hbase-server-1.3.1.jar importtsv -Dimporttsv.co lumns=HBASE_ROW_KEY,into:name,into:color fiudd hdfs://linuxzhb8:9000/文件名

2.用MR清洗HBase的数据在导入hbase的另一个表里

1.Mapper阶段

//TableMapper固定了inputkey和value的值为<ImmutableBytesWritable,Result>ImmutableBytesWritable存放的是行号，Result存放一行的内容
public class HBase_Mapper extends TableMapper<ImmutableBytesWritable,Put>{
    @Override
    protected void map(ImmutableBytesWritable key, Result value, Context context) throws IOException, InterruptedException {
        //创建一个Put对象用来存放要增加数据的信息
        Put put = new Put(key.get());
        //因为value是Result类型所有它有listCells方法可以返回一行
        List<Cell> cells = value.listCells();
        //因为一行可以有多个列限制对象所有需要循环
        for (Cell cell:cells){
            //获取列限定的符的内容转换成字符串
            String s = Bytes.toString(CellUtil.cloneQualifier(cell));
            //和name对比，如果相同，输出
            if ("name".equals(s)){
                //将这一行的信息放入put对象中
                put.addColumn(CellUtil.cloneFamily(cell),CellUtil.cloneQualifier(cell),CellUtil.cloneValue(cell));
                context.write(key,put);
            }
        }
    }
}

2.Reducer阶段

public class HBase_Reducer extends TableReducer<ImmutableBytesWritable,Put,NullWritable>{
    @Override
    protected void reduce(ImmutableBytesWritable key, Iterable<Put> values, Context context) throws IOException, InterruptedException {
        //循环遍历Put的值，输出
        for (Put row:values){
            context.write(NullWritable.get(),row);
        }
    }
}

3.Driver阶段

public class HBase_Driver extends Configuration implements Tool {
    //定义一个Configuration对象
    Configuration conf = null;

    public static void main(String[] args) throws Exception {
        //创建一个Hbase的configuration对象获取Hbase的配置文件
        Configuration configuration = HBaseConfiguration.create();
        //configuration.set("hbase.zookeeper.quorum","192.168.245.139");
        //将配置文件传给this.conf，并执行run方法
        ToolRunner.run(configuration,new HBase_Driver(),args);
    }
    @Override
    public int run(String[] args) throws Exception {
        Job job = Job.getInstance(conf);
        job.setJarByClass(HBase_Driver.class);
        //mapper提交信息
        TableMapReduceUtil.initTableMapperJob("fiudd",new Scan(),HBase_Mapper.class,ImmutableBytesWritable.class, Put.class,job);
        //reducer提交信息
        TableMapReduceUtil.initTableReducerJob("fiudd_mr",HBase_Reducer.class,job);
        //job提交任务
        return job.waitForCompletion(true)?0:1;
    }

    @Override
    //将配置文件传给this.conf
    public void setConf(Configuration conf) {
        this.conf=conf;
    }

    @Override
    public Configuration getConf() {
        return conf;
    }
}

3.用MR将Hdfs上的数据导入到hbase的表中

1. Mapper阶段

public class HdfsTotableMapper extends Mapper<LongWritable,Text,ImmutableBytesWritable,Put> {
    ImmutableBytesWritable k = new ImmutableBytesWritable();
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String[] split = value.toString().split("\t");
        //创建Put对象，并将行号传入
        Put put = new Put(split[0].getBytes());
        //将行号传到key中
        k.set(split[0].getBytes());
        //放数据
        put.addColumn(Bytes.toBytes("into"),Bytes.toBytes("name"),Bytes.toBytes(split[1]));
        put.addColumn(Bytes.toBytes("into"),Bytes.toBytes("color"),Bytes.toBytes(split[2]));
        //传到reduce
        context.write(k,put);
    }
}

2.Reducer阶段

public class HdfsTotableReducer extends TableReducer<ImmutableBytesWritable,Put,NullWritable> {
    @Override
    protected void reduce(ImmutableBytesWritable key, Iterable<Put> values, Context context) throws IOException, InterruptedException {
        //循环提交
        for (Put put:values){
            context.write(NullWritable.get(),put);
        }
    }
}

3.Driver阶段

public class HdfsTotableDriver extends Configuration implements Tool {
    Configuration conf = null;
    public static void main(String[] args) throws Exception {
        Configuration configuration = HBaseConfiguration.create();
        configuration.set("hbase.zookeeper.quorum","192.168.245.139");
        ToolRunner.run(configuration,new HdfsTotableDriver(),args);
    }
    @Override
    public int run(String[] args) throws Exception {
        Job job = Job.getInstance(conf);
        //因为map继承的是mapper所有用传统的job
        job.setMapOutputKeyClass(ImmutableBytesWritable.class);
        job.setMapOutputValueClass(Put.class);
        job.setMapperClass(HdfsTotableMapper.class);
        TableMapReduceUtil.initTableReducerJob("fiudd",HdfsTotableReducer.class,job);
        FileInputFormat.setInputPaths(job,new Path("hdfs://192.168.245.139:9000/fruit.tsv"));
        return job.waitForCompletion(true)?0:1;
    }

    @Override
    public void setConf(Configuration conf) {
        this.conf=conf;
    }

    @Override
    public Configuration getConf() {
        return conf;
    }
}

4.用MR将在HBase上的数据导出到hdfs上

1.Mapper阶段

public class TableToHdfsMapper extends TableMapper<Text,Text> {
    Text k = new Text();
    @Override
    protected void map(ImmutableBytesWritable key, Result value, Context context) throws IOException, InterruptedException {
        List<Cell> cells = value.listCells();
        for (Cell cell:cells){
            k.set(new String(CellUtil.cloneRow(cell)));
            String s = new String(CellUtil.cloneFamily(cell));
            String s1 = new String(CellUtil.cloneQualifier(cell));
            String s2 = new String(CellUtil.cloneValue(cell));
            String s3 = new String(CellUtil.cloneRow(cell));
            String all=s3+"\t"+s+"\t"+s1+"\t"+s2;
            context.write(k,new Text(all));
        }
    }
}

2.Reducer阶段

public class TableToHdfsReducer extends Reducer<Text,Text,Text,NullWritable> {
    @Override
    protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
        for (Text row : values){
            context.write(row,NullWritable.get());
        }
    }
}

3.Driver阶段

public class TableToHdfsDriver extends Configuration implements Tool {
    Configuration conf = null;
    public static void main(String[] args) throws Exception {
        Configuration configuration = HBaseConfiguration.create();
        configuration.set("hbase.zookeeper.quorum","192.168.245.139");
        ToolRunner.run(configuration,new TableToHdfsDriver(),args);
    }
    @Override
    public int run(String[] args) throws Exception {
        Job job = Job.getInstance(conf);
        TableMapReduceUtil.initTableMapperJob("fiudd",new Scan(),TableToHdfsMapper.class, Text.class, Text.class,job);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(NullWritable.class);
        job.setReducerClass(TableToHdfsReducer.class);
        FileOutputFormat.setOutputPath(job,new Path("hdfs://192.168.245.139:9000/out/hbaseout"));
        return job.waitForCompletion(true)?0:1;
    }

    @Override
    public void setConf(Configuration conf) {
        this.conf=conf;
    }

    @Override
    public Configuration getConf() {
        return conf;
    }
}