hbase删除某个列的数据

　　碰到这样一个事：我们往hbase里面导数据, 补了快一年的数据了，结果发现某个列的数据有几个月是有问题的，不能用，所以需要将这个列的有问题的几个月数据全部干掉, 查了hbase的命令，发现没有这种根据rowkey范围直接删除某个列的命令. 所以只能自己写了: 可以采用客户端编程的方式，也可以采用hbase on mr的方式，我这里采用的是hbase on mr的方式。原因是如果采用客户端编程的方式，需要scan所有的主键,然后判断rowkey是否符合删除的要求，如果符合则删除，因为数据量很大，这种方式可能太慢，其次是怕把客户端直接给弄死了。采用mr分布式的做法就不用担心这方面的问题。 

    注：

　　　　1. hbase的版本是: HBase 0.98.9

　　　　2. rowkey的形式是  userid+yyyyMMdd的形式, 比如: 0000120181103, 这里需要把20180406之前的数据的某个列( f:cl )干掉,代码如下:

　　

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.GenericOptionsParser;

import java.io.IOException;

public class HbaseDelColMr {

    static class DelColMapper extends TableMapper<Text, NullWritable> {

        private Text dekKey = new Text();

        // key: rowkey
        // result: 一行的数据
        @Override
        public void map(ImmutableBytesWritable key, Result result, Context context) throws IOException, InterruptedException {

            //拿到 rowkey
            String rowkey = Bytes.toString(key.get());

            // 判断 rowkey 是否需要删除 rowkey的类型类似这种字符串 12556565620180405
            String dateStr = rowkey.substring(rowkey.length() - 8, rowkey.length());

            //如果在20180406之前的数据全部需要删掉
            if (Integer.parseInt(dateStr) < 20180406) {
                dekKey.set(rowkey);
                context.write(dekKey, NullWritable.get());
            }
        }
    }

    static class DelColReducer extends TableReducer<Text, NullWritable, ImmutableBytesWritable> {
        @Override
        public void reduce(Text delKey, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException {
            // delKey 这就是要删除的rowkey
            Delete delete = new Delete(Bytes.toBytes(delKey.toString()));

            //设置要删除的列
            delete.deleteColumn(Bytes.toBytes("f"), Bytes.toBytes("cl"));
            context.write(new ImmutableBytesWritable(Bytes.toBytes(delKey.toString())), delete);
        }
    }

    public static void main(String[] args) throws Exception {
        Configuration configuration = HBaseConfiguration.create();
        configuration.set("hbase.zookeeper.quorum", "zk_1,zk_2,zk_3,zk_4,zk_5");
        configuration.set("hbase.zookeeper.property.clientPort", "2181");
        //configuration.set("hbase.local.dir", "/tmp/hbase-local-dir_test");

        String[] otherArgs = new GenericOptionsParser(configuration, args).getRemainingArgs();

        for (String ar:otherArgs) {
            System.out.println(ar+" ======================================");
        }

        Job job = Job.getInstance(configuration);
        job.setJobName("HbaseDelColMr");
        job.setJarByClass(HbaseDelColMr.class);

        Scan scan = new Scan();
        scan.addColumn(Bytes.toBytes("f"), Bytes.toBytes("cl"));
        scan.setCaching(500);
        scan.setCacheBlocks(false);

        TableMapReduceUtil.initTableMapperJob(
                otherArgs[0], //输入表 "dt_list_detail_test"
                scan, // scan 对象
                DelColMapper.class,
                Text.class, //mapper输出的key类型
                NullWritable.class, //mapper输出的value类型
                job
        );

        TableMapReduceUtil.initTableReducerJob(
                otherArgs[0],// 输出表 "dt_list_detail_test"
                DelColReducer.class,
                job);

        job.setNumReduceTasks(10);

        boolean b = job.waitForCompletion(true);

        if (!b) {
            throw new IOException("任务出错.....");
        }
    }
}

　　还有一种效率更高更加简便的方式, 就是去掉reduce阶段, 如下：

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.GenericOptionsParser;

import java.io.IOException;

public class HbaseDelColMr2 {

    static class DelColMapper extends TableMapper<ImmutableBytesWritable, Delete> {
        @Override
        public void map(ImmutableBytesWritable key, Result result, Context context) throws IOException, InterruptedException {

            String rowkey = Bytes.toString(key.get()); //拿到 rowkey

            // 判断 rowkey 是否需要删除 rowkey的类型类似这种字符串 12556565620180405
            String dateStr = rowkey.substring(rowkey.length() - 8, rowkey.length());

            //如果在20180406之前的数据全部需要删掉
            if (Integer.parseInt(dateStr) < 20180406) {
                //设置要删除的列
                Delete delete = new Delete(Bytes.toBytes(rowkey));
                delete.deleteColumn(Bytes.toBytes("f"), Bytes.toBytes("cl"));
                context.write(key, delete);  //需要测试如果没有reduce阶段,这里是否会直接写入到hbase, 补充:结论是可以的
            }
        }
    }

    public static void main(String[] args) throws Exception {
        Configuration configuration = HBaseConfiguration.create();
        configuration.set("hbase.zookeeper.quorum", "zk_1,zk_2,zk_3,zk_4,zk_5");
        configuration.set("hbase.zookeeper.property.clientPort", "2181");
        //configuration.set("hbase.local.dir", "/tmp/hbase-local-dir_test");

        String[] otherArgs = new GenericOptionsParser(configuration, args).getRemainingArgs();

        for (String ar:otherArgs) {
            System.out.println(ar+" ======================================");
        }

        Job job = Job.getInstance(configuration);
        job.setJobName("HbaseDelColMr2");
        job.setJarByClass(HbaseDelColMr2.class);

        Scan scan = new Scan();
        scan.addColumn(Bytes.toBytes("f"), Bytes.toBytes("cl"));
        scan.setCaching(500);
        scan.setCacheBlocks(false);

        TableMapReduceUtil.initTableMapperJob(
                otherArgs[0], //输入表 "dt_list_detail_test"
                scan, // scan 对象
                DelColMapper.class,
                null, //没有输出,直接写入hbase
                null, //没有输出,直接写入hbase
                job
        );

        TableMapReduceUtil.initTableReducerJob(
                otherArgs[0],// 输出表 "dt_list_detail_test"
                null,
                job);

        job.setNumReduceTasks(0);

        boolean b = job.waitForCompletion(true);

        if (!b) {
            throw new IOException("任务出错.....");
        }
    }
}

　　打包调用: 

　export HADOOP_CLASSPATH=`hbase classpath`

  yarn jar ./hbaseDeltest.jar xxx.HbaseDelColMr -D mapreduce.job.queuename=xxx dt_list_detail_test

　这样子就可以啦，上面两种方式随便选一种就ok了。。。。。。

转载于:https://www.cnblogs.com/wuxilc/p/9904572.html