HBase过滤器的代码实现

package hmr.jr.hbase.first;

import java.util.Iterator;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.ColumnPaginationFilter;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.FamilyFilter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
import org.apache.hadoop.hbase.filter.PageFilter;
import org.apache.hadoop.hbase.filter.QualifierFilter;
import org.apache.hadoop.hbase.filter.RegexStringComparator;
import org.apache.hadoop.hbase.filter.RowFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.filter.SubstringComparator;
import org.apache.hadoop.hbase.filter.ValueFilter;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Test;

/**
 * 测试过滤器
 * @author Administrator
 *
 */
public class TestFilter {
/**
 * 二进制对比
 * @throws Exception
 */
    @Test
    public void rowFilter() throws Exception{
        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        
        Table t = conn.getTable(TableName.valueOf("ns1:t3"));
    //    Get get = new Get(Bytes.toBytes("row1"));
        Scan scan = new Scan();
        //查询rowkey <='row6'
        scan.setFilter(new RowFilter(CompareFilter.CompareOp.LESS_OR_EQUAL,new BinaryComparator(Bytes.toBytes("row6"))));
        ResultScanner scanner = t.getScanner(scan);
        Iterator<Result> it = scanner.iterator();
        while(it.hasNext()){
            Result r = it.next();
        System.out.println(toStr(r,"f1","id"));
        System.out.println(toStr(r,"f1","name"));
        System.out.println(toStr(r,"f1","age"));
        System.out.println(toStr(r,"f2","id"));
        System.out.println(toStr(r,"f2","name"));
        System.out.println(toStr(r,"f2","age"));
        }
    }
    
/**
 * 正则串对比器
 * @throws Exception
 */
    @Test
    public void rowFilterRegex() throws Exception{
        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        
        Table t = conn.getTable(TableName.valueOf("ns1:t3"));
    //    Get get = new Get(Bytes.toBytes("row1"));
        Scan scan = new Scan();
        //行过滤器，按rowkey过滤，查询是1或2或3的rowkey
        scan.setFilter(new RowFilter(CompareFilter.CompareOp.EQUAL,new RegexStringComparator(".*[123]")));
        ResultScanner scanner = t.getScanner(scan);
        Iterator<Result> it = scanner.iterator();
        while(it.hasNext()){
            Result r = it.next();
        System.out.println(toStr(r,"f1","id"));
        System.out.println(toStr(r,"f1","name"));
        System.out.println(toStr(r,"f1","age"));
        System.out.println(toStr(r,"f2","id"));
        System.out.println(toStr(r,"f2","name"));
        System.out.println(toStr(r,"f2","age"));
        }
    }
        
/**
 * 子串对比器
 * @throws Exception
 */
    @Test
    public void rowFilterSubstring() throws Exception{
        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        
        Table t = conn.getTable(TableName.valueOf("ns1:t3"));
    //    Get get = new Get(Bytes.toBytes("row1"));
        Scan scan = new Scan();
        //行过滤器，按rowkey过滤，查询是1或2或3的rowkey
        scan.setFilter(new RowFilter(CompareFilter.CompareOp.EQUAL,new SubstringComparator("w5")));
        ResultScanner scanner = t.getScanner(scan);
        Iterator<Result> it = scanner.iterator();
        while(it.hasNext()){
            Result r = it.next();
        System.out.println(toStr(r,"f1","id"));
        System.out.println(toStr(r,"f1","name"));
        System.out.println(toStr(r,"f1","age"));
        System.out.println(toStr(r,"f2","id"));
        System.out.println(toStr(r,"f2","name"));
        System.out.println(toStr(r,"f2","age"));
        }
    }        

/**
 * 列族过滤器
 * @throws Exception
 */
    @Test
    public void familyFilterSubstring() throws Exception{
        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        
        Table t = conn.getTable(TableName.valueOf("ns1:t3"));
    //    Get get = new Get(Bytes.toBytes("row1"));
        Scan scan = new Scan();
        //列族过滤器
        scan.setFilter(new FamilyFilter(CompareFilter.CompareOp.EQUAL,new BinaryComparator(Bytes.toBytes("f1"))));
        ResultScanner scanner = t.getScanner(scan);
        Iterator<Result> it = scanner.iterator();
        while(it.hasNext()){
            Result r = it.next();
        System.out.println(toStr(r,"f1","id"));
        System.out.println(toStr(r,"f1","name"));
        System.out.println(toStr(r,"f1","age"));
        System.out.println(toStr(r,"f2","id"));
        System.out.println(toStr(r,"f2","name"));
        System.out.println(toStr(r,"f2","age"));
        }
    }        
/**
 * 列过滤器  Qualifier
 * @throws Exception
 */
    @Test
    public void QualifierFilterSubstring() throws Exception{
        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        
        Table t = conn.getTable(TableName.valueOf("ns1:t3"));
    //    Get get = new Get(Bytes.toBytes("row1"));
        Scan scan = new Scan();
        //列族过滤器
        scan.setFilter(new QualifierFilter(CompareFilter.CompareOp.EQUAL,new BinaryComparator(Bytes.toBytes("name"))));
        ResultScanner scanner = t.getScanner(scan);
        Iterator<Result> it = scanner.iterator();
        while(it.hasNext()){
            Result r = it.next();
        System.out.println(toStr(r,"f1","id"));
        System.out.println(toStr(r,"f1","name"));
        System.out.println(toStr(r,"f1","age"));
        System.out.println(toStr(r,"f2","id"));
        System.out.println(toStr(r,"f2","name"));
        System.out.println(toStr(r,"f2","age"));
        }
    }    
    
/**
 * 值过滤器  
 * @throws Exception
 */
    @Test
    public void ValueFilterSubstring() throws Exception{
        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        
        Table t = conn.getTable(TableName.valueOf("ns1:t3"));
        Scan scan = new Scan();
        //列族过滤器
        scan.setFilter(new ValueFilter(CompareFilter.CompareOp.EQUAL,new RegexStringComparator("^tom.*")));
        ResultScanner scanner = t.getScanner(scan);
        Iterator<Result> it = scanner.iterator();
        while(it.hasNext()){
            Result r = it.next();
        System.out.println(toStr(r,"f1","id"));
        System.out.println(toStr(r,"f1","name"));
        System.out.println(toStr(r,"f1","age"));
        System.out.println(toStr(r,"f2","id"));
        System.out.println(toStr(r,"f2","name"));
        System.out.println(toStr(r,"f2","age"));
        }
    }                        
            
    /**
     * 单列值过滤器  取一行值
     * @throws Exception
     */
        @Test
        public void singleColValFilter() throws Exception{
            Configuration conf = HBaseConfiguration.create();
            Connection conn = ConnectionFactory.createConnection(conf);
            
            Table t = conn.getTable(TableName.valueOf("ns1:t3"));
            Scan scan = new Scan();
            SingleColumnValueFilter f=new SingleColumnValueFilter(Bytes.toBytes("f1"),Bytes.toBytes("name"),CompareFilter.CompareOp.EQUAL,Bytes.toBytes("tom2"));
            //排它性，把自己排除掉再显示剩下的一行值
            //SingleColumnValueFilter f1=new SingleColumnValueExcludeFilter(Bytes.toBytes("f1"),Bytes.toBytes("name"),CompareFilter.CompareOp.EQUAL,Bytes.toBytes("tom2"));
            scan.setFilter(f);
            ResultScanner scanner = t.getScanner(scan);
            Iterator<Result> it = scanner.iterator();
            while(it.hasNext()){
                Result r = it.next();
            System.out.println(toStr(r,"f1","id"));
            System.out.println(toStr(r,"f1","name"));
            System.out.println(toStr(r,"f1","age"));
            System.out.println(toStr(r,"f2","id"));
            System.out.println(toStr(r,"f2","name"));
            System.out.println(toStr(r,"f2","age"));
            }
        }
        
/**
 * 分页过滤器   类似于limit
 * @throws Exception
 */
    @Test
    public void pageFilter() throws Exception{
        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        
        Table t = conn.getTable(TableName.valueOf("ns1:t3"));
        Scan scan = new Scan();
        
        PageFilter p= new PageFilter(5);
        
        scan.setFilter(p);
        ResultScanner scanner = t.getScanner(scan);
        Iterator<Result> it = scanner.iterator();
        while(it.hasNext()){
            Result r = it.next();
        System.out.println(toStr(r,"f1","id"));
        System.out.println(toStr(r,"f1","name"));
        System.out.println(toStr(r,"f1","age"));
        System.out.println(toStr(r,"f2","id"));
        System.out.println(toStr(r,"f2","name"));
        System.out.println(toStr(r,"f2","age"));
        }
    }
    
/**
 * 只返回key,不返回value
 * @throws Exception
 */
    @Test
    public void keyOnlyFilter() throws Exception{
        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        
        Table t = conn.getTable(TableName.valueOf("ns1:t3"));
        Scan scan = new Scan();
        KeyOnlyFilter k= new KeyOnlyFilter();
        scan.setFilter(k);
        ResultScanner scanner = t.getScanner(scan);
        Iterator<Result> it = scanner.iterator();
        while(it.hasNext()){
            Result r = it.next();
        System.out.println(toStr(r,"f1","id"));
        System.out.println(toStr(r,"f1","name"));
        System.out.println(toStr(r,"f1","age"));
        System.out.println(toStr(r,"f2","id"));
        System.out.println(toStr(r,"f2","name"));
        System.out.println(toStr(r,"f2","age"));
        }
    }
    
/**
 * 列分页过滤器，类似于limit
 * @throws Exception
 */
    @Test
    public void columnPageFilter() throws Exception{
        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        
        Table t = conn.getTable(TableName.valueOf("ns1:t3"));
        Scan scan = new Scan();
        //一行显示两列，起始列的位置，表示从第1列开始读。
        ColumnPaginationFilter k= new ColumnPaginationFilter(2,1);
        scan.setFilter(k);
        ResultScanner scanner = t.getScanner(scan);
        Iterator<Result> it = scanner.iterator();
        while(it.hasNext()){
            Result r = it.next();
        System.out.println(toStr(r,"f1","id"));
        System.out.println(toStr(r,"f1","name"));
        System.out.println(toStr(r,"f1","age"));
        System.out.println(toStr(r,"f2","id"));
        System.out.println(toStr(r,"f2","name"));
        System.out.println(toStr(r,"f2","age"));
        }
    }
    
/**
 * 组合过滤器链，FilterList
 * @throws Exception
 */
    @Test
    public void compositeFilter() throws Exception{
        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        
        Table t = conn.getTable(TableName.valueOf("ns1:t3"));
        Scan scan = new Scan();
        //where ((name like 't%') and (age < 20))
        //                   or
        //      ((name like '%t') and (age >=20))
        
        FilterList list = new FilterList(FilterList.Operator.MUST_PASS_ALL);
        //name like 't%'
        SingleColumnValueFilter ftl = new SingleColumnValueFilter(bytes("f1"),bytes("name"),CompareFilter.CompareOp.EQUAL,new RegexStringComparator("^t"));
        //age < 20
        SingleColumnValueFilter ftr = new SingleColumnValueFilter(bytes("f1"),bytes("age"),CompareFilter.CompareOp.LESS,new BinaryComparator(Bytes.toBytes(20)));
        //name like 't%' and age < 20
        FilterList ft = new FilterList(FilterList.Operator.MUST_PASS_ALL,ftl,ftr);
        
        //name like '%t'
        SingleColumnValueFilter fbl = new SingleColumnValueFilter(bytes("f1"),bytes("name"),CompareFilter.CompareOp.EQUAL,new RegexStringComparator("t$"));
        //age >= 20
        SingleColumnValueFilter fbr = new SingleColumnValueFilter(bytes("f1"),bytes("age"),CompareFilter.CompareOp.GREATER_OR_EQUAL,new BinaryComparator(Bytes.toBytes(20)));
        //name like '%t' and age >= 20
        FilterList fb = new FilterList(FilterList.Operator.MUST_PASS_ALL,fbl,fbr);
        
        FilterList fall =  new FilterList(FilterList.Operator.MUST_PASS_ONE,ft,fb);
        scan.setFilter(fall);
        ResultScanner scanner = t.getScanner(scan);
        Iterator<Result> it = scanner.iterator();
        while(it.hasNext()){
            Result r = it.next();
        System.out.println(toStr(r,"f1","id"));
        System.out.println(toStr(r,"f1","name"));
        System.out.println(toStr(r,"f1","age"));
        System.out.println(toStr(r,"f2","id"));
        System.out.println(toStr(r,"f2","name"));
        System.out.println(toStr(r,"f2","age"));
        }
    }
        
    private byte[] bytes(String str){
        return Bytes.toBytes(str);
    }
        
    private String toStr(Result r,String f1,String c1){
        Cell cell = r.getColumnLatestCell(Bytes.toBytes(f1), Bytes.toBytes(c1));
        if(cell != null){
            String row=Bytes.toString(cell.getRow());
            return new String (row+"/"+f1+":"+c1+"="+Bytes.toString(cell.getValue()));
        }
        return new String("/"+f1+":"+c1+"=null ");
    }
}