HBase过滤器实现表复杂查询

HBase过滤器实现表复杂查询

一、引入maven依赖

    <dependencies>
        <dependency>
            <groupId>org.apache.hbase</groupId>
            <artifactId>hbase-client</artifactId>
            <version>1.2.3</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/junit/junit -->
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
        </dependency>

    </dependencies>

二、代码实现

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.filter.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Test;

import java.io.IOException;
import java.util.Iterator;
import java.util.Map;
import java.util.NavigableMap;


/**
 * @Author      王磊
 * @Date        2019/1/6
 * @ClassName   HBaseFilterDemo
 * @Description HBase条件查询demo
 **/
public class HBaseFilterDemo {
    /**
     * 通过表结果转化的迭代器对象将表查询详细信息进行打印
     * @param it
     */
    public static void getIteratorDetailInfo(Iterator<Result> it){
        while (it.hasNext()){
            Result rows = it.next();
            NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> row = rows.getMap();
            for(Map.Entry<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> row_ : row.entrySet()){
                System.out.println("family:" + Bytes.toString(row_.getKey()));
                NavigableMap<byte[], NavigableMap<Long, byte[]>> cols = row_.getValue();
                for(Map.Entry<byte[], NavigableMap<Long, byte[]>> cols_ : cols.entrySet()){
                    System.out.println("col:" + Bytes.toString(cols_.getKey()));
                    NavigableMap<Long, byte[]> fields = cols_.getValue();
                    for(Map.Entry<Long, byte[]> fields_ : fields.entrySet()){
                        System.out.println("timeStamp:" + fields_.getKey());
                        System.out.println("value" + Bytes.toString(fields_.getValue()));
                    }
                }
            }
        }
    }

    /**
     * 行过滤器
     */
    @Test
    public static void RowFilterMethod() throws IOException {
        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        Table tb = conn.getTable(TableName.valueOf("nameSpaceName:tableName"));
        Scan scan = new Scan();
        // 行过滤器，设置过滤rowkey小于row20的扫描行数据
        RowFilter rowFilter = new RowFilter(CompareFilter.CompareOp.LESS, new BinaryComparator(Bytes.toBytes("row20")));
        scan.setFilter(rowFilter);
        ResultScanner rs = tb.getScanner(scan);
        Iterator<Result> it = rs.iterator();
        while (it.hasNext()){
            Result row = it.next();
            String rowName = Bytes.toString(row.getRow());
            System.out.println(rowName);
        }
    }

    /**
     * 列簇过滤器
     * @param familyName 查询列簇名称
     * @throws IOException
     */
    @Test
    public static void testFamilyFilter(String familyName) throws IOException {
        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        Table tb = conn.getTable(TableName.valueOf("nameSpaceName:tableName"));
        Scan scan = new Scan();
        FamilyFilter ff = new FamilyFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(familyName)));
        scan.setFilter(ff);
        ResultScanner rs = tb.getScanner(scan);
        Iterator<Result> it = rs.iterator();
        getIteratorDetailInfo(it);
    }

    /**
     * 列过滤器
     * @param tableName
     * @param qualiName
     * @throws IOException
     */
    @Test
    public static void testQualifierFilter(String nameSpaceName, String tableName, String qualiName) throws IOException {
        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        Table tb = conn.getTable(TableName.valueOf(nameSpaceName + ":" + tableName));
        QualifierFilter qf = new QualifierFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(qualiName)));
        Scan scan = new Scan();
        scan.setFilter(qf);
        ResultScanner rs = tb.getScanner(scan);
        Iterator<Result> it = rs.iterator();
        getIteratorDetailInfo(it);
    }

    /**
     * 值过滤器(任何列)
     * @param nameSpaceName
     * @param tableName
     * @param value
     * @throws IOException
     */
    @Test
    public static void testValueFilter(String nameSpaceName, String tableName, String value) throws IOException {
        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        Table tb = conn.getTable(TableName.valueOf(nameSpaceName + ":" + tableName));
        ValueFilter vf = new ValueFilter(
                CompareFilter.CompareOp.EQUAL,
                new BinaryComparator(Bytes.toBytes(value))
        );
        Scan scan = new Scan();
        scan.setFilter(vf);
        getIteratorDetailInfo(tb.getScanner(scan).iterator());
    }

    /**
     * 依赖过滤器
     * @throws IOException
     */
    @Test
    public static void testDependentColumnFilter() throws IOException {
        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        Table tb = conn.getTable(TableName.valueOf("nameSpaceName:tableName"));
        DependentColumnFilter dcf = new DependentColumnFilter(
                Bytes.toBytes("familyName"),
                Bytes.toBytes("qualifierName"),
                false
        );
        Scan scan = new Scan();
        scan.setFilter(dcf);
        getIteratorDetailInfo(tb.getScanner(scan).iterator());
    }

    /**
     * 单列值过滤(value不满足，整行过滤)
     * 查询以列名为qualifierName值以hi开头的数据（整行）,要返回qualifierName字段
     * @throws IOException
     */
    @Test
    public static void testSingleColumnValueFilter() throws IOException {
        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        Table tb = conn.getTable(TableName.valueOf("nameSpaceName:tableName"));
        SingleColumnValueFilter scvf = new SingleColumnValueFilter(
                Bytes.toBytes("familyName"),
                Bytes.toBytes("qualifierName"),
                CompareFilter.CompareOp.EQUAL,
                new RegexStringComparator("^hi")
        );
        Scan scan = new Scan();
        scan.setFilter(scvf);
        getIteratorDetailInfo(tb.getScanner(scan).iterator());
    }

    /**
     * 单列值过滤(value不满足，整行过滤)
     * 查询以列名为qualifierName值以hi开头的数据（整行）,不返回qualifierName字段
     */
    @Test
    public static void testSingleColumnValueExcludeFilter() throws IOException {
        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        Table tb = conn.getTable(TableName.valueOf("nameSpaceName:tableName"));
        SingleColumnValueExcludeFilter scvef = new SingleColumnValueExcludeFilter(
                Bytes.toBytes("familyName"),
                Bytes.toBytes("qualifierName"),
                CompareFilter.CompareOp.EQUAL,
                new RegexStringComparator("^hi")
        );
        Scan scan = new Scan();
        scan.setFilter(scvef);
        getIteratorDetailInfo(tb.getScanner(scan).iterator());
    }

    /**
     * rowkey前缀过滤器
     * @param prefix rowkey的前缀
     * @throws IOException
     */
    @Test
    public static void testPrefixFilter(String prefix) throws IOException {
        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        Table tb = conn.getTable(TableName.valueOf("nameSpaceName:tableName"));
        PrefixFilter pf = new PrefixFilter(Bytes.toBytes(prefix));
        Scan scan = new Scan();
        scan.setFilter(pf);
        getIteratorDetailInfo(tb.getScanner(scan).iterator());
    }

    /**
     * 分页过滤器
     * 返回的数据为表的区域数与pageSize的乘积数
     * @param pageSize
     * @throws IOException
     */
    @Test
    public static void testPageFilter(Long pageSize) throws IOException {
        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        Table tb = conn.getTable(TableName.valueOf("nameSpaceName:tableName"));
        PageFilter pf = new PageFilter(pageSize);
        Scan scan = new Scan();
        scan.setFilter(pf);
        getIteratorDetailInfo(tb.getScanner(scan).iterator());
    }

    /**
     * key值过滤器
     * 只提取Key，不提取value（定位坐标【rowKey、列簇、列】）
     */
    @Test
    public static void testKeyOnlyFilter() throws IOException {
        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        Table tb = conn.getTable(TableName.valueOf("nameSpaceName:tableName"));
        KeyOnlyFilter kof = new KeyOnlyFilter();
        Scan scan = new Scan();
        scan.setFilter(kof);
        getIteratorDetailInfo(tb.getScanner(scan).iterator());
    }

    /**
     * 多条件查询
     * @throws IOException
     * 此处测试语句：select * from tableName where id > 5 and name like 'hi' or id < 3 and name like '%hi';
     */
    @Test
    public static void testMultiListFilter() throws IOException {
        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        Table tb = conn.getTable(TableName.valueOf("nameSpaceName:tableName"));

        // 前半段
        SingleColumnValueFilter scvf1 = new SingleColumnValueFilter(
                Bytes.toBytes("familyName"),
                Bytes.toBytes("id"),
                CompareFilter.CompareOp.GREATER,
                new BinaryComparator(Bytes.toBytes(5))
        );
        SingleColumnValueFilter scvf2 = new SingleColumnValueFilter(
                Bytes.toBytes("familyName"),
                Bytes.toBytes("name"),
                CompareFilter.CompareOp.EQUAL,
                new RegexStringComparator("^hi")
        );
        FilterList fl1 = new FilterList();
        fl1.addFilter(scvf1);
        fl1.addFilter(scvf2);
        // 后半段
        SingleColumnValueFilter scvf3 = new SingleColumnValueFilter(
                Bytes.toBytes("familyName"),
                Bytes.toBytes("id"),
                CompareFilter.CompareOp.LESS,
                new BinaryComparator(Bytes.toBytes(3))
        );
        SingleColumnValueFilter scvf4 = new SingleColumnValueFilter(
                Bytes.toBytes("familyName"),
                Bytes.toBytes("name"),
                CompareFilter.CompareOp.EQUAL,
                new RegexStringComparator("hi$")
        );
        FilterList fl2 = new FilterList();
        fl2.addFilter(scvf3);
        fl2.addFilter(scvf4);

        // 合并
        FilterList fl = new FilterList();
        fl.addFilter(fl1);
        fl.addFilter(fl2);

        Scan scan = new Scan();
        ResultScanner resultScanner = tb.getScanner(scan);
        getIteratorDetailInfo(resultScanner.iterator());
    }
}