Java API连接HBase

boyzwz

已于 2022-06-12 15:11:28 修改

阅读量1.3k

点赞数

文章标签： hbase java hadoop

于 2022-06-10 21:28:00 首次发布

本文链接：https://blog.csdn.net/zwyoozwz/article/details/125228420

版权

导入依赖

        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hbase</groupId>
            <artifactId>hbase-client</artifactId>
            <version>1.7.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.7.6</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>2.7.6</version>
        </dependency>

相关操作

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

public class ToHbase {
    private HConnection conn;
    private HBaseAdmin hAdmin;

    @Before
    public void connect() {
        try {
            //获取Hadoop相关配置
            Configuration conf = new Configuration();
            //获取zookeeper配置
            conf.set("hbase.zookeeper.quorum", "master:2181,node1:2181,node2:2181");
            //获取与hbase的连接
            conn = HConnectionManager.createConnection(conf);
            //HMaster
            hAdmin = new HBaseAdmin(conf);
            System.out.println("连接：" + conn + "-HMaster:" + hAdmin);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    @Test   //建表
    public void createTable() {
        try {
            //创建表对象
            HTableDescriptor students = new HTableDescriptor("students");
            //创建表，至少指定一个列簇
            //列簇描述器
            HColumnDescriptor info = new HColumnDescriptor("info");
            //将列簇添加到表中
            students.addFamily(info);
            //创建表执行是由HMaster
            hAdmin.createTable(students);
            System.out.println(students.getNameAsString() + "表创建成功");
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    @Test   //插入数据
    public void putData() {
        try {
            //通过表名，获取表的实例
            HTableInterface students = conn.getTable("students");
            //创建插入数据的实例，并添加行键
            Put put = new Put("1001".getBytes());
            //为行键对应的值，添加列簇、列名和具体的值
            put.add("info".getBytes(), "name".getBytes(), "xiaohei".getBytes());
            //表的实例将表的数据添加到表中
            students.put(put);
            System.out.println(students.getName().toString() + "成功插入数据");
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    @Test   //获取表中的一条数据
    public void getData() {
        try {
            //通过表名获取表实例
            HTableInterface students = conn.getTable("students");
            //创建获取数据的实例,并添加行键
            Get get = new Get("1500100018".getBytes());
            //表的实例get方法获取结果
            Result result = students.get(get);
            //获取行键
            String rk = Bytes.toString(get.getRow());
            //通过结果获取值
            String s = Bytes.toString(result.getValue("info".getBytes(), "name".getBytes()));
            System.out.println(rk + ":" + s);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    @Test   //删除表
    public void deleteTable() {
        try {
            //先检查改表是否存在
            if (hAdmin.tableExists("students")) {
                //删除表前，需将表禁用
                hAdmin.disableTable("students");
                //删除表
                hAdmin.deleteTable("students");
                System.out.println(conn.getTable("students").getName().toString() + "表成功删除");
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    @Test   //插入多行数据
    public void putAll() {
        try {
            //通过表名，获取表的实例对象
            HTableInterface students = conn.getTable("students");
            BufferedReader br = new BufferedReader(new FileReader("F:\\soft\\IDEAprojects\\hadoop-bigdata\\hadoop-hbase\\students.txt"));
            //也可创建一个集合，一次性插入数据
            ArrayList<Put> list = new ArrayList<>();
            String line = null;
            while ((line = br.readLine()) != null) {
                String[] split = line.split(",");
                //为插入数据创建实例，并添加行键，以第一列为行键
                Put put = new Put(split[0].getBytes());
                //插入数据
                put.add("info".getBytes(), "name".getBytes(), split[1].getBytes());
                put.add("info".getBytes(), "age".getBytes(), split[2].getBytes());
                put.add("info".getBytes(), "gender".getBytes(), split[3].getBytes());
                put.add("info".getBytes(), "clazz".getBytes(), split[4].getBytes());
                //将每一行数据对应的put实例，添加到集合中
                list.add(put);
                //将每一行数据插入
                //students.put(put);
            }
            //将集合中的数据插入表中
            students.put(list);
            System.out.println(students.getName().toString() + "表数据插入成功");
            br.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    @Test   //scan扫描查询数据
    public void dataQuery() {
        try {
            //通过表名获取表的实例对象
            HTableInterface students = conn.getTable("students");
            //使用scan扫描数据
            Scan scan = new Scan();
            //若不需要全局扫描，可设定开始和结束行[start,end)
            scan.setStartRow("1500100001".getBytes());
            scan.setStopRow("1500100011".getBytes());
            //将scan加入表中
            ResultScanner scanner = students.getScanner(scan);
            //调用ResultScanner中的next()方法，next()是指向下一个数据的索引。有点类似于我之前迭代器的指针
            Result rs = null;
            while ((rs = scanner.next()) != null) {
                //若知道表的列簇和列名，结果可以通过列簇、列名获取表中的值
//                String name = Bytes.toString(rs.getValue("info".getBytes(), "name".getBytes()));
//                String age = Bytes.toString(rs.getValue("info".getBytes(), "age".getBytes()));
//                String gender = Bytes.toString(rs.getValue("info".getBytes(), "gender".getBytes()));
//                String clazz = Bytes.toString(rs.getValue("info".getBytes(), "clazz".getBytes()));
                //获取行键
                String rk = Bytes.toString(rs.getRow());
                System.out.print(rk);
                System.out.print("\t");
//                System.out.println(rk + ":" + "name:" + name + ",age:" + age + ",gender:" + gender + ",clazz:" + clazz);

                //2、也可以获取一行中所有单元格集合
                List<Cell> cells = rs.listCells();
                for (Cell cell : cells) {
                    //获得列名
                    String col = Bytes.toString(CellUtil.cloneQualifier(cell));
                    //获取列值
                    String s = Bytes.toString(CellUtil.cloneValue(cell));
                    System.out.print(col + ":" + s);
                    System.out.print("\t");
                }
                System.out.println();
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    @After  //关闭资源
    public void close() {
        if (conn != null) {
            try {
                conn.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        if (hAdmin != null) {
            try {
                hAdmin.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}

使用过滤器过滤数据

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.filter.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.util.List;

public class FilterHbase {
    private HConnection conn;
    private HBaseAdmin hAdmin;

    @Before     //连接
    public void connect() {
        try {
            //获取Hadoop相关配置
            Configuration conf = new Configuration();
            //获取zookeeper配置
            conf.set("hbase.zookeeper.quorum", "master:2181,node1:2181,node2:2181");
            //获取与hbase的连接
            conn = HConnectionManager.createConnection(conf);
            //HMaster
            hAdmin = new HBaseAdmin(conf);
            System.out.println("连接：" + conn + "-HMaster:" + hAdmin);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * rowKey过滤器：RowFilter  行键过滤器
     * <p>
     * 通过RowFilter与BinaryComparator过滤比rowKey 1500100010小的所有值出来
     */
    @Test
    public void rowFilter() {
        try {
            //获取表的实例
            HTableInterface students = conn.getTable("students");
            //创建BinaryComparator比较器
            BinaryComparator binaryComparator = new BinaryComparator("1500100010".getBytes());
            //创建RowFilter过滤器对象
            RowFilter rowFilter = new RowFilter(CompareFilter.CompareOp.LESS, binaryComparator);

            Scan scan = new Scan();
            scan.setFilter(rowFilter);
            ResultScanner scanner = students.getScanner(scan);
            print2(scanner);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 列簇过滤器：FamilyFilter
     * <p>
     * 通过FamilyFilter与SubstringComparator查询列簇名包含in的所有列簇下面的数据
     */
    @Test
    public void familyFilter() {
        try {
            //获取表对象
            HTableInterface students = conn.getTable("students");
            //创建SubstringComparator比较器
            SubstringComparator substringComparator = new SubstringComparator("in");
            //创建FamilyFilter过滤器
            FamilyFilter familyFilter = new FamilyFilter(CompareFilter.CompareOp.EQUAL, substringComparator);

            Scan scan = new Scan();
            scan.setFilter(familyFilter);
            ResultScanner scanner = students.getScanner(scan);
            print1(scanner);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 通过FamilyFilter与 BinaryPrefixComparator 过滤出列簇以i开头的列簇下的所有数据
     */
    @Test
    public void familyFilter2() {
        try {
            //获取表的实例
            HTableInterface students = conn.getTable("students");
            //创建比较器
            BinaryPrefixComparator binaryPrefixComparator = new BinaryPrefixComparator("i".getBytes());
            //创建过滤器
            FamilyFilter familyFilter = new FamilyFilter(CompareFilter.CompareOp.EQUAL, binaryPrefixComparator);
            //创建扫描数据
            Scan scan = new Scan();
            scan.setFilter(familyFilter);
            //表添加扫描
            ResultScanner scanner = students.getScanner(scan);
            //输出数据
            print1(scanner);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 列过滤器：QualifierFilter
     * <p>
     * 通过QualifierFilter与SubstringComparator查询列名包含ge的列的值
     */
    @Test
    public void qualifierFilter() {
        try {
            //创建表的实例
            HTableInterface students = conn.getTable("students");
            //创建比较器
            SubstringComparator substringComparator = new SubstringComparator("ge");
            //创建过滤器
            QualifierFilter qualifierFilter = new QualifierFilter(CompareFilter.CompareOp.EQUAL, substringComparator);

            Scan scan = new Scan();
            scan.setFilter(qualifierFilter);
            ResultScanner scanner = students.getScanner(scan);
            print2(scanner);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 过滤出列的名字中 包含 "am" 所有的列 及列的值
     */
    @Test
    public void qualifierFilter2() {
        try {
            //创建表的实例
            HTableInterface students = conn.getTable("students");
            //创建比较器
            SubstringComparator substringComparator = new SubstringComparator("am");
            //创建过滤器
            QualifierFilter qualifierFilter = new QualifierFilter(CompareFilter.CompareOp.EQUAL, substringComparator);

            Scan scan = new Scan();
            scan.setFilter(qualifierFilter);
            ResultScanner scanner = students.getScanner(scan);
            print2(scanner);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 列值过滤器：ValueFilter
     * <p>
     * 通过ValueFilter与BinaryPrefixComparator过滤出所有的cell中值以 "张" 开头的学生
     */
    @Test
    public void valueFilter() {
        try {
            //创建表的实例
            HTableInterface students = conn.getTable("students");
            //创建比较器
            BinaryPrefixComparator binaryPrefixComparator = new BinaryPrefixComparator("张".getBytes());
            //创建过滤器
            ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, binaryPrefixComparator);

            Scan scan = new Scan();
            scan.setFilter(valueFilter);
            ResultScanner scanner = students.getScanner(scan);
            print2(scanner);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 过滤出文科的学生，只会返回以文科开头的数据列，其他列的数据不符合条件，不会返回
     */
    @Test
    public void valueFilter2() {
        try {
            //创建表的实例
            HTableInterface students = conn.getTable("students");
            //创建比较器
            BinaryPrefixComparator binaryPrefixComparator = new BinaryPrefixComparator("文科".getBytes());
            //创建过滤器
            ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, binaryPrefixComparator);

            Scan scan = new Scan();
            scan.setFilter(valueFilter);
            ResultScanner scanner = students.getScanner(scan);
            print2(scanner);
        } catch (IOException e) {
            e.printStackTrace();
        }

    }

    /**
     * ==========================================专用过滤器==========================================
     * 单列值过滤器：SingleColumnValueFilter
     * <p>
     * SingleColumnValueFilter会返回满足条件的cell所在行的所有cell的值（即会返回一行数据）
     * <p>
     * 通过SingleColumnValueFilter与查询文科班所有学生信息
     */
    @Test
    public void singleColumnValueFilter() {
        try {
            //获取表的实例
            HTableInterface students = conn.getTable("students");
            //创建比较器
            BinaryPrefixComparator binaryPrefixComparator = new BinaryPrefixComparator("文科".getBytes());
            //创建过滤器
            SingleColumnValueFilter singleColumnValueFilter = new SingleColumnValueFilter("info".getBytes(), "clazz".getBytes(), CompareFilter.CompareOp.EQUAL, binaryPrefixComparator);

            Scan scan = new Scan();
            scan.setFilter(singleColumnValueFilter);
            ResultScanner scanner = students.getScanner(scan);
            print1(scanner);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 列值排除过滤器：SingleColumnValueExcludeFilter
     * <p>
     * 与SingleColumnValueFilter相反，会排除掉指定的列，其他的列全部返回
     * <p>
     * 通过SingleColumnValueExcludeFilter与BinaryComparator查询文科一班所有学生信息，最终不返回clazz列
     */
    @Test
    public void singleColumnValueExcludeFilter() {
        try {
            //获取表的实例
            HTableInterface students = conn.getTable("students");
            //创建比较器
            BinaryComparator binaryComparator = new BinaryComparator("文科一班".getBytes());
            //创建过滤器
            SingleColumnValueExcludeFilter singleColumnValueExcludeFilter = new SingleColumnValueExcludeFilter("info".getBytes(), "clazz".getBytes(), CompareFilter.CompareOp.EQUAL, binaryComparator);

            Scan scan = new Scan();
            scan.setFilter(singleColumnValueExcludeFilter);
            ResultScanner scanner = students.getScanner(scan);
            print2(scanner);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * rowkey前缀过滤器：PrefixFilter
     * <p>
     * 通过PrefixFilter查询以150010008开头的所有前缀的rowkey
     */
    @Test
    public void rowKey() {
        try {
            //获取表的实例
            HTableInterface students = conn.getTable("students");
            //创建过滤器
            PrefixFilter prefixFilter = new PrefixFilter("150010008".getBytes());

            Scan scan = new Scan();
            scan.setFilter(prefixFilter);
            ResultScanner scanner = students.getScanner(scan);
            print1(scanner);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 多过滤器综合查询
     * <p>
     * 以上都是讲解单个过滤器的作用，当需要多个过滤器共同作用于一次查询的时候，就需要使用 `FilterList`。`FilterList` 支持通过构造器或者 `addFilter` 方法传入多个过滤器。
     * <p>
     * 通过运用4种比较器过滤出姓于，年纪大于23岁，性别为女，且是理科的学生。
     */
    @Test
    public void multiFilter() {
        try {
            //获取表的实例
            HTableInterface students = conn.getTable("students");
            //过滤出性于的条件
            //创建比较器
            BinaryPrefixComparator binaryPrefixComparator = new BinaryPrefixComparator("于".getBytes());
            //创建过滤器
            SingleColumnValueFilter singleColumnValueFilter = new SingleColumnValueFilter("info".getBytes(), "name".getBytes(), CompareFilter.CompareOp.EQUAL, binaryPrefixComparator);

            //过滤年纪大于23的条件
            //创建比较器
            BinaryComparator binaryComparator = new BinaryComparator("23".getBytes());
            //创建过滤器
            SingleColumnValueFilter singleColumnValueFilter2 = new SingleColumnValueFilter("info".getBytes(), "age".getBytes(), CompareFilter.CompareOp.GREATER_OR_EQUAL, binaryComparator);

            //过滤出性别为女的条件
            //创建比较器
            BinaryComparator binaryComparator2 = new BinaryComparator("女".getBytes());
            //创建过滤器
            SingleColumnValueFilter singleColumnValueFilter3 = new SingleColumnValueFilter("info".getBytes(), "gender".getBytes(), CompareFilter.CompareOp.EQUAL, binaryComparator2);

            //过滤出理科学生的条件
            //创建比较器
            BinaryPrefixComparator binaryPrefixComparator2 = new BinaryPrefixComparator("理科".getBytes());
            //创建过滤器
            SingleColumnValueFilter singleColumnValueFilter4 = new SingleColumnValueFilter("info".getBytes(), "clazz".getBytes(), CompareFilter.CompareOp.EQUAL, binaryPrefixComparator2);
            //创建过滤器集合
            FilterList filterList = new FilterList();
            filterList.addFilter(singleColumnValueFilter);
            filterList.addFilter(singleColumnValueFilter2);
            filterList.addFilter(singleColumnValueFilter3);
            filterList.addFilter(singleColumnValueFilter4);

            Scan scan = new Scan();
            scan.setFilter(filterList);
            ResultScanner scanner = students.getScanner(scan);
            print1(scanner);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 过滤出学号是以15001001开头的文科学生
     */
    @Test
    public void multiFilter2() {
        try {
            //获取表的实例
            HTableInterface students = conn.getTable("students");
            //过滤出学号以15001001开头的条件
            //创建过滤器
            PrefixFilter prefixFilter = new PrefixFilter("15001001".getBytes());

            //过滤出文科学生的条件
            //创建比较器
            BinaryPrefixComparator binaryPrefixComparator = new BinaryPrefixComparator("文科".getBytes());
            //创建过滤器
            SingleColumnValueFilter singleColumnValueFilter = new SingleColumnValueFilter("info".getBytes(), "clazz".getBytes(), CompareFilter.CompareOp.EQUAL, binaryPrefixComparator);

            //创建过滤器集合
            FilterList filterList = new FilterList();
            filterList.addFilter(prefixFilter);
            filterList.addFilter(singleColumnValueFilter);

            Scan scan = new Scan();
            scan.setFilter(filterList);
            ResultScanner scanner = students.getScanner(scan);
            print1(scanner);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    //输出方法1
    public void print1(ResultScanner scanner) {
        try {
            Result rs = null;
            while (((rs = scanner.next()) != null)) {
                String rk = Bytes.toString(rs.getRow());
                String name = Bytes.toString(rs.getValue("info".getBytes(), "name".getBytes()));
                String age = Bytes.toString(rs.getValue("info".getBytes(), "age".getBytes()));
                String gender = Bytes.toString(rs.getValue("info".getBytes(), "gender".getBytes()));
                String clazz = Bytes.toString(rs.getValue("info".getBytes(), "clazz".getBytes()));
                System.out.println(rk + ":" + "name:" + name + ",age:" + age + ",gender:" + gender + ",clazz:" + clazz);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    //输出方法2     获取一行的所有单元格
    public void print2(ResultScanner scanner) throws IOException {
        Result rs = null;
        while ((rs = scanner.next()) != null) {
            List<Cell> cells = rs.listCells();
            //获取行键
            String rk = Bytes.toString(rs.getRow());
            System.out.print(rk + "\t");
            for (Cell cell : cells) {
                //获取列名
                String col = Bytes.toString(CellUtil.cloneQualifier(cell));
                //获取列值
                String s = Bytes.toString(CellUtil.cloneValue(cell));
                System.out.print(col + ":" + s + "\t");
            }
            System.out.println();
        }
    }

    @After  //关闭
    public void close() {
        if (conn != null) {
            try {
                conn.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        if (hAdmin != null) {
            try {
                hAdmin.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}