HBase工具类

import com.google.common.collect.Maps;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.RowFilter;
import org.apache.hadoop.hbase.filter.SubstringComparator;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.Assert;
import org.springframework.util.StringUtils;

import java.io.IOException;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.*;

import static org.apache.hadoop.hbase.CompareOperator.EQUAL;

@Component
public class HbaseUtil {

    @Autowired
    private Connection connection;


    /**
     * 创建表
     *
     * @param tableName 表名
     * @param families  列族
     */
    public void createTable(String tableName, List<String> families, Compression.Algorithm type) throws IOException {
        TableName name = TableName.valueOf(tableName);
        try (Admin admin = connection.getAdmin()) {
            Assert.isTrue(!admin.tableExists(name), "This table already exists");
            TableDescriptorBuilder desc = TableDescriptorBuilder.newBuilder(name);
            for (String cf : families) {
                ColumnFamilyDescriptorBuilder builder = ColumnFamilyDescriptorBuilder.newBuilder(cf.getBytes(StandardCharsets.UTF_8));
                if (type != null) {
                    builder.setCompressionType(type);
                }
                //指定最大版本1，值会被覆盖
                builder.setMaxVersions(1);
                desc.setColumnFamily(builder.build());
            }
            admin.createTable(desc.build());
        } catch (IOException e) {
            throw e;
        }
    }

    public void addFamily2Table(String tableName, List<String> families, Compression.Algorithm type) throws IOException {
        TableName table = TableName.valueOf(tableName);
        try (Admin admin = connection.getAdmin()) {
            Assert.isTrue(admin.tableExists(table), "This table not exists");
            for (String cf : families) {
                ColumnFamilyDescriptorBuilder builder = ColumnFamilyDescriptorBuilder.newBuilder(cf.getBytes(StandardCharsets.UTF_8));
                if (type != null) {
                    builder.setCompressionType(type);
                }
                builder.setMaxVersions(1);//指定最大版本1，值会被覆盖(当时间戳大于等于已存数据的时间戳时会覆盖原来的值)
                admin.addColumnFamily(table, builder.build());
            }
        } catch (IOException e) {
            throw e;
        }
    }

    /**
     * 删除表操作
     *
     * @param tableName
     */
    public void deleteTable(String tableName) throws IOException {
        TableName name = TableName.valueOf(tableName);
        try (Admin admin = connection.getAdmin()) {
            Assert.isTrue(admin.tableExists(name), "This table not exists");
            admin.disableTable(name);
            admin.deleteTable(name);
        } catch (IOException e) {
            throw e;
        }

    }

    /**
     * 插入记录（单行单列族-单列单值-指定时间戳）
     *
     * @param tableName 表名
     * @param rowKey    行名
     * @param family    列族名
     * @param column    列名
     * @param value     值
     * @param timestamp 值对应的时间戳
     */
    public void insertDouble(String tableName, String rowKey, String family, String column, Double value, Long timestamp) throws IOException {
        insertObject(tableName, rowKey, family, column, value, timestamp);
    }

    /**
     * 插入记录（单行单列族-单列单值-不指定时间戳）
     *
     * @param tableName 表名
     * @param rowKey    行名
     * @param family    列族名
     * @param column    列名
     * @param value     值
     */
    public void insertDouble(String tableName, String rowKey, String family, String column, Double value) throws IOException {
        insertObject(tableName, rowKey, family, column, value, null);
    }

    /**
     * 插入记录（单行单列族-单列单值）
     *
     * @param tableName 表名
     * @param rowKey    行名
     * @param family    列族名
     * @param column    列名
     * @param value     值
     * @param timestamp 值对应的时间戳
     */
    public void insertObject(String tableName, String rowKey, String family, String column, Object value, Long timestamp) throws IOException {
        TableName name = TableName.valueOf(tableName);
        try (Table table = connection.getTable(name)) {
            Put put = new Put(Bytes.toBytes(rowKey));
            if (timestamp != null) {
                put.setTimestamp(timestamp);
            }
            put.addColumn(Bytes.toBytes(family), Bytes.toBytes(column), getBytes(value));
            table.put(put);
        } catch (IOException e) {
            throw e;
        }
    }

    /**
     * 插入记录（单行单列族-多列多值）
     *
     * @param tableName 表名
     * @param rowKey    行名
     * @param family    列族名
     * @param columns   列名（数组）
     * @param values    值（数组）（且需要和列一一对应）
     */
    public void insertDoubles(String tableName, String rowKey, String family, List<String> columns, List<Double> values) throws IOException {
        TableName name = TableName.valueOf(tableName);
        try (Table table = connection.getTable(name)) {
            Put put = new Put(Bytes.toBytes(rowKey));
            for (int i = 0; i < columns.size(); i++) {
                put.addColumn(Bytes.toBytes(family), Bytes.toBytes(columns.get(i)), Bytes.toBytes(values.get(i)));
                table.put(put);
            }
        } catch (IOException e) {
            throw e;
        }
    }

    /**
     * 查找一行记录
     *
     * @param tableName 表名
     * @param rowKey    行名
     */
    public Result selectData(String tableName, String rowKey) throws IOException {
        return this.selectData(tableName, rowKey, null);
    }

    /**
     * 查找一行记录
     *
     * @param tableName 表名
     * @param rowKey    行名
     * @param family    列族
     */
    public Result selectData(String tableName, String rowKey, String family) throws IOException {
        return this.selectData(tableName, rowKey, family, null);
    }

    /**
     * 查找一行记录
     *
     * @param tableName 表名
     * @param rowKey    行名
     * @param family    列族
     * @param qualifier 列名
     */
    public Result selectData(String tableName, String rowKey, String family, String qualifier) throws IOException {
        TableName name = TableName.valueOf(tableName);
        try (Table table = connection.getTable(name)) {
            Get g = new Get(rowKey.getBytes(StandardCharsets.UTF_8));
            if (!StringUtils.isEmpty(family)) {
                g.addFamily(Bytes.toBytes(family));
                if (!StringUtils.isEmpty(qualifier)) {
                    g.addColumn(Bytes.toBytes(family), Bytes.toBytes(qualifier));
                }
            }
            Result rs = table.get(g);
            return rs;
        } catch (IOException e) {
            throw e;
        }
    }

    /**
     * @param tableName   表格名称
     * @param rowKeyStart 开始rowKey
     * @param rowKeyEnd   结束rowKey
     * @return
     * @throws IOException
     */
    public ResultScanner selectDataList(String tableName, String rowKeyStart, String rowKeyEnd) throws IOException {
        return this.selectDataList(tableName, rowKeyStart, rowKeyEnd, null);
    }

    /**
     * @param tableName   表格名称
     * @param rowKeyStart 开始rowKey
     * @param rowKeyEnd   结束rowKey
     * @param family      列族
     * @return
     * @throws IOException
     */
    public ResultScanner selectDataList(String tableName, String rowKeyStart, String rowKeyEnd, String family) throws IOException {
        return this.selectDataList(tableName, rowKeyStart, rowKeyEnd, family, null);
    }

    /**
     * @param tableName   表格名称
     * @param rowKeyStart 开始rowKey
     * @param rowKeyEnd   结束rowKey
     * @param family      列族
     * @param qualifier   列
     * @return
     * @throws IOException
     */
    public ResultScanner selectDataList(String tableName, String rowKeyStart, String rowKeyEnd, String family, String qualifier) throws IOException {
        TableName name = TableName.valueOf(tableName);
        try (Table table = connection.getTable(name)) {
            Scan scan = new Scan();
            boolean isHasFamily = StringUtils.isEmpty(family);
            boolean isHasQualifier = StringUtils.isEmpty(qualifier);
            if (!isHasFamily) {
                scan.addFamily(Bytes.toBytes(family));
                if (!isHasQualifier) {
                    scan.addColumn(Bytes.toBytes(family), Bytes.toBytes(qualifier));
                }
            }
            scan.withStartRow(Bytes.toBytes(rowKeyStart));
            scan.withStopRow(Bytes.toBytes(rowKeyEnd));
            ResultScanner scanner = table.getScanner(scan);
            return scanner;
        } catch (IOException e) {
            throw e;
        }
    }

    /**
     * @param qualifierMap 列集合
     * @param entries      列族集合
     * @return 经过转换的返回值
     */
    private List<Map<String, Object>> assembleReturnData(Map<String, List<Object>> qualifierMap, Set<Map.Entry<String, Set<String>>> entries) {
        List<Map<String, Object>> families = Lists.newArrayList();
        for (Iterator<Map.Entry<String, Set<String>>> it = entries.iterator(); it.hasNext(); ) {
            Map.Entry<String, Set<String>> next = it.next();
            String key = next.getKey();
            Set<String> value = next.getValue();
            Map<String, Object> familyData = Maps.newHashMap();
            List<Map<String, Object>> qualifiers = Lists.newArrayList();
            familyData.put("family", key);
            familyData.put("qualifiers", qualifiers);
            families.add(familyData);
            for (String cell : value) {
                Map<String, Object> qualifierData = Maps.newHashMap();
                qualifierData.put("qualifier", cell);
                qualifierData.put("times", qualifierMap.get(cell + "Times"));
                qualifierData.put("values", qualifierMap.get(cell + "Values"));
                qualifiers.add(qualifierData);
            }
        }
        return families;
    }

    /**
     * 组装数据
     *
     * @param familyList    列族集合
     * @param qualifierList 列集合
     * @param rs
     * @param rowKeyList 行键集合
     */
    private void assembleCellData(List<String> familyList, List<Map<String, String>> qualifierList, Result rs, List<String> rowKeyList) {
        if (rs.size() == 0) {
            return;
        }

        for (Cell cell : rs.listCells()) {

            String family = Bytes.toString(CellUtil.cloneFamily(cell));
            String qualifier = Bytes.toString(CellUtil.cloneQualifier(cell));
            String value = Bytes.toString(CellUtil.cloneValue(cell));
            String rowKey = Bytes.toString(CellUtil.cloneRow(cell));

            familyList.add(family);
            if(!rowKeyList.contains(rowKey)){
                rowKeyList.add(rowKey);
            }
            Map<String, String> map = new HashMap<>();
            map.put("key", qualifier);
            map.put("value", value);
            map.put("row",rowKey);
            qualifierList.add(map);
        }
    }

    /**
     * 删除一行记录
     *
     * @param tableName 表名
     * @param rowKey    行名
     */
    public void deleteRow(String tableName, String rowKey) throws IOException {
        TableName name = TableName.valueOf(tableName);
        try (Table table = connection.getTable(name)) {
            Delete d = new Delete(rowKey.getBytes(StandardCharsets.UTF_8));
            table.delete(d);
        } catch (IOException e) {
            throw e;
        }
    }

    /**
     * 删除单行单列族记录
     *
     * @param tableName 表名
     * @param rowKey    行名
     * @param family    列族名
     */
    public void deleteFamily(String tableName, String rowKey, String family) throws IOException {
        TableName name = TableName.valueOf(tableName);
        try (Table table = connection.getTable(name)) {
            Delete d = new Delete(rowKey.getBytes(StandardCharsets.UTF_8)).addFamily(Bytes.toBytes(family));
            table.delete(d);
        } catch (IOException e) {
            throw e;
        }
    }

    /**
     * 删除单行单列族单列记录
     *
     * @param tableName 表名
     * @param rowKey    行名
     * @param family    列族名
     * @param qualifier 列名
     */
    public void deleteColumn(String tableName, String rowKey, String family, String qualifier) throws IOException {
        TableName name = TableName.valueOf(tableName);
        try (Table table = connection.getTable(name)) {
            Delete d = new Delete(rowKey.getBytes(StandardCharsets.UTF_8)).addColumn(Bytes.toBytes(family), Bytes.toBytes(qualifier));
            table.delete(d);
        } catch (IOException e) {
            throw e;
        }
    }


    /**
     * 查找单行单列族单列记录
     *
     * @param tableName 表名
     * @param rowKey    行名
     * @param family    列族名
     * @param column    列名
     * @return
     */
    public Object selectValue(String tableName, String rowKey, String family, String column) throws IOException {
        TableName name = TableName.valueOf(tableName);
        Table table = connection.getTable(name);
        Get g = new Get(rowKey.getBytes(StandardCharsets.UTF_8));
        g.addColumn(Bytes.toBytes(family), Bytes.toBytes(column));
        Result rs = table.get(g);
        return getObject(rs.value(), String.class);
    }

    /**
     * 根据rowKey关键字查询报告记录
     *
     * @param tableName
     * @param rowKeyword
     * @return
     */
    public List scanReportDataByRowKeyword(String tableName, String rowKeyword) throws IOException {
        List list = new ArrayList<>();

        Table table = connection.getTable(TableName.valueOf(tableName));
        Scan scan = new Scan();

        //添加行键过滤器，根据关键字匹配
        RowFilter rowFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, new SubstringComparator(rowKeyword));
        scan.setFilter(rowFilter);

        ResultScanner scanner = table.getScanner(scan);
        try {
            for (Result result : scanner) {
                //TODO 此处根据业务来自定义实现
                list.add(null);
            }
        } finally {
            if (scanner != null) {
                scanner.close();
            }
        }

        return list;
    }

    /**
     * 根据rowKey关键字和时间戳范围查询报告记录
     *
     * @param tableName
     * @param rowKeyword
     * @return
     */
    public List scanReportDataByRowKeywordTimestamp(String tableName, String rowKeyword, Long minStamp, Long maxStamp) throws IOException {
        ArrayList list = new ArrayList<>();

        Table table = connection.getTable(TableName.valueOf(tableName));
        Scan scan = new Scan();
        //添加scan的时间范围
        scan.setTimeRange(minStamp, maxStamp);

        RowFilter rowFilter = new RowFilter(EQUAL, new SubstringComparator(rowKeyword));
        scan.setFilter(rowFilter);

        ResultScanner scanner = table.getScanner(scan);
        try {
            for (Result result : scanner) {
                //TODO 此处根据业务来自定义实现
                list.add(null);
            }
        } finally {
            if (scanner != null) {
                scanner.close();
            }
        }

        return list;
    }


    /**
     * 利用协处理器进行全表count统计
     *
     * @param tableName
     */
    /*public Long countRowsWithCoprocessor(String tableName) throws Throwable {
        TableName name = TableName.valueOf(tableName);
        Admin admin = connection.getAdmin();
        HTableDescriptor descriptor = admin.getTableDescriptor(name);
        String coprocessorClass = "org.apache.hadoop.hbase.coprocessor.AggregateImplementation";
        if (!descriptor.hasCoprocessor(coprocessorClass)) {
            admin.disableTable(name);
            descriptor.addCoprocessor(coprocessorClass);
            admin.modifyTable(name, descriptor);
            admin.enableTable(name);
        }
        //计时
        StopWatch stopWatch = new StopWatch();
        stopWatch.start();

        Scan scan = new Scan();
        //Hbase服务端需要开启该服务
        AggregationClient aggregationClient = new AggregationClient(connection.getConfiguration());
        Long count = aggregationClient.rowCount(name, new LongColumnInterpreter(), scan);
        stopWatch.stop();
        log.debug("RowCount:{} ,全表count统计耗时:{}", count, stopWatch.getTotalTimeMillis());
        return count;
    }*/

    /**
     * 根据不同的类型获取不同的byte数组
     *
     * @param object
     * @return
     */
    private static byte[] getBytes(Object object) {
        Class className = object.getClass();
        if (className.equals(java.lang.Boolean.class)) {
            return Bytes.toBytes((Boolean) object);
        }
        if (className.equals(java.lang.Byte.class)) {
            return Bytes.toBytes((Byte) object);
        }
        if (className.equals(java.lang.Short.class)) {
            return Bytes.toBytes((Short) object);
        }
        if (className.equals(java.lang.Character.class)) {
            return Bytes.toBytes((Character) object);
        }
        if (className.equals(java.lang.Integer.class)) {
            return Bytes.toBytes((Integer) object);
        }
        if (className.equals(java.lang.Long.class)) {
            return Bytes.toBytes((Long) object);
        }
        if (className.equals(java.lang.Float.class)) {
            return Bytes.toBytes((Float) object);
        }
        if (className.equals(java.lang.Double.class)) {
            return Bytes.toBytes((Double) object);
        }
        if (className.equals(java.lang.String.class)) {
            return Bytes.toBytes((String) object);
        }
        if (className.equals(java.nio.ByteBuffer.class)) {
            return Bytes.toBytes((ByteBuffer) object);
        }
        if (className.equals(java.math.BigDecimal.class)) {
            return Bytes.toBytes((BigDecimal) object);
        }
        return ByteUtil.toBytes(object);
    }

    /**
     * 根据不同的byte数组获取不同的对象
     *
     * @param bytes
     * @return
     */
    private static Object getObject(byte[] bytes, Class className) {
        if (className.equals(java.lang.Boolean.class)) {
            return Bytes.toBoolean(bytes);
        }
        if (className.equals(java.lang.Byte.class)) {
            return Bytes.toShort(bytes);
        }
        if (className.equals(java.lang.Short.class)) {
            return Bytes.toShort(bytes);
        }
        if (className.equals(java.lang.Character.class)) {
            return Bytes.toString(bytes);
        }
        if (className.equals(java.lang.Integer.class)) {
            return Bytes.toInt(bytes);
        }
        if (className.equals(java.lang.Long.class)) {
            return Bytes.toLong(bytes);
        }
        if (className.equals(java.lang.Float.class)) {
            return Bytes.toFloat(bytes);
        }
        if (className.equals(java.lang.Double.class)) {
            return Bytes.toDouble(bytes);
        }
        if (className.equals(java.lang.String.class)) {
            return Bytes.toString(bytes);
        }
        if (className.equals(java.nio.ByteBuffer.class)) {
            return ByteBuffer.wrap(bytes);
        }
        if (className.equals(java.math.BigDecimal.class)) {
            return Bytes.toBigDecimal(bytes);
        }
        return ByteUtil.toObject(bytes);
    }

}