SpringBoot学习：整合Hbase

1. 所需pom依赖

<!--hbase依赖-->
<hbase-client.version>2.0.0</hbase-client.version>
<lombak.version>1.16.10</lombak.version>
<dependency>
    <groupId>org.apache.hbase</groupId>
    <artifactId>hbase-client</artifactId>
    <version>${hbase-client.version}</version>
    <!--排除以下与springboot的冲突包，guava自己引入，故而排除。-->
    <exclusions>
        <exclusion>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
        </exclusion>
        <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
        </exclusion>
        <exclusion>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
        </exclusion>
        <exclusion>
            <groupId>javax.servlet</groupId>
            <artifactId>servlet-api</artifactId>
        </exclusion>
        <exclusion>
            <groupId>com.google.guava</groupId>
            <artifactId>guava</artifactId>
        </exclusion>
        <exclusion>
            <groupId>org.elasticsearch</groupId>
            <artifactId>elasticsearch</artifactId>
        </exclusion>
        <exclusion>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-lang3</artifactId>
        </exclusion>
    </exclusions>
</dependency>

<dependency>
    <groupId>org.projectlombok</groupId>
    <artifactId>lombok</artifactId>
    <version>${lombak.version}</version>
</dependency>

2. yaml文件

hbase:
  config:
    hbase.zookeeper.quorum: 10.16.1.111
    hbase.zookeeper.property.clientPort: 2181
    zookeeper.znode.parent: /hbase
    zookeeper.session.timeout: 60000

3. 配置类

package com.staryea.invocate.config;

/**
 * @author: hs
 * @Date: 2019/3/8 15:54
 * @Description:
 */
@Configuration
@EnableConfigurationProperties(HBaseConfig.HbaseProperties.class)
public class HBaseConfig {

    private static final String HBASE_PREFIX = "hbase";

    private final HbaseProperties PROPERTIES;

    public HBaseConfig(HbaseProperties properties) {
        this.PROPERTIES = properties;
    }

    public org.apache.hadoop.conf.Configuration configuration() {
        org.apache.hadoop.conf.Configuration configuration = HBaseConfiguration.create();
        Map<String, String> config = PROPERTIES.getConfig();
        config.forEach((key, value) -> configuration.set(key, value));
        return configuration;
    }

    /*@Bean
    public HbaseTemplate hbaseTemplate(){
        HbaseTemplate hbaseTemplate = new HbaseTemplate();
        hbaseTemplate.setConfiguration(configuration());
        hbaseTemplate.setAutoFlush(true);
        return  hbaseTemplate;
    }*/

    @Data
    @ConfigurationProperties(prefix = HBASE_PREFIX)
    static class HbaseProperties {
        private Map<String, String> config;
    }


}

4. 辅助类

/**
 * @Author: hs
 * @Descriptions: HBaseDao操作公共类
 */

@Component
public class HBaseBaseDao {

    private final Logger logger = LoggerFactory.getLogger(HBaseBeanUtils.class);

    private final HBaseConfig HBASE_CONFIG;

    private Connection conn = null;

    @Autowired
    public HBaseBaseDao(HBaseConfig hbaseConfig) {
        this.HBASE_CONFIG = hbaseConfig;
    }

    public Connection getConn() {
        try {
            return ConnectionFactory.createConnection(HBASE_CONFIG.configuration());
        } catch (IOException e) {
            logger.error("连接异常：" + e.getMessage(), e);
        }
        return null;
    }


    /*// 关闭连接
    public static void close() {
        try {
            if (conn != null) {
                conn.close();
            }
        } catch (IOException e) {
            log.error("关闭habse连接异常："+e.getMessage(),e);
        }

    }*/

    /**
     * @param tableName
     * @param familyColumn
     * @Descripton: 创建表
     * @Author: hs
     */
    public void createTable(String tableName, Set<String> familyColumn) {
        try (Admin admin = getConn().getAdmin();) {
            TableDescriptorBuilder htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName));
            for (String fc : familyColumn) {
                ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(fc)).build();
                htd.setColumnFamily(hcd);
            }
            admin.createTable(htd.build());
        } catch (IOException e) {
            logger.error("创建" + tableName + "表失败：" + e.getMessage(), e);
        }
    }

    /**
     * @Descripton: 删除表
     * @Author: hs
     */
    public void dropTable(String tableName) {
        TableName tn = TableName.valueOf(tableName);
        try (Admin admin = getConn().getAdmin();) {
            admin.disableTable(tn);
            admin.deleteTable(tn);
        } catch (IOException e) {
            logger.error("删除" + tableName + "表失败：" + e.getMessage(), e);
        }
    }

    /**
     * @param obj
     * @param param
     * @Descripton: 根据条件过滤查询
     * @Author: hs
     */
    public <T> List<T> queryScan(T obj, Map<String, String> param) throws Exception {
        List<T> objs = new ArrayList<T>();
        String tableName = getORMTable(obj);
        if (StringUtils.isBlank(tableName)) {
            return null;
        }
        try (Table table = getConn().getTable(TableName.valueOf(tableName));) {
            Scan scan = new Scan();
            for (Map.Entry<String, String> entry : param.entrySet()) {
                Class<?> clazz = obj.getClass();
                Field[] fields = clazz.getDeclaredFields();
                for (Field field : fields) {
                    if (!field.isAnnotationPresent(HBaseColumn.class)) {
                        continue;
                    }
                    field.setAccessible(true);
                    HBaseColumn orm = field.getAnnotation(HBaseColumn.class);
                    String family = orm.family();
                    String qualifier = orm.qualifier();
                    if (qualifier.equals(entry.getKey())) {
                        Filter filter = new SingleColumnValueFilter(Bytes.toBytes(family), Bytes.toBytes(entry.getKey()), CompareOperator.EQUAL, Bytes.toBytes(entry.getValue()));
                        scan.setFilter(filter);
                    }
                }
            }
            ResultScanner scanner = table.getScanner(scan);
            for (Result result : scanner) {
                T beanClone = (T) BeanUtils.cloneBean(HBaseBeanUtils.resultToBean(result, obj));
                objs.add(beanClone);
            }
        } catch (Exception e) {
            logger.error("查询失败：" + e.getMessage(), e);
            throw new Exception(e);
        }
        return objs;
    }

    /**
     * @param obj
     * @param rowkeys
     * @Descripton: 根据rowkey查询
     * @Author: hs
     */
    public <T> List<T> get(T obj, String... rowkeys) {
        List<T> objs = new ArrayList<T>();
        String tableName = getORMTable(obj);
        if (StringUtils.isBlank(tableName)) {
            return objs;
        }
        try (Admin admin = getConn().getAdmin();) {
            if (!admin.isTableAvailable(TableName.valueOf(tableName))) {
                return objs;
            }
            List<Result> results = getResults(tableName, rowkeys);
            if (results.isEmpty()) {
                return objs;
            }
            for (int i = 0; i < results.size(); i++) {
                T bean = null;
                Result result = results.get(i);
                if (result == null || result.isEmpty()) {
                    continue;
                }
                try {
                    bean = HBaseBeanUtils.resultToBean(result, obj);
                    objs.add(bean);
                } catch (Exception e) {
                    logger.error("查询异常：" + e.getMessage(), e);
                }
            }
        } catch (Exception e) {
            logger.error("" + e.getMessage(), e);
        }
        return objs;
    }


    /**
     * @param objs
     * @Descripton: 保存实体对象
     * @Author: hs
     */
    public <T> boolean save(T... objs) {
        List<Put> puts = new ArrayList<Put>();
        String tableName = "";
        try (Admin admin = getConn().getAdmin();) {
            for (Object obj : objs) {
                if (obj == null) {
                    continue;
                }
                tableName = getORMTable(obj);
                // 表不存在，先获取family创建表
                if (!admin.isTableAvailable(TableName.valueOf(tableName))) {
                    // 获取family, 创建表
                    Class<?> clazz = obj.getClass();
                    Field[] fields = clazz.getDeclaredFields();
                    Set<String> set = new HashSet<>(10);
                    for (int i = 0; i < fields.length; i++) {
                        if (!fields[i].isAnnotationPresent(HBaseColumn.class)) {
                            continue;
                        }
                        fields[i].setAccessible(true);
                        HBaseColumn orm = fields[i].getAnnotation(HBaseColumn.class);
                        String family = orm.family();
                        if ("rowkey".equalsIgnoreCase(family)) {
                            continue;
                        }
                        set.add(family);
                    }
                    // 创建表
                    createTable(tableName, set);
                }
                Put put = HBaseBeanUtils.beanToPut(obj);
                puts.add(put);
            }
        } catch (Exception e) {
            logger.error("保存Hbase异常：" + e.getMessage(), e);
        }
        return savePut(puts, tableName);
    }

    /**
     * @param tableName
     * @param objs
     * @Descripton: 根据tableName保存
     * @Author: hs
     */
    public <T> void save(String tableName, T... objs) {
        List<Put> puts = new ArrayList<Put>();
        for (Object obj : objs) {
            if (obj == null) {
                continue;
            }
            try {
                Put put = HBaseBeanUtils.beanToPut(obj);
                puts.add(put);
            } catch (Exception e) {
                logger.warn("", e);
            }
        }
        savePut(puts, tableName);
    }

    /**
     * @param obj
     * @param rowkeys
     * @Descripton: 删除
     * @Author: hs
     */
    public <T> void delete(T obj, String... rowkeys) {
        String tableName = "";
        tableName = getORMTable(obj);
        if (StringUtils.isBlank(tableName)) {
            return;
        }
        List<Delete> deletes = new ArrayList<Delete>();
        for (String rowkey : rowkeys) {
            if (StringUtils.isBlank(rowkey)) {
                continue;
            }
            deletes.add(new Delete(Bytes.toBytes(rowkey)));
        }
        delete(deletes, tableName);
    }


    /**
     * @param deletes
     * @param tableName
     * @Descripton: 批量删除
     * @Author: hs
     */
    private void delete(List<Delete> deletes, String tableName) {
        try (Table table = getConn().getTable(TableName.valueOf(tableName));) {
            if (StringUtils.isBlank(tableName)) {
                logger.info("tableName为空！");
                return;
            }
            table.delete(deletes);
        } catch (IOException e) {
            logger.error("删除失败：" + e.getMessage(), e);
        }
    }

    /**
     * @param tableName
     * @Descripton: 根据tableName获取列簇名称
     * @Author: hs
     */
    public List<String> familys(String tableName) {
        try (Table table = getConn().getTable(TableName.valueOf(tableName));) {
            List<String> columns = new ArrayList<String>();
            if (table == null) {
                return columns;
            }
            ColumnFamilyDescriptor[] columnDescriptors = table.getDescriptor().getColumnFamilies();
            for (ColumnFamilyDescriptor columnDescriptor : columnDescriptors) {
                String columnName = columnDescriptor.getNameAsString();
                columns.add(columnName);
            }
            return columns;
        } catch (Exception e) {
            logger.error("查询列簇名称失败：" + e.getMessage(), e);
        }
        return new ArrayList<String>();
    }

    /**
     * 保存方法
     * @param puts
     * @param tableName
     * @return
     */
    private boolean savePut(List<Put> puts, String tableName) {
        if (StringUtils.isBlank(tableName)) {
            return false;
        }
        try (Table table = getConn().getTable(TableName.valueOf(tableName));) {
            table.put(puts);
            return true;
        } catch (IOException e) {
            logger.error("连接异常：" + e.getMessage(), e);
            return false;
        }
    }

    /**
     * 获取tableName
     * @param obj
     * @return
     */
    private String getORMTable(Object obj) {
        HBaseTable table = obj.getClass().getAnnotation(HBaseTable.class);
        return table.tableName();
    }

    /**
     * 获取查询结果
     * @param tableName
     * @param rowkeys
     * @return
     */
    private List<Result> getResults(String tableName, String... rowkeys) {
        List<Result> resultList = new ArrayList<Result>();
        List<Get> gets = new ArrayList<Get>();
        for (String rowkey : rowkeys) {
            if (StringUtils.isBlank(rowkey)) {
                continue;
            }
            Get get = new Get(Bytes.toBytes(rowkey));
            gets.add(get);
        }
        try (Table table = getConn().getTable(TableName.valueOf(tableName));) {
            Result[] results = table.get(gets);
            Collections.addAll(resultList, results);
            return resultList;
        } catch (Exception e) {
            e.printStackTrace();
            return resultList;
        }
    }

    /**
     * @param obj
     * @param param
     * @Descripton: 根据条件过滤查询（大于等于）
     * @Author: hs
     */
    public <T> List<T> queryScanGreater(T obj, Map<String, String> param) throws Exception {
        List<T> objs = new ArrayList<T>();
        String tableName = getORMTable(obj);
        if (StringUtils.isBlank(tableName)) {
            return null;
        }
        try (Table table = getConn().getTable(TableName.valueOf(tableName));
        ) {

            Scan scan = new Scan();
            for (Map.Entry<String, String> entry : param.entrySet()) {
                Class<?> clazz = obj.getClass();
                Field[] fields = clazz.getDeclaredFields();
                for (Field field : fields) {
                    if (!field.isAnnotationPresent(HBaseColumn.class)) {
                        continue;
                    }
                    field.setAccessible(true);
                    HBaseColumn orm = field.getAnnotation(HBaseColumn.class);
                    String family = orm.family();
                    String qualifier = orm.qualifier();
                    if (qualifier.equals(entry.getKey())) {
                        Filter filter = new SingleColumnValueFilter(Bytes.toBytes(family), Bytes.toBytes(entry.getKey()), CompareOperator.GREATER_OR_EQUAL, Bytes.toBytes(entry.getValue()));
                        scan.setFilter(filter);
                    }
                }
            }
            ResultScanner scanner = table.getScanner(scan);
            for (Result result : scanner) {
                T beanClone = (T) BeanUtils.cloneBean(HBaseBeanUtils.resultToBean(result, obj));
                objs.add(beanClone);
            }
        } catch (Exception e) {
            logger.error("查询失败！" + e.getMessage(), e);
            throw new Exception(e);
        }
        return objs;
    }

    /**
     * 根据rowkey查询记录
     *
     * @param obj
     * @param rowkey
     * @param <T>
     * @return
     */
    public <T> List<T> queryScanRowkey(T obj, String rowkey) {
        List<T> objs = new ArrayList<T>();
        String tableName = getORMTable(obj);
        if (StringUtils.isBlank(tableName)) {
            return null;
        }
        ResultScanner scanner = null;
        try (Table table
                     = getConn().getTable(TableName.valueOf(tableName));) {
            Scan scan = new Scan();
            scan.setRowPrefixFilter(Bytes.toBytes(rowkey));
            scanner = table.getScanner(scan);
            for (Result result : scanner) {
                T beanClone = (T) BeanUtils.cloneBean(HBaseBeanUtils.resultToBean(result, obj));
                objs.add(beanClone);
            }
        } catch (Exception e) {
            logger.error("queryScanRowkey:查询失败：" + e.getMessage(), e);
        } finally {
            if (scanner != null) {
                try {
                    scanner.close();
                } catch (Exception e) {
                    logger.error("queryScan:关闭流异常：" + e.getMessage(), e);
                }
            }
        }
        return objs;
    }
}

5. Util

/**
 * @author: hs
 * @Date: 2019/3/11 11:14
 * @Description:
 */
@Slf4j
public class HBaseBeanUtils {

    /**
     * JavaBean转换为Put
     *
     * @param <T>
     * @param obj
     * @return
     * @throws Exception
     */
    public static <T> Put beanToPut(T obj) {
        Put put = new Put(Bytes.toBytes(parseObjId(obj)));
        Class<?> clazz = obj.getClass();
        Field[] fields = clazz.getDeclaredFields();
        for (Field field : fields) {
            if (!field.isAnnotationPresent(HBaseColumn.class)) {
                continue;
            }
            field.setAccessible(true);
            HBaseColumn orm = field.getAnnotation(HBaseColumn.class);
            String family = orm.family();
            String qualifier = orm.qualifier();
            if (StringUtils.isBlank(family) || StringUtils.isBlank(qualifier)) {
                continue;
            }
            try {
                Object fieldObj = field.get(obj);
                if (fieldObj.getClass().isArray()) {
                    log.error("nonsupport");
                }
                if ("rowkey".equalsIgnoreCase(qualifier) || "rowkey".equalsIgnoreCase(family)) {
                    continue;
                }
                if (field.get(obj) != null || StringUtils.isNotBlank(field.get(obj).toString())) {
                    put.addColumn(Bytes.toBytes(family), Bytes.toBytes(qualifier), Bytes.toBytes(fieldObj.toString()));
                }
            } catch (IllegalAccessException e) {
                log.error("" + e.getMessage(), e);
            }
        }
        return put;
    }

    /**
     * 获取Bean中的id,作为Rowkey
     *
     * @param <T>
     * @param obj
     * @return
     */
    public static <T> String parseObjId(T obj) {
        Class<?> clazz = obj.getClass();
        try {
            Field field = clazz.getDeclaredField("id");
            field.setAccessible(true);
            Object object = field.get(obj);
            if (object == null) {
                return "";
            }
            return object.toString();
        } catch (NoSuchFieldException e) {
            log.error("error:" + e.getMessage(), e);
        } catch (SecurityException e) {
            log.error("error:" + e.getMessage(), e);
        } catch (IllegalArgumentException e) {
            log.error("error:" + e.getMessage(), e);
        } catch (IllegalAccessException e) {
            log.error("error:" + e.getMessage(), e);
        }
        return "";
    }

    /**
     * HBase result 转换为 bean
     *
     * @param <T>
     * @param result
     * @param obj
     * @return
     * @throws Exception
     */
    public static <T> T resultToBean(Result result, T obj) throws Exception {
        if (result == null) {
            return null;
        }
        Class<?> clazz = obj.getClass();
        Field[] fields = clazz.getDeclaredFields();
        for (Field field : fields) {
            if (!field.isAnnotationPresent(HBaseColumn.class)) {
                continue;
            }
            HBaseColumn orm = field.getAnnotation(HBaseColumn.class);
            String family = orm.family();
            String qualifier = orm.qualifier();
            boolean timeStamp = orm.timestamp();
            if (StringUtils.isBlank(family) || StringUtils.isBlank(qualifier)) {
                continue;
            }
            String fieldName = field.getName();
            String value = "";
            if ("rowkey".equalsIgnoreCase(family)) {
                //这里不可改成toString。
                value = new String(result.getRow());
            } else {
                value = getResultValueByType(result, family, qualifier, timeStamp);
            }
            String firstLetter = fieldName.substring(0, 1).toUpperCase();
            String setMethodName = "set" + firstLetter + fieldName.substring(1);
            Method setMethod = clazz.getMethod(setMethodName, new Class[]{field.getType()});
            setMethod.invoke(obj, new Object[]{value});
        }
        return obj;
    }

    /**
     * @param result
     * @param family
     * @param qualifier
     * @param timeStamp
     * @return
     */
    private static String getResultValueByType(Result result, String family, String qualifier, boolean timeStamp) {
        if (!timeStamp) {
            return new String(result.getValue(Bytes.toBytes(family), Bytes.toBytes(qualifier)));
        }
        List<Cell> cells = result.getColumnCells(Bytes.toBytes(family), Bytes.toBytes(qualifier));
        if (cells.size() == 1) {
            Cell cell = cells.get(0);
            return cell.getTimestamp() + "";
        }
        return "";
    }
}