hbase的基本封装

背景：目前在做项目的时候，需要使用hbase进行数据的查询，一些评论记录，消费数据，交易日志等数据都是存储在hbase中的，因为都是海量的数据。了解hbase的原理是必要的，因此必须要学习好 rowkey的设计，否则会出现热点问题。可以百度这个rowkey 的设计，比如一般使用：随机，md5，分桶，反转等基本的手段就可以了。还需要了解hadoop的hdfs，因为hbase的HFile存储在hadoop上。

接下来就是基本你的api的封装，用于业务的调用

package hbase.annotation;

import hbase.convertor.HbaseConvertor;
import hbase.convertor.SimpleHbaseConvertor;

import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;

@Retention(RetentionPolicy.RUNTIME)
@Target(value={ElementType.FIELD})
public @interface HbColumn {

    /**
     * 列名，一张hbase表中列名必须唯一，子类可以覆盖父类，为空表示与db一致
     * @return
     */
    String value() default "";

    String familyName() default "f";

    Class<? extends HbaseConvertor> convertClass() default SimpleHbaseConvertor.class ;

    String[] args() default {};


}

package hbase.condition;

public class HbaseQueryCond {

    private String startRowKey;

    private String endRowKey;

    private int limit=50;

    private int offset=0;

    private boolean needRevers;


    public String getStartRowKey() {
        return startRowKey;
    }

    public void setStartRowKey(String startRowKey) {
        this.startRowKey = startRowKey;
    }

    public String getEndRowKey() {
        return endRowKey;
    }

    public void setEndRowKey(String endRowKey) {
        this.endRowKey = endRowKey;
    }

    public int getLimit() {
        return limit;
    }

    public void setLimit(int limit) {
        this.limit = limit;
    }

    public int getOffset() {
        return offset;
    }

    public void setOffset(int offset) {
        this.offset = offset;
    }

    public boolean isNeedRevers() {
        return needRevers;
    }

    public void setNeedRevers(boolean needRevers) {
        this.needRevers = needRevers;
    }
}

package hbase.convertor;

public interface HbaseConvertor {

    byte[] toBytes(Object value,String[] args);

    Object toObject(Class<?> objectType,byte[] bytes,String[] args);
}

package hbase.convertor;

import com.alibaba.fastjson.JSON;
import com.sun.org.apache.xpath.internal.operations.Bool;
import org.apache.hadoop.hbase.util.Bytes;

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class SimpleHbaseConvertor implements HbaseConvertor {

    @Override
    public byte[] toBytes(Object value, String[] args) {
        if(value==null){
            return null;
        }
        if(value.getClass()==byte[].class){
            return (byte[]) value;
        }
        if(value.getClass()==String.class){
            return Bytes.toBytes((String)value);
        }
        if(value.getClass()==Integer.class){
            return Bytes.toBytes((Integer)value);
        }
        if(value.getClass()==Long.class){
            return Bytes.toBytes((Long)value);
        }
        if(value.getClass()==Short.class){
            return Bytes.toBytes((Short)value);
        }
        if(value.getClass()==Boolean.class){
            return Bytes.toBytes((Boolean)value);
        }
        if(value.getClass()==Double.class){
            return Bytes.toBytes((Double)value);
        }
        if(value.getClass()==Float.class){
            return Bytes.toBytes((Float)value);
        }
        if(value.getClass()== Date.class){
            String date=new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format((Date)value);
            return Bytes.toBytes(date);
        }
        if(value.getClass()== Map.class){
            String mapjson= JSON.toJSONString((Map)value);
            return Bytes.toBytes(mapjson);
        }
        if(value.getClass()== List.class){
            String listjson= JSON.toJSONString((List)value);
            return Bytes.toBytes(listjson);
        }
        throw new RuntimeException("SimpleHbaseConvertor,不支持类型的转换"+value.getClass());
    }

    @Override
    public Object toObject(Class<?> objectType, byte[] bytes, String[] args) {
        if(bytes==null){
            return null;
        }
        if(objectType==byte[].class){
            return bytes;
        }
        if(objectType==String.class){
            return  Bytes.toString(bytes);
        }
        if(objectType==Integer.class || objectType==Integer.TYPE){
            return  Bytes.toInt(bytes);
        }
        if(objectType==Long.class || objectType==Long.TYPE){
            return  Bytes.toLong(bytes);
        }
        if(objectType== Boolean.class||objectType== Boolean.TYPE){
            return  Bytes.toBoolean(bytes);
        }
        if(objectType==Double.class || objectType==Double.TYPE){
            return  Bytes.toDouble(bytes);
        }
        if(objectType==Short.class || objectType==Short.TYPE){
            return  Bytes.toShort(bytes);
        }
        if(objectType==Float.class || objectType==Float.TYPE){
            return  Bytes.toFloat(bytes);
        }
        if(objectType==Date.class){
            String date=Bytes.toString(bytes);
            try {
                return  new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse(date);
            } catch (ParseException e) {
            }
        }
        if(objectType==Map.class || objectType== HashMap.class){
            String mapBytes=Bytes.toString(bytes);
            return JSON.parseObject(mapBytes,Map.class);
        }
        if(objectType==List.class){
            String listBytes=Bytes.toString(bytes);
            return JSON.parseObject(listBytes,List.class);
        }
        throw new RuntimeException("SimpleHbaseConvertor,不支持类型的转换"+objectType);
    }
}

package hbase.model;

public class BaseHbaseDo {

    private String rowKey;

    public String getRowKey() {
        return rowKey;
    }

    public void setRowKey(String rowKey) {
        this.rowKey = rowKey;
    }
}

package hbase.model;

import hbase.annotation.HbColumn;

public class CommentDo extends BaseHbaseDo {
    @HbColumn
    private String id;

    @HbColumn
    private String comment;

    public String getId() {
        return id;
    }

    public void setId(String id) {
        this.id = id;
    }

    public String getComment() {
        return comment;
    }

    public void setComment(String comment) {
        this.comment = comment;
    }
}

package hbase.model;

import org.apache.hadoop.hbase.client.HTableInterface;

public interface HbaseDataSource {

    public HTableInterface getHTable(String storeName,String tableName);

}

package hbase;

import hbase.annotation.HbColumn;
import hbase.condition.HbaseQueryCond;
import hbase.convertor.HbaseConvertor;
import hbase.model.BaseHbaseDo;
import hbase.model.HbaseDataSource;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;

import java.io.IOException;
import java.lang.reflect.Field;
import java.util.*;

public abstract class HbaseBaseDAO<T extends BaseHbaseDo> {

    private HbaseDataSource hbaseDataSource;

    private Map<String,Column> columnMap;

    private Map<Class<? extends HbaseConvertor>,HbaseConvertor> convertorClassMap;

    public void init() throws InstantiationException, IllegalAccessException {
        convertorClassMap=new HashMap<>();
        columnMap=parseHabseDOClass(getHbaseDOClass());
    }

    public void put(T hbaseDo) throws IllegalAccessException {
        HTableInterface table=null;
        try {
            hbaseDataSource.getHTable(getStoreName(),getTableName());

        Put put=new Put(Bytes.toBytes(hbaseDo.getRowKey()));
        for(Column column:columnMap.values()){
            putColumn(put,column,hbaseDo);
        }

            table.put(put);
        } catch (IOException e) {
            e.printStackTrace();
        }finally {
            if(table!=null){
                try {
                    table.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }


    public void put(List<T> hbaseDos) throws IllegalAccessException {
        HTableInterface table=null;
        table=hbaseDataSource.getHTable(getStoreName(),getTableName());
        List<Put> puts=new ArrayList<>();
        for(T hbaseDo:hbaseDos){
            Put put=new Put(Bytes.toBytes(hbaseDo.getRowKey()));
            for(Column column:columnMap.values()){
                putColumn(put,column,hbaseDo);
            }
            puts.add(put);
        }
        try {
            table.put(puts);
        } catch (IOException e) {
            e.printStackTrace();
        }finally {
            if(table!=null){
                try {
                    table.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }

    public T get(String rowKey) throws InstantiationException, IllegalAccessException {
        HTableInterface table=null;
        try {
        table=hbaseDataSource.getHTable(getStoreName(),getTableName());
        Get get=new Get(Bytes.toBytes(rowKey));

            Result result=table.get(get);
            return parseResult(result);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }finally {
            if(table!=null){
                try {
                    table.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }

    public List<T> batchget(List<String> rowKeyList) throws IOException, IllegalAccessException, InstantiationException {
        if(CollectionUtils.isEmpty(rowKeyList)){
            return Collections.emptyList();
        }
        HTableInterface table=null;
        table=hbaseDataSource.getHTable(getStoreName(),getTableName());
        List<Get> getList=new ArrayList<>();
        for(String rowKey:rowKeyList){
            Get get=new Get(Bytes.toBytes(rowKey));
            getList.add(get);
        }
        Result[] results=table.get(getList);
        return parseResultList(results);
    }

    public List<T> scan(HbaseQueryCond hbaseQueryCond) throws IOException {
        if(hbaseQueryCond==null){
            return  Collections.emptyList();
        }
        ResultScanner rs=null;
        HTableInterface table=null;
        try {
        table=hbaseDataSource.getHTable(getStoreName(),getTableName());
        rs=table.getScanner(buildScan(hbaseQueryCond));

            return buildResultScanner(rs);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }finally {
            if(table!=null){
                table.close();
            }
        }
    }


    public List<T> pageScan(HbaseQueryCond cond){
        if(cond==null){
            return  Collections.emptyList();
        }
        ResultScanner rs=null;
        HTableInterface table=null;
        try {
            table=hbaseDataSource.getHTable(getStoreName(),getTableName());
            rs=table.getScanner(buildPageScan(cond));

            return buildResultScanner(rs);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }finally {
            if(table!=null){
                try {
                    table.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }

    }


    public void delete(String rowKey) throws IOException {
        HTableInterface table=null;
        try {
            table = hbaseDataSource.getHTable(getStoreName(), getTableName());
            Delete delete=new Delete(Bytes.toBytes(rowKey));
            table.delete(delete);
        }catch (Exception e){
            if(table!=null){
                table.close();
            }
        }
    }

    private List<T> parseResultScanner(ResultScanner rs,int limit) throws InstantiationException, IllegalAccessException {
        List<T> hbaseDoList=new ArrayList<>();
        int count=0;
        for(Result result:rs){
            T hbaseDo=parseResult(result);
            if(hbaseDo!=null){
                hbaseDoList.add(hbaseDo);
                count++;
            }
            if(count>=limit){
                break;
            }
        }

        return  hbaseDoList;
    }

    private List<T> buildResultScanner(ResultScanner rs) throws InstantiationException, IllegalAccessException {
        List<T> hbaseDOList=new ArrayList<>();
        if(rs==null){
            return  Collections.emptyList();
        }
        for(Result result:rs){
             T hbaseDo=parseResult(result);
            hbaseDOList.add(hbaseDo);
        }
        return hbaseDOList;
    }

    private Scan buildScan(HbaseQueryCond cond){
        Scan scan=new Scan();
        if(!StringUtils.isEmpty(cond.getStartRowKey())){
            scan.setStartRow(Bytes.toBytes(cond.getStartRowKey()));
        }
        if(!StringUtils.isEmpty(cond.getEndRowKey())){
            scan.setStopRow(Bytes.toBytes(cond.getEndRowKey()));
        }
        scan.setBatch(1000);
        return scan;
    }

    public Scan buildPageScan(HbaseQueryCond cond){
        Scan scan=new Scan();
        if(!StringUtils.isEmpty(cond.getStartRowKey())){
            scan.setStartRow(Bytes.toBytes(cond.getStartRowKey()));
        }
        if(!StringUtils.isEmpty(cond.getEndRowKey())){
            scan.setStopRow(Bytes.toBytes(cond.getEndRowKey()));
        }
        if(cond.getLimit()>200){
            scan.setCaching(200);
        }
        scan.setCaching(cond.getLimit()+1);
        scan.setBatch(1000);
        return scan;

    }
    private T parseResult(Result result) throws IllegalAccessException, InstantiationException {
        if(result.isEmpty()){
            return null;
        }
        T hbaseDo=getHbaseDOClass().newInstance();
        for(Column column:columnMap.values()){
            byte[] bytes=result.getValue(Bytes.toBytes(column.familyName),Bytes.toBytes(column.columnName));
            if(bytes!=null){
                Object value=column.hbaseConvertor.toObject(column.field.getType(),bytes,column.args);
                column.field.set(hbaseDo,value);
            }
        }
        hbaseDo.setRowKey(Bytes.toString(result.getRow()));
        return hbaseDo;
    }


    private List<T> parseResultList(Result[] results) throws InstantiationException, IllegalAccessException {
        List<T> hbaseDoList=new ArrayList<>();
        if(results==null||results.length==0){
            return hbaseDoList;
        }
        for(Result result:results){
            T hbaseDo=parseResult(result);
            if(hbaseDo!=null){
                hbaseDoList.add(hbaseDo);
            }
        }
        return hbaseDoList;

    }
    private void putColumn(Put put,Column column,T hbaseDo) throws IllegalAccessException {
        Object doValue=column.field.get(hbaseDo);
        if(doValue!=null){
            byte[] valueBytes=column.hbaseConvertor.toBytes(doValue,column.args);
            if(valueBytes!=null){
                put.add(Bytes.toBytes(column.familyName),Bytes.toBytes(column.columnName),valueBytes);
            }
        }

    }

    protected abstract  String getStoreName();

    protected abstract String getTableName();


    private Map<String,Column> parseHabseDOClass(Class<?> clazz) throws IllegalAccessException, InstantiationException {
        if(clazz.getSuperclass()==null){
            return Collections.emptyMap();
        }
        Map<String,Column> columnMap=new HashMap<>();
        columnMap.putAll(parseHabseDOClass(clazz.getSuperclass()));

        Field[] fields=clazz.getDeclaredFields();
        for(Field field:fields){
            Column column=buildColumn(field);
            if(column!=null){
                columnMap.put(column.columnName,column);
            }
        }
        return columnMap;
    }

    private Column buildColumn(Field field) throws InstantiationException, IllegalAccessException {
        HbColumn hbColumn=field.getAnnotation(HbColumn.class);
        if(hbColumn!=null){
            field.setAccessible(true);
            Column column=new Column();
            column.field=field;
            if(StringUtils.isEmpty(hbColumn.value())){
                column.columnName=field.getName();
            }else{
                column.columnName=hbColumn.value();
            }
            column.familyName=hbColumn.familyName();
            column.args=hbColumn.args();
            column.hbaseConvertor=getConvert(hbColumn.convertClass());
            return column;
        }
        return null;
    }

    private HbaseConvertor getConvert(Class<? extends HbaseConvertor> convertorClass) throws IllegalAccessException, InstantiationException {
        if(convertorClassMap.containsKey(convertorClass)){
            return convertorClassMap.get(convertorClass);
        }
        HbaseConvertor convertor=convertorClass.newInstance();
        convertorClassMap.put(convertorClass,convertor);
        return convertor;

    }
    protected abstract Class<T> getHbaseDOClass();

    private static class Column{
        String columnName;
        String familyName;
        Field field;
        HbaseConvertor hbaseConvertor;
        String[] args;
    }
}

package hbase;

import hbase.model.CommentDo;

import java.util.List;

public interface HbaseCommentDAO  {

    public void insert(CommentDo commentDo);

    List<CommentDo> queryById(List<String> ids);
}

package hbase;

import hbase.model.CommentDo;

import java.io.IOException;
import java.util.List;

public class HbaseCommentDAOImpl extends HbaseBaseDAO<CommentDo> implements HbaseCommentDAO {


    @Override
    public void insert(CommentDo commentDo) {
        try {
            put(commentDo);
        } catch (IllegalAccessException e) {
        }
    }

    @Override
    public List<CommentDo> queryById(List<String> ids) {
        try {
            return batchget(ids);
        } catch (IOException e) {
            e.printStackTrace();
        } catch (IllegalAccessException e) {
            e.printStackTrace();
        } catch (InstantiationException e) {
            e.printStackTrace();
        }
        return null;
    }


    @Override
    protected String getStoreName() {
        return null;
    }

    @Override
    protected String getTableName() {
        return null;
    }

    @Override
    protected Class<CommentDo> getHbaseDOClass() {
        return CommentDo.class;
    }
}