背景:目前在做项目的时候,需要使用hbase进行数据的查询,一些评论记录,消费数据,交易日志等数据都是存储在hbase中的,因为都是海量的数据。了解hbase的原理是必要的,因此必须要学习好 rowkey的设计,否则会出现热点问题。可以百度这个rowkey 的设计,比如一般使用:随机,md5,分桶,反转等基本的手段就可以了。还需要了解hadoop的hdfs,因为hbase的HFile存储在hadoop上。
接下来就是基本你的api的封装,用于业务的调用
package hbase.annotation;
import hbase.convertor.HbaseConvertor;
import hbase.convertor.SimpleHbaseConvertor;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Retention(RetentionPolicy.RUNTIME)
@Target(value={ElementType.FIELD})
public @interface HbColumn {
/**
* 列名,一张hbase表中列名必须唯一,子类可以覆盖父类,为空表示与db一致
* @return
*/
String value() default "";
String familyName() default "f";
Class<? extends HbaseConvertor> convertClass() default SimpleHbaseConvertor.class ;
String[] args() default {};
}
package hbase.condition;
public class HbaseQueryCond {
private String startRowKey;
private String endRowKey;
private int limit=50;
private int offset=0;
private boolean needRevers;
public String getStartRowKey() {
return startRowKey;
}
public void setStartRowKey(String startRowKey) {
this.startRowKey = startRowKey;
}
public String getEndRowKey() {
return endRowKey;
}
public void setEndRowKey(String endRowKey) {
this.endRowKey = endRowKey;
}
public int getLimit() {
return limit;
}
public void setLimit(int limit) {
this.limit = limit;
}
public int getOffset() {
return offset;
}
public void setOffset(int offset) {
this.offset = offset;
}
public boolean isNeedRevers() {
return needRevers;
}
public void setNeedRevers(boolean needRevers) {
this.needRevers = needRevers;
}
}
package hbase.convertor;
public interface HbaseConvertor {
byte[] toBytes(Object value,String[] args);
Object toObject(Class<?> objectType,byte[] bytes,String[] args);
}
package hbase.convertor;
import com.alibaba.fastjson.JSON;
import com.sun.org.apache.xpath.internal.operations.Bool;
import org.apache.hadoop.hbase.util.Bytes;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class SimpleHbaseConvertor implements HbaseConvertor {
@Override
public byte[] toBytes(Object value, String[] args) {
if(value==null){
return null;
}
if(value.getClass()==byte[].class){
return (byte[]) value;
}
if(value.getClass()==String.class){
return Bytes.toBytes((String)value);
}
if(value.getClass()==Integer.class){
return Bytes.toBytes((Integer)value);
}
if(value.getClass()==Long.class){
return Bytes.toBytes((Long)value);
}
if(value.getClass()==Short.class){
return Bytes.toBytes((Short)value);
}
if(value.getClass()==Boolean.class){
return Bytes.toBytes((Boolean)value);
}
if(value.getClass()==Double.class){
return Bytes.toBytes((Double)value);
}
if(value.getClass()==Float.class){
return Bytes.toBytes((Float)value);
}
if(value.getClass()== Date.class){
String date=new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format((Date)value);
return Bytes.toBytes(date);
}
if(value.getClass()== Map.class){
String mapjson= JSON.toJSONString((Map)value);
return Bytes.toBytes(mapjson);
}
if(value.getClass()== List.class){
String listjson= JSON.toJSONString((List)value);
return Bytes.toBytes(listjson);
}
throw new RuntimeException("SimpleHbaseConvertor,不支持类型的转换"+value.getClass());
}
@Override
public Object toObject(Class<?> objectType, byte[] bytes, String[] args) {
if(bytes==null){
return null;
}
if(objectType==byte[].class){
return bytes;
}
if(objectType==String.class){
return Bytes.toString(bytes);
}
if(objectType==Integer.class || objectType==Integer.TYPE){
return Bytes.toInt(bytes);
}
if(objectType==Long.class || objectType==Long.TYPE){
return Bytes.toLong(bytes);
}
if(objectType== Boolean.class||objectType== Boolean.TYPE){
return Bytes.toBoolean(bytes);
}
if(objectType==Double.class || objectType==Double.TYPE){
return Bytes.toDouble(bytes);
}
if(objectType==Short.class || objectType==Short.TYPE){
return Bytes.toShort(bytes);
}
if(objectType==Float.class || objectType==Float.TYPE){
return Bytes.toFloat(bytes);
}
if(objectType==Date.class){
String date=Bytes.toString(bytes);
try {
return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse(date);
} catch (ParseException e) {
}
}
if(objectType==Map.class || objectType== HashMap.class){
String mapBytes=Bytes.toString(bytes);
return JSON.parseObject(mapBytes,Map.class);
}
if(objectType==List.class){
String listBytes=Bytes.toString(bytes);
return JSON.parseObject(listBytes,List.class);
}
throw new RuntimeException("SimpleHbaseConvertor,不支持类型的转换"+objectType);
}
}
package hbase.model;
public class BaseHbaseDo {
private String rowKey;
public String getRowKey() {
return rowKey;
}
public void setRowKey(String rowKey) {
this.rowKey = rowKey;
}
}
package hbase.model;
import hbase.annotation.HbColumn;
public class CommentDo extends BaseHbaseDo {
@HbColumn
private String id;
@HbColumn
private String comment;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getComment() {
return comment;
}
public void setComment(String comment) {
this.comment = comment;
}
}
package hbase.model;
import org.apache.hadoop.hbase.client.HTableInterface;
public interface HbaseDataSource {
public HTableInterface getHTable(String storeName,String tableName);
}
package hbase;
import hbase.annotation.HbColumn;
import hbase.condition.HbaseQueryCond;
import hbase.convertor.HbaseConvertor;
import hbase.model.BaseHbaseDo;
import hbase.model.HbaseDataSource;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.*;
public abstract class HbaseBaseDAO<T extends BaseHbaseDo> {
private HbaseDataSource hbaseDataSource;
private Map<String,Column> columnMap;
private Map<Class<? extends HbaseConvertor>,HbaseConvertor> convertorClassMap;
public void init() throws InstantiationException, IllegalAccessException {
convertorClassMap=new HashMap<>();
columnMap=parseHabseDOClass(getHbaseDOClass());
}
public void put(T hbaseDo) throws IllegalAccessException {
HTableInterface table=null;
try {
hbaseDataSource.getHTable(getStoreName(),getTableName());
Put put=new Put(Bytes.toBytes(hbaseDo.getRowKey()));
for(Column column:columnMap.values()){
putColumn(put,column,hbaseDo);
}
table.put(put);
} catch (IOException e) {
e.printStackTrace();
}finally {
if(table!=null){
try {
table.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
public void put(List<T> hbaseDos) throws IllegalAccessException {
HTableInterface table=null;
table=hbaseDataSource.getHTable(getStoreName(),getTableName());
List<Put> puts=new ArrayList<>();
for(T hbaseDo:hbaseDos){
Put put=new Put(Bytes.toBytes(hbaseDo.getRowKey()));
for(Column column:columnMap.values()){
putColumn(put,column,hbaseDo);
}
puts.add(put);
}
try {
table.put(puts);
} catch (IOException e) {
e.printStackTrace();
}finally {
if(table!=null){
try {
table.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
public T get(String rowKey) throws InstantiationException, IllegalAccessException {
HTableInterface table=null;
try {
table=hbaseDataSource.getHTable(getStoreName(),getTableName());
Get get=new Get(Bytes.toBytes(rowKey));
Result result=table.get(get);
return parseResult(result);
} catch (IOException e) {
throw new RuntimeException(e);
}finally {
if(table!=null){
try {
table.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
public List<T> batchget(List<String> rowKeyList) throws IOException, IllegalAccessException, InstantiationException {
if(CollectionUtils.isEmpty(rowKeyList)){
return Collections.emptyList();
}
HTableInterface table=null;
table=hbaseDataSource.getHTable(getStoreName(),getTableName());
List<Get> getList=new ArrayList<>();
for(String rowKey:rowKeyList){
Get get=new Get(Bytes.toBytes(rowKey));
getList.add(get);
}
Result[] results=table.get(getList);
return parseResultList(results);
}
public List<T> scan(HbaseQueryCond hbaseQueryCond) throws IOException {
if(hbaseQueryCond==null){
return Collections.emptyList();
}
ResultScanner rs=null;
HTableInterface table=null;
try {
table=hbaseDataSource.getHTable(getStoreName(),getTableName());
rs=table.getScanner(buildScan(hbaseQueryCond));
return buildResultScanner(rs);
} catch (Exception e) {
throw new RuntimeException(e);
}finally {
if(table!=null){
table.close();
}
}
}
public List<T> pageScan(HbaseQueryCond cond){
if(cond==null){
return Collections.emptyList();
}
ResultScanner rs=null;
HTableInterface table=null;
try {
table=hbaseDataSource.getHTable(getStoreName(),getTableName());
rs=table.getScanner(buildPageScan(cond));
return buildResultScanner(rs);
} catch (Exception e) {
throw new RuntimeException(e);
}finally {
if(table!=null){
try {
table.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
public void delete(String rowKey) throws IOException {
HTableInterface table=null;
try {
table = hbaseDataSource.getHTable(getStoreName(), getTableName());
Delete delete=new Delete(Bytes.toBytes(rowKey));
table.delete(delete);
}catch (Exception e){
if(table!=null){
table.close();
}
}
}
private List<T> parseResultScanner(ResultScanner rs,int limit) throws InstantiationException, IllegalAccessException {
List<T> hbaseDoList=new ArrayList<>();
int count=0;
for(Result result:rs){
T hbaseDo=parseResult(result);
if(hbaseDo!=null){
hbaseDoList.add(hbaseDo);
count++;
}
if(count>=limit){
break;
}
}
return hbaseDoList;
}
private List<T> buildResultScanner(ResultScanner rs) throws InstantiationException, IllegalAccessException {
List<T> hbaseDOList=new ArrayList<>();
if(rs==null){
return Collections.emptyList();
}
for(Result result:rs){
T hbaseDo=parseResult(result);
hbaseDOList.add(hbaseDo);
}
return hbaseDOList;
}
private Scan buildScan(HbaseQueryCond cond){
Scan scan=new Scan();
if(!StringUtils.isEmpty(cond.getStartRowKey())){
scan.setStartRow(Bytes.toBytes(cond.getStartRowKey()));
}
if(!StringUtils.isEmpty(cond.getEndRowKey())){
scan.setStopRow(Bytes.toBytes(cond.getEndRowKey()));
}
scan.setBatch(1000);
return scan;
}
public Scan buildPageScan(HbaseQueryCond cond){
Scan scan=new Scan();
if(!StringUtils.isEmpty(cond.getStartRowKey())){
scan.setStartRow(Bytes.toBytes(cond.getStartRowKey()));
}
if(!StringUtils.isEmpty(cond.getEndRowKey())){
scan.setStopRow(Bytes.toBytes(cond.getEndRowKey()));
}
if(cond.getLimit()>200){
scan.setCaching(200);
}
scan.setCaching(cond.getLimit()+1);
scan.setBatch(1000);
return scan;
}
private T parseResult(Result result) throws IllegalAccessException, InstantiationException {
if(result.isEmpty()){
return null;
}
T hbaseDo=getHbaseDOClass().newInstance();
for(Column column:columnMap.values()){
byte[] bytes=result.getValue(Bytes.toBytes(column.familyName),Bytes.toBytes(column.columnName));
if(bytes!=null){
Object value=column.hbaseConvertor.toObject(column.field.getType(),bytes,column.args);
column.field.set(hbaseDo,value);
}
}
hbaseDo.setRowKey(Bytes.toString(result.getRow()));
return hbaseDo;
}
private List<T> parseResultList(Result[] results) throws InstantiationException, IllegalAccessException {
List<T> hbaseDoList=new ArrayList<>();
if(results==null||results.length==0){
return hbaseDoList;
}
for(Result result:results){
T hbaseDo=parseResult(result);
if(hbaseDo!=null){
hbaseDoList.add(hbaseDo);
}
}
return hbaseDoList;
}
private void putColumn(Put put,Column column,T hbaseDo) throws IllegalAccessException {
Object doValue=column.field.get(hbaseDo);
if(doValue!=null){
byte[] valueBytes=column.hbaseConvertor.toBytes(doValue,column.args);
if(valueBytes!=null){
put.add(Bytes.toBytes(column.familyName),Bytes.toBytes(column.columnName),valueBytes);
}
}
}
protected abstract String getStoreName();
protected abstract String getTableName();
private Map<String,Column> parseHabseDOClass(Class<?> clazz) throws IllegalAccessException, InstantiationException {
if(clazz.getSuperclass()==null){
return Collections.emptyMap();
}
Map<String,Column> columnMap=new HashMap<>();
columnMap.putAll(parseHabseDOClass(clazz.getSuperclass()));
Field[] fields=clazz.getDeclaredFields();
for(Field field:fields){
Column column=buildColumn(field);
if(column!=null){
columnMap.put(column.columnName,column);
}
}
return columnMap;
}
private Column buildColumn(Field field) throws InstantiationException, IllegalAccessException {
HbColumn hbColumn=field.getAnnotation(HbColumn.class);
if(hbColumn!=null){
field.setAccessible(true);
Column column=new Column();
column.field=field;
if(StringUtils.isEmpty(hbColumn.value())){
column.columnName=field.getName();
}else{
column.columnName=hbColumn.value();
}
column.familyName=hbColumn.familyName();
column.args=hbColumn.args();
column.hbaseConvertor=getConvert(hbColumn.convertClass());
return column;
}
return null;
}
private HbaseConvertor getConvert(Class<? extends HbaseConvertor> convertorClass) throws IllegalAccessException, InstantiationException {
if(convertorClassMap.containsKey(convertorClass)){
return convertorClassMap.get(convertorClass);
}
HbaseConvertor convertor=convertorClass.newInstance();
convertorClassMap.put(convertorClass,convertor);
return convertor;
}
protected abstract Class<T> getHbaseDOClass();
private static class Column{
String columnName;
String familyName;
Field field;
HbaseConvertor hbaseConvertor;
String[] args;
}
}
package hbase;
import hbase.model.CommentDo;
import java.util.List;
public interface HbaseCommentDAO {
public void insert(CommentDo commentDo);
List<CommentDo> queryById(List<String> ids);
}
package hbase;
import hbase.model.CommentDo;
import java.io.IOException;
import java.util.List;
public class HbaseCommentDAOImpl extends HbaseBaseDAO<CommentDo> implements HbaseCommentDAO {
@Override
public void insert(CommentDo commentDo) {
try {
put(commentDo);
} catch (IllegalAccessException e) {
}
}
@Override
public List<CommentDo> queryById(List<String> ids) {
try {
return batchget(ids);
} catch (IOException e) {
e.printStackTrace();
} catch (IllegalAccessException e) {
e.printStackTrace();
} catch (InstantiationException e) {
e.printStackTrace();
}
return null;
}
@Override
protected String getStoreName() {
return null;
}
@Override
protected String getTableName() {
return null;
}
@Override
protected Class<CommentDo> getHbaseDOClass() {
return CommentDo.class;
}
}