前言
Hadoop中的Hbase，作为分布式数据库，其重要性不言而喻，在这里不再讲述Hbase的相关理论知识，直接展示的是Hbase的API操作。相较于网上的大多数Hbase的API操作而言，本篇博客的API是针对于新的Hbase的，在API中很多旧的方法都不能再用了，已经被淘汰了。
提示：以下是本篇文章正文内容，下面案例可供参考
一、项目成分
这里展示的不仅仅是操作的代码，还有其他一些比较零散的代码，所以先展示整个代码的成分，两个配置文件，两个代码文件。
二、代码
1.配置文件
hbase.properties的内容如下，写的是连接的节点对象，三台节点
hbase.zookeeper.quorum=node1:2181,node2:2181,node3:2181
log4j.properties的内容如下
log4j.rootLogger=info,appender
log4j.appender.appender=org.apache.log4j.ConsoleAppender
log4j.appender.appender.layout=org.apache.log4j.PatternLayout
log4j.appender.appender.layout.ConversionPattern=%d{yy-MM-dd HH:mm:ss,SSS} [hbase] %p [%t] %C.%M(%L) %m%n
2.API
下面是HBaseUtil里面的代码，这个类里面写的都是相关的API操作，对集群上的表进行的一系列的操作，代码有点长。。。。。。
import org.apache.commons.beanutils.ConvertUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.util.*;
/**
* @ClassName HBaseUtil
* @Author MerLiang
* @Date 2021_05_10_11:07
*/
public class HBaseUtil {
private static Connection conn;
private static Properties props;
private static Logger log = LoggerFactory.getLogger(HBaseUtil.class);
/**
* 读取配置文件，连接Hbase
*/
static {
props = new Properties();
try {
props.load(new InputStreamReader(Objects.requireNonNull(HBaseUtil.class.getClassLoader().getResourceAsStream("hbase.properties")), "UTF-8"));
} catch (IOException e) {
log.error("配置文件读取异常", e);
}
Configuration conf = HBaseConfiguration.create();
String value = props.getProperty("hbase.zookeeper.quorum");
if (StringUtils.isNotBlank(value)) {
conf.set("hbase.zookeeper.quorum", value.trim());
}
try {
conn = ConnectionFactory.createConnection(conf);
} catch (IOException e) {
log.error("获取HBase连接异常", e);
}
}
/**
* 获取Hbase的无参连接
*
* @return
*/
public static Connection getConnection() {
return conn;
}
/**
* Hbase的有参连接
*
* @param quorum
* @return
*/
public static Connection getConnection(String quorum) {
Configuration conf = HBaseConfiguration.create();
conf.set("hbase.zookeeper.quorum", quorum);
try {
return ConnectionFactory.createConnection(conf);
} catch (IOException e) {
log.error("获取HBase连接异常", e);
}
return null;
}
/**
* 设置有参Hbase连接对象
*
* @param connection
*/
public static void setConnection(Connection connection) {
conn = connection;
}
/**
* 先判断Hbase的连接是否正常，再获取Admin
*
* @return
*/
public static Admin getAdmin() {
if (check()) {
try {
return conn.getAdmin();
} catch (IOException e) {
log.error("获取Admin对象异常", e);
}
}
return null;
}
/**
* 先判断Hbase的连接是否正常，再获取表
*
* @param tableName
* @return
*/
public static Table getTable(String tableName) {
if (check()) {
try {
return conn.getTable(TableName.valueOf(tableName));
} catch (IOException e) {
log.error("获取Table对象异常", e);
}
}
return null;
}
/**
* 创建表格 表名，列族，最小版本号，最大版本号(测试成功)
*
* @param tableName
* @param family
* @param minVersion
* @param maxVersion
* @return
*/
public static boolean createTable(String tableName, String family, int minVersion, int maxVersion) {
List<String> families = new LinkedList();
families.add(family);
return createTable(tableName, families, minVersion, maxVersion);
}
/**
* 创建表格 表名，列族，版本号(测试成功)
*
* @param tableName
* @param family
* @param version
* @return
*/
public static boolean createTable(String tableName, String family, int version) {
return createTable(tableName, family, version, version);
}
/**
* 创建表格 表名，列族，默认版本号1(测试成功)
*
* @param tableName
* @param family
* @return
*/
public static boolean createTable(String tableName, String family) {
return createTable(tableName, family, 1, 1);
}
/**
* 创建表格 表名，最小版本号，最大版本号，可变参数列族（测试成功）
*
* @param tableName
* @param minVersion
* @param maxVersion
* @param family
* @return
*/
public static boolean createTable(String tableName, int minVersion, int maxVersion, String... family) {
List<String> families = Arrays.asList(family);
return createTable(tableName, families, minVersion, maxVersion);
}
/**
* 创建表格 表名，版本号，可变参数列族（测试成功）
*
* @param tableName
* @param version
* @param family
* @return
*/
public static boolean createTable(String tableName, int version, String... family) {
return createTable(tableName, version, version, family);
}
/**
* 创建表格 表名，可变参数列族，默认版本号1（测试成功）
*
* @param tableName
* @param family
* @return
*/
public static boolean createTable(String tableName, String... family) {
return createTable(tableName, 1, 1, family);
}
/**
* 创建表格 表名，String类型的List集合列族，最小版本号，最大版本号（测试成功）
*
* @param tableName
* @param families
* @param minVersion
* @param maxVersion
* @return
*/
public static boolean createTable(String tableName, List<String> families, int minVersion, int maxVersion) {
if (families == null || families.isEmpty()) {
log.error("列族数据为空，创建表失败");
return false;
}
Admin admin = getAdmin();
boolean status = false;
checkVersion(minVersion, maxVersion);
if (admin != null) {
List<ColumnFamilyDescriptor> familyDescriptors = CommonUtil.convert(families, family -> ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)).setMinVersions(minVersion).setMaxVersions(maxVersion).build());
System.out.println(familyDescriptors);
TableDescriptor desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName)).setColumnFamilies(familyDescriptors).build();
try {
admin.createTable(desc);
status = true;
} catch (IOException e) {
log.error("创建表操作异常", e);
} finally {
if (admin != null) {
try {
admin.close();
} catch (IOException e) {
log.error("关闭Admin对象异常", e);
}
}
}
}
return status;
}
/**
* 创建表格 表名，String类型的List集合列族，版本号（测试成功）
*
* @param tableName
* @param families
* @param version
* @return
*/
public static boolean createTable(String tableName, List<String> families, int version) {
return createTable(tableName, families, 1, version);
}
/**
* 创建表格 表名，String类型的List集合列族（测试成功）
*
* @param tableName
* @param families
* @return
*/
public static boolean createTable(String tableName, List<String> families) {
return createTable(tableName, families, 1);
}
/**
* (测试通过)
* <p>
* 添加一行数据 表名，行名，列族名，多个数值（String类型的列族，Object类型的值）
* 添加多个数据，数据的批量插入
*
* @param tableName
* @param rowKey
* @param family
* @param map
* @return
*/
public static boolean putByRowKey(String tableName, String rowKey, String family, Map<String, Object> map) {
if (map == null || map.isEmpty()) {
log.warn("数据为空，未添加任何数据");
return false;
}
Table table = getTable(tableName);
boolean result = false;
if (table != null) {
try {
Put put = new Put(Bytes.toBytes(rowKey));
//map的key是列名，value是列值
map.forEach((key, value) -> {
put.addColumn(Bytes.toBytes(family), Bytes.toBytes(key), Bytes.toBytes(value == null ? "null" : value.toString()));
});
table.put(put);
result = true;
} catch (IOException e) {
log.error("插入数据异常", e);
} finally {
try {
table.close();
} catch (IOException e) {
log.error("关闭表对象异常", e);
}
}
}
return result;
}
/**
* (测试通过)
* 添加一行数据 表名，行名，列族名，列名，值
* 单一数据的插入
*
* @param tableName
* @param rowKey
* @param family
* @param column
* @param value
* @return
*/
public static boolean putByRowKey(String tableName, String rowKey, String family, String column, Object value) {
Table table = getTable(tableName);
boolean result = false;
if (table != null) {
try {
Put put = new Put(Bytes.toBytes(rowKey));
put.addColumn(Bytes.toBytes(family), Bytes.toBytes(column), Bytes.toBytes(value == null ? "null" : value.toString()));
table.put(put);
result = true;
} catch (IOException e) {
log.error("插入数据异常", e);
} finally {
try {
table.close();
} catch (IOException e) {
log.error("关闭表对象异常", e);
}
}
}
return result;
}
/**
* (测试通过)
* <p>
* 添加一行数据 表名，行名，列族名，范类
* 先将范类转化为String类型的map
*
* @param tableName
* @param rowKey
* @param family
* @param bean
* @param <T>
* @return
*/
public static <T> boolean putByRowKey(String tableName, String rowKey, String family, T bean) {
return putByRowKey(tableName, rowKey, family, CommonUtil.convert(bean));
}
/**
* (测试通过)
* <p>
* 添加数据 表名，列族，范类
*
* @param tableName
* @param family
* @param bean
* @param <T>
* @return
*/
public static <T> boolean put(String tableName, String family, T bean) {
String rowKey;
try {
// 先判断范类中是否有id字段，id字段即为HBase中的rowKey，如果有，则保存数据
rowKey = CommonUtil.getFieldValue(bean, "id");
if (rowKey == null) {
log.error("对象的id字段数据为空，添加数据失败");
return false;
}
} catch (NoSuchFieldException e) {
log.error("对象必须有id字段，但是接收到的对象不具备id字段", e);
return false;
} catch (IllegalAccessException e) {
log.error("字段id不具备访问权限", e);
return false;
}
return putByRowKey(tableName, rowKey, family, CommonUtil.convert(bean));
}
/**
* (测试通过)
* <p>
* 添加数据 表名，列族，数据链表
* 输入一个链表类型的数据，作为要输入的数据
*
* @param tableName
* @param family
* @param beans
* @param <T>
* @return
*/
public static <T> boolean puts(String tableName, String family, List<T> beans) {
// 先判断链表是否为空，为空直接不输入
if (beans == null || beans.isEmpty()) {
log.warn("数据为空，未添加任何数据");
return false;
}
Table table = getTable(tableName);
boolean result = false;
if (table != null) {
try {
List<Put> puts = new ArrayList<>();
for (T bean : beans) {
puts.add(convertToPut(bean, family));
}
table.put(puts);
result = true;
} catch (IOException e) {
log.error("插入数据异常", e);
} catch (NoSuchFieldException e) {
log.error("对象必须有id字段，但是接收到的对象不具备id字段", e);
} catch (IllegalAccessException e) {
log.error("字段 id 不具备访问权限", e);
} finally {
try {
table.close();
} catch (IOException e) {
log.error("关闭表对象异常", e);
}
}
}
return result;
}
/**
* @param tableName
* @param rowKey
* @param family
* @param beanType
* @param <T>
* @return
*/
public static <T> T getBeanByRowKey(String tableName, String rowKey, String family, Class<T> beanType) {
Table table = getTable(tableName);
T t = null;
if (table != null) {
try {
Result result = table.get(new Get(Bytes.toBytes(rowKey)));
Map<byte[], byte[]> familyMap = result.getFamilyMap(Bytes.toBytes(family));
Constructor<T> constructor = beanType.getConstructor();
t = constructor.newInstance();
for (Map.Entry<byte[], byte[]> entry : familyMap.entrySet()) {
try {
Field field = beanType.getDeclaredField(Bytes.toString(entry.getKey()));
field.setAccessible(true);
field.set(t, ConvertUtils.convert(Bytes.toString(entry.getValue()), field.getType()));
} catch (NoSuchFieldException | IllegalAccessException e) {
log.warn("{} 不是 {} 的成员", Bytes.toString(entry.getKey()), beanType);
}
}
} catch (IOException e) {
log.error("根据 RowKey 获取数据失败", e);
} catch (IllegalAccessException e) {
log.error("实例化Bean对象异常，BeanType没有公共的无参构造器", e);
} catch (InvocationTargetException e) {
e.printStackTrace();
} catch (NoSuchMethodException e) {
log.error("BeanType必须具备无参构造器, 而当前对象不具备无参构造器", e);
} catch (InstantiationException e) {
log.error("实例化Bean对象异常", e);
}
}
return t;
}
/**
* 修改表名(测试通过)
*
* @param oldTableName
* @param newTableName
*/
public static void alterTableName(String oldTableName, String newTableName) {
Admin admin = getAdmin();
try {
if (admin.tableExists(TableName.valueOf(oldTableName))) {
String snapshotName = oldTableName + "_snap";
// 禁用表
admin.disableTable(TableName.valueOf(oldTableName));
// 为表做快照
admin.snapshot(snapshotName, TableName.valueOf(oldTableName));
// 从快照用克隆出一个新的表
admin.cloneSnapshot(snapshotName, TableName.valueOf(newTableName));
//删除快照
admin.deleteSnapshot(snapshotName);
// 删除旧表
admin.deleteTable(TableName.valueOf(oldTableName));
}
} catch (IOException e) {
log.error(oldTableName + "不存在", e);
}
}
/**
* 向列族插入数据 表名，列数据
*
* @param tableName
* @param map
*/
public static void putFamilies(String tableName, Map<String, String[]> map) {
Admin admin = getAdmin();
try {
if (!admin.tableExists(TableName.valueOf(tableName))) {
return;
}
Table table = getTable(tableName);
List<Put> list = new ArrayList<>();
Iterator iterator = map.keySet().iterator();
while (iterator.hasNext()) {
String key = iterator.next().toString();
Put put = new Put(Bytes.toBytes(key));
String[] arr = map.get(key);
for (int i = 0; i < arr.length; i += 3) {
put.addColumn(Bytes.toBytes(arr[i]), Bytes.toBytes(arr[i + 1]), Bytes.toBytes(arr[i + 2]));
}
list.add(put);
}
table.put(list);
} catch (IOException e) {
log.error(tableName + "不存在");
}
}
/**
* 查询指定行的数据
*
* @param tableName
* @param rowKey
*/
public static void getResult(String tableName, String rowKey) {
Get get = new Get(Bytes.toBytes(rowKey));
try {
Table table = getTable(tableName);
Result result = table.get(get);
for (Cell keyValue : result.listCells()) {
// 行名
System.out.println("rowKey" + Bytes.toString(keyValue.getRowArray()));
//列族名
log.info("family :" + Bytes.toString(keyValue.getFamilyArray()));
// 列名
log.info("qualifier" + Bytes.toString(keyValue.getQualifierArray()));
//列对应的值
log.info("value" + Bytes.toString(keyValue.getValueArray()));
// 时间戳
log.info("timestamp" + keyValue.getTimestamp());
}
} catch (IOException e) {
log.error(tableName + "不存在", e);
}
}
/**
* (测试通过)
* 删除某行数据
*
* @param tablename
* @param rowkey
*/
public static void deleteRow(String tablename, String rowkey) {
Admin admin = getAdmin();
try {
if (!admin.tableExists(TableName.valueOf(tablename))) {
return;
}
Table table = getTable(tablename);
Delete delete = new Delete(Bytes.toBytes(rowkey));
table.delete(delete);
} catch (IOException e) {
log.error(tablename + "不存在", e);
}
}
/**
* (测试通过)
* 删除指定行指定列族中的指定列数据
*
* @param tableName
* @param rowkey
* @param colFamily
* @param col
*/
public static void deleteOneCol(String tableName, String rowkey, String colFamily, String col) {
Admin admin = getAdmin();
try {
if (!admin.tableExists(TableName.valueOf(tableName))) {
return;
}
Table table = getTable(tableName);
Delete delete = new Delete(Bytes.toBytes(rowkey));
delete.addColumn(Bytes.toBytes(colFamily), Bytes.toBytes(col));
table.delete(delete);
} catch (IOException e) {
log.error(tableName + "不存在", e);
}
}
/**
* 删除表
*
* @param tableName
*/
public void deleteTable(String tableName) {
Admin admin = getAdmin();
try {
if (admin.tableExists(TableName.valueOf(tableName))) {
admin.disableTable(TableName.valueOf(tableName));
admin.deleteTable(TableName.valueOf(tableName));
}
} catch (IOException e) {
log.error(tableName + "不存在", e);
}
}
/**
* 查询表全部的列族的名字
*
* @param tableName
* @return
*/
public static List getColumnNames(String tableName) {
Table table = getTable(tableName);
List<String> list = new ArrayList<>();
try {
TableDescriptor desc = table.getDescriptor();
for (ColumnFamilyDescriptor des : desc.getColumnFamilies()) {
list.add(des.getNameAsString());
}
} catch (IOException e) {
e.printStackTrace();
}
return list;
}
/**
* 查询表的结构
*
* @param tableName
*/
public static void descTable(String tableName) {
Table table = getTable(tableName);
try {
TableDescriptor tableDescriptor = table.getDescriptor();
ColumnFamilyDescriptor[] columns = tableDescriptor.getColumnFamilies();
for (ColumnFamilyDescriptor cfd : columns) {
log.info(Bytes.toString(cfd.getName()));
}
} catch (IOException e) {
log.error(tableName + "不存在", e);
}
}
/**
* 查询所有表名(测试通过)
*/
public static void listTableNames() {
Admin admin = getAdmin();
try {
TableName[] tableNames = admin.listTableNames();
Arrays.stream(tableNames).forEach(System.out::println);
} catch (IOException e) {
log.error("没有表", e);
}
}
/**
* 检查Hbase是否连接成功
*
* @return
*/
private static boolean check() {
if (conn == null) {
log.error("创建HBase连接失败，请重新配置HBase的Zookeeper地址");
return false;
}
return true;
}
/**
* 检查版本号是否错误
*
* @param minVersion
* @param maxVersion
* @throws IllegalArgumentException
*/
private static void checkVersion(int minVersion, int maxVersion) throws IllegalArgumentException {
if (minVersion > maxVersion) {
throw new IllegalArgumentException(maxVersion + "必须大于等于" + minVersion);
}
if (minVersion < 1) {
throw new IllegalArgumentException(minVersion + "必须大于等于1");
}
}
/**
* 将输入的类转化为String类型的map
*
* @param map
* @return
*/
private static Map<String, String> convert(Map<byte[], byte[]> map) {
Map<String, String> result = new HashMap<>();
if (map != null) {
map.forEach((byte[] k, byte[] v) -> result.put(Bytes.toString(k), Bytes.toString(v)));
}
return result;
}
/**
* 传入范类和列族名，返回列名和值
*
* @param bean
* @param family
* @param <T>
* @return
* @throws NoSuchFieldException
* @throws IllegalAccessException
*/
private static <T> Put convertToPut(T bean, String family) throws NoSuchFieldException, IllegalAccessException {
Put put = new Put(Bytes.toBytes(CommonUtil.getFieldValue(bean, "id")));
CommonUtil.convert(bean).forEach((key, value) -> put.addColumn(Bytes.toBytes(family), Bytes.toBytes(key), Bytes.toBytes(value == null ? "null" : value.toString())));
return put;
}
/**
* 判断表中是否有指定列族
*
* @param tableName
* @param columnName
* @return
*/
private static boolean isColumnExists(String tableName, String columnName) {
for (Object s : getColumnNames(tableName)) {
if (s.toString() == columnName) {
return true;
}
}
return false;
}
/**
* 判断是否所有列族都在表中
*
* @param tableName
* @param columnName
* @return
*/
private static boolean isColumnExists(String tableName, String... columnName) {
int sum = 0;
for (String s : columnName) {
if (isColumnExists(tableName, s)) {
sum += 1;
}
}
return sum == columnName.length;
}
}
还剩下最后一个类，就是关于HBaseUtil类中方法的调用，代码如下
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
/**
* @ClassName CommonUtil
* @Author MerLiang
* @Date 2021_05_10_10:42
*/
public class CommonUtil {
private static Logger log = LoggerFactory.getLogger(CommonUtil.class);
public static <T> Map<String, Object> convert(T bean) {
return convert(bean, true);
}
public static <T> Map<String, Object> convert(T bean, boolean isIgnoreNull) {
Map<String, Object> result = new HashMap<>();
if (bean != null) {
Class clazz = bean.getClass();
Field[] fields = clazz.getDeclaredFields();
for (Field field : fields) {
field.setAccessible(true);
try {
Object o = field.get(bean);
if (!isIgnoreNull || o != null) {
result.put(field.getName(), o.toString());
}
} catch (IllegalAccessException e) {
log.error("访问权限异常", e);
} catch (Exception e) {
log.error("转换异常", e);
}
}
}
return result;
}
public static <E, T> List<E> convert(List<T> list, Function<T, E> func) {
return list.stream().collect(ArrayList::new, (li, p) -> li.add(func.apply(p)), List::addAll);
}
/**
* 得到字段fieldName的值
*
* @param bean
* @param fieldName
* @param <T>
* @return
* @throws NoSuchFieldException
* @throws IllegalAccessException
*/
public static <T> String getFieldValue(T bean, String fieldName) throws NoSuchFieldException, IllegalAccessException {
if (bean != null) {
Class clazz = bean.getClass();
Field id = clazz.getDeclaredField(fieldName);
id.setAccessible(true);
Object value = id.get(bean);
return value == null ? null : value.toString();
}
return null;
}
public static void main(String[] args) {
HBaseUtil.createTable("s223", "school");
List<String> list = null;
list.add("ml");
list.add("ls");
HBaseUtil.createTable("s2", list);
System.out.println(HBaseUtil.getColumnNames("s1"));
HBaseUtil.alterTableName("Students", "Studentes");
HBaseUtil.putByRowKey("s223", "1001", "school", "xbie", "nan");
Map<String, Object> map = new HashMap<>();
map.put("age", 12);
map.put("name", "merliang");
HBaseUtil.putByRowKey("s1", "1001", "school", map);
HBaseUtil.deleteRow("s1", "1001");
HBaseUtil.deleteOneCol("s2", "1001", "info", "name");
Map<String, String[]> maps = new HashMap<>();
map.put("1001", new String[]{"info", "name", "一", "info", "age", "21"});
map.put("1002", new String[]{"info", "name", "二", "info", "age", "22"});
map.put("1003", new String[]{"info", "name", "三", "info", "age", "23"});
HBaseUtil.putFamilies("s1", maps);
HBaseUtil.getResult("s1", "1001");
HBaseUtil.listTableNames();
System.out.println(HBaseUtil.getColumnNames("s223"));
}
}
总结
以上的类中的所有方法都经过测试，都是可以使用的，该API测试时使用的HBase版本为2.2.6。
扫一扫
477
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
