Springboot整合HBase客户端

1.HBase单机集群搭建
2.springboot整合HBase客户端

1.Hbase单机伪集群搭建

拉取镜像

docker pull harisekhon/hbase

启动容器

docker run -d -h docker-hbase \
        -p 2181:2181 \
        -p 8099:8080 \
        -p 8085:8085 \
        -p 9090:9090 \
        -p 9000:9000 \
        -p 9095:9095 \
        -p 16000:16000 \
        -p 16010:16010 \
        -p 16201:16201 \
        -p 16301:16301 \
        -p 16020:16020\
        --name hbase \
        harisekhon/hbase

设置host

在服务的host文件中设置如下映射
127.0.0.1 docker-hbase

访问测试

http://localhost:16010/master-status

进入容器

docker exec -it hbase /bin/bash

基本命令

创建表
	create 'table1' , 'cf1'
删除表
	 disable 't1'
	 drop 't1'
查看表结构
	 describe 't1'
全表扫描
	scan 'test2'
统计表行数
	 count  'OPC_TEST'
查看表信息
	desc  'OPC_TEST'
列出所有命名空间
	 list_namespace 
查看命名空间下的表
	list_namespace_tables 'default'
查看命名空间
	describe_namespace 'hbase'
直接进入hbase shell
	docker exec -ti hbase1 /hbase/bin/hbase shell

全表扫描，匹配rowkey的前缀
	scan 'OPC_TEST',{FILTER=>"RowFilter(=,'substring:1122XZI8904S')"}
显示rowkey字节顺序大于002的键值对
	scan 'OPC_TEST',FILTER=>"RowFilter(>,'binary:002')"
扫描前缀为001的rowkey
	scan 'OPC_TEST',FILTER=>"PrefixFilter('001')"
按时间戳
	scan 'OPC_TEST',{COLUMNS=> 'OPC_COLUMNS',TIMERANGE =>[1592464809160,1592464809166]}
扫描全表，查询指定时间范围和匹配rowkey前缀
	scan 'OPC_TEST',{COLUMNS=> 'OPC_COLUMNS',TIMERANGE =>[1592464809160,1592464809166],FILTER=>"RowFilter(=,'substring:118883XHZ9904S')"}

2.Springboot 整合 HBase客户端

引入pom

**注意jar包版本和服务器的对应关系
这里服务器版本是2.1.3，jar包是2.1.1

<dependency>
  <groupId>org.apache.hbase</groupId>
    <artifactId>hbase-client</artifactId>
    <version>2.1.1</version>
     <exclusions>
       <exclusion>
            <groupId>javax.servlet</groupId>
            <artifactId>servlet-api</artifactId>
        </exclusion>
    </exclusions>
</dependency>

application.yml中增加配置

## HBase 配置
hbase:
  config:
    hbase.zookeeper.quorum: xx.x.x.xx
    hbase.zookeeper.port: 2181
    hbase.zookeeper.znode: /hbase
    hbase.client.keyvalue.maxsize: 1572864000

配置类

@Data
@ConfigurationProperties(prefix = "hbase")
public class HbaseProperties {
	
    private Map<String, String> config;
}

@Configuration
@EnableConfigurationProperties(HbaseProperties.class)
public class HbaseConfig {
	
	private final HbaseProperties properties;
	
	public HbaseConfig(HbaseProperties properties) {
		this.properties = properties;
	}
 
	@Bean
	public org.apache.hadoop.conf.Configuration configuration() {
		
		org.apache.hadoop.conf.Configuration configuration = HBaseConfiguration.create();
		
		Map<String, String> config = properties.getConfig();
		
		Set<String> keySet = config.keySet();
		for (String key : keySet) {
			configuration.set(key, config.get(key));
		}
		return configuration;
	}
}

Hbase工具类

@Slf4j
@Component
@DependsOn("hbaseConfig")
public class HBaseClient {

	@Autowired
	private HbaseConfig config;

	private static Admin admin = null;
	
	public static Configuration conf=null;
	
	private static Connection connection = null;
	
	private ThreadLocal<List<Put>> threadLocal = new ThreadLocal<List<Put>>();
	
	private static final int CACHE_LIST_SIZE = 1000;
	
	  
	
	@PostConstruct
	private void init() {
		if (connection != null) {
			return;
		}
		
		try {
		 	connection = ConnectionFactory.createConnection(config.configuration());
			admin = connection.getAdmin();
		} catch (IOException e) {
			log.error("HBase create connection failed: {}", e);
		}
	}

	/**
	 * 创建表
	 * experssion : create 'tableName','[Column Family 1]','[Column Family 2]'
	 * @param tableName 	  表名
	 * @param columnFamilies 列族名
	 * @throws IOException
	 */
	public void createTable(String tableName, String... columnFamilies) throws IOException {
		TableName name = TableName.valueOf(tableName);
		boolean isExists = this.tableExists(tableName);
		
		if (isExists) {
			throw new TableExistsException(tableName + "is exists!");
		}
		
		TableDescriptorBuilder descriptorBuilder = TableDescriptorBuilder.newBuilder(name);
		List<ColumnFamilyDescriptor> columnFamilyList = new ArrayList<>();
		
		for (String columnFamily : columnFamilies) {
			ColumnFamilyDescriptor columnFamilyDescriptor = ColumnFamilyDescriptorBuilder
					.newBuilder(columnFamily.getBytes()).build();
			columnFamilyList.add(columnFamilyDescriptor);
		}
		
		descriptorBuilder.setColumnFamilies(columnFamilyList);
		TableDescriptor tableDescriptor = descriptorBuilder.build();
		admin.createTable(tableDescriptor);
	}

	/**
	 * 插入或更新
	 *  experssion : put <tableName>,<rowKey>,<family:column>,<value>,<timestamp>
	 * @param tableName 	表名
	 * @param rowKey		行id
	 * @param columnFamily  列族名
	 * @param column 		列
	 * @param value 		值
	 * @throws IOException
	 */
	public void insertOrUpdate(String tableName, String rowKey, String columnFamily, String column, String value)
			throws IOException {
		this.insertOrUpdate(tableName, rowKey, columnFamily, new String[]{column}, new String[]{value});
	}

	/** 
	 *   插入或更新多个字段
	 * experssion : put <tableName>,<rowKey>,<family:column>,<value>,<timestamp>
	 * @param tableName 	表名
	 * @param rowKey        行id
	 * @param columnFamily  列族名
	 * @param columns		列
	 * @param values		值
	 * @throws IOException
	 */
	public void insertOrUpdate(String tableName, String rowKey, String columnFamily, String[] columns, String[] values)
			throws IOException {
		Table table = connection.getTable(TableName.valueOf(tableName));
		
		Put put = new Put(Bytes.toBytes(rowKey));
		
		for (int i = 0; i < columns.length; i++) {
			put.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(columns[i]), Bytes.toBytes(values[i]));
			table.put(put);
		}
	}

	/** 
	 * 删除行
	 * @param tableName		表名
	 * @param rowKey		行id
	 * @throws IOException
	 */
	public void deleteRow(String tableName, String rowKey) throws IOException {
		Table table = connection.getTable(TableName.valueOf(tableName));
		
		Delete delete = new Delete(rowKey.getBytes());
		
		table.delete(delete);
	}

	/**
	 * 删除列族
	 * @param tableName		表名
	 * @param rowKey		行id
	 * @param columnFamily	列族名
	 * @throws IOException
	 */
	public void deleteColumnFamily(String tableName, String rowKey, String columnFamily) throws IOException {
		Table table = connection.getTable(TableName.valueOf(tableName));
		
		Delete delete = new Delete(rowKey.getBytes());
		delete.addFamily(Bytes.toBytes(columnFamily));
		
		table.delete(delete);
	}

	/**
	 * 删除列
	 * experssion : delete 'tableName','rowKey','columnFamily:column'
	 * @param tableName		表名
	 * @param rowKey		行id
	 * @param columnFamily	列族名
	 * @param column		列名
	 * @throws IOException	
	 */
	public void deleteColumn(String tableName, String rowKey, String columnFamily, String column) throws IOException {
		Table table = connection.getTable(TableName.valueOf(tableName));
		
		Delete delete = new Delete(rowKey.getBytes());
		delete.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(column));
		
		table.delete(delete);
	}

	/**
	 * 删除表
	 * experssion : disable 'tableName' 之后 drop 'tableName'
	 * @param tableName 	表名
	 * @throws IOException
	 */
	public void deleteTable(String tableName) throws IOException {
		boolean isExists = this.tableExists(tableName);
		
		if (!isExists) {
			return;
		}
		
		TableName name = TableName.valueOf(tableName);
		admin.disableTable(name);
		admin.deleteTable(name);
	}

	/**
	 * 获取值
	 * experssion : get 'tableName','rowkey','family:column'
	 * @param tableName		表名
	 * @param rowkey		行id
	 * @param family		列族名
	 * @param column		列名
	 * @return
	 */ 
	public String getValue(String tableName, String rowkey, String family, String column) {
		Table table = null;
		String value = "";
		
		if (StringUtils.isBlank(tableName) || StringUtils.isBlank(family) || StringUtils.isBlank(rowkey) || StringUtils
				.isBlank(column)) {
			return null;
		}
		
		try {
			table = connection.getTable(TableName.valueOf(tableName));
			Get g = new Get(rowkey.getBytes());
			g.addColumn(family.getBytes(), column.getBytes());
			Result result = table.get(g);
			List<Cell> ceList = result.listCells();
			if (ceList != null && ceList.size() > 0) {
				for (Cell cell : ceList) {
					value = Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
				}
			}
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			try {
				table.close();
				connection.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		return value;
	}

	/**
	 * 查询指定行
	 * experssion : get 'tableName','rowKey'
	 * @param tableName		表名
	 * @param rowKey		行id
	 * @return
	 * @throws IOException
	 */
	public String selectOneRow(String tableName, String rowKey) throws IOException {
		Table table = connection.getTable(TableName.valueOf(tableName));
		Get get = new Get(rowKey.getBytes());
		Result result = table.get(get);
		NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map = result.getMap();
		
		for (Cell cell : result.rawCells()) {
			String row = Bytes.toString(cell.getRowArray());
			String columnFamily = Bytes.toString(cell.getFamilyArray());
			String column = Bytes.toString(cell.getQualifierArray());
			String value = Bytes.toString(cell.getValueArray());
			// 可以通过反射封装成对象(列名和Java属性保持一致)
			System.out.println(row);
			System.out.println(columnFamily);
			System.out.println(column);
			System.out.println(value);
		}
		return null;
	}
	

	/**
	 *   根据条件取出点位指定时间内的所有记录
	 * @param tableName		表名("OPC_TEST")
	 * @param family 列簇名("OPC_COLUMNS")
	 * @param column 列名("site")
	 * @param value 值(采集点标识)
	 * @param startMillis 开始时间毫秒值(建议传递当前时间前一小时的毫秒值，在保证查询效率的前提下获取到点位最新的记录)
	 * @param endMMillis 结束时间毫秒值(当前时间)
	 * @return	
	 * @throws IOException
	 */    
	@SuppressWarnings("finally")
	public Map<String,String> scanBatchOfTable(String tableName,String family,String [] column,String [] value,Long startMillis,Long endMillis) throws IOException {
		
	 	if(Objects.isNull(column) || Objects.isNull(column) || column.length != value.length) {
	 		return null;
	 	}
	 	
	 	FilterList filterList = new FilterList();
	 	
	 	for (int i = 0; i < column.length; i++) {
	 		SingleColumnValueFilter filter =  new SingleColumnValueFilter(Bytes.toBytes(family), Bytes.toBytes(column[i]), CompareOperator.EQUAL, Bytes.toBytes(value[i]));
	 		filterList.addFilter(filter);
		}
	 	
		Table table = connection.getTable(TableName.valueOf(tableName));
		
	 	Scan scan = new Scan();
	 	scan.setFilter(filterList);
	 	
	 	if(startMillis != null && endMillis != null) {
	 		scan.setTimeRange(startMillis,endMillis);
	 	}
		
		ResultScanner scanner = table.getScanner(scan);
		Map<String,String> resultMap = new HashMap<>();
		 
		try {
			 for(Result result:scanner){
		         for(Cell cell:result.rawCells()){
		             String values=Bytes.toString(CellUtil.cloneValue(cell));
		             String qualifier=Bytes.toString(CellUtil.cloneQualifier(cell));
		             
		             resultMap.put(qualifier, values);		     
		         }
		       }
		} finally {
			if (scanner != null) {
				scanner.close();
			}
			return resultMap;
		}
	}

	/**
	 *   根据条件取出点位最近时间的一条记录
	 * experssion : scan 't1',{FILTER=>"PrefixFilter('2015')"}
	 * @param tableName		表名("OPC_TEST")
	 * @param family 列簇名("OPC_COLUMNS")
	 * @param column 列名("site")
	 * @param value 值(采集点标识)
	 * @param startMillis 开始时间毫秒值(建议传递当前时间前一小时的毫秒值，在保证查询效率的前提下获取到点位最新的记录)
	 * @param endMMillis 结束时间毫秒值(当前时间)
	 * @return	
	 * @throws IOException
	 */    
	@SuppressWarnings("finally")
	public Map<String,String> scanOneOfTable(String tableName,String family,String column,String value,Long startMillis,Long endMillis) throws IOException {
		Table table = connection.getTable(TableName.valueOf(tableName));
		
	 	Scan scan = new Scan();
	 	scan.setReversed(true);

	 	PageFilter pageFilter = new PageFilter(1); // 
	 	scan.setFilter(pageFilter);
	 	
	 	if(startMillis != null && endMillis != null) {
	 		scan.setTimeRange(startMillis,endMillis);
	 	}
	 	
		if (StringUtils.isNotBlank(column)) {
		    SingleColumnValueFilter filter =  new SingleColumnValueFilter(Bytes.toBytes(family), Bytes.toBytes(column), CompareOperator.EQUAL, Bytes.toBytes(value));
		    scan.setFilter(filter);
		}
		
		ResultScanner scanner = table.getScanner(scan);
		Map<String,String> resultMap = new HashMap<>();
		 
		try {
			 for(Result result:scanner){
		         for(Cell cell:result.rawCells()){
		             String values=Bytes.toString(CellUtil.cloneValue(cell));
		             String qualifier=Bytes.toString(CellUtil.cloneQualifier(cell));
		             
		             resultMap.put(qualifier, values);		     
		         }
		       }
		} finally {
			if (scanner != null) {
				scanner.close();
			}
			return resultMap;
		}
	}


	/**
	 * 判断表是否已经存在，这里使用间接的方式来实现
	 * @param tableName 表名
	 * @return
	 * @throws IOException
	 */
	public boolean tableExists(String tableName) throws IOException {
		TableName[] tableNames = admin.listTableNames();
		if (tableNames != null && tableNames.length > 0) {
			for (int i = 0; i < tableNames.length; i++) {
				if (tableName.equals(tableNames[i].getNameAsString())) {
					return true;
				}
			}
		}
		
		return false;
	}
	
	 
    /**
         *  批量添加
     *
     * @param tableName HBase表名
     * @param rowkey    HBase表的rowkey
     * @param cf        HBase表的columnFamily
     * @param column    HBase表的列key
     * @param values     写入HBase表的值value
     * @param flag      提交标识符号。需要立即提交时，传递，值为 “end”
     */
    public void bulkput(String tableName, String rowkey, String columnFamily, String [] columns, String [] values,String flag) {
        try {
            List<Put> list = threadLocal.get();
            if (list == null) {
                list = new ArrayList<Put>();
            }
           
            Put put = new Put(Bytes.toBytes(rowkey));
            
            for (int i = 0; i < columns.length; i++) {
    			put.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(columns[i]), Bytes.toBytes(values[i]));
    			list.add(put);
    		}
            
            if (list.size() >= HBaseClient.CACHE_LIST_SIZE || flag.equals("end")) {
            	
                Table table = connection.getTable(TableName.valueOf(tableName));
                table.put(list);
               
                list.clear();
            } else {
                threadLocal.set(list);
            }
            
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}