本文是使用Java来编写的,要使用scala语言请参考scala的hbase api使用
先介绍一下集群环境:
hadoop version: 3.0.0
hbase version 2.1.0
- 修改pom.xml, 加入依赖,仓库没有比较新的jar,所以本文引用了较低的版本
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
</dependency>
<!-- 读取bootstrap配置文件必须引入该包, 不然默认读取application相关的配置文件 -->
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-context</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-hadoop-hbase</artifactId>
<version>2.5.0.RELEASE</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>1.1.2</version>
</dependency>
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-hadoop</artifactId>
<version>2.5.0.RELEASE</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>2.5.0</version>
</dependency>
- 我的spring boot的配置文件是bootstrap.yml,添加hbase相关配置
hbase:
config:
hbase.zookeeper.quorum: 181.181.0.28
hbase.zookeeper.property.clientPort: 2181
- 编写HBaseProps.java来读取配置文件bootstrap.yml,获取hbase参数
import org.springframework.boot.context.properties.ConfigurationProperties;
import java.util.HashMap;
import java.util.Map;
@ConfigurationProperties(prefix = "hbase")
public class HBaseProps {
private Map<String, String> config = new HashMap<>();
public Map<String, String> getConfig() {
return config;
}
public void setConfig(Map<String, String> config) {
this.config = config;
}
}
- 编写HBaseConfig.java来配置初始化HBase,连接上hbase
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.hadoop.hbase.HbaseTemplate;
import java.io.IOException;
import java.util.Map;
@Configuration
@EnableConfigurationProperties(HBaseProps.class)
public class HBaseConfig {
private final HBaseProps properties;
public HBaseConfig(HBaseProps properties) {
this.properties = properties;
}
public org.apache.hadoop.conf.Configuration configuration() {
org.apache.hadoop.conf.Configuration configuration = HBaseConfiguration.create();
Map<String, String> config = properties.getConfig();
for(Map.Entry<String, String> map : config.entrySet()){
configuration.set(map.getKey(), map.getValue());
}
return configuration;
}
@Bean
public HbaseTemplate hbaseTemplate() {
HbaseTemplate hbaseTemplate = new HbaseTemplate();
hbaseTemplate.setConfiguration(configuration());
hbaseTemplate.setAutoFlush(true);
return hbaseTemplate;
}
@Bean
public Admin admin() {
Admin admin = null;
try {
Connection connection = ConnectionFactory.createConnection(configuration());
admin = connection.getAdmin();
} catch (IOException e) {
e.printStackTrace();
}
return admin;
}
}
- 编写HBaseUtils.java工具类,实现创建表、插入数据、读取数据
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.hadoop.hbase.HbaseTemplate;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeSet;
@Service
public class HBaseUtils {
@Autowired
private HbaseTemplate hbaseTemplate;
@Autowired
private Admin hbaseAdmin;
/**
* 判断表是否存在
*
* @param tableName 表名
* @return true/false
*/
public boolean isExists(String tableName) {
boolean tableExists = false;
try {
TableName table = TableName.valueOf(tableName);
tableExists = hbaseAdmin.tableExists(table);
} catch (IOException e) {
e.printStackTrace();
}
return tableExists;
}
/**
* 创建表
* @param tableName 表名
* @param columnFamily 列族
* @return true/false
*/
public boolean createTable(String tableName, List<String> columnFamily) {
return createTable(tableName, columnFamily, null);
}
/**
* 预分区创建表
* @param tableName 表名
* @param columnFamily 列族
* @param keys 分区集合
* @return true/false
*/
public boolean createTable(String tableName, List<String> columnFamily, List<String> keys) {
if (!isExists(tableName)) {
try {
TableName table = TableName.valueOf(tableName);
HTableDescriptor desc = new HTableDescriptor(table);
for (String cf : columnFamily) {
desc.addFamily(new HColumnDescriptor(cf));
}
if (keys == null) {
hbaseAdmin.createTable(desc);
} else {
byte[][] splitKeys = getSplitKeys(keys);
hbaseAdmin.createTable(desc, splitKeys);
}
return true;
} catch (IOException e) {
e.printStackTrace();
}
} else {
System.out.println(tableName + "is exists!!!");
return false;
}
return false;
}
/**
* 插入数据
* @param tableName 表名
* @param rowKey rowKey
* @param columnFamily 列族
* @param columns 列
* @param values 值
* @return true/false
*/
public boolean putData(String tableName, String rowKey, String columnFamily, List<String> columns, List<String> values) {
try {
Table table = hbaseAdmin.getConnection().getTable(TableName.valueOf(tableName));
Put put = new Put(Bytes.toBytes(rowKey));
for (int i=0; i<columns.size(); i++) {
put.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(columns.get(i)), Bytes.toBytes(values.get(i)));
}
table.put(put);
return true;
} catch (IOException e) {
e.printStackTrace();
return false;
}
}
/**
* 获取数据
* @param tableName 表名
* @param rowKey rowKey
* @return map
*/
public Map<String, String> getData(String tableName, String rowKey) {
HashMap<String, String> map = new HashMap<>();
try {
Table table = hbaseAdmin.getConnection().getTable(TableName.valueOf(tableName));
Get get = new Get(Bytes.toBytes(rowKey));
Result result = table.get(get);
if (result != null && !result.isEmpty()) {
for (Cell cell : result.listCells()) {
map.put(Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()),
Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
}
}
} catch (IOException e) {
e.printStackTrace();
}
return map;
}
/**
* 分区【10, 20, 30】 -> ( ,10] (10,20] (20,30] (30, )
* @param keys 分区集合[10, 20, 30]
* @return byte二维数组
*/
private byte[][] getSplitKeys(List<String> keys) {
byte[][] splitKeys = new byte[keys.size()][];
TreeSet<byte[]> rows = new TreeSet<>(Bytes.BYTES_COMPARATOR);
for(String key : keys) {
rows.add(Bytes.toBytes(key));
}
int i = 0;
for (byte[] row : rows) {
splitKeys[i] = row;
i ++;
}
return splitKeys;
}
}
- 编写HBaseTest.java测试类
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.ArrayList;
@SpringBootTest
@RunWith(SpringRunner.class)
public class HBaseTest {
@Autowired
private HBaseUtils hBaseUtils;
@Test
public void test() {
String tableName = "hbase";
String columnFamily = "cf";
String rowKey = "100001";
ArrayList<String> familyList = new ArrayList<>();
familyList.add(columnFamily);
ArrayList<String> columns = new ArrayList<>();
columns.add("name");
columns.add("age");
ArrayList<String> values = new ArrayList<>();
values.add("Jack");
values.add("18");
boolean exists = hBaseUtils.isExists(tableName);
System.out.println(exists);
boolean hbase = hBaseUtils.createTable("hbase", familyList);
System.out.println(hbase);
boolean putData = hBaseUtils.putData(tableName, rowKey, columnFamily, columns, values);
System.out.println(putData);
}
}
好了,以上就是Spring Boot集成HBase的全部内容