Spring Boot集成HBase

本文是使用Java来编写的，要使用scala语言请参考scala的hbase api使用

先介绍一下集群环境：

hadoop version: 3.0.0

hbase version 2.1.0

• 修改pom.xml， 加入依赖，仓库没有比较新的jar，所以本文引用了较低的版本

<dependency>
       <groupId>org.springframework.boot</groupId>
       <artifactId>spring-boot-starter-web</artifactId>
</dependency>

<dependency>
       <groupId>org.springframework.boot</groupId>
       <artifactId>spring-boot-starter-test</artifactId>
</dependency>

<!-- 读取bootstrap配置文件必须引入该包， 不然默认读取application相关的配置文件 -->
<dependency>
       <groupId>org.springframework.cloud</groupId>
       <artifactId>spring-cloud-context</artifactId>
</dependency>

<dependency>
       <groupId>org.springframework.data</groupId>
       <artifactId>spring-data-hadoop-hbase</artifactId>
       <version>2.5.0.RELEASE</version>
</dependency>

<dependency>
       <groupId>org.apache.hbase</groupId>
       <artifactId>hbase-client</artifactId>
       <version>1.1.2</version>
</dependency>

<dependency>
       <groupId>org.springframework.data</groupId>
       <artifactId>spring-data-hadoop</artifactId>
       <version>2.5.0.RELEASE</version>
</dependency>

<dependency>
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>
       <version>2.5.0</version>
</dependency>

• 我的spring boot的配置文件是bootstrap.yml，添加hbase相关配置

hbase:
  config:
    hbase.zookeeper.quorum: 181.181.0.28
    hbase.zookeeper.property.clientPort: 2181

• 编写HBaseProps.java来读取配置文件bootstrap.yml，获取hbase参数

import org.springframework.boot.context.properties.ConfigurationProperties;

import java.util.HashMap;
import java.util.Map;

@ConfigurationProperties(prefix = "hbase")
public class HBaseProps {

    private Map<String, String> config = new HashMap<>();

    public Map<String, String> getConfig() {
        return config;
    }

    public void setConfig(Map<String, String> config) {
        this.config = config;
    }
}

• 编写HBaseConfig.java来配置初始化HBase，连接上hbase

import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.hadoop.hbase.HbaseTemplate;

import java.io.IOException;
import java.util.Map;

@Configuration
@EnableConfigurationProperties(HBaseProps.class)
public class HBaseConfig {

    private final HBaseProps properties;

    public HBaseConfig(HBaseProps properties) {
        this.properties = properties;
    }

    public org.apache.hadoop.conf.Configuration configuration() {
        org.apache.hadoop.conf.Configuration configuration = HBaseConfiguration.create();
        Map<String, String> config = properties.getConfig();
        for(Map.Entry<String, String> map : config.entrySet()){
            configuration.set(map.getKey(), map.getValue());
        }
        return configuration;
    }

    @Bean
    public HbaseTemplate hbaseTemplate() {
        HbaseTemplate hbaseTemplate = new HbaseTemplate();
        hbaseTemplate.setConfiguration(configuration());
        hbaseTemplate.setAutoFlush(true);
        return hbaseTemplate;
    }

    @Bean
    public Admin admin() {
        Admin admin = null;
        try {
            Connection connection = ConnectionFactory.createConnection(configuration());
            admin = connection.getAdmin();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return admin;
    }
}

• 编写HBaseUtils.java工具类，实现创建表、插入数据、读取数据

import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.hadoop.hbase.HbaseTemplate;
import org.springframework.stereotype.Service;

import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeSet;

@Service
public class HBaseUtils {
    @Autowired
    private HbaseTemplate hbaseTemplate;

    @Autowired
    private Admin hbaseAdmin;

    /**
     * 判断表是否存在
     *
     * @param tableName 表名
     * @return true/false
     */
    public boolean isExists(String tableName) {
        boolean tableExists = false;
        try {
            TableName table = TableName.valueOf(tableName);
            tableExists = hbaseAdmin.tableExists(table);
        } catch (IOException e) {
            e.printStackTrace();
        }
        return tableExists;
    }

    /**
     * 创建表
     * @param tableName 表名
     * @param columnFamily 列族
     * @return true/false
     */
    public boolean createTable(String tableName, List<String> columnFamily) {
        return createTable(tableName, columnFamily, null);
    }

    /**
     * 预分区创建表
     * @param tableName 表名
     * @param columnFamily 列族
     * @param keys 分区集合
     * @return true/false
     */
    public boolean createTable(String tableName, List<String> columnFamily, List<String> keys) {
        if (!isExists(tableName)) {
            try {
                TableName table = TableName.valueOf(tableName);
                HTableDescriptor desc = new HTableDescriptor(table);
                for (String cf : columnFamily) {
                    desc.addFamily(new HColumnDescriptor(cf));
                }
                if (keys == null) {
                    hbaseAdmin.createTable(desc);
                } else {
                    byte[][] splitKeys = getSplitKeys(keys);
                    hbaseAdmin.createTable(desc, splitKeys);
                }
                return true;
            } catch (IOException e) {
                e.printStackTrace();
            }
        } else {
            System.out.println(tableName + "is exists!!!");
            return false;
        }
        return false;
    }

    /**
     * 插入数据
     * @param tableName 表名
     * @param rowKey rowKey
     * @param columnFamily 列族
     * @param columns 列
     * @param values 值
     * @return true/false
     */
    public boolean putData(String tableName, String rowKey, String columnFamily, List<String> columns, List<String> values) {
        try {
            Table table = hbaseAdmin.getConnection().getTable(TableName.valueOf(tableName));
            Put put = new Put(Bytes.toBytes(rowKey));
            for (int i=0; i<columns.size(); i++) {
                put.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(columns.get(i)), Bytes.toBytes(values.get(i)));
            }
            table.put(put);
            return true;
        } catch (IOException e) {
            e.printStackTrace();
            return false;
        }
    }

    /**
     * 获取数据
     * @param tableName 表名
     * @param rowKey rowKey
     * @return map
     */
    public Map<String, String> getData(String tableName, String rowKey) {
        HashMap<String, String> map = new HashMap<>();
        try {
            Table table = hbaseAdmin.getConnection().getTable(TableName.valueOf(tableName));
            Get get = new Get(Bytes.toBytes(rowKey));
            Result result = table.get(get);
            if (result != null && !result.isEmpty()) {
                for (Cell cell : result.listCells()) {
                    map.put(Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()),
                            Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        return map;
    }

    /**
     * 分区【10, 20, 30】 -> ( ,10] (10,20] (20,30] (30, )
     * @param keys 分区集合[10, 20, 30]
     * @return byte二维数组
     */
    private byte[][] getSplitKeys(List<String> keys) {
        byte[][] splitKeys = new byte[keys.size()][];
        TreeSet<byte[]> rows = new TreeSet<>(Bytes.BYTES_COMPARATOR);
        for(String key : keys) {
            rows.add(Bytes.toBytes(key));
        }
        int i = 0;
        for (byte[] row : rows) {
            splitKeys[i] = row;
            i ++;
        }
        return splitKeys;
    }
}

• 编写HBaseTest.java测试类

import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;

import java.util.ArrayList;

@SpringBootTest
@RunWith(SpringRunner.class)
public class HBaseTest {

    @Autowired
    private HBaseUtils hBaseUtils;

    @Test
    public void test() {
        String tableName = "hbase";
        String columnFamily = "cf";
        String rowKey = "100001";
        ArrayList<String> familyList = new ArrayList<>();
        familyList.add(columnFamily);
        ArrayList<String> columns = new ArrayList<>();
        columns.add("name");
        columns.add("age");
        ArrayList<String> values = new ArrayList<>();
        values.add("Jack");
        values.add("18");

        boolean exists = hBaseUtils.isExists(tableName);
        System.out.println(exists);
        boolean hbase = hBaseUtils.createTable("hbase", familyList);
        System.out.println(hbase);
        boolean putData = hBaseUtils.putData(tableName, rowKey, columnFamily, columns, values);
        System.out.println(putData);
    }
}

好了，以上就是Spring Boot集成HBase的全部内容