大数据调试环境配置（1）：外部链接HBase调试环境配置

一、目的

在开发过程中，不可避免需要在集群外部调试程序，以便快速开发，测试。

二、环境

1.IDEA

2.CDH 5.7.0（Hbase、HADOOP）

三、步骤

1.创建Maven项目

2.加载对应CDH需求依赖

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>CDH</groupId>
    <artifactId>hbase</artifactId>
    <version>1.0</version>

    <repositories>
        <repository>
            <id>cloudera</id>
            <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
        </repository>
    </repositories>
    <properties>
        <hadoop.version>2.6.0-cdh5.7.0</hadoop.version>
        <hbase.version>1.2.0-cdh5.7.0</hbase.version>

        <maven.compiler.source>1.8</maven.compiler.source>
        <maven.compiler.target>1.8</maven.compiler.target>
    </properties>

    <dependencies>
        <!--            <dependency>-->
        <!--                <groupId>org.apache.hadoop</groupId>-->
        <!--                <artifactId>hadoop-common</artifactId>-->
        <!--                <version>2.5.0-cdh5.2.0</version>-->
        <!--            </dependency>-->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>${hadoop.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hbase</groupId>
            <artifactId>hbase-client</artifactId>
            <version>${hbase.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hbase</groupId>
            <artifactId>hbase-server</artifactId>
            <version>${hbase.version}</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java -->
        <dependency>
            <groupId>com.google.protobuf</groupId>
            <artifactId>protobuf-java</artifactId>
            <version>2.5.0</version>
        </dependency>

        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.10</version>
            <scope>test</scope>
        </dependency>


        <!-- https://mvnrepository.com/artifact/commons-logging/commons-logging -->
<!--        <dependency>-->
<!--            <groupId>commons-logging</groupId>-->
<!--            <artifactId>commons-logging</artifactId>-->
<!--            <version>1.2</version>-->
<!--        </dependency>-->

    </dependencies>

</project>

3.加载hbase-site.xml、hdfs-site.xml、core-site.xml到resources目录下

4.将hadoop-2.6.0-cdh5.7.0的tar包直接解压缩到本地，并且配置Hadoop环境变量

备注：这个不一定需要hadoop源码，但是需要配置HADOOP_HOME变量，原因参考

https://blog.csdn.net/iamlihongwei/article/details/79876947

5.添加winutils.exe到hadoop-2.6.0-cdh5.7.0\bin目录下，并且配置winutils.exe到系统Path中

 

6.开启hdfs和zookeeper以及hbase

备注：如果不是cdh，则需要单个启动每个服务，参考：https://blog.csdn.net/u010886217/article/details/83759070

7.重启idea才可以

四、链接HBase测试代码

1.创建namespace并且创建表

import org.apache.commons.math.linear.NonSquareMatrixException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import java.io.IOException;
import java.util.NavigableMap;

public class HbaseUtils {

    public static Configuration getHBaseConf(){
        Configuration conf = HBaseConfiguration.create();
        conf.set("hbase.rootdir","hdfs://hadoop01:8020/hbase");
        conf.set("hbase.zookeeper.quorum","hadoop01");
        conf.set("hbase.zookeeper.property.clientPort","2181");
        return conf;
    }
 
    /**
     * createTable：创建namespace，表名，列簇。
     * @throws Exception
     */
    @SuppressWarnings("resource")
//	@Test
    public static void createTable() throws Exception {
        String nameSpace = "ns2";
        String tbName = "actor";
        String colFamily = "info";
        // 1.读取配置文件
        //的hbase-default.xml，也可以把hbase-site.xml放到resources中
        Configuration conf=HbaseUtils.getHBaseConf();


        Connection connection;
        connection = ConnectionFactory.createConnection(conf);

        // 2.创建一个管理员，管理表格的创建和删除
//        HBaseAdmin admin = new HBaseAdmin(conf);
        Admin admin=connection.getAdmin();

        // 3.创建命名空间描述器
        NamespaceDescriptor ns = NamespaceDescriptor.create(nameSpace).build();

        // 4.创建表格，表格对象
        TableName tableName = TableName.valueOf(nameSpace + ":" + tbName);
        // 创建表的描述器
        HTableDescriptor tableDesc = new HTableDescriptor(tableName);
        // 5.创建列簇的描述器
        HColumnDescriptor columnDesc = new HColumnDescriptor(colFamily);
        // 6.将列簇添加到表的描述器
        tableDesc.addFamily(columnDesc);
        // 7.创建命名空间，创建表
        admin.createNamespace(ns);
        admin.createTable(tableDesc);
        System.out.println("创建表   " + nameSpace + ":" + tbName + "  成功");

    }
}

2.结果

创建成功！