高可用(HA)Hadoop搭建Hive on Spark

高可用(HA)Hadoop搭建Hive on Spark

本文只介绍hive on spark搭建,Hadoop高可用(HA)请自行操作,

安装HIVE

1. 解压hive

tar -zxvf apache-hive-3.1.2-bin.tar.gz -C 指定目录

2. 配置hive

mysql自己配置

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>javax.jdo.option.ConnectionURL</name>
        <value>jdbc:mysql://bigdataserver1:3306/hiveSource?useSSL=false&amp;useUnicode=true&amp;characterEncoding=UTF-8
        </value>
    </property>

    <property>
        <name>javax.jdo.option.ConnectionDriverName</name>
        <value>com.mysql.jdbc.Driver</value>
    </property>

    <property>
        <name>javax.jdo.option.ConnectionUserName</name>
        <value>root</value>
    </property>

    <property>
        <name>javax.jdo.option.ConnectionPassword</name>
        <value>123456</value>
    </property>

    <property>
        <name>hive.metastore.warehouse.dir</name>
        <value>/user/hive/warehouse</value>
    </property>

    <property>
        <name>hive.metastore.schema.verification</name>
        <value>false</value>
    </property>

    <property>
        <name>hive.server2.thrift.port</name>
        <value>10000</value>
    </property>

    <property>
        <name>hive.server2.thrift.bind.host</name>
        <value>bigdataserver1</value>
    </property>

    <!--    <property>
         <name>hive.metastore.uris</name>
         <value>thrift://hadoop102:9083</value>
        </property>  -->


    <property>
        <name>hive.metastore.event.db.notification.api.auth</name>
        <value>false</value>
    </property>

    <property>
        <name>hive.cli.print.header</name>
        <value>true</value>
    </property>

    <property>
        <name>hive.cli.print.current.db</name>
        <value>true</value>
    </property>
    
    <!--这里需要将纯spark的jar传到你指定的目录上-->
    <!--注意:这里还要将你hadoop->share/hadoop/mapreduce下面的jar也上传,我不传要报错/-->
    <property>
        <name>spark.yarn.jars</name>
        <!--这个hadoopcluster就是你hdfs-site.xml里面dfs.nameservices配置-->
        <value>hdfs://hadoopcluster/spark的jar包/*</value>
    </property>

    <!--Hive 执行引擎-->
    <property>
        <name>hive.execution.engine</name>
        <value>spark</value>
    </property>
    <!--Hive 和 Spark 连接超时时间-->
    <property>
        <name>hive.spark.client.connect.timeout</name>
        <value>60000ms</value>
    </property>
    <!-- hiveserver2的高可用参数，开启此参数可以提高hiveserver2的启动速度 -->
    <property>
        <name>hive.server2.active.passive.ha.enable</name>
        <value>true</value>
    </property>
</configuration>

3. 启动hive

初始化hive===>schematool -initSchema -dbType mysql -verbose
bin/hive
create table test(id int,name string);
insert into test values(1,‘fjds’),(2,‘jdfkfdf’);
完成

4. 代码

如果需要在代码里面连接,需要设置一个hadoop的用户
System.setProperty(“HADOOP_USER_NAME”, “用户”)
如果找不到集群名字,可以将hdsf-site.xml放到resources配置文件下,或者直接代码配置都可以