HIVE 双机集群HA-部署

HIVE双机HA-部署

1.1、系统环境初始化

防火墙关闭：

server iptables stop

chkconfig iptables off

selinux关闭：

 

用户创建：

vim yonghu.txt

hbase hdfs hive impala Impala kudu Kudu spark wxl zookeeper

 

#!/bin/sh

for i in `cat /root/hadoop-cdh/yonghu.txt`;

do

    echo $i

    useradd $i

done

 

免密码登录:其他用户更改root名称XXX

config-ssh-root.sh

#!/bin/sh

 

expect -c "

spawn ssh-keygen -t rsa

expect {

\".ssh/id_rsa): \" {send \"\r\";exp_continue }

\"Enter passphrase (empty for no passphrase): \" {send \"\r\";exp_continue }

\"Enter same passphrase again: \" {send \"\r\" }

}

expect eof

"

expect -c "

    spawn ssh-copy-id -i /root/.ssh/id_rsa.pub zk-master-la01

    expect {

        yes/no { send \"yes\r\"; exp_continue }

        *assword* { send \"pass@word1\r\" }

    }

    expect {

        *assword* { send \"pass@word1\r\" }

    }

expect eof

"

expect -c "

    spawn ssh-copy-id -i /root/.ssh/id_rsa.pub zk-master-la02

    expect {

        yes/no { send \"yes\r\"; exp_continue }

        *assword* { send \"pass@word1\r\" }

    }

    expect {

        *assword* { send \"pass@word1\r\" }

    }

expect eof

"

expect -c "

spawn ssh-copy-id -i /root/.ssh/id_rsa.pub zk-la03

expect {

yes/no { send \"yes\r\"; exp_continue }

*assword* { send \"pass@word1\r\" }

}

expect {

*assword* { send \"pass@word1\r\" }

}

expect eof

"

expect -c "

spawn ssh-copy-id -i /root/.ssh/id_rsa.pub nagios_server

expect {

yes/no { send \"yes\r\"; exp_continue }

*assword* { send \"pass@word1\r\" }

}

expect {

*assword* { send \"pass@word1\r\" }

}

expect eof

"

for slave in $(</tmp/hadoop-slaves)

do

#ecl=${EXPECT_CMD/remoteHost/$slave}

#echo $ecl

expect -c "

spawn ssh-copy-id -i /root/.ssh/id_rsa.pub $slave

expect {

yes/no { send \"yes\r\"; exp_continue }

*assword* { send \"pass@word1\r\" }

}

expect {

*assword* { send \"pass@word1\r\" }

}

"

done

 

1.2、zookeeper安装配置：

JDK1.8配置生效

tar -zxvf jdk-8u121-linux-x64.gz -C /usr/local/

 

vim /home/hadoop/.bash_profile

 

export JAVA_HOME=/usr/local/jdk1.8.0_121

export PATH=$PATH:$JAVA_HOME/bin

 

source /home/hadoop/.bash_profile

zk-master-la01节点操作

tar -zxvf zookeeper-3.4.61.tar.gz –C /usr/local/

cd /usr/local/zookeeper-3.4.6/conf

 

配置zookeerper文件：

vim zoo.cfg

 

tickTime=2000

initLimit=10

syncLimit=5

dataDir=/opt/zookeeper/data

clientPort=2181

server.1=zk-master-la01:2888:3888

server.2=zk-master-la02:2888:3888

server.3=slave-01:2888:3888

 

#创建data数据目录

mkdir -p /opt/zookeeper/data/

echo "1" >> /opt/zookeeper/data/myid

#scp分别传输多台节点

vim host.txt

zk-master-la01

zk-master-la02

zk-la03

slave-01

slave-02

 

for i in `cat /root/host.txt`;do echo $i;scp -r /usr/local/zookeeper-3.4.5-cdh5.10.0 root@$i:/usr/local/zookeeper-3.4.5-cdh5.10.0 ;done

 

zk-master-la02操作

#注意改下myid 2

mkdir –p /opt/zookeeper/data/

echo "2" >> /opt/zookeeper/data/myid

 

zk-la03操作

#注意改下myid 3

mkdir -p /opt/zookeeper/data/

echo "3" >> /opt/zookeeper/data/myid

 

#zk-master-la01操作

/usr/local/zookeeper-3.4.6/bin/zkServer.sh start

/usr/local/zookeeper-3.4.6/bin/zkServer.sh stop

//usr/local/zookeeper-3.4.6/bin/zkServer.sh status

 

#zk-master-la02操作

/usr/local/zookeeper-3.4.6/bin/zkServer.sh start

 

#zk-la03操作

/usr/local/zookeeper-3.4.6/bin/zkServer.sh start

 

1.3、部署hadoop双机HA-namenode

hadoop解压到/usr目录

tar -zxvf hadoop-2.6.0-cdh5.7.5.tar.gz -C /usr/local/

 

cd /usr/local/hadoop-2.6.0-cdh5.7.5/etc/hadoop

编辑修改内容如下：

vim core-site.xml

<?xml version="1.0" encoding="utf-8"?>

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

 

<configuration>

<property>

<na