Spark高可用集群搭建（HA）

zk安装

tar -zxvf zookeeper-3.4.14.tar.gz
mv zookeeper-3.4.14/ zookeeper
cd /zookeeper/conf
mv zoo_sample.cfg zoo.cfg
mkdir {data,logs}
vim zoo.cfg

tickTime=2000 
initLimit=10   ##集群中的follower服务器(F)与leader服务器(L)之间初始连接时能容忍的最多心跳数
syncLimit=5    ##集群中的follower服务器与leader服务器之间请求和应答之间能容忍的最多心跳数
clientPort=2181   
maxClientCnxns=100 
dataDir=/itcast/zookeeper/data
dataLogDir=/itcast/zookeeper/logs  
server.1=192.168.66.101:2888:3888  
server.2=192.168.66.102:2888:3888  
server.3=192.168.66.103:2888:3888 

cd data
echo “1” > myid
echo “2” > myid
echo “3” > myid

spark HA安装

修改spark-env.sh
#export SPARK_MASTER_HOST=hadoop101
export SPARK_DAEMON_JAVA_OPTS="-Dspark.deploy.recoveryMode=ZOOKEEPER -Dspark.deploy.zookeeper.url=192.168.66.101:2181,192.168.66.102:2181,192.168.66.103:2181 -Dspark.deploy.zookeeper.dir=/spark"

然后每节点依次启动
./start-master.sh
./start-slave.sh spark://hadoop101:7077

测试
kill掉master

kill -9 6757