hadoop、spark、zookeeper、hive集群搭建脚本

mkdir /letv/bigdata

tar xzf zookeeper-3.4.8.tar.gz -C /letv/bigdata/
tar xzf hadoop-2.7.2.tar.gz -C /letv/bigdata/
tar xzf apache-hive-2.1.0-bin.tar.gz -C /letv/bigdata/
tar xzf scala-2.11.8.tgz -C /letv/bigdata/
tar xzf spark-2.0.0-preview-bin-hadoop2.7.tgz -C /letv/bigdata/
tar xzf jdk-8u91-linux-x64.gz -C /letv/bigdata/
ls /letv/bigdata/



THIS_SCRIPT_DIR=$(cd $( dirname $0) && pwd)

mkdir /letv/bigdata/hadoop-2.7.2/pid_dir

cp -a ./apache-hive-2.1.0-config/* /letv/bigdata/apache-hive-2.1.0-bin/
cp -a ./hadoop-2.7.2-config/* /letv/bigdata/hadoop-2.7.2/
cp -a ./spark-2.0.0-config/* /letv/bigdata/spark-2.0.0-preview-bin-hadoop2.7/

cp -a ./zookeeper-3.4.8-config/* /letv/bigdata/zookeeper-3.4.8/


rsync /letv/bigdata/ root@THadoop2:/letv/bigdata/ -avz --delete
rsync /letv/bigdata/ root@THadoop3:/letv/bigdata/ -avz --delete
rsync /letv/bigdata/ root@THadoop4:/letv/bigdata/ -avz --delete
rsync /letv/bigdata/ root@THadoop5:/letv/bigdata/ -avz --delete
rsync /letv/bigdata/ root@THadoop6:/letv/bigdata/ -avz --delete
rsync /letv/bigdata/ root@THadoop7:/letv/bigdata/ -avz --delete
rsync /letv/bigdata/ root@THadoop8:/letv/bigdata/ -avz --delete


#### zoo keeper
ssh root@THadoop5 "echo 5 > /letv/bigdata/zookeeper-3.4.8/data/myid; cat /letv/bigdata/zookeeper-3.4.8/data/myid;"
ssh root@THadoop6 "echo 6 > /letv/bigdata/zookeeper-3.4.8/data/myid; cat /letv/bigdata/zookeeper-3.4.8/data/myid;"
ssh root@THadoop7 "echo 7 > /letv/bigdata/zookeeper-3.4.8/data/myid; cat /letv/bigdata/zookeeper-3.4.8/data/myid;"

ssh root@THadoop5 "source ~/.bash_profile; cd /letv/bigdata/zookeeper-3.4.8/bin; ./zkServer.sh start; "
ssh root@THadoop6 "source ~/.bash_profile; cd /letv/bigdata/zookeeper-3.4.8/bin; ./zkServer.sh start; "
ssh root@THadoop7 "source ~/.bash_profile; cd /letv/bigdata/zookeeper-3.4.8/bin; ./zkServer.sh start; "

ssh root@THadoop5 "source ~/.bash_profile; cd /letv/bigdata/zookeeper-3.4.8/bin; ./zkServer.sh status;"
ssh root@THadoop6 "source ~/.bash_profile; cd /letv/bigdata/zookeeper-3.4.8/bin; ./zkServer.sh status;"
ssh root@THadoop7 "source ~/.bash_profile; cd /letv/bigdata/zookeeper-3.4.8/bin; ./zkServer.sh status;"


ssh root@THadoop5 "source ~/.bash_profile; /letv/bigdata/hadoop-2.7.2/sbin/hadoop-daemon.sh start journalnode;"
ssh root@THadoop6 "source ~/.bash_profile; /letv/bigdata/hadoop-2.7.2/sbin/hadoop-daemon.sh start journalnode;"
ssh root@THadoop7 "source ~/.bash_profile; /letv/bigdata/hadoop-2.7.2/sbin/hadoop-daemon.sh start journalnode;"

# 在Master1上格式化namenode
# http://eksliang.iteye.com/blog/2226986
# http://blog.csdn.net/jackli31742/article/details/51483876

#在1上格式化NameNode
ssh root@THadoop1 "source ~/.bash_profile; hdfs zkfc -formatZK; hdfs namenode -format; /letv/bigdata/hadoop-2.7.2/sbin/hadoop-daemon.sh start namenode; "

#将NameNode的数据同步到2上
ssh root@THadoop2 "source ~/.bash_profile; hdfs namenode -bootstrapStandby; /letv/bigdata/hadoop-2.7.2/sbin/hadoop-daemon.sh start namenode; " 

#启动 datanode
#ssh root@THadoop1 "source ~/.bash_profile; /letv/bigdata/hadoop-2.7.2/sbin/hadoop-daemons.sh start datanode; /letv/bigdata/hadoop-2.7.2/sbin/start-yarn.sh; /letv/bigdata/hadoo
p-2.7.2/sbin/hadoop-daemon.sh start zkfc; "
ssh root@THadoop1 "source ~/.bash_profile; /letv/bigdata/hadoop-2.7.2/sbin/start-all.sh "
##开启负载均衡
/letv/bigdata/hadoop-2.7.2/sbin/start-balancer.sh -threshold 1


##启动hive服务
nohup hive --service metastore -v &

##启动spark
/letv/bigdata/spark-2.0.0-preview-bin-hadoop2.7/sbin/start-all.sh 

##启动spark sql cli
#hadoop fs -chmod -R +w /tmp/hive
#/letv/bigdata/spark-2.0.0-preview-bin-hadoop2.7/bin/spark-sql --master  spark://92:7077 yarn-client