shell一键安装spark(3节点)

shell一键部署jdk_三台
shell一键部署Scala_三台

1. 准备好三台已经安装好jdk和scala的机器

172.16.149.150 cdh-master
172.16.149.151 cdh-worker1
172.16.149.152 cdh-worker2

2. 编写脚本

#!/bin/bash
ipath=/opt/bigdata
echo "安装spark集群"
installpath=$(cd `dirname $0`; pwd)
echo ${installpath}
filepath=${installpath}/files
echo "解压spark压缩包"
tar -xvf ${filepath}/spark.tar.gz > /dev/null 2>&1
cd spark-* && sparkname=`pwd | awk -F '/' '{print $NF}'`
echo "spark版本：${sparkname}"
echo "因为压缩包是已经配置好的文件，而且spark各节点配置文件一致，所以此处未做修改"
echo "移动解压包到指定spark路径"
cd ${installpath} && cp -rf ${sparkname}/ ${ipath}/spark/
echo "cp -r ${sparkname} ${ipath}/spark/"
echo "配置spark环境变量"
echo "#spark home"
echo "export SPARK_HOME=${ipath}/spark/${sparkname}" >> ~/.bash_profile
echo "刷新环境变量"
source ~/.bash_profile > /dev/null 2>&1
echo "解压包分发到其他节点"
# $( seq 1 10 ) = {1..10} 此处循环分发已验证
for i in {1..2}
do
	scp -r ${ipath}/spark/${sparkname} root@cdh-worker$i:${ipath}/spark
done
echo "启动Spark 。。。"
echo;
echo "先执行关闭操作。。。"
${ipath}/spark/${sparkname}/sbin/stop-all.sh
echo "启动start-all.sh"
${ipath}/spark/${sparkname}/sbin/start-all.sh
echo "上传spark的 jars到 hdfs 否则无法运行 spark on yarn"
/opt/bigdata/hadoop/hadoop-3.2.0/bin/hadoop fs -put /opt/bigdata/spark/spark-2.4.3-bin-hadoop2.7/jars/* /spark/jars/