@@spark 安装
@这三台机器当做hadoop 的客户端,需要把 hadoop 的文件全部导入这个三台机器 /etc/hosts 设置
192.168.x.87 spark1.sxw.com spark1
192.168.x.145 spark2.sxw.com spark2
192.168.x.147 spark3.sxw.com spark3
设置主机名
hostnamectl set-hostname spark1 spark2 spark3
##ntp 时间认证服务
for i in spark1 spark2 spark3;do ssh $i yum install ntp -y ;done
for i in spark1 spark2 spark3;do ssh $i "systemctl enable ntpd && systemctl start ntpd";done
##安装java 所有spark 节点
#set java path
export JAVA_HOME=/usr/local/java
export PATH=$JAVA_HOME/bin:$PATH
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
###所有spark节点安装 scala 环境
wget https://downloads.lightbend.com/scala/2.10.7/scala-2.10.7.rpm
for i in spark1 spark2 spark3;do ssh $i "yum localinstall scala -y ";done
##下载spark
wget http://mirrors.shu.edu.cn/apache/spark/spark-2.3.3/spark-2.3.3-bin-hadoop2.7.tgz
tar -zxvf spark-2.3.3-bin-hadoop2.7.tgz
mv spark-2.3.3-bin-hadoop2.7 spark
cp -r spark /usr/local/
cd /usr/local/spark/conf
cp spark-env.sh.template spark-env.sh
##到hadoop 的namenode 节点上执行,作为客户端 访问hadoop 集群
scp -r /usr/local/hadoop root@spark1:/usr/local/
##设置变量
vim spark-env.sh
export JAVA_HOME=/usr/local/java
export HADOOP_HOME=/usr/local/hadoop
export HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop
export LD_LIBRARY_PATH=$HADOOP_HOME/lib/native
export SPARK_WORKER_MEMORY=500m
export SPARK_WORKER_CORES=1
export SPARK_DAEMON_JAVA_OPTS="-Dspark.deploy.recoveryMode=ZOOKEEPER -Dspark.deploy.zookeeper.url=zk1:2181,zk2:2181,zk3:2181 -Dspark.deploy.zookeeper.dir=/spark"
##设置master
[root@spark1 conf]# cat /usr/local/spark/conf/slaves
spark1
spark2
spark3
##设置log
mv /usr/local/spark/conf/log4j.properties.template /usr/local/spark/conf/log4j.properties
##设置/etc/profile
#set spark path
export SPARK_HOME=/usr/local/spark
export PATH=$PATH:$SPARK_HOME/bin
#set hadoop path
export HADOOP_HOME=/usr/local/hadoop
export PATH=$PATH:$HADOOP_HOME/bin
##把配置同步到其他spark2 spark3 ##启动服务
/usr/local/spark/sbin/start-all.sh
web spark1:8080
##测试
/usr/local/spark/bin/spark-submit \
--class org.apache.spark.examples.SparkPi \
--master spark://spark1.sxw.com:6066 \
--deploy-mode cluster \
--supervise \
--executor-memory 1G \
--total-executor-cores 10 \
/usr/local/spark/examples/jars/spark-examples_2.11-2.1.0.jar \
100000