Spark 集群安装

@@spark 安装

@这三台机器当做hadoop 的客户端，需要把 hadoop 的文件全部导入这个三台机器 /etc/hosts 设置
 

192.168.x.87 spark1.sxw.com spark1
192.168.x.145 spark2.sxw.com spark2
192.168.x.147 spark3.sxw.com spark3

设置主机名

hostnamectl set-hostname spark1 spark2 spark3

##ntp 时间认证服务

for i in spark1 spark2 spark3;do ssh $i yum install ntp -y ;done
for i in spark1 spark2 spark3;do ssh $i "systemctl enable ntpd && systemctl start ntpd";done

##安装java 所有spark 节点

#set java path
export JAVA_HOME=/usr/local/java
export PATH=$JAVA_HOME/bin:$PATH
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

###所有spark节点安装 scala 环境

wget https://downloads.lightbend.com/scala/2.10.7/scala-2.10.7.rpm

for i in spark1 spark2 spark3;do ssh $i "yum localinstall scala -y ";done

##下载spark

wget  http://mirrors.shu.edu.cn/apache/spark/spark-2.3.3/spark-2.3.3-bin-hadoop2.7.tgz

tar -zxvf spark-2.3.3-bin-hadoop2.7.tgz 
mv spark-2.3.3-bin-hadoop2.7 spark 

cp -r spark /usr/local/
cd /usr/local/spark/conf 

cp spark-env.sh.template spark-env.sh

##到hadoop 的namenode 节点上执行，作为客户端 访问hadoop 集群
scp -r /usr/local/hadoop root@spark1:/usr/local/
##设置变量

vim spark-env.sh 
export JAVA_HOME=/usr/local/java
export HADOOP_HOME=/usr/local/hadoop
export HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop
export LD_LIBRARY_PATH=$HADOOP_HOME/lib/native
export SPARK_WORKER_MEMORY=500m
export SPARK_WORKER_CORES=1
export SPARK_DAEMON_JAVA_OPTS="-Dspark.deploy.recoveryMode=ZOOKEEPER -Dspark.deploy.zookeeper.url=zk1:2181,zk2:2181,zk3:2181 -Dspark.deploy.zookeeper.dir=/spark"

##设置master 

[root@spark1 conf]# cat /usr/local/spark/conf/slaves 
spark1
spark2
spark3

##设置log

mv /usr/local/spark/conf/log4j.properties.template /usr/local/spark/conf/log4j.properties
##设置/etc/profile

#set spark path
export SPARK_HOME=/usr/local/spark
export PATH=$PATH:$SPARK_HOME/bin
#set hadoop path
export HADOOP_HOME=/usr/local/hadoop
export PATH=$PATH:$HADOOP_HOME/bin

##把配置同步到其他spark2 spark3 ##启动服务

/usr/local/spark/sbin/start-all.sh 

web spark1:8080

##测试

/usr/local/spark/bin/spark-submit \
  --class org.apache.spark.examples.SparkPi \
  --master spark://spark1.sxw.com:6066 \
  --deploy-mode cluster \
  --supervise \
  --executor-memory 1G \
  --total-executor-cores 10 \
  /usr/local/spark/examples/jars/spark-examples_2.11-2.1.0.jar \
  100000