Spark on Hadoop Yarn 部署

前提：haoop yarn已经部署完毕，关于hadoop yarn的部署已经在前面文章中说明

1. 安装scala

下载scala-2.11.7.tgz

sudo ln -s /home/tizen/share/software/scala-2.11.7 scala

export SCALA_HOME=/usr/local/scala
export PATH=$PATH:$SCALA_HOME/bin

2. 安装spark

spark-1.6.0-bin-hadoop2.6.tgz

sudo ln -s ~/share/software/spark-1.6.0-bin-hadoop2.6 spark

export SPARK_HOME=/usr/local/spark
export PATH=$PATH:$SPARK_HOME/bin

3. 修改配置文件

cd /usr/local/spark/conf
cp spark-env.sh.template spark-env.sh

vim spark-env.sh
export JAVA_HOME=/usr/local/jdk
export SCALA_HOME=/usr/local/scala
export HADOOP_HOME=/usr/local/hadoop

cp log4j.properties.template log4j.properties

cp slaves.template slaves

vim slaves
namenode
datanode

4. master 节点执行任务

cd /usr/local/spark

sbin/start-all.sh

5. jps

5813 Jps
5760 Worker
5634 Master

/usr/local/hadoop/sbin

start-all.sh

jps

6114 DataNode
5760 Worker
6483 ResourceManager
6929 Jps
6615 NodeManager
6327 SecondaryNameNode
5988 NameNode
5634 Master

http://192.168.0.101:8080/

$ cat spark.sh
SPARK_JAR=/usr/local/spark/lib/spark-assembly-1.6.0-hadoop2.6.0.jar \
/usr/local/spark/bin/spark-class org.apache.spark.deploy.yarn.Client \
--jar /usr/local/spark/lib/spark-examples-1.6.0-hadoop2.6.0.jar \
--class org.apache.spark.examples.SparkPi \
--args yarn-standalone \
--num-workers 2 \
--master-memory 1g \
--worker-memory 1g \
--worker-cores 1