spark单机版搭建记录

spark单机版搭建记录

1.关闭防火墙 & selinux disable

service iptables stop && chkconfig iptables off  

2.免ssh登陆设置

网上很多，这里写上最后一步  

cat id_rsa.pub >> authorized_keys

3.下载scala、jdk、spark版本

[root@spark1 opt]# ll
total 364104
drwxr-xr-x.  8 uucp  143      4096 Mar 18 04:03 jdk1.8.0_05
-rw-r--r--.  1 root root 159910666 Jun 24 22:56 jdk-8u5-linux-i586.gz
drwxrwxr-x.  6 2000 2000      4096 May 21 04:12 scala
-rw-r--r--.  1 root root  25685521 Jun 25 06:53 scala-2.11.1.tgz
drwxrwxr-x. 11 1000 1000      4096 Jun 25 08:32 spark
-rw-r--r--.  1 root root 187224108 Jun 24 22:53 spark-1.0.0-bin-hadoop2.tgz

4.配置/etc/profile文件

JAVA_HOME=/opt/jdk1.8.0_05
CLASSPATH=.:$JAVA_HOME/lib.tools.jar
PATH=$JAVA_HOME/bin:$PATH
export JAVA_HOME CLASSPATH PATH
export SCALA_HOME=/opt/scala
export SPARK_HOME=/opt/spark
export PATH=$SCALA_HOME/bin:$SPARK_HOME/bin:$PATH

source  /etc/profile生效

5.配置spark文件

export SPARK_MASTER_IP=192.168.43.188
export SPARK_WORKER_CORES=1
export SPARK_WORKER_INSTANCES=1
export SPARK_MASTER_PORT=7074
export SPARK_WORKER_MEMORY=512m
export MASTER=spark://${SPARK_MASTER_IP}:${SPARK_MASTER_PORT}
[root@spark1 spark]# cat  conf/spark-env.sh

[root@spark1 conf]# cat slaves 
# A Spark Worker will be started on each of the machines listed below.
192.168.43.188

6.启动集群

[root@spark1 ~]# cd /opt/spark/sbin/
[root@spark1 sbin]# ls
slaves.sh        spark-daemon.sh   spark-executor  start-history-server.sh  start-slave.sh   stop-all.sh             stop-master.sh
spark-config.sh  spark-daemons.sh  start-all.sh    start-master.sh          start-slaves.sh  stop-history-server.sh  stop-slaves.sh
[root@spark1 sbin]# ./stop-all.sh 
192.168.43.188: stopping org.apache.spark.deploy.worker.Worker
stopping org.apache.spark.deploy.master.Master
[root@spark1 sbin]# ./start-all.sh 
starting org.apache.spark.deploy.master.Master, logging to /opt/spark/sbin/../logs/spark-root-org.apache.spark.deploy.master.Master-1-spark1.ouyang.cn.out
192.168.43.188: starting org.apache.spark.deploy.worker.Worker, logging to /opt/spark/sbin/../logs/spark-root-org.apache.spark.deploy.worker.Worker-1-spark1.ouyang.cn.out
[root@spark1 sbin]# 

7.查看启动界面