Spark 配置

 

1. 解压

   root@master:~# cd /usr/local
   root@master:/usr/local# tar -zxvf spark-2.3.0-bin-hadoop2.7.tgz

2. 文件夹重命名

   root@master:/usr/local# mv spark-2.3.0-bin-hadoop2.7 spark

    

3. 配置环境变量

   root@master:/usr/local# vim /etc/profile
   export SPARK_HOME=/usr/local/spark
   export PATH=$PATH:$SPARK_HOME/bin:$SPARK_HOME/sbin

    

4. 同样，需要该环境变量生效

   root@master:/usr/local# source /etc/profile

    

5. 复制并重命名配置文件 spark-env.sh

   root@master:~# cd /usr/local/spark/conf
   root@master:/usr/local/spark/conf# cp spark-env.sh.template spark-env.sh 

    

6. 编辑spark-env.,sh文件，在第一行添加以下配置信息

   root@master:/usr/local/spark/conf# vim spark-env.sh

   export JAVA_HOME=/usr/local/java
   export SCALA_HOME=/usr/local/scala
   export HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop
   export HADOOP_HDFS_HOME=/usr/local/hadoop
   export SPARK_HOME=/usr/local/spark
   export SPARK_MASTER_IP=master
   export SPARK_MASTER_PORT=7077
   export SPARK_MASTER_HOST=master
   export SPARK_WORKER_CORES=2
   export SPARK_WORKER_PORT=8901
   export SPARK_WORKER_INSTANCES=1
   export SPARK_WORKER_MEMORY=2g
   export SPARK_DIST_CLASSPATH=$(/usr/local/hadoop/bin/hadoop classpath)
   export SPARK_MASTER_WEBUI_PORT=8079

    

7. 保存并生效

   root@master:/usr/local/spark/conf# source spark-env.sh

    

8. 复制slaves

   先复制slaves.template 并重命名slaves节点文件
   root@master:/usr/local/spark/conf# cp slaves.template slaves
   root@master:/usr/local/spark/conf# vim slaves 

   删除默认的localhost 增加主从节点
   hadoop01
   hadoop02

    

9. 传送到其他节点上

   root@master:/usr/local/spark/conf# scp -r /usr/local/spark
   root@slave1:/usr/local
   root@master:/usr/local/spark/conf# scp -r /usr/local/spark
   root@slave2:/usr/local

    

10. 测试运行

    root@master:/usr/local/hbase/conf# cd
    启动spark
    root@master:~# /usr/local/spark/sbin/start-all.sh
    运行spark自带的例子求pi值
    root@master:~# /usr/local/spark/bin/run-example SparkPi 2>&1 | grep "Pi is"