1. 通过xmanager的Xftp上传spark-1.4.1-bin-hadoop2.6.tgz文件到/usr/local目录
2. 解压缩spark-1.4.1-bin-hadoop2.6.tgz文件
tar -zxf spark-1.4.1-bin-hadoop2.6.tgz
解压后即可,看到/usr/local/spark-1.4.1-bin-hadoop2.6文件夹
3. 配置Spark ,进入目录
cd /usr/local/spark-1.4.1-bin-hadoop2.6/conf
3.1 配置文件spark-env.sh(从spark-env.sh.template拷贝即可)
export JAVA_HOME=/usr/java/jdk1.7.0_80
export HADOOP_CONF_DIR=/usr/local/hadoop-2.6.4/etc/hadoop
export SPARK_MASTER_IP=master
export SPARK_MASTER_PORT=7077
export SPARK_WORKER_MEMORY=512m
export SPARK_WORKER_CORES=1
export SPARK_WORKER_INSTANCES=1
3.2 配置文件 slaves(从slaves.template拷贝)
删除内容,并添加
slave1
slave2
slave3
3.3 配置文件spark-defaults.conf (从spark-defaults.conf.template拷贝)
spark.master spark://master:7077
spark.eventLog.enabled true
spark.eventLog.dir hdfs://master:8020/spark-log
在HDFS中新建目录:
hadoop fs -mkdir /spark-log
hadoop fs -mkdir /applogs/spark
先拷贝:
scp -r /usr/local/spark-1.4.1-bin-hadoop2.6/ slava1:/usr/local
4. 启动Spark
进入目录:
cd /usr/local/spark-1.4.1-bin-hadoop2.6/sbin
启动spark
./start-all.sh
浏览器查看:
http://192.168.128.130:8080
5.命令行执行Spark
./spark-submit --class demo.WordCount --master spark://master:7077 --executor-memory 512m --total-executor-cores 2 /opt/wordcount.jar /user/root/aaa.txt
//用MR运行
./spark-submit --master yarn --class demo.ALSTrainer /opt/als.jar /root/ratings.dat /root/als_output 10 10 0.01
//3就是所有的资源
2. 解压缩spark-1.4.1-bin-hadoop2.6.tgz文件
tar -zxf spark-1.4.1-bin-hadoop2.6.tgz
解压后即可,看到/usr/local/spark-1.4.1-bin-hadoop2.6文件夹
3. 配置Spark ,进入目录
cd /usr/local/spark-1.4.1-bin-hadoop2.6/conf
3.1 配置文件spark-env.sh(从spark-env.sh.template拷贝即可)
export JAVA_HOME=/usr/java/jdk1.7.0_80
export HADOOP_CONF_DIR=/usr/local/hadoop-2.6.4/etc/hadoop
export SPARK_MASTER_IP=master
export SPARK_MASTER_PORT=7077
export SPARK_WORKER_MEMORY=512m
export SPARK_WORKER_CORES=1
export SPARK_WORKER_INSTANCES=1
3.2 配置文件 slaves(从slaves.template拷贝)
删除内容,并添加
slave1
slave2
slave3
3.3 配置文件spark-defaults.conf (从spark-defaults.conf.template拷贝)
spark.master spark://master:7077
spark.eventLog.enabled true
spark.eventLog.dir hdfs://master:8020/spark-log
在HDFS中新建目录:
hadoop fs -mkdir /spark-log
hadoop fs -mkdir /applogs/spark
先拷贝:
scp -r /usr/local/spark-1.4.1-bin-hadoop2.6/ slava1:/usr/local
4. 启动Spark
进入目录:
cd /usr/local/spark-1.4.1-bin-hadoop2.6/sbin
启动spark
./start-all.sh
浏览器查看:
http://192.168.128.130:8080
5.命令行执行Spark
./spark-submit --class demo.WordCount --master spark://master:7077 --executor-memory 512m --total-executor-cores 2 /opt/wordcount.jar /user/root/aaa.txt
//用MR运行
./spark-submit --master yarn --class demo.ALSTrainer /opt/als.jar /root/ratings.dat /root/als_output 10 10 0.01
//3就是所有的资源