链接:https://pan.baidu.com/s/1C7bKh8SPbFTw3zaPR5XNgA 提取码:sgzw
上面是直接提取配置文件一步操作 只需解压 执行是start-spark 关闭是stop-spark.sh
两个进程
链接:https://pan.baidu.com/s/1PMXP3kishCQcBpxzQHyxcg 提取码:l6n1
2.0.2版本
cd /usr/local/
#上传spark
tar xf spark-2.0.2-bin-hadoop2.7.tgz
mv spark-2.0.2-bin-hadoop2.7 spark
vim /etc/profile
export SPARK_HOME=/usr/local/spark
:$SPARK_HOME/bin
source /etc/profile
cd spark/conf/
mv spark-env.sh.template spark-env.sh
mv slaves.template slaves
mv spark-defaults.conf.template spark-defaults.conf
#修改配置文件
vim spark-env.sh
export SCALA_HOME=/usr/scala
export SPARK_HOME=/usr/local/spark
export HADOOP_HOME=/usr/local/hadoop
export JAVA_HOME=/usr/java
export SPARK_WORKER_MEMORY=1g
export SPARK_MASTER_IP=hadoop
export SPARK_WORKER_CORES=2
################################
cd ../sbin/
#编写启动脚本
vim start-spark.sh
#!/bin/bash
/usr/local/spark/sbin/start-all.sh
#保存退出
chmod 755 start-spark.sh
vim /etc/profile
:$SPARK_HOME/sbin
source /etc/profile
cp start-spark.sh stop-spark.sh
vim stop-spark.sh
#!/bin/bash
/usr/local/spark/sbin/stop-all.sh
chmod 755 stop-spark.sh
#启动spark
start-all.sh
start-spark.sh
spark-shell
hadoop下配置文件
year-sist.sh
<property>
<name>yarn.nodemanager.vmem-check-enabled</name>
<value>false</value>
</property>
<property>
<name>yarn.nodemanager.pmem-check-enabled</name>
<value>false</value>
</property>
IDEA软件maven配置pox.xml
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>##scala版本
<version>2.0.2</version>
</dependency>
spark代码圆周率
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import scala.math.random
object SparkPi {
def main(args: Array[String]){
val conf = new SparkConf().setAppName("Spark Pi")
val spark = new SparkContext(conf)
val slices = if (args.length > 0) args(0).toInt else 2
val n = math.min(100000L * slices, Int.MaxValue).toInt
val count = spark.parallelize(1 until n, slices).map { i =>
val x = random * 2 - 1
val y = random * 2 - 1
if (x * x + y * y < 1) 1 else 0
}.reduce(_+_)
println("Pi is roughly" + 4.0 * count / n)
spark.stop()
}
}
执行
spark-submit --master yarn --class SparkPi aaaaaaaa.jar 100
class 加包名字 。/