安装spark-on-yarn

配置hadoop

添加

vim /opts/hadoop/etc/hadoop/yarn-site.xml

<property>
        <name>yarn.nodemanager.pmem-check-enabled</name> 
        <value>false</value>
        </property>
<property>
        <name>yarn.nodemanager.vmem-check-enabled</name>
        <value>false</value>
        </property>

修改spark

mv /opts/spark/conf/spark-env.sh.template /opts/spark/conf/spark-env.sh

vim spark-env.sh

export JAVA_HOME=/opts/jdk
YARN_CONF_DIR=/opts/hadoop/etc/hadoop

mv spark-defaults.conf.template spark-defaults.conf

spark.eventLog.enabled	true
spark.eventLog.dir	hdfs://node01:8020/directory

export SPARK_HISTORY_OPTS="
-Dspark.history.ui.port=18080
-Dspark.history.fs.logDirectory=hdfs://node01/directory
-Dspark.history.retainedApplications=30"

测试spark

spark-submit \
--class org.apache.spark.examples.SparkPi \
--master yarn \
--deploy-mode cluster \
/opts/spark/examples/jars/spark-examples_2.12-3.1.2.jar