spark Historyserver配置

yarn-site.xml

<property>

  <name>yarn.log-aggregation-enable</name>

<value>true</value>

</property>

 

spark-env.sh

 

export SPARK_HISTORY_OPTS="-Dspark.history.retainedApplications=20 -Dspark.history.fs.logDirectory=hdfs://192.168.1.191:9000/spark/log/"

 

 

spark-defaults.conf

spark.eventLog.enabled           true
spark.eventLog.compress          true
spark.eventLog.dir               hdfs://192.168.1.191:9000/spark/log/
spark.yarn.historyServer.address 192.168.1.191:18080