安装spark:
tar -zxvf ./spark-2.4.3-bin-hadoop2.7.tgz -C ./spark
export SPARK_HOME=/home/service/spark-2.4.5-bin-hadoop2.7
export PATH=$SPARK_HOME/bin:$PATH
安装hadoop:
tar -zxvf ./hadoop-2.6.0-cdh5.8.5.tar.gz -C ./hadoop
export HADOOP_HOME=/home/service/hadoop-2.7.7
export PATH=$HADOOP_HOME/bin:$PATH
安装scala:
tar -zxvf ./scala-2.13.0.tgz -C ./scala
export SCALA_HOME=/home/service/scala-2.11.0
export PATH=$SCALA_HOME/bin:$PATH
source ~/.bashrc
安装pyspark:
pip install pyspark
参考:
linux下spark-submit遇到的问题:
--master local模型下通过conf.get函数获取spark中driver和executor相关的属性值均为None
spark-submit任务--master yarn模式报错:
问题1:Exception in thread "main" org.apache.spark.SparkException: When running with master 'yarn' e