解决方法:
spark-config.sh 中添加jdk 路径
export JAVA_HOME=/usr/local/java/jdk1.8.0_171
如下:
if [ -z "${PYSPARK_PYTHONPATH_SET}" ]; then
export PYTHONPATH="${SPARK_HOME}/python:${PYTHONPATH}"
export PYTHONPATH="${SPARK_HOME}/python/lib/py4j-0.10.3-src.zip:${PYTHONPATH}"
export PYSPARK_PYTHONPATH_SET=1
export JAVA_HOME=/usr/local/java/jdk1.8.0_171
fi