本地spark安装运行spark-shell

1、scala安装

1.1 jdk安装

安装jdk1.8

wget --no-cookies \
--no-check-certificate \
--header "Cookie: oraclelicense=accept-securebackup-cookie" \
http://download.oracle.com/otn-pub/java/jdk/8u151-b12/jdk-8u151-linux-x64.tar.gz \

-O jdk-8-linux-x64.tar.gz

tar -zvxf 

export JAVA_HOME=/home/duhj/duhj/jdk1.8
export JRE_HOME=${JAVA_HOME}/jre
export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib 
export PATH=${JAVA_HOME}/bin:$PATH

1.2 scala安装

https://scala-lang.org/download/2.11.8.html　安装2.11

export SCALA_HOME=/home/duhj/duhj/scala-2.11.8/bin

export PATH=$PATH:$SCALA_HOME/bin:$PATH

2、hadoop 安装

export HADOOP_HOME=/home/duhj/hadoop-2.9.0  
export HADOOP_CONF_DIR=${HADOOP_HOME}/etc/hadoop  
export HADOOP_COMMON_LIB_NATIVE_DIR=${HADOOP_HOME}/lib/native  

export HADOOP_OPTS="-Djava.library.path=${HADOOP_HOME}/lib" 

３、spark安装

https://spark.apache.org/downloads.html

export SPARK_HOME=/home/duhj/spark-2.3.0-bin-hadoop2.7

同时编辑/home/duhj/spark-2.3.0-bin-hadoop2.7/conf/spark-env.sh

export SPARK_DIST_CLASSPATH=$(/home/duhj/hadoop-2.9.0/bin/hadoop classpath)
export JAVA_HOME=/home/duhj/duhj/jdk1.8
export HADOOP_CONF_DIR=/home/duhj/hadoop-2.9.0/etc/hadoop/