Spark安装编译

安装maven，java，hadoop，scala

vi /etc/profile

export MAVEN_HOME=/usr/local/apache-maven-3.3.9
export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m"
export PATH=$MAVEN_HOME/bin:$JAVA_HOME/bin:$PATH


export JAVA_HOME=/usr/java/jdk1.7.0_80


export HIVE_HOME=/usr/local/hive
export PATH=$HIVE_HOME/bin:$PATH


export HADOOP_HOME=/usr/local/hadoop
export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH


export SQOOP_HOME=/usr/local/sqoop
export PATH=$SQOOP_HOME/bin:$PATH


export SCALA_HOME=/usr/local/scala-2.11.8
export PATH=$SCALA_HOME/bin:$PATH

http://spark.apache.org/docs/latest/building-spark.html

cd /usr/local/spark-2.2.0

mvn -Pyarn -Phive -Phive-thriftserver -Phadoop-2.6 -Dhadoop.version=2.6.0-cdh5.7.1 -DskipTests clean package

#没有tgz包

./dev/make-distribution.sh --name custom-spark --pip --r --tgz -Psparkr -Phadoop-2.7 -Phive -Phive-thriftserver -Pmesos -Pyarn

./dev/make-distribution.sh --name 2.6.0-cdh5.7.1 --tgz -Dhadoop.version=2.6.0-cdh5.7.1 -Phadoop-2.6 -Phive -Phive-thriftserver -Pyarn

vi dev/make-distribution.sh 

#编译提速

VERSION=2.2.0

SCALA_VERSION=2.11

SPARK_HADOOP_VERSION=2.6.0-cdh5.7.1

SPARK_HIVE=1

vi pom.xml 

    <repository>
      <id>cloudera</id>
      <name>cloudera Repository</name>
      <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
    </repository>

https://github.com/apache/spark

【来自@若泽大数据】