spark编译

一、编译

1.安装maven

#maven
export MAVEN_HOME=/home/pxj/app/apache-maven-3.6.3
export PATH=$MAVEN_HOME/bin:$PATH
#ant
export ANT_HOME=/home/pxj/app/apache-ant-1.10.7
export PATH=${ANT_HOME}/bin:$PATH

1.2.修改

vim  settings.xml
 <mirror>
    <id>alimaven</id>
    <name>aliyun maven</name>
    <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
    <mirrorOf>central</mirrorOf>
</mirror>

2.spark的编译

[pxj@pxj /home/pxj/app/spark-2.4.5]$cat pom.xml 
<repository>
      <!--
        This is used as a fallback when the first try fails.
      -->
      <id>cloudera</id>
     
      <url>http://repository.cloudera.com/artifactory/cloudera-repos/</url>
     
    </repository>

2.1make-distribution.sh 修改

[pxj@pxj /home/pxj/app/spark-2.4.5/dev]$vim make-distribution.sh 
VERSION=2.4.5
SCALA_VERSION=2.12.10
SPARK_HADOOP_VERSION=2.6.0-cdh5.16.2
SPARK_HIVE=1
#VERSION=("$MVN" help:evaluate -Dexpression=project.version $@ 2>/dev/null\
#    | grep -v "INFO"\
#    | grep -v "WARNING"\
#    | tail -n 1)
#SCALA_VERSION=$("$MVN" help:evaluate -Dexpression=scala.binary.version $@ 2>/dev/null\
#    | grep -v "INFO"\
#    | grep -v "WARNING"\
#    | tail -n 1)
#SPARK_HADOOP_VERSION=$("$MVN" help:evaluate -Dexpression=hadoop.version $@ 2>/dev/null\
#    | grep -v "INFO"\
#    | grep -v "WARNING"\
#    | tail -n 1)
#SPARK_HIVE=$("$MVN" help:evaluate -Dexpression=project.activeProfiles -pl sql/hive $@ 2>/dev/null\
#    | grep -v "INFO"\
#    | grep -v "WARNING"\
#    | fgrep --count "<id>hive</id>";\
#    # Reset exit status to 0, otherwise the script stops here if the last grep finds nothing\
#    # because we use "set -o pipefail"
#    echo -n)

作者：潘陈
日期：2020-03-21 0：45