hadoop 自编译(Linux)

1. 准备JDK,maven,ant,protobuf, hadoop 源码包

2. 下载包链接: https://pan.baidu.com/s/1a16hxPheMdm99g01yZvjSw?pwd=rzm9 提取码: rzm9

3. 上传包到Linux 中

4. 解压安装 JDK,Maven,ant

   
   tar -zxvf jdk-8u301-linux-x64.tar.gz 
   mv jdk1.8.0_301/ /usr/local/
   tar -zxvf apache-maven-3.6.3-bin.tar.gz 
   mv apache-maven-3.6.3/ /usr/local/
   
   tar -zvf apache-ant-1.10.7-bin.tar.gz     
   mv apache-ant-1.10.7/ /usr/local/
   
   

5. 配置环境变量(vi /etc/profile)

   #JAVA_HOME 
   export JAVA_HOME=/usr/local/jdk1.8.0_301
   export PATH=$PATH:$JAVA_HOME/bin
   #MAVEN_HOM 
   export MAVEN_HOME=/usr/local/apache-maven-3.6.3
   export PATH=$PATH:$MAVEN_HOME/bin
   #ANT_HOME 
   export ANT_HOME=/usr/local/apache-ant-1.10.7
   export PATH=$PATH:$ANT_HOME/bin ```
   

6. source /etc/profile (使配置生效)

7. 配置maven源下载(vi /usr/local/apache-maven-3.6.3/conf/settings.xml)

   <mirrors>
    <!-- mirror
     | Specifies a repository mirror site to use instead of a given repository. The repository that
     | this mirror serves has an ID that matches the mirrorOf element of this mirror. IDs are used
     | for inheritance and direct lookup purposes, and must be unique across the set of mirrors.
     |
    
        <mirror>
        <id>alimaven</id>
        <mirrorOf>central</mirrorOf>
         <name>aliyun maven</name>
        <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
        </mirror>
        <mirror>
        <id>mirrorId</id>
        <mirrorOf>repositoryId</mirrorOf>
        <name>Human Readable Name for this Mirror.</name>
        <url>http://my.repository.com/repo/path</url>
        </mirror>
     -->
     </mirrors>
   

8. 配置protobuf

   yum -y install gcc c++
   yum  -y install  gcc-c++
   yum -y install  make
   tar -zxvf protobuf-2.5.0.tar.gz 
   mv protobuf-2.5.0/ /usr/local/
   cd /usr/local/protobuf-2.5.0/
   ./configure 
   
   #配置环境变量(vi /etc/profile)
   #protobuf
    export LD_LIBRARY_PATH=/usr/local/protobuf-2.5.0
    export PATH=$PATH:$LD_LIBRARY_PATH
    ##配置生效
    source /etc/profile
   

9. 各种依赖命令

   yum -y install svn ncurses-devel gcc*  
   yum -y install lzo-devel zlib-devel autoconf automake libtool openssl-devel
   #安装cmake 3.16+
   tar -zxvf cmake
   mv cmake-3.16.5 /usr/local/
   cd /usr/local/cmake-3.16.5
   ./bootstrap
   gmake
   yum remove cmake -y
   ln -s /usr/local/cmake-3.16.5 /usr/bin/
   vi /etc/profile.d/cmake.sh
   # cmake安装方式二
   export CMAKE_HOME=/usr/local/cmake-3.16.5
   export PATH=$PATH:$CMAKE_HOME/bin
   cmake --version
   
   

10. 编译hadoop

    tar -zxvf hadoop-3.3.1-src.tar.gz 
    mv hadoop-3.3.1-src /usr/local/ 
    cd /usr/local/hadoop-3.3.1-src/
    方式一
    mvn clean   package -Pdist,native -DskipTests -Dtar
    方式二
    mvn clean package -Pdist,native  -DskipTests -Dtar -Dmaven.javadoc.skip=true   (约30分钟左右耐心等待)
    

11. 成功后在 hadoop-dist/target/ 下找生成的包