Hudi的编译安装

hudi搭建

对于hudi的安装，我们需要自己下载包进行编译。hudi下载地址Index of /dist/hudi (apache.org) ，下面是hudi编译的详细步骤：

1、下载maven安装包

maven下载地址为：https://maven.apache.org/download.cgi

#利用wget工具进行maven资源拉取
wget https://dlcdn.apache.org/maven/maven-3/3.8.4/binaries/apache-maven-3.8.4-bin.tar.gz

2.解压并配置环境变量

1.解压安装包

#解压环境
tar -zxvf apache-maven-3.8.4-bin.tar.gz -C /opt/software/
#重命名
mv apache-maven-3.8.4/ maven384

2.配置环境变量

#打开环境变量文件，输入以下内容
vim /etc/profile.d/my.sh
#---------------------------------------
#MAVEN
export MAVEN_HOME=/opt/software/maven384
export PATH=$PATH:$MAVEN_HOME/bin
#---------------------------------------
#生效环境变量
source /etc/profile

3、maven仓库配置

1.打开文件

#打开配置文件
vim /opt/software/maven384/conf/settings.xml
#创建本地仓库
mkdir /opt/software/maven384/ck

2.输入以下内容

<!--本地maven仓库地址-->
<localRepository>/opt/software/maven384/ck</localRepository>
<!--配置阿里的maven仓库-->
    <mirror>
       <id>alimaven</id>
       <name>aliyun maven</name>
           <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
       <mirrorOf>central</mirrorOf>
    </mirror>
<!--配置华为的maven仓库-->
    <mirror>
      <id>huaweicloud</id>
      <mirrorOf>*</mirrorOf>
      <url>https://mirrors.huaweicloud.com/repository/maven/</url>
    </mirror>

4、验证maven仓库是否配置完成

#查看maven版本信息，验证maven是否安装成功
mvn -v
#---------------------------------------------------------------------------------------
Apache Maven 3.8.4 (9b656c72d54e5bacbed989b64718c159fe39b537)
Maven home: /opt/software/maven384
Java version: 1.8.0_171, vendor: Oracle Corporation, runtime: /opt/software/jdk8/jre
Default locale: en_US, platform encoding: UTF-8
OS name: "linux", version: "3.10.0-1160.45.1.el7.x86_64", arch: "amd64", family: "unix"
#---------------------------------------------------------------------------------------

5、编译打包hudi

hudi原生包的下载地址为：https://hudi.apache.org/releases/download

1.解压文件

#wget下载hudi
wget https://www.apache.org/dyn/closer.lua/hudi/0.10.0/hudi-0.10.0.src.tgz
#解压hudi安装包
tar -zxvf hudi-0.10.0.src.tgz -C /opt/software/

2.配置相关配置信息

#打开配置文件,添加远程镜像源
vim /opt/software/hudi-0.10.0/pom.xml
#--------------------------------------------------------------
    <repository>
     <id>nexus-aliyun</id>
      <name>nexus-aliyun</name>
      <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
      <releases>
        <enabled>true</enabled>
      </releases>
      <snapshots>
        <enabled>false</enabled>
      </snapshots>
    </repository>
    
    <repository>
      <id>huaweicloud</id>
      <mirrorOf>*</mirrorOf>
      <url>https://mirrors.huaweicloud.com/repository/maven/</url>
      <releases>
        <enabled>true</enabled>
      </releases>
      <snapshots>
        <enabled>false</enabled>
      </snapshots>
    </repository>
    <repository>
         <id>maven-ossez</id>
         <name>OSSEZ Repository</name>
         <url>https://maven.ossez.com/repository/internal</url>
         <releases>
        <enabled>true</enabled>
      </releases>
      <snapshots>
        <enabled>false</enabled>
      </snapshots>
     </repository>
#--------------------------------------------------------------

3.编译

#对hudi进行版本编译，支持scala2.12和spark3
mvn clean package -DskipTests -DskipITs -Dscala-2.12 -Dspark3