Hadoop-3.22+ snappy + lzo源码编译安装

编译安装cmake 3.21.4

编译cmake
yum install openssl-devel -y
wget https://github.com/Kitware/CMake/releases/download/v3.21.4/cßmake-3.21.4.tar.gz
tar xf cmake-3.21.4.tar.gz
cd cmake-3.21.4
./bootstrap && make -j4 && sudo make install

安装maven

#!/bin/bash
install_maven(){
  source_url='https://mirrors.tuna.tsinghua.edu.cn/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz'
  cd ~ && wget --no-check-certificate ${source_url} -O apache-maven-3.6.3-bin.tar.gz 
  tar xvf apache-maven-3.6.3-bin.tar.gz -C /usr/local/
if [ -d /usr/local/apache-maven-3.6.3 ];then
cat > /etc/profile.d/maven.sh <<-EOF
#!/bin/bash
export M2_HOME=/usr/local/apache-maven-3.6.3
export PATH=\$PATH:\$M2_HOME/bin
EOF
  chmod 0744 /etc/profile.d/maven.sh
  source /etc/profile.d/maven.sh
  rm -f ~/apache-maven-3.6.3-bin.tar.gz
fi
}

install_maven

cat > /usr/local/apache-maven-3.6.3/conf/settings.xml <<-EOF
<?xml version="1.0" encoding="UTF-8"?>
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
    <localRepository>/root/.m2/repository</localRepository>
    <pluginGroups>
    </pluginGroups>
    <proxies>
    </proxies>
    <servers>
    </servers>
    <mirrors>
        <mirror>
            <id>aliyunmaven</id>
            <mirrorOf>*</mirrorOf>
            <name>阿里云公共仓库</name>
            <url>https://maven.aliyun.com/repository/public</url>
        </mirror>
        <mirror>
            <id>aliyun-central</id>
            <mirrorOf>central</mirrorOf>
            <name>阿里云central仓库</name>
            <url>https://maven.aliyun.com/repository/central</url>
        </mirror>
        <mirror>
            <id>aliyun-google</id>
            <mirrorOf>central</mirrorOf>
            <name>阿里云google仓库</name>
            <url>
                https://maven.aliyun.com/repository/google
            </url>
        </mirror>
        <mirror>
            <id>aliyun-gradle</id>
            <mirrorOf>central</mirrorOf>
            <name>阿里云gradle仓库</name>
            <url>
                https://maven.aliyun.com/repository/gradle-plugin
            </url>
        </mirror>
        <mirror>
            <id>aliyun-spring</id>
            <mirrorOf>central</mirrorOf>
            <name>阿里云spring仓库</name>
            <url>
                https://maven.aliyun.com/repository/spring
            </url>
        </mirror>
        <mirror>
            <id>aliyun-spring-plugin</id>
            <mirrorOf>central</mirrorOf>
            <name>阿里云spring-plugin仓库</name>
            <url>
                https://maven.aliyun.com/repository/spring-plugin
            </url>
        </mirror>
        <mirror>
            <id>aliyun-grails-core</id>
            <mirrorOf>central</mirrorOf>
            <name>阿里云grails-core仓库</name>
            <url>
                https://maven.aliyun.com/repository/grails-core
            </url>
        </mirror>
        <mirror>
            <id>aliyun-apache-snapshots</id>
            <mirrorOf>central</mirrorOf>
            <name>阿里云apache-snapshots仓库</name>
            <url>
                https://maven.aliyun.com/repository/apache-snapshots
            </url>
        </mirror>
        <mirror>
            <id>nexus-mirror</id>
            <mirrorOf>central</mirrorOf>
            <url>https://maven2.meifute.com/repository/maven-public/</url>
        </mirror>
    </mirrors>
    <profiles>
    </profiles>
</settings>
EOF

编译安装protobuf

安装编译protobuf
# 编译protoc 版本要求是2.5.0
wget https://github.com/protocolbuffers/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz
$ tar xf protobuf-2.5.0.tar.gz
$ ./configure
$ make -j 8
$ make install
$ protoc --version
libprotoc 2.5.0

编译安装findbugs

wget https://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-3.0.1.tar.gz/download --no-check-certificate

下载并编译lzo

# yum install lzo-devel -y
wget http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz

tar -zxvf lzo-2.10.tar.gz

cd lzo-2.10
./configure -prefix=/usr/local/hadoop/lzo/
make
make install


# 编译hadoop-lzo源码
wget https://github.com/twitter/hadoop-lzo/archive/master.zip
修改pom.xml
<hadoop.current.version>3.2.2</hadoop.current.version>

export C_INCLUDE_PATH=/usr/local/hadoop/lzo/include
export LIBRARY_PATH=/usr/local/hadoop/lzo/lib
mvn package -Dmaven.test.skip=true

将编译的包重命名后发到hadoop的common文件夹
cp hadoop-lzo-0.4.21.jar hadoop/current/share/hadoop/common/

下载源码包

yum install -y snappy-devel openssl-devel gcc c++ autoconf automake libtool libzstd-devel zlib-devel bzip2-devel lz4-devel lzo-devel lzop ncurses-devel build-essential cyrus-sasl* libgsasl-devel* 

wget https://dlcdn.apache.org/hadoop/common/hadoop-3.2.2/hadoop-3.2.2-src.tar.gz --no-check-certificate



mvn clean package -Pdist,native -DskipTests -Dtar -Dsnappy.lib=/usr/lib64 -Dbundle.snappy -Dzstd.lib=/usr/lib64 -Dbundle.zstd -Dbzip2.lib=/usr/lib64 -Dbundle.bzip2 -Dopenssl.lib=/usr/lib64 -Dbundle.openssl -Drequire.lz4 

cd hadoop-dist/target

包在里面
记得将lzo的jar包放到 hadoop/share/hadoop/common/里