CentOS and Ubuntu 开发环境搭建

Common

下载或copy以下开发环境到目标开发机

# ll
total 20188
drwxr-xr-x 8 root root      255 May 12 04:32 jdk
drwxr-xr-x 6 root root       99 May 12 04:32 maven
drwxr-xr-x 5 root root       51 May 12 04:32 sbt
drwxrwxr-x 6 root root       79 May 12 04:32 scala
drwxrwxr-x 6 root root       79 May 12 04:32 scala-2.12.10
-rw-r--r-- 1 root root 20669259 May 12 04:32 scala-2.12.10.tgz

然后设置环境变量即可

export JAVA_HOME=/opt/dev/jdk
export PATH=/opt/dev/jdk/bin:/opt/dev/jdk/sbin:$PATH

export MAVEN_HOME=/opt/dev/maven
export PATH=/opt/dev/maven/bin:/opt/dev/maven/sbin:$PATH

export SBT_HOME=/opt/dev/sbt
export PATH=/opt/dev/sbt/bin:/opt/dev/sbt/sbin:$PATH

export SCALA_HOME=/opt/dev/scala
export PATH=/opt/dev/scala/bin:$PATH

export LD_LIBRARY_PATH=/usr/lib64/:/usr/local/lib64/:$LD_LIBRARY_PATH

copy maven repo

从成熟的环境中 copy .m2 到目标机器

设置LD_LIBRARY_PATH

export LD_LIBRARY_PATH=/usr/lib64/:/usr/local/lib64/:$LD_LIBRARY_PATH

install jemalloc

wget https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2
tar -xvf jemalloc-5.2.1.tar.bz2
cd ./jemalloc-5.2.1/
./configure --prefix=/usr/
make -j$(nproc)
make install     -j$(nproc)
cmake --version

查看 Glibc 版本

# ldd --version
ldd (GNU libc) 2.17
Copyright (C) 2012 Free Software Foundation, Inc.
This is free software; see the source for copying conditions.  There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
Written by Roland McGrath and Ulrich Drepper.

以下的方式也可以查看版本号

# ls -l /lib64/libc.so.6
lrwxrwxrwx 1 root root 12 May 29 03:16 /lib64/libc.so.6 -> libc-2.17.so

CentOS

安装基础环境

yum -y install gcc gcc-c++ python-devel libcurl-devel python3 git
yum -y install automake autoconf libtool
yum -y install gmp-devel mpfr-devel libmpc-devel

安装GCC 9.3

注意GCC 编译安装耗时比较长，半小时左右

wget https://bigsearcher.com/mirrors/gcc/releases/gcc-9.3.0/gcc-9.3.0.tar.xz --no-check-certificate
tar -xvf gcc-9.3.0.tar.xz
cd gcc-9.3.0
./configure --prefix=/usr --disable-multilib
make -j$(nproc)
make install     -j$(nproc)
cd ..

安装 openssl-1.1.1n

openssl version
wget https://www.openssl.org/source/openssl-1.1.1n.tar.gz --no-check-certificate
tar zxvf openssl-1.1.1n.tar.gz
cd openssl-1.1.1n
#安装在/usr/，可以覆盖旧的openssl
./config --prefix=/usr/
make -j$(nproc)
make install     -j$(nproc)
#如果
./config --prefix=/usr/local/openssl
echo "xxxx" >> /etc/ld.so.conf
ldconfig
openssl version
cd ..

安装cmake-3.19

wget https://cmake.org/files/v3.19/cmake-3.19.4.tar.gz
tar -zxvf cmake-3.19.4.tar.gz
cd cmake-3.19.4
./configure --prefix=/usr/
make -j$(nproc)
make install     -j$(nproc)
cmake --version
cd ..

如果安装 LLVM-7

wget -t 0 -c --no-check-certificate http://releases.llvm.org/7.0.1/llvm-7.0.1.src.tar.xz
tar xf llvm-7.0.1.src.tar.xz
cd llvm-7.0.1.src/
mkdir -p tools
cd tools
wget -t 0 -c --no-check-certificate http://releases.llvm.org/7.0.1/cfe-7.0.1.src.tar.xz
tar xf cfe-7.0.1.src.tar.xz
mv cfe-7.0.1.src clang
 cd ..
mkdir -p build
cd build
cmake -DCMAKE_BUILD_TYPE=Release ..
make -j$(nproc)
make install     -j$(nproc)

安装LLVM-11

wget https://github.com/llvm/llvm-project/releases/download/llvmorg-11.0.0/llvm-11.0.0.src.tar.xz
tar xf llvm-11.0.0.src.tar.xz
cd llvm-11.0.0.src
mkdir build
cd build
cmake .. -DCMAKE_BUILD_TYPE=Release
cmake --build . -j
cmake --build . --target install
llvm-config --version
cd ../../

安装clang-11

wget https://github.com/llvm/llvm-project/releases/download/llvmorg-11.0.0/clang-11.0.0.src.tar.xz
tar xf clang-11.0.0.src.tar.xz
cd clang-11.0.0.src
mkdir build
cd build
cmake .. -DCMAKE_BUILD_TYPE=Release
make -j$(nproc)
make install     -j$(nproc)
cd ../../

安装boost-1.7.3

wget https://boostorg.jfrog.io/artifactory/main/release/1.73.0/source/boost_1_73_0.tar.gz
tar zxvf boost_1_73_0.tar.gz
cd boost_1_73_0
./bootstrap.sh --prefix=/usr/
./b2 -j$(nproc)
./b2 install -j$(nproc)
ldconfig
cd ../

安装protobuf

wget https://github.com/protocolbuffers/protobuf/releases/download/v3.13.0/protobuf-all-3.13.0.tar.gz
tar -zxvf protobuf-all-3.13.0.tar.gz
cd protobuf-3.13.0/
./autogen.sh
./configure
make -j$(nproc)
make install     -j$(nproc)
ldconfig # refresh shared library cache.

或者其他版本

wget https://github.com/protocolbuffers/protobuf/releases/download/v3.21.4/protobuf-all-3.21.4.tar.gz

安装 googletest

git clone https://github.com/google/googletest.git
cd googletest
git checkout release-1.10.0
mkdir build
cd build
cmake -DBUILD_GTEST=ON -DBUILD_GMOCK=ON -DINSTALL_GTEST=ON -DINSTALL_GMOCK=ON -DBUILD_SHARED_LIBS=ON ..
make -j$(nproc)
make install     -j$(nproc)
cd ../../

安装 google benchmark

wget https://github.com/google/benchmark/archive/refs/tags/v1.6.0.tar.gz
tar -zxvf  v1.6.0.tar.gz
cd benchmark-1.6.0
mkdir build
cd build
cmake -DBENCHMARK_DOWNLOAD_DEPENDENCIES=OFF -DBENCHMARK_ENABLE_GTEST_TESTS=OFF -DBENCHMARK_ENABLE_TESTING=OFF -DCMAKE_BUILD_TYPE=Release ../
make -j$(nproc)
make install     -j$(nproc)
cd ../../

Ubuntu

apt install

	apt install gcc -y
	apt install g++ -y
	apt install cmake -y
	apt install openssl -y
	apt install libssl-dev -y
	apt install libcurl4-openssl-dev -y
	apt install libboost-all-dev -y

install llvm-12

apt-get install llvm-12 clang-12 -y

install from source

install llvm-11

该安装方式，存在 segment fault， 建议采取以上 llvm-12 的安装方式。

https://github.com/llvm/llvm-project/releases/download/llvmorg-11.0.0/llvm-11.0.0.src.tar.xz

tar xf llvm-11.0.0.src.tar.xz
cd llvm-11.0.0.src
mkdir build
cd build
cmake .. -DCMAKE_BUILD_TYPE=Release
cmake --build . -j
cmake --build . --target install
llvm-config --version

boost

如果不想使用 apt install libboost-all-dev，也可以下载source 编译安装
wget https://boostorg.jfrog.io/artifactory/main/release/1.73.0/source/boost_1_73_0.tar.gz

protobuf

https://github.com/protocolbuffers/protobuf/releases/download/v3.13.0/protobuf-all-3.13.0.tar.gz

follow https://github.com/protocolbuffers/protobuf/blob/main/src/README.md

	tar -zxvf protobuf-all-3.13.0.tar.gz
	cd protobuf-3.13.0/
	./autogen.sh
	./configure
	make -j
	#make check
	make install
	ldconfig # refresh shared library cache.

make check 的目的是看看有没有error。可省略，输出如下

PASS: protobuf-test
PASS: protobuf-lazy-descriptor-test
PASS: protobuf-lite-test
PASS: google/protobuf/compiler/zip_output_unittest.sh
PASS: google/protobuf/io/gzip_stream_unittest.sh
PASS: protobuf-lite-arena-test
PASS: no-warning-test
============================================================================
Testsuite summary for Protocol Buffers 3.13.0
============================================================================
# TOTAL: 7
# PASS:  7
# SKIP:  0
# XFAIL: 0
# FAIL:  0
# XPASS: 0
# ERROR: 0
============================================================================
make[3]: Leaving directory '/home/shen/software/protobuf-3.13.0/src'
make[2]: Leaving directory '/home/shen/software/protobuf-3.13.0/src'
make[1]: Leaving directory '/home/shen/software/protobuf-3.13.0/src'

查看版本

# openssl version
OpenSSL 1.1.1f  31 Mar 2020

# lsb_release -a
No LSB modules are available.
Distributor ID: Ubuntu
Description:    Ubuntu 20.04.2 LTS
Release:        20.04
Codename:       focal

other setups for Gluten

install Velox dependencies

apt install libxml2-dev libkrb5-dev libgsasl7-dev  libuuid1  uuid-dev

# Get the Velox Source
git clone --recursive https://github.com/oap-project/velox.git
cd velox
# Running Ubuntu setup script to install all dependencies
./scripts/setup-ubuntu.sh

install 之后，执行 ldconfig

re2

apt remove libre2-dev
git clone https://code.googlesource.com/re2
cd re2
git checkout 2021-04-01
make
make test
make install
make testinstall

install 之后，执行 ldconfig

FQA

case

/bin/sh: Python3_EXECUTABLE-NOTFOUND: command not found

安装 python3， 然后重新cmake

case

An exception or error caused a run to abort: /tmp/spark_columnar_plugin_2660379366566816190/libspark_columnar_jni.so: libprotobuf.so.26: cannot open shared object file: No such file or directory 
java.lang.UnsatisfiedLinkError: /tmp/spark_columnar_plugin_2660379366566816190/libspark_columnar_jni.so: libprotobuf.so.26: cannot open shared object file: No such file or directory
	at java.lang.ClassLoader$NativeLibrary.load(Native Method)

不能直接从其他机器copy已经编译好的成勋运行。可以直接copy source code 在当前机器编译

Others

安装nasm

wget https://www.nasm.us/pub/nasm/releasebuilds/2.13/nasm-2.13.tar.gz --no-check-certificate
tar -zxvf nasm-2.13.tar.gz
cd nasm-2.13/
./configure
make -j
make install

安装 yasm

wget http://www.tortall.net/projects/yasm/releases/yasm-1.3.0.tar.gz
tar -zxvf yasm-1.3.0.tar.gz
cd yasm-1.3.0/
./configure
make -j
make install

CTYunOS

安装 gengetopt

yum install -y gperf help2man

#安装 makeinfo
yum install texinfo -y

wget https://mirrors.tuna.tsinghua.edu.cn/gnu/gengetopt/gengetopt-2.23.tar.xz
xz -d gengetopt-2.23.tar.xz
tar xvf gengetopt-2.23.tar
cd gengetopt-2.23
./configure
make -j
make install

安装 gsasl

#gsasl 库
git clone https://gitlab.com/gsasl/gsasl.git
#或
https://ftp.gnu.org/gnu/gsasl/

./bootstrap
./configure
make -j
make install