RedHat系列系统 Clickhouse v18 源码编译安装

需要GCC7.3以上版本支持。

下载源码

git clone -b v18.12.17-stable https://github.com/ClickHouse/ClickHouse.git

下载依赖

git依赖

cd Clickhouse
sed -i 's/github.com/github.com.cnpmjs.org/g' .gitmodules	# 更改下载地址，提高速度，如果连接github速度快可不改
git submodule sync											# 将新的URL更新到.git/config
git submodule update --init --recursive						# 循环下载依赖

包依赖

yum -y install lz4-devel openssl-devel zlib-devel zstd-devel protobuf-devel libicu-devel readline-devel gperf

编译

禁用unwind

不禁用unwind时会报错

vi ClickHouse/libs/libdaemon/cmake/find_unwind.cmake

注释：

option (ENABLE_UNWIND "Enable libunwind (better stacktraces)" ON)

添加：

option (ENABLE_UNWIND "Enable libunwind (better stacktraces)" OFF)

生成Makefile

mkdir build && cd build/
cmake3 .. -DHAVE_SSE41=0 -DHAVE_SSE42=0 -DHAVE_POPCNT=0 -DHAVE_PCLMULQDQ_INTRIN=0 -DPCLMULFLAG=' ' -DENABLE_TCMALLOC=OFF -DENABLE_JEMALLOC=OFF -DCMAKE_INSTALL_PREFIX=/opt/clickhouse

编译

注:16核make正常耗时大概四五十分钟

export CC=gcc
export CXX=g++
make -j16

错误&解决

问题1

问题描述

./ClickHouse/contrib/librdkafka/src/crc32c.c: In function ‘crc32c_global_init’:
./ClickHouse/contrib/librdkafka/src/crc32c.c:349:9: error: unknown register name ‘%edx’ in ‘asm’

解决方案

cd ./ClickHouse/contrib/librdkafka
./configure --clean
./configure --build=mips64el-linux-gnuabi64

问题2

问题描述

/root/tmpv2/ClickHouse/dbms/src/Columns/ColumnVector.cpp: In instantiation of ‘DB::ColumnPtr DB::ColumnVector<T>::index(const DB::IColumn&, size_t) const [with T = unsigned char; DB::ColumnPtr = COWPtr<DB::IColumn>::immutable_ptr<DB::IColumn>; size_t = long unsigned int]’:
/root/tmpv2/ClickHouse/dbms/src/Columns/ColumnVector.cpp:322:16: required from here
/root/tmpv2/ClickHouse/dbms/src/Columns/ColumnVector.cpp:239:27: error: ‘selectIndexImpl’ was not declared in this scope
return selectIndexImpl(*this, indexes, limit);
~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~
/root/tmpv2/ClickHouse/dbms/src/Columns/ColumnVector.cpp:239:27: note: suggested alternative: ‘indexImpl’
return selectIndexImpl(*this, indexes, limit);
~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~

解决方案

vi ClickHouse/dbms/src/Columns/ColumnVector.cpp

#if __SSE2__
    #include <emmintrin.h>
#include <Columns/ColumnsCommon.h>

#endif

改为：

#include <Columns/ColumnsCommon.h>
#if __SSE2__
    #include <emmintrin.h>
#endif

问题3

问题描述

/root/tmpv2/ClickHouse/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp: In member function ‘void DB::MergeTreeRangeReader::executePrewhereActionsAndFilterColumns(DB::MergeTreeRangeReader::ReadResult&)’:
/root/tmpv2/ClickHouse/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp:634:84: error: ‘DataTypeNothing’ was not declared in this scope
result.block.insert({ColumnNothing::create(getNumRows()), std::make_shared<DataTypeNothing>(), "_nothing"});

解决方案

vi ClickHouse/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp

#if __SSE2__
#include <emmintrin.h>
#include <DataTypes/DataTypeNothing.h>
#endif

改为：

#include <DataTypes/DataTypeNothing.h>
#if __SSE2__
#include <emmintrin.h>
#endif

问题4

问题描述

/usr/local/lib/librdkafka.a(rdkafka_lz4.o): In functionrd_kafka_lz4_decompress':
/home/ilan/librdkafka-master/src/rdkafka_lz4.c:184: undefined reference to LZ4F_createDecompressionContext' /home/ilan/librdkafka-master/src/rdkafka_lz4.c:185: undefined reference to LZ4F_isError'

或

/usr/bin/ld: ../../contrib/librdkafka-cmake/librdkafka.a(rdkafka_conf.c.o):(.data.rel.ro+0x6100): undefined reference to `rd_kafka_plugins_conf_set'
/usr/bin/ld: ../../contrib/librdkafka-cmake/librdkafka.a(rdkafka_sasl.c.o): in function `rd_kafka_sasl_select_provider':
/root/build_clickhouse/ClickHouse/contrib/librdkafka/src/rdkafka_sasl.c:268: undefined reference to `rd_kafka_sasl_scram_provider'
collect2: 错误：ld 返回 1

解决方案

vi ClickHouse/contrib/librdkafka/Makefile.config

三处地方更改

ENABLE_LZ4_EXT= y
LIBS+= -lssl -lcrypto -llz4 -lcrypto -lz -ldl -lpthread -lrt
WITH_LZ4_EXT= y

改为：

ENABLE_LZ4_EXT= n
LIBS+= -lssl -lcrypto -lcrypto -lz -ldl -lpthread -lrt
WITH_LZ4_EXT= n

单独编译librdkafka并将静态库文件拷贝到build目录下

cd ClickHouse/contrib/librdkafka/
make -j8
cp ClickHouse/contrib/librdkafka/src/librdkafka.a ClickHouse/build/contrib/librdkafka-cmake/librdkafka.a

问题5

问题描述

/home/build_mod/build_clickhouse/ClickHouse/dbms/src/IO/LZ4_decompress_faster.cpp:212:103: error: ‘copyOverlap8Shuffle’ was not declared in this scope
 template <> void inline copyOverlap<8, true>(UInt8 * op, const UInt8 *& match, const size_t offset) { copyOverlap8Shuffle(op, match, offset); }
                                                                                                       ^~~~~~~~~~~~~~~~~~~
/home/build_mod/build_clickhouse/ClickHouse/dbms/src/IO/LZ4_decompress_faster.cpp:212:103: note: suggested alternative: ‘copyOverlap8’
 template <> void inline copyOverlap<8, true>(UInt8 * op, const UInt8 *& match, const size_t offset) { copyOverlap8Shuffle(op, match, offset); }
                                                                                                       ^~~~~~~~~~~~~~~~~~~
                                                                                                       copyOverlap8

解决方案

vi /home/build_mod/build_clickhouse/ClickHouse/dbms/src/IO/LZ4_decompress_faster.cpp

template <> void inline copyOverlap<8, true>(UInt8 * op, const UInt8 *& match, const size_t offset) { copyOverlap8Shuffle(op, match, offset); }

改为

template <> void inline copyOverlap<8, true>(UInt8 * op, const UInt8 *& match, const size_t offset) { copyOverlap8(op, match, offset); }

问题6

问题描述

In file included from /root/build_clickhouse/ClickHouse/utils/test-data-generator/main.cpp:9:0:
/root/build_clickhouse/ClickHouse/dbms/src/Common/HashTable/Hash.h: 在函数‘DB::UInt64 intHashCRC32(DB::UInt64)’中:
/root/build_clickhouse/ClickHouse/dbms/src/Common/HashTable/Hash.h:52:30: 错误：大整数隐式截断为无符号类型 [-Werror=overflow]
     return __crc32cd(-1ULL, x);

解决方案

vi /root/build_clickhouse/ClickHouse/dbms/src/Common/HashTable/Hash.h

注释以下三行代码

#if __aarch64__
//#include <arm_acle.h>
#include <arm_neon.h>
#endif

inline DB::UInt64 intHashCRC32(DB::UInt64 x)
{
#if __SSE4_2__
    return _mm_crc32_u64(-1ULL, x);
//#elif __aarch64__
//    return __crc32cd(-1ULL, x);

问题7

问题描述

In file included from /root/build_clickhouse/ClickHouse/dbms/src/Common/HashTable/Hash.h:43:0,
                 from /root/build_clickhouse/ClickHouse/dbms/src/Interpreters/AggregationCommon.h:8,
                 from /root/build_clickhouse/ClickHouse/dbms/src/Common/tests/parallel_aggregation2.cpp:8:
/usr/lib/gcc/aarch64-linux-gnu/7.3.0/include/arm_acle.h:65:1: 错误：inlining failed in call to always_inline ‘uint32_t __crc32cd(uint32_t, uint64_t)’: target specific option mismatch
 __crc32cd (uint32_t __a, uint64_t __b)

解决方案

cd /root/build_clickhouse/ClickHouse/dbms/
sed -i 's/CXX_FLAGS =/CXX_FLAGS = -march=armv8-a+crc+simd/g' `find ./ -name flags.make`
sed -i "s/-Werror//g" `find ./ -name flags.make`
sed -i "s/-Werror//g" `find ./ -name link.txt`

cd /root/build_clickhouse/ClickHouse/utils/
sed -i 's/CXX_FLAGS =/CXX_FLAGS = -march=armv8-a+crc+simd/g' `find ./ -name flags.make`
sed -i "s/-Werror//g" `find ./ -name flags.make`
sed -i "s/-Werror//g" `find ./ -name link.txt`

问题8

问题描述

In file included from /data_old/build_mod/build_clickhouse/ClickHouse/build/dbms/src/Functions/generated/registerFunctionBitRotateLeft.cpp:4:
/data_old/build_mod/build_clickhouse/ClickHouse/dbms/src/Functions/FunctionsArithmetic.h: In static member function ‘static void DB::BinaryOperationImplBase<A, B, Op, ResultType_>::vector_constant(DB::PaddedPODArray<T>&, B, DB::PaddedPODArray<ResultType_>&) [with A = unsigned char; B = long unsigned int; Op = DB::BitRotateLeftImpl<unsigned char, long unsigned int>; ResultType_ = long unsigned int]’:
/data_old/build_mod/build_clickhouse/ClickHouse/dbms/src/Functions/FunctionsArithmetic.h:73:13: internal compiler error: in expand_shift_1, at expmed.c:2631
             c[i] = Op::template apply<ResultType>(a[i], b);

解决方案

vi /root/build_clickhouse/ClickHouse/dbms/src/Functions/FunctionsArithmetic.h

调整，添加四行，注释两行

static void NO_INLINE vector_constant(const PaddedPODArray<A> & a, B b, PaddedPODArray<ResultType> & c)
{
    size_t size = a.size();
    // added
    PaddedPODArray<B> bb(size, B{});
    for(size_t i = 0; i < size; ++i)
        bb[i] = b;
    vector_vector(a, bb, c);
    // noted
    //for (size_t i = 0; i < size; ++i)
    //  c[i] = Op::template apply<ResultType>(a[i], bb[0]);
}

安装

make install

配置

添加环境变量

vi /etc/profile

添加：

# add for clickhouse 
export PATH=/opt/clickhouse/bin:$PATH
export LD_LIBRARY_PATH=/opt/clickhouse/lib:$LD_LIBRARY_PATH

source /etc/profile

创建用户和用户组

groupadd clickhouse
groupmod -g 498 clickhouse
useradd -g clickhouse clickhouse -s /sbin/nologin
usermod -d /var/lib/clickhouse -u 498 -c "Clickhouse Server" clickhouse

配置文件与目录

mkdir -p /data/clickhouse
mkdir -p /etc/clickhouse-server
mkdir -p /var/log/clickhouse-server
mkdir -p /var/lib/clickhouse
mkdir -p /var/run/clickhouse-server
cp config.xml /etc/clickhouse-server/
cp users.xml /etc/clickhouse-server/
cp clickhouse-server_initd /etc/init.d/
cp clickhouse-server_crond /etc/cron.d/
cp clickhouse-server.service /etc/systemd/system/

配置文件权限

chown -R clickhouse:clickhouse /data/clickhouse
chown -R clickhouse:clickhouse /var/log/clickhouse-server
chown -R clickhouse:clickhouse /var/lib/clickhouse
chmod 750 /var/lib/clickhouse
chmod 755 /var/run/clickhouse-server
chmod 755 /data/clickhouse
chmod 755 -R /etc/clickhouse-server
chmod 777 /etc/init.d/clickhouse-server_initd
chmod 777 /etc/cron.d/clickhouse-server_crond
chmod 777 /etc/systemd/system/clickhouse-server.service

# 添加到系统服务,系统需要用systemctl，就要执行这个
chkconfig --add clickhouse-server

确认安装成功

#启动
service clickhouse-server start
service clickhouse-server status
#安装成功
[root@localhost ~]# clickhouse client
ClickHouse client version 18.12.17.
Connecting to localhost:9000.
Connected to ClickHouse server version 18.12.17 revision 54407.

localhost :)

出处：
https://blog.csdn.net/Reddragon2010/article/details/118159422?spm=1001.2014.3001.5501