TBase 安装

说明

参考 https://github.com/Tencent/TBase/wiki/1%E3%80%81TBase_Quick_Start
对 pgxc_ctl.conf 进行了修改，dnslave节点正常，gtmslave已启动，但是monitor显示not run。

机器配置

CPU	MEM	SPACE	IP	SYSTEM	Service
4U	8G	500G	172.19.129.112	CentOS7.8	GTMMaster/CN01/DN01Master/DN02Slaver
4U	8G	500G	172.19.129.113	CentOS7.8	GTMSlaver/CN02/DN01Slaver/DN02Master

节点名称	IP	数据目录
GTM主机	172.19.129.112	/Data/tbase/data/gtm
GTM从机	172.19.129.113	/Data/tbase/data/gtm
CN1	172.19.129.112	/data/tbase/data/coord
CN2	172.19.129.113	/data/tbase/data/coord
DN1主	172.19.129.112	/Data/tbase/data/dn001
DN1从	172.19.129.113	/Data/tbase/data/dn001
DN2主	172.19.129.113	/Data/tbase/data/dn002
DN2从	172.19.129.112	/Data/tbase/data/dn002

环境部署

系统优化(所有机器)

echo "export PS1='[\u@\h \t \w]\$ '" >> /etc/profile && source /etc/profile

# 优化OS rancher版本
cat > /etc/sysctl.conf << EOF
net.bridge.bridge-nf-call-ip6tables=1
net.bridge.bridge-nf-call-iptables=1
net.ipv4.ip_forward=1
net.ipv4.conf.all.forwarding=1
net.ipv4.neigh.default.gc_thresh1=4096
net.ipv4.neigh.default.gc_thresh2=6144
net.ipv4.neigh.default.gc_thresh3=8192
net.ipv4.neigh.default.gc_interval=60
net.ipv4.neigh.default.gc_stale_time=120

# 参考 https://github.com/prometheus/node_exporter#disabled-by-default
kernel.perf_event_paranoid=-1

#sysctls for k8s node config
net.ipv4.tcp_slow_start_after_idle=0
net.core.rmem_max=16777216
fs.inotify.max_user_watches=524288
kernel.softlockup_all_cpu_backtrace=1

kernel.softlockup_panic=0

kernel.watchdog_thresh=30
fs.file-max=2097152
fs.inotify.max_user_instances=8192
fs.inotify.max_queued_events=16384
vm.max_map_count=262144
fs.may_detach_mounts=1
net.core.netdev_max_backlog=16384
net.ipv4.tcp_wmem=4096 12582912 16777216
net.core.wmem_max=16777216
net.core.somaxconn=32768
net.ipv4.ip_forward=1
net.ipv4.tcp_max_syn_backlog=8096
net.ipv4.tcp_rmem=4096 12582912 16777216

net.ipv6.conf.all.disable_ipv6=1
net.ipv6.conf.default.disable_ipv6=1
net.ipv6.conf.lo.disable_ipv6=1

kernel.yama.ptrace_scope=0
vm.swappiness=0

# 可以控制core文件的文件名中是否添加pid作为扩展。
kernel.core_uses_pid=1

# Do not accept source routing
net.ipv4.conf.default.accept_source_route=0
net.ipv4.conf.all.accept_source_route=0

# Promote secondary addresses when the primary address is removed
net.ipv4.conf.default.promote_secondaries=1
net.ipv4.conf.all.promote_secondaries=1

# Enable hard and soft link protection
fs.protected_hardlinks=1
fs.protected_symlinks=1

# 源路由验证
# see details in https://help.aliyun.com/knowledge_detail/39428.html
net.ipv4.conf.all.rp_filter=0
net.ipv4.conf.default.rp_filter=0
net.ipv4.conf.default.arp_announce = 2
net.ipv4.conf.lo.arp_announce=2
net.ipv4.conf.all.arp_announce=2

# see details in https://help.aliyun.com/knowledge_detail/41334.html
net.ipv4.tcp_max_tw_buckets=5000
net.ipv4.tcp_syncookies=1
net.ipv4.tcp_fin_timeout=30
net.ipv4.tcp_synack_retries=2
kernel.sysrq=1
EOF

cat > /etc/security/limits.conf << EOF
root soft nofile 65535
root hard nofile 65535
* soft nofile 65535
* hard nofile 65535
EOF
sysctl -p

# 关闭selinux
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
setenforce 0
# 关闭防火墙
systemctl stop firewalld.service && systemctl disable firewalld.service
# 修改时区
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
# 关闭swap
swapoff -a && sysctl -w vm.swappiness=0 && sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
# 重启
reboot

安装依赖

yum install -y git readline-devel uuid zlib zlib-devel openssl-devel bison  gcc gcc-g++ make flex flex-devel uuid-devel

编译

编译后的文件保存，用于之后分发。

获取源码

# 当前为基础路径
git clone https://github.com/Tencent/TBase

环境变量

export SOURCECODE_PATH=$PWD/TBase
export INSTALL_PATH=$PWD/install

源码编译

cd ${SOURCECODE_PATH}
rm -rf ${INSTALL_PATH}/tbase_bin_v2.0
chmod +x configure*
./configure --prefix=${INSTALL_PATH}/tbase_bin_v2.0  --enable-user-switch --with-openssl  --with-ossp-uuid CFLAGS=-g
make clean
make -sj
make install
chmod +x contrib/pgxc_ctl/make_signature
cd contrib
make -sj
make install

配置数据服务器环境

创建tbase用户 (2台tbase服务器上执行)

mkdir /data && useradd -d /data/tbase tbase
# 设置密码(大于8位)pentakill
passwd tbase
su tbase
cat >> ~/.bashrc << 'EOF'
export TBASE_HOME=~/install/tbase_bin_v2.0
export PATH="$PATH:$TBASE_HOME/bin"
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$TBASE_HOME/lib"
EOF
source ~/.bashrc

免密登录

# 112
su tbase
ssh-keygen -t rsa
ssh-copy-id -i ~/.ssh/id_rsa.pub tbase@172.19.129.113
# 113
su tbase
ssh-keygen -t rsa
ssh-copy-id -i ~/.ssh/id_rsa.pub tbase@172.19.129.112

安装 （OP）

tbase环境配置

# 当前为基础路径
export TBASE_HOME=$PWD/install/tbase_bin_v2.0
export PATH="$PATH:$TBASE_HOME/bin"
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$TBASE_HOME/lib"
# op  方便分发
ssh-keygen -t rsa
ssh-copy-id -i ~/.ssh/id_rsa.pub tbase@172.19.129.113
ssh-copy-id -i ~/.ssh/id_rsa.pub tbase@172.19.129.112

配置文件

vi ./pgxc_ctl/pgxc_ctl.conf

#!/bin/bash
m1=172.19.129.112
m2=172.19.129.113
pgxcInstallDir=./install/tbase_bin_v2.0
pgxcOwner=tbase
defaultDatabase=postgres
pgxcUser=$pgxcOwner
tmpDir=/tmp
localTmpDir=$tmpDir
configBackup=n
configBackupHost=pgxc-linker
configBackupDir=$HOME/pgxc
configBackupFile=pgxc_ctl.bak


#---- GTM ----------
gtmName=gtm
gtmMasterServer=$m1
gtmMasterPort=50001
gtmMasterDir=/data/tbase/data/gtm
gtmExtraConfig=none
gtmMasterSpecificExtraConfig=none
gtmSlave=y
# 新增,否则slave配置中为null
gtmSlaveName=gtm
gtmSlaveServer=$m2
gtmSlavePort=50001
gtmSlaveDir=/data/tbase/data/gtm
gtmSlaveSpecificExtraConfig=none

#---- Coordinators -------
coordMasterDir=/data/tbase/data/coord
coordMasterDir=/data/tbase/data/coord
coordArchLogDir=/data/tbase/data/coord_archlog

coordNames=(cn001 cn002 )
coordPorts=(30004 30004 )
poolerPorts=(31110 31110 )
coordPgHbaEntries=(0.0.0.0/0)
coordMasterServers=($m1 $m2)
coordMasterDirs=($coordMasterDir $coordMasterDir)
coordMaxWALsernder=2
coordMaxWALSenders=($coordMaxWALsernder $coordMaxWALsernder )
coordSlave=n
coordSlaveSync=n
coordArchLogDirs=($coordArchLogDir $coordArchLogDir)

coordExtraConfig=coordExtraConfig
cat > $coordExtraConfig <<EOF
#================================================
# Added to all the coordinator postgresql.conf
# Original: $coordExtraConfig
# 此处适应性修改后 dn slave可正常启动
listen_addresses = '0.0.0.0'
max_connections = 500
max_pool_size = 65535
shared_buffers = 1GB
max_prepared_transactions = 2000
maintenance_work_mem = 256MB
wal_level = logical
max_wal_senders = 64
max_wal_size = 1GB
min_wal_size = 256MB
log_destination = 'csvlog'
logging_collector = on
log_directory = 'log'
log_filename = 'postgresql-%A-%H.log'
synchronous_commit = local
synchronous_standby_names = ''

EOF

coordSpecificExtraConfig=(none none)
coordExtraPgHba=coordExtraPgHba
cat > $coordExtraPgHba <<EOF
local   all             all                                     trust
host    all             all             0.0.0.0/0               trust
host    replication     all             0.0.0.0/0               trust
host    all             all             ::1/128                 trust
host    replication     all             ::1/128                 trust
host    replication    all    $m1/32    trust 
host    all            all    $m1/32    trust 
host    replication    all    $m2/32    trust 
host    all            all    $m2/32    trust
host    all            all       0.0.0.0/0        md5

EOF

coordSpecificExtraPgHba=(none none)
coordAdditionalSlaves=n	
cad1_Sync=n

#---- Datanodes ---------------------
dn1MstrDir=/data/tbase/data/dn001
dn2MstrDir=/data/tbase/data/dn002
dn1SlvDir=/data/tbase/data/dn001
dn2SlvDir=/data/tbase/data/dn002
dn1ALDir=/data/tbase/data/datanode_archlog
dn2ALDir=/data/tbase/data/datanode_archlog

primaryDatanode=dn001
datanodeNames=(dn001 dn002)
datanodePorts=(40004 40004)
datanodePoolerPorts=(41110 41110)
datanodePgHbaEntries=(0.0.0.0/0)
datanodeMasterServers=($m1 $m2)
datanodeMasterDirs=($dn1MstrDir $dn2MstrDir)
dnWALSndr=4
datanodeMaxWALSenders=($dnWALSndr $dnWALSndr)

datanodeSlave=y
datanodeSlaveServers=($m2 $m1)
datanodeSlavePorts=(50004 50004)
datanodeSlavePoolerPorts=(51110 51110)
datanodeSlaveSync=n
datanodeSlaveDirs=($dn1SlvDir $dn2SlvDir)
datanodeArchLogDirs=($dn1ALDir/dn001 $dn2ALDir/dn002)

datanodeExtraConfig=datanodeExtraConfig
cat > $datanodeExtraConfig <<EOF
#================================================
# Added to all the coordinator postgresql.conf
# Original: $datanodeExtraConfig
# 此处适应性修改后 dn slave可正常启动
listen_addresses = '0.0.0.0'
max_connections = 500
max_pool_size = 65535
shared_buffers = 1GB
max_prepared_transactions = 2000
maintenance_work_mem = 256MB
wal_level = logical
max_wal_senders = 64
max_wal_size = 1GB
min_wal_size = 256MB
log_destination = 'csvlog'
logging_collector = on
log_directory = 'log'
log_filename = 'postgresql-%A-%H.log'
synchronous_commit = local
synchronous_standby_names = ''

EOF

datanodeSpecificExtraConfig=(none none)
datanodeExtraPgHba=datanodeExtraPgHba
cat > $datanodeExtraPgHba <<EOF

local   all             all                                     trust
host    all             all             0.0.0.0/0               trust
host    replication     all             0.0.0.0/0               trust
host    all             all             ::1/128                 trust
host    replication     all             ::1/128                 trust
host    replication    all    $m1/32    trust 
host    all            all    $m1/32    trust 
host    replication    all    $m2/32    trust 
host    all            all    $m2/32    trust
host    all            all       0.0.0.0/0        md5

EOF

datanodeSpecificExtraPgHba=(none none)

datanodeAdditionalSlaves=n
walArchive=n

操作

# 指定基础目录，指定配置文件
pgxc_ctl --home $PWD -c ./pgxc_ctl/pgxc_ctl.conf
# 二进制分发
> deploy all
# 集群初始化
> init all
# 显示配置
> show config all
# 启停服务
> stop all -m fast
> start all
# 监控集群
> monitor all
# 清除集群：用于重新部署
> clean all

集群访问

psql -h 172.19.129.112 -p 30004 -d postgres -U tbase

创建default group以及sharding

TBase使用datanode group来增加节点的管理灵活度，要求有一个default group才能使用；

TBase数据分布为了增加灵活度，加了中间逻辑层来维护数据记录到物理节点的映射，叫sharding，要预先创建

create default node group default_group  with (dn001,dn002);
create sharding group to group default_group;

使用

与操作 postgres 数据库相同

问题