openGauss_3.0 一主一从一级联部署

一、环境介绍

网络主机名
node1 192.168.18.166
node2 192.168.18.167
node3 192.168.18.168

下载地址：https://opengauss.org/zh/download/archive/

二、安装前准备

1.修改hosts

echo "192.168.18.166 node1 " >>/etc/hosts
echo "192.168.18.167 node2 " >>/etc/hosts
echo "192.168.18.168 node3 " >>/etc/hosts

2.创建目录修改环境变量

mkdir -p /app/software/openGauss
chmod 775 -R /app/software/openGauss

3.关闭防火墙

systemctl stop firewalld.service
systemctl disable firewalld.service

4.关闭SELinux

sed -i '/^SELINUX=/d' /etc/selinux/config
echo "SELINUX=disabled" >> /etc/selinux/config

5.设置操作系统字符集编码

echo "LANG=en_US.UTF-8" >> /etc/profile
source /etc/profile
echo $LANG

6.设置操作系统时区

rm -fr /etc/localtime
ln -s /usr/share/zoneinfo/Asia/Shanghai  /etc/localtime
date -R
hwclock

7.关闭SWAP分区

sed -i '/swap/s/^/#/' etc/fstab
swapoff -a
free -m

8.配置SSH服务，关闭Banner，允许root远程登录

sed -i '/Banner/s/^/#/' /etc/ssh/sshd_config
sed -i '/PermitRootLogin/s/^/#/'  /etc/ssh/sshd_config
echo -e "\n" >> /etc/ssh/sshd_config
echo "Banner none " >> /etc/ssh/sshd_config
echo "PermitRootLogin yes" >> /etc/ssh/sshd_config
cat /etc/ssh/sshd_config |grep -v ^#|grep -E 'PermitRoot|Banner'

9.配置 sysctl.conf 和 performance.sh

cat >> /etc/sysctl.conf << EOF
net.ipv4.tcp_retries1 = 5
net.ipv4.tcp_syn_retries = 5
net.sctp.path_max_retrans = 10
net.sctp.max_init_retransmits = 10
EOF
sysctl -p

10.配置资源限制

echo "* soft stack 3072" >> /etc/security/limits.conf
echo "* hard stack 3072" >> /etc/security/limits.conf
echo "* soft nofile 1000000" >> /etc/security/limits.conf
echo "* hard nofile 1000000" >> /etc/security/limits.conf
echo "* soft nproc unlimited" >> /etc/security/limits.d/90-nproc.conf
tail -n 4 /etc/security/limits.conf
tail -n 1 /etc/security/limits.d/90-nproc.conf

11.关闭透明大页[Only for CentOS]

cat >>/etc/rc.d/rc.local<<EOF
if test -f /sys/kernel/mm/transparent_hugepage/enabled; then
   echo never > /sys/kernel/mm/transparent_hugepage/enabled
fi
if test -f /sys/kernel/mm/transparent_hugepage/defrag; then
   echo never > /sys/kernel/mm/transparent_hugepage/defrag
fi
EOF
chmod +x /etc/rc.d/rc.local /usr/bin/sh /etc/rc.d/rc.local
cat /sys/kernel/mm/transparent_hugepage/enabled
cat /sys/kernel/mm/transparent_hugepage/defrag

12.yum源配置和安装依赖

cd /etc/yum.repos.d/
[root@node1 yum.repos.d]# mkdir -p  /etc/yum.repos.d/yum_bak
[root@node1 yum.repos.d]# mv /etc/yum.repos.d//*.repo /etc/yum.repos.d/yum_bak

echo "[local]" >> /etc/yum.repos.d/local.repo
echo "name=centos7-yum" >> /etc/yum.repos.d/local.repo
echo "baseurl=file:///mnt" >> /etc/yum.repos.d/local.repo
echo "enabled=1"  >> /etc/yum.repos.d/local.repo
echo "gpgcheck=0" >> /etc/yum.repos.d/local.repo
echo "gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7"  >> /etc/yum.repos.d/local.repo


mount /dev/cdrom /mnt

yum clean all
yum makecache
yum install -y libaio-devel libnsl flex bison ncurses-devel glibc-devel patch readline-devel net-tools tar  expect

13.安装Python3

#安装依赖
yum -y install zlib-devel bzip2-devel openssl-devel ncurses-devel sqlite-devel readline-devel tk-devel gcc make

#解压安装包
cd /soft
tar -zxf Python-3.6.8.tgz

#配置，需要加上--enable-shared
cd Python-3.6.8
 ./configure  --enable-shared CFLAGS=-fPIC

#安装
make && make install
python3 --version
Python 3.6.8


#修改命令路径
 rm -rf /usr/bin/python
 ln -s /usr/local/bin/python3 /usr/bin/python
 python --version
Python 3.6.8

#python3会造成yum报错，下面方法可以恢复
 rm -rf /usr/bin/python
ln -s /usr/bin/python2 /usr/bin/python 

14.创建对应目录

mkdir -p /app/opengauss/install
mkdir -p /app/opengauss/{log,tmp,corefile}
mkdir -p /app/opengauss/log/omm

node1
mkdir -p /app/opengauss/install/data/d1
chmod 775 -R /app/opengauss/install/data/d1
node2:
mkdir -p /app/opengauss/install/data/d2
chmod 775 -R /app/opengauss/install/data/d1
node3
mkdir -p /app/opengauss/install/data/d3
chmod 775 -R /app/opengauss/install/data/d1

三、安装步骤

1.解压安装包

cluster_config.xml  openGauss-3.0.5-CentOS-64bit-all.tar.gz
 tar -zxf openGauss-3.0.5-CentOS-64bit-all.tar.gz 
[root@node1 openGauss]# ll
total 205868
-rw-r--r-- 1 root root      2582 Oct 23 17:36 cluster_config.xml
-rw-r--r-- 1 root root 104970830 Oct 23 17:41 openGauss-3.0.5-CentOS-64bit-all.tar.gz
-rw-r--r-- 1 root root       105 Sep 23 09:39 openGauss-3.0.5-CentOS-64bit-cm.sha256
-rw-r--r-- 1 root root   6651856 Sep 23 09:39 openGauss-3.0.5-CentOS-64bit-cm.tar.gz
-rw-r--r-- 1 root root        65 Sep 23 09:38 openGauss-3.0.5-CentOS-64bit-om.sha256
-rw-r--r-- 1 root root  11928462 Sep 23 09:38 openGauss-3.0.5-CentOS-64bit-om.tar.gz
-rw-r--r-- 1 root root        65 Sep 23 09:39 openGauss-3.0.5-CentOS-64bit.sha256
-rw-r--r-- 1 root root  86779055 Sep 23 09:39 openGauss-3.0.5-CentOS-64bit.tar.bz2
-rw------- 1 root root        65 Sep 23 09:35 upgrade_sql.sha256
-rw------- 1 root root    447618 Sep 23 09:35 upgrade_sql.tar.gz
[root@node1 openGauss]# tar -zxf openGauss-3.0.5-CentOS-64bit-om.tar.gz 

2.编辑配置文件

[root@node1 openGauss]# more cluster_config.xml 
<?xml version="1.0" encoding="UTF-8"?>
<ROOT>
    <!-- openGauss整体信息 -->
    <CLUSTER>
        <PARAM name="clusterName" value="Cluster_openGauss" />
        <PARAM name="nodeNames" value="node1,node2,node3" />

        <PARAM name="gaussdbAppPath" value="/app/opengauss/install/app" />
        <PARAM name="gaussdbLogPath" value="/app/opengauss/log/omm" />
        <PARAM name="tmpMppdbPath" value="/app/opengauss/tmp"/>
        <PARAM name="gaussdbToolPath" value="/app/opengauss/install/om" />
        <PARAM name="corePath" value="/app/opengauss/corefile"/>
        <PARAM name="backIp1s" value="192.168.18.166,192.168.18.167,192.168.18.168"/>
    
    </CLUSTER>
    <!-- 每台服务器上的节点部署信息 -->
    <DEVICELIST>
        <!-- node1上的节点部署信息 -->
        <DEVICE sn="node1">
            <PARAM name="name" value="node1"/>
            <PARAM name="azName" value="AZ1"/>
            <PARAM name="azPriority" value="1"/>
            <!-- 如果服务器只有一个网卡可用，将backIP1和sshIP1配置成同一个IP -->
            <PARAM name="backIp1" value="192.168.18.166"/>
            <PARAM name="sshIp1" value="192.168.18.166"/>
    
      <!--dn-->
            <PARAM name="dataNum" value="1"/>
      <PARAM name="dataPortBase" value="15400"/>
      <PARAM name="dataNode1" value="/app/opengauss/install/data/d1,node2,/app/opengauss/install/data/d2,node3,/app/opengauss/install/data/d3"/>
            <PARAM name="dataNode1_syncNum" value="0"/>
        </DEVICE>
    
        <!-- node2上的节点部署信息，其中“name”的值配置为主机名称 -->
        <DEVICE sn="node2">
            <PARAM name="name" value="node2"/>
            <PARAM name="azName" value="AZ1"/>
            <PARAM name="azPriority" value="1"/>
            <!-- 如果服务器只有一个网卡可用，将backIP1和sshIP1配置成同一个IP -->
            <PARAM name="backIp1" value="192.168.18.167"/>
            <PARAM name="sshIp1" value="192.168.18.167"/>

  </DEVICE>

        <!-- node3上的节点部署信息，其中“name”的值配置为主机名称 -->
        <DEVICE sn="node3">
            <PARAM name="name" value="node3"/>
            <PARAM name="azName" value="AZ1"/>
            <PARAM name="azPriority" value="1"/>
            <!-- 如果服务器只有一个网卡可用，将backIP1和sshIP1配置成同一个IP -->
            <PARAM name="backIp1" value="192.168.18.168"/>
            <PARAM name="sshIp1" value="192.168.18.168"/>
            <PARAM name="cascadeRole" value="on"/>

  </DEVICE>
    </DEVICELIST>
</ROOT>

3.安装前配置

 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/

 cd /app/software/openGauss/script/
 ./gs_preinstall -U omm -G dbgrp -X /app/software/openGauss/cluster_config.xml

4.安装

su - omm

gs_install -X  /app/software/openGauss/cluster_config.xml --gsinit-parameter="--encoding=UTF8"  --dn-guc="max_process_memory=2GB" --dn-guc="shared_buffers=128MB" --dn-guc="cstore_buffers=16MB" 
[omm@node1 ~]$ gs_install -X  /app/software/openGauss/cluster_config.xml --gsinit-parameter="--encoding=UTF8"  --dn-guc="max_process_memory=2GB" --dn-guc="shared_buffers=128MB" --dn-guc="cstore_buffers=16MB" 
Parsing the configuration file.
Check preinstall on every node.
Successfully checked preinstall on every node.
Creating the backup directory.
Successfully created the backup directory.
begin deploy..
Installing the cluster.
begin prepare Install Cluster..
Checking the installation environment on all nodes.
begin install Cluster..
Installing applications on all nodes.
Successfully installed APP.
begin init Instance..
encrypt cipher and rand files for database.
Please enter password for database:
Please repeat for database:
[GAUSS-50322] : Failed to encrypt the password for databaseError:
 Try "gs_guc --help" for more information.
Invalid password,it must contain at least eight characters

Please enter password for database:       >>>>>>1qaz@WSX
Please repeat for database:
begin to create CA cert files
The sslcert will be generated in /app/opengauss/install/app/share/sslcert/om
NO cm_server instance, no need to create CA for CM.
Cluster installation is completed.
Configuring.
Deleting instances from all nodes.
Successfully deleted instances from all nodes.
Checking node configuration on all nodes.
Initializing instances on all nodes.
Updating instance configuration on all nodes.
Check consistence of memCheck and coresCheck on database nodes.
Successful check consistence of memCheck and coresCheck on all nodes.
Configuring pg_hba on all nodes.
Configuration is completed.
Successfully started cluster.
Successfully installed application.
end deploy.. 

四、验证

[omm@node1 ~]$ gs_om -t status
-----------------------------------------------------------------------

cluster_name    : Cluster_openGauss
cluster_state   : Normal
redistributing  : No

-----------------------------------------------------------------------

[omm@node1 ~]$ gs_om -t status --detail
[   Cluster State   ]

cluster_state   : Normal
redistributing  : No
current_az      : AZ_ALL

[  Datanode State   ]

    node node_ip         port      instance                               state

-----------------------------------------------------------------------------------------------

1  node1 192.168.18.166  15400      6001 /app/opengauss/install/data/d1   P Primary Normal
2  node2 192.168.18.167  15400      6002 /app/opengauss/install/data/d2   S Standby Normal
3  node3 192.168.18.168  15400      6003 /app/opengauss/install/data/d3   C Cascade Normal
[omm@node1 ~]$  

五、问题

1.问题1

问题现象：

[root@node1 script]# ./gs_preinstall -U omm -G dbgrp -X /app/software/openGauss/cluster_config.xml
Warnging version of python doesnot meet the expection, maybe third-party libs need to be compiled by yourself
Traceback (most recent call last):
  File "./gs_preinstall", line 38, in <module>
    from gspylib.common.Common import DefaultValue
  File "/app/software/openGauss/script/gspylib/common/Common.py", line 34, in <module>
    import secrets
ImportError: No module named 'secrets'

解决办法：

安装指南要求Python版本为3.6+

Python 版本3.5 版本升级到3.6.8 解决

2.问题2

问题现象：

[root@node1 script]# ./gs_preinstall -U omm -G dbgrp -X /app/software/openGauss/cluster_config.xml
Traceback (most recent call last):
  File "/app/software/openGauss/script/gspylib/common/Common.py", line 63, in <module>
    import psutil
  File "/app/software/openGauss/script/gspylib/common/../../../lib/psutil/__init__.py", line 99, in <module>
    from . import _pslinux as _psplatform
  File "/app/software/openGauss/script/gspylib/common/../../../lib/psutil/_pslinux.py", line 26, in <module>
    from . import _psutil_linux as cext
  File "/app/software/openGauss/script/gspylib/common/../../../lib/psutil/_psutil_linux.py", line 7, in <module>
    __bootstrap__()
  File "/app/software/openGauss/script/gspylib/common/../../../lib/psutil/_psutil_linux.py", line 6, in __bootstrap__
    imp.load_dynamic(__name__,__file__)
  File "/usr/local/lib/python3.6/imp.py", line 343, in load_dynamic
    return _load(spec)
ImportError: /app/software/openGauss/script/gspylib/common/../../../lib/psutil/_psutil_linux.so: cannot open shared object file: No such file or directory

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "./gs_preinstall", line 38, in <module>
    from gspylib.common.Common import DefaultValue
  File "/app/software/openGauss/script/gspylib/common/Common.py", line 97, in <module>
    import psutil
  File "/app/software/openGauss/script/gspylib/common/../../../lib/psutil/__init__.py", line 99, in <module>
    from . import _pslinux as _psplatform
  File "/app/software/openGauss/script/gspylib/common/../../../lib/psutil/_pslinux.py", line 26, in <module>
    from . import _psutil_linux as cext
ImportError: libpython3.6m.so.1.0: cannot open shared object file: No such file or directory

解决办法：

安装时未添加 --enable-shared参数，导致没有缺少一些C库，重新编译安装Python3
./configure  --enable-shared CFLAGS=-fPIC 
配置环境变量：
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/
将 libpython3.6m.so.1.0复制到/app/software/openGauss/script/gspylib/clib
cp /usr/local/lib/libpython3.6m.so.1.0 /app/software/openGauss/script/gspylib/clib/

3.问题4

问题现象：

[root@node1 script]#  ./gs_preinstall -U omm -G dbgrp -X /app/software/openGauss/cluster_config.xml
Parsing the configuration file.
Successfully parsed the configuration file.
Installing the tools on the local node.
Successfully installed the tools on the local node.
Are you sure you want to create trust for root (yes/no)?yes
Please enter password for root
[GAUSS-51632] : Failed to do gs_sshexkey.Error: python3: error while loading shared libraries: libpython3.6m.so.1.0: cannot open shared object file: No such file or directory

解决办法：

原因和问题3一样，缺少C库

[root@node1 script]# cd gspylib/
[root@node1 gspylib]# ls
clib  common  component  etc  hardware  __init__.py  inspection  os  pssh  __pycache__  threads
[root@node1 gspylib]# cd clib/
[root@node1 clib]# ls
encrypt  libcrypto.so.1.1  libssl.so.1.1  libstdc++.so.6
[root@node1 clib]# cp  /usr/local/lib/libpython3.6m.so.1.0 ./
[root@node1 clib]# ll
total 37052
-rwxr-xr-x 1 root root    85088 Sep 23 09:33 encrypt
-rw-r--r-- 1 root root 11215104 Sep 23 09:38 libcrypto.so.1.1
-r-xr-xr-x 1 root root 12388048 Oct 23 22:51 libpython3.6m.so.1.0
-rw-r--r-- 1 root root  2706048 Sep 23 09:38 libssl.so.1.1
-rwx------ 1 root root 11535640 Sep 23 09:38 libstdc++.so.6
[root@node1 clib]# 

5.问题5

问题现象：

Successfully distribute package to package path.
[FAILURE] node2:
[GAUSS-51251] : The /app/opengauss/install/app cannot be a root user group or a link.
[FAILURE] node3:
[GAUSS-51251] : The /app/opengauss/install/app cannot be a root user group or a link.

解决办法：

创建非root用户和组，赋权
官方指南不需要提前创建/app/opengauss/install下的子目录，
groupadd dbgrp
useradd omm -g dbgrp
chown omm:dbgrp /app/opengauss/install/app
chown omm:dbgrp /app/opengauss/tmp
chown omm:dbgrp /app/opengauss/install/data/d2
chown omm:dbgrp /app/opengauss/install/data/d3

6.问题6

问题现象：

[FAILURE] node1:
[GAUSS-51248] : The hostname(node2) may be not same with hostname(/etc/hostname) Command: "pssh -s -H node2 'cat /etc/hostname'". Error: 

[FAILURE] node2:
[GAUSS-51248] : The hostname(node2) may be not same with hostname(/etc/hostname) Command: "pssh -s -H node2 'cat /etc/hostname'". Error: 

[FAILURE] node3:
[GAUSS-51248] : The hostname(node2) may be not same with hostname(/etc/hostname) Command: "pssh -s -H node2 'cat /etc/hostname'". Error: 

解决办法：

节点2从其他节点复制而来，/etc/hostname不存在，创建该文件，加入主机名解决
[root@node2 install]# vi /etc/hostname 
node2
~

7.问题7

问题现象：

[FAILURE] node1:
[GAUSS-50200] : The /app/opengauss/install/app already exists. Please remove it. It should be a symbolic link to $GAUSSHOME if it exists
[FAILURE] node2:
[GAUSS-50200] : The /app/opengauss/install/app already exists. Please remove it. It should be a symbolic link to $GAUSSHOME if it exists
[FAILURE] node3:
[GAUSS-50200] : The /app/opengauss/install/app already exists. Please remove it. It should be a symbolic link to $GAUSSHOME if it exists

解决办法

不应该提前创建/app/opengauss/install/app该目录，预安装程序会自动创建，3个节点删除该路径

8.问题8

问题现象：

[FAILURE] node1:
[GAUSS-50202] : The /app/opengauss/install must be empty. Or user [omm] has write permission to directory /app/opengauss/install. Because it will create symbolic link [/app/opengauss/install/app] to install path [/app/opengauss/install/app_b54d05de] in gs_install process with this user.
[FAILURE] node2:
[GAUSS-50202] : The /app/opengauss/install must be empty. Or user [omm] has write permission to directory /app/opengauss/install. Because it will create symbolic link [/app/opengauss/install/app] to install path [/app/opengauss/install/app_b54d05de] in gs_install process with this user.
[SUCCESS] node3:

解决办法

对安装目录赋权
chmod 775 -R   /app/opengauss/install

六、参考

https://opengauss.org/zh/blogs/July/openGauss%E4%B8%80%E4%B8%BB%E4%B8%80%E5%A4%87%E4%B8%80%E7%BA%A7%E5%AE%89%E8%A3%85.html