Centos 7 部署 OpenStack_Rocky版高可用集群3-1

Centos 7 部署 OpenStack_Rocky版高可用集群3-1

1、前期准备

1.1 软硬件准备

控制主机三台（三网卡）+计算节点三台（双网卡、兼分布式存储OSD）+存储节点一台（双网卡部署MOD）

1.1.1组件

组件	版本	备注
CentOS	CentOS Linux release 7.7.1908 (Core)
Openstack	Rocky
Ceph	ceph version 12.2.12 luminous (stable)

1.1.2 网络拓扑（逻辑）

1.控制节点(cont01、cont02、cont03)运行MariaDB galeria,Haproxy,pacemaker,corosync,keystone, glance,horizon,nova&neutron&cinder管理相关组件和openstack相关的基础服务

2.计算节点(comp01、comp02、comp03)运行Nova-compute,neutron-openswitch-agent,ceph-osd等

3.存储控制节点(mon01)运行ceph-mon,ceph-mgr(如生产环境，建议也采用三个节点部署)

4.控制节点网络：

​ 管理网络–>eth0:192.168.10.0/24 用于对openstack集群host os,api等网络管理

​ 租户隧道VM网络–>eth1:10.10.10.0/24 用于集群内部的guest os之间的数据交换，采用vxlan/gre等方式

​ 外部External网络–>eth2:主要用户计算节点访问internet/外部的floating ip

ps:此处的管理网络合并Management网络+API网络

5.计算节点网络：

​ 管理网络–>eth0:192.168.10.0/24 用于对openstack集群host os,api等网络管理，

​ 租户隧道网络–>eth1:10.10.10.0/24 用于集群内部的guest os之间的数据交换，采用vxlan/gre等方式

6.前端采用Haproxy做高可用，数据库采用MariaDB galeria，存储采用ceph分布式存储集群

7.无状态的服务，如xxx-api，采取active/active的模式运行；有状态的服务，如neturon-xxx-agent，cinder-volume等，建议采取active/passive的模式运行（因前端采用haproxy，客户端的多次请求可能会被转发到不同的控制节点，如果客户端请求被负载到无状态信息的控制节点，可能会导致操作请求失败）；自身具有集群机制的服务，如rabbitmq，memcached等采用本身的集群机制。

1.1.3 物理拓扑（部分参考）

1.2 配置hosts文件(集群所有电脑配置)

[root@cont02:/root]# vim /etc/hosts
192.168.10.21   cont01
192.168.10.22   cont02
192.168.10.23	  cont03
192.168.10.20	  VirtualIP
192.168.10.24   mon01
192.168.10.19   comp01
192.168.10.18	  comp02
192.168.10.17	  comp03
[root@cont02:/root]# scp /etc/hosts cont02:/etc/
[root@cont02:/root]# scp /etc/hosts cont03:/etc/
[root@cont02:/root]# scp /etc/hosts mon01:/etc/
[root@cont02:/root]# scp /etc/hosts comp01:/etc/
[root@cont02:/root]# scp /etc/hosts comp02:/etc/
[root@cont02:/root]# scp /etc/hosts comp03:/etc/

1.3 节点间设置免密互相登陆

[root@cont01:/root]# ssh-keygen
ssh-copy-id cont02
ssh-copy-id cont03
ssh-copy-id mon01
ssh-copy-id comp01
ssh-copy-id comp02
ssh-copy-id comp03
[root@cont02:/root]# ssh-keygen
ssh-copy-id cont01
ssh-copy-id cont03
ssh-copy-id mon01
ssh-copy-id comp01
ssh-copy-id comp02
ssh-copy-id comp03
[root@cont03:/root]# ssh-keygen
ssh-copy-id cont01
ssh-copy-id cont02
ssh-copy-id mon01
ssh-copy-id comp01
ssh-copy-id comp02
ssh-copy-id comp03

1.4 配置NTP服务

1.4.1 在控制节点进行配置（cont01、cont02、cont03）

[root@cont02:/root]# yum install system-config-date -y
[root@cont02:/root]# vim /etc/chrony.conf
  3 server 0.centos.pool.ntp.org iburst
  4 server 192.168.10.21    iburst
  5 server 192.168.10.22    iburst
  6 server 192.168.10.23    iburst
 25 # Allow NTP client access from local network.
 26 allow 192.168.0.0/16
[root@cont02:/root]# scp /etc/chrony.conf cont01:/etc/chrony.conf
[root@cont02:/root]# scp /etc/chrony.conf cont03:/etc/chrony.conf  
[root@cont01:/root]# systemctl enable chronyd && systemctl restart chronyd
[root@cont01:/root]# systemctl status chronyd
[root@cont02:/root]# systemctl enable chronyd && systemctl restart chronyd
[root@cont02:/root]# systemctl status chronyd
[root@cont03:/root]# systemctl enable chronyd && systemctl restart chronyd
[root@cont03:/root]# systemctl status chronyd
[root@cont01:/root]# chronyc sources
210 Number of sources = 3
MS Name/IP address         Stratum Poll Reach LastRx Last sample               
===============================================================================
^? cont02                        0   7     0     -     +0ns[   +0ns] +/-    0ns
^? cont03                        0   7     0     -     +0ns[   +0ns] +/-    0ns
^* 119.28.206.193                2   6    37    21   +197us[+1847us] +/-   55ms

[root@cont02:/root]# chronyc sources
210 Number of sources = 3
MS Name/IP address         Stratum Poll Reach LastRx Last sample               
===============================================================================
^? cont01                        0   7     0     -     +0ns[   +0ns] +/-    0ns
^? cont03                        0   7     0     -     +0ns[   +0ns] +/-    0ns
^* 162.159.200.123               3   6    37    51  +2273us[ +143us] +/-  121ms

[root@cont03:/root]# chronyc sources
210 Number of sources = 3
MS Name/IP address         Stratum Poll Reach LastRx Last sample               
===============================================================================
^? cont01                        0   8     0     -     +0ns[   +0ns] +/-    0ns
^? cont02                        0   8     0     -     +0ns[   +0ns] +/-    0ns
^? tick.ntp.infomaniak.ch        0   8     0     -     +0ns[   +0ns] +/-    0ns

[root@cont02:/root]# ssh cont01 'date'; ssh cont03 'date' ; date
Tue Feb 11 20:32:16 CST 2020
Tue Feb 11 20:32:16 CST 2020
Tue Feb 11 20:32:16 CST 2020

1.4.2 在其他节点上安装NTP软件（mon01、comp01、comp02、comp03）

[root@mon01:/root]# yum install chrony -y
[root@comp01:/root]# yum install chrony -y
[root@comp02:/root]# yum install chrony -y
[root@comp03:/root]# yum install chrony -y
[root@mon01:/root]# vim /etc/chrony.conf
  3 server 192.168.10.22    iburst
  4 server 192.168.10.21    iburst
  5 server 192.168.10.23    iburst
  6 #server 3.centos.pool.ntp.org iburst

[root@comp01:/root]# vim /etc/chrony.conf
  3 server 192.168.10.22    iburst
  4 server 192.168.10.21    iburst
  5 server 192.168.10.23    iburst
  6 #server 3.centos.pool.ntp.org iburst

[root@comp02:/root]# vim /etc/chrony.conf
  3 server 192.168.10.22    iburst
  4 server 192.168.10.21    iburst
  5 server 192.168.10.23    iburst
  6 #server 3.centos.pool.ntp.org iburst
  
  [root@comp03:/root]# vim /etc/chrony.conf
  3 server 192.168.10.22    iburst
  4 server 192.168.10.21    iburst
  5 server 192.168.10.23    iburst
  6 #server 3.centos.pool.ntp.org iburst

[root@mon01:/root]# systemctl enable chronyd && systemctl restart chronyd
[root@mon01:/root]# systemctl status chronyd
[root@comp01:/root]# systemctl enable chronyd && systemctl restart chronyd
[root@comp01:/root]# systemctl status chronyd
[root@comp02:/root]# systemctl enable chronyd && systemctl restart chronyd
[root@comp02:/root]# systemctl status chronyd
[root@comp03:/root]# systemctl enable chronyd && systemctl restart chronyd
[root@comp03:/root]# systemctl status chronyd

[root@mon01:/root]# chronyc sources
210 Number of sources = 3
MS Name/IP address         Stratum Poll Reach LastRx Last sample               
===============================================================================
^? cont02                        0   7     0     -     +0ns[   +0ns] +/-    0ns
^? cont01                        0   7     0     -     +0ns[   +0ns] +/-    0ns
^? cont03                        0   7     0     -     +0ns[   +0ns] +/-    0ns
[root@mon01:/root]# timedatectl
      Local time: Wed 2020-01-15 11:22:23 CST
  Universal time: Wed 2020-01-15 03:22:23 UTC
        RTC time: Wed 2020-01-15 03:22:24
       Time zone: Asia/Shanghai (CST, +0800)
     NTP enabled: yes
NTP synchronized: no
 RTC in local TZ: no
      DST active: n/a
[root@comp01:/root]# chronyc sources
210 Number of sources = 3
MS Name/IP address         Stratum Poll Reach LastRx Last sample               
===============================================================================
^? cont02                        0   7     0     -     +0ns[   +0ns] +/-    0ns
^? cont01                        0   7     0     -     +0ns[   +0ns] +/-    0ns
^? cont03                        0   7     0     -     +0ns[   +0ns] +/-    0ns
[root@comp01:/root]# timedatectl
      Local time: Wed 2020-01-15 11:22:24 CST
  Universal time: Wed 2020-01-15 03:22:24 UTC
        RTC time: Wed 2020-01-15 03:22:24
       Time zone: Asia/Shanghai (CST, +0800)
     NTP enabled: yes
NTP synchronized: no
 RTC in local TZ: no
      DST active: n/a
[root@comp02:/root]# chronyc sources
210 Number of sources = 3
MS Name/IP address         Stratum Poll Reach LastRx Last sample               
===============================================================================
^? cont02                        0   7     0     -     +0ns[   +0ns] +/-    0ns
^? cont01                        0   7     0     -     +0ns[   +0ns] +/-    0ns
^? cont03                        0   7     0     -     +0ns[   +0ns] +/-    0ns
[root@comp02:/root]# timedatectl
      Local time: Wed 2020-01-15 11:22:24 CST
  Universal time: Wed 2020-01-15 03:22:24 UTC
        RTC time: Wed 2020-01-15 03:22:24
       Time zone: Asia/Shanghai (CST, +0800)
     NTP enabled: yes
NTP synchronized: no
 RTC in local TZ: no
      DST active: n/a

1.5 系统初始设置

1.5.1 所有节点关闭防火墙和selinux

systemctl stop firewalld && systemctl disable firewalld
setenforce 0
sed -i 's/SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config

1.5.2 所有节点编辑limits.conf和sysctl.conf文件

cat >>/etc/security/limits.conf<<eof
   * soft nofile 65536
   * hard nofile 65536
eof
   
cat >>/etc/sysctl.conf<<eof   
   fs.file-max=655350
   net.ipv4.ip_local_port_range = 1025 65000
   net.ipv4.tcp_tw_recycle = 1
   net.ipv4.ip_nonlocal_bind = 1
eof
   
sysctl -p

2、部署MariaDB Galeria集群

2.1 控制节点安装mariadb(cont01、cont02、cont03)

Galera是Galera Cluster，是一种为数据库设计的新型的、数据不共享的、高度冗余的高可用方案，galera mariadb就是集成了Galera插件的MariaDB集群，Galera本身是具有多主特性的，所以galera mariadb不是传统的主备模式的集群，而是多主节点架构。

2.1.1 控制节点添加mariadb-10.4.11的yum源

[root@cont01:/root]# vim /etc/yum.repos.d/mariadb.repo
[mariadb]
name = MariaDB
baseurl = http://mirror.netinch.com/pub/mariadb//mariadb-10.4.11/yum/centos7-amd64/
gpgcheck = 0
enabled = 1
[root@cont01:/root]# yum makecache
[root@cont02:/root]# 
cat > /etc/yum.repos.d/mariadb.repo << EOF
[mariadb]
name = MariaDB
baseurl = http://mirror.netinch.com/pub/mariadb//mariadb-10.4.11/yum/centos7-amd64/
gpgcheck = 0
enabled = 1
EOF
[root@cont02:/root]# yum makecache
[root@cont02:/root]# scp /etc/yum.repos.d/mariadb.repo cont03:/etc/yum.repos.d/
[root@cont03:/root]# yum makecache

2.1.2 控制节点yum安装mariadb软件包并开启服务

[root@cont01:/root]# yum -y install MariaDB-server MariaDB-client galera-4 rsync xinetd wget ntpdate
[root@cont01:/root]# systemctl enable mariadb && systemctl start mariadb
[root@cont01:/root]# systemctl status mariadb
[root@cont02:/root]# yum -y install MariaDB-server MariaDB-client galera-4 rsync xinetd wget ntpdate
[root@cont02:/root]# systemctl enable mariadb && systemctl start mariadb
[root@cont02:/root]# systemctl status mariadb
[root@cont03:/root]# yum -y install MariaDB-server MariaDB-client galera-4 rsync xinetd wget ntpdate
[root@cont03:/root]# systemctl enable mariadb && systemctl start mariadb
[root@cont03:/root]# systemctl status mariadb

2.1.3 控制节点数据库安全设置、登陆数据库设置密码并授权

[root@cont02:/root]# mysql_secure_installation
注：Disallow root login remotely? [Y/n] n
密码：typora#2019
[root@cont02:/root]# mysql
MariaDB [(none)]> SET password=PASSWORD('typora#2019');
//密码设置过的可以省略
Query OK, 0 rows affected (0.002 sec)

MariaDB [(none)]> grant all privileges on *.* to 'root'@'%' identified by 'typora#2019' with grant option;
Query OK, 0 rows affected (0.002 sec)

MariaDB [(none)]> grant all privileges on *.* to 'mysql'@'%' identified by 'typora#2019' with grant option;
Query OK, 0 rows affected (0.008 sec)

MariaDB [(none)]> grant all privileges on *.* to 'root'@'localhost' identified by 'typora#2019' with grant option;
Query OK, 0 rows affected (0.002 sec)

MariaDB [(none)]> grant all privileges on *.* to 'mysql'@'localhost' identified by 'typora#2019' with grant option;
Query OK, 0 rows affected (0.008 sec)

MariaDB [(none)]> flush privileges;
Query OK, 0 rows affected (0.001 sec)

MariaDB [(none)]> exit
Bye


[root@cont01:/root]# mysql_secure_installation
注：Disallow root login remotely? [Y/n] n
密码：typora#2019
[root@cont01:/root]# mysql
MariaDB [(none)]> SET password=PASSWORD('typora#2019');
//密码设置过的可以省略
Query OK, 0 rows affected (0.012 sec)

MariaDB [(none)]> grant all privileges on *.* to 'root'@'%' identified by 'typora#2019' with grant option;
Query OK, 0 rows affected (0.008 sec)

MariaDB [(none)]> grant all privileges on *.* to 'mysql'@'%' identified by 'typora#2019' with grant option;
Query OK, 0 rows affected (0.008 sec)

MariaDB [(none)]> grant all privileges on *.* to 'root'@'localhost' identified by 'typora#2019' with grant option;
Query OK, 0 rows affected (0.002 sec)

MariaDB [(none)]> grant all privileges on *.* to 'mysql'@'localhost' identified by 'typora#2019' with grant option;
Query OK, 0 rows affected (0.008 sec)

MariaDB [(none)]> flush privileges;
Query OK, 0 rows affected (0.001 sec)

MariaDB [(none)]> exit
Bye

[root@cont03:/root]# mysql_secure_installation
注：Disallow root login remotely? [Y/n] n
密码：typora#2019
[root@cont03:/root]#  mysql
MariaDB [(none)]>  SET password=PASSWORD('typora#2019');
//密码设置过的可以省略
Query OK, 0 rows affected (0.009 sec)

MariaDB [(none)]> grant all privileges on *.* to 'root'@'localhost' identified by 'typora#2019' with grant option;
Query OK, 0 rows affected (0.002 sec)

MariaDB [(none)]> grant all privileges on *.* to 'mysql'@'localhost' identified by 'typora#2019' with grant option;
Query OK, 0 rows affected (0.008 sec)

MariaDB [(none)]> grant all privileges on *.* to 'root'@'%' identified by 'typora#2019' with grant option;
Query OK, 0 rows affected (0.016 sec)

MariaDB [(none)]> grant all privileges on *.* to 'mysql'@'%' identified by 'typora#2019' with grant option;
Query OK, 0 rows affected (0.008 sec)

MariaDB [(none)]> flush privileges;
Query OK, 0 rows affected (0.001 sec)

MariaDB [(none)]>  exit
Bye

2.1.4 配置MariaDB Galeria 集群 (控制节点全修改配置文件/etc/my.cnf.d/server.cnf)

[root@cont02:/root]# vim /etc/my.cnf.d/server.cnf
[server]
# this is only for the mysqld standalone daemon
[mysqld]
#
# * Galera-related settings
#
[galera]
# Mandatory settings
##wsrep_on=ON   #开启全同步复制模式
wsrep_on=ON
wsrep_provider=/usr/lib64/galera-4/libgalera_smm.so
wsrep_cluster_address="gcomm://192.168.10.22,192.168.10.23,192.168.10.21"
binlog_format=row
default_storage_engine=InnoDB
innodb_autoinc_lock_mode=2
##innodb_autoinc_lock_mode=2   #主键自增模式修改为交叉模式
##wsrep_slave_threads=8  #开启并行复制线程，根据CPU核数设置
#
# Allow server to accept connections on all interfaces.
#
bind-address=192.168.10.22
#
# Optional setting
#wsrep_slave_threads=1
#innodb_flush_log_at_trx_commit=0   #事务提交每隔1秒刷盘 
##wsrep_certify_nonPK=ON #为没有显式申明主键的表生成一个用于certificationtest的主键，默认为ON
##query_cache_size=0      #关闭查询缓存 
##wsrep_provider_options="gcache.size=4G" #同步复制缓冲池
wsrep_provider_options="gcache.size=1G"
wsrep_cluster_name=MariaDB-Galera-Cluster
wsrep_node_name=cont02
wsrep_node_address=192.168.10.22
wsrep_sst_method=rsync
##innodb_buffer_pool_size=2G
# this is only for embedded server
[embedded]
# This group is only read by MariaDB servers, not by MySQL.
# If you use the same .cnf file for MySQL and MariaDB,
# you can put MariaDB-only options here
[mariadb]
# This group is only read by MariaDB-10.4 servers.
# If you use the same .cnf file for MariaDB of different versions,
# use this group for options that older servers don't understand
[mariadb-10.4]

[root@cont01:/root]# vim /etc/my.cnf.d/server.cnf
[server]
# this is only for the mysqld standalone daemon
[mysqld]
#
# * Galera-related settings
#
[galera]
# Mandatory settings
wsrep_on=ON
wsrep_provider=/usr/lib64/galera-4/libgalera_smm.so
wsrep_cluster_address="gcomm://192.168.10.21,192.168.10.22,192.168.10.23"
binlog_format=row
default_storage_engine=InnoDB
innodb_autoinc_lock_mode=2
#
# Allow server to accept connections on all interfaces.
#
bind-address=192.168.10.21
#
# Optional setting
#wsrep_slave_threads=1
#innodb_flush_log_at_trx_commit=0
wsrep_provider_options="gcache.size=1G"
wsrep_cluster_name=MariaDB-Galera-Cluster
wsrep_node_name=cont01
wsrep_node_address=192.168.10.21
wsrep_sst_method=rsync
# this is only for embedded server
[embedded]
# This group is only read by MariaDB servers, not by MySQL.
# If you use the same .cnf file for MySQL and MariaDB,
# you can put MariaDB-only options here
[mariadb]
# This group is only read by MariaDB-10.4 servers.
# If you use the same .cnf file for MariaDB of different versions,
# use this group for options that older servers don't understand
[mariadb-10.4]

[root@cont03:/root]# vim /etc/my.cnf.d/server.cnf
[server]

# this is only for the mysqld standalone daemon
[mysqld]

#
# * Galera-related settings
#
[galera]
# Mandatory settings
wsrep_on=ON
wsrep_provider=/usr/lib64/galera-4/libgalera_smm.so
wsrep_cluster_address="gcomm://192.168.10.23,192.168.10.22,192.168.10.21"
binlog_format=row
default_storage_engine=InnoDB
innodb_autoinc_lock_mode=2
#
# Allow server to accept connections on all interfaces.
#
bind-address=192.168.10.23
#
# Optional setting
#wsrep_slave_threads=1
#innodb_flush_log_at_trx_commit=0
wsrep_provider_options="gcache.size=1G"
wsrep_cluster_name=MariaDB-Galera-Cluster
wsrep_node_name=cont03
wsrep_node_address=192.168.10.23
wsrep_sst_method=rsync
# this is only for embedded server
[embedded]

# This group is only read by MariaDB servers, not by MySQL.
# If you use the same .cnf file for MySQL and MariaDB,
# you can put MariaDB-only options here
[mariadb]

# This group is only read by MariaDB-10.4 servers.
# If you use the same .cnf file for MariaDB of different versions,
# use this group for options that older servers don't understand
[mariadb-10.4]

2.1.5 控制节点停止mariadb.service

[root@cont01:/root]# systemctl daemon-reload
[root@cont01:/root]# systemctl stop mariadb.service
[root@cont02:/root]# systemctl daemon-reload
[root@cont02:/root]# systemctl stop mariadb.service
[root@cont03:/root]# systemctl daemon-reload
[root@cont03:/root]# systemctl stop mariadb.service

2.1.6 启动初始化集群

[root@cont02:/root]