mysql集群从节点无法启动_一次galera cluster集群故障节点无法启动问题排查

现象

环境：

Server version: 10.0.25-MariaDB-wsrep MariaDB Server, wsrep_25.13.raf7f02e

配置文件：

[root@node-23 mariadb]# more /etc/my.cnf

[mysqld]

server_id=3

bind_address = node-23

port = 3306

datadir=/var/lib/mysql

log-error=/var/log/mariadb/mariadb-error.log

collation-server = utf8_general_ci

init-connect = 'SET NAMES utf8'

character-set-server = utf8

skip-name-resolve

default-storage-engine = innodb

innodb_autoinc_lock_mode=2

binlog_format = ROW

# LOGGING #

log-queries-not-using-indexes = 0

slow-query-log = 0

slow-query-log-file = /var/log/mariadb/mariadb-slow.log

log_error = /var/log/mariadb/mariadb-error.log

log-bin = /var/lib/mysql/mariadb-bin

log-bin-index = /var/lib/mysql/mariadb-bin.index

expire-logs-days = 7

log_slave_updates = 1

# SAFETY #

max-allowed-packet = 16M

max-connect-errors = 100

max_connections = 10000

wait_timeout = 3600

# CACHES AND LIMITS #

tmp-table-size = 32M

max-heap-table-size = 32M

query-cache-type = 0

query-cache-size = 0M

thread-cache-size = 50

open-files-limit = 65535

#table-definition-cache = 4096

table-open-cache = 1024

# INNODB #

innodb-flush-method = O_DIRECT

#innodb-log-file-size = 10240M

innodb-flush-log-at-trx-commit = 2

innodb-file-per-table = 1

#innodb-buffer-pool-size = 4096M

# Depending on number of cores and disk sub

innodb-read-io-threads = 4

innodb-write-io-threads = 4

innodb-doublewrite = 0

#innodb-log-buffer-size = 128M

innodb-buffer-pool-instances = 8

innodb-log-files-in-group = 2

innodb-thread-concurrency = 64

# avoid statistics update when doing e.g show tables

innodb_stats_on_metadata = 0

wsrep_provider=/usr/lib64/galera/libgalera_smm.so

# wsrep_provider_options="pc.ignore_quorum = true; pc.ignore_sb=false; gmcast.listen_addr=tcp://node-23:4567;gcs.fc_limit = 256; gcs.fc_factor = 0.99; gcs.fc_master_sl

ave=yes"

wsrep_cluster_address=gcomm://node-22,node-21,node-23

wsrep_cluster_name="openstack-controller"

wsrep_node_address="node-23"

wsrep_node_name="mysql-galera-node-23"

#wsrep_sst_method=rsync

wsrep_sst_method=xtrabackup-v2

wsrep_sst_auth=sst_user:XXXXXXXXXX

wsrep_slave_threads=4

# to enable debug level logging, set this to 1

wsrep_debug = 0

# how many times to retry deadlocked autocommits

wsrep_retry_autocommit = 3

[xtrabackup]

compress

compact

parallel = 4

compress-threads = 4

rebuild-threads = 4

[mysqldump]

quick

quote-names

max_allowed_packet = 16M

三节点的galera cluster集群，有两个节点down了，时间较长，重启后都不能加入集群了。

报错节点启动日志：

正常节点日志：

思路

1、日志没发现什么具体的报错信息

2、怀疑网络不通，发现没有

3、怀疑是不是日志不足所有导致初始化有问题，但是目前就是SSt全量的方式

4、查看主节点innobackup.backup.log日志也没有具体信息

解决

备份方式改为rsync模式，启动后改为xtrabackup-v2，重新启动容器