greenplum数据库无法启动 error: Do not have enough valid segments to start the array.

         greenplum数据库异常无法启动，启动报错error: Do not have enough valid segments to start the array.查看数据库日志报错FATAL","XX000","Global sequence number 20930983 less than maximum value 20930998 found in scan ('gp_persistent_relation_node') (cdbpersistentstore.c:600)"

启动报错信息如下：

20190327:00:29:14:014215 gpstart:mas:gpadmin-[INFO]:-Commencing parallel primary and mirror segment instance startup, please wait............
20190327:00:31:17:014215 gpstart:mas:gpadmin-[INFO]:-Process results...
20190327:00:31:17:014215 gpstart:mas:gpadmin-[ERROR]:-No segment started for content: 0.
20190327:00:31:17:014215 gpstart:mas:gpadmin-[INFO]:-dumping success segments: ['seg15:/data1/pg_system/mirror/gpseg53:content=53:dbid=135:mode=s:status=u', 'seg15:/data1/pg_system/mirror/gpseg52:content=52:dbid=134:mode=s:status=u', 'seg15:/data2/pg_system/primary/gpseg58:content=58:dbid=60:mode=s:status=u', 'seg15:/data2/pg_system/primary/gpseg59:content=59:dbid=61:mode=s:status=u', 'seg15:/data1/pg_system/primary/gpseg57:content=57:dbid=59:mode=s:status=u', 'seg15:/data1/pg_system/primary/gpseg56:content=56:dbid=58:mode=s:status=u', 'seg15:/data2/pg_system/mirror/gpseg54:content=54:dbid=136:mode=s:status=u', 'seg15:/data2/pg_system/mirror/gpseg55:content=55:dbid=137:mode=s:status=u', 'seg09:/data1/pg_system/mirror/gpseg28:content=28:dbid=110:mode=s:status=u', 'seg09:/data1/pg_system/mirror/gpseg29:content=29:dbid=111:mode=s:status=u', 'seg09:/data2/pg_system/mirror/gpseg30:content=30:dbid=112:mode=s:status=u', 'seg09:/data2/pg_system/mirror/gpseg31:content=31:dbid=113:mode=s:status=u', 'seg09:/data1/pg_system/primary/gpseg33:content=33:dbid=35:mode=s:status=u', 'seg09:/data1/pg_system/primary/gpseg32:content=32:dbid=34:mode=s:status=u', 'seg09:/data2/pg_system/primary/gpseg34:content=34:dbid=36:mode=s:status=u', 'seg09:/data2/pg_system/primary/gpseg35:content=35:dbid=37:mode=s:status=u', 'seg14:/data1/pg_system/mirror/gpseg48:content=48:dbid=130:mode=s:status=u', 'seg14:/data1/pg_system/mirror/gpseg49:content=49:dbid=131:mode=s:status=u', 'seg14:/data2/pg_system/primary/gpseg54:content=54:dbid=56:mode=s:status=u', 'seg14:/data2/pg_system/primary/gpseg55:content=55:dbid=57:mode=s:status=u', 'seg14:/data1/pg_system/primary/gpseg53:content=53:dbid=55:mode=s:status=u', 'seg14:/data1/pg_system/primary/gpseg52:content=52:dbid=54:mode=s:status=u', 'seg14:/data2/pg_system/mirror/gpseg50:content=50:dbid=132:mode=s:status=u', 'seg14:/data2/pg_system/mirror/gpseg51:content=51:dbid=133:mode=s:status=u', 'seg08:/data2/pg_system/mirror/gpseg27:content=27:dbid=109:mode=s:status=u', 'seg08:/data2/pg_system/mirror/gpseg26:content=26:dbid=108:mode=s:status=u', 'seg08:/data1/pg_system/mirror/gpseg24:content=24:dbid=106:mode=s:status=u', 'seg08:/data1/pg_system/mirror/gpseg25:content=25:dbid=107:mode=s:status=u', 'seg08:/data2/pg_system/primary/gpseg30:content=30:dbid=32:mode=s:status=u', 'seg08:/data2/pg_system/primary/gpseg31:content=31:dbid=33:mode=s:status=u', 'seg08:/data1/pg_system/primary/gpseg28:content=28:dbid=30:mode=s:status=u', 'seg08:/data1/pg_system/primary/gpseg29:content=29:dbid=31:mode=s:status=u', 'seg10:/data2/pg_system/mirror/gpseg34:content=34:dbid=116:mode=s:status=u', 'seg10:/data2/pg_system/mirror/gpseg35:content=35:dbid=117:mode=s:status=u', 'seg10:/data1/pg_system/primary/gpseg37:content=37:dbid=39:mode=s:status=u', 'seg10:/data1/pg_system/primary/gpseg36:content=36:dbid=38:mode=s:status=u', 'seg10:/data1/pg_system/mirror/gpseg33:content=33:dbid=115:mode=s:status=u', 'seg10:/data1/pg_system/mirror/gpseg32:content=32:dbid=114:mode=s:status=u', 'seg10:/data2/pg_system/primary/gpseg38:content=38:dbid=40:mode=s:status=u', 'seg10:/data2/pg_system/primary/gpseg39:content=39:dbid=41:mode=s:status=u', 'seg18:/data1/pg_system/primary/gpseg68:content=68:dbid=70:mode=s:status=u', 'seg18:/data1/pg_system/primary/gpseg69:content=69:dbid=71:mode=s:status=u', 'seg18:/data2/pg_system/mirror/gpseg67:content=67:dbid=149:mode=s:status=u', 'seg18:/data2/pg_system/mirror/gpseg66:content=66:dbid=148:mode=s:status=u', 'seg18:/data1/pg_system/mirror/gpseg64:content=64:dbid=146:mode=s:status=u', 'seg18:/data1/pg_system/mirror/gpseg65:content=65:dbid=147:mode=s:status=u', 'seg18:/data2/pg_system/primary/gpseg70:content=70:dbid=72:mode=s:status=u', 'seg18:/data2/pg_system/primary/gpseg71:content=71:dbid=73:mode=s:status=u', 'seg19:/data2/pg_system/mirror/gpseg70:content=70:dbid=152:mode=s:status=u', 'seg19:/data2/pg_system/mirror/gpseg71:content=71:dbid=153:mode=s:status=u', 'seg19:/data1/pg_system/primary/gpseg73:content=73:dbid=75:mode=s:status=u', 'seg19:/data1/pg_system/primary/gpseg72:content=72:dbid=74:mode=s:status=u', 'seg19:/data1/pg_system/mirror/gpseg68:content=68:dbid=150:mode=s:status=u', 'seg19:/data1/pg_system/mirror/gpseg69:content=69:dbid=151:mode=s:status=u', 'seg19:/data2/pg_system/primary/gpseg74:content=74:dbid=76:mode=s:status=u', 'seg19:/data2/pg_system/primary/gpseg75:content=75:dbid=77:mode=s:status=u', 'seg16:/data1/pg_system/primary/gpseg60:content=60:dbid=62:mode=s:status=u', 'seg16:/data1/pg_system/primary/gpseg61:content=61:dbid=63:mode=s:status=u', 'seg16:/data1/pg_system/mirror/gpseg57:content=57:dbid=139:mode=s:status=u', 'seg16:/data1/pg_system/mirror/gpseg56:content=56:dbid=138:mode=s:status=u', 'seg16:/data2/pg_system/mirror/gpseg58:content=58:dbid=140:mode=s:status=u', 'seg16:/data2/pg_system/mirror/gpseg59:content=59:dbid=141:mode=s:status=u', 'seg16:/data2/pg_system/primary/gpseg63:content=63:dbid=65:mode=s:status=u', 'seg16:/data2/pg_system/primary/gpseg62:content=62:dbid=64:mode=s:status=u', 'seg02:/data1/pg_system/primary/gpseg5:content=5:dbid=7:mode=s:status=u', 'seg02:/data1/pg_system/primary/gpseg4:content=4:dbid=6:mode=s:status=u', 'seg02:/data1/pg_system/mirror/gpseg1:content=1:dbid=83:mode=s:status=u', 'seg02:/data2/pg_system/primary/gpseg6:content=6:dbid=8:mode=s:status=u', 'seg02:/data2/pg_system/primary/gpseg7:content=7:dbid=9:mode=s:status=u', 'seg02:/data2/pg_system/mirror/gpseg2:content=2:dbid=84:mode=s:status=u', 'seg02:/data2/pg_system/mirror/gpseg3:content=3:dbid=85:mode=s:status=u', 'seg04:/data2/pg_system/mirror/gpseg10:content=10:dbid=92:mode=s:status=u', 'seg04:/data2/pg_system/mirror/gpseg11:content=11:dbid=93:mode=s:status=u', 'seg04:/data2/pg_system/primary/gpseg14:content=14:dbid=16:mode=s:status=u', 'seg04:/data2/pg_system/primary/gpseg15:content=15:dbid=17:mode=s:status=u', 'seg04:/data1/pg_system/mirror/gpseg9:content=9:dbid=91:mode=s:status=u', 'seg04:/data1/pg_system/mirror/gpseg8:content=8:dbid=90:mode=s:status=u', 'seg04:/data1/pg_system/primary/gpseg13:content=13:dbid=15:mode=s:status=u', 'seg04:/data1/pg_system/primary/gpseg12:content=12:dbid=14:mode=s:status=u', 'seg11:/data2/pg_system/mirror/gpseg38:content=38:dbid=120:mode=s:status=u', 'seg11:/data2/pg_system/mirror/gpseg39:content=39:dbid=121:mode=s:status=u', 'seg11:/data2/pg_system/primary/gpseg43:content=43:dbid=45:mode=s:status=u', 'seg11:/data2/pg_system/primary/gpseg42:content=42:dbid=44:mode=s:status=u', 'seg11:/data1/pg_system/primary/gpseg40:content=40:dbid=42:mode=s:status=u', 'seg11:/data1/pg_system/primary/gpseg41:content=41:dbid=43:mode=s:status=u', 'seg11:/data1/pg_system/mirror/gpseg37:content=37:dbid=119:mode=s:status=u', 'seg11:/data1/pg_system/mirror/gpseg36:content=36:dbid=118:mode=s:status=u', 'seg13:/data2/pg_system/primary/gpseg50:content=50:dbid=52:mode=s:status=u', 'seg13:/data2/pg_system/primary/gpseg51:content=51:dbid=53:mode=s:status=u', 'seg13:/data1/pg_system/primary/gpseg48:content=48:dbid=50:mode=s:status=u', 'seg13:/data1/pg_system/primary/gpseg49:content=49:dbid=51:mode=s:status=u', 'seg13:/data1/pg_system/mirror/gpseg44:content=44:dbid=126:mode=s:status=u', 'seg13:/data1/pg_system/mirror/gpseg45:content=45:dbid=127:mode=s:status=u', 'seg13:/data2/pg_system/mirror/gpseg47:content=47:dbid=129:mode=s:status=u', 'seg13:/data2/pg_system/mirror/gpseg46:content=46:dbid=128:mode=s:status=u', 'seg01:/data1/pg_system/mirror/gpseg77:content=77:dbid=159:mode=s:status=u', 'seg01:/data1/pg_system/mirror/gpseg76:content=76:dbid=158:mode=s:status=u', 'seg01:/data2/pg_system/mirror/gpseg78:content=78:dbid=160:mode=s:status=u', 'seg01:/data2/pg_system/mirror/gpseg79:content=79:dbid=161:mode=s:status=u', 'seg01:/data1/pg_system/primary/gpseg1:content=1:dbid=3:mode=s:status=u', 'seg01:/data2/pg_system/primary/gpseg2:content=2:dbid=4:mode=s:status=u', 'seg01:/data2/pg_system/primary/gpseg3:content=3:dbid=5:mode=s:status=u', 'seg17:/data2/pg_system/mirror/gpseg63:content=63:dbid=145:mode=s:status=u', 'seg17:/data2/pg_system/mirror/gpseg62:content=62:dbid=144:mode=s:status=u', 'seg17:/data1/pg_system/mirror/gpseg60:content=60:dbid=142:mode=s:status=u', 'seg17:/data1/pg_system/mirror/gpseg61:content=61:dbid=143:mode=s:status=u', 'seg17:/data1/pg_system/primary/gpseg64:content=64:dbid=66:mode=s:status=u', 'seg17:/data1/pg_system/primary/gpseg65:content=65:dbid=67:mode=s:status=u', 'seg17:/data2/pg_system/primary/gpseg67:content=67:dbid=69:mode=s:status=u', 'seg17:/data2/pg_system/primary/gpseg66:content=66:dbid=68:mode=s:status=u', 'seg03:/data1/pg_system/mirror/gpseg5:content=5:dbid=87:mode=s:status=u', 'seg03:/data1/pg_system/mirror/gpseg4:content=4:dbid=86:mode=s:status=u', 'seg03:/data2/pg_system/primary/gpseg10:content=10:dbid=12:mode=s:status=u', 'seg03:/data2/pg_system/primary/gpseg11:content=11:dbid=13:mode=s:status=u', 'seg03:/data1/pg_system/primary/gpseg9:content=9:dbid=11:mode=s:status=u', 'seg03:/data1/pg_system/primary/gpseg8:content=8:dbid=10:mode=s:status=u', 'seg03:/data2/pg_system/mirror/gpsegnew7:content=7:dbid=89:mode=s:status=u', 'seg03:/data2/pg_system/mirror/gpsegnew6:content=6:dbid=88:mode=s:status=u', 'seg12:/data2/pg_system/primary/gpseg47:content=47:dbid=49:mode=s:status=u', 'seg12:/data2/pg_system/primary/gpseg46:content=46:dbid=48:mode=s:status=u', 'seg12:/data1/pg_system/mirror/gpseg40:content=40:dbid=122:mode=s:status=u', 'seg12:/data1/pg_system/mirror/gpseg41:content=41:dbid=123:mode=s:status=u', 'seg12:/data1/pg_system/primary/gpseg44:content=44:dbid=46:mode=s:status=u', 'seg12:/data1/pg_system/primary/gpseg45:content=45:dbid=47:mode=s:status=u', 'seg12:/data2/pg_system/mirror/gpseg43:content=43:dbid=125:mode=s:status=u', 'seg12:/data2/pg_system/mirror/gpseg42:content=42:dbid=124:mode=s:status=u', 'seg20:/data2/pg_system/mirror/gpseg74:content=74:dbid=156:mode=s:status=u', 'seg20:/data2/pg_system/mirror/gpseg75:content=75:dbid=157:mode=s:status=u', 'seg20:/data1/pg_system/mirror/gpseg73:content=73:dbid=155:mode=s:status=u', 'seg20:/data1/pg_system/mirror/gpseg72:content=72:dbid=154:mode=s:status=u', 'seg20:/data1/pg_system/primary/gpseg77:content=77:dbid=79:mode=s:status=u', 'seg20:/data1/pg_system/primary/gpseg76:content=76:dbid=78:mode=s:status=u', 'seg20:/data2/pg_system/primary/gpseg78:content=78:dbid=80:mode=s:status=u', 'seg20:/data2/pg_system/primary/gpseg79:content=79:dbid=81:mode=s:status=u', 'seg07:/data2/pg_system/mirror/gpseg23:content=23:dbid=105:mode=s:status=u', 'seg07:/data2/pg_system/mirror/gpseg22:content=22:dbid=104:mode=s:status=u', 'seg07:/data1/pg_system/mirror/gpseg20:content=20:dbid=102:mode=s:status=u', 'seg07:/data1/pg_system/mirror/gpseg21:content=21:dbid=103:mode=s:status=u', 'seg07:/data1/pg_system/primary/gpseg24:content=24:dbid=26:mode=s:status=u', 'seg07:/data1/pg_system/primary/gpseg25:content=25:dbid=27:mode=s:status=u', 'seg07:/data2/pg_system/primary/gpseg27:content=27:dbid=29:mode=s:status=u', 'seg07:/data2/pg_system/primary/gpseg26:content=26:dbid=28:mode=s:status=u', 'seg05:/data2/pg_system/mirror/gpseg14:content=14:dbid=96:mode=s:status=u', 'seg05:/data2/pg_system/mirror/gpseg15:content=15:dbid=97:mode=s:status=u', 'seg05:/data1/pg_system/mirror/gpseg13:content=13:dbid=95:mode=s:status=u', 'seg05:/data1/pg_system/mirror/gpseg12:content=12:dbid=94:mode=s:status=u', 'seg05:/data1/pg_system/primary/gpseg17:content=17:dbid=19:mode=s:status=u', 'seg05:/data1/pg_system/primary/gpseg16:content=16:dbid=18:mode=s:status=u', 'seg05:/data2/pg_system/primary/gpseg18:content=18:dbid=20:mode=s:status=u', 'seg05:/data2/pg_system/primary/gpseg19:content=19:dbid=21:mode=s:status=u', 'seg06:/data2/pg_system/primary/gpseg23:content=23:dbid=25:mode=s:status=u', 'seg06:/data2/pg_system/primary/gpseg22:content=22:dbid=24:mode=s:status=u', 'seg06:/data2/pg_system/mirror/gpseg18:content=18:dbid=100:mode=s:status=u', 'seg06:/data2/pg_system/mirror/gpseg19:content=19:dbid=101:mode=s:status=u', 'seg06:/data1/pg_system/mirror/gpseg17:content=17:dbid=99:mode=s:status=u', 'seg06:/data1/pg_system/mirror/gpseg16:content=16:dbid=98:mode=s:status=u', 'seg06:/data1/pg_system/primary/gpseg20:content=20:dbid=22:mode=s:status=u', 'seg06:/data1/pg_system/primary/gpseg21:content=21:dbid=23:mode=s:status=u']
20190327:00:31:17:014215 gpstart:mas:gpadmin-[INFO]:-----------------------------------------------------
20190327:00:31:17:014215 gpstart:mas:gpadmin-[INFO]:-DBID:82  FAILED  host:'seg02' datadir:'/data1/pg_system/mirror/gpseg0' with reason:'Segment postmaster has exited; check segment logfile'
20190327:00:31:17:014215 gpstart:mas:gpadmin-[INFO]:-----------------------------------------------------
20190327:00:31:17:014215 gpstart:mas:gpadmin-[INFO]:-----------------------------------------------------
20190327:00:31:17:014215 gpstart:mas:gpadmin-[INFO]:-   Successful segment starts                                            = 158
20190327:00:31:17:014215 gpstart:mas:gpadmin-[WARNING]:-Failed segment starts                                                = 1     <<<<<<<<
20190327:00:31:17:014215 gpstart:mas:gpadmin-[WARNING]:-Skipped segment starts (segments are marked down in configuration)   = 1     <<<<<<<<
20190327:00:31:17:014215 gpstart:mas:gpadmin-[INFO]:-----------------------------------------------------
20190327:00:31:17:014215 gpstart:mas:gpadmin-[INFO]:-
20190327:00:31:17:014215 gpstart:mas:gpadmin-[INFO]:-Successfully started 158 of 159 segment instances, skipped 1 other segments <<<<<<<<
20190327:00:31:17:014215 gpstart:mas:gpadmin-[INFO]:-----------------------------------------------------
20190327:00:31:17:014215 gpstart:mas:gpadmin-[WARNING]:-Segment instance startup failures reported
20190327:00:31:17:014215 gpstart:mas:gpadmin-[WARNING]:-Failed start 1 of 159 segment instances <<<<<<<<
20190327:00:31:17:014215 gpstart:mas:gpadmin-[WARNING]:-Review /home/gpadmin/gpAdminLogs/gpstart_20190327.log
20190327:00:31:17:014215 gpstart:mas:gpadmin-[INFO]:-----------------------------------------------------
20190327:00:31:17:014215 gpstart:mas:gpadmin-[WARNING]:-****************************************************************************
20190327:00:31:17:014215 gpstart:mas:gpadmin-[WARNING]:-There are 1 segment(s) marked down in the database
20190327:00:31:17:014215 gpstart:mas:gpadmin-[WARNING]:-To recover from this current state, review usage of the gprecoverseg
20190327:00:31:17:014215 gpstart:mas:gpadmin-[WARNING]:-management utility which will recover failed segment instance databases.
20190327:00:31:17:014215 gpstart:mas:gpadmin-[WARNING]:-****************************************************************************
20190327:00:31:17:014215 gpstart:mas:gpadmin-[INFO]:-Commencing parallel segment instance shutdown, please wait...
20190327:00:31:21:014215 gpstart:mas:gpadmin-[ERROR]:-gpstart error: Do not have enough valid segments to start the array.
You have new mail in /var/spool/mail/gpadmin
[gpadmin@mas ~]$ 
数据库异常挂了一个实例，而这个实例的primary和primary均无法启动，导致整个集群无法启动数据库无法使用，启动master管理模式查看gp_segment_configuration系统表异常实例信息如下：

     2 |       0 | m    | p              | s    | d      | 40000 | seg01    | seg01   |            41000 | 
   82 |       0 | p     | m             | c    | u      | 50000 | seg02    | seg02   |            51000 | 

登录异常节点查看该实例数据库日志，详细异常信息如下：

2019-03-27 01:25:52.804801 CST,,,p23484,th2100778784,,,,0,,,seg-1,,,,,"LOG","00000","next transaction ID: 0/250355250; next OID: 383240004",,,,,,,0,,"xlog.c",6621,
2019-03-27 01:25:52.804817 CST,,,p23484,th2100778784,,,,0,,,seg-1,,,,,"LOG","00000","next MultiXactId: 7676039; next MultiXactOffset: 151107044",,,,,,,0,,"xlog.c",6624,
2019-03-27 01:25:52.804857 CST,,,p23484,th2100778784,,,,0,,,seg-1,,,,,"LOG","00000","database system was not properly shut down; automatic recovery in progress",,,,,,,0,,"xlog.c",6721,
2019-03-27 01:25:52.830638 CST,,,p23484,th2100778784,,,,0,,,seg-1,,,,,"LOG","00000","redo starts at 158C/AF169BA8",,,,,,,0,,"xlog.c",6853,
2019-03-27 01:25:52.835367 CST,,,p23484,th2100778784,,,,0,,,seg-1,,,,,"LOG","00000","incorrect resource manager data checksum in record at 158C/AF3C7D08",,,,,,,0,,"xlog.c",3425,
2019-03-27 01:25:52.835412 CST,,,p23484,th2100778784,,,,0,,,seg-1,,,,,"LOG","00000","redo done at 158C/AF3C7AE8",,,,,,,0,,"xlog.c",6942,
2019-03-27 01:25:52.835432 CST,,,p23484,th2100778784,,,,0,,,seg-1,,,,,"LOG","00000","end of transaction log location is 158C/AF3C7D08",,,,,,,0,,"xlog.c",6988,
2019-03-27 01:25:53.047107 CST,,,p23484,th2100778784,,,,0,,,seg-1,,,,,"FATAL","XX000","Global sequence number 20930983 less than maximum value 20930998 found in scan ('gp_persistent_relation_node') (cdbpersistentstore.c:600)",,,,,,,0,,"cdbpersistentstore.c",600,"Stack trace:
1    0xb03bda postgres <symbol not found> (elog.c:502)
2    0xb05be8 postgres elog_finish (elog.c:1446)
3    0xcb301e postgres <symbol not found> (cdbpersistentstore.c:600)
4    0xcb3e17 postgres PersistentStore_InitScanUnderLock (cdbpersistentstore.c:659)
5    0xc9a19c postgres PersistentFileSysObj_StartupInitScan (cdbpersistentfilesysobj.c:673)
6    0x5643f5 postgres StartupXLOG (xlog.c:7171)
7    0x566616 postgres StartupProcessMain (xlog.c:10970)
8    0x5f6715 postgres AuxiliaryProcessMain (bootstrap.c:463)
9    0x8ede24 postgres <symbol not found> (postmaster.c:7589)
10   0x8ee04d postgres StartMasterOrPrimaryPostmasterProcesses

多番尝试后最终解决方法为登录异常实例进入数据目录在配置文件postgresql.conf中设置参数gp_persistent_repair_global_sequence=true，然后启动集群恢复异常实例，恢复数据库后恢复还原修改的参数，最后使用gpcheckcat命令再检查数据库信息并针对详细异常信息恢复数据库元数据信息。