一、检查副本集状态
handong1:PRIMARY> rs.status()
{
"set" : "handong1",
"date" : ISODate("2021-03-18T11:43:08.167Z"),
"myState" : 1,
"term" : NumberLong(3),
"syncingTo" : "",
"syncSourceHost" : "",
"syncSourceId" : -1,
"heartbeatIntervalMillis" : NumberLong(2000),
"majorityVoteCount" : 2,
"writeMajorityCount" : 2,
"optimes" : {
"lastCommittedOpTime" : {
"ts" : Timestamp(1616067784, 1),
"t" : NumberLong(3)
},
"lastCommittedWallTime" : ISODate("2021-03-18T11:43:04.894Z"),
"readConcernMajorityOpTime" : {
"ts" : Timestamp(1616067784, 1),
"t" : NumberLong(3)
},
"readConcernMajorityWallTime" : ISODate("2021-03-18T11:43:04.894Z"),
"appliedOpTime" : {
"ts" : Timestamp(1616067784, 1),
"t" : NumberLong(3)
},
"durableOpTime" : {
"ts" : Timestamp(1616067784, 1),
"t" : NumberLong(3)
},
"lastAppliedWallTime" : ISODate("2021-03-18T11:43:04.894Z"),
"lastDurableWallTime" : ISODate("2021-03-18T11:43:04.894Z")
},
"lastStableRecoveryTimestamp" : Timestamp(1616067764, 1),
"lastStableCheckpointTimestamp" : Timestamp(1616067764, 1),
"electionCandidateMetrics" : {
"lastElectionReason" : "electionTimeout",
"lastElectionDate" : ISODate("2021-03-18T09:05:47.089Z"),
"electionTerm" : NumberLong(3),
"lastCommittedOpTimeAtElection" : {
"ts" : Timestamp(0, 0),
"t" : NumberLong(-1)
},
"lastSeenOpTimeAtElection" : {
"ts" : Timestamp(1616055916, 1),
"t" : NumberLong(2)
},
"numVotesNeeded" : 1,
"priorityAtElection" : 1,
"electionTimeoutMillis" : NumberLong(10000),
"newTermStartDate" : ISODate("2021-03-18T09:05:47.098Z"),
"wMajorityWriteAvailabilityDate" : ISODate("2021-03-18T09:05:47.129Z")
},
"members" : [
{
"_id" : 0,
"name" : "172.16.254.131:27017",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 9444,
"optime" : {
"ts" : Timestamp(1616067784, 1),
"t" : NumberLong(3)
},
"optimeDate" : ISODate("2021-03-18T11:43:04Z"),
"syncingTo" : "",
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"electionTime" : Timestamp(1616058347, 1),
"electionDate" : ISODate("2021-03-18T09:05:47Z"),
"configVersion" : 6,
"self" : true,
"lastHeartbeatMessage" : ""
},
{
"_id" : 1,
"name" : "mongo2:27017",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 8089,
"optime" : {
"ts" : Timestamp(1616067784, 1),
"t" : NumberLong(3)
},
"optimeDurable" : {
"ts" : Timestamp(1616067784, 1),
"t" : NumberLong(3)
},
"optimeDate" : ISODate("2021-03-18T11:43:04Z"),
"optimeDurableDate" : ISODate("2021-03-18T11:43:04Z"),
"lastHeartbeat" : ISODate("2021-03-18T11:43:07.136Z"),
"lastHeartbeatRecv" : ISODate("2021-03-18T11:43:07.113Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "",
"syncingTo" : "172.16.254.131:27017",
"syncSourceHost" : "172.16.254.131:27017",
"syncSourceId" : 0,
"infoMessage" : "",
"configVersion" : 6
},
{
"_id" : 2,
"name" : "mongo3:27017",
"health" : 1,
"state" : 7,
"stateStr" : "ARBITER",
"uptime" : 8062,
"lastHeartbeat" : ISODate("2021-03-18T11:43:07.135Z"),
"lastHeartbeatRecv" : ISODate("2021-03-18T11:43:06.741Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "",
"syncingTo" : "",
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"configVersion" : 6
}
],
"ok" : 1,
"$clusterTime" : {
"clusterTime" : Timestamp(1616067784, 1),
"signature" : {
"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
"keyId" : NumberLong(0)
}
},
"operationTime" : Timestamp(1616067784, 1)
}
二、测试复制
172.16.254.131
handong1:PRIMARY> use db3
switched to db db3
handong1:PRIMARY> db.db3.insert({"name":"jianghe"})
WriteResult({ "nInserted" : 1 })
handong1:PRIMARY> db.db3.find()
{ "_id" : ObjectId("6051d08ac74d627f4b561349"), "name" : "mongodb" }
{ "_id" : ObjectId("6053225bdebdeda2639f026e"), "name" : "mongodb4.2" }
{ "_id" : ObjectId("60533d88504b2400d4ae3091"), "name" : "jianghe" }
172.16.254.132
handong1:SECONDARY> rs.secondaryOk()
handong1:SECONDARY> db.db3.find()
{ "_id" : ObjectId("6051d08ac74d627f4b561349"), "name" : "mongodb" }
{ "_id" : ObjectId("6053225bdebdeda2639f026e"), "name" : "mongodb4.2" }
{ "_id" : ObjectId("60533d88504b2400d4ae3091"), "name" : "jianghe" }
目前测试正常。
三、测试故障转移
1、关闭主节点服务器
2、查看集群状态
172.16.254.132
handong1:PRIMARY> rs.status()
{
"set" : "handong1",
"date" : ISODate("2021-03-18T13:05:51.862Z"),
"myState" : 1,
"term" : NumberLong(5),
"syncingTo" : "",
"syncSourceHost" : "",
"syncSourceId" : -1,
"heartbeatIntervalMillis" : NumberLong(2000),
"majorityVoteCount" : 2,
"writeMajorityCount" : 2,
"optimes" : {
"lastCommittedOpTime" : {
"ts" : Timestamp(1616071857, 1),
"t" : NumberLong(3)
},
"lastCommittedWallTime" : ISODate("2021-03-18T12:50:57.574Z"),
"readConcernMajorityOpTime" : {
"ts" : Timestamp(1616071857, 1),
"t" : NumberLong(3)
},
"readConcernMajorityWallTime" : ISODate("2021-03-18T12:50:57.574Z"),
"appliedOpTime" : {
"ts" : Timestamp(1616072744, 1),
"t" : NumberLong(5)
},
"durableOpTime" : {
"ts" : Timestamp(1616072744, 1),
"t" : NumberLong(5)
},
"lastAppliedWallTime" : ISODate("2021-03-18T13:05:44.997Z"),
"lastDurableWallTime" : ISODate("2021-03-18T13:05:44.997Z")
},
"lastStableRecoveryTimestamp" : Timestamp(1616071857, 1),
"lastStableCheckpointTimestamp" : Timestamp(1616071857, 1),
"electionCandidateMetrics" : {
"lastElectionReason" : "electionTimeout",
"lastElectionDate" : ISODate("2021-03-18T12:51:14.081Z"),
"electionTerm" : NumberLong(5),
"lastCommittedOpTimeAtElection" : {
"ts" : Timestamp(1616071857, 1),
"t" : NumberLong(3)
},
"lastSeenOpTimeAtElection" : {
"ts" : Timestamp(1616071857, 1),
"t" : NumberLong(3)
},
"numVotesNeeded" : 2,
"priorityAtElection" : 1,
"electionTimeoutMillis" : NumberLong(10000),
"numCatchUpOps" : NumberLong(0),
"newTermStartDate" : ISODate("2021-03-18T12:51:14.471Z")
},
"members" : [
{
"_id" : 0,
"name" : "172.16.254.131:27017",
"health" : 0,
"state" : 8,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : {
"ts" : Timestamp(0, 0),
"t" : NumberLong(-1)
},
"optimeDurable" : {
"ts" : Timestamp(0, 0),
"t" : NumberLong(-1)
},
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"optimeDurableDate" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2021-03-18T13:05:42.846Z"),
"lastHeartbeatRecv" : ISODate("2021-03-18T12:51:03.822Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "Error connecting to 172.16.254.131:27017 :: caused by :: No route to host",
"syncingTo" : "",
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"configVersion" : -1
},
{
"_id" : 1,
"name" : "mongo2:27017",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 14396,
"optime" : {
"ts" : Timestamp(1616072744, 1),
"t" : NumberLong(5)
},
"optimeDate" : ISODate("2021-03-18T13:05:44Z"),
"syncingTo" : "",
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"electionTime" : Timestamp(1616071874, 1),
"electionDate" : ISODate("2021-03-18T12:51:14Z"),
"configVersion" : 6,
"self" : true,
"lastHeartbeatMessage" : ""
},
{
"_id" : 2,
"name" : "mongo3:27017",
"health" : 1,
"state" : 7,
"stateStr" : "ARBITER",
"uptime" : 13029,
"lastHeartbeat" : ISODate("2021-03-18T13:05:51.848Z"),
"lastHeartbeatRecv" : ISODate("2021-03-18T13:05:51.782Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "",
"syncingTo" : "",
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"configVersion" : 6
}
],
"ok" : 1,
"$clusterTime" : {
"clusterTime" : Timestamp(1616072744, 1),
"signature" : {
"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
"keyId" : NumberLong(0)
}
},
"operationTime" : Timestamp(1616072744, 1)
}
可以看到原来的primary已经not reachable, 172.16.254.132原来昨晚recondary(副本)现在变成了primary。
测试副本集可用性
[mongodb@mongo2 ~]$ mongo --port 27017
handong1:PRIMARY> db.db3.insert({"name":"dandan"})
WriteResult({ "nInserted" : 1 })
handong1:PRIMARY> db.db3.find()
{ "_id" : ObjectId("6051d08ac74d627f4b561349"), "name" : "mongodb" }
{ "_id" : ObjectId("6053225bdebdeda2639f026e"), "name" : "mongodb4.2" }
{ "_id" : ObjectId("60533d88504b2400d4ae3091"), "name" : "jianghe" }
{ "_id" : ObjectId("60534e662b429cdfe156324a"), "name" : "dandan" }
可以看到当前的primary已经可以进行读写操作。
此时启动 172.16.254.131服务器,并启动mongodb服务。
在 172.16.254.131查看副本集信息:
handong1:PRIMARY> rs.status()
{
"set" : "handong1",
"date" : ISODate("2021-03-18T13:35:29.218Z"),
"myState" : 1,
"term" : NumberLong(6),
"syncingTo" : "",
"syncSourceHost" : "",
"syncSourceId" : -1,
"heartbeatIntervalMillis" : NumberLong(2000),
"majorityVoteCount" : 2,
"writeMajorityCount" : 2,
"optimes" : {
"lastCommittedOpTime" : {
"ts" : Timestamp(1616074523, 1),
"t" : NumberLong(6)
},
"lastCommittedWallTime" : ISODate("2021-03-18T13:35:23.542Z"),
"readConcernMajorityOpTime" : {
"ts" : Timestamp(1616074523, 1),
"t" : NumberLong(6)
},
"readConcernMajorityWallTime" : ISODate("2021-03-18T13:35:23.542Z"),
"appliedOpTime" : {
"ts" : Timestamp(1616074523, 1),
"t" : NumberLong(6)
},
"durableOpTime" : {
"ts" : Timestamp(1616074523, 1),
"t" : NumberLong(6)
},
"lastAppliedWallTime" : ISODate("2021-03-18T13:35:23.542Z"),
"lastDurableWallTime" : ISODate("2021-03-18T13:35:23.542Z")
},
"lastStableRecoveryTimestamp" : Timestamp(1616074513, 1),
"lastStableCheckpointTimestamp" : Timestamp(1616074513, 1),
"electionCandidateMetrics" : {
"lastElectionReason" : "priorityTakeover",
"lastElectionDate" : ISODate("2021-03-18T13:34:33.453Z"),
"electionTerm" : NumberLong(6),
"lastCommittedOpTimeAtElection" : {
"ts" : Timestamp(1616074466, 1),
"t" : NumberLong(5)
},
"lastSeenOpTimeAtElection" : {
"ts" : Timestamp(1616074466, 1),
"t" : NumberLong(5)
},
"numVotesNeeded" : 2,
"priorityAtElection" : 2,
"electionTimeoutMillis" : NumberLong(10000),
"priorPrimaryMemberId" : 1,
"numCatchUpOps" : NumberLong(0),
"newTermStartDate" : ISODate("2021-03-18T13:34:33.518Z"),
"wMajorityWriteAvailabilityDate" : ISODate("2021-03-18T13:34:34.476Z")
},
"members" : [
{
"_id" : 0,
"name" : "172.16.254.131:27017",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 71,
"optime" : {
"ts" : Timestamp(1616074523, 1),
"t" : NumberLong(6)
},
"optimeDate" : ISODate("2021-03-18T13:35:23Z"),
"syncingTo" : "",
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"electionTime" : Timestamp(1616074473, 1),
"electionDate" : ISODate("2021-03-18T13:34:33Z"),
"configVersion" : 6,
"self" : true,
"lastHeartbeatMessage" : ""
},
{
"_id" : 1,
"name" : "mongo2:27017",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 67,
"optime" : {
"ts" : Timestamp(1616074523, 1),
"t" : NumberLong(6)
},
"optimeDurable" : {
"ts" : Timestamp(1616074523, 1),
"t" : NumberLong(6)
},
"optimeDate" : ISODate("2021-03-18T13:35:23Z"),
"optimeDurableDate" : ISODate("2021-03-18T13:35:23Z"),
"lastHeartbeat" : ISODate("2021-03-18T13:35:27.692Z"),
"lastHeartbeatRecv" : ISODate("2021-03-18T13:35:29.131Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "",
"syncingTo" : "172.16.254.131:27017",
"syncSourceHost" : "172.16.254.131:27017",
"syncSourceId" : 0,
"infoMessage" : "",
"configVersion" : 6
},
{
"_id" : 2,
"name" : "mongo3:27017",
"health" : 1,
"state" : 7,
"stateStr" : "ARBITER",
"uptime" : 67,
"lastHeartbeat" : ISODate("2021-03-18T13:35:27.692Z"),
"lastHeartbeatRecv" : ISODate("2021-03-18T13:35:28.445Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "",
"syncingTo" : "",
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"configVersion" : 6
}
],
"ok" : 1,
"$clusterTime" : {
"clusterTime" : Timestamp(1616074523, 1),
"signature" : {
"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
"keyId" : NumberLong(0)
}
},
"operationTime" : Timestamp(1616074523, 1)
}
可以看到 172.16.254.131上的主节点又变成了primary。
在 172.16.254.131上插入数据:
handong1:PRIMARY> use db3
switched to db db3
handong1:PRIMARY> db.db3.insert({"name":"doudou"})
WriteResult({ "nInserted" : 1 })
handong1:PRIMARY> db.db3.find()
{ "_id" : ObjectId("6051d08ac74d627f4b561349"), "name" : "mongodb" }
{ "_id" : ObjectId("6053225bdebdeda2639f026e"), "name" : "mongodb4.2" }
{ "_id" : ObjectId("60533d88504b2400d4ae3091"), "name" : "jianghe" }
{ "_id" : ObjectId("60534e662b429cdfe156324a"), "name" : "dandan" }
{ "_id" : ObjectId("605359a1ab6994b3520da444"), "name" : "doudou" }
在 172.16.254.132查询数据:
handong1:SECONDARY> use db3
switched to db db3
handong1:SECONDARY> rs.secondaryOk()
handong1:SECONDARY> db.db3.find()
{ "_id" : ObjectId("6051d08ac74d627f4b561349"), "name" : "mongodb" }
{ "_id" : ObjectId("6053225bdebdeda2639f026e"), "name" : "mongodb4.2" }
{ "_id" : ObjectId("60533d88504b2400d4ae3091"), "name" : "jianghe" }
{ "_id" : ObjectId("60534e662b429cdfe156324a"), "name" : "dandan" }
{ "_id" : ObjectId("605359a1ab6994b3520da444"), "name" : "doudou" }
可以看到,副本集数据恢复了同步。