Mongo replicaSet dispersed over all continents
https://dba.stackexchange.com/questions/192450
-
10-10-2020 - |
italiano
english
français
española
中国
日本の
العربية
Deutsch
한국어
Português
RussianFrage
I have problem in operating mongo replicaSet which works on all continents. The primary data is located in Europe and want to replicate it distant locations in US, Brasil, Tokyo, Sydney and Dublin.
Unfortunately I'm keep getting errors like this
05/12/2017 18:23:192017-12-05T17:23:19.990+0000 I REPL [ReplicationExecutor] Error in heartbeat request to mongo-pcluster-7:27017; ExceededTimeLimit: Couldn't get a connection within the time limit
05/12/2017 18:23:192017-12-05T17:23:19.991+0000 I REPL [ReplicationExecutor] Error in heartbeat request to mongo-pcluster-3:27017; ExceededTimeLimit: Couldn't get a connection within the time limit
05/12/2017 18:23:192017-12-05T17:23:19.991+0000 I REPL [ReplicationExecutor] Error in heartbeat request to mongo-pcluster-6:27017; ExceededTimeLimit: Couldn't get a connection within the time limit
05/12/2017 18:23:192017-12-05T17:23:19.992+0000 I REPL [ReplicationExecutor] Error in heartbeat request to mongo-pcluster-2:27017; ExceededTimeLimit: Couldn't get a connection within the time limit
05/12/2017 18:23:192017-12-05T17:23:19.992+0000 I ASIO [NetworkInterfaceASIO-Replication-0] Failed to connect to mongo-pcluster-7:27017 - ExceededTimeLimit: Operation timed out
05/12/2017 18:23:192017-12-05T17:23:19.993+0000 I ASIO [NetworkInterfaceASIO-Replication-0] Failed to connect to mongo-pcluster-3:27017 - ExceededTimeLimit: Operation timed out
05/12/2017 18:23:192017-12-05T17:23:19.993+0000 I ASIO [NetworkInterfaceASIO-Replication-0] Failed to connect to mongo-pcluster-6:27017 - ExceededTimeLimit: Operation timed out
05/12/2017 18:23:192017-12-05T17:23:19.993+0000 I ASIO [NetworkInterfaceASIO-Replication-0] Failed to connect to mongo-pcluster-2:27017 - ExceededTimeLimit: Operation timed out
05/12/2017 18:23:242017-12-05T17:23:24.990+0000 I ASIO [NetworkInterfaceASIO-Replication-0] Connecting to mongo-pcluster-7:27017
05/12/2017 18:23:242017-12-05T17:23:24.992+0000 I ASIO [NetworkInterfaceASIO-Replication-0] Connecting to mongo-pcluster-3:27017
05/12/2017 18:23:242017-12-05T17:23:24.992+0000 I ASIO [NetworkInterfaceASIO-Replication-0] Connecting to mongo-pcluster-6:27017
05/12/2017 18:23:242017-12-05T17:23:24.992+0000 I ASIO [NetworkInterfaceASIO-Replication-0] Connecting to mongo-pcluster-2:27017
and the status of the cluster keep failing into recovering.
rs0:PRIMARY> rs.status()
{
"set" : "rs0",
"date" : ISODate("2017-12-05T17:11:54.400Z"),
"myState" : 1,
"term" : NumberLong(7),
"heartbeatIntervalMillis" : NumberLong(2000),
"members" : [
{
"_id" : 0,
"name" : "mongo-pcluster-4:27017",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 1107523,
"optime" : {
"ts" : Timestamp(1512493794, 15),
"t" : NumberLong(7)
},
"optimeDate" : ISODate("2017-12-05T17:09:54Z"),
"electionTime" : Timestamp(1512298906, 1),
"electionDate" : ISODate("2017-12-03T11:01:46Z"),
"configVersion" : 10,
"self" : true
},
{
"_id" : 1,
"name" : "mongo-pcluster-5:27017",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 1106965,
"optime" : {
"ts" : Timestamp(1512493794, 15),
"t" : NumberLong(7)
},
"optimeDate" : ISODate("2017-12-05T17:09:54Z"),
"lastHeartbeat" : ISODate("2017-12-05T17:11:54.093Z"),
"lastHeartbeatRecv" : ISODate("2017-12-05T17:11:52.845Z"),
"pingMs" : NumberLong(3),
"syncingTo" : "mongo-pcluster-4:27017",
"configVersion" : 10
},
{
"_id" : 2,
"name" : "mongo-pcluster-1:27017",
"health" : 1,
"state" : 3,
"stateStr" : "RECOVERING",
"uptime" : 195019,
"optime" : {
"ts" : Timestamp(1512492648, 63),
"t" : NumberLong(7)
},
"optimeDate" : ISODate("2017-12-05T16:50:48Z"),
"lastHeartbeat" : ISODate("2017-12-05T17:11:54.371Z"),
"lastHeartbeatRecv" : ISODate("2017-12-05T17:11:52.608Z"),
"pingMs" : NumberLong(326),
"syncingTo" : "mongo-pcluster-5:27017",
"configVersion" : 10
},
{
"_id" : 3,
"name" : "mongo-pcluster-2:27017",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 195016,
"optime" : {
"ts" : Timestamp(1512493498, 10),
"t" : NumberLong(7)
},
"optimeDate" : ISODate("2017-12-05T17:04:58Z"),
"lastHeartbeat" : ISODate("2017-12-05T17:11:52.537Z"),
"lastHeartbeatRecv" : ISODate("2017-12-05T17:11:53.447Z"),
"pingMs" : NumberLong(27),
"syncingTo" : "mongo-pcluster-5:27017",
"configVersion" : 10
},
{
"_id" : 4,
"name" : "mongo-pcluster-3:27017",
"health" : 1,
"state" : 3,
"stateStr" : "RECOVERING",
"uptime" : 195024,
"optime" : {
"ts" : Timestamp(1512492759, 138),
"t" : NumberLong(7)
},
"optimeDate" : ISODate("2017-12-05T16:52:39Z"),
"lastHeartbeat" : ISODate("2017-12-05T17:11:53.259Z"),
"lastHeartbeatRecv" : ISODate("2017-12-05T17:11:52.384Z"),
"pingMs" : NumberLong(131),
"configVersion" : 10
},
{
"_id" : 5,
"name" : "mongo-pcluster-6:27017",
"health" : 1,
"state" : 3,
"stateStr" : "RECOVERING",
"uptime" : 195000,
"optime" : {
"ts" : Timestamp(1512492675, 19),
"t" : NumberLong(7)
},
"optimeDate" : ISODate("2017-12-05T16:51:15Z"),
"lastHeartbeat" : ISODate("2017-12-05T17:11:53.476Z"),
"lastHeartbeatRecv" : ISODate("2017-12-05T17:11:49.488Z"),
"pingMs" : NumberLong(241),
"configVersion" : 10
},
{
"_id" : 6,
"name" : "mongo-pcluster-7:27017",
"health" : 1,
"state" : 3,
"stateStr" : "RECOVERING",
"uptime" : 195015,
"optime" : {
"ts" : Timestamp(1512492651, 65),
"t" : NumberLong(7)
},
"optimeDate" : ISODate("2017-12-05T16:50:51Z"),
"lastHeartbeat" : ISODate("2017-12-05T17:11:54.027Z"),
"lastHeartbeatRecv" : ISODate("2017-12-05T17:11:52.632Z"),
"pingMs" : NumberLong(306),
"configVersion" : 10
}
],
"ok" : 1
}
rs0:PRIMARY>
How to make the settings for this replicaSet to work steady on long distance links.
My current config is as below. I increased timeout to 60seconds which seems to be way then needed on this distances but the increase has changed nothing.
rs0:PRIMARY> cfg = rs.conf()
{
"_id" : "rspreredner0",
"version" : 10,
"protocolVersion" : NumberLong(1),
"members" : [
{
"_id" : 0,
"host" : "mongo-pcluster-4:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 500,
"tags" : {
},
"slaveDelay" : NumberLong(0),
"votes" : 1
},
{
"_id" : 1,
"host" : "mongo-pcluster-5:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 200,
"tags" : {
},
"slaveDelay" : NumberLong(0),
"votes" : 1
},
{
"_id" : 2,
"host" : "mongo-pcluster-1:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"slaveDelay" : NumberLong(0),
"votes" : 1
},
{
"_id" : 3,
"host" : "mongo-pcluster-2:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"slaveDelay" : NumberLong(0),
"votes" : 1
},
{
"_id" : 4,
"host" : "mongo-pcluster-3:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"slaveDelay" : NumberLong(0),
"votes" : 1
},
{
"_id" : 5,
"host" : "mongo-pcluster-6:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"slaveDelay" : NumberLong(0),
"votes" : 1
},
{
"_id" : 6,
"host" : "mongo-pcluster-7:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"slaveDelay" : NumberLong(0),
"votes" : 1
}
],
"settings" : {
"chainingAllowed" : true,
"heartbeatIntervalMillis" : 2000,
"heartbeatTimeoutSecs" : 60,
"electionTimeoutMillis" : 10000,
"getLastErrorModes" : {
},
"getLastErrorDefaults" : {
"w" : 1,
"wtimeout" : 0
},
"replicaSetId" : ObjectId("5a15edbfe2c98aee31ae2c4f")
}
}
rs0:PRIMARY>
Lösung
The reason why it didn't change anything is that your config have "protocolVersion" : NumberLong(1) and this heartbeatTimeoutSecs applies only when protocolVersion is zero (0).
You should use settings.electionTimeoutMillis wich description you can find here.
Lizenziert unter: CC-BY-SA mit Zuschreibung
Nicht verbunden mit dba.stackexchange
