59aa00823c
In some cases, like read more than write scenario, the replication offset of the replicas are the same. When the primary fails, the replicas have the same rankings (rank == 0). They issue the election at the same time (although we have a random 500), the simultaneous elections may lead to the failure of the election due to quorum. In clusterGetReplicaRank, when we calculates the rank, if the offsets are the same, the one with the smaller node name will have a better rank to avoid this situation. --------- Signed-off-by: Binbin <binloveplay1314@qq.com>
134 lines
3.3 KiB
Tcl
134 lines
3.3 KiB
Tcl
# Check the basic monitoring and failover capabilities.
|
|
|
|
start_cluster 5 5 {tags {external:skip cluster}} {
|
|
|
|
test "Cluster is up" {
|
|
wait_for_cluster_state ok
|
|
}
|
|
|
|
test "Cluster is writable" {
|
|
cluster_write_test [srv 0 port]
|
|
}
|
|
|
|
test "Instance #5 is a slave" {
|
|
assert {[s -5 role] eq {slave}}
|
|
}
|
|
|
|
test "Instance #5 synced with the master" {
|
|
wait_for_condition 1000 50 {
|
|
[s -5 master_link_status] eq {up}
|
|
} else {
|
|
fail "Instance #5 master link status is not up"
|
|
}
|
|
}
|
|
|
|
set current_epoch [CI 1 cluster_current_epoch]
|
|
|
|
set paused_pid [srv 0 pid]
|
|
test "Killing one master node" {
|
|
pause_process $paused_pid
|
|
}
|
|
|
|
test "Wait for failover" {
|
|
wait_for_condition 1000 50 {
|
|
[CI 1 cluster_current_epoch] > $current_epoch
|
|
} else {
|
|
fail "No failover detected"
|
|
}
|
|
}
|
|
|
|
test "Cluster should eventually be up again" {
|
|
for {set j 0} {$j < [llength $::servers]} {incr j} {
|
|
if {[process_is_paused [srv -$j pid]]} continue
|
|
wait_for_condition 1000 50 {
|
|
[CI $j cluster_state] eq "ok"
|
|
} else {
|
|
fail "Cluster node $j cluster_state:[CI $j cluster_state]"
|
|
}
|
|
}
|
|
}
|
|
|
|
test "Cluster is writable" {
|
|
cluster_write_test [srv -1 port]
|
|
}
|
|
|
|
test "Instance #5 is now a master" {
|
|
assert {[s -5 role] eq {master}}
|
|
}
|
|
|
|
test "Restarting the previously killed master node" {
|
|
resume_process $paused_pid
|
|
}
|
|
|
|
test "Instance #0 gets converted into a slave" {
|
|
wait_for_condition 1000 50 {
|
|
[s 0 role] eq {slave}
|
|
} else {
|
|
fail "Old master was not converted into slave"
|
|
}
|
|
wait_for_cluster_propagation
|
|
}
|
|
|
|
} ;# start_cluster
|
|
|
|
start_cluster 3 6 {tags {external:skip cluster}} {
|
|
|
|
test "Cluster is up" {
|
|
wait_for_cluster_state ok
|
|
}
|
|
|
|
test "Cluster is writable" {
|
|
cluster_write_test [srv 0 port]
|
|
}
|
|
|
|
set current_epoch [CI 1 cluster_current_epoch]
|
|
|
|
set paused_pid [srv 0 pid]
|
|
test "Killing the first primary node" {
|
|
pause_process $paused_pid
|
|
}
|
|
|
|
test "Wait for failover" {
|
|
wait_for_condition 1000 50 {
|
|
[CI 1 cluster_current_epoch] > $current_epoch
|
|
} else {
|
|
fail "No failover detected"
|
|
}
|
|
}
|
|
|
|
test "Cluster should eventually be up again" {
|
|
for {set j 0} {$j < [llength $::servers]} {incr j} {
|
|
if {[process_is_paused [srv -$j pid]]} continue
|
|
wait_for_condition 1000 50 {
|
|
[CI $j cluster_state] eq "ok"
|
|
} else {
|
|
fail "Cluster node $j cluster_state:[CI $j cluster_state]"
|
|
}
|
|
}
|
|
}
|
|
|
|
test "Restarting the previously killed primary node" {
|
|
resume_process $paused_pid
|
|
}
|
|
|
|
test "Instance #0 gets converted into a replica" {
|
|
wait_for_condition 1000 50 {
|
|
[s 0 role] eq {slave}
|
|
} else {
|
|
fail "Old primary was not converted into replica"
|
|
}
|
|
wait_for_cluster_propagation
|
|
}
|
|
|
|
test "Make sure the replicas always get the different ranks" {
|
|
if {[s -3 role] == "master"} {
|
|
verify_log_message -3 "*Start of election*rank #0*" 0
|
|
verify_log_message -6 "*Start of election*rank #1*" 0
|
|
} else {
|
|
verify_log_message -3 "*Start of election*rank #1*" 0
|
|
verify_log_message -6 "*Start of election*rank #0*" 0
|
|
}
|
|
}
|
|
|
|
} ;# start_cluster
|