103 lines
3.9 KiB
Tcl
103 lines
3.9 KiB
Tcl
# Issue 3899 regression test.
|
|
# We create a chain of three instances: master -> slave -> slave2
|
|
# and continuously break the link while traffic is generated by
|
|
# keydb-benchmark. At the end we check that the data is the same
|
|
# everywhere.
|
|
|
|
start_server {tags {"psync2"} overrides {active-replica {yes} multi-master {yes} client-output-buffer-limit {replica 200mb 10mb 999999} } } {
|
|
start_server {overrides {active-replica {yes} multi-master {yes} client-output-buffer-limit {replica 200mb 10mb 999999} } } {
|
|
start_server {overrides {active-replica {yes} multi-master {yes} client-output-buffer-limit {replica 200mb 10mb 999999} } } {
|
|
# Config
|
|
set debug_msg 0 ; # Enable additional debug messages
|
|
|
|
set no_exit 0 ; # Do not exit at end of the test
|
|
|
|
set duration 20 ; # Total test seconds
|
|
|
|
for {set j 0} {$j < 3} {incr j} {
|
|
set R($j) [srv [expr 0-$j] client]
|
|
set R_host($j) [srv [expr 0-$j] host]
|
|
set R_port($j) [srv [expr 0-$j] port]
|
|
set R_unixsocket($j) [srv [expr 0-$j] unixsocket]
|
|
if {$debug_msg} {puts "Log file: [srv [expr 0-$j] stdout]"}
|
|
}
|
|
|
|
# Setup the replication and backlog parameters
|
|
test "PSYNC2 #3899 regression: setup" {
|
|
$R(0) slaveof $R_host(1) $R_port(1)
|
|
$R(0) slaveof $R_host(2) $R_port(2)
|
|
$R(1) slaveof $R_host(0) $R_port(0)
|
|
$R(1) slaveof $R_host(2) $R_port(2)
|
|
$R(2) slaveof $R_host(0) $R_port(0)
|
|
$R(2) slaveof $R_host(1) $R_port(1)
|
|
|
|
$R(0) set foo bar
|
|
wait_for_condition 50 1000 {
|
|
[status $R(1) master_link_status] == "up" &&
|
|
[status $R(2) master_link_status] == "up" &&
|
|
[$R(1) dbsize] == 1 &&
|
|
[$R(2) dbsize] == 1
|
|
} else {
|
|
fail "Replicas not replicating from master"
|
|
}
|
|
|
|
$R(0) config set repl-backlog-size 200mb
|
|
$R(1) config set repl-backlog-size 200mb
|
|
$R(2) config set repl-backlog-size 200mb
|
|
|
|
}
|
|
|
|
set cycle_start_time [clock milliseconds]
|
|
set bench_pid [exec src/keydb-benchmark -s $R_unixsocket(0) -n 10000000 -r 1000 incr __rand_int__ > /dev/null &]
|
|
while 1 {
|
|
set elapsed [expr {[clock milliseconds]-$cycle_start_time}]
|
|
if {$elapsed > $duration*1000} break
|
|
if {rand() < .05} {
|
|
test "PSYNC2 #3899 regression: kill first replica" {
|
|
$R(1) client kill type master
|
|
}
|
|
}
|
|
if {rand() < .05} {
|
|
test "PSYNC2 #3899 regression: kill chained replica" {
|
|
$R(2) client kill type master
|
|
}
|
|
}
|
|
after 100
|
|
}
|
|
exec kill -9 $bench_pid
|
|
|
|
if {$debug_msg} {
|
|
for {set j 0} {$j < 100} {incr j} {
|
|
if {
|
|
[$R(0) debug digest] == [$R(1) debug digest] &&
|
|
[$R(1) debug digest] == [$R(2) debug digest]
|
|
} break
|
|
puts [$R(0) debug digest]
|
|
puts [$R(1) debug digest]
|
|
puts [$R(2) debug digest]
|
|
after 1000
|
|
}
|
|
}
|
|
|
|
test "PSYNC2 #3899 regression: verify consistency" {
|
|
wait_for_condition 50 1000 {
|
|
([$R(0) debug digest] eq [$R(1) debug digest]) &&
|
|
([$R(1) debug digest] eq [$R(2) debug digest])
|
|
} else {
|
|
set csv3 [csvdump {r -2}]
|
|
set csv2 [csvdump {r -1}]
|
|
set csv1 [csvdump r]
|
|
set fd [open /tmp/repldump1.txt w]
|
|
puts -nonewline $fd $csv1
|
|
close $fd
|
|
set fd [open /tmp/repldump2.txt w]
|
|
puts -nonewline $fd $csv2
|
|
close $fd
|
|
set fd [open /tmp/repldump3.txt w]
|
|
puts -nonewline $fd $csv3
|
|
close $fd
|
|
fail [format "The three instances have different data sets:\n\tnode 1: %s\n\tnode 2: %s\n\tnode 3: %s\nRun diff -u against /tmp/repldump*.txt for more info" [$R(0) debug digest] [$R(1) debug digest] [$R(2) debug digest] ]
|
|
}
|
|
}
|
|
}}}
|