added tests for flash psync
Former-commit-id: 79f7376a4fea2de9bd06c06d96f3f98ee7308874
This commit is contained in:
parent
a59b192174
commit
0dacd68163
135
tests/integration/replication-psync-flash.tcl
Normal file
135
tests/integration/replication-psync-flash.tcl
Normal file
@ -0,0 +1,135 @@
|
||||
# Creates a master-slave pair and breaks the link continuously to force
|
||||
# partial resyncs attempts, all this while flooding the master with
|
||||
# write queries.
|
||||
#
|
||||
# You can specify backlog size, ttl, delay before reconnection, test duration
|
||||
# in seconds, and an additional condition to verify at the end.
|
||||
#
|
||||
# If reconnect is > 0, the test actually try to break the connection and
|
||||
# reconnect with the master, otherwise just the initial synchronization is
|
||||
# checked for consistency.
|
||||
proc test_psync {descr duration backlog_size backlog_ttl delay cond mdl sdl reconnect} {
|
||||
start_server {tags {"repl"}} {
|
||||
start_server [list tags {flash} overrides [list storage-provider {flash ./rocks.db} delete-on-evict no storage-flush-period 10]] {
|
||||
|
||||
set master [srv -1 client]
|
||||
set master_host [srv -1 host]
|
||||
set master_port [srv -1 port]
|
||||
set slave [srv 0 client]
|
||||
|
||||
set load_handle0 [start_bg_complex_data $master_host $master_port 9 100000]
|
||||
set load_handle1 [start_bg_complex_data $master_host $master_port 11 100000]
|
||||
set load_handle2 [start_bg_complex_data $master_host $master_port 12 100000]
|
||||
|
||||
test {Slave should be able to synchronize with the master} {
|
||||
$slave slaveof $master_host $master_port
|
||||
wait_for_condition 50 100 {
|
||||
[lindex [r role] 0] eq {slave} &&
|
||||
[lindex [r role] 3] eq {connected}
|
||||
} else {
|
||||
fail "Replication not started."
|
||||
}
|
||||
}
|
||||
|
||||
# Check that the background clients are actually writing.
|
||||
test {Detect write load to master} {
|
||||
wait_for_condition 50 1000 {
|
||||
[$master dbsize] > 100
|
||||
} else {
|
||||
fail "Can't detect write load from background clients."
|
||||
}
|
||||
}
|
||||
|
||||
test "Test replication partial resync: $descr (diskless: $mdl, $sdl, reconnect: $reconnect)" {
|
||||
# Now while the clients are writing data, break the maste-slave
|
||||
# link multiple times.
|
||||
if ($reconnect) {
|
||||
for {set j 0} {$j < $duration*10} {incr j} {
|
||||
after 100
|
||||
# catch {puts "MASTER [$master dbsize] keys, REPLICA [$slave dbsize] keys"}
|
||||
|
||||
if {($j % 20) == 0} {
|
||||
catch {
|
||||
$slave debug restart
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
stop_bg_complex_data $load_handle0
|
||||
stop_bg_complex_data $load_handle1
|
||||
stop_bg_complex_data $load_handle2
|
||||
|
||||
# Wait for the slave to reach the "online"
|
||||
# state from the POV of the master.
|
||||
set retry 5000
|
||||
while {$retry} {
|
||||
set info [$master info]
|
||||
if {[string match {*slave0:*state=online*} $info]} {
|
||||
break
|
||||
} else {
|
||||
incr retry -1
|
||||
after 100
|
||||
}
|
||||
}
|
||||
if {$retry == 0} {
|
||||
error "assertion:Slave not correctly synchronized"
|
||||
}
|
||||
|
||||
# Wait that slave acknowledge it is online so
|
||||
# we are sure that DBSIZE and DEBUG DIGEST will not
|
||||
# fail because of timing issues. (-LOADING error)
|
||||
wait_for_condition 5000 100 {
|
||||
[lindex [$slave role] 3] eq {connected}
|
||||
} else {
|
||||
fail "Slave still not connected after some time"
|
||||
}
|
||||
|
||||
set retry 10
|
||||
while {$retry && ([$master debug digest] ne [$slave debug digest])}\
|
||||
{
|
||||
after 1000
|
||||
incr retry -1
|
||||
}
|
||||
assert {[$master dbsize] > 0}
|
||||
|
||||
if {[$master debug digest] ne [$slave debug digest]} {
|
||||
set csv1 [csvdump r]
|
||||
set csv2 [csvdump {r -1}]
|
||||
set fd [open /tmp/repldump1.txt w]
|
||||
puts -nonewline $fd $csv1
|
||||
close $fd
|
||||
set fd [open /tmp/repldump2.txt w]
|
||||
puts -nonewline $fd $csv2
|
||||
close $fd
|
||||
puts "Master - Replica inconsistency"
|
||||
puts "Run diff -u against /tmp/repldump*.txt for more info"
|
||||
}
|
||||
assert_equal [r debug digest] [r -1 debug digest]
|
||||
eval $cond
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach mdl {no yes} {
|
||||
foreach sdl {disabled swapdb} {
|
||||
test_psync {no reconnection, just sync} 6 1000000 3600 0 {
|
||||
} $mdl $sdl 0
|
||||
|
||||
test_psync {ok psync} 6 100000000 3600 0 {
|
||||
assert {[s -1 sync_partial_ok] > 0}
|
||||
} $mdl $sdl 1
|
||||
|
||||
test_psync {no backlog} 6 100 3600 0.5 {
|
||||
assert {[s -1 sync_partial_err] > 0}
|
||||
} $mdl $sdl 1
|
||||
|
||||
test_psync {ok after delay} 3 100000000 3600 3 {
|
||||
assert {[s -1 sync_partial_ok] > 0}
|
||||
} $mdl $sdl 1
|
||||
|
||||
test_psync {backlog expired} 3 100000000 1 3 {
|
||||
assert {[s -1 sync_partial_err] > 0}
|
||||
} $mdl $sdl 1
|
||||
}
|
||||
}
|
@ -47,6 +47,7 @@ set ::all_tests {
|
||||
integration/replication-3
|
||||
integration/replication-4
|
||||
integration/replication-psync
|
||||
integration/replication-psync-flash
|
||||
integration/replication-active
|
||||
integration/replication-multimaster
|
||||
integration/replication-multimaster-connect
|
||||
|
Loading…
x
Reference in New Issue
Block a user