From e258a1c0877d825c1675251620f817f5333c1446 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 26 May 2020 11:00:48 +0300 Subject: [PATCH 1/5] tests: each test client work on a distinct port range apparently when running tests in parallel (the default of --clients 16), there's a chance for two tests to use the same port. specifically, one test might shutdown a master and still have the replica up, and then another test will re-use the port number of master for another master, and then that replica will connect to the master of the other test. this can cause a master to count too many full syncs and fail a test if we run the tests with --single integration/psync2 --loop --stop see Probmem 2 in #7314 --- tests/instances.tcl | 3 ++- tests/support/server.tcl | 21 ++++++++++----------- tests/support/util.tcl | 8 ++++---- tests/test_helper.tcl | 30 +++++++++++++++++++++--------- tests/unit/other.tcl | 4 ++-- 5 files changed, 39 insertions(+), 27 deletions(-) diff --git a/tests/instances.tcl b/tests/instances.tcl index 0a0cbab12..3a4fadca0 100644 --- a/tests/instances.tcl +++ b/tests/instances.tcl @@ -25,6 +25,7 @@ set ::sentinel_instances {} set ::redis_instances {} set ::sentinel_base_port 20000 set ::redis_base_port 30000 +set ::redis_port_count 1024 set ::pids {} ; # We kill everything at exit set ::dirs {} ; # We remove all the temp dirs at exit set ::run_matching {} ; # If non empty, only tests matching pattern are run. @@ -57,7 +58,7 @@ proc exec_instance {type cfgfile} { # Spawn a redis or sentinel instance, depending on 'type'. proc spawn_instance {type base_port count {conf {}}} { for {set j 0} {$j < $count} {incr j} { - set port [find_available_port $base_port] + set port [find_available_port $base_port $::redis_port_count] incr base_port puts "Starting $type #$j at port $port" diff --git a/tests/support/server.tcl b/tests/support/server.tcl index d086366dc..146ebc72c 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -214,14 +214,14 @@ proc start_server {options {code undefined}} { dict set config dir [tmpdir server] # start every server on a different port - set ::port [find_available_port [expr {$::port+1}]] + set port [find_available_port $::baseport $::portcount] if {$::tls} { dict set config "port" 0 - dict set config "tls-port" $::port + dict set config "tls-port" $port dict set config "tls-cluster" "yes" dict set config "tls-replication" "yes" } else { - dict set config port $::port + dict set config port $port } set unixsocket [file normalize [format "%s/%s" [dict get $config "dir"] "socket"]] @@ -243,10 +243,10 @@ proc start_server {options {code undefined}} { set server_started 0 while {$server_started == 0} { if {$::verbose} { - puts -nonewline "=== ($tags) Starting server ${::host}:${::port} " + puts -nonewline "=== ($tags) Starting server ${::host}:${port} " } - send_data_packet $::test_server_fd "server-spawning" "port $::port" + send_data_packet $::test_server_fd "server-spawning" "port $port" if {$::valgrind} { set pid [exec valgrind --track-origins=yes --suppressions=src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full src/redis-server $config_file > $stdout 2> $stderr &] @@ -291,19 +291,19 @@ proc start_server {options {code undefined}} { # for availability. Other test clients may grab the port before we # are able to do it for example. if {$port_busy} { - puts "Port $::port was already busy, trying another port..." - set ::port [find_available_port [expr {$::port+1}]] + puts "Port $port was already busy, trying another port..." + set port [find_available_port $::baseport $::portcount] if {$::tls} { - dict set config "tls-port" $::port + dict set config "tls-port" $port } else { - dict set config port $::port + dict set config port $port } create_server_config_file $config_file $config continue; # Try again } if {$code ne "undefined"} { - set serverisup [server_is_up $::host $::port $retrynum] + set serverisup [server_is_up $::host $port $retrynum] } else { set serverisup 1 } @@ -324,7 +324,6 @@ proc start_server {options {code undefined}} { # setup properties to be able to initialize a client object set port_param [expr $::tls ? {"tls-port"} : {"port"}] set host $::host - set port $::port if {[dict exists $config bind]} { set host [dict get $config bind] } if {[dict exists $config $port_param]} { set port [dict get $config $port_param] } diff --git a/tests/support/util.tcl b/tests/support/util.tcl index 7ecf5b79c..a5ded67a3 100644 --- a/tests/support/util.tcl +++ b/tests/support/util.tcl @@ -344,8 +344,8 @@ proc roundFloat f { format "%.10g" $f } -proc find_available_port start { - for {set j $start} {$j < $start+1024} {incr j} { +proc find_available_port {start count} { + for {set j $start} {$j < $start+$count} {incr j} { if {[catch {set fd1 [socket 127.0.0.1 $j]}] && [catch {set fd2 [socket 127.0.0.1 [expr $j+10000]]}]} { return $j @@ -356,8 +356,8 @@ proc find_available_port start { } } } - if {$j == $start+1024} { - error "Can't find a non busy port in the $start-[expr {$start+1023}] range." + if {$j == $start+$count} { + error "Can't find a non busy port in the $start-[expr {$start+$count-1}] range." } } diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index de0a64728..7fc0e6f00 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -70,7 +70,9 @@ set ::all_tests { set ::next_test 0 set ::host 127.0.0.1 -set ::port 21111 +set ::port 6379; # port for external server +set ::baseport 21111; # initial port for spawned redis servers +set ::portcount 8000; # we don't wanna use more than 10000 to avoid collision with cluster bus ports set ::traceleaks 0 set ::valgrind 0 set ::tls 0 @@ -228,26 +230,26 @@ proc test_server_main {} { set tclsh [info nameofexecutable] # Open a listening socket, trying different ports in order to find a # non busy one. - set port [find_available_port 11111] + set clientport [find_available_port 11111 32] if {!$::quiet} { - puts "Starting test server at port $port" + puts "Starting test server at port $clientport" } - socket -server accept_test_clients -myaddr 127.0.0.1 $port + socket -server accept_test_clients -myaddr 127.0.0.1 $clientport # Start the client instances set ::clients_pids {} if {$::external} { set p [exec $tclsh [info script] {*}$::argv \ - --client $port --port $::port &] + --client $clientport &] lappend ::clients_pids $p } else { - set start_port [expr {$::port+100}] + set start_port $::baseport + set port_count [expr {$::portcount / $::numclients}] for {set j 0} {$j < $::numclients} {incr j} { - set start_port [find_available_port $start_port] set p [exec $tclsh [info script] {*}$::argv \ - --client $port --port $start_port &] + --client $clientport --baseport $start_port --portcount $port_count &] lappend ::clients_pids $p - incr start_port 10 + incr start_port $port_count } } @@ -510,6 +512,10 @@ proc print_help_screen {} { "--loop Execute the specified set of tests forever." "--wait-server Wait after server is started (so that you can attach a debugger)." "--tls Run tests in TLS mode." + "--host Run tests against an external host." + "--port TCP port to use against external host." + "--baseport Initial port number for spawned redis servers." + "--portcount Port range for spawned redis servers." "--help Print this help screen." } "\n"] } @@ -560,6 +566,12 @@ for {set j 0} {$j < [llength $argv]} {incr j} { } elseif {$opt eq {--port}} { set ::port $arg incr j + } elseif {$opt eq {--baseport}} { + set ::baseport $arg + incr j + } elseif {$opt eq {--portcount}} { + set ::portcount $arg + incr j } elseif {$opt eq {--accurate}} { set ::accurate 1 } elseif {$opt eq {--force-failure}} { diff --git a/tests/unit/other.tcl b/tests/unit/other.tcl index 7720c055a..20a32e795 100644 --- a/tests/unit/other.tcl +++ b/tests/unit/other.tcl @@ -167,9 +167,9 @@ start_server {tags {"other"}} { tags {protocol} { test {PIPELINING stresser (also a regression for the old epoll bug)} { if {$::tls} { - set fd2 [::tls::socket $::host $::port] + set fd2 [::tls::socket [srv host] [srv port]] } else { - set fd2 [socket $::host $::port] + set fd2 [socket [srv host] [srv port]] } fconfigure $fd2 -encoding binary -translation binary puts -nonewline $fd2 "SELECT 9\r\n" From 1cf33a46d588a8e2591665cfcf1c121cbe0f18d5 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 27 May 2020 16:12:30 +0300 Subject: [PATCH 2/5] tests: find_available_port start search from next port i.e. don't start the search from scratch hitting the used ones again. this will also reduce the likelihood of collisions (if there are any left) by increasing the time until we re-use a port we did use in the past. --- tests/support/util.tcl | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/support/util.tcl b/tests/support/util.tcl index a5ded67a3..8bec95374 100644 --- a/tests/support/util.tcl +++ b/tests/support/util.tcl @@ -344,21 +344,26 @@ proc roundFloat f { format "%.10g" $f } +set ::last_port_attempted 0 proc find_available_port {start count} { - for {set j $start} {$j < $start+$count} {incr j} { - if {[catch {set fd1 [socket 127.0.0.1 $j]}] && - [catch {set fd2 [socket 127.0.0.1 [expr $j+10000]]}]} { - return $j + set port [expr $::last_port_attempted + 1] + for {set attempts 0} {$attempts < $count} {incr attempts} { + if {$port < $start || $port >= $start+$count} { + set port $start + } + if {[catch {set fd1 [socket 127.0.0.1 $port]}] && + [catch {set fd2 [socket 127.0.0.1 [expr $port+10000]]}]} { + set ::last_port_attempted $port + return $port } else { catch { close $fd1 close $fd2 } } + incr port } - if {$j == $start+$count} { - error "Can't find a non busy port in the $start-[expr {$start+$count-1}] range." - } + error "Can't find a non busy port in the $start-[expr {$start+$count-1}] range." } # Test if TERM looks like to support colors From 15bcb813d47dbfec343c7ed9ad328319c0bac989 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 27 May 2020 18:09:09 +0300 Subject: [PATCH 3/5] 32bit CI needs to build modules correctly --- .github/workflows/daily.yml | 4 +++- tests/modules/Makefile | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index c22d49895..acc4dd33a 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -47,7 +47,9 @@ jobs: sudo apt-get install tcl8.5 ./runtest --accurate --verbose - name: module api test - run: ./runtest-moduleapi --verbose + run: | + make -C tests/modules 32bit # the script below doesn't have an argument, we must build manually ahead of time + ./runtest-moduleapi --verbose test-tls: runs-on: ubuntu-latest diff --git a/tests/modules/Makefile b/tests/modules/Makefile index 363231a87..39b8e6efa 100644 --- a/tests/modules/Makefile +++ b/tests/modules/Makefile @@ -28,11 +28,14 @@ TEST_MODULES = \ all: $(TEST_MODULES) +32bit: + $(MAKE) CFLAGS="-m32" LDFLAGS="-melf_i386" + %.xo: %.c ../../src/redismodule.h $(CC) -I../../src $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@ %.so: %.xo - $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc + $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LDFLAGS) $(LIBS) -lc .PHONY: clean From 90f3856fd5a1c7d30bf4450772923bf4a16993b6 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 28 May 2020 08:21:24 +0300 Subject: [PATCH 4/5] revive meaningful offset tests --- tests/integration/psync2-pingoff.tcl | 212 +++++++++++++++++++++++++++ tests/test_helper.tcl | 1 + 2 files changed, 213 insertions(+) create mode 100644 tests/integration/psync2-pingoff.tcl diff --git a/tests/integration/psync2-pingoff.tcl b/tests/integration/psync2-pingoff.tcl new file mode 100644 index 000000000..5d8459a92 --- /dev/null +++ b/tests/integration/psync2-pingoff.tcl @@ -0,0 +1,212 @@ +# Test the meaningful offset implementation to make sure masters +# are able to PSYNC with replicas even if the replication stream +# has pending PINGs at the end. + +start_server {tags {"psync2"}} { +start_server {} { + # Config + set debug_msg 0 ; # Enable additional debug messages + + for {set j 0} {$j < 2} {incr j} { + set R($j) [srv [expr 0-$j] client] + set R_host($j) [srv [expr 0-$j] host] + set R_port($j) [srv [expr 0-$j] port] + $R($j) CONFIG SET repl-ping-replica-period 1 + if {$debug_msg} {puts "Log file: [srv [expr 0-$j] stdout]"} + } + + # Setup replication + test "PSYNC2 meaningful offset: setup" { + $R(1) replicaof $R_host(0) $R_port(0) + $R(0) set foo bar + wait_for_condition 50 1000 { + [status $R(1) master_link_status] == "up" && + [$R(0) dbsize] == 1 && [$R(1) dbsize] == 1 + } else { + fail "Replicas not replicating from master" + } + } + + test "PSYNC2 meaningful offset: write and wait replication" { + $R(0) INCR counter + $R(0) INCR counter + $R(0) INCR counter + wait_for_condition 50 1000 { + [$R(0) GET counter] eq [$R(1) GET counter] + } else { + fail "Master and replica don't agree about counter" + } + } + + # In this test we'll make sure the replica will get stuck, but with + # an active connection: this way the master will continue to send PINGs + # every second (we modified the PING period earlier) + test "PSYNC2 meaningful offset: pause replica and promote it" { + $R(1) MULTI + $R(1) DEBUG SLEEP 5 + $R(1) SLAVEOF NO ONE + $R(1) EXEC + $R(1) ping ; # Wait for it to return back available + } + + test "Make the old master a replica of the new one and check conditions" { + set sync_partial [status $R(1) sync_partial_ok] + assert {$sync_partial == 0} + $R(0) REPLICAOF $R_host(1) $R_port(1) + wait_for_condition 50 1000 { + [status $R(1) sync_partial_ok] == 1 + } else { + fail "The new master was not able to partial sync" + } + } +}} + + +start_server {tags {"psync2"}} { +start_server {} { +start_server {} { +start_server {} { +start_server {} { + test {pings at the end of replication stream are ignored for psync} { + set master [srv -4 client] + set master_host [srv -4 host] + set master_port [srv -4 port] + set replica1 [srv -3 client] + set replica2 [srv -2 client] + set replica3 [srv -1 client] + set replica4 [srv -0 client] + + $replica1 replicaof $master_host $master_port + $replica2 replicaof $master_host $master_port + $replica3 replicaof $master_host $master_port + $replica4 replicaof $master_host $master_port + wait_for_condition 50 1000 { + [status $master connected_slaves] == 4 + } else { + fail "replicas didn't connect" + } + + $master incr x + wait_for_condition 50 1000 { + [$replica1 get x] == 1 && [$replica2 get x] == 1 && + [$replica3 get x] == 1 && [$replica4 get x] == 1 + } else { + fail "replicas didn't get incr" + } + + # disconnect replica1 and replica2 + # and wait for the master to send a ping to replica3 and replica4 + $replica1 replicaof no one + $replica2 replicaof 127.0.0.1 1 ;# we can't promote it to master since that will cycle the replication id + $master config set repl-ping-replica-period 1 + after 1500 + + # make everyone sync from the replica1 that didn't get the last ping from the old master + # replica4 will keep syncing from the old master which now syncs from replica1 + # and replica2 will re-connect to the old master (which went back in time) + set new_master_host [srv -3 host] + set new_master_port [srv -3 port] + $replica3 replicaof $new_master_host $new_master_port + $master replicaof $new_master_host $new_master_port + $replica2 replicaof $master_host $master_port + wait_for_condition 50 1000 { + [status $replica2 master_link_status] == "up" && + [status $replica3 master_link_status] == "up" && + [status $replica4 master_link_status] == "up" && + [status $master master_link_status] == "up" + } else { + fail "replicas didn't connect" + } + + # make sure replication is still alive and kicking + $replica1 incr x + wait_for_condition 50 1000 { + [$replica2 get x] == 2 && + [$replica3 get x] == 2 && + [$replica4 get x] == 2 && + [$master get x] == 2 + } else { + fail "replicas didn't get incr" + } + + # make sure there are full syncs other than the initial ones + assert_equal [status $master sync_full] 4 + assert_equal [status $replica1 sync_full] 0 + assert_equal [status $replica2 sync_full] 0 + assert_equal [status $replica3 sync_full] 0 + assert_equal [status $replica4 sync_full] 0 + + # force psync + $master client kill type master + $replica2 client kill type master + $replica3 client kill type master + $replica4 client kill type master + + # make sure replication is still alive and kicking + $replica1 incr x + wait_for_condition 50 1000 { + [$replica2 get x] == 3 && + [$replica3 get x] == 3 && + [$replica4 get x] == 3 && + [$master get x] == 3 + } else { + fail "replicas didn't get incr" + } + + # make sure there are full syncs other than the initial ones + assert_equal [status $master sync_full] 4 + assert_equal [status $replica1 sync_full] 0 + assert_equal [status $replica2 sync_full] 0 + assert_equal [status $replica3 sync_full] 0 + assert_equal [status $replica4 sync_full] 0 +} +}}}}} + +start_server {tags {"psync2"}} { +start_server {} { +start_server {} { + + for {set j 0} {$j < 3} {incr j} { + set R($j) [srv [expr 0-$j] client] + set R_host($j) [srv [expr 0-$j] host] + set R_port($j) [srv [expr 0-$j] port] + $R($j) CONFIG SET repl-ping-replica-period 1 + } + + test "Chained replicas disconnect when replica re-connect with the same master" { + # Add a second replica as a chained replica of the current replica + $R(1) replicaof $R_host(0) $R_port(0) + $R(2) replicaof $R_host(1) $R_port(1) + wait_for_condition 50 1000 { + [status $R(2) master_link_status] == "up" + } else { + fail "Chained replica not replicating from its master" + } + + # Do a write on the master, and wait for 3 seconds for the master to + # send some PINGs to its replica + $R(0) INCR counter2 + after 2000 + set sync_partial_master [status $R(0) sync_partial_ok] + set sync_partial_replica [status $R(1) sync_partial_ok] + $R(0) CONFIG SET repl-ping-replica-period 100 + + # Disconnect the master's direct replica + $R(0) client kill type replica + wait_for_condition 50 1000 { + [status $R(1) master_link_status] == "up" && + [status $R(2) master_link_status] == "up" && + [status $R(0) sync_partial_ok] == $sync_partial_master + 1 && + [status $R(1) sync_partial_ok] == $sync_partial_replica + 1 + } else { + fail "Disconnected replica failed to PSYNC with master" + } + + # Verify that the replica and its replica's meaningful and real + # offsets match with the master + assert_equal [status $R(0) master_repl_offset] [status $R(1) master_repl_offset] + assert_equal [status $R(0) master_repl_offset] [status $R(2) master_repl_offset] + assert_equal [status $R(0) master_repl_meaningful_offset] [status $R(1) master_repl_meaningful_offset] + assert_equal [status $R(0) master_repl_meaningful_offset] [status $R(2) master_repl_meaningful_offset] + } +}}} diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 05e45b999..de0a64728 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -47,6 +47,7 @@ set ::all_tests { integration/logging integration/psync2 integration/psync2-reg + integration/psync2-pingoff unit/pubsub unit/slowlog unit/scripting From 2a8af8e675755f2bc4beed7bbe9bcfc86a41e958 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 28 May 2020 09:10:51 +0300 Subject: [PATCH 5/5] adjust revived meaningful offset tests these tests create several edge cases that are otherwise uncovered (at least not consistently) by the test suite, so although they're no longer testing what they were meant to test, it's still a good idea to keep them in hope that they'll expose some issue in the future. --- tests/integration/psync2-pingoff.tcl | 59 ++++++++++++++++++---------- 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/tests/integration/psync2-pingoff.tcl b/tests/integration/psync2-pingoff.tcl index 5d8459a92..5a9a46d16 100644 --- a/tests/integration/psync2-pingoff.tcl +++ b/tests/integration/psync2-pingoff.tcl @@ -1,6 +1,9 @@ -# Test the meaningful offset implementation to make sure masters -# are able to PSYNC with replicas even if the replication stream -# has pending PINGs at the end. +# These tests were added together with the meaningful offset implementation +# in redis 6.0.0, which was later abandoned in 6.0.4, they used to test that +# servers are able to PSYNC with replicas even if the replication stream has +# PINGs at the end which present in one sever and missing on another. +# We keep these tests just because they reproduce edge cases in the replication +# logic in hope they'll be able to spot some problem in the future. start_server {tags {"psync2"}} { start_server {} { @@ -16,7 +19,7 @@ start_server {} { } # Setup replication - test "PSYNC2 meaningful offset: setup" { + test "PSYNC2 pingoff: setup" { $R(1) replicaof $R_host(0) $R_port(0) $R(0) set foo bar wait_for_condition 50 1000 { @@ -27,7 +30,7 @@ start_server {} { } } - test "PSYNC2 meaningful offset: write and wait replication" { + test "PSYNC2 pingoff: write and wait replication" { $R(0) INCR counter $R(0) INCR counter $R(0) INCR counter @@ -41,7 +44,7 @@ start_server {} { # In this test we'll make sure the replica will get stuck, but with # an active connection: this way the master will continue to send PINGs # every second (we modified the PING period earlier) - test "PSYNC2 meaningful offset: pause replica and promote it" { + test "PSYNC2 pingoff: pause replica and promote it" { $R(1) MULTI $R(1) DEBUG SLEEP 5 $R(1) SLAVEOF NO ONE @@ -50,14 +53,22 @@ start_server {} { } test "Make the old master a replica of the new one and check conditions" { - set sync_partial [status $R(1) sync_partial_ok] - assert {$sync_partial == 0} + assert_equal [status $R(1) sync_full] 0 $R(0) REPLICAOF $R_host(1) $R_port(1) wait_for_condition 50 1000 { - [status $R(1) sync_partial_ok] == 1 + [status $R(1) sync_full] == 1 } else { - fail "The new master was not able to partial sync" + fail "The new master was not able to sync" } + + # make sure replication is still alive and kicking + $R(1) incr x + wait_for_condition 50 1000 { + [$R(0) get x] == 1 + } else { + fail "replica didn't get incr" + } + assert_equal [status $R(0) master_repl_offset] [status $R(1) master_repl_offset] } }} @@ -67,7 +78,7 @@ start_server {} { start_server {} { start_server {} { start_server {} { - test {pings at the end of replication stream are ignored for psync} { + test {test various edge cases of repl topology changes with missing pings at the end} { set master [srv -4 client] set master_host [srv -4 host] set master_port [srv -4 port] @@ -129,9 +140,9 @@ start_server {} { fail "replicas didn't get incr" } - # make sure there are full syncs other than the initial ones - assert_equal [status $master sync_full] 4 - assert_equal [status $replica1 sync_full] 0 + # make sure we have the right amount of full syncs + assert_equal [status $master sync_full] 6 + assert_equal [status $replica1 sync_full] 2 assert_equal [status $replica2 sync_full] 0 assert_equal [status $replica3 sync_full] 0 assert_equal [status $replica4 sync_full] 0 @@ -153,9 +164,9 @@ start_server {} { fail "replicas didn't get incr" } - # make sure there are full syncs other than the initial ones - assert_equal [status $master sync_full] 4 - assert_equal [status $replica1 sync_full] 0 + # make sure we have the right amount of full syncs + assert_equal [status $master sync_full] 6 + assert_equal [status $replica1 sync_full] 2 assert_equal [status $replica2 sync_full] 0 assert_equal [status $replica3 sync_full] 0 assert_equal [status $replica4 sync_full] 0 @@ -197,7 +208,7 @@ start_server {} { [status $R(1) master_link_status] == "up" && [status $R(2) master_link_status] == "up" && [status $R(0) sync_partial_ok] == $sync_partial_master + 1 && - [status $R(1) sync_partial_ok] == $sync_partial_replica + 1 + [status $R(1) sync_partial_ok] == $sync_partial_replica } else { fail "Disconnected replica failed to PSYNC with master" } @@ -206,7 +217,15 @@ start_server {} { # offsets match with the master assert_equal [status $R(0) master_repl_offset] [status $R(1) master_repl_offset] assert_equal [status $R(0) master_repl_offset] [status $R(2) master_repl_offset] - assert_equal [status $R(0) master_repl_meaningful_offset] [status $R(1) master_repl_meaningful_offset] - assert_equal [status $R(0) master_repl_meaningful_offset] [status $R(2) master_repl_meaningful_offset] + + # make sure replication is still alive and kicking + $R(0) incr counter2 + wait_for_condition 50 1000 { + [$R(1) get counter2] == 2 && [$R(2) get counter2] == 2 + } else { + fail "replicas didn't get incr" + } + assert_equal [status $R(0) master_repl_offset] [status $R(1) master_repl_offset] + assert_equal [status $R(0) master_repl_offset] [status $R(2) master_repl_offset] } }}}