From 0dacd68163bd5454e0108e7dd02826624b775fb7 Mon Sep 17 00:00:00 2001
From: christianEQ <christian@eqalpha.com>
Date: Thu, 11 Nov 2021 01:41:28 +0000
Subject: [PATCH] added tests for flash psync

Former-commit-id: 79f7376a4fea2de9bd06c06d96f3f98ee7308874
---
 tests/integration/replication-psync-flash.tcl | 135 ++++++++++++++++++
 tests/test_helper.tcl                         |   1 +
 2 files changed, 136 insertions(+)
 create mode 100644 tests/integration/replication-psync-flash.tcl

diff --git a/tests/integration/replication-psync-flash.tcl b/tests/integration/replication-psync-flash.tcl
new file mode 100644
index 000000000..5cad127f3
--- /dev/null
+++ b/tests/integration/replication-psync-flash.tcl
@@ -0,0 +1,135 @@
+# Creates a master-slave pair and breaks the link continuously to force
+# partial resyncs attempts, all this while flooding the master with
+# write queries.
+#
+# You can specify backlog size, ttl, delay before reconnection, test duration
+# in seconds, and an additional condition to verify at the end.
+#
+# If reconnect is > 0, the test actually try to break the connection and
+# reconnect with the master, otherwise just the initial synchronization is
+# checked for consistency.
+proc test_psync {descr duration backlog_size backlog_ttl delay cond mdl sdl reconnect} {
+    start_server {tags {"repl"}} {
+        start_server [list tags {flash} overrides [list storage-provider {flash ./rocks.db} delete-on-evict no storage-flush-period 10]] {
+
+            set master [srv -1 client]
+            set master_host [srv -1 host]
+            set master_port [srv -1 port]
+            set slave [srv 0 client]
+
+            set load_handle0 [start_bg_complex_data $master_host $master_port 9 100000]
+            set load_handle1 [start_bg_complex_data $master_host $master_port 11 100000]
+            set load_handle2 [start_bg_complex_data $master_host $master_port 12 100000]
+
+            test {Slave should be able to synchronize with the master} {
+                $slave slaveof $master_host $master_port
+                wait_for_condition 50 100 {
+                    [lindex [r role] 0] eq {slave} &&
+                    [lindex [r role] 3] eq {connected}
+                } else {
+                    fail "Replication not started."
+                }
+            }
+
+            # Check that the background clients are actually writing.
+            test {Detect write load to master} {
+                wait_for_condition 50 1000 {
+                    [$master dbsize] > 100
+                } else {
+                    fail "Can't detect write load from background clients."
+                }
+            }
+
+            test "Test replication partial resync: $descr (diskless: $mdl, $sdl, reconnect: $reconnect)" {
+                # Now while the clients are writing data, break the maste-slave
+                # link multiple times.
+                if ($reconnect) {
+                    for {set j 0} {$j < $duration*10} {incr j} {
+                        after 100
+                        # catch {puts "MASTER [$master dbsize] keys, REPLICA [$slave dbsize] keys"}
+
+                        if {($j % 20) == 0} {
+                            catch {
+                                $slave debug restart
+                            }
+                        }
+                    }
+                }
+                stop_bg_complex_data $load_handle0
+                stop_bg_complex_data $load_handle1
+                stop_bg_complex_data $load_handle2
+
+                # Wait for the slave to reach the "online"
+                # state from the POV of the master.
+                set retry 5000
+                while {$retry} {
+                    set info [$master info]
+                    if {[string match {*slave0:*state=online*} $info]} {
+                        break
+                    } else {
+                        incr retry -1
+                        after 100
+                    }
+                }
+                if {$retry == 0} {
+                    error "assertion:Slave not correctly synchronized"
+                }
+
+                # Wait that slave acknowledge it is online so
+                # we are sure that DBSIZE and DEBUG DIGEST will not
+                # fail because of timing issues. (-LOADING error)
+                wait_for_condition 5000 100 {
+                    [lindex [$slave role] 3] eq {connected}
+                } else {
+                    fail "Slave still not connected after some time"
+                }  
+
+                set retry 10
+                while {$retry && ([$master debug digest] ne [$slave debug digest])}\
+                {
+                    after 1000
+                    incr retry -1
+                }
+                assert {[$master dbsize] > 0}
+
+                if {[$master debug digest] ne [$slave debug digest]} {
+                    set csv1 [csvdump r]
+                    set csv2 [csvdump {r -1}]
+                    set fd [open /tmp/repldump1.txt w]
+                    puts -nonewline $fd $csv1
+                    close $fd
+                    set fd [open /tmp/repldump2.txt w]
+                    puts -nonewline $fd $csv2
+                    close $fd
+                    puts "Master - Replica inconsistency"
+                    puts "Run diff -u against /tmp/repldump*.txt for more info"
+                }
+                assert_equal [r debug digest] [r -1 debug digest]
+                eval $cond
+            }
+        }
+    }
+}
+
+foreach mdl {no yes} {
+    foreach sdl {disabled swapdb} {
+        test_psync {no reconnection, just sync} 6 1000000 3600 0 {
+        } $mdl $sdl 0
+
+        test_psync {ok psync} 6 100000000 3600 0 {
+        assert {[s -1 sync_partial_ok] > 0}
+        } $mdl $sdl 1
+
+        test_psync {no backlog} 6 100 3600 0.5 {
+        assert {[s -1 sync_partial_err] > 0}
+        } $mdl $sdl 1
+
+        test_psync {ok after delay} 3 100000000 3600 3 {
+        assert {[s -1 sync_partial_ok] > 0}
+        } $mdl $sdl 1
+
+        test_psync {backlog expired} 3 100000000 1 3 {
+        assert {[s -1 sync_partial_err] > 0}
+        } $mdl $sdl 1
+    }
+}
diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 2e200c35a..2ac904d20 100644
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -47,6 +47,7 @@ set ::all_tests {
     integration/replication-3
     integration/replication-4
     integration/replication-psync
+    integration/replication-psync-flash
     integration/replication-active
     integration/replication-multimaster
     integration/replication-multimaster-connect