From e258a1c0877d825c1675251620f817f5333c1446 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 26 May 2020 11:00:48 +0300 Subject: [PATCH 1/2] tests: each test client work on a distinct port range apparently when running tests in parallel (the default of --clients 16), there's a chance for two tests to use the same port. specifically, one test might shutdown a master and still have the replica up, and then another test will re-use the port number of master for another master, and then that replica will connect to the master of the other test. this can cause a master to count too many full syncs and fail a test if we run the tests with --single integration/psync2 --loop --stop see Probmem 2 in #7314 --- tests/instances.tcl | 3 ++- tests/support/server.tcl | 21 ++++++++++----------- tests/support/util.tcl | 8 ++++---- tests/test_helper.tcl | 30 +++++++++++++++++++++--------- tests/unit/other.tcl | 4 ++-- 5 files changed, 39 insertions(+), 27 deletions(-) diff --git a/tests/instances.tcl b/tests/instances.tcl index 0a0cbab12..3a4fadca0 100644 --- a/tests/instances.tcl +++ b/tests/instances.tcl @@ -25,6 +25,7 @@ set ::sentinel_instances {} set ::redis_instances {} set ::sentinel_base_port 20000 set ::redis_base_port 30000 +set ::redis_port_count 1024 set ::pids {} ; # We kill everything at exit set ::dirs {} ; # We remove all the temp dirs at exit set ::run_matching {} ; # If non empty, only tests matching pattern are run. @@ -57,7 +58,7 @@ proc exec_instance {type cfgfile} { # Spawn a redis or sentinel instance, depending on 'type'. proc spawn_instance {type base_port count {conf {}}} { for {set j 0} {$j < $count} {incr j} { - set port [find_available_port $base_port] + set port [find_available_port $base_port $::redis_port_count] incr base_port puts "Starting $type #$j at port $port" diff --git a/tests/support/server.tcl b/tests/support/server.tcl index d086366dc..146ebc72c 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -214,14 +214,14 @@ proc start_server {options {code undefined}} { dict set config dir [tmpdir server] # start every server on a different port - set ::port [find_available_port [expr {$::port+1}]] + set port [find_available_port $::baseport $::portcount] if {$::tls} { dict set config "port" 0 - dict set config "tls-port" $::port + dict set config "tls-port" $port dict set config "tls-cluster" "yes" dict set config "tls-replication" "yes" } else { - dict set config port $::port + dict set config port $port } set unixsocket [file normalize [format "%s/%s" [dict get $config "dir"] "socket"]] @@ -243,10 +243,10 @@ proc start_server {options {code undefined}} { set server_started 0 while {$server_started == 0} { if {$::verbose} { - puts -nonewline "=== ($tags) Starting server ${::host}:${::port} " + puts -nonewline "=== ($tags) Starting server ${::host}:${port} " } - send_data_packet $::test_server_fd "server-spawning" "port $::port" + send_data_packet $::test_server_fd "server-spawning" "port $port" if {$::valgrind} { set pid [exec valgrind --track-origins=yes --suppressions=src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full src/redis-server $config_file > $stdout 2> $stderr &] @@ -291,19 +291,19 @@ proc start_server {options {code undefined}} { # for availability. Other test clients may grab the port before we # are able to do it for example. if {$port_busy} { - puts "Port $::port was already busy, trying another port..." - set ::port [find_available_port [expr {$::port+1}]] + puts "Port $port was already busy, trying another port..." + set port [find_available_port $::baseport $::portcount] if {$::tls} { - dict set config "tls-port" $::port + dict set config "tls-port" $port } else { - dict set config port $::port + dict set config port $port } create_server_config_file $config_file $config continue; # Try again } if {$code ne "undefined"} { - set serverisup [server_is_up $::host $::port $retrynum] + set serverisup [server_is_up $::host $port $retrynum] } else { set serverisup 1 } @@ -324,7 +324,6 @@ proc start_server {options {code undefined}} { # setup properties to be able to initialize a client object set port_param [expr $::tls ? {"tls-port"} : {"port"}] set host $::host - set port $::port if {[dict exists $config bind]} { set host [dict get $config bind] } if {[dict exists $config $port_param]} { set port [dict get $config $port_param] } diff --git a/tests/support/util.tcl b/tests/support/util.tcl index 7ecf5b79c..a5ded67a3 100644 --- a/tests/support/util.tcl +++ b/tests/support/util.tcl @@ -344,8 +344,8 @@ proc roundFloat f { format "%.10g" $f } -proc find_available_port start { - for {set j $start} {$j < $start+1024} {incr j} { +proc find_available_port {start count} { + for {set j $start} {$j < $start+$count} {incr j} { if {[catch {set fd1 [socket 127.0.0.1 $j]}] && [catch {set fd2 [socket 127.0.0.1 [expr $j+10000]]}]} { return $j @@ -356,8 +356,8 @@ proc find_available_port start { } } } - if {$j == $start+1024} { - error "Can't find a non busy port in the $start-[expr {$start+1023}] range." + if {$j == $start+$count} { + error "Can't find a non busy port in the $start-[expr {$start+$count-1}] range." } } diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index de0a64728..7fc0e6f00 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -70,7 +70,9 @@ set ::all_tests { set ::next_test 0 set ::host 127.0.0.1 -set ::port 21111 +set ::port 6379; # port for external server +set ::baseport 21111; # initial port for spawned redis servers +set ::portcount 8000; # we don't wanna use more than 10000 to avoid collision with cluster bus ports set ::traceleaks 0 set ::valgrind 0 set ::tls 0 @@ -228,26 +230,26 @@ proc test_server_main {} { set tclsh [info nameofexecutable] # Open a listening socket, trying different ports in order to find a # non busy one. - set port [find_available_port 11111] + set clientport [find_available_port 11111 32] if {!$::quiet} { - puts "Starting test server at port $port" + puts "Starting test server at port $clientport" } - socket -server accept_test_clients -myaddr 127.0.0.1 $port + socket -server accept_test_clients -myaddr 127.0.0.1 $clientport # Start the client instances set ::clients_pids {} if {$::external} { set p [exec $tclsh [info script] {*}$::argv \ - --client $port --port $::port &] + --client $clientport &] lappend ::clients_pids $p } else { - set start_port [expr {$::port+100}] + set start_port $::baseport + set port_count [expr {$::portcount / $::numclients}] for {set j 0} {$j < $::numclients} {incr j} { - set start_port [find_available_port $start_port] set p [exec $tclsh [info script] {*}$::argv \ - --client $port --port $start_port &] + --client $clientport --baseport $start_port --portcount $port_count &] lappend ::clients_pids $p - incr start_port 10 + incr start_port $port_count } } @@ -510,6 +512,10 @@ proc print_help_screen {} { "--loop Execute the specified set of tests forever." "--wait-server Wait after server is started (so that you can attach a debugger)." "--tls Run tests in TLS mode." + "--host Run tests against an external host." + "--port TCP port to use against external host." + "--baseport Initial port number for spawned redis servers." + "--portcount Port range for spawned redis servers." "--help Print this help screen." } "\n"] } @@ -560,6 +566,12 @@ for {set j 0} {$j < [llength $argv]} {incr j} { } elseif {$opt eq {--port}} { set ::port $arg incr j + } elseif {$opt eq {--baseport}} { + set ::baseport $arg + incr j + } elseif {$opt eq {--portcount}} { + set ::portcount $arg + incr j } elseif {$opt eq {--accurate}} { set ::accurate 1 } elseif {$opt eq {--force-failure}} { diff --git a/tests/unit/other.tcl b/tests/unit/other.tcl index 7720c055a..20a32e795 100644 --- a/tests/unit/other.tcl +++ b/tests/unit/other.tcl @@ -167,9 +167,9 @@ start_server {tags {"other"}} { tags {protocol} { test {PIPELINING stresser (also a regression for the old epoll bug)} { if {$::tls} { - set fd2 [::tls::socket $::host $::port] + set fd2 [::tls::socket [srv host] [srv port]] } else { - set fd2 [socket $::host $::port] + set fd2 [socket [srv host] [srv port]] } fconfigure $fd2 -encoding binary -translation binary puts -nonewline $fd2 "SELECT 9\r\n" From 1cf33a46d588a8e2591665cfcf1c121cbe0f18d5 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 27 May 2020 16:12:30 +0300 Subject: [PATCH 2/2] tests: find_available_port start search from next port i.e. don't start the search from scratch hitting the used ones again. this will also reduce the likelihood of collisions (if there are any left) by increasing the time until we re-use a port we did use in the past. --- tests/support/util.tcl | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/support/util.tcl b/tests/support/util.tcl index a5ded67a3..8bec95374 100644 --- a/tests/support/util.tcl +++ b/tests/support/util.tcl @@ -344,21 +344,26 @@ proc roundFloat f { format "%.10g" $f } +set ::last_port_attempted 0 proc find_available_port {start count} { - for {set j $start} {$j < $start+$count} {incr j} { - if {[catch {set fd1 [socket 127.0.0.1 $j]}] && - [catch {set fd2 [socket 127.0.0.1 [expr $j+10000]]}]} { - return $j + set port [expr $::last_port_attempted + 1] + for {set attempts 0} {$attempts < $count} {incr attempts} { + if {$port < $start || $port >= $start+$count} { + set port $start + } + if {[catch {set fd1 [socket 127.0.0.1 $port]}] && + [catch {set fd2 [socket 127.0.0.1 [expr $port+10000]]}]} { + set ::last_port_attempted $port + return $port } else { catch { close $fd1 close $fd2 } } + incr port } - if {$j == $start+$count} { - error "Can't find a non busy port in the $start-[expr {$start+$count-1}] range." - } + error "Can't find a non busy port in the $start-[expr {$start+$count-1}] range." } # Test if TERM looks like to support colors