From 3adcc6efc7feb4693d7b6f68c7e6a3d83e7826c9 Mon Sep 17 00:00:00 2001 From: benschermel Date: Tue, 16 Mar 2021 01:57:26 +0000 Subject: [PATCH 01/32] re-label keydb-pro-server to keydb-server Former-commit-id: eb6f61ad372f0f4e573b9b731eac6c7f2ada0d9f --- .gitignore | 6 +++--- README.md | 12 ++++++------ deps/hiredis/Makefile | 4 ++-- src/Makefile | 4 ++-- src/asciilogo.h | 2 +- src/debug.cpp | 2 +- src/keydb-pro-server.REMOVED.git-id | 1 + src/server.cpp | 26 +++++++++++++------------- tests/instances.tcl | 2 +- tests/support/server.tcl | 6 +++--- tests/support/test.tcl | 2 +- utils/create-cluster/create-cluster | 2 +- utils/install_server.sh | 5 +++-- utils/redis_init_script | 2 +- utils/speed-regression.tcl | 4 ++-- 15 files changed, 41 insertions(+), 39 deletions(-) create mode 100644 src/keydb-pro-server.REMOVED.git-id diff --git a/.gitignore b/.gitignore index c29512367..f85cfbc81 100644 --- a/.gitignore +++ b/.gitignore @@ -10,9 +10,9 @@ core !**/keydb-sentinel.service.d *.log dump.rdb -src/keydb-pro-server -**/bin/keydb-pro-server -**/app/keydb-pro-server +src/keydb-server +**/bin/keydb-server +**/app/keydb-server *.deb *.rpm src/keydb-cli diff --git a/README.md b/README.md index 5cd8c02b5..1decad4b0 100644 --- a/README.md +++ b/README.md @@ -185,19 +185,19 @@ Running KeyDB To run KeyDB with the default configuration, just type: % cd src - % ./keydb-pro-server + % ./keydb-server If you want to provide your keydb.conf, you have to run it using an additional parameter (the path of the configuration file): % cd src - % ./keydb-pro-server /path/to/keydb.conf + % ./keydb-server /path/to/keydb.conf It is possible to alter the KeyDB configuration by passing parameters directly as options using the command line. Examples: - % ./keydb-pro-server --port 9999 --replicaof 127.0.0.1 6379 - % ./keydb-pro-server /etc/keydb/6379.conf --loglevel debug + % ./keydb-server --port 9999 --replicaof 127.0.0.1 6379 + % ./keydb-server /etc/keydb/6379.conf --loglevel debug All the options in keydb.conf are also supported as options using the command line, with exactly the same name. @@ -213,7 +213,7 @@ how to use Redis with TLS. Playing with KeyDB ------------------ -You can use keydb-cli to play with KeyDB. Start a keydb-pro-server instance, +You can use keydb-cli to play with KeyDB. Start a keydb-server instance, then in another terminal try the following: % cd src @@ -278,7 +278,7 @@ Simply make a directory you would like to have the latest binaries dumped in, th ``` $ docker run -it --rm -v /path-to-dump-binaries:/keydb_bin eqalpha/keydb-build-bin ``` -You should receive the following files: keydb-benchmark, keydb-check-aof, keydb-check-rdb, keydb-cli, keydb-sentinel, keydb-pro-server +You should receive the following files: keydb-benchmark, keydb-check-aof, keydb-check-rdb, keydb-cli, keydb-sentinel, keydb-server If you are looking to enable flash support with the build (make MALLOC=memkind) then use the following command: ``` diff --git a/deps/hiredis/Makefile b/deps/hiredis/Makefile index c231a9b93..841990d5c 100644 --- a/deps/hiredis/Makefile +++ b/deps/hiredis/Makefile @@ -29,9 +29,9 @@ INSTALL_INCLUDE_PATH= $(DESTDIR)$(PREFIX)/$(INCLUDE_PATH) INSTALL_LIBRARY_PATH= $(DESTDIR)$(PREFIX)/$(LIBRARY_PATH) INSTALL_PKGCONF_PATH= $(INSTALL_LIBRARY_PATH)/$(PKGCONF_PATH) -# keydb-pro-server configuration used for testing +# keydb-server configuration used for testing REDIS_PORT=56379 -REDIS_SERVER=keydb-pro-server +REDIS_SERVER=keydb-server define REDIS_TEST_CONFIG daemonize yes pidfile /tmp/hiredis-test-redis.pid diff --git a/src/Makefile b/src/Makefile index d76255fd4..5f9458eb2 100644 --- a/src/Makefile +++ b/src/Makefile @@ -310,7 +310,7 @@ QUIET_LINK = @printf ' %b %b\n' $(LINKCOLOR)LINK$(ENDCOLOR) $(BINCOLOR)$@$(EN QUIET_INSTALL = @printf ' %b %b\n' $(LINKCOLOR)INSTALL$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR); endif -REDIS_SERVER_NAME=keydb-pro-server$(PROG_SUFFIX) +REDIS_SERVER_NAME=keydb-server$(PROG_SUFFIX) REDIS_SENTINEL_NAME=keydb-sentinel$(PROG_SUFFIX) REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o t_nhash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o acl.o storage.o rdb-s3.o fastlock.o new.o tracking.o cron.o connection.o tls.o sha256.o motd.o timeout.o setcpuaffinity.o AsyncWorkQueue.o snapshot.o storage/rocksdb.o storage/rocksdbfactory.o storage/teststorageprovider.o keydbutils.o StorageCache.o $(ASM_OBJ) REDIS_CLI_NAME=keydb-cli$(PROG_SUFFIX) @@ -368,7 +368,7 @@ else endif @touch $@ -# keydb-pro-server +# keydb-server $(REDIS_SERVER_NAME): $(REDIS_SERVER_OBJ) $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/lua/src/liblua.a ../deps/rocksdb/librocksdb.a $(FINAL_LIBS) diff --git a/src/asciilogo.h b/src/asciilogo.h index dbb3cfcd6..d4dfe6a77 100644 --- a/src/asciilogo.h +++ b/src/asciilogo.h @@ -32,7 +32,7 @@ const char *ascii_logo = " _ \n" " _-(+)-_ \n" " _-- / \\ --_ \n" -" _-- / \\ --_ KeyDB Pro %s (%s/%d) %s bit \n" +" _-- / \\ --_ KeyDB Enterprise %s (%s/%d) %s bit \n" " __-- / \\ --__ \n" " (+) _ / \\ _ (+) Running in %s mode\n" " | -- / \\ -- | Port: %d\n" diff --git a/src/debug.cpp b/src/debug.cpp index 23a7c9c02..cd5fe7909 100644 --- a/src/debug.cpp +++ b/src/debug.cpp @@ -1817,7 +1817,7 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) { "\n=== KEYDB BUG REPORT END. Make sure to include from START to END. ===\n\n" " Please report the crash by opening an issue on github:\n\n" " https://github.com/JohnSully/KeyDB/issues\n\n" -" Suspect RAM error? Use keydb-pro-server --test-memory to verify it.\n\n" +" Suspect RAM error? Use keydb-server --test-memory to verify it.\n\n" ); /* free(messages); Don't call free() with possibly corrupted memory. */ diff --git a/src/keydb-pro-server.REMOVED.git-id b/src/keydb-pro-server.REMOVED.git-id new file mode 100644 index 000000000..5116b94ce --- /dev/null +++ b/src/keydb-pro-server.REMOVED.git-id @@ -0,0 +1 @@ +39d678ef2074bab53ae57cbcd3ed3bf5ffba5aed \ No newline at end of file diff --git a/src/server.cpp b/src/server.cpp index 56c476aa8..d1700223c 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -5660,19 +5660,19 @@ void version(void) { } void usage(void) { - fprintf(stderr,"Usage: ./keydb-pro-server [/path/to/keydb.conf] [options]\n"); - fprintf(stderr," ./keydb-pro-server - (read config from stdin)\n"); - fprintf(stderr," ./keydb-pro-server -v or --version\n"); - fprintf(stderr," ./keydb-pro-server -h or --help\n"); - fprintf(stderr," ./keydb-pro-server --test-memory \n\n"); + fprintf(stderr,"Usage: ./keydb-server [/path/to/keydb.conf] [options]\n"); + fprintf(stderr," ./keydb-server - (read config from stdin)\n"); + fprintf(stderr," ./keydb-server -v or --version\n"); + fprintf(stderr," ./keydb-server -h or --help\n"); + fprintf(stderr," ./keydb-server --test-memory \n\n"); fprintf(stderr,"Examples:\n"); - fprintf(stderr," ./keydb-pro-server (run the server with default conf)\n"); - fprintf(stderr," ./keydb-pro-server /etc/redis/6379.conf\n"); - fprintf(stderr," ./keydb-pro-server --port 7777\n"); - fprintf(stderr," ./keydb-pro-server --port 7777 --replicaof 127.0.0.1 8888\n"); - fprintf(stderr," ./keydb-pro-server /etc/mykeydb.conf --loglevel verbose\n\n"); + fprintf(stderr," ./keydb-server (run the server with default conf)\n"); + fprintf(stderr," ./keydb-server /etc/redis/6379.conf\n"); + fprintf(stderr," ./keydb-server --port 7777\n"); + fprintf(stderr," ./keydb-server --port 7777 --replicaof 127.0.0.1 8888\n"); + fprintf(stderr," ./keydb-server /etc/mykeydb.conf --loglevel verbose\n\n"); fprintf(stderr,"Sentinel mode:\n"); - fprintf(stderr," ./keydb-pro-server /etc/sentinel.conf --sentinel\n"); + fprintf(stderr," ./keydb-server /etc/sentinel.conf --sentinel\n"); exit(1); } @@ -5717,7 +5717,7 @@ void redisAsciiArt(void) { if (cserver.license_key == nullptr && !g_pserver->sentinel_mode) { #ifndef NO_LICENSE_CHECK - serverLog(LL_WARNING, "!!!! KeyDB Pro is being run in trial mode !!!!"); + serverLog(LL_WARNING, "!!!! KeyDB Enterprise is being run in trial mode !!!!"); serverLog(LL_WARNING, "!!!! Execution will terminate in %d minutes !!!!", cserver.trial_timeout); #endif } @@ -6260,7 +6260,7 @@ int main(int argc, char **argv) { exit(0); } else { fprintf(stderr,"Please specify the amount of memory to test in megabytes.\n"); - fprintf(stderr,"Example: ./keydb-pro-server --test-memory 4096\n\n"); + fprintf(stderr,"Example: ./keydb-server --test-memory 4096\n\n"); exit(1); } } diff --git a/tests/instances.tcl b/tests/instances.tcl index d4f3581a2..99d3984c7 100644 --- a/tests/instances.tcl +++ b/tests/instances.tcl @@ -41,7 +41,7 @@ if {[catch {cd tmp}]} { # the provided configuration file. Returns the PID of the process. proc exec_instance {type dirname cfgfile} { if {$type eq "redis"} { - set prgname keydb-pro-server + set prgname keydb-server } elseif {$type eq "sentinel"} { set prgname keydb-sentinel } else { diff --git a/tests/support/server.tcl b/tests/support/server.tcl index c83517a19..cb21f0b9c 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -178,11 +178,11 @@ proc create_server_config_file {filename config} { proc spawn_server {config_file stdout stderr} { if {$::valgrind} { - set pid [exec valgrind --track-origins=yes --trace-children=yes --suppressions=[pwd]/src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full src/keydb-pro-server $config_file >> $stdout 2>> $stderr &] + set pid [exec valgrind --track-origins=yes --trace-children=yes --suppressions=[pwd]/src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full src/keydb-server $config_file >> $stdout 2>> $stderr &] } elseif ($::stack_logging) { - set pid [exec /usr/bin/env MallocStackLogging=1 MallocLogFile=/tmp/malloc_log.txt src/keydb-pro-server $config_file >> $stdout 2>> $stderr &] + set pid [exec /usr/bin/env MallocStackLogging=1 MallocLogFile=/tmp/malloc_log.txt src/keydb-server $config_file >> $stdout 2>> $stderr &] } else { - set pid [exec src/keydb-pro-server $config_file >> $stdout 2>> $stderr &] + set pid [exec src/keydb-server $config_file >> $stdout 2>> $stderr &] } if {$::wait_server} { diff --git a/tests/support/test.tcl b/tests/support/test.tcl index 65100269e..79acb6d5e 100644 --- a/tests/support/test.tcl +++ b/tests/support/test.tcl @@ -187,7 +187,7 @@ proc test {name code {okpattern undefined} {options undefined}} { } if {$::traceleaks} { - set output [exec leaks keydb-pro-server] + set output [exec leaks keydb-server] if {![string match {*0 leaks*} $output]} { send_data_packet $::test_server_fd err "Detected a memory leak in test '$name': $output" } diff --git a/utils/create-cluster/create-cluster b/utils/create-cluster/create-cluster index c07a45963..4733f0ccb 100755 --- a/utils/create-cluster/create-cluster +++ b/utils/create-cluster/create-cluster @@ -26,7 +26,7 @@ then while [ $((PORT < ENDPORT)) != "0" ]; do PORT=$((PORT+1)) echo "Starting $PORT" - $BIN_PATH/keydb-pro-server --port $PORT --protected-mode $PROTECTED_MODE --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes ${ADDITIONAL_OPTIONS} + $BIN_PATH/keydb-server --port $PORT --protected-mode $PROTECTED_MODE --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes ${ADDITIONAL_OPTIONS} done exit 0 fi diff --git a/utils/install_server.sh b/utils/install_server.sh index 2e3ee9d4a..ee809b82b 100755 --- a/utils/install_server.sh +++ b/utils/install_server.sh @@ -1,3 +1,4 @@ + #!/bin/sh # Copyright 2011 Dvir Volk . All rights reserved. @@ -37,7 +38,7 @@ # REDIS_CONFIG_FILE=/etc/redis/1234.conf \ # REDIS_LOG_FILE=/var/log/redis_1234.log \ # REDIS_DATA_DIR=/var/lib/redis/1234 \ -# REDIS_EXECUTABLE=`command -v keydb-pro-server` ./utils/install_server.sh +# REDIS_EXECUTABLE=`command -v keydb-server` ./utils/install_server.sh # # This generates a redis config file and an /etc/init.d script, and installs them. # @@ -129,7 +130,7 @@ fi if [ ! -x "$REDIS_EXECUTABLE" ] ; then _MANUAL_EXECUTION=true #get the redis executable path - _REDIS_EXECUTABLE=`command -v keydb-pro-server` + _REDIS_EXECUTABLE=`command -v keydb-server` read -p "Please select the redis executable path [$_REDIS_EXECUTABLE] " REDIS_EXECUTABLE if [ ! -x "$REDIS_EXECUTABLE" ] ; then REDIS_EXECUTABLE=$_REDIS_EXECUTABLE diff --git a/utils/redis_init_script b/utils/redis_init_script index 589792ee3..bee5545ef 100755 --- a/utils/redis_init_script +++ b/utils/redis_init_script @@ -12,7 +12,7 @@ ### END INIT INFO REDISPORT=6379 -EXEC=/usr/local/bin/keydb-pro-server +EXEC=/usr/local/bin/keydb-server CLIEXEC=/usr/local/bin/keydb-cli PIDFILE=/var/run/redis_${REDISPORT}.pid diff --git a/utils/speed-regression.tcl b/utils/speed-regression.tcl index 073f01a19..8d5220c75 100755 --- a/utils/speed-regression.tcl +++ b/utils/speed-regression.tcl @@ -27,8 +27,8 @@ proc run-tests branches { } # Start the Redis server - puts " starting the server... [exec ./keydb-pro-server -v]" - set pids [exec echo "port $::port\nloglevel warning\n" | ./keydb-pro-server - > /dev/null 2> /dev/null &] + puts " starting the server... [exec ./keydb-server -v]" + set pids [exec echo "port $::port\nloglevel warning\n" | ./keydb-server - > /dev/null 2> /dev/null &] puts " pids: $pids" after 1000 puts " running the benchmark" From 05b2d1e7984b3128624b448f773582d6985a8071 Mon Sep 17 00:00:00 2001 From: benschermel Date: Wed, 17 Mar 2021 02:30:34 +0000 Subject: [PATCH 02/32] update all packaging to support enterprise dist Former-commit-id: 1fd4f27cd271090fc66dba3b09a3800bf1c33b0e --- keydb.conf | 2 +- pkg/deb/deb-buildsource.sh | 10 ++-- .../debian/bin/generate-systemd-service-files | 12 +--- pkg/deb/debian/control | 23 ++++---- pkg/deb/debian/copyright | 4 +- ...sentinel.1 => keydb-enterprise-sentinel.1} | 0 ...tall => keydb-enterprise-sentinel.install} | 0 ...te => keydb-enterprise-sentinel.logrotate} | 0 .../debian/keydb-enterprise-sentinel.manpages | 1 + ...nst => keydb-enterprise-sentinel.postinst} | 0 ...ostrm => keydb-enterprise-sentinel.postrm} | 0 ...pro-server.1 => keydb-enterprise-server.1} | 0 ...rver.docs => keydb-enterprise-server.docs} | 0 ...nstall => keydb-enterprise-server.install} | 0 ...tate => keydb-enterprise-server.logrotate} | 0 .../debian/keydb-enterprise-server.manpages | 1 + ...tinst => keydb-enterprise-server.postinst} | 0 ....postrm => keydb-enterprise-server.postrm} | 0 ...amples => keydb-enterprise-tools.examples} | 0 .../keydb-enterprise-tools.install} | 2 +- ...npages => keydb-enterprise-tools.manpages} | 0 ...stinst => keydb-enterprise-tools.postinst} | 0 ...s.postrm => keydb-enterprise-tools.postrm} | 0 pkg/deb/debian/keydb-pro-sentinel.manpages | 1 - pkg/deb/debian/keydb-pro-server.manpages | 1 - pkg/deb/debian/source/lintian-overrides | 2 +- pkg/deb/debian/watch | 2 +- pkg/deb/debian_dh9/control | 22 ++++---- pkg/deb/debian_dh9/copyright | 4 +- pkg/deb/debian_dh9/files | 1 - ...sentinel.1 => keydb-enterprise-sentinel.1} | 0 ...tall => keydb-enterprise-sentinel.install} | 0 ...te => keydb-enterprise-sentinel.logrotate} | 0 ... => keydb-enterprise-sentinel.maintscript} | 0 .../keydb-enterprise-sentinel.manpages | 1 + ...nst => keydb-enterprise-sentinel.postinst} | 0 ...ostrm => keydb-enterprise-sentinel.postrm} | 0 ....prerm => keydb-enterprise-sentinel.prerm} | 0 ...pro-server.1 => keydb-enterprise-server.1} | 0 ...rver.docs => keydb-enterprise-server.docs} | 0 ...nstall => keydb-enterprise-server.install} | 0 ...tate => keydb-enterprise-server.logrotate} | 0 ...pt => keydb-enterprise-server.maintscript} | 0 .../keydb-enterprise-server.manpages | 1 + ...tinst => keydb-enterprise-server.postinst} | 0 ....postrm => keydb-enterprise-server.postrm} | 0 ...er.prerm => keydb-enterprise-server.prerm} | 0 ...amples => keydb-enterprise-tools.examples} | 0 .../keydb-enterprise-tools.install} | 2 +- ...npages => keydb-enterprise-tools.manpages} | 0 ...stinst => keydb-enterprise-tools.postinst} | 0 ...s.postrm => keydb-enterprise-tools.postrm} | 0 .../debian_dh9/keydb-pro-sentinel.manpages | 1 - pkg/deb/debian_dh9/keydb-pro-server.manpages | 1 - pkg/deb/debian_dh9/source/include-binaries | 4 +- pkg/deb/debian_dh9/source/lintian-overrides | 2 +- pkg/deb/debian_dh9/watch | 2 +- pkg/deb/master_changelog | 55 +------------------ pkg/docker/Dockerfile | 2 +- pkg/rpm/generate_rpms.sh | 4 +- pkg/rpm/keydb_build/keydb.spec | 4 +- .../usr/lib/systemd/system/keydb.service | 2 +- 62 files changed, 55 insertions(+), 114 deletions(-) rename pkg/deb/debian/{keydb-pro-sentinel.1 => keydb-enterprise-sentinel.1} (100%) rename pkg/deb/debian/{keydb-pro-sentinel.install => keydb-enterprise-sentinel.install} (100%) rename pkg/deb/debian/{keydb-pro-sentinel.logrotate => keydb-enterprise-sentinel.logrotate} (100%) create mode 100644 pkg/deb/debian/keydb-enterprise-sentinel.manpages rename pkg/deb/debian/{keydb-pro-sentinel.postinst => keydb-enterprise-sentinel.postinst} (100%) rename pkg/deb/debian/{keydb-pro-sentinel.postrm => keydb-enterprise-sentinel.postrm} (100%) rename pkg/deb/debian/{keydb-pro-server.1 => keydb-enterprise-server.1} (100%) rename pkg/deb/debian/{keydb-pro-server.docs => keydb-enterprise-server.docs} (100%) rename pkg/deb/debian/{keydb-pro-server.install => keydb-enterprise-server.install} (100%) rename pkg/deb/debian/{keydb-pro-server.logrotate => keydb-enterprise-server.logrotate} (100%) create mode 100644 pkg/deb/debian/keydb-enterprise-server.manpages rename pkg/deb/debian/{keydb-pro-server.postinst => keydb-enterprise-server.postinst} (100%) rename pkg/deb/debian/{keydb-pro-server.postrm => keydb-enterprise-server.postrm} (100%) rename pkg/deb/debian/{keydb-pro-tools.examples => keydb-enterprise-tools.examples} (100%) rename pkg/deb/{debian_dh9/keydb-pro-tools.install => debian/keydb-enterprise-tools.install} (87%) rename pkg/deb/debian/{keydb-pro-tools.manpages => keydb-enterprise-tools.manpages} (100%) rename pkg/deb/debian/{keydb-pro-tools.postinst => keydb-enterprise-tools.postinst} (100%) rename pkg/deb/debian/{keydb-pro-tools.postrm => keydb-enterprise-tools.postrm} (100%) delete mode 100644 pkg/deb/debian/keydb-pro-sentinel.manpages delete mode 100644 pkg/deb/debian/keydb-pro-server.manpages delete mode 100644 pkg/deb/debian_dh9/files rename pkg/deb/debian_dh9/{keydb-pro-sentinel.1 => keydb-enterprise-sentinel.1} (100%) rename pkg/deb/debian_dh9/{keydb-pro-sentinel.install => keydb-enterprise-sentinel.install} (100%) rename pkg/deb/debian_dh9/{keydb-pro-sentinel.logrotate => keydb-enterprise-sentinel.logrotate} (100%) rename pkg/deb/debian_dh9/{keydb-pro-sentinel.maintscript => keydb-enterprise-sentinel.maintscript} (100%) create mode 100644 pkg/deb/debian_dh9/keydb-enterprise-sentinel.manpages rename pkg/deb/debian_dh9/{keydb-pro-sentinel.postinst => keydb-enterprise-sentinel.postinst} (100%) rename pkg/deb/debian_dh9/{keydb-pro-sentinel.postrm => keydb-enterprise-sentinel.postrm} (100%) rename pkg/deb/debian_dh9/{keydb-pro-sentinel.prerm => keydb-enterprise-sentinel.prerm} (100%) rename pkg/deb/debian_dh9/{keydb-pro-server.1 => keydb-enterprise-server.1} (100%) rename pkg/deb/debian_dh9/{keydb-pro-server.docs => keydb-enterprise-server.docs} (100%) rename pkg/deb/debian_dh9/{keydb-pro-server.install => keydb-enterprise-server.install} (100%) rename pkg/deb/debian_dh9/{keydb-pro-server.logrotate => keydb-enterprise-server.logrotate} (100%) rename pkg/deb/debian_dh9/{keydb-pro-server.maintscript => keydb-enterprise-server.maintscript} (100%) create mode 100644 pkg/deb/debian_dh9/keydb-enterprise-server.manpages rename pkg/deb/debian_dh9/{keydb-pro-server.postinst => keydb-enterprise-server.postinst} (100%) rename pkg/deb/debian_dh9/{keydb-pro-server.postrm => keydb-enterprise-server.postrm} (100%) rename pkg/deb/debian_dh9/{keydb-pro-server.prerm => keydb-enterprise-server.prerm} (100%) rename pkg/deb/debian_dh9/{keydb-pro-tools.examples => keydb-enterprise-tools.examples} (100%) rename pkg/deb/{debian/keydb-pro-tools.install => debian_dh9/keydb-enterprise-tools.install} (87%) rename pkg/deb/debian_dh9/{keydb-pro-tools.manpages => keydb-enterprise-tools.manpages} (100%) rename pkg/deb/debian_dh9/{keydb-pro-tools.postinst => keydb-enterprise-tools.postinst} (100%) rename pkg/deb/debian_dh9/{keydb-pro-tools.postrm => keydb-enterprise-tools.postrm} (100%) delete mode 100644 pkg/deb/debian_dh9/keydb-pro-sentinel.manpages delete mode 100644 pkg/deb/debian_dh9/keydb-pro-server.manpages diff --git a/keydb.conf b/keydb.conf index 48c321fdc..3874c5acd 100644 --- a/keydb.conf +++ b/keydb.conf @@ -3,7 +3,7 @@ # Note that in order to read the configuration file, KeyDB must be # started with the file path as first argument: # -# ./keydb-pro-server /path/to/keydb.conf +# ./keydb-server /path/to/keydb.conf # Note on units: when memory size is needed, it is possible to specify # it in the usual form of 1k 5GB 4M and so forth: diff --git a/pkg/deb/deb-buildsource.sh b/pkg/deb/deb-buildsource.sh index d51c179fb..ed055a7b6 100755 --- a/pkg/deb/deb-buildsource.sh +++ b/pkg/deb/deb-buildsource.sh @@ -19,11 +19,11 @@ elif [ "$distributor" == "Ubuntu" ]; then fi codename=$(lsb_release --codename --short) date=$(date +%a," "%d" "%b" "%Y" "%T) -pkg_name=keydb-pro-$majorv:$version$distname +pkg_name=keydb-enterprise-$majorv:$version$distname # create build tree cd ../../../ -tar -czvf keydb-pro_$version.orig.tar.gz --force-local KeyDB-Pro +tar -czvf keydb-enterprise_$version.orig.tar.gz --force-local KeyDB-Pro cd KeyDB-Pro/pkg/deb/ mkdir -p $pkg_name/tmp if [[ "$codename" == "xenial" ]] || [[ "$codename" == "stretch" ]]; then @@ -32,9 +32,9 @@ else cp -r debian $pkg_name/tmp fi cp master_changelog $pkg_name/tmp/debian/changelog -mv ../../../keydb-pro_$version.orig.tar.gz ./$pkg_name +mv ../../../keydb-enterprise_$version.orig.tar.gz ./$pkg_name cd $pkg_name/tmp -changelog_str="keydb-pro ($majorv:$version-$build$distname) $codename; urgency=medium\n\n * $version $changelog_comments \n\n -- Ben Schermel $date +0000\n\n" +changelog_str="keydb-enterprise ($majorv:$version-$build$distname) $codename; urgency=medium\n\n * $version $changelog_comments \n\n -- Ben Schermel $date +0000\n\n" if [ $# -eq 0 ]; then sed -i "1s/^/$changelog_str\n/" debian/changelog elif [ $# -eq 1 ] && [ "$1" != "None" ]; then @@ -61,4 +61,4 @@ sudo pbuilder --build *.dsc --logfile /mnt/pbuilderlog.log cp /var/cache/pbuilder/result/*$version*.deb ../deb_files_generated sudo pbuilder clean cd ../ -rm -rf $pkg_name +#rm -rf $pkg_name diff --git a/pkg/deb/debian/bin/generate-systemd-service-files b/pkg/deb/debian/bin/generate-systemd-service-files index 9ee38792e..4a3df9a12 100755 --- a/pkg/deb/debian/bin/generate-systemd-service-files +++ b/pkg/deb/debian/bin/generate-systemd-service-files @@ -25,28 +25,22 @@ do case "${BINARY}" in keydb-server) NAME="keydb" - BINARY_EXEC="keydb-server" - BINARY_NAME="keydb-pro-server" - TARGET_NAME="keydb-pro-server" ;; keydb-sentinel) NAME="sentinel" - BINARY_EXEC="keydb-sentinel" - BINARY_NAME="keydb-sentinel" - TARGET_NAME="keydb-pro-sentinel" ;; esac case "${MODE}" in default) EXTRA="Alias=${NAME}.service" - TARGET="debian/${TARGET_NAME}.${BINARY}.service" + TARGET="debian/${BINARY}.service" NAMESPACED="${NAME}" DESCRIPTION="Advanced key-value store" ;; templated) EXTRA="" - TARGET="debian/${TARGET_NAME}.${BINARY}@.service" + TARGET="debian/${BINARY}@.service" NAMESPACED="${NAME}-%i" DESCRIPTION="Advanced key-value store (%I)" ;; @@ -94,7 +88,7 @@ Documentation=https://docs.keydb.dev, man:${BINARY}(1) [Service] Type=forking -ExecStart=/usr/bin/${BINARY_NAME} /etc/keydb/${NAMESPACED}.conf +ExecStart=/usr/bin/${BINARY} /etc/keydb/${NAMESPACED}.conf ExecStop=/bin/kill -s TERM \$MAINPID PIDFile=/var/run/${NAMESPACED}/${BINARY}.pid TimeoutStopSec=0 diff --git a/pkg/deb/debian/control b/pkg/deb/debian/control index 5a46e1848..6354eaa85 100644 --- a/pkg/deb/debian/control +++ b/pkg/deb/debian/control @@ -1,4 +1,4 @@ -Source: keydb-pro +Source: keydb-enterprise Section: database Priority: optional Maintainer: Ben Schermel @@ -7,6 +7,7 @@ Build-Depends: dpkg-dev (>= 1.17.5), systemd, procps , + pkg-config , build-essential , tcl , tcl-dev , @@ -24,14 +25,14 @@ Build-Depends: libzstd-dev Standards-Version: 4.2.1 Homepage: https://docs.keydb.dev/ -Vcs-Git: https://github.com/JohnSully/KeyDB-Pro.git -Vcs-Browser: https://github.com/JohnSully/KeyDB-Pro +Vcs-Git: https://gitlab.eqalpha.com/keydb-dev/KeyDB-Pro.git +Vcs-Browser: https://gitlab.eqalpha.com/keydb-dev/KeyDB-Pro -Package: keydb-pro +Package: keydb-enterprise Architecture: all Depends: - keydb-pro-server (<< ${binary:Version}.1~), - keydb-pro-server (>= ${binary:Version}), + keydb-enterprise-server (<< ${binary:Version}.1~), + keydb-enterprise-server (>= ${binary:Version}), ${misc:Depends}, Description: Persistent key-value database with network interface (metapackage) keydb is a key-value database in a similar vein to memcache but the dataset @@ -42,11 +43,11 @@ Description: Persistent key-value database with network interface (metapackage) . This package depends on the keydb-server package. -Package: keydb-pro-sentinel +Package: keydb-enterprise-sentinel Architecture: any Depends: lsb-base (>= 3.2-14), - keydb-pro-tools (= ${binary:Version}), + keydb-enterprise-tools (= ${binary:Version}), ${misc:Depends}, Description: Persistent key-value database with network interface (monitoring) keydb is a key-value database in a similar vein to memcache but the dataset @@ -55,11 +56,11 @@ Description: Persistent key-value database with network interface (monitoring) . This package contains the keydb Sentinel monitoring software. -Package: keydb-pro-server +Package: keydb-enterprise-server Architecture: any Depends: lsb-base (>= 3.2-14), - keydb-pro-tools (= ${binary:Version}), + keydb-enterprise-tools (= ${binary:Version}), ${misc:Depends}, Description: Persistent key-value database with network interface keydb is a key-value database in a similar vein to memcache but the dataset @@ -68,7 +69,7 @@ Description: Persistent key-value database with network interface . The dataset is stored entirely in memory and periodically flushed to disk. -Package: keydb-pro-tools +Package: keydb-enterprise-tools Architecture: any Depends: adduser, diff --git a/pkg/deb/debian/copyright b/pkg/deb/debian/copyright index c7e2428e7..57591384f 100644 --- a/pkg/deb/debian/copyright +++ b/pkg/deb/debian/copyright @@ -1,7 +1,7 @@ Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Contact: John Sully -Upstream-Name: keydb-pro -Source: https://github.com/JohnSully/KeyDB-Pro +Upstream-Name: keydb-enterprise +Source: https://gitlab.eqalpha.com/keydb-dev/KeyDB-Pro Files: * Copyright: © 2006-2014 Salvatore Sanfilippo diff --git a/pkg/deb/debian/keydb-pro-sentinel.1 b/pkg/deb/debian/keydb-enterprise-sentinel.1 similarity index 100% rename from pkg/deb/debian/keydb-pro-sentinel.1 rename to pkg/deb/debian/keydb-enterprise-sentinel.1 diff --git a/pkg/deb/debian/keydb-pro-sentinel.install b/pkg/deb/debian/keydb-enterprise-sentinel.install similarity index 100% rename from pkg/deb/debian/keydb-pro-sentinel.install rename to pkg/deb/debian/keydb-enterprise-sentinel.install diff --git a/pkg/deb/debian/keydb-pro-sentinel.logrotate b/pkg/deb/debian/keydb-enterprise-sentinel.logrotate similarity index 100% rename from pkg/deb/debian/keydb-pro-sentinel.logrotate rename to pkg/deb/debian/keydb-enterprise-sentinel.logrotate diff --git a/pkg/deb/debian/keydb-enterprise-sentinel.manpages b/pkg/deb/debian/keydb-enterprise-sentinel.manpages new file mode 100644 index 000000000..036eda5a6 --- /dev/null +++ b/pkg/deb/debian/keydb-enterprise-sentinel.manpages @@ -0,0 +1 @@ +debian/keydb-enterprise-sentinel.1 diff --git a/pkg/deb/debian/keydb-pro-sentinel.postinst b/pkg/deb/debian/keydb-enterprise-sentinel.postinst similarity index 100% rename from pkg/deb/debian/keydb-pro-sentinel.postinst rename to pkg/deb/debian/keydb-enterprise-sentinel.postinst diff --git a/pkg/deb/debian/keydb-pro-sentinel.postrm b/pkg/deb/debian/keydb-enterprise-sentinel.postrm similarity index 100% rename from pkg/deb/debian/keydb-pro-sentinel.postrm rename to pkg/deb/debian/keydb-enterprise-sentinel.postrm diff --git a/pkg/deb/debian/keydb-pro-server.1 b/pkg/deb/debian/keydb-enterprise-server.1 similarity index 100% rename from pkg/deb/debian/keydb-pro-server.1 rename to pkg/deb/debian/keydb-enterprise-server.1 diff --git a/pkg/deb/debian/keydb-pro-server.docs b/pkg/deb/debian/keydb-enterprise-server.docs similarity index 100% rename from pkg/deb/debian/keydb-pro-server.docs rename to pkg/deb/debian/keydb-enterprise-server.docs diff --git a/pkg/deb/debian/keydb-pro-server.install b/pkg/deb/debian/keydb-enterprise-server.install similarity index 100% rename from pkg/deb/debian/keydb-pro-server.install rename to pkg/deb/debian/keydb-enterprise-server.install diff --git a/pkg/deb/debian/keydb-pro-server.logrotate b/pkg/deb/debian/keydb-enterprise-server.logrotate similarity index 100% rename from pkg/deb/debian/keydb-pro-server.logrotate rename to pkg/deb/debian/keydb-enterprise-server.logrotate diff --git a/pkg/deb/debian/keydb-enterprise-server.manpages b/pkg/deb/debian/keydb-enterprise-server.manpages new file mode 100644 index 000000000..f66cf18d1 --- /dev/null +++ b/pkg/deb/debian/keydb-enterprise-server.manpages @@ -0,0 +1 @@ +debian/keydb-enterprise-server.1 diff --git a/pkg/deb/debian/keydb-pro-server.postinst b/pkg/deb/debian/keydb-enterprise-server.postinst similarity index 100% rename from pkg/deb/debian/keydb-pro-server.postinst rename to pkg/deb/debian/keydb-enterprise-server.postinst diff --git a/pkg/deb/debian/keydb-pro-server.postrm b/pkg/deb/debian/keydb-enterprise-server.postrm similarity index 100% rename from pkg/deb/debian/keydb-pro-server.postrm rename to pkg/deb/debian/keydb-enterprise-server.postrm diff --git a/pkg/deb/debian/keydb-pro-tools.examples b/pkg/deb/debian/keydb-enterprise-tools.examples similarity index 100% rename from pkg/deb/debian/keydb-pro-tools.examples rename to pkg/deb/debian/keydb-enterprise-tools.examples diff --git a/pkg/deb/debian_dh9/keydb-pro-tools.install b/pkg/deb/debian/keydb-enterprise-tools.install similarity index 87% rename from pkg/deb/debian_dh9/keydb-pro-tools.install rename to pkg/deb/debian/keydb-enterprise-tools.install index 346ad4597..166faeafd 100644 --- a/pkg/deb/debian_dh9/keydb-pro-tools.install +++ b/pkg/deb/debian/keydb-enterprise-tools.install @@ -3,5 +3,5 @@ src/keydb-benchmark /usr/bin src/keydb-check-aof /usr/bin src/keydb-check-rdb /usr/bin src/keydb-cli /usr/bin -src/keydb-pro-server /usr/bin +src/keydb-server /usr/bin src/keydb-sentinel /usr/bin diff --git a/pkg/deb/debian/keydb-pro-tools.manpages b/pkg/deb/debian/keydb-enterprise-tools.manpages similarity index 100% rename from pkg/deb/debian/keydb-pro-tools.manpages rename to pkg/deb/debian/keydb-enterprise-tools.manpages diff --git a/pkg/deb/debian/keydb-pro-tools.postinst b/pkg/deb/debian/keydb-enterprise-tools.postinst similarity index 100% rename from pkg/deb/debian/keydb-pro-tools.postinst rename to pkg/deb/debian/keydb-enterprise-tools.postinst diff --git a/pkg/deb/debian/keydb-pro-tools.postrm b/pkg/deb/debian/keydb-enterprise-tools.postrm similarity index 100% rename from pkg/deb/debian/keydb-pro-tools.postrm rename to pkg/deb/debian/keydb-enterprise-tools.postrm diff --git a/pkg/deb/debian/keydb-pro-sentinel.manpages b/pkg/deb/debian/keydb-pro-sentinel.manpages deleted file mode 100644 index 08ce51ff5..000000000 --- a/pkg/deb/debian/keydb-pro-sentinel.manpages +++ /dev/null @@ -1 +0,0 @@ -debian/keydb-pro-sentinel.1 diff --git a/pkg/deb/debian/keydb-pro-server.manpages b/pkg/deb/debian/keydb-pro-server.manpages deleted file mode 100644 index 06ff67ee3..000000000 --- a/pkg/deb/debian/keydb-pro-server.manpages +++ /dev/null @@ -1 +0,0 @@ -debian/keydb-pro-server.1 diff --git a/pkg/deb/debian/source/lintian-overrides b/pkg/deb/debian/source/lintian-overrides index 44dec9b63..ebc49b26e 100644 --- a/pkg/deb/debian/source/lintian-overrides +++ b/pkg/deb/debian/source/lintian-overrides @@ -1,2 +1,2 @@ # Upstream do not provide signed tarballs. -keydb-pro source: debian-watch-does-not-check-gpg-signature +keydb-enterprise source: debian-watch-does-not-check-gpg-signature diff --git a/pkg/deb/debian/watch b/pkg/deb/debian/watch index acca0f199..2dac38351 100644 --- a/pkg/deb/debian/watch +++ b/pkg/deb/debian/watch @@ -1,3 +1,3 @@ version=6 opts=uversionmangle=s/-?(alpha|beta|rc)/~$1/ \ - https://github.com/JohnSully/KeyDB-Pro/releases .*/archive/(.*).tar.gz + https://gitlab.eqalpha.com/keydb-dev/KeyDB-Pro/releases .*/archive/(.*).tar.gz diff --git a/pkg/deb/debian_dh9/control b/pkg/deb/debian_dh9/control index f847ad345..1932b27fe 100644 --- a/pkg/deb/debian_dh9/control +++ b/pkg/deb/debian_dh9/control @@ -1,4 +1,4 @@ -Source: keydb-pro +Source: keydb-enterprise Section: database Priority: optional Maintainer: Ben Schermel @@ -29,14 +29,14 @@ Build-Depends: libzstd-dev Standards-Version: 4.2.1 Homepage: https://docs.keydb.dev/ -Vcs-Git: https://github.com/JohnSully/KeyDB-Pro.git -Vcs-Browser: https://github.com/JohnSully/KeyDB-Pro +Vcs-Git: https://gitlab.eqalpha.com/keydb-dev/KeyDB-Pro.git +Vcs-Browser: https://gitlab.eqalpha.com/keydb-dev/KeyDB-Pro -Package: keydb-pro +Package: keydb-enterprise Architecture: all Depends: - keydb-pro-server (<< ${binary:Version}.1~), - keydb-pro-server (>= ${binary:Version}), + keydb-enterprise-server (<< ${binary:Version}.1~), + keydb-enterprise-server (>= ${binary:Version}), ${misc:Depends}, Description: Persistent key-value database with network interface (metapackage) keydb is a key-value database in a similar vein to memcache but the dataset @@ -47,11 +47,11 @@ Description: Persistent key-value database with network interface (metapackage) . This package depends on the keydb-server package. -Package: keydb-pro-sentinel +Package: keydb-enterprise-sentinel Architecture: any Depends: lsb-base (>= 3.2-14), - keydb-pro-tools (= ${binary:Version}), + keydb-enterprise-tools (= ${binary:Version}), ${misc:Depends}, Description: Persistent key-value database with network interface (monitoring) keydb is a key-value database in a similar vein to memcache but the dataset @@ -60,11 +60,11 @@ Description: Persistent key-value database with network interface (monitoring) . This package contains the keydb Sentinel monitoring software. -Package: keydb-pro-server +Package: keydb-enterprise-server Architecture: any Depends: lsb-base (>= 3.2-14), - keydb-pro-tools (= ${binary:Version}), + keydb-enterprise-tools (= ${binary:Version}), ${misc:Depends}, Description: Persistent key-value database with network interface keydb is a key-value database in a similar vein to memcache but the dataset @@ -73,7 +73,7 @@ Description: Persistent key-value database with network interface . The dataset is stored entirely in memory and periodically flushed to disk. -Package: keydb-pro-tools +Package: keydb-enterprise-tools Architecture: any Depends: adduser, diff --git a/pkg/deb/debian_dh9/copyright b/pkg/deb/debian_dh9/copyright index c7e2428e7..57591384f 100644 --- a/pkg/deb/debian_dh9/copyright +++ b/pkg/deb/debian_dh9/copyright @@ -1,7 +1,7 @@ Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Contact: John Sully -Upstream-Name: keydb-pro -Source: https://github.com/JohnSully/KeyDB-Pro +Upstream-Name: keydb-enterprise +Source: https://gitlab.eqalpha.com/keydb-dev/KeyDB-Pro Files: * Copyright: © 2006-2014 Salvatore Sanfilippo diff --git a/pkg/deb/debian_dh9/files b/pkg/deb/debian_dh9/files deleted file mode 100644 index 4fa490963..000000000 --- a/pkg/deb/debian_dh9/files +++ /dev/null @@ -1 +0,0 @@ -keydb-pro_5.3.3-1~bionic1_source.buildinfo database optional diff --git a/pkg/deb/debian_dh9/keydb-pro-sentinel.1 b/pkg/deb/debian_dh9/keydb-enterprise-sentinel.1 similarity index 100% rename from pkg/deb/debian_dh9/keydb-pro-sentinel.1 rename to pkg/deb/debian_dh9/keydb-enterprise-sentinel.1 diff --git a/pkg/deb/debian_dh9/keydb-pro-sentinel.install b/pkg/deb/debian_dh9/keydb-enterprise-sentinel.install similarity index 100% rename from pkg/deb/debian_dh9/keydb-pro-sentinel.install rename to pkg/deb/debian_dh9/keydb-enterprise-sentinel.install diff --git a/pkg/deb/debian_dh9/keydb-pro-sentinel.logrotate b/pkg/deb/debian_dh9/keydb-enterprise-sentinel.logrotate similarity index 100% rename from pkg/deb/debian_dh9/keydb-pro-sentinel.logrotate rename to pkg/deb/debian_dh9/keydb-enterprise-sentinel.logrotate diff --git a/pkg/deb/debian_dh9/keydb-pro-sentinel.maintscript b/pkg/deb/debian_dh9/keydb-enterprise-sentinel.maintscript similarity index 100% rename from pkg/deb/debian_dh9/keydb-pro-sentinel.maintscript rename to pkg/deb/debian_dh9/keydb-enterprise-sentinel.maintscript diff --git a/pkg/deb/debian_dh9/keydb-enterprise-sentinel.manpages b/pkg/deb/debian_dh9/keydb-enterprise-sentinel.manpages new file mode 100644 index 000000000..036eda5a6 --- /dev/null +++ b/pkg/deb/debian_dh9/keydb-enterprise-sentinel.manpages @@ -0,0 +1 @@ +debian/keydb-enterprise-sentinel.1 diff --git a/pkg/deb/debian_dh9/keydb-pro-sentinel.postinst b/pkg/deb/debian_dh9/keydb-enterprise-sentinel.postinst similarity index 100% rename from pkg/deb/debian_dh9/keydb-pro-sentinel.postinst rename to pkg/deb/debian_dh9/keydb-enterprise-sentinel.postinst diff --git a/pkg/deb/debian_dh9/keydb-pro-sentinel.postrm b/pkg/deb/debian_dh9/keydb-enterprise-sentinel.postrm similarity index 100% rename from pkg/deb/debian_dh9/keydb-pro-sentinel.postrm rename to pkg/deb/debian_dh9/keydb-enterprise-sentinel.postrm diff --git a/pkg/deb/debian_dh9/keydb-pro-sentinel.prerm b/pkg/deb/debian_dh9/keydb-enterprise-sentinel.prerm similarity index 100% rename from pkg/deb/debian_dh9/keydb-pro-sentinel.prerm rename to pkg/deb/debian_dh9/keydb-enterprise-sentinel.prerm diff --git a/pkg/deb/debian_dh9/keydb-pro-server.1 b/pkg/deb/debian_dh9/keydb-enterprise-server.1 similarity index 100% rename from pkg/deb/debian_dh9/keydb-pro-server.1 rename to pkg/deb/debian_dh9/keydb-enterprise-server.1 diff --git a/pkg/deb/debian_dh9/keydb-pro-server.docs b/pkg/deb/debian_dh9/keydb-enterprise-server.docs similarity index 100% rename from pkg/deb/debian_dh9/keydb-pro-server.docs rename to pkg/deb/debian_dh9/keydb-enterprise-server.docs diff --git a/pkg/deb/debian_dh9/keydb-pro-server.install b/pkg/deb/debian_dh9/keydb-enterprise-server.install similarity index 100% rename from pkg/deb/debian_dh9/keydb-pro-server.install rename to pkg/deb/debian_dh9/keydb-enterprise-server.install diff --git a/pkg/deb/debian_dh9/keydb-pro-server.logrotate b/pkg/deb/debian_dh9/keydb-enterprise-server.logrotate similarity index 100% rename from pkg/deb/debian_dh9/keydb-pro-server.logrotate rename to pkg/deb/debian_dh9/keydb-enterprise-server.logrotate diff --git a/pkg/deb/debian_dh9/keydb-pro-server.maintscript b/pkg/deb/debian_dh9/keydb-enterprise-server.maintscript similarity index 100% rename from pkg/deb/debian_dh9/keydb-pro-server.maintscript rename to pkg/deb/debian_dh9/keydb-enterprise-server.maintscript diff --git a/pkg/deb/debian_dh9/keydb-enterprise-server.manpages b/pkg/deb/debian_dh9/keydb-enterprise-server.manpages new file mode 100644 index 000000000..f66cf18d1 --- /dev/null +++ b/pkg/deb/debian_dh9/keydb-enterprise-server.manpages @@ -0,0 +1 @@ +debian/keydb-enterprise-server.1 diff --git a/pkg/deb/debian_dh9/keydb-pro-server.postinst b/pkg/deb/debian_dh9/keydb-enterprise-server.postinst similarity index 100% rename from pkg/deb/debian_dh9/keydb-pro-server.postinst rename to pkg/deb/debian_dh9/keydb-enterprise-server.postinst diff --git a/pkg/deb/debian_dh9/keydb-pro-server.postrm b/pkg/deb/debian_dh9/keydb-enterprise-server.postrm similarity index 100% rename from pkg/deb/debian_dh9/keydb-pro-server.postrm rename to pkg/deb/debian_dh9/keydb-enterprise-server.postrm diff --git a/pkg/deb/debian_dh9/keydb-pro-server.prerm b/pkg/deb/debian_dh9/keydb-enterprise-server.prerm similarity index 100% rename from pkg/deb/debian_dh9/keydb-pro-server.prerm rename to pkg/deb/debian_dh9/keydb-enterprise-server.prerm diff --git a/pkg/deb/debian_dh9/keydb-pro-tools.examples b/pkg/deb/debian_dh9/keydb-enterprise-tools.examples similarity index 100% rename from pkg/deb/debian_dh9/keydb-pro-tools.examples rename to pkg/deb/debian_dh9/keydb-enterprise-tools.examples diff --git a/pkg/deb/debian/keydb-pro-tools.install b/pkg/deb/debian_dh9/keydb-enterprise-tools.install similarity index 87% rename from pkg/deb/debian/keydb-pro-tools.install rename to pkg/deb/debian_dh9/keydb-enterprise-tools.install index 346ad4597..166faeafd 100644 --- a/pkg/deb/debian/keydb-pro-tools.install +++ b/pkg/deb/debian_dh9/keydb-enterprise-tools.install @@ -3,5 +3,5 @@ src/keydb-benchmark /usr/bin src/keydb-check-aof /usr/bin src/keydb-check-rdb /usr/bin src/keydb-cli /usr/bin -src/keydb-pro-server /usr/bin +src/keydb-server /usr/bin src/keydb-sentinel /usr/bin diff --git a/pkg/deb/debian_dh9/keydb-pro-tools.manpages b/pkg/deb/debian_dh9/keydb-enterprise-tools.manpages similarity index 100% rename from pkg/deb/debian_dh9/keydb-pro-tools.manpages rename to pkg/deb/debian_dh9/keydb-enterprise-tools.manpages diff --git a/pkg/deb/debian_dh9/keydb-pro-tools.postinst b/pkg/deb/debian_dh9/keydb-enterprise-tools.postinst similarity index 100% rename from pkg/deb/debian_dh9/keydb-pro-tools.postinst rename to pkg/deb/debian_dh9/keydb-enterprise-tools.postinst diff --git a/pkg/deb/debian_dh9/keydb-pro-tools.postrm b/pkg/deb/debian_dh9/keydb-enterprise-tools.postrm similarity index 100% rename from pkg/deb/debian_dh9/keydb-pro-tools.postrm rename to pkg/deb/debian_dh9/keydb-enterprise-tools.postrm diff --git a/pkg/deb/debian_dh9/keydb-pro-sentinel.manpages b/pkg/deb/debian_dh9/keydb-pro-sentinel.manpages deleted file mode 100644 index 08ce51ff5..000000000 --- a/pkg/deb/debian_dh9/keydb-pro-sentinel.manpages +++ /dev/null @@ -1 +0,0 @@ -debian/keydb-pro-sentinel.1 diff --git a/pkg/deb/debian_dh9/keydb-pro-server.manpages b/pkg/deb/debian_dh9/keydb-pro-server.manpages deleted file mode 100644 index 06ff67ee3..000000000 --- a/pkg/deb/debian_dh9/keydb-pro-server.manpages +++ /dev/null @@ -1 +0,0 @@ -debian/keydb-pro-server.1 diff --git a/pkg/deb/debian_dh9/source/include-binaries b/pkg/deb/debian_dh9/source/include-binaries index 55422f4bb..aa70b47d3 100644 --- a/pkg/deb/debian_dh9/source/include-binaries +++ b/pkg/deb/debian_dh9/source/include-binaries @@ -1,3 +1,3 @@ -keydb-pro_5.0.6.orig.tar.gz -keydb-pro_5.0.6-1~bionic1.debian.tar.xz +keydb-enterprise_5.0.6.orig.tar.gz +keydb-enterprise_5.0.6-1~bionic1.debian.tar.xz diff --git a/pkg/deb/debian_dh9/source/lintian-overrides b/pkg/deb/debian_dh9/source/lintian-overrides index 44dec9b63..ebc49b26e 100644 --- a/pkg/deb/debian_dh9/source/lintian-overrides +++ b/pkg/deb/debian_dh9/source/lintian-overrides @@ -1,2 +1,2 @@ # Upstream do not provide signed tarballs. -keydb-pro source: debian-watch-does-not-check-gpg-signature +keydb-enterprise source: debian-watch-does-not-check-gpg-signature diff --git a/pkg/deb/debian_dh9/watch b/pkg/deb/debian_dh9/watch index acca0f199..2dac38351 100644 --- a/pkg/deb/debian_dh9/watch +++ b/pkg/deb/debian_dh9/watch @@ -1,3 +1,3 @@ version=6 opts=uversionmangle=s/-?(alpha|beta|rc)/~$1/ \ - https://github.com/JohnSully/KeyDB-Pro/releases .*/archive/(.*).tar.gz + https://gitlab.eqalpha.com/keydb-dev/KeyDB-Pro/releases .*/archive/(.*).tar.gz diff --git a/pkg/deb/master_changelog b/pkg/deb/master_changelog index e92f8cc95..c5b91f9fa 100644 --- a/pkg/deb/master_changelog +++ b/pkg/deb/master_changelog @@ -1,58 +1,5 @@ -keydb-pro (6:6.0.12-1distribution_placeholder) codename_placeholder; urgency=medium +keydb-enterprise (6:6.0.12-1distribution_placeholder) codename_placeholder; urgency=medium * 6.0.12 Enable SCAN for MVCC -- Ben Schermel Fri, 10 Jul 2020 20:00:37 +0000 - -keydb-pro (6:6.0.11-1distribution_placeholder) codename_placeholder; urgency=medium - - * 6.0.11 fixes applied related to cluster usage and expires - - -- Ben Schermel Wed, 17 Jun 2020 20:00:37 +0000 - -keydb-pro (6:6.0.9-1distribution_placeholder) codename_placeholder; urgency=medium - - * 6.0.9 Addressing issues 187 - cpu lockup with subkey expire, 190 - missing sentinel binary in keydb-tools. - * FLASH performance updates, including updating client handling to mitigate SSD overload behavior on heavy writes. - * flash_memory parameter has been added to INFO memory - - -- Ben Schermel Sun, 07 Jun 2020 18:00:37 +0000 - -keydb-pro (6:6.0.8-1distribution_placeholder) codename_placeholder; urgency=medium - - * This is our first release to fully support all Redis 6.0.4 features, including but not limited to: TLS Support (fully supports multithreading!), Client side caching, RESP 3 Support - * KeyDB Has also added the following new features: Improved memory efficiency for short strings, Fastlock autotuning, KeyDB.HRENAME (rename a member of a hash)In addition we've spent a lot of time focussing on stability. - * The following bug fixes are resolved: #150 - Deadlock in ReplicationFeedMonitors, #170 - KeyDB dying via SIGABORT, #180 - crash after setting maxclients via cmd line, #169 - Write performance of a master dropped sharply when a slave is added - * In addition untracked issues were resolved: Potential deadlock when entering futex sleep, Pub/Sub Async messages may not be sent in a timely manner when load is low, Excessive logging during failed RREPLAY, KeyDB unresponsive handling clients on different threads during RDB load - * Updated deb package builds to build from source and phasing out init.d. dh_installsystemd from deb helper 11 used where applicable - * Naming conventions are now updated [ keydbpackage_version-build~distribution_architecture] and master changelog included and maintained - * keydb-pro-server binary is no longer inluded in the open source package - - -- Ben Schermel Mon, 1 Jun 2020 8:00:37 +0000 - -keydb-pro (5:5.1.12-1chl1distribution_placeholder) codename_placeholder; urgency=medium - - * 5.1.1 update. This update fixes several rare deadlock scenarios. Deadlock detection is also added. - - -- Ben Schermel Fri, 25 Oct 2019 8:00:37 +0000 - - -keydb-pro (5:5.1.11-1chl1distribution_placeholder) codename_placeholder; urgency=medium - - * 5.1 release. This release includes subkey expires (EXPIREMEMBER/EXPIREMEMBERAT), with updates to PTTL/TTL accordingly. New OBJECT LASTMODIFIED, BITIOP LSHIFT & BITOP RSHIFT commands. See https://docs.keydb.dev/blog/2019/10/20/blog-post/ for detailed review of release. - - -- Ben Schermel Mon, 21 Oct 2019 8:00:37 +0000 - - -keydb-pro (5:5.0.1-1chl1distribution_placeholder) codename_placeholder; urgency=medium - - * Arm build now included for bionic package - - -- Ben Schermel Wed, 21 Aug 2019 22:58:37 +0000 - - -keydb-pro (5:5.0.0-1chl1distribution_placeholder) codename_placeholder; urgency=medium - - * Initial release of KeyDB PPA. This PPA was originally derived from https://launchpad.net/~chris-lea/+archive/ubuntu/redis-server - - -- Ben Schermel Wed, 21 Aug 2019 2:58:37 +0000 diff --git a/pkg/docker/Dockerfile b/pkg/docker/Dockerfile index c71353b00..8528aabb5 100644 --- a/pkg/docker/Dockerfile +++ b/pkg/docker/Dockerfile @@ -88,4 +88,4 @@ ENTRYPOINT ["docker-entrypoint.sh"] # Expose container port and start KeyDB by default on running container EXPOSE 6379 -CMD ["keydb-pro-server", "/etc/keydb/keydb.conf"] +CMD ["keydb-server", "/etc/keydb/keydb.conf"] diff --git a/pkg/rpm/generate_rpms.sh b/pkg/rpm/generate_rpms.sh index 2f5959c89..93d2980f4 100755 --- a/pkg/rpm/generate_rpms.sh +++ b/pkg/rpm/generate_rpms.sh @@ -31,8 +31,8 @@ sed -i '3d' $DIR/keydb_build/keydb.spec sed -i -E "2a\Release : $release%{?dist}" $DIR/keydb_build/keydb.spec # yum install -y scl-utils centos-release-scl rpm-build -mkdir -p /root/rpmbuild/BUILDROOT/keydb-pro-$version-$release.$dist.$arch -cp -r $DIR/keydb_build/keydb_rpm/* /root/rpmbuild/BUILDROOT/keydb-pro-$version-$release.$dist.$arch/ +mkdir -p /root/rpmbuild/BUILDROOT/keydb-enterprise-$version-$release.$dist.$arch +cp -r $DIR/keydb_build/keydb_rpm/* /root/rpmbuild/BUILDROOT/keydb-enterprise-$version-$release.$dist.$arch/ rpmbuild -bb $DIR/keydb_build/keydb.spec mv /root/rpmbuild/RPMS/$arch/* $DIR/rpm_files_generated diff --git a/pkg/rpm/keydb_build/keydb.spec b/pkg/rpm/keydb_build/keydb.spec index 8c47c178c..8fd826937 100755 --- a/pkg/rpm/keydb_build/keydb.spec +++ b/pkg/rpm/keydb_build/keydb.spec @@ -1,4 +1,4 @@ -Name : keydb-pro +Name : keydb-enterprise Version : 6.0.5 Release : 1%{?dist} Group : Unspecified @@ -27,7 +27,7 @@ getent group keydb &> /dev/null || \ groupadd -r keydb &> /dev/null getent passwd keydb &> /dev/null || \ useradd -r -g keydb -d /var/lib/keydb -s /sbin/nologin \ --c 'KeyDB Professional Database Server' keydb &> /dev/null +-c 'KeyDB Enterprise Database Server' keydb &> /dev/null exit 0 #postinstall scriptlet (using /bin/sh): diff --git a/pkg/rpm/keydb_build/keydb_rpm/usr/lib/systemd/system/keydb.service b/pkg/rpm/keydb_build/keydb_rpm/usr/lib/systemd/system/keydb.service index 861c7a384..e36f5f2cf 100644 --- a/pkg/rpm/keydb_build/keydb_rpm/usr/lib/systemd/system/keydb.service +++ b/pkg/rpm/keydb_build/keydb_rpm/usr/lib/systemd/system/keydb.service @@ -5,7 +5,7 @@ Documentation=https://docs.keydb.dev, man:keydb-server(1) [Service] Type=forking -ExecStart=/usr/bin/keydb-pro-server /etc/keydb/keydb.conf +ExecStart=/usr/bin/keydb-server /etc/keydb/keydb.conf ExecStop=/bin/kill -s TERM $MAINPID PIDFile=/var/run/keydb/keydb-server.pid TimeoutStopSec=0 From d0e424039cdcd36363a527489d7493bee3a48f85 Mon Sep 17 00:00:00 2001 From: benschermel Date: Wed, 17 Mar 2021 05:11:01 +0000 Subject: [PATCH 03/32] fix rpm script for centos8 Former-commit-id: 6790d9b793b1995b2faae7dbfb5d09dce90a878c --- pkg/rpm/generate_rpms.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/rpm/generate_rpms.sh b/pkg/rpm/generate_rpms.sh index 93d2980f4..925509935 100755 --- a/pkg/rpm/generate_rpms.sh +++ b/pkg/rpm/generate_rpms.sh @@ -6,6 +6,9 @@ version=$(grep KEYDB_REAL_VERSION $DIR/../../src/version.h | awk '{ printf $3 }' release=1 # by default this will always be 1 for keydb version structure. If build release version needs to be update you can modify here arch=$(uname -m) dist=el$(rpm -q --queryformat '%{VERSION}' centos-release | cut -d. -f1) +if [[ "$dist" == "elpackage centos-release is not installed" ]]; then + dist=el$(rpm -q --queryformat '%{VERSION}' centos-linux-release | cut -d. -f1) +fi if [[ "$arch" != "aarch64" ]] && [[ "$arch" != "x86_64" ]]; then echo "This script is only valid and tested for aarch64 and x86_64 architectures. You are trying to use: $arch" From 3f9a80917a4796906c283e933b7da578ae8d08c0 Mon Sep 17 00:00:00 2001 From: benschermel Date: Thu, 18 Mar 2021 01:16:57 +0000 Subject: [PATCH 04/32] minor fixes Former-commit-id: ad7a27d41fb9e8e47966bbee72b28a4be66bbd8f --- pkg/deb/debian_dh9/bin/generate-systemd-service-files | 6 +----- src/keydb-pro-server.REMOVED.git-id | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) delete mode 100644 src/keydb-pro-server.REMOVED.git-id diff --git a/pkg/deb/debian_dh9/bin/generate-systemd-service-files b/pkg/deb/debian_dh9/bin/generate-systemd-service-files index 738e2f542..4a3df9a12 100755 --- a/pkg/deb/debian_dh9/bin/generate-systemd-service-files +++ b/pkg/deb/debian_dh9/bin/generate-systemd-service-files @@ -25,13 +25,9 @@ do case "${BINARY}" in keydb-server) NAME="keydb" - BINARY_EXEC="keydb-server" - BINARY_NAME="keydb-pro-server" ;; keydb-sentinel) NAME="sentinel" - BINARY_EXEC="keydb-sentinel" - BINARY_NAME="keydb-sentinel" ;; esac @@ -92,7 +88,7 @@ Documentation=https://docs.keydb.dev, man:${BINARY}(1) [Service] Type=forking -ExecStart=/usr/bin/${BINARY_NAME} /etc/keydb/${NAMESPACED}.conf +ExecStart=/usr/bin/${BINARY} /etc/keydb/${NAMESPACED}.conf ExecStop=/bin/kill -s TERM \$MAINPID PIDFile=/var/run/${NAMESPACED}/${BINARY}.pid TimeoutStopSec=0 diff --git a/src/keydb-pro-server.REMOVED.git-id b/src/keydb-pro-server.REMOVED.git-id deleted file mode 100644 index 5116b94ce..000000000 --- a/src/keydb-pro-server.REMOVED.git-id +++ /dev/null @@ -1 +0,0 @@ -39d678ef2074bab53ae57cbcd3ed3bf5ffba5aed \ No newline at end of file From 88e6c73906948b111506536117549e27f7f2afe8 Mon Sep 17 00:00:00 2001 From: benschermel Date: Sun, 21 Mar 2021 05:19:28 +0000 Subject: [PATCH 05/32] update enterprise wording for keydb.config lk Former-commit-id: feb2d31e6544763c94f4abaea69bc98bd1cb8b17 --- keydb.conf | 8 ++++---- pkg/deb/conf/keydb.conf | 8 ++++---- pkg/rpm/keydb_build/keydb_rpm/etc/keydb/keydb.conf | 8 ++++---- src/config.cpp | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/keydb.conf b/keydb.conf index 3874c5acd..50f9784a1 100644 --- a/keydb.conf +++ b/keydb.conf @@ -1828,9 +1828,9 @@ server-threads 2 # bringing up replicas can result in data loss (the first master will win). # active-replica yes -# Enable Pro? KeyDB pro provides support for pro only features -# note: you may omit the license key to demo pro features for a limited time -# enable-pro [License Key] +# Enable Enterprise? KeyDB Enterprise provides support for Enterprise only features +# note: you may omit the license key to demo Enterprise features for a limited time +# enable-enterprise [License Key] -# Enable FLASH support? (Pro Only) +# Enable FLASH support? (Enterprise Only) # storage-provider flash /path/to/flash/db diff --git a/pkg/deb/conf/keydb.conf b/pkg/deb/conf/keydb.conf index 9cf1773c7..821591b32 100644 --- a/pkg/deb/conf/keydb.conf +++ b/pkg/deb/conf/keydb.conf @@ -1568,9 +1568,9 @@ server-threads 2 # bringing up replicas can result in data loss (the first master will win). # active-replica yes -# Enable Pro? KeyDB pro provides support for pro only features -# note: you may omit the license key to demo pro features for a limited time -# enable-pro [License Key] +# Enable Enterprise? KeyDB Enterprise provides support for Enterprise only features +# note: you may omit the license key to demo Enterprise features for a limited time +# enable-enterprise [License Key] -# Enable FLASH support? (Pro Only) +# Enable FLASH support? (Enterprise Only) # storage-provider flash /path/to/flash/db diff --git a/pkg/rpm/keydb_build/keydb_rpm/etc/keydb/keydb.conf b/pkg/rpm/keydb_build/keydb_rpm/etc/keydb/keydb.conf index 9cf1773c7..821591b32 100644 --- a/pkg/rpm/keydb_build/keydb_rpm/etc/keydb/keydb.conf +++ b/pkg/rpm/keydb_build/keydb_rpm/etc/keydb/keydb.conf @@ -1568,9 +1568,9 @@ server-threads 2 # bringing up replicas can result in data loss (the first master will win). # active-replica yes -# Enable Pro? KeyDB pro provides support for pro only features -# note: you may omit the license key to demo pro features for a limited time -# enable-pro [License Key] +# Enable Enterprise? KeyDB Enterprise provides support for Enterprise only features +# note: you may omit the license key to demo Enterprise features for a limited time +# enable-enterprise [License Key] -# Enable FLASH support? (Pro Only) +# Enable FLASH support? (Enterprise Only) # storage-provider flash /path/to/flash/db diff --git a/src/config.cpp b/src/config.cpp index ca9181788..9a608e6f8 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -730,7 +730,7 @@ void loadServerConfigFromString(char *config) { g_sdsProvider = sdsdup(argv[1]); if (argc > 2) g_sdsArgs = sdsdup(argv[2]); - } else if (!strcasecmp(argv[0],"enable-pro") && (argc == 1 || argc == 2)) { + } else if (!strcasecmp(argv[0],"enable-enterprise") && (argc == 1 || argc == 2)) { if (argc == 2) { if (!FValidKey(argv[1], strlen(argv[1]))) { From cd48001c21cdc84068155f0b7842511a9931f4c2 Mon Sep 17 00:00:00 2001 From: benschermel Date: Tue, 23 Mar 2021 18:41:59 +0000 Subject: [PATCH 06/32] addressing comments from merge review Former-commit-id: 955aa69ca16ca7230d5099ad16f6f825402b5dd5 --- pkg/deb/deb-buildsource.sh | 2 +- src/config.cpp | 2 +- src/replication.cpp | 4 ++-- src/server.cpp | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/deb/deb-buildsource.sh b/pkg/deb/deb-buildsource.sh index ed055a7b6..b1109642b 100755 --- a/pkg/deb/deb-buildsource.sh +++ b/pkg/deb/deb-buildsource.sh @@ -61,4 +61,4 @@ sudo pbuilder --build *.dsc --logfile /mnt/pbuilderlog.log cp /var/cache/pbuilder/result/*$version*.deb ../deb_files_generated sudo pbuilder clean cd ../ -#rm -rf $pkg_name +rm -rf $pkg_name diff --git a/src/config.cpp b/src/config.cpp index 9a608e6f8..99505d444 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -1855,7 +1855,7 @@ int rewriteConfig(char *path, int force_all) { rewriteConfigClientoutputbufferlimitOption(state); rewriteConfigYesNoOption(state,"active-replica",g_pserver->fActiveReplica,CONFIG_DEFAULT_ACTIVE_REPLICA); rewriteConfigStringOption(state, "version-override",KEYDB_SET_VERSION,KEYDB_REAL_VERSION); - rewriteConfigStringOption(state, "enable-pro", cserver.license_key, CONFIG_DEFAULT_LICENSE_KEY); + rewriteConfigStringOption(state, "enable-enterprise", cserver.license_key, CONFIG_DEFAULT_LICENSE_KEY); rewriteConfigOOMScoreAdjValuesOption(state); /* Rewrite Sentinel config if in Sentinel mode. */ diff --git a/src/replication.cpp b/src/replication.cpp index 0589868cb..9f13dcda6 100644 --- a/src/replication.cpp +++ b/src/replication.cpp @@ -2747,8 +2747,8 @@ void syncWithMaster(connection *conn) { err = sendSynchronousCommand(mi, SYNC_CMD_READ,conn,NULL); if (err[0] == '-') { if (err[1] == 'E' && err[2] == 'R' && err[3] == 'R') { - // Replicating with non-pro - serverLog(LL_WARNING, "Replicating with non-pro server."); + // Replicating with non-enterprise + serverLog(LL_WARNING, "Replicating with non-enterprise server."); } else { serverLog(LL_WARNING, "Recieved error from client: %s", err); sdsfree(err); diff --git a/src/server.cpp b/src/server.cpp index d1700223c..70fb762c4 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -71,7 +71,7 @@ int g_fTestMode = false; const char *motd_url = "http://api.keydb.dev/motd/motd_server_pro.txt"; -const char *motd_cache_file = "/.keydb-pro-server-motd"; +const char *motd_cache_file = "/.keydb-enterprise-server-motd"; #ifdef __linux__ #include @@ -5426,7 +5426,7 @@ sds genRedisInfoString(const char *section) { if (sections++) info = sdscat(info,"\r\n"); info = sdscatprintf(info, "# KeyDB\r\n" - "variant:pro\r\n" + "variant:enterprise\r\n" "license_status:%s\r\n" "mvcc_depth:%d\r\n", cserver.license_key ? "OK" : "Trial", From 301e4f6a16b8a7c9ebaf9cde50527ed720e944e1 Mon Sep 17 00:00:00 2001 From: christianEQ Date: Wed, 24 Mar 2021 19:27:39 +0000 Subject: [PATCH 07/32] added condition variable to time thread; wake on afterSleep, sleep on beforeSleep Former-commit-id: ff2f2a3aceff2ba4a74951197348d67fc39568b2 --- src/server.cpp | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/server.cpp b/src/server.cpp index fbe0458e3..6c3f740af 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -61,6 +61,7 @@ #include #include #include +#include #include "aelocker.h" #include "keycheck.h" #include "motd.h" @@ -97,6 +98,10 @@ redisServer *g_pserver = &GlobalHidden::server; struct redisServerConst cserver; thread_local struct redisServerThreadVars *serverTL = NULL; // thread local server vars volatile unsigned long lru_clock; /* Server global current LRU time. */ +std::mutex time_thread_mutex; +std::condition_variable time_thread_cv; +bool time_thread_running = false; +void wakeTimeThread(); /* Our command table. * @@ -2509,6 +2514,8 @@ void beforeSleep(struct aeEventLoop *eventLoop) { serverAssert(g_pserver->repl_batch_offStart < 0); runAndPropogateToReplicas(processClients); + time_thread_running = false; + /* Handle precise timeouts of blocked clients. */ handleBlockedClientsTimeout(); @@ -2655,6 +2662,8 @@ void afterSleep(struct aeEventLoop *eventLoop) { /* Aquire the modules GIL so that their threads won't touch anything. */ if (moduleCount()) moduleAcquireGIL(TRUE /*fServerThread*/); + wakeTimeThread(); + serverAssert(serverTL->gcEpoch.isReset()); serverTL->gcEpoch = g_pserver->garbageCollector.startEpoch(); for (int idb = 0; idb < cserver.dbnum; ++idb) @@ -5731,6 +5740,8 @@ static void sigShutdownHandler(int sig) { msg = "Received shutdown signal, scheduling shutdown..."; }; + wakeTimeThread(); + /* SIGINT is often delivered via Ctrl+C in an interactive session. * If we receive the signal the second time, we interpret this as * the user really wanting to quit ASAP without waiting to persist @@ -6076,15 +6087,24 @@ void OnTerminate() serverPanic("std::teminate() called"); } +void wakeTimeThread() { + time_thread_running = true; + time_thread_cv.notify_one(); +} + void *timeThreadMain(void*) { timespec delay; delay.tv_sec = 0; delay.tv_nsec = 100; while (true) { + std::unique_lock lock(time_thread_mutex); + if (!time_thread_running) { + time_thread_cv.wait(lock); + } updateCachedTime(); clock_nanosleep(CLOCK_REALTIME, 0, &delay, NULL); } -} +} void *workerThreadMain(void *parg) { From ecf2d0aeb54bbb79530c6ab75c7740c007dd1cc1 Mon Sep 17 00:00:00 2001 From: christianEQ Date: Wed, 24 Mar 2021 20:24:07 +0000 Subject: [PATCH 08/32] removed extraneous wakeTimeThread cal Former-commit-id: cfb5c5d7dc1e09c51ca416a2dbb19eab4bbaa77a --- src/server.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/server.cpp b/src/server.cpp index 6c3f740af..0f503ece3 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -5740,8 +5740,6 @@ static void sigShutdownHandler(int sig) { msg = "Received shutdown signal, scheduling shutdown..."; }; - wakeTimeThread(); - /* SIGINT is often delivered via Ctrl+C in an interactive session. * If we receive the signal the second time, we interpret this as * the user really wanting to quit ASAP without waiting to persist From d71956e1ce7f1a65fd8f2fc8a6e337c6e49174b3 Mon Sep 17 00:00:00 2001 From: John Sully Date: Wed, 24 Mar 2021 19:58:51 +0000 Subject: [PATCH 09/32] Fix bug where we skip valid dict elements in dictGetRandomKey Former-commit-id: 626b56b00824573660af0c47b210fd1e8d2cfeb2 --- src/dict.cpp | 11 +++++++++-- src/dict.h | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/dict.cpp b/src/dict.cpp index c682e2ec9..831a32f8f 100644 --- a/src/dict.cpp +++ b/src/dict.cpp @@ -373,6 +373,7 @@ int dictRehash(dict *d, int n) { dictAsyncRehashCtl::dictAsyncRehashCtl(struct dict *d, dictAsyncRehashCtl *next) : dict(d), next(next) { queue.reserve(c_targetQueueSize); __atomic_fetch_add(&d->refcount, 1, __ATOMIC_RELEASE); + this->rehashIdxBase = d->rehashidx; } dictAsyncRehashCtl *dictRehashAsyncStart(dict *d, int buckets) { @@ -931,12 +932,18 @@ dictEntry *dictGetRandomKey(dict *d) if (dictSize(d) == 0) return NULL; if (dictIsRehashing(d)) _dictRehashStep(d); if (dictIsRehashing(d)) { + long rehashidx = d->rehashidx; + auto async = d->asyncdata; + while (async != nullptr) { + rehashidx = std::min((long)async->rehashIdxBase, rehashidx); + async = async->next; + } do { /* We are sure there are no elements in indexes from 0 * to rehashidx-1 */ - h = d->rehashidx + (random() % (d->ht[0].size + + h = rehashidx + (random() % (d->ht[0].size + d->ht[1].size - - d->rehashidx)); + rehashidx)); he = (h >= d->ht[0].size) ? d->ht[1].table[h - d->ht[0].size] : d->ht[0].table[h]; } while(he == NULL); diff --git a/src/dict.h b/src/dict.h index ab57a7d7f..f24108d32 100644 --- a/src/dict.h +++ b/src/dict.h @@ -100,6 +100,7 @@ struct dictAsyncRehashCtl { struct dict *dict = nullptr; std::vector queue; size_t hashIdx = 0; + long rehashIdxBase; dictAsyncRehashCtl *next = nullptr; std::atomic done { false }; std::atomic abondon { false }; From 9dde193c1dae35362a34610a39cfb98f3acf8e99 Mon Sep 17 00:00:00 2001 From: christianEQ Date: Wed, 24 Mar 2021 20:33:26 +0000 Subject: [PATCH 10/32] counter for time thread sleeping threads Former-commit-id: 6270939d6a2d7820fecac9c5aedf0ff985531f6b --- src/server.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/server.cpp b/src/server.cpp index 0f503ece3..7827879a5 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -100,7 +100,7 @@ thread_local struct redisServerThreadVars *serverTL = NULL; // thread local se volatile unsigned long lru_clock; /* Server global current LRU time. */ std::mutex time_thread_mutex; std::condition_variable time_thread_cv; -bool time_thread_running = false; +int sleeping_threads = cserver.cthreads; void wakeTimeThread(); /* Our command table. @@ -2514,7 +2514,9 @@ void beforeSleep(struct aeEventLoop *eventLoop) { serverAssert(g_pserver->repl_batch_offStart < 0); runAndPropogateToReplicas(processClients); - time_thread_running = false; + time_thread_mutex.lock(); + sleeping_threads++; + time_thread_mutex.unlock(); /* Handle precise timeouts of blocked clients. */ handleBlockedClientsTimeout(); @@ -6086,7 +6088,9 @@ void OnTerminate() } void wakeTimeThread() { - time_thread_running = true; + time_thread_mutex.lock(); + sleeping_threads--; + time_thread_mutex.unlock(); time_thread_cv.notify_one(); } @@ -6096,7 +6100,7 @@ void *timeThreadMain(void*) { delay.tv_nsec = 100; while (true) { std::unique_lock lock(time_thread_mutex); - if (!time_thread_running) { + if (sleeping_threads >= cserver.cthreads) { time_thread_cv.wait(lock); } updateCachedTime(); From ce8705451a44626b0d73ed8602aed27b1b0a41ae Mon Sep 17 00:00:00 2001 From: christianEQ Date: Wed, 24 Mar 2021 20:44:40 +0000 Subject: [PATCH 11/32] serverAsserts on sleeping_threads, remove bare locks Former-commit-id: 8c64ca2333d8eb2ee92d835907474ec63e127b62 --- src/server.cpp | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/server.cpp b/src/server.cpp index 7827879a5..26bf59b79 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -2514,9 +2514,11 @@ void beforeSleep(struct aeEventLoop *eventLoop) { serverAssert(g_pserver->repl_batch_offStart < 0); runAndPropogateToReplicas(processClients); - time_thread_mutex.lock(); - sleeping_threads++; - time_thread_mutex.unlock(); + { + std::lock_guard lock(time_thread_mutex); + sleeping_threads++; + serverAssert(sleeping_threads <= cserver.cthreads); + } /* Handle precise timeouts of blocked clients. */ handleBlockedClientsTimeout(); @@ -6088,9 +6090,9 @@ void OnTerminate() } void wakeTimeThread() { - time_thread_mutex.lock(); + std::lock_guard lock(time_thread_mutex); sleeping_threads--; - time_thread_mutex.unlock(); + serverAssert(sleeping_threads >= 0); time_thread_cv.notify_one(); } @@ -6099,9 +6101,11 @@ void *timeThreadMain(void*) { delay.tv_sec = 0; delay.tv_nsec = 100; while (true) { - std::unique_lock lock(time_thread_mutex); - if (sleeping_threads >= cserver.cthreads) { - time_thread_cv.wait(lock); + { + std::unique_lock lock(time_thread_mutex); + if (sleeping_threads >= cserver.cthreads) { + time_thread_cv.wait(lock); + } } updateCachedTime(); clock_nanosleep(CLOCK_REALTIME, 0, &delay, NULL); From de376f6b93d8b2980b8d3cdda21c31f96fe54cab Mon Sep 17 00:00:00 2001 From: christianEQ Date: Wed, 24 Mar 2021 21:16:21 +0000 Subject: [PATCH 12/32] moved time thread code to end of beforeSleep Former-commit-id: ac1022c772c7357571829f24c87aa3dc2deade72 --- src/server.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/server.cpp b/src/server.cpp index 26bf59b79..97a29f1a3 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -2514,12 +2514,6 @@ void beforeSleep(struct aeEventLoop *eventLoop) { serverAssert(g_pserver->repl_batch_offStart < 0); runAndPropogateToReplicas(processClients); - { - std::lock_guard lock(time_thread_mutex); - sleeping_threads++; - serverAssert(sleeping_threads <= cserver.cthreads); - } - /* Handle precise timeouts of blocked clients. */ handleBlockedClientsTimeout(); @@ -2651,6 +2645,13 @@ void beforeSleep(struct aeEventLoop *eventLoop) { locker.disarm(); if (!fSentReplies) handleClientsWithPendingWrites(iel, aof_state); + + { + std::lock_guard lock(time_thread_mutex); + sleeping_threads++; + serverAssert(sleeping_threads <= cserver.cthreads); + } + if (moduleCount()) moduleReleaseGIL(TRUE /*fServerThread*/); /* Do NOT add anything below moduleReleaseGIL !!! */ From 788ff61b4b0403de63c4a847f5bd10f68cef9627 Mon Sep 17 00:00:00 2001 From: John Sully Date: Thu, 25 Mar 2021 23:14:48 +0000 Subject: [PATCH 13/32] Prevent unnecessary copies in replication scenarios Former-commit-id: 84c671cdafae7e84beef722fcc8a90b92918c89d --- src/networking.cpp | 10 ++++++++++ src/replication.cpp | 3 +++ 2 files changed, 13 insertions(+) diff --git a/src/networking.cpp b/src/networking.cpp index 15aa6f43a..0774694ae 100644 --- a/src/networking.cpp +++ b/src/networking.cpp @@ -312,6 +312,16 @@ int prepareClientToWrite(client *c) { * Low level functions to add more data to output buffers. * -------------------------------------------------------------------------- */ +void _clientAsyncReplyBufferReserve(client *c, size_t len) { + if (c->replyAsync != nullptr) + return; + size_t newsize = std::max(len, (size_t)PROTO_ASYNC_REPLY_CHUNK_BYTES); + clientReplyBlock *replyNew = (clientReplyBlock*)zmalloc(sizeof(clientReplyBlock) + newsize); + replyNew->size = zmalloc_usable(replyNew) - sizeof(clientReplyBlock); + replyNew->used = 0; + c->replyAsync = replyNew; +} + /* Attempts to add the reply to the static buffer in the client struct. * Returns C_ERR if the buffer is full, or the reply list is not empty, * in which case the reply must be added to the reply list. */ diff --git a/src/replication.cpp b/src/replication.cpp index 9f13dcda6..2533bae52 100644 --- a/src/replication.cpp +++ b/src/replication.cpp @@ -4449,6 +4449,7 @@ void replicateSubkeyExpire(redisDb *db, robj_roptr key, robj_roptr subkey, long sdsfree(szFromObj(&objTtl)); } +void _clientAsyncReplyBufferReserve(client *c, size_t len); void flushReplBacklogToClients() { serverAssert(GlobalLocksAcquired()); @@ -4486,6 +4487,8 @@ void flushReplBacklogToClients() addReplyProto(replica, g_pserver->repl_backlog + g_pserver->repl_batch_idxStart, cbCopy); } else { auto cbPhase1 = g_pserver->repl_backlog_size - g_pserver->repl_batch_idxStart; + if (fAsyncWrite) + _clientAsyncReplyBufferReserve(replica, cbPhase1 + g_pserver->repl_backlog_idx); addReplyProto(replica, g_pserver->repl_backlog + g_pserver->repl_batch_idxStart, cbPhase1); addReplyProto(replica, g_pserver->repl_backlog, g_pserver->repl_backlog_idx); serverAssert((cbPhase1 + g_pserver->repl_backlog_idx) == (g_pserver->master_repl_offset - g_pserver->repl_batch_offStart)); From 931a5cce447735fa1fb82913e8c896c11517671d Mon Sep 17 00:00:00 2001 From: John Sully Date: Fri, 26 Mar 2021 02:54:33 +0000 Subject: [PATCH 14/32] Fix issue where timethread wakes too slowly Former-commit-id: 60f875f5eb7cb541e537f0f440ce8d5aeac57862 --- src/server.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/server.cpp b/src/server.cpp index 97a29f1a3..f7002b9bc 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -6091,6 +6091,7 @@ void OnTerminate() } void wakeTimeThread() { + updateCachedTime(); std::lock_guard lock(time_thread_mutex); sleeping_threads--; serverAssert(sleeping_threads >= 0); From 1a2803a1ac2b7fce05d3fe55619081b2880b436e Mon Sep 17 00:00:00 2001 From: John Sully Date: Fri, 26 Mar 2021 23:44:42 +0000 Subject: [PATCH 15/32] Eliminate unnecessary lookup in ensure when there is no snapshot Former-commit-id: 04540531b95ebadf6d60c90c6681c11060eaf907 --- src/db.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/db.cpp b/src/db.cpp index 20b70fe02..524cdb66c 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -2581,6 +2581,8 @@ void redisDbPersistentData::updateValue(dict_iter itr, robj *val) void redisDbPersistentData::ensure(const char *key) { + if (m_pdbSnapshot == nullptr && m_spstorage == nullptr) + return; dictEntry *de = dictFind(m_pdict, key); ensure(key, &de); } From c356822897fc5c9d413af35821dc70f36720ef90 Mon Sep 17 00:00:00 2001 From: John Sully Date: Fri, 26 Mar 2021 23:48:24 +0000 Subject: [PATCH 16/32] Prefetch keys even in pure RAM scenarios Former-commit-id: 54c0f11db0b35f34c9dfc3c8b4b76b7b7921bacd --- src/db.cpp | 25 ++++++++++++++++++++++++- src/networking.cpp | 4 ++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/db.cpp b/src/db.cpp index 524cdb66c..9978a8ca5 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -35,6 +35,11 @@ #include #include +// Needed for prefetch +#if defined(__x86_64__) || defined(__i386__) +#include +#endif + /* Database backup. */ struct dbBackup { const redisDbPersistentDataSnapshot **dbarray; @@ -3002,8 +3007,26 @@ int dbnumFromDb(redisDb *db) void redisDbPersistentData::prefetchKeysAsync(client *c, parsed_command &command) { - if (m_spstorage == nullptr) + if (m_spstorage == nullptr) { +#if defined(__x86_64__) || defined(__i386__) + // We do a quick 'n dirty check for set & get. Anything else is too slow. + // Should the user do something weird like remap them then the worst that will + // happen is we don't prefetch or we prefetch wrong data. A mild perf hit, but + // not dangerous + const char *cmd = szFromObj(command.argv[0]); + if (!strcasecmp(cmd, "set") || !strcasecmp(cmd, "get")) { + auto h = dictSdsHash(szFromObj(command.argv[1])); + for (int iht = 0; iht < 2; ++iht) { + auto hT = h & c->db->m_pdict->ht[iht].sizemask; + if (c->db->m_pdict->ht[iht].table != nullptr) + _mm_prefetch(c->db->m_pdict->ht[iht].table[hT], _MM_HINT_T1); + if (!dictIsRehashing(c->db->m_pdict)) + break; + } + } +#endif return; + } AeLocker lock; diff --git a/src/networking.cpp b/src/networking.cpp index 0774694ae..7607146ad 100644 --- a/src/networking.cpp +++ b/src/networking.cpp @@ -2368,8 +2368,8 @@ void parseClientCommandBuffer(client *c) { serverAssert(c->vecqueuedcmd.back().reploff >= 0); } - /* Prefetch if we have a storage provider and we're not in the global lock */ - if (cqueries < c->vecqueuedcmd.size() && g_pserver->m_pstorageFactory != nullptr && !GlobalLocksAcquired()) { + /* Prefetch outside the lock for better perf */ + if (cqueries < c->vecqueuedcmd.size() && !GlobalLocksAcquired()) { auto &query = c->vecqueuedcmd.back(); if (query.argc > 0 && query.argc == query.argcMax) { c->db->prefetchKeysAsync(c, query); From 73bc5aec956e8880bf4b6ba949e33daef753e39b Mon Sep 17 00:00:00 2001 From: John Sully Date: Sun, 28 Mar 2021 17:58:43 +0000 Subject: [PATCH 17/32] Fix thread safety issues with the cache prefetch logic Former-commit-id: a80a128bb64b81115c095d6dd91896ff73048b3d --- src/db.cpp | 22 +++++++++++++--------- src/dict.cpp | 8 +++++--- src/redis-benchmark.cpp | 4 ++++ src/redis-cli.c | 5 +++++ src/server.cpp | 9 +++++++++ src/server.h | 20 ++++++++++++++++++++ 6 files changed, 56 insertions(+), 12 deletions(-) diff --git a/src/db.cpp b/src/db.cpp index 9978a8ca5..59eb08a68 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -3013,15 +3013,19 @@ void redisDbPersistentData::prefetchKeysAsync(client *c, parsed_command &command // Should the user do something weird like remap them then the worst that will // happen is we don't prefetch or we prefetch wrong data. A mild perf hit, but // not dangerous - const char *cmd = szFromObj(command.argv[0]); - if (!strcasecmp(cmd, "set") || !strcasecmp(cmd, "get")) { - auto h = dictSdsHash(szFromObj(command.argv[1])); - for (int iht = 0; iht < 2; ++iht) { - auto hT = h & c->db->m_pdict->ht[iht].sizemask; - if (c->db->m_pdict->ht[iht].table != nullptr) - _mm_prefetch(c->db->m_pdict->ht[iht].table[hT], _MM_HINT_T1); - if (!dictIsRehashing(c->db->m_pdict)) - break; + if (command.argc >= 2) { + const char *cmd = szFromObj(command.argv[0]); + if (!strcasecmp(cmd, "set") || !strcasecmp(cmd, "get")) { + auto h = dictSdsHash(szFromObj(command.argv[1])); + for (int iht = 0; iht < 2; ++iht) { + auto hT = h & c->db->m_pdict->ht[iht].sizemask; + dictEntry **table; + __atomic_load(&c->db->m_pdict->ht[iht].table, &table, __ATOMIC_RELAXED); + if (table != nullptr) + _mm_prefetch(table[hT], _MM_HINT_T2); + if (!dictIsRehashing(c->db->m_pdict)) + break; + } } } #endif diff --git a/src/dict.cpp b/src/dict.cpp index 831a32f8f..4b9f0f6d1 100644 --- a/src/dict.cpp +++ b/src/dict.cpp @@ -74,6 +74,8 @@ static int _dictInit(dict *ht, dictType *type, void *privDataPtr); static uint8_t dict_hash_function_seed[16]; +extern "C" void asyncFreeDictTable(dictEntry **de); + void dictSetHashFunctionSeed(uint8_t *seed) { memcpy(dict_hash_function_seed,seed,sizeof(dict_hash_function_seed)); } @@ -359,7 +361,7 @@ int dictRehash(dict *d, int n) { /* Check if we already rehashed the whole table... */ if (d->ht[0].used == 0) { - zfree(d->ht[0].table); + asyncFreeDictTable(d->ht[0].table); d->ht[0] = d->ht[1]; _dictReset(&d->ht[1]); d->rehashidx = -1; @@ -487,7 +489,7 @@ void dictCompleteRehashAsync(dictAsyncRehashCtl *ctl, bool fFree) { /* Check if we already rehashed the whole table... */ if (d->ht[0].used == 0 && d->asyncdata == nullptr) { - zfree(d->ht[0].table); + asyncFreeDictTable(d->ht[0].table); d->ht[0] = d->ht[1]; _dictReset(&d->ht[1]); d->rehashidx = -1; @@ -762,7 +764,7 @@ int _dictClear(dict *d, dictht *ht, void(callback)(void *)) { } } /* Free the table and the allocated cache structure */ - zfree(ht->table); + asyncFreeDictTable(ht->table); /* Re-initialize the table */ _dictReset(ht); return DICT_OK; /* never fails */ diff --git a/src/redis-benchmark.cpp b/src/redis-benchmark.cpp index 41449099d..3b130679d 100644 --- a/src/redis-benchmark.cpp +++ b/src/redis-benchmark.cpp @@ -813,6 +813,10 @@ static int ipow(int base, int exp) { return result; } +extern "C" void asyncFreeDictTable(dictEntry **de) { + zfree(de); +} + static void showLatencyReport(void) { int i, curlat = 0; int usbetweenlat = ipow(10, MAX_LATENCY_PRECISION-config.precision); diff --git a/src/redis-cli.c b/src/redis-cli.c index c555a711e..1bfe4c8a2 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -144,6 +144,11 @@ static void cliRefreshPrompt(void) { sdsfree(prompt); } +struct dictEntry; +void asyncFreeDictTable(struct dictEntry **de) { + zfree(de); +} + /* Return the name of the dotfile for the specified 'dotfilename'. * Normally it just concatenates user $HOME to the file specified * in 'dotfilename'. However if the environment variable 'envoverride' diff --git a/src/server.cpp b/src/server.cpp index f7002b9bc..fbf05c8fa 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -2484,6 +2484,15 @@ int serverCronLite(struct aeEventLoop *eventLoop, long long id, void *clientData return 1000/g_pserver->hz; } +extern "C" void asyncFreeDictTable(dictEntry **de) +{ + if (de == nullptr || serverTL == nullptr || serverTL->gcEpoch.isReset()) { + zfree(de); + } else { + g_pserver->garbageCollector.enqueueCPtr(serverTL->gcEpoch, de); + } +} + extern int ProcessingEventsWhileBlocked; void processClients(); diff --git a/src/server.h b/src/server.h index 07f025c46..3612d93f0 100644 --- a/src/server.h +++ b/src/server.h @@ -1786,6 +1786,19 @@ class GarbageCollectorCollection GarbageCollector garbageCollectorSnapshot; GarbageCollector garbageCollectorGeneric; + class CPtrCollectable : public ICollectable + { + void *m_pv; + + public: + CPtrCollectable(void *pv) + : m_pv(pv) + {} + ~CPtrCollectable() { + zfree(m_pv); + } + }; + public: struct Epoch { @@ -1831,6 +1844,13 @@ public: { garbageCollectorGeneric.enqueue(e.epochGeneric, std::move(sp)); } + + template + void enqueueCPtr(Epoch e, T p) + { + auto sp = std::make_unique(reinterpret_cast(p)); + enqueue(e, std::move(sp)); + } }; // Per-thread variabels that may be accessed without a lock From 5a28256b8063a6cf78b8cc217b9148adaabe6362 Mon Sep 17 00:00:00 2001 From: John Sully Date: Sun, 28 Mar 2021 17:59:02 +0000 Subject: [PATCH 18/32] Excessive rehashing adds latency Former-commit-id: 9db2277b0fea032abcde089f617fe7e4f5ce12e3 --- src/dict.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dict.cpp b/src/dict.cpp index 4b9f0f6d1..9acc01dc5 100644 --- a/src/dict.cpp +++ b/src/dict.cpp @@ -546,7 +546,7 @@ int dictRehashMilliseconds(dict *d, int ms) { static void _dictRehashStep(dict *d) { unsigned iterators; __atomic_load(&d->iterators, &iterators, __ATOMIC_RELAXED); - if (iterators == 0) dictRehash(d,2); + if (iterators == 0) dictRehash(d,1); } /* Add an element to the target hash table */ From 526761b6f9169ca57cf5fae3faa19e87e5e82f56 Mon Sep 17 00:00:00 2001 From: John Sully Date: Sun, 28 Mar 2021 18:27:00 +0000 Subject: [PATCH 19/32] Make some asserts debug only for perf Former-commit-id: c22740df4f7c1468fbe19fd97dfaec20e011b6f1 --- src/networking.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/networking.cpp b/src/networking.cpp index 7607146ad..e2819ff19 100644 --- a/src/networking.cpp +++ b/src/networking.cpp @@ -1672,7 +1672,7 @@ int writeToClient(client *c, int handler_installed) { ssize_t nwritten = 0, totwritten = 0; clientReplyBlock *o; - AssertCorrectThread(c); + serverAssertDebug(FCorrectThread(c)); std::unique_locklock)> lock(c->lock); @@ -1891,7 +1891,7 @@ int handleClientsWithPendingWrites(int iel, int aof_state) { processed += (int)vec.size(); for (client *c : vec) { - AssertCorrectThread(c); + serverAssertDebug(FCorrectThread(c)); uint64_t flags = c->flags.fetch_and(~CLIENT_PENDING_WRITE, std::memory_order_relaxed); @@ -2438,8 +2438,8 @@ void readQueryFromClient(connection *conn) { int nread, readlen; size_t qblen; - serverAssert(FCorrectThread(c)); - serverAssert(!GlobalLocksAcquired()); + serverAssertDebug(FCorrectThread(c) sdfsdf); + serverAssertDebug(!GlobalLocksAcquired()); AeLocker aelock; AssertCorrectThread(c); From bb629c6f1e5f8199542436bee7c24848b3242757 Mon Sep 17 00:00:00 2001 From: John Sully Date: Sun, 28 Mar 2021 18:27:14 +0000 Subject: [PATCH 20/32] Enable LTO Former-commit-id: 7241bfc25525e8b27193fb0d4f9eae2e5e98e8d6 --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index 5f9458eb2..966ce4400 100644 --- a/src/Makefile +++ b/src/Makefile @@ -15,7 +15,7 @@ release_hdr := $(shell sh -c './mkreleasehdr.sh') uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') uname_M := $(shell sh -c 'uname -m 2>/dev/null || echo not') -OPTIMIZATION?=-O2 +OPTIMIZATION?=-O2 -flto DEPENDENCY_TARGETS=hiredis linenoise lua rocksdb NODEPS:=clean distclean From 2bbff8ae4daaff28e8a529c0ebff60404718742c Mon Sep 17 00:00:00 2001 From: Jesse Date: Tue, 30 Mar 2021 13:12:23 -0700 Subject: [PATCH 21/32] Add additional libraries that need to be installed for building pro Former-commit-id: 303a0646b5510e310dfd03c7288d747f76ce3d62 --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1decad4b0..bcb3e850e 100644 --- a/README.md +++ b/README.md @@ -88,9 +88,13 @@ KeyDB can be compiled and is tested for use on Linux. KeyDB currently relies on More on CentOS/Archlinux/Alpine/Debian/Ubuntu dependencies and builds can be found here: https://docs.keydb.dev/docs/build/ +Init and clone submodule dependencies: + + % git submodule init && git submodule update + Install dependencies: - % sudo apt install build-essential nasm autotools-dev autoconf libjemalloc-dev tcl tcl-dev uuid-dev libcurl4-openssl-dev + % sudo apt install build-essential nasm autotools-dev autoconf libjemalloc-dev tcl tcl-dev uuid-dev libcurl4-openssl-dev libbz2-dev libzstd-dev liblz4-dev libsnappy-dev Compiling is as simple as: From 32926467aa4098b0cd7f504596f251f684abc7d9 Mon Sep 17 00:00:00 2001 From: John Sully Date: Tue, 30 Mar 2021 20:44:22 +0000 Subject: [PATCH 22/32] Fix crash in RDB save Former-commit-id: ef1e122fcb039d25f60752ca27c9feb5e8cfbb31 --- src/gc.h | 6 +++--- src/server.h | 31 +++++++++++++++++++++++++++---- tests/unit/memefficiency.tcl | 7 ++++--- 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/src/gc.h b/src/gc.h index 4715bc8de..5d92e38cb 100644 --- a/src/gc.h +++ b/src/gc.h @@ -52,7 +52,7 @@ public: void endEpoch(uint64_t epoch, bool fNoFree = false) { std::unique_lock lock(m_lock); - assert(m_setepochOutstanding.find(epoch) != m_setepochOutstanding.end()); + serverAssert(m_setepochOutstanding.find(epoch) != m_setepochOutstanding.end()); bool fMinElement = *std::min_element(m_setepochOutstanding.begin(), m_setepochOutstanding.end()); m_setepochOutstanding.erase(epoch); if (fNoFree) @@ -91,8 +91,8 @@ public: void enqueue(uint64_t epoch, std::unique_ptr &&sp) { std::unique_lock lock(m_lock); - assert(m_setepochOutstanding.find(epoch) != m_setepochOutstanding.end()); - assert(sp->FWillFreeChildDebug() == false); + serverAssert(m_setepochOutstanding.find(epoch) != m_setepochOutstanding.end()); + serverAssert(sp->FWillFreeChildDebug() == false); auto itr = std::find(m_vecepochs.begin(), m_vecepochs.end(), m_epochNext+1); if (itr == m_vecepochs.end()) diff --git a/src/server.h b/src/server.h index 3612d93f0..1fbb77c60 100644 --- a/src/server.h +++ b/src/server.h @@ -1794,7 +1794,13 @@ class GarbageCollectorCollection CPtrCollectable(void *pv) : m_pv(pv) {} - ~CPtrCollectable() { + + CPtrCollectable(CPtrCollectable &&move) { + m_pv = move.m_pv; + move.m_pv = nullptr; + } + + virtual ~CPtrCollectable() { zfree(m_pv); } }; @@ -1810,6 +1816,20 @@ public: epochGeneric = 0; } + Epoch() = default; + + Epoch (const Epoch &other) { + epochSnapshot = other.epochSnapshot; + epochGeneric = other.epochGeneric; + } + + Epoch &operator=(const Epoch &other) { + serverAssert(isReset()); + epochSnapshot = other.epochSnapshot; + epochGeneric = other.epochGeneric; + return *this; + } + bool isReset() const { return epochSnapshot == 0 && epochGeneric == 0; } @@ -1823,10 +1843,13 @@ public: return e; } - void endEpoch(Epoch e, bool fNoFree = false) + void endEpoch(Epoch &e, bool fNoFree = false) { - garbageCollectorSnapshot.endEpoch(e.epochSnapshot, fNoFree); - garbageCollectorGeneric.endEpoch(e.epochGeneric, fNoFree); + auto epochSnapshot = e.epochSnapshot; + auto epochGeneric = e.epochGeneric; + e.reset(); // We must do this early as GC'd dtors can themselves try to enqueue more data + garbageCollectorSnapshot.endEpoch(epochSnapshot, fNoFree); + garbageCollectorGeneric.endEpoch(epochGeneric, fNoFree); } void shutdown() diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl index c0a6ec4d7..5bf69787b 100644 --- a/tests/unit/memefficiency.tcl +++ b/tests/unit/memefficiency.tcl @@ -154,7 +154,7 @@ start_server {tags {"defrag"} overrides {server-threads 1} } { $rd read ; # Discard replies } - set expected_frag 1.7 + set expected_frag 1.5 if {$::accurate} { # scale the hash to 1m fields in order to have a measurable the latency for {set j 10000} {$j < 1000000} {incr j} { @@ -265,7 +265,7 @@ start_server {tags {"defrag"} overrides {server-threads 1} } { # create big keys with 10k items set rd [redis_deferring_client] - set expected_frag 1.7 + set expected_frag 1.5 # add a mass of list nodes to two lists (allocations are interlaced) set val [string repeat A 100] ;# 5 items of 100 bytes puts us in the 640 bytes bin, which has 32 regs, so high potential for fragmentation set elements 500000 @@ -543,4 +543,5 @@ start_server {tags {"defrag"} overrides {server-threads 1 active-replica yes} } } {OK} } } -} ;# run solo \ No newline at end of file +} ;# run solo + From 57b242a5ba1860a31252ba8e51f0f433efc92022 Mon Sep 17 00:00:00 2001 From: John Sully Date: Tue, 30 Mar 2021 23:30:17 +0000 Subject: [PATCH 23/32] Fix issue #300 Former-commit-id: bee7f398c9c3b4c01b687d26194913bf215dca36 --- src/networking.cpp | 4 ++++ src/server.h | 6 ++---- tests/unit/scripting.tcl | 15 +++++++++++++++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/networking.cpp b/src/networking.cpp index e2819ff19..72d08cc80 100644 --- a/src/networking.cpp +++ b/src/networking.cpp @@ -2403,6 +2403,7 @@ void processInputBuffer(client *c, bool fParse, int callFlags) { /* Return if we're still parsing this command */ auto &cmd = c->vecqueuedcmd.front(); if (cmd.argc != cmd.argcMax) break; + if (c->flags & CLIENT_EXECUTING_COMMAND) break; if (!FClientReady(c)) break; @@ -2421,13 +2422,16 @@ void processInputBuffer(client *c, bool fParse, int callFlags) { if (c->argc == 0) { resetClient(c); } else { + c->flags |= CLIENT_EXECUTING_COMMAND; /* We are finally ready to execute the command. */ if (processCommandAndResetClient(c, callFlags) == C_ERR) { /* If the client is no longer valid, we avoid exiting this * loop and trimming the client buffer later. So we return * ASAP in that case. */ + c->flags &= ~CLIENT_EXECUTING_COMMAND; return; } + c->flags &= ~CLIENT_EXECUTING_COMMAND; } } } diff --git a/src/server.h b/src/server.h index 1fbb77c60..449bc77ee 100644 --- a/src/server.h +++ b/src/server.h @@ -506,10 +506,8 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT]; #define CLIENT_PENDING_READ (1<<29) /* The client has pending reads and was put in the list of clients we can read from. */ -#define CLIENT_PENDING_COMMAND (1<<30) /* Used in threaded I/O to signal after - we return single threaded that the - client has already pending commands - to be executed. */ +#define CLIENT_EXECUTING_COMMAND (1<<30) /* Used to handle reentrency cases in processCommandWhileBlocked + to ensure we don't process a client already executing */ #define CLIENT_TRACKING (1ULL<<31) /* Client enabled keys tracking in order to perform client side caching. */ #define CLIENT_TRACKING_BROKEN_REDIR (1ULL<<32) /* Target client is invalid. */ diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index b41956f9a..a4960d222 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -430,6 +430,21 @@ start_server {tags {"scripting"}} { set res } {102} + test {EVAL with pipelined command (No crash)} { + r flushall + r config set lua-time-limit 1 + set rd [redis_deferring_client] + $rd eval {for i=1,1000000 do redis.call('set', i, 'sdfdsfd') end} 0 + $rd set testkey foo + $rd get testkey + after 1200 + catch {r echo "foo"} err + assert_match {BUSY*} $err + $rd read + $rd close + } + + test {EVAL timeout from AOF} { # generate a long running script that is propagated to the AOF as script # make sure that the script times out during loading From d58a415e295b117bb6d18cf67443d8cca57099a2 Mon Sep 17 00:00:00 2001 From: John Sully Date: Fri, 9 Apr 2021 01:05:55 +0000 Subject: [PATCH 24/32] DO not set the timethread as high priority, it can starve the server threads Former-commit-id: faeac65371af9d6b6effe0886bcbdefaec24ad6d --- src/server.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/server.cpp b/src/server.cpp index fbf05c8fa..bb0e6edae 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -6471,9 +6471,6 @@ int main(int argc, char **argv) { serverAssert(cserver.cthreads > 0 && cserver.cthreads <= MAX_EVENT_LOOPS); pthread_create(&cserver.time_thread_id, nullptr, timeThreadMain, nullptr); - struct sched_param time_thread_priority; - time_thread_priority.sched_priority = sched_get_priority_max(SCHED_FIFO); - pthread_setschedparam(cserver.time_thread_id, SCHED_FIFO, &time_thread_priority); pthread_attr_t tattr; pthread_attr_init(&tattr); From df66c598e9f25506235272e17299c7083fa21055 Mon Sep 17 00:00:00 2001 From: John Sully Date: Thu, 8 Apr 2021 19:52:38 +0000 Subject: [PATCH 25/32] Make prefetch configurable Former-commit-id: 3b660347d70cc25d57119080bd43fb4671e36488 --- src/config.cpp | 1 + src/networking.cpp | 4 ++-- src/server.h | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/config.cpp b/src/config.cpp index 99505d444..9d7f14007 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -2535,6 +2535,7 @@ standardConfig configs[] = { createBoolConfig("use-fork", NULL, IMMUTABLE_CONFIG, cserver.fForkBgSave, 0, NULL, NULL), createBoolConfig("allow-write-during-load", NULL, MODIFIABLE_CONFIG, g_pserver->fWriteDuringActiveLoad, 0, NULL, NULL), createBoolConfig("io-threads-do-reads", NULL, IMMUTABLE_CONFIG, fDummy, 0, NULL, NULL), + createBoolConfig("prefetch-enabled", NULL, MODIFIABLE_CONFIG, g_pserver->prefetch_enabled, 1, NULL, NULL), /* String Configs */ createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, g_pserver->acl_filename, "", NULL, NULL), diff --git a/src/networking.cpp b/src/networking.cpp index 72d08cc80..c6daefe1f 100644 --- a/src/networking.cpp +++ b/src/networking.cpp @@ -2353,7 +2353,7 @@ void parseClientCommandBuffer(client *c) { } } - size_t cqueries = c->vecqueuedcmd.size(); + size_t cqueriesStart = c->vecqueuedcmd.size(); if (c->reqtype == PROTO_REQ_INLINE) { if (processInlineBuffer(c) != C_OK) break; } else if (c->reqtype == PROTO_REQ_MULTIBULK) { @@ -2369,7 +2369,7 @@ void parseClientCommandBuffer(client *c) { } /* Prefetch outside the lock for better perf */ - if (cqueries < c->vecqueuedcmd.size() && !GlobalLocksAcquired()) { + if (g_pserver->prefetch_enabled && cqueriesStart < c->vecqueuedcmd.size() && !GlobalLocksAcquired()) { auto &query = c->vecqueuedcmd.back(); if (query.argc > 0 && query.argc == query.argcMax) { c->db->prefetchKeysAsync(c, query); diff --git a/src/server.h b/src/server.h index 449bc77ee..9eed26e95 100644 --- a/src/server.h +++ b/src/server.h @@ -2391,6 +2391,8 @@ struct redisServer { char *aof_rewrite_cpulist; /* cpu affinity list of aof rewrite process. */ char *bgsave_cpulist; /* cpu affinity list of bgsave process. */ + int prefetch_enabled = 1; + long long repl_batch_offStart = -1; long long repl_batch_idxStart = -1; From e6782d8f1b6e3ec74e6c2f0600d2bb63a41d94d4 Mon Sep 17 00:00:00 2001 From: John Sully Date: Mon, 12 Apr 2021 03:22:22 +0000 Subject: [PATCH 26/32] Reduce P99 latency with async rehash Former-commit-id: 7ea4c26fd82c0cdfa377183083f38a43336e480b --- src/dict.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dict.cpp b/src/dict.cpp index 9acc01dc5..88ad116bb 100644 --- a/src/dict.cpp +++ b/src/dict.cpp @@ -384,7 +384,7 @@ dictAsyncRehashCtl *dictRehashAsyncStart(dict *d, int buckets) { d->asyncdata = new dictAsyncRehashCtl(d, d->asyncdata); - int empty_visits = buckets * 10; + int empty_visits = buckets; while (d->asyncdata->queue.size() < (size_t)buckets && (size_t)d->rehashidx < d->ht[0].size) { dictEntry *de; From 1b121723e2871cae3c7ae570b933a50d59515ac1 Mon Sep 17 00:00:00 2001 From: John Sully Date: Mon, 12 Apr 2021 03:23:49 +0000 Subject: [PATCH 27/32] Don't prefetch when lock contention is low, it increases latency Former-commit-id: 0d21614e0e5aba28acd364231823d51a3073081f --- src/ae.cpp | 16 ++++++++++++++++ src/ae.h | 2 ++ src/networking.cpp | 3 ++- src/server.cpp | 21 +++++++++++++++++++-- src/server.h | 5 +++++ 5 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/ae.cpp b/src/ae.cpp index d88328ded..f96ef4f6c 100644 --- a/src/ae.cpp +++ b/src/ae.cpp @@ -870,3 +870,19 @@ int aeThreadOwnsLock() { return g_lock.fOwnLock(); } + +int aeLockContested(int threshold) +{ + return g_lock.m_ticket.m_active < static_cast(g_lock.m_ticket.m_avail - threshold); +} + +int aeLockContention() +{ + ticket ticketT; + __atomic_load(&g_lock.m_ticket.u, &ticketT.u, __ATOMIC_RELAXED); + int32_t avail = ticketT.m_avail; + int32_t active = ticketT.m_active; + if (avail < active) + avail += 0x10000; + return avail - active; +} \ No newline at end of file diff --git a/src/ae.h b/src/ae.h index 3a240877d..aec1df154 100644 --- a/src/ae.h +++ b/src/ae.h @@ -169,6 +169,8 @@ void aeAcquireLock(); int aeTryAcquireLock(int fWeak); void aeReleaseLock(); int aeThreadOwnsLock(); +int aeLockContested(int threshold); +int aeLockContention(); // returns the number of instantaneous threads waiting on the lock #ifdef __cplusplus } diff --git a/src/networking.cpp b/src/networking.cpp index c6daefe1f..36eb033aa 100644 --- a/src/networking.cpp +++ b/src/networking.cpp @@ -2369,7 +2369,8 @@ void parseClientCommandBuffer(client *c) { } /* Prefetch outside the lock for better perf */ - if (g_pserver->prefetch_enabled && cqueriesStart < c->vecqueuedcmd.size() && !GlobalLocksAcquired()) { + if (g_pserver->prefetch_enabled && cqueriesStart < c->vecqueuedcmd.size() && + (g_pserver->m_pstorageFactory || aeLockContested(cserver.cthreads/2)) && !GlobalLocksAcquired()) { auto &query = c->vecqueuedcmd.back(); if (query.argc > 0 && query.argc == query.argcMax) { c->db->prefetchKeysAsync(c, query); diff --git a/src/server.cpp b/src/server.cpp index bb0e6edae..bbee218ac 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -2442,6 +2442,14 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { /* CRON functions may trigger async writes, so do this last */ ProcessPendingAsyncWrites(); + // Measure lock contention from a different thread to be more accurate + g_pserver->asyncworkqueue->AddWorkFunction([]{ + g_pserver->rglockSamples[g_pserver->ilockRingHead] = (uint16_t)aeLockContention(); + ++g_pserver->ilockRingHead; + if (g_pserver->ilockRingHead >= redisServer::s_lockContentionSamples) + g_pserver->ilockRingHead = 0; + }); + g_pserver->cronloops++; return 1000/g_pserver->hz; } @@ -5134,6 +5142,11 @@ sds genRedisInfoString(const char *section) { /* Stats */ if (allsections || defsections || !strcasecmp(section,"stats")) { + double avgLockContention = 0; + for (unsigned i = 0; i < redisServer::s_lockContentionSamples; ++i) + avgLockContention += g_pserver->rglockSamples[i]; + avgLockContention /= redisServer::s_lockContentionSamples; + if (sections++) info = sdscat(info,"\r\n"); info = sdscatprintf(info, "# Stats\r\n" @@ -5169,7 +5182,9 @@ sds genRedisInfoString(const char *section) { "tracking_total_prefixes:%lld\r\n" "unexpected_error_replies:%lld\r\n" "total_reads_processed:%lld\r\n" - "total_writes_processed:%lld\r\n", + "total_writes_processed:%lld\r\n" + "instantaneous_lock_contention:%d\r\n" + "avg_lock_contention:%f\r\n", g_pserver->stat_numconnections, g_pserver->stat_numcommands, getInstantaneousMetric(STATS_METRIC_COMMAND), @@ -5202,7 +5217,9 @@ sds genRedisInfoString(const char *section) { (unsigned long long) trackingGetTotalPrefixes(), g_pserver->stat_unexpected_error_replies, g_pserver->stat_total_reads_processed.load(std::memory_order_relaxed), - g_pserver->stat_total_writes_processed.load(std::memory_order_relaxed)); + g_pserver->stat_total_writes_processed.load(std::memory_order_relaxed), + aeLockContention(), + avgLockContention); } /* Replication */ diff --git a/src/server.h b/src/server.h index 9eed26e95..678d83225 100644 --- a/src/server.h +++ b/src/server.h @@ -2396,6 +2396,11 @@ struct redisServer { long long repl_batch_offStart = -1; long long repl_batch_idxStart = -1; + /* Lock Contention Ring Buffer */ + static const size_t s_lockContentionSamples = 64; + uint16_t rglockSamples[s_lockContentionSamples]; + unsigned ilockRingHead = 0; + bool FRdbSaveInProgress() const { return rdbThreadVars.fRdbThreadActive; } }; From 14250faddd19822394ec422eff07f41a8fba5c9a Mon Sep 17 00:00:00 2001 From: John Sully Date: Mon, 12 Apr 2021 03:24:05 +0000 Subject: [PATCH 28/32] Make prefetch more aggressive Former-commit-id: 25a5cfefcf7fa3451e92500f5d310290c4b6bbde --- src/db.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/db.cpp b/src/db.cpp index 59eb08a68..77d3c1a66 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -3021,8 +3021,13 @@ void redisDbPersistentData::prefetchKeysAsync(client *c, parsed_command &command auto hT = h & c->db->m_pdict->ht[iht].sizemask; dictEntry **table; __atomic_load(&c->db->m_pdict->ht[iht].table, &table, __ATOMIC_RELAXED); - if (table != nullptr) - _mm_prefetch(table[hT], _MM_HINT_T2); + if (table != nullptr) { + dictEntry *de = table[hT]; + while (de != nullptr) { + _mm_prefetch(dictGetKey(de), _MM_HINT_T2); + de = de->next; + } + } if (!dictIsRehashing(c->db->m_pdict)) break; } From ea2bbfb5c1721c345910d010d67c689970241011 Mon Sep 17 00:00:00 2001 From: John Sully Date: Wed, 7 Apr 2021 17:30:59 +0000 Subject: [PATCH 29/32] Fix issue where GC is not free'd until a BGSAVE Former-commit-id: 38523e6b508cf5f4b40c178dfe98554abea8f6bd --- src/server.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/server.cpp b/src/server.cpp index bbee218ac..6d297c937 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -2450,6 +2450,16 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { g_pserver->ilockRingHead = 0; }); + run_with_period(10) { + // Server threads don't free the GC, but if we don't have a + // a bgsave or some other async task then we'll hold onto the + // data for too long + g_pserver->asyncworkqueue->AddWorkFunction([]{ + auto epoch = g_pserver->garbageCollector.startEpoch(); + g_pserver->garbageCollector.endEpoch(epoch); + }); + } + g_pserver->cronloops++; return 1000/g_pserver->hz; } From 211b0e5f80c84fc372fdff5313341b9d840b1626 Mon Sep 17 00:00:00 2001 From: John Sully Date: Wed, 7 Apr 2021 17:41:20 +0000 Subject: [PATCH 30/32] Avoid posting unnecessary async tasks Former-commit-id: 1661d2a05943f96992be195f5dc32dd9a67c0e68 --- src/gc.h | 8 +++++++- src/server.cpp | 16 +++++++++------- src/server.h | 5 +++++ 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/gc.h b/src/gc.h index 5d92e38cb..8ed834ae9 100644 --- a/src/gc.h +++ b/src/gc.h @@ -49,6 +49,12 @@ public: m_setepochOutstanding.clear(); } + bool empty() const + { + std::unique_lock lock(m_lock); + return m_vecepochs.empty(); + } + void endEpoch(uint64_t epoch, bool fNoFree = false) { std::unique_lock lock(m_lock); @@ -109,7 +115,7 @@ public: } private: - fastlock m_lock { "Garbage Collector"}; + mutable fastlock m_lock { "Garbage Collector"}; std::vector m_vecepochs; std::unordered_set m_setepochOutstanding; diff --git a/src/server.cpp b/src/server.cpp index 6d297c937..3d547f748 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -2451,13 +2451,15 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { }); run_with_period(10) { - // Server threads don't free the GC, but if we don't have a - // a bgsave or some other async task then we'll hold onto the - // data for too long - g_pserver->asyncworkqueue->AddWorkFunction([]{ - auto epoch = g_pserver->garbageCollector.startEpoch(); - g_pserver->garbageCollector.endEpoch(epoch); - }); + if (!g_pserver->garbageCollector.empty()) { + // Server threads don't free the GC, but if we don't have a + // a bgsave or some other async task then we'll hold onto the + // data for too long + g_pserver->asyncworkqueue->AddWorkFunction([]{ + auto epoch = g_pserver->garbageCollector.startEpoch(); + g_pserver->garbageCollector.endEpoch(epoch); + }); + } } g_pserver->cronloops++; diff --git a/src/server.h b/src/server.h index 678d83225..878154bc5 100644 --- a/src/server.h +++ b/src/server.h @@ -1850,6 +1850,11 @@ public: garbageCollectorGeneric.endEpoch(epochGeneric, fNoFree); } + bool empty() + { + return garbageCollectorGeneric.empty() && garbageCollectorSnapshot.empty(); + } + void shutdown() { garbageCollectorSnapshot.shutdown(); From b2ffe82640c6069e9e6468614efa3bfe9fdfc81a Mon Sep 17 00:00:00 2001 From: John Sully Date: Mon, 12 Apr 2021 04:44:52 +0000 Subject: [PATCH 31/32] Allow prefetch even with a single thread Former-commit-id: 2e68821b330a6bae8a352e37c3da643d504b2ce3 --- src/networking.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/networking.cpp b/src/networking.cpp index 36eb033aa..574a8bc6c 100644 --- a/src/networking.cpp +++ b/src/networking.cpp @@ -2370,7 +2370,7 @@ void parseClientCommandBuffer(client *c) { /* Prefetch outside the lock for better perf */ if (g_pserver->prefetch_enabled && cqueriesStart < c->vecqueuedcmd.size() && - (g_pserver->m_pstorageFactory || aeLockContested(cserver.cthreads/2)) && !GlobalLocksAcquired()) { + (g_pserver->m_pstorageFactory || aeLockContested(cserver.cthreads/2) || cserver.cthreads == 1) && !GlobalLocksAcquired()) { auto &query = c->vecqueuedcmd.back(); if (query.argc > 0 && query.argc == query.argcMax) { c->db->prefetchKeysAsync(c, query); From 12044bab091043d43c306ddb3a043ff625710daf Mon Sep 17 00:00:00 2001 From: christian Date: Mon, 3 May 2021 21:41:04 +0000 Subject: [PATCH 32/32] Set repl_down_since in freeClientAsync Former-commit-id: fb4bd2984651837fadd1dbbbb56327e8452b4a2a --- src/networking.cpp | 4 +++- src/replication.cpp | 8 +++++++- src/server.cpp | 2 +- src/server.h | 1 + 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/networking.cpp b/src/networking.cpp index 15aa6f43a..df1aca947 100644 --- a/src/networking.cpp +++ b/src/networking.cpp @@ -158,6 +158,7 @@ client *createClient(connection *conn, int iel) { c->reploff_cmd = 0; c->repl_ack_off = 0; c->repl_ack_time = 0; + c->repl_down_since = 0; c->slave_listening_port = 0; c->slave_ip[0] = '\0'; c->slave_capa = SLAVE_CAPA_NONE; @@ -1606,7 +1607,8 @@ void freeClientAsync(client *c) { AeLocker lock; lock.arm(c); if (c->flags & CLIENT_CLOSE_ASAP || c->flags & CLIENT_LUA) return; // race condition after we acquire the lock - c->flags |= CLIENT_CLOSE_ASAP; + c->flags |= CLIENT_CLOSE_ASAP; + c->repl_down_since = g_pserver->unixtime; std::unique_lock ul(lockasyncfree); listAddNodeTail(g_pserver->clients_to_close,c); } diff --git a/src/replication.cpp b/src/replication.cpp index 0589868cb..f3b0ebed1 100644 --- a/src/replication.cpp +++ b/src/replication.cpp @@ -3213,9 +3213,14 @@ void replicationHandleMasterDisconnection(redisMaster *mi) { moduleFireServerEvent(REDISMODULE_EVENT_MASTER_LINK_CHANGE, REDISMODULE_SUBEVENT_MASTER_LINK_DOWN, NULL); + if (mi->master && mi->master->repl_down_since) { + mi->repl_down_since = mi->master->repl_down_since; + } + else { + mi->repl_down_since = g_pserver->unixtime; + } mi->master = NULL; mi->repl_state = REPL_STATE_CONNECT; - mi->repl_down_since = g_pserver->unixtime; /* We lost connection with our master, don't disconnect slaves yet, * maybe we'll be able to PSYNC with our master later. We'll disconnect * the slaves only if we'll have to do a full resync with our master. */ @@ -3534,6 +3539,7 @@ void replicationResurrectCachedMaster(redisMaster *mi, connection *conn) { mi->master->lastinteraction = g_pserver->unixtime; mi->repl_state = REPL_STATE_CONNECTED; mi->repl_down_since = 0; + mi->master->repl_down_since = 0; /* Normally changing the thread of a client is a BIG NONO, but this client was unlinked so its OK here */ diff --git a/src/server.cpp b/src/server.cpp index 96d62e4ea..d4805e8d6 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -6082,7 +6082,7 @@ void *timeThreadMain(void*) { delay.tv_nsec = 100; while (true) { updateCachedTime(); - clock_nanosleep(CLOCK_REALTIME, 0, &delay, NULL); + clock_nanosleep(CLOCK_MONOTONIC, 0, &delay, NULL); } } diff --git a/src/server.h b/src/server.h index 07f025c46..f556c6077 100644 --- a/src/server.h +++ b/src/server.h @@ -1509,6 +1509,7 @@ struct client { off_t repldboff; /* Replication DB file offset. */ off_t repldbsize; /* Replication DB file size. */ sds replpreamble; /* Replication DB preamble. */ + time_t repl_down_since; /* When client lost connection. */ long long read_reploff; /* Read replication offset if this is a master. */ long long reploff; /* Applied replication offset if this is a master. */ long long reploff_skipped; /* Repl backlog we did not send to this client */