From ac4e639df55f1f31e4ca4733d24573136fa1da6e Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 6 Mar 2020 11:09:52 +0100 Subject: [PATCH 01/11] Fix release notes spelling mistake. --- 00-RELEASENOTES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index ad2648bcb..9847d37dd 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -30,7 +30,7 @@ New features and improvements: * ACL LOG: log denied commands, keys accesses and authentications. * Client side caching redesigned. Now we use keys not caching slots. * Client side caching: Broadcasting mode implemented. -* Client side caching: OPTIN/OUTPUT modes implemented. +* Client side caching: OPTIN/OPTOUT modes implemented. * Remove RDB files used for replication in persistence-less instances (option). Fixes (only selected ones, see commits for all the fixes): From 75d5ed8938739c9dfbb7842340ae86b194d445fb Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 12 Mar 2020 12:59:44 +0100 Subject: [PATCH 02/11] ae.c: fix crash when resizing the event loop. See #6964. The root cause is that the event loop may be resized from an event callback itself, causing the event pointer to be invalid. --- src/ae.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ae.c b/src/ae.c index d2248fe5c..1bf6cbfbf 100644 --- a/src/ae.c +++ b/src/ae.c @@ -464,6 +464,7 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) if (!invert && fe->mask & mask & AE_READABLE) { fe->rfileProc(eventLoop,fd,fe->clientData,mask); fired++; + fe = &eventLoop->events[fd]; /* Refresh in case of resize. */ } /* Fire the writable event. */ @@ -476,8 +477,11 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) /* If we have to invert the call, fire the readable event now * after the writable one. */ - if (invert && fe->mask & mask & AE_READABLE) { - if (!fired || fe->wfileProc != fe->rfileProc) { + if (invert) { + fe = &eventLoop->events[fd]; /* Refresh in case of resize. */ + if ((fe->mask & mask & AE_READABLE) && + (!fired || fe->wfileProc != fe->rfileProc)) + { fe->rfileProc(eventLoop,fd,fe->clientData,mask); fired++; } From cde46df309268421c553433a574464306120cf52 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 5 Mar 2020 16:55:14 +0200 Subject: [PATCH 03/11] fix for flaky psync2 test *** [err]: PSYNC2: total sum of full synchronizations is exactly 4 in tests/integration/psync2.tcl Expected 5 == 4 (context: type eval line 6 cmd {assert {$sum == 4}} proc ::test) issue was that sometime the test got an unexpected full sync since it tried to switch to the replica before it was in sync with it's master. --- tests/integration/psync2.tcl | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/integration/psync2.tcl b/tests/integration/psync2.tcl index d1212b640..333736ffa 100644 --- a/tests/integration/psync2.tcl +++ b/tests/integration/psync2.tcl @@ -114,6 +114,27 @@ start_server {} { } } + # wait for all the slaves to be in sync with the master + set master_ofs [status $R($master_id) master_repl_offset] + wait_for_condition 500 100 { + $master_ofs == [status $R(0) master_repl_offset] && + $master_ofs == [status $R(1) master_repl_offset] && + $master_ofs == [status $R(2) master_repl_offset] && + $master_ofs == [status $R(3) master_repl_offset] && + $master_ofs == [status $R(4) master_repl_offset] + } else { + if {$debug_msg} { + for {set j 0} {$j < 5} {incr j} { + puts "$j: sync_full: [status $R($j) sync_full]" + puts "$j: id1 : [status $R($j) master_replid]:[status $R($j) master_repl_offset]" + puts "$j: id2 : [status $R($j) master_replid2]:[status $R($j) second_repl_offset]" + puts "$j: backlog : firstbyte=[status $R($j) repl_backlog_first_byte_offset] len=[status $R($j) repl_backlog_histlen]" + puts "---" + } + } + fail "Slaves are not in sync with the master after too long time." + } + # Put down the old master so that it cannot generate more # replication stream, this way in the next master switch, the time at # which we move slaves away is not important, each will have full From 9f1b635067886d8972b817698f95ffd893f903b8 Mon Sep 17 00:00:00 2001 From: Johannes Truschnigg Date: Thu, 19 Dec 2019 21:47:24 +0100 Subject: [PATCH 04/11] Signal systemd readiness atfer Partial Resync "Partial Resynchronization" is a special variant of replication success that we have to tell systemd about if it is managing redis-server via a Type=Notify service unit. --- src/replication.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/replication.c b/src/replication.c index 429e1d8a8..31e14d7fe 100644 --- a/src/replication.c +++ b/src/replication.c @@ -2284,6 +2284,10 @@ void syncWithMaster(connection *conn) { if (psync_result == PSYNC_CONTINUE) { serverLog(LL_NOTICE, "MASTER <-> REPLICA sync: Master accepted a Partial Resynchronization."); + if (server.supervised_mode == SUPERVISED_SYSTEMD) { + redisCommunicateSystemd("STATUS=MASTER <-> REPLICA sync: Partial Resynchronization accepted. Ready to accept connections.\n"); + redisCommunicateSystemd("READY=1\n"); + } return; } From 2880ed0b90c8fd3056c5c0bb3253ef1c81c6ba5d Mon Sep 17 00:00:00 2001 From: Jamie Scott Date: Tue, 3 Mar 2020 18:03:16 -0800 Subject: [PATCH 05/11] Update Redis.conf to improve TLS usability When using TLS with a Redis.conf file the line for TLS reading tls-cert-file redis.crt tls-key-file redis.key is interpreted as one complete directive. I am separating this on two separate lines to improve usability so users do not get the below error. ubuntu@ip-172-31-29-250:~/redis-6.0-rc1$ ./src/redis-server redis.conf *** FATAL CONFIG FILE ERROR *** Reading the configuration file, at line 145 >>> 'tls-cert-file redis.crt tls-key-file redis.key' wrong number of arguments ubuntu@ip-172-31-29-250:~/redis-6.0-rc1$ vi redis.conf ubuntu@ip-172-31-29-250:~/redis-6.0-rc1$ ./src/redis-server redis.conf 23085:C 04 Mar 2020 01:58:12.631 # oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo 23085:C 04 Mar 2020 01:58:12.631 # Redis version=5.9.101, bits=64, commit=00000000, modified=0, pid=23085, just started 23085:C 04 Mar 2020 01:58:12.631 # Configuration loaded 23085:M 04 Mar 2020 01:58:12.632 * Increased maximum number of open files to 10032 (it was originally set to 1024). --- redis.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/redis.conf b/redis.conf index 8609a9f57..6fa5e87c0 100644 --- a/redis.conf +++ b/redis.conf @@ -142,7 +142,8 @@ tcp-keepalive 300 # server to connected clients, masters or cluster peers. These files should be # PEM formatted. # -# tls-cert-file redis.crt tls-key-file redis.key +# tls-cert-file redis.crt +# tls-key-file redis.key # Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange: # From 6e3f53a46fc4757b51308e431dd69280f5483ed8 Mon Sep 17 00:00:00 2001 From: Jamie Scott Date: Mon, 9 Mar 2020 12:53:44 -0700 Subject: [PATCH 06/11] Remove default guidance in Redis.conf Removing the default guidance in Redis.conf since this is not an available value. --- redis.conf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/redis.conf b/redis.conf index 6fa5e87c0..c9d256bef 100644 --- a/redis.conf +++ b/redis.conf @@ -176,8 +176,7 @@ tcp-keepalive 300 # tls-cluster yes # Explicitly specify TLS versions to support. Allowed values are case insensitive -# and include "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3" (OpenSSL >= 1.1.1) or -# "default" which is currently >= TLSv1.1. +# and include "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3" (OpenSSL >= 1.1.1) # # tls-protocols TLSv1.2 From 109296b438c6f7192d6093f7524a6433f2d10852 Mon Sep 17 00:00:00 2001 From: lifubang Date: Thu, 5 Mar 2020 18:13:43 +0800 Subject: [PATCH 07/11] update linenoise to https://github.com/antirez/linenoise/tree/fc9667a81d43911a6690fb1e68c16e6e3bb8df05 Signed-off-by: lifubang --- deps/linenoise/README.markdown | 25 ++++++++++++++++++++++++- deps/linenoise/example.c | 5 +++++ deps/linenoise/linenoise.c | 31 ++++++++++++++++++++++++++++--- deps/linenoise/linenoise.h | 2 ++ 4 files changed, 59 insertions(+), 4 deletions(-) diff --git a/deps/linenoise/README.markdown b/deps/linenoise/README.markdown index e01642cf8..1afea2ae6 100644 --- a/deps/linenoise/README.markdown +++ b/deps/linenoise/README.markdown @@ -21,7 +21,7 @@ So what usually happens is either: The result is a pollution of binaries without line editing support. -So I spent more or less two hours doing a reality check resulting in this little library: is it *really* needed for a line editing library to be 20k lines of code? Apparently not, it is possibe to get a very small, zero configuration, trivial to embed library, that solves the problem. Smaller programs will just include this, supporing line editing out of the box. Larger programs may use this little library or just checking with configure if readline/libedit is available and resorting to Linenoise if not. +So I spent more or less two hours doing a reality check resulting in this little library: is it *really* needed for a line editing library to be 20k lines of code? Apparently not, it is possibe to get a very small, zero configuration, trivial to embed library, that solves the problem. Smaller programs will just include this, supporting line editing out of the box. Larger programs may use this little library or just checking with configure if readline/libedit is available and resorting to Linenoise if not. ## Terminals, in 2010. @@ -126,6 +126,24 @@ Linenoise has direct support for persisting the history into an history file. The functions `linenoiseHistorySave` and `linenoiseHistoryLoad` do just that. Both functions return -1 on error and 0 on success. +## Mask mode + +Sometimes it is useful to allow the user to type passwords or other +secrets that should not be displayed. For such situations linenoise supports +a "mask mode" that will just replace the characters the user is typing +with `*` characters, like in the following example: + + $ ./linenoise_example + hello> get mykey + echo: 'get mykey' + hello> /mask + hello> ********* + +You can enable and disable mask mode using the following two functions: + + void linenoiseMaskModeEnable(void); + void linenoiseMaskModeDisable(void); + ## Completion Linenoise supports completion, which is the ability to complete the user @@ -222,3 +240,8 @@ Sometimes you may want to clear the screen as a result of something the user typed. You can do this by calling the following function: void linenoiseClearScreen(void); + +## Related projects + +* [Linenoise NG](https://github.com/arangodb/linenoise-ng) is a fork of Linenoise that aims to add more advanced features like UTF-8 support, Windows support and other features. Uses C++ instead of C as development language. +* [Linenoise-swift](https://github.com/andybest/linenoise-swift) is a reimplementation of Linenoise written in Swift. diff --git a/deps/linenoise/example.c b/deps/linenoise/example.c index 3a544d3c6..74358c323 100644 --- a/deps/linenoise/example.c +++ b/deps/linenoise/example.c @@ -55,6 +55,7 @@ int main(int argc, char **argv) { * * The typed string is returned as a malloc() allocated string by * linenoise, so the user needs to free() it. */ + while((line = linenoise("hello> ")) != NULL) { /* Do something with the string. */ if (line[0] != '\0' && line[0] != '/') { @@ -65,6 +66,10 @@ int main(int argc, char **argv) { /* The "/historylen" command will change the history len. */ int len = atoi(line+11); linenoiseHistorySetMaxLen(len); + } else if (!strncmp(line, "/mask", 5)) { + linenoiseMaskModeEnable(); + } else if (!strncmp(line, "/unmask", 7)) { + linenoiseMaskModeDisable(); } else if (line[0] == '/') { printf("Unreconized command: %s\n", line); } diff --git a/deps/linenoise/linenoise.c b/deps/linenoise/linenoise.c index fce14a7c5..01c3b9350 100644 --- a/deps/linenoise/linenoise.c +++ b/deps/linenoise/linenoise.c @@ -125,6 +125,7 @@ static linenoiseHintsCallback *hintsCallback = NULL; static linenoiseFreeHintsCallback *freeHintsCallback = NULL; static struct termios orig_termios; /* In order to restore at exit.*/ +static int maskmode = 0; /* Show "***" instead of input. For passwords. */ static int rawmode = 0; /* For atexit() function to check if restore is needed*/ static int mlmode = 0; /* Multi line mode. Default is single line. */ static int atexit_registered = 0; /* Register atexit just 1 time. */ @@ -197,6 +198,19 @@ FILE *lndebug_fp = NULL; /* ======================= Low level terminal handling ====================== */ +/* Enable "mask mode". When it is enabled, instead of the input that + * the user is typing, the terminal will just display a corresponding + * number of asterisks, like "****". This is useful for passwords and other + * secrets that should not be displayed. */ +void linenoiseMaskModeEnable(void) { + maskmode = 1; +} + +/* Disable mask mode. */ +void linenoiseMaskModeDisable(void) { + maskmode = 0; +} + /* Set if to use or not the multi line mode. */ void linenoiseSetMultiLine(int ml) { mlmode = ml; @@ -485,6 +499,8 @@ void refreshShowHints(struct abuf *ab, struct linenoiseState *l, int plen) { if (bold == 1 && color == -1) color = 37; if (color != -1 || bold != 0) snprintf(seq,64,"\033[%d;%d;49m",bold,color); + else + seq[0] = '\0'; abAppend(ab,seq,strlen(seq)); abAppend(ab,hint,hintlen); if (color != -1 || bold != 0) @@ -523,7 +539,11 @@ static void refreshSingleLine(struct linenoiseState *l) { abAppend(&ab,seq,strlen(seq)); /* Write the prompt and the current buffer content */ abAppend(&ab,l->prompt,strlen(l->prompt)); - abAppend(&ab,buf,len); + if (maskmode == 1) { + while (len--) abAppend(&ab,"*",1); + } else { + abAppend(&ab,buf,len); + } /* Show hits if any. */ refreshShowHints(&ab,l,plen); /* Erase to right */ @@ -577,7 +597,11 @@ static void refreshMultiLine(struct linenoiseState *l) { /* Write the prompt and the current buffer content */ abAppend(&ab,l->prompt,strlen(l->prompt)); - abAppend(&ab,l->buf,l->len); + if (maskmode == 1) { + for (uint i = 0; i < l->len; i++) abAppend(&ab,"*",1); + } else { + abAppend(&ab,l->buf,l->len); + } /* Show hits if any. */ refreshShowHints(&ab,l,plen); @@ -645,7 +669,8 @@ int linenoiseEditInsert(struct linenoiseState *l, char c) { if ((!mlmode && l->plen+l->len < l->cols && !hintsCallback)) { /* Avoid a full update of the line in the * trivial case. */ - if (write(l->ofd,&c,1) == -1) return -1; + char d = (maskmode==1) ? '*' : c; + if (write(l->ofd,&d,1) == -1) return -1; } else { refreshLine(l); } diff --git a/deps/linenoise/linenoise.h b/deps/linenoise/linenoise.h index ed20232c5..6dfee73bc 100644 --- a/deps/linenoise/linenoise.h +++ b/deps/linenoise/linenoise.h @@ -65,6 +65,8 @@ int linenoiseHistoryLoad(const char *filename); void linenoiseClearScreen(void); void linenoiseSetMultiLine(int ml); void linenoisePrintKeyCodes(void); +void linenoiseMaskModeEnable(void); +void linenoiseMaskModeDisable(void); #ifdef __cplusplus } From 8e3c92fff84a4edb2c3468e54dd2b19079b6db8c Mon Sep 17 00:00:00 2001 From: lifubang Date: Thu, 5 Mar 2020 18:17:32 +0800 Subject: [PATCH 08/11] add askpass mode Signed-off-by: lifubang --- src/redis-cli.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 54898f42e..b44db9a1e 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -229,6 +229,7 @@ static struct config { int hotkeys; int stdinarg; /* get last arg from stdin. (-x option) */ char *auth; + int askpass; char *user; int output; /* output mode, see OUTPUT_* defines */ sds mb_delim; @@ -1450,6 +1451,8 @@ static int parseOptions(int argc, char **argv) { config.dbnum = atoi(argv[++i]); } else if (!strcmp(argv[i], "--no-auth-warning")) { config.no_auth_warning = 1; + } else if (!strcmp(argv[i], "--askpass")) { + config.askpass = 1; } else if ((!strcmp(argv[i],"-a") || !strcmp(argv[i],"--pass")) && !lastarg) { @@ -1690,6 +1693,9 @@ static void usage(void) { " (if both are used, this argument takes predecence).\n" " --user Used to send ACL style 'AUTH username pass'. Needs -a.\n" " --pass Alias of -a for consistency with the new --user option.\n" +" --askpass Force user to input password with mask from STDIN.\n" +" If this argument is used, '-a' and " REDIS_CLI_AUTH_ENV "\n" +" environment variable will be ignored.\n" " -u Server URI.\n" " -r Execute specified command N times.\n" " -i When -r is used, waits seconds per command.\n" @@ -7858,6 +7864,13 @@ static void intrinsicLatencyMode(void) { } } +static sds askPassword() { + linenoiseMaskModeEnable(); + sds auth = linenoise("Please input password: "); + linenoiseMaskModeDisable(); + return auth; +} + /*------------------------------------------------------------------------------ * Program main() *--------------------------------------------------------------------------- */ @@ -7894,6 +7907,7 @@ int main(int argc, char **argv) { config.hotkeys = 0; config.stdinarg = 0; config.auth = NULL; + config.askpass = 0; config.user = NULL; config.eval = NULL; config.eval_ldb = 0; @@ -7935,6 +7949,10 @@ int main(int argc, char **argv) { parseEnv(); + if (config.askpass) { + config.auth = askPassword(); + } + #ifdef USE_OPENSSL if (config.tls) { ERR_load_crypto_strings(); @@ -8044,4 +8062,4 @@ int main(int argc, char **argv) { } else { return noninteractive(argc,convertToSds(argc,argv)); } -} +} \ No newline at end of file From 317496405af981dc6a4c2cb657ec7637ea142aa4 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 12 Mar 2020 15:53:08 +0100 Subject: [PATCH 09/11] Update linenoise. --- deps/linenoise/linenoise.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deps/linenoise/linenoise.c b/deps/linenoise/linenoise.c index 01c3b9350..cfe51e768 100644 --- a/deps/linenoise/linenoise.c +++ b/deps/linenoise/linenoise.c @@ -598,7 +598,8 @@ static void refreshMultiLine(struct linenoiseState *l) { /* Write the prompt and the current buffer content */ abAppend(&ab,l->prompt,strlen(l->prompt)); if (maskmode == 1) { - for (uint i = 0; i < l->len; i++) abAppend(&ab,"*",1); + unsigned int i; + for (i = 0; i < l->len; i++) abAppend(&ab,"*",1); } else { abAppend(&ab,l->buf,l->len); } From e8c11fe29dce1a6004d99f871d9d7dce28622532 Mon Sep 17 00:00:00 2001 From: fengpf <18221167541@163.com> Date: Thu, 12 Mar 2020 20:44:32 +0800 Subject: [PATCH 10/11] fix comments in latency.c --- .gitignore | 1 + src/latency.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index de626d61b..e445fd201 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ deps/lua/src/liblua.a *.dSYM Makefile.dep .vscode/* +.idea/* diff --git a/src/latency.c b/src/latency.c index 74ced72a5..9a291ac9b 100644 --- a/src/latency.c +++ b/src/latency.c @@ -85,7 +85,7 @@ int THPGetAnonHugePagesSize(void) { /* ---------------------------- Latency API --------------------------------- */ /* Latency monitor initialization. We just need to create the dictionary - * of time series, each time serie is craeted on demand in order to avoid + * of time series, each time serie is created on demand in order to avoid * having a fixed list to maintain. */ void latencyMonitorInit(void) { server.latency_events = dictCreate(&latencyTimeSeriesDictType,NULL); From c609bf3f2c7f0982f632f82623ee4802868b8ef1 Mon Sep 17 00:00:00 2001 From: "bodong.ybd" Date: Wed, 11 Mar 2020 20:55:51 +0800 Subject: [PATCH 11/11] Fix bug of tcl test using external server --- tests/support/server.tcl | 7 +++++-- tests/test_helper.tcl | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/support/server.tcl b/tests/support/server.tcl index d086366dc..400017c5f 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -159,9 +159,12 @@ proc start_server {options {code undefined}} { if {$::external} { if {[llength $::servers] == 0} { set srv {} + # In test_server_main(tests/test_helper.tcl:215~218), increase the value of start_port + # and assign it to ::port through the `--port` option, so we need to reduce it. + set baseport [expr {$::port-100}] dict set srv "host" $::host - dict set srv "port" $::port - set client [redis $::host $::port 0 $::tls] + dict set srv "port" $baseport + set client [redis $::host $baseport 0 $::tls] dict set srv "client" $client $client select 9 diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index fa5579669..4dbead193 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -505,6 +505,9 @@ for {set j 0} {$j < [llength $argv]} {incr j} { } elseif {$opt eq {--host}} { set ::external 1 set ::host $arg + # If we use an external server, we can only set numclients to 1, + # otherwise the port will be miscalculated. + set ::numclients 1 incr j } elseif {$opt eq {--port}} { set ::port $arg