From b3e4aa6797f484ed08d3189af0dd7f143dbaf421 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 6 Mar 2020 11:09:52 +0100 Subject: [PATCH 01/11] Fix release notes spelling mistake. --- 00-RELEASENOTES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index ad2648bcb..9847d37dd 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -30,7 +30,7 @@ New features and improvements: * ACL LOG: log denied commands, keys accesses and authentications. * Client side caching redesigned. Now we use keys not caching slots. * Client side caching: Broadcasting mode implemented. -* Client side caching: OPTIN/OUTPUT modes implemented. +* Client side caching: OPTIN/OPTOUT modes implemented. * Remove RDB files used for replication in persistence-less instances (option). Fixes (only selected ones, see commits for all the fixes): From 70e0e49952ef40675b515a9483767be898640609 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 12 Mar 2020 12:59:44 +0100 Subject: [PATCH 02/11] ae.c: fix crash when resizing the event loop. See #6964. The root cause is that the event loop may be resized from an event callback itself, causing the event pointer to be invalid. --- src/ae.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ae.c b/src/ae.c index d2248fe5c..1bf6cbfbf 100644 --- a/src/ae.c +++ b/src/ae.c @@ -464,6 +464,7 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) if (!invert && fe->mask & mask & AE_READABLE) { fe->rfileProc(eventLoop,fd,fe->clientData,mask); fired++; + fe = &eventLoop->events[fd]; /* Refresh in case of resize. */ } /* Fire the writable event. */ @@ -476,8 +477,11 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) /* If we have to invert the call, fire the readable event now * after the writable one. */ - if (invert && fe->mask & mask & AE_READABLE) { - if (!fired || fe->wfileProc != fe->rfileProc) { + if (invert) { + fe = &eventLoop->events[fd]; /* Refresh in case of resize. */ + if ((fe->mask & mask & AE_READABLE) && + (!fired || fe->wfileProc != fe->rfileProc)) + { fe->rfileProc(eventLoop,fd,fe->clientData,mask); fired++; } From 6173815408a379c5a060283c589e83c8f2f1134f Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 5 Mar 2020 16:55:14 +0200 Subject: [PATCH 03/11] fix for flaky psync2 test *** [err]: PSYNC2: total sum of full synchronizations is exactly 4 in tests/integration/psync2.tcl Expected 5 == 4 (context: type eval line 6 cmd {assert {$sum == 4}} proc ::test) issue was that sometime the test got an unexpected full sync since it tried to switch to the replica before it was in sync with it's master. --- tests/integration/psync2.tcl | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/integration/psync2.tcl b/tests/integration/psync2.tcl index d1212b640..333736ffa 100644 --- a/tests/integration/psync2.tcl +++ b/tests/integration/psync2.tcl @@ -114,6 +114,27 @@ start_server {} { } } + # wait for all the slaves to be in sync with the master + set master_ofs [status $R($master_id) master_repl_offset] + wait_for_condition 500 100 { + $master_ofs == [status $R(0) master_repl_offset] && + $master_ofs == [status $R(1) master_repl_offset] && + $master_ofs == [status $R(2) master_repl_offset] && + $master_ofs == [status $R(3) master_repl_offset] && + $master_ofs == [status $R(4) master_repl_offset] + } else { + if {$debug_msg} { + for {set j 0} {$j < 5} {incr j} { + puts "$j: sync_full: [status $R($j) sync_full]" + puts "$j: id1 : [status $R($j) master_replid]:[status $R($j) master_repl_offset]" + puts "$j: id2 : [status $R($j) master_replid2]:[status $R($j) second_repl_offset]" + puts "$j: backlog : firstbyte=[status $R($j) repl_backlog_first_byte_offset] len=[status $R($j) repl_backlog_histlen]" + puts "---" + } + } + fail "Slaves are not in sync with the master after too long time." + } + # Put down the old master so that it cannot generate more # replication stream, this way in the next master switch, the time at # which we move slaves away is not important, each will have full From 23d5e8b8e46c706bd5a93ed3b3e0782909e926b2 Mon Sep 17 00:00:00 2001 From: Johannes Truschnigg Date: Thu, 19 Dec 2019 21:47:24 +0100 Subject: [PATCH 04/11] Signal systemd readiness atfer Partial Resync "Partial Resynchronization" is a special variant of replication success that we have to tell systemd about if it is managing redis-server via a Type=Notify service unit. --- src/replication.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/replication.c b/src/replication.c index 429e1d8a8..31e14d7fe 100644 --- a/src/replication.c +++ b/src/replication.c @@ -2284,6 +2284,10 @@ void syncWithMaster(connection *conn) { if (psync_result == PSYNC_CONTINUE) { serverLog(LL_NOTICE, "MASTER <-> REPLICA sync: Master accepted a Partial Resynchronization."); + if (server.supervised_mode == SUPERVISED_SYSTEMD) { + redisCommunicateSystemd("STATUS=MASTER <-> REPLICA sync: Partial Resynchronization accepted. Ready to accept connections.\n"); + redisCommunicateSystemd("READY=1\n"); + } return; } From d28cbaf770734d5e839b42853ff96dc2d1dd6816 Mon Sep 17 00:00:00 2001 From: Jamie Scott Date: Tue, 3 Mar 2020 18:03:16 -0800 Subject: [PATCH 05/11] Update Redis.conf to improve TLS usability When using TLS with a Redis.conf file the line for TLS reading tls-cert-file redis.crt tls-key-file redis.key is interpreted as one complete directive. I am separating this on two separate lines to improve usability so users do not get the below error. ubuntu@ip-172-31-29-250:~/redis-6.0-rc1$ ./src/redis-server redis.conf *** FATAL CONFIG FILE ERROR *** Reading the configuration file, at line 145 >>> 'tls-cert-file redis.crt tls-key-file redis.key' wrong number of arguments ubuntu@ip-172-31-29-250:~/redis-6.0-rc1$ vi redis.conf ubuntu@ip-172-31-29-250:~/redis-6.0-rc1$ ./src/redis-server redis.conf 23085:C 04 Mar 2020 01:58:12.631 # oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo 23085:C 04 Mar 2020 01:58:12.631 # Redis version=5.9.101, bits=64, commit=00000000, modified=0, pid=23085, just started 23085:C 04 Mar 2020 01:58:12.631 # Configuration loaded 23085:M 04 Mar 2020 01:58:12.632 * Increased maximum number of open files to 10032 (it was originally set to 1024). --- redis.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/redis.conf b/redis.conf index 8609a9f57..6fa5e87c0 100644 --- a/redis.conf +++ b/redis.conf @@ -142,7 +142,8 @@ tcp-keepalive 300 # server to connected clients, masters or cluster peers. These files should be # PEM formatted. # -# tls-cert-file redis.crt tls-key-file redis.key +# tls-cert-file redis.crt +# tls-key-file redis.key # Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange: # From e5a063bcb405399ea71ec4a6fdead61ad4b78bb2 Mon Sep 17 00:00:00 2001 From: Jamie Scott Date: Mon, 9 Mar 2020 12:53:44 -0700 Subject: [PATCH 06/11] Remove default guidance in Redis.conf Removing the default guidance in Redis.conf since this is not an available value. --- redis.conf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/redis.conf b/redis.conf index 6fa5e87c0..c9d256bef 100644 --- a/redis.conf +++ b/redis.conf @@ -176,8 +176,7 @@ tcp-keepalive 300 # tls-cluster yes # Explicitly specify TLS versions to support. Allowed values are case insensitive -# and include "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3" (OpenSSL >= 1.1.1) or -# "default" which is currently >= TLSv1.1. +# and include "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3" (OpenSSL >= 1.1.1) # # tls-protocols TLSv1.2 From e1c29434b2f79425a1d006d7dd6737dd1fbbcc15 Mon Sep 17 00:00:00 2001 From: lifubang Date: Thu, 5 Mar 2020 18:13:43 +0800 Subject: [PATCH 07/11] update linenoise to https://github.com/antirez/linenoise/tree/fc9667a81d43911a6690fb1e68c16e6e3bb8df05 Signed-off-by: lifubang --- deps/linenoise/README.markdown | 25 ++++++++++++++++++++++++- deps/linenoise/example.c | 5 +++++ deps/linenoise/linenoise.c | 31 ++++++++++++++++++++++++++++--- deps/linenoise/linenoise.h | 2 ++ 4 files changed, 59 insertions(+), 4 deletions(-) diff --git a/deps/linenoise/README.markdown b/deps/linenoise/README.markdown index e01642cf8..1afea2ae6 100644 --- a/deps/linenoise/README.markdown +++ b/deps/linenoise/README.markdown @@ -21,7 +21,7 @@ So what usually happens is either: The result is a pollution of binaries without line editing support. -So I spent more or less two hours doing a reality check resulting in this little library: is it *really* needed for a line editing library to be 20k lines of code? Apparently not, it is possibe to get a very small, zero configuration, trivial to embed library, that solves the problem. Smaller programs will just include this, supporing line editing out of the box. Larger programs may use this little library or just checking with configure if readline/libedit is available and resorting to Linenoise if not. +So I spent more or less two hours doing a reality check resulting in this little library: is it *really* needed for a line editing library to be 20k lines of code? Apparently not, it is possibe to get a very small, zero configuration, trivial to embed library, that solves the problem. Smaller programs will just include this, supporting line editing out of the box. Larger programs may use this little library or just checking with configure if readline/libedit is available and resorting to Linenoise if not. ## Terminals, in 2010. @@ -126,6 +126,24 @@ Linenoise has direct support for persisting the history into an history file. The functions `linenoiseHistorySave` and `linenoiseHistoryLoad` do just that. Both functions return -1 on error and 0 on success. +## Mask mode + +Sometimes it is useful to allow the user to type passwords or other +secrets that should not be displayed. For such situations linenoise supports +a "mask mode" that will just replace the characters the user is typing +with `*` characters, like in the following example: + + $ ./linenoise_example + hello> get mykey + echo: 'get mykey' + hello> /mask + hello> ********* + +You can enable and disable mask mode using the following two functions: + + void linenoiseMaskModeEnable(void); + void linenoiseMaskModeDisable(void); + ## Completion Linenoise supports completion, which is the ability to complete the user @@ -222,3 +240,8 @@ Sometimes you may want to clear the screen as a result of something the user typed. You can do this by calling the following function: void linenoiseClearScreen(void); + +## Related projects + +* [Linenoise NG](https://github.com/arangodb/linenoise-ng) is a fork of Linenoise that aims to add more advanced features like UTF-8 support, Windows support and other features. Uses C++ instead of C as development language. +* [Linenoise-swift](https://github.com/andybest/linenoise-swift) is a reimplementation of Linenoise written in Swift. diff --git a/deps/linenoise/example.c b/deps/linenoise/example.c index 3a544d3c6..74358c323 100644 --- a/deps/linenoise/example.c +++ b/deps/linenoise/example.c @@ -55,6 +55,7 @@ int main(int argc, char **argv) { * * The typed string is returned as a malloc() allocated string by * linenoise, so the user needs to free() it. */ + while((line = linenoise("hello> ")) != NULL) { /* Do something with the string. */ if (line[0] != '\0' && line[0] != '/') { @@ -65,6 +66,10 @@ int main(int argc, char **argv) { /* The "/historylen" command will change the history len. */ int len = atoi(line+11); linenoiseHistorySetMaxLen(len); + } else if (!strncmp(line, "/mask", 5)) { + linenoiseMaskModeEnable(); + } else if (!strncmp(line, "/unmask", 7)) { + linenoiseMaskModeDisable(); } else if (line[0] == '/') { printf("Unreconized command: %s\n", line); } diff --git a/deps/linenoise/linenoise.c b/deps/linenoise/linenoise.c index fce14a7c5..01c3b9350 100644 --- a/deps/linenoise/linenoise.c +++ b/deps/linenoise/linenoise.c @@ -125,6 +125,7 @@ static linenoiseHintsCallback *hintsCallback = NULL; static linenoiseFreeHintsCallback *freeHintsCallback = NULL; static struct termios orig_termios; /* In order to restore at exit.*/ +static int maskmode = 0; /* Show "***" instead of input. For passwords. */ static int rawmode = 0; /* For atexit() function to check if restore is needed*/ static int mlmode = 0; /* Multi line mode. Default is single line. */ static int atexit_registered = 0; /* Register atexit just 1 time. */ @@ -197,6 +198,19 @@ FILE *lndebug_fp = NULL; /* ======================= Low level terminal handling ====================== */ +/* Enable "mask mode". When it is enabled, instead of the input that + * the user is typing, the terminal will just display a corresponding + * number of asterisks, like "****". This is useful for passwords and other + * secrets that should not be displayed. */ +void linenoiseMaskModeEnable(void) { + maskmode = 1; +} + +/* Disable mask mode. */ +void linenoiseMaskModeDisable(void) { + maskmode = 0; +} + /* Set if to use or not the multi line mode. */ void linenoiseSetMultiLine(int ml) { mlmode = ml; @@ -485,6 +499,8 @@ void refreshShowHints(struct abuf *ab, struct linenoiseState *l, int plen) { if (bold == 1 && color == -1) color = 37; if (color != -1 || bold != 0) snprintf(seq,64,"\033[%d;%d;49m",bold,color); + else + seq[0] = '\0'; abAppend(ab,seq,strlen(seq)); abAppend(ab,hint,hintlen); if (color != -1 || bold != 0) @@ -523,7 +539,11 @@ static void refreshSingleLine(struct linenoiseState *l) { abAppend(&ab,seq,strlen(seq)); /* Write the prompt and the current buffer content */ abAppend(&ab,l->prompt,strlen(l->prompt)); - abAppend(&ab,buf,len); + if (maskmode == 1) { + while (len--) abAppend(&ab,"*",1); + } else { + abAppend(&ab,buf,len); + } /* Show hits if any. */ refreshShowHints(&ab,l,plen); /* Erase to right */ @@ -577,7 +597,11 @@ static void refreshMultiLine(struct linenoiseState *l) { /* Write the prompt and the current buffer content */ abAppend(&ab,l->prompt,strlen(l->prompt)); - abAppend(&ab,l->buf,l->len); + if (maskmode == 1) { + for (uint i = 0; i < l->len; i++) abAppend(&ab,"*",1); + } else { + abAppend(&ab,l->buf,l->len); + } /* Show hits if any. */ refreshShowHints(&ab,l,plen); @@ -645,7 +669,8 @@ int linenoiseEditInsert(struct linenoiseState *l, char c) { if ((!mlmode && l->plen+l->len < l->cols && !hintsCallback)) { /* Avoid a full update of the line in the * trivial case. */ - if (write(l->ofd,&c,1) == -1) return -1; + char d = (maskmode==1) ? '*' : c; + if (write(l->ofd,&d,1) == -1) return -1; } else { refreshLine(l); } diff --git a/deps/linenoise/linenoise.h b/deps/linenoise/linenoise.h index ed20232c5..6dfee73bc 100644 --- a/deps/linenoise/linenoise.h +++ b/deps/linenoise/linenoise.h @@ -65,6 +65,8 @@ int linenoiseHistoryLoad(const char *filename); void linenoiseClearScreen(void); void linenoiseSetMultiLine(int ml); void linenoisePrintKeyCodes(void); +void linenoiseMaskModeEnable(void); +void linenoiseMaskModeDisable(void); #ifdef __cplusplus } From c0c67c9be30fa197ec86ba33915c6f6186a5f166 Mon Sep 17 00:00:00 2001 From: lifubang Date: Thu, 5 Mar 2020 18:17:32 +0800 Subject: [PATCH 08/11] add askpass mode Signed-off-by: lifubang --- src/redis-cli.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 54898f42e..b44db9a1e 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -229,6 +229,7 @@ static struct config { int hotkeys; int stdinarg; /* get last arg from stdin. (-x option) */ char *auth; + int askpass; char *user; int output; /* output mode, see OUTPUT_* defines */ sds mb_delim; @@ -1450,6 +1451,8 @@ static int parseOptions(int argc, char **argv) { config.dbnum = atoi(argv[++i]); } else if (!strcmp(argv[i], "--no-auth-warning")) { config.no_auth_warning = 1; + } else if (!strcmp(argv[i], "--askpass")) { + config.askpass = 1; } else if ((!strcmp(argv[i],"-a") || !strcmp(argv[i],"--pass")) && !lastarg) { @@ -1690,6 +1693,9 @@ static void usage(void) { " (if both are used, this argument takes predecence).\n" " --user Used to send ACL style 'AUTH username pass'. Needs -a.\n" " --pass Alias of -a for consistency with the new --user option.\n" +" --askpass Force user to input password with mask from STDIN.\n" +" If this argument is used, '-a' and " REDIS_CLI_AUTH_ENV "\n" +" environment variable will be ignored.\n" " -u Server URI.\n" " -r Execute specified command N times.\n" " -i When -r is used, waits seconds per command.\n" @@ -7858,6 +7864,13 @@ static void intrinsicLatencyMode(void) { } } +static sds askPassword() { + linenoiseMaskModeEnable(); + sds auth = linenoise("Please input password: "); + linenoiseMaskModeDisable(); + return auth; +} + /*------------------------------------------------------------------------------ * Program main() *--------------------------------------------------------------------------- */ @@ -7894,6 +7907,7 @@ int main(int argc, char **argv) { config.hotkeys = 0; config.stdinarg = 0; config.auth = NULL; + config.askpass = 0; config.user = NULL; config.eval = NULL; config.eval_ldb = 0; @@ -7935,6 +7949,10 @@ int main(int argc, char **argv) { parseEnv(); + if (config.askpass) { + config.auth = askPassword(); + } + #ifdef USE_OPENSSL if (config.tls) { ERR_load_crypto_strings(); @@ -8044,4 +8062,4 @@ int main(int argc, char **argv) { } else { return noninteractive(argc,convertToSds(argc,argv)); } -} +} \ No newline at end of file From 916dd79f190b51c78fdff7677a3a00d81e2088bf Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 12 Mar 2020 15:53:08 +0100 Subject: [PATCH 09/11] Update linenoise. --- deps/linenoise/linenoise.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deps/linenoise/linenoise.c b/deps/linenoise/linenoise.c index 01c3b9350..cfe51e768 100644 --- a/deps/linenoise/linenoise.c +++ b/deps/linenoise/linenoise.c @@ -598,7 +598,8 @@ static void refreshMultiLine(struct linenoiseState *l) { /* Write the prompt and the current buffer content */ abAppend(&ab,l->prompt,strlen(l->prompt)); if (maskmode == 1) { - for (uint i = 0; i < l->len; i++) abAppend(&ab,"*",1); + unsigned int i; + for (i = 0; i < l->len; i++) abAppend(&ab,"*",1); } else { abAppend(&ab,l->buf,l->len); } From 0e5820d893467b2607515e3ab21c824b4ce3fa8c Mon Sep 17 00:00:00 2001 From: fengpf <18221167541@163.com> Date: Thu, 12 Mar 2020 20:44:32 +0800 Subject: [PATCH 10/11] fix comments in latency.c --- .gitignore | 1 + src/latency.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index de626d61b..e445fd201 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ deps/lua/src/liblua.a *.dSYM Makefile.dep .vscode/* +.idea/* diff --git a/src/latency.c b/src/latency.c index 74ced72a5..9a291ac9b 100644 --- a/src/latency.c +++ b/src/latency.c @@ -85,7 +85,7 @@ int THPGetAnonHugePagesSize(void) { /* ---------------------------- Latency API --------------------------------- */ /* Latency monitor initialization. We just need to create the dictionary - * of time series, each time serie is craeted on demand in order to avoid + * of time series, each time serie is created on demand in order to avoid * having a fixed list to maintain. */ void latencyMonitorInit(void) { server.latency_events = dictCreate(&latencyTimeSeriesDictType,NULL); From 76d57161d960f27ac92765118e3b80fef8847b8f Mon Sep 17 00:00:00 2001 From: "bodong.ybd" Date: Wed, 11 Mar 2020 20:55:51 +0800 Subject: [PATCH 11/11] Fix bug of tcl test using external server --- tests/support/server.tcl | 7 +++++-- tests/test_helper.tcl | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/support/server.tcl b/tests/support/server.tcl index d086366dc..400017c5f 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -159,9 +159,12 @@ proc start_server {options {code undefined}} { if {$::external} { if {[llength $::servers] == 0} { set srv {} + # In test_server_main(tests/test_helper.tcl:215~218), increase the value of start_port + # and assign it to ::port through the `--port` option, so we need to reduce it. + set baseport [expr {$::port-100}] dict set srv "host" $::host - dict set srv "port" $::port - set client [redis $::host $::port 0 $::tls] + dict set srv "port" $baseport + set client [redis $::host $baseport 0 $::tls] dict set srv "client" $client $client select 9 diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index fa5579669..4dbead193 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -505,6 +505,9 @@ for {set j 0} {$j < [llength $argv]} {incr j} { } elseif {$opt eq {--host}} { set ::external 1 set ::host $arg + # If we use an external server, we can only set numclients to 1, + # otherwise the port will be miscalculated. + set ::numclients 1 incr j } elseif {$opt eq {--port}} { set ::port $arg