From 5907dccdaef66e78cc986783ca5df2a1a8830042 Mon Sep 17 00:00:00 2001 From: John Sully Date: Sun, 5 Jan 2020 20:19:28 -0500 Subject: [PATCH 1/7] Add new flags to example configuration file Former-commit-id: bb38628f0ef4f417db968154cc37d103f11b4c63 --- keydb.conf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/keydb.conf b/keydb.conf index 5c31305d8..03087b0c0 100644 --- a/keydb.conf +++ b/keydb.conf @@ -1561,3 +1561,10 @@ server-threads 2 # replicas will still sync in the normal way and incorrect ordering when # bringing up replicas can result in data loss (the first master will win). # active-replica yes + +# Enable Pro? KeyDB pro provides support for pro only features +# note: you may omit the license key to demo pro features for a limited time +# enable-pro [License Key] + +# Enable FLASH support? (Pro Only) +# storage-provider flash /path/to/flash/db From 174a8b886d64df99802d439aacad7a561f7014af Mon Sep 17 00:00:00 2001 From: John Sully Date: Mon, 6 Jan 2020 14:24:37 -0500 Subject: [PATCH 2/7] pro launch should respect KEYDB_PRO_DIRECTORY Former-commit-id: d5f8df59977194ee0cfce798364eb5620435e6f3 --- src/server.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/server.cpp b/src/server.cpp index f1618419a..12dff6a85 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -5199,7 +5199,10 @@ int main(int argc, char **argv) { } if (cserver.fUsePro) { - execv("keydb-pro-server", argv); + const char *keydb_pro_dir = getenv("KEYDB_PRO_DIRECTORY"); + sds path = sdsnew(keydb_pro_dir); + path = sdscat(path, "keydb-pro-server"); + execv(path, argv); perror("Failed launch the pro binary"); exit(EXIT_FAILURE); } From 09ff507ee9000fb9aa830c14668dac8730088981 Mon Sep 17 00:00:00 2001 From: John Sully Date: Fri, 10 Jan 2020 14:19:44 -0500 Subject: [PATCH 3/7] Update graphs Former-commit-id: 7475ea1b0b7d8b2f7a8dfb6c6b754126d5160e5b --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8d8c59ff0..e4841e23d 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,9 @@ KeyDB maintains full compatibility with the Redis protocol, modules, and scripts On the same hardware KeyDB can perform twice as many queries per second as Redis, with 60% lower latency. Active-Replication simplifies hot-spare failover allowing you to easily distribute writes over replicas and use simple TCP based load balancing/failover. KeyDB's higher performance allows you to do more on less hardware which reduces operation costs and complexity. - + + +See the full benchmark results and setup information here: https://docs.keydb.dev/blog/2019/10/07/blog-post/ Why fork Redis? --------------- From 900e596440b9bbabf05a55d25acf55e168512174 Mon Sep 17 00:00:00 2001 From: John Sully Date: Fri, 10 Jan 2020 14:21:11 -0500 Subject: [PATCH 4/7] Update README.md Former-commit-id: 0ba0a6ab0a2c552e17a4344f9935b40f26c694e1 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e4841e23d..70b879d52 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ KeyDB maintains full compatibility with the Redis protocol, modules, and scripts On the same hardware KeyDB can perform twice as many queries per second as Redis, with 60% lower latency. Active-Replication simplifies hot-spare failover allowing you to easily distribute writes over replicas and use simple TCP based load balancing/failover. KeyDB's higher performance allows you to do more on less hardware which reduces operation costs and complexity. - + See the full benchmark results and setup information here: https://docs.keydb.dev/blog/2019/10/07/blog-post/ From 18cc4b3ebe4675eb17bb44624aa0ebe5709eb5b9 Mon Sep 17 00:00:00 2001 From: John Sully Date: Fri, 10 Jan 2020 17:46:02 -0500 Subject: [PATCH 5/7] Ignore other flags after pro is enabled since we won't understand them Former-commit-id: 049b48ef79fdde4c9b1c9f227a329bd9223e2b1d --- src/config.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/config.cpp b/src/config.cpp index f1e562f6b..c056b98dc 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -804,6 +804,7 @@ void loadServerConfigFromString(char *config) { // NOP, handled in main } else if (!strcasecmp(argv[0],"enable-pro")) { cserver.fUsePro = true; + break; } else { err = "Bad directive or wrong number of arguments"; goto loaderr; } From b9b8272724bed931fa86b72c2c1ecd6386be3a99 Mon Sep 17 00:00:00 2001 From: John Sully Date: Sat, 11 Jan 2020 16:34:09 -0500 Subject: [PATCH 6/7] Avoid crash due to excessive posted functions for AOF rewrite Former-commit-id: aa6409f2e8a37288eb4953fbcf3a82e02545348b --- src/aof.cpp | 12 ++++++++---- src/server.h | 1 + 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/aof.cpp b/src/aof.cpp index d48b664d2..b82be9a34 100644 --- a/src/aof.cpp +++ b/src/aof.cpp @@ -165,10 +165,14 @@ void aofRewriteBufferAppend(unsigned char *s, unsigned long len) { /* Install a file event to send data to the rewrite child if there is * not one already. */ - aePostFunction(g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN].el, []{ - if (g_pserver->aof_pipe_write_data_to_child >= 0) - aeCreateFileEvent(g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN].el, g_pserver->aof_pipe_write_data_to_child, AE_WRITABLE, aofChildWriteDiffData, NULL); - }); + if (!g_pserver->aof_rewrite_pending) { + g_pserver->aof_rewrite_pending = true; + aePostFunction(g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN].el, [] { + g_pserver->aof_rewrite_pending = false; + if (g_pserver->aof_pipe_write_data_to_child >= 0) + aeCreateFileEvent(g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN].el, g_pserver->aof_pipe_write_data_to_child, AE_WRITABLE, aofChildWriteDiffData, NULL); + }); + } } /* Write the buffer (possibly composed of multiple blocks) into the specified diff --git a/src/server.h b/src/server.h index 5733264e0..6b8903092 100644 --- a/src/server.h +++ b/src/server.h @@ -1733,6 +1733,7 @@ struct redisServer { int aof_stop_sending_diff; /* If true stop sending accumulated diffs to child process. */ sds aof_child_diff; /* AOF diff accumulator child side. */ + int aof_rewrite_pending = 0; /* is a call to aofChildWriteDiffData already queued? */ /* RDB persistence */ long long dirty; /* Changes to DB from the last save */ long long dirty_before_bgsave; /* Used to restore dirty on failed BGSAVE */ From 793f0bf274cd360975731cc743be3e8731f68280 Mon Sep 17 00:00:00 2001 From: John Sully Date: Sat, 11 Jan 2020 19:28:13 -0500 Subject: [PATCH 7/7] Avoid locking as late as possible Former-commit-id: acc894fe41d4e8869f28928cb0feffa1792c11c4 --- src/aelocker.h | 4 +- src/server.cpp | 101 ++++++++++++++++++++++++++----------------------- 2 files changed, 57 insertions(+), 48 deletions(-) diff --git a/src/aelocker.h b/src/aelocker.h index eca15f491..528955380 100644 --- a/src/aelocker.h +++ b/src/aelocker.h @@ -11,9 +11,11 @@ public: void arm(client *c) // if a client is passed, then the client is already locked { + if (m_fArmed) + return; + if (c != nullptr) { - serverAssert(!m_fArmed); serverAssert(c->lock.fOwnLock()); if (!aeTryAcquireLock(true /*fWeak*/)) // avoid locking the client if we can diff --git a/src/server.cpp b/src/server.cpp index 12dff6a85..b99cc97a3 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -3585,9 +3585,6 @@ int processCommand(client *c, int callFlags) { } incrementMvccTstamp(); - - if (!locker.isArmed()) - locker.arm(c); /* Handle the maxmemory directive. * @@ -3596,6 +3593,7 @@ int processCommand(client *c, int callFlags) { * condition, to avoid mixing the propagation of scripts with the * propagation of DELs due to eviction. */ if (g_pserver->maxmemory && !g_pserver->lua_timedout) { + locker.arm(c); int out_of_memory = freeMemoryIfNeededAndSafe() == C_ERR; /* freeMemoryIfNeeded may flush replica output buffers. This may result * into a replica, that may be the active client, to be freed. */ @@ -3615,44 +3613,48 @@ int processCommand(client *c, int callFlags) { /* Don't accept write commands if there are problems persisting on disk * and if this is a master instance. */ - int deny_write_type = writeCommandsDeniedByDiskError(); - if (deny_write_type != DISK_ERROR_TYPE_NONE && - listLength(g_pserver->masters) == 0 && - (c->cmd->flags & CMD_WRITE || - c->cmd->proc == pingCommand)) + if (c->cmd->flags & CMD_WRITE || c->cmd->proc == pingCommand) { - flagTransaction(c); - if (deny_write_type == DISK_ERROR_TYPE_RDB) - addReply(c, shared.bgsaveerr); - else - addReplySds(c, - sdscatprintf(sdsempty(), - "-MISCONF Errors writing to the AOF file: %s\r\n", - strerror(g_pserver->aof_last_write_errno))); - return C_OK; - } + locker.arm(c); + int deny_write_type = writeCommandsDeniedByDiskError(); + if (deny_write_type != DISK_ERROR_TYPE_NONE && + listLength(g_pserver->masters) == 0 && + (c->cmd->flags & CMD_WRITE || + c->cmd->proc == pingCommand)) + { + flagTransaction(c); + if (deny_write_type == DISK_ERROR_TYPE_RDB) + addReply(c, shared.bgsaveerr); + else + addReplySds(c, + sdscatprintf(sdsempty(), + "-MISCONF Errors writing to the AOF file: %s\r\n", + strerror(g_pserver->aof_last_write_errno))); + return C_OK; + } - /* Don't accept write commands if there are not enough good slaves and - * user configured the min-slaves-to-write option. */ - if (listLength(g_pserver->masters) == 0 && - g_pserver->repl_min_slaves_to_write && - g_pserver->repl_min_slaves_max_lag && - c->cmd->flags & CMD_WRITE && - g_pserver->repl_good_slaves_count < g_pserver->repl_min_slaves_to_write) - { - flagTransaction(c); - addReply(c, shared.noreplicaserr); - return C_OK; - } + /* Don't accept write commands if there are not enough good slaves and + * user configured the min-slaves-to-write option. */ + if (listLength(g_pserver->masters) == 0 && + g_pserver->repl_min_slaves_to_write && + g_pserver->repl_min_slaves_max_lag && + c->cmd->flags & CMD_WRITE && + g_pserver->repl_good_slaves_count < g_pserver->repl_min_slaves_to_write) + { + flagTransaction(c); + addReply(c, shared.noreplicaserr); + return C_OK; + } - /* Don't accept write commands if this is a read only replica. But - * accept write commands if this is our master. */ - if (listLength(g_pserver->masters) && g_pserver->repl_slave_ro && - !(c->flags & CLIENT_MASTER) && - c->cmd->flags & CMD_WRITE) - { - addReply(c, shared.roslaveerr); - return C_OK; + /* Don't accept write commands if this is a read only replica. But + * accept write commands if this is our master. */ + if (listLength(g_pserver->masters) && g_pserver->repl_slave_ro && + !(c->flags & CLIENT_MASTER) && + c->cmd->flags & CMD_WRITE) + { + addReply(c, shared.roslaveerr); + return C_OK; + } } /* Only allow a subset of commands in the context of Pub/Sub if the @@ -3667,16 +3669,20 @@ int processCommand(client *c, int callFlags) { return C_OK; } - /* Only allow commands with flag "t", such as INFO, SLAVEOF and so on, - * when replica-serve-stale-data is no and we are a replica with a broken - * link with master. */ - if (FBrokenLinkToMaster() && - g_pserver->repl_serve_stale_data == 0 && - !(c->cmd->flags & CMD_STALE)) + if (listLength(g_pserver->masters)) { - flagTransaction(c); - addReply(c, shared.masterdownerr); - return C_OK; + locker.arm(c); + /* Only allow commands with flag "t", such as INFO, SLAVEOF and so on, + * when replica-serve-stale-data is no and we are a replica with a broken + * link with master. */ + if (FBrokenLinkToMaster() && + g_pserver->repl_serve_stale_data == 0 && + !(c->cmd->flags & CMD_STALE)) + { + flagTransaction(c); + addReply(c, shared.masterdownerr); + return C_OK; + } } /* Loading DB? Return an error if the command has not the @@ -3711,6 +3717,7 @@ int processCommand(client *c, int callFlags) { queueMultiCommand(c); addReply(c,shared.queued); } else { + locker.arm(c); call(c,callFlags); c->woff = g_pserver->master_repl_offset; if (listLength(g_pserver->ready_keys))