diff --git a/deps/Makefile b/deps/Makefile index d02882b1b..190a648a8 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -103,7 +103,7 @@ jemalloc: .make-prerequisites rocksdb: .make-prerequisites @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) ifeq ($(uname_M),x86_64) - cd rocksdb && PORTABLE=1 USE_SSE=1 FORCE_SSE42=1 $(MAKE) static_lib + cd rocksdb && CFLAGS=-Wno-error PORTABLE=1 USE_SSE=1 FORCE_SSE42=1 $(MAKE) static_lib else cd rocksdb && PORTABLE=1 $(MAKE) static_lib endif diff --git a/src/Makefile b/src/Makefile index f48e93821..d432c6fc5 100644 --- a/src/Makefile +++ b/src/Makefile @@ -90,7 +90,6 @@ endif ifeq ($(COMPILER_NAME),clang) CXXFLAGS+= -stdlib=libc++ - LDFLAGS+= -latomic endif # To get ARM stack traces if KeyDB crashes we need a special C flag. @@ -104,12 +103,6 @@ ifneq (,$(findstring armv,$(uname_M))) endif endif -ifneq (,$(filter aarch64 armv,$(uname_M))) - LICENSE_LIB_DIR=../deps/license/arm64/ -else - LICENSE_LIB_DIR=../deps/license/x64/ -endif - # Backwards compatibility for selecting an allocator ifeq ($(USE_TCMALLOC),yes) MALLOC=tcmalloc @@ -135,11 +128,11 @@ endif # Override default settings if possible -include .make-settings +DEBUG=-g -ggdb FINAL_CFLAGS=$(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(KEYDB_CFLAGS) $(REDIS_CFLAGS) FINAL_CXXFLAGS=$(CXX_STD) $(WARN) $(OPT) $(DEBUG) $(CXXFLAGS) $(KEYDB_CFLAGS) $(REDIS_CFLAGS) FINAL_LDFLAGS=$(LDFLAGS) $(KEYDB_LDFLAGS) $(DEBUG) -FINAL_LIBS+=-lm -lz -latomic -L$(LICENSE_LIB_DIR) -lkey -lcrypto -lbz2 -lzstd -llz4 -lsnappy -DEBUG=-g -ggdb +FINAL_LIBS+=-lm -lz -lcrypto -lbz2 -lzstd -llz4 -lsnappy ifneq ($(uname_S),Darwin) FINAL_LIBS+=-latomic @@ -249,14 +242,15 @@ endif ifdef OPENSSL_PREFIX OPENSSL_CFLAGS=-I$(OPENSSL_PREFIX)/include + OPENSSL_CXXFLAGS=-I$(OPENSSL_PREFIX)/include OPENSSL_LDFLAGS=-L$(OPENSSL_PREFIX)/lib # Also export OPENSSL_PREFIX so it ends up in deps sub-Makefiles export OPENSSL_PREFIX endif # Include paths to dependencies -FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram -I../deps/license/ -FINAL_CXXFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram -I../deps/rocksdb/include/ -I../deps/license -I../deps/concurrentqueue +FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram +FINAL_CXXFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram -I../deps/rocksdb/include/ -I../deps/concurrentqueue # Determine systemd support and/or build preference (defaulting to auto-detection) BUILD_WITH_SYSTEMD=no diff --git a/src/config.cpp b/src/config.cpp index 870d2b6c3..5d416fcde 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -35,9 +35,9 @@ #include #include +#ifdef __linux__ #include -#include "keycheck.h" - +#endif const char *KEYDB_SET_VERSION = KEYDB_REAL_VERSION; @@ -371,6 +371,7 @@ bool initializeStorageProvider(const char **err) // We need to set max memory to a sane default so keys are actually evicted properly if (g_pserver->maxmemory == 0 && g_pserver->maxmemory_policy == MAXMEMORY_NO_EVICTION) { +#ifdef __linux__ struct sysinfo sys; if (sysinfo(&sys) == 0) { @@ -378,6 +379,9 @@ bool initializeStorageProvider(const char **err) g_pserver->maxmemory = static_cast(sys.totalram / 2.2); g_pserver->maxmemory_policy = MAXMEMORY_ALLKEYS_LRU; } +#else + serverLog(LL_WARNING, "Unable to dynamically set maxmemory, please set maxmemory and maxmemory-policy if you are using a storage provier"); +#endif } else if (g_pserver->maxmemory_policy == MAXMEMORY_NO_EVICTION) { @@ -753,15 +757,6 @@ void loadServerConfigFromString(char *config) { g_sdsProvider = sdsdup(argv[1]); if (argc > 2) g_sdsArgs = sdsdup(argv[2]); - } else if (!strcasecmp(argv[0],"enable-enterprise") && (argc == 1 || argc == 2)) { - if (argc == 2) - { - if (!FValidKey(argv[1], strlen(argv[1]))) { - err = "Invalid license key"; - goto loaderr; - } - cserver.license_key = zstrdup(argv[1]); - } } else { err = "Bad directive or wrong number of arguments"; goto loaderr; } @@ -1904,7 +1899,6 @@ int rewriteConfig(char *path, int force_all) { rewriteConfigClientoutputbufferlimitOption(state); rewriteConfigYesNoOption(state,"active-replica",g_pserver->fActiveReplica,CONFIG_DEFAULT_ACTIVE_REPLICA); rewriteConfigStringOption(state, "version-override",KEYDB_SET_VERSION,KEYDB_REAL_VERSION); - rewriteConfigStringOption(state, "enable-enterprise", cserver.license_key, CONFIG_DEFAULT_LICENSE_KEY); rewriteConfigOOMScoreAdjValuesOption(state); /* Rewrite Sentinel config if in Sentinel mode. */ diff --git a/src/dict.cpp b/src/dict.cpp index d94f17701..34efe1e82 100644 --- a/src/dict.cpp +++ b/src/dict.cpp @@ -575,7 +575,7 @@ int dictRehashMilliseconds(dict *d, int ms) { * dictionary so that the hash table automatically migrates from H1 to H2 * while it is actively used. */ static void _dictRehashStep(dict *d) { - uint16_t pauserehash; + int16_t pauserehash; __atomic_load(&d->pauserehash, &pauserehash, __ATOMIC_RELAXED); if (pauserehash == 0) dictRehash(d,1); } @@ -1519,7 +1519,7 @@ void dictGetStats(char *buf, size_t bufsize, dict *d) { void dictForceRehash(dict *d) { - uint16_t pauserehash; + int16_t pauserehash; __atomic_load(&d->pauserehash, &pauserehash, __ATOMIC_RELAXED); while (pauserehash == 0 && dictIsRehashing(d)) _dictRehashStep(d); } diff --git a/src/module.cpp b/src/module.cpp index 906493f86..6f4315c32 100644 --- a/src/module.cpp +++ b/src/module.cpp @@ -4289,10 +4289,9 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch } { - aeAcquireLock(); + AeLocker locker; locker.arm(nullptr); std::unique_lock ul(c->lock); call(c,call_flags); - aeReleaseLock(); } g_pserver->replication_allowed = prev_replication_allowed; diff --git a/src/networking.cpp b/src/networking.cpp index 98b6aa735..2b8ac08c9 100644 --- a/src/networking.cpp +++ b/src/networking.cpp @@ -2766,9 +2766,9 @@ void readQueryFromClient(connection *conn) { } else { // If we're single threaded its actually better to just process the command here while the query is hot in the cache // multithreaded lock contention dominates and batching is better - aeAcquireLock(); + AeLocker locker; + locker.arm(c); runAndPropogateToReplicas(processInputBuffer, c, true /*fParse*/, CMD_CALL_FULL); - aeReleaseLock(); } } diff --git a/src/rdb.cpp b/src/rdb.cpp index 31fe905a6..4d6cde7d3 100644 --- a/src/rdb.cpp +++ b/src/rdb.cpp @@ -1552,6 +1552,7 @@ struct rdbSaveThreadArgs void *rdbSaveThread(void *vargs) { + serverAssert(!g_pserver->rdbThreadVars.fDone); rdbSaveThreadArgs *args = reinterpret_cast(vargs); serverAssert(serverTL == nullptr); redisServerThreadVars vars; @@ -1577,6 +1578,7 @@ void *rdbSaveThread(void *vargs) "RDB",cbDiff/(1024*1024)); } + g_pserver->rdbThreadVars.fDone = true; return (retval == C_OK) ? (void*)0 : (void*)1; } @@ -2945,6 +2947,7 @@ public: serverTL = &vars; aeSetThreadOwnsLockOverride(true); +#ifdef __linux__ // We will inheret the server thread's affinity mask, clear it as we want to run on a different core. cpu_set_t *cpuset = CPU_ALLOC(std::thread::hardware_concurrency()); if (cpuset != nullptr) { @@ -2956,6 +2959,7 @@ public: pthread_setaffinity_np(pthread_self(), size, cpuset); CPU_FREE(cpuset); } +#endif for (;;) { if (queue.queueJobs.size_approx() == 0) { @@ -3624,6 +3628,7 @@ struct rdbSaveSocketThreadArgs }; void *rdbSaveToSlavesSocketsThread(void *vargs) { + serverAssert(!g_pserver->rdbThreadVars.fDone); /* Child */ serverAssert(serverTL == nullptr); rdbSaveSocketThreadArgs *args = (rdbSaveSocketThreadArgs*)vargs; @@ -3664,6 +3669,7 @@ void *rdbSaveToSlavesSocketsThread(void *vargs) close(args->safe_to_exit_pipe); zfree(args); + g_pserver->rdbThreadVars.fDone = true; return (retval == C_OK) ? (void*)0 : (void*)1; } diff --git a/src/redis-benchmark.cpp b/src/redis-benchmark.cpp index 48465df9c..397b557c6 100644 --- a/src/redis-benchmark.cpp +++ b/src/redis-benchmark.cpp @@ -42,6 +42,7 @@ #include #include #include +#include extern "C" { #include /* Use hiredis' sds compat header that maps sds calls to their hi_ variants */ #include /* Use hiredis sds. */ diff --git a/src/replication.cpp b/src/replication.cpp index 1b095f031..3a506a62a 100644 --- a/src/replication.cpp +++ b/src/replication.cpp @@ -190,7 +190,11 @@ int bg_unlink(const char *filename) { bool createDiskBacklog() { // Lets create some disk backed pages and add them here std::string path = "./repl-backlog-temp" + std::to_string(gettid()); +#ifdef __APPLE__ + int fd = open(path.c_str(), O_CREAT | O_RDWR, S_IRUSR | S_IWUSR); +#else int fd = open(path.c_str(), O_CREAT | O_RDWR | O_LARGEFILE, S_IRUSR | S_IWUSR); +#endif if (fd < 0) { return false; } @@ -1559,14 +1563,7 @@ LError: void processReplconfLicense(client *c, robj *arg) { - if (cserver.license_key != nullptr) - { - if (strcmp(cserver.license_key, szFromObj(arg)) == 0) { - addReplyError(c, "Each replica must have a unique license key"); - c->flags |= CLIENT_CLOSE_AFTER_REPLY; - return; - } - } + // Only for back-compat addReply(c, shared.ok); } @@ -3632,41 +3629,6 @@ retry_connect: } sdsfree(err); err = NULL; - mi->repl_state = REPL_STATE_SEND_KEY; - // fallthrough - } - - /* Send LICENSE Key */ - if (mi->repl_state == REPL_STATE_SEND_KEY) - { - if (cserver.license_key == nullptr) - { - mi->repl_state = REPL_STATE_SEND_PSYNC; - } - else - { - err = sendCommand(conn,"REPLCONF","license",cserver.license_key,NULL); - if (err) goto write_error; - mi->repl_state = REPL_STATE_KEY_ACK; - return; - } - } - - /* LICENSE Key Ack */ - if (mi->repl_state == REPL_STATE_KEY_ACK) - { - err = receiveSynchronousResponse(mi, conn); - if (err[0] == '-') { - if (err[1] == 'E' && err[2] == 'R' && err[3] == 'R') { - // Replicating with non-enterprise - serverLog(LL_WARNING, "Replicating with non-enterprise server."); - } else { - serverLog(LL_WARNING, "Recieved error from client: %s", err); - sdsfree(err); - goto error; - } - } - sdsfree(err); mi->repl_state = REPL_STATE_SEND_PSYNC; // fallthrough } @@ -5598,7 +5560,7 @@ void flushReplBacklogToClients() if (g_pserver->repl_batch_offStart != g_pserver->master_repl_offset) { bool fAsyncWrite = false; - long long min_offset = LONG_LONG_MAX; + long long min_offset = LLONG_MAX; // Ensure no overflow serverAssert(g_pserver->repl_batch_offStart < g_pserver->master_repl_offset); if (g_pserver->master_repl_offset - g_pserver->repl_batch_offStart > g_pserver->repl_backlog_size) { @@ -5657,7 +5619,7 @@ LDone: // This may be called multiple times per "frame" so update with our progress flushing to clients g_pserver->repl_batch_idxStart = g_pserver->repl_backlog_idx; g_pserver->repl_batch_offStart = g_pserver->master_repl_offset; - g_pserver->repl_lowest_off.store(min_offset == LONG_LONG_MAX ? -1 : min_offset, std::memory_order_seq_cst); + g_pserver->repl_lowest_off.store(min_offset == LLONG_MAX ? -1 : min_offset, std::memory_order_seq_cst); } } diff --git a/src/server.cpp b/src/server.cpp index 119f91bb4..977191e7c 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -65,7 +65,6 @@ #include #include #include "aelocker.h" -#include "keycheck.h" #include "motd.h" #include "t_nhash.h" #include "readwritelock.h" @@ -2200,8 +2199,8 @@ void checkChildrenDone(void) { if (g_pserver->FRdbSaveInProgress() && !cserver.fForkBgSave) { void *rval = nullptr; - int err; - if ((err = pthread_tryjoin_np(g_pserver->rdbThreadVars.rdb_child_thread, &rval))) + int err = EAGAIN; + if (!g_pserver->rdbThreadVars.fDone || (err = pthread_join(g_pserver->rdbThreadVars.rdb_child_thread, &rval))) { if (err != EBUSY && err != EAGAIN) serverLog(LL_WARNING, "Error joining the background RDB save thread: %s\n", strerror(errno)); @@ -2211,6 +2210,7 @@ void checkChildrenDone(void) { int exitcode = (int)reinterpret_cast(rval); backgroundSaveDoneHandler(exitcode,g_pserver->rdbThreadVars.fRdbThreadCancel); g_pserver->rdbThreadVars.fRdbThreadCancel = false; + g_pserver->rdbThreadVars.fDone = false; if (exitcode == 0) receiveChildInfo(); } } @@ -7513,13 +7513,6 @@ int main(int argc, char **argv) { serverLog(LL_WARNING, "Configuration loaded"); } -#ifndef NO_LICENSE_CHECK - if (!g_pserver->sentinel_mode && (cserver.license_key == nullptr || !FValidKey(cserver.license_key, strlen(cserver.license_key)))){ - serverLog(LL_WARNING, "Error: %s license key provided, exiting immediately.", cserver.license_key == nullptr ? "No" : "Invalid"); - exit(1); - } -#endif - validateConfiguration(); const char *err; diff --git a/src/server.h b/src/server.h index 4a0b9f55c..743011f0d 100644 --- a/src/server.h +++ b/src/server.h @@ -373,8 +373,6 @@ inline bool operator!=(const void *p, const robj_sharedptr &rhs) #define CONFIG_DEFAULT_ACTIVE_REPLICA 0 #define CONFIG_DEFAULT_ENABLE_MULTIMASTER 0 -#define CONFIG_DEFAULT_LICENSE_KEY "" - #define ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP 64 /* Loopkups per loop. */ #define ACTIVE_EXPIRE_CYCLE_SUBKEY_LOOKUPS_PER_LOOP 16384 /* Subkey loopkups per loop. */ #define ACTIVE_EXPIRE_CYCLE_FAST_DURATION 1000 /* Microseconds */ @@ -586,8 +584,6 @@ typedef enum { REPL_STATE_RECEIVE_IP_REPLY, /* Wait for REPLCONF reply */ REPL_STATE_RECEIVE_CAPA_REPLY, /* Wait for REPLCONF reply */ REPL_STATE_RECEIVE_UUID, /* they should ack with their UUID */ - REPL_STATE_SEND_KEY, - REPL_STATE_KEY_ACK, REPL_STATE_SEND_PSYNC, /* Send PSYNC */ REPL_STATE_RECEIVE_PSYNC_REPLY, /* Wait for PSYNC reply */ /* --- End of handshake states --- */ @@ -2179,7 +2175,6 @@ struct redisServerConst { int enable_motd; /* Flag to retrieve the Message of today using CURL request*/ - sds license_key = nullptr; int delete_on_evict = false; // Only valid when a storage provider is set int thread_min_client_threshold = 50; int multimaster_no_forward; @@ -2371,6 +2366,7 @@ struct redisServer { struct _rdbThreadVars { std::atomic fRdbThreadCancel {false}; + std::atomic fDone {false}; int tmpfileNum = 0; pthread_t rdb_child_thread; int fRdbThreadActive = false;