From cf03a62ba78f1939b16a81d53c0de6d1aef73294 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 2 Apr 2018 18:36:17 +0300 Subject: [PATCH 001/101] Add redis-cli support for diskless replication (CAPA EOF) when setting repl-diskless-sync yes, and sending SYNC. redis-cli needs to be able to understand the EOF marker protocol in order to be able to skip or download the rdb file --- src/redis-cli.c | 112 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 102 insertions(+), 10 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index d80973e75..6fffa9024 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1793,9 +1793,31 @@ static void latencyDistMode(void) { * Slave mode *--------------------------------------------------------------------------- */ +#define RDB_EOF_MARK_SIZE 40 + +void sendReplconf(const char* arg1, const char* arg2) { + printf("sending REPLCONF %s %s\n", arg1, arg2); + redisReply *reply = redisCommand(context, "REPLCONF %s %s", arg1, arg2); + + /* Handle any error conditions */ + if(reply == NULL) { + fprintf(stderr, "\nI/O error\n"); + exit(1); + } else if(reply->type == REDIS_REPLY_ERROR) { + fprintf(stderr, "REPLCONF %s error: %s\n", arg1, reply->str); + /* non fatal, old versions may not support it */ + } + freeReplyObject(reply); +} + +void sendCapa() { + sendReplconf("capa", "eof"); +} + /* Sends SYNC and reads the number of bytes in the payload. Used both by - * slaveMode() and getRDB(). */ -unsigned long long sendSync(int fd) { + * slaveMode() and getRDB(). + * returns 0 in case an EOF marker is used. */ +unsigned long long sendSync(int fd, char *out_eof) { /* To start we need to send the SYNC command and return the payload. * The hiredis client lib does not understand this part of the protocol * and we don't want to mess with its buffers, so everything is performed @@ -1825,17 +1847,33 @@ unsigned long long sendSync(int fd) { printf("SYNC with master failed: %s\n", buf); exit(1); } + if (strncmp(buf+1,"EOF:",4) == 0 && strlen(buf+5) >= RDB_EOF_MARK_SIZE) { + memcpy(out_eof, buf+5, RDB_EOF_MARK_SIZE); + return 0; + } return strtoull(buf+1,NULL,10); } static void slaveMode(void) { int fd = context->fd; - unsigned long long payload = sendSync(fd); + static char eofmark[RDB_EOF_MARK_SIZE]; + static char lastbytes[RDB_EOF_MARK_SIZE]; + static int usemark = 0; + unsigned long long payload = sendSync(fd, eofmark); char buf[1024]; int original_output = config.output; - fprintf(stderr,"SYNC with master, discarding %llu " - "bytes of bulk transfer...\n", payload); + if (payload == 0) { + payload = ULLONG_MAX; + memset(lastbytes,0,RDB_EOF_MARK_SIZE); + usemark = 1; + fprintf(stderr,"SYNC with master, discarding " + "bytes of bulk transfer until EOF marker...\n"); + } else { + fprintf(stderr,"SYNC with master, discarding %llu " + "bytes of bulk transfer...\n", payload); + } + /* Discard the payload. */ while(payload) { @@ -1847,8 +1885,29 @@ static void slaveMode(void) { exit(1); } payload -= nread; + + if (usemark) { + /* Update the last bytes array, and check if it matches our delimiter.*/ + if (nread >= RDB_EOF_MARK_SIZE) { + memcpy(lastbytes,buf+nread-RDB_EOF_MARK_SIZE,RDB_EOF_MARK_SIZE); + } else { + int rem = RDB_EOF_MARK_SIZE-nread; + memmove(lastbytes,lastbytes+nread,rem); + memcpy(lastbytes+rem,buf,nread); + } + if (memcmp(lastbytes,eofmark,RDB_EOF_MARK_SIZE) == 0) + break; + } } - fprintf(stderr,"SYNC done. Logging commands from master.\n"); + + if (usemark) { + unsigned long long offset = ULLONG_MAX - payload; + fprintf(stderr,"SYNC done after %llu bytes. Logging commands from master.\n", offset); + /* put the slave online */ + sleep(1); + sendReplconf("ACK", "0"); + } else + fprintf(stderr,"SYNC done. Logging commands from master.\n"); /* Now we can use hiredis to read the incoming protocol. */ config.output = OUTPUT_CSV; @@ -1865,11 +1924,22 @@ static void slaveMode(void) { static void getRDB(void) { int s = context->fd; int fd; - unsigned long long payload = sendSync(s); + static char eofmark[RDB_EOF_MARK_SIZE]; + static char lastbytes[RDB_EOF_MARK_SIZE]; + static int usemark = 0; + unsigned long long payload = sendSync(s, eofmark); char buf[4096]; - fprintf(stderr,"SYNC sent to master, writing %llu bytes to '%s'\n", - payload, config.rdb_filename); + if (payload == 0) { + payload = ULLONG_MAX; + memset(lastbytes,0,RDB_EOF_MARK_SIZE); + usemark = 1; + fprintf(stderr,"SYNC sent to master, writing bytes of bulk transfer until EOF marker to '%s'\n", + config.rdb_filename); + } else { + fprintf(stderr,"SYNC sent to master, writing %llu bytes to '%s'\n", + payload, config.rdb_filename); + } /* Write to file. */ if (!strcmp(config.rdb_filename,"-")) { @@ -1898,11 +1968,31 @@ static void getRDB(void) { exit(1); } payload -= nread; + + if (usemark) { + /* Update the last bytes array, and check if it matches our delimiter.*/ + if (nread >= RDB_EOF_MARK_SIZE) { + memcpy(lastbytes,buf+nread-RDB_EOF_MARK_SIZE,RDB_EOF_MARK_SIZE); + } else { + int rem = RDB_EOF_MARK_SIZE-nread; + memmove(lastbytes,lastbytes+nread,rem); + memcpy(lastbytes+rem,buf,nread); + } + if (memcmp(lastbytes,eofmark,RDB_EOF_MARK_SIZE) == 0) + break; + } + } + if (usemark) { + payload = ULLONG_MAX - payload - RDB_EOF_MARK_SIZE; + if (ftruncate(fd, payload) == -1) + fprintf(stderr,"ftruncate failed: %s.\n", strerror(errno)); + fprintf(stderr,"Transfer finished with success after %llu bytes\n", payload); + } else { + fprintf(stderr,"Transfer finished with success.\n"); } close(s); /* Close the file descriptor ASAP as fsync() may take time. */ fsync(fd); close(fd); - fprintf(stderr,"Transfer finished with success.\n"); exit(0); } @@ -2893,12 +2983,14 @@ int main(int argc, char **argv) { /* Slave mode */ if (config.slave_mode) { if (cliConnect(0) == REDIS_ERR) exit(1); + sendCapa(); slaveMode(); } /* Get RDB mode. */ if (config.getrdb_mode) { if (cliConnect(0) == REDIS_ERR) exit(1); + sendCapa(); getRDB(); } From bac51d5ac4e1663d9de980baabad316a4ee40322 Mon Sep 17 00:00:00 2001 From: John Sully Date: Tue, 29 Jan 2019 18:10:46 -0500 Subject: [PATCH 002/101] Custom flash heap --- .vscode/settings.json | 5 ++ src/Makefile | 8 +- src/acl.c | 4 +- src/adlist.c | 10 +-- src/ae.c | 8 +- src/ae_epoll.c | 4 +- src/ae_evport.c | 2 +- src/ae_kqueue.c | 2 +- src/ae_select.c | 2 +- src/aof.c | 6 +- src/bio.c | 2 +- src/bitops.c | 6 +- src/blocked.c | 4 +- src/cluster.c | 24 +++--- src/config.c | 6 +- src/db.c | 16 ++-- src/debug.c | 2 +- src/defrag.c | 4 +- src/dict.c | 8 +- src/evict.c | 2 +- src/geo.c | 4 +- src/intset.c | 2 +- src/latency.c | 2 +- src/listpack_malloc.h | 2 +- src/lolwut5.c | 4 +- src/module.c | 40 +++++----- src/multi.c | 4 +- src/networking.c | 16 ++-- src/object.c | 15 ++-- src/pubsub.c | 2 +- src/quicklist.c | 18 ++--- src/rax_malloc.h | 2 +- src/rdb.c | 22 ++--- src/redis-benchmark.c | 13 +-- src/redis-check-aof.c | 2 +- src/redis-cli.c | 70 ++++++++-------- src/replication.c | 4 +- src/rio.c | 4 +- src/scripting.c | 2 +- src/sds.c | 16 ++-- src/sdsalloc.h | 10 ++- src/sentinel.c | 16 ++-- src/server.c | 11 ++- src/slowlog.c | 4 +- src/sort.c | 4 +- src/sparkline.c | 4 +- src/storage.c | 155 ++++++++++++++++++++++++++++++++++++ src/storage.h | 23 ++++++ src/t_hash.c | 2 +- src/t_list.c | 2 +- src/t_set.c | 6 +- src/t_stream.c | 12 +-- src/t_zset.c | 10 +-- src/ziplist.c | 2 +- src/zipmap.c | 2 +- src/zmalloc.c | 22 +++-- src/zmalloc.h | 11 ++- utils/hashtable/rehashing.c | 2 +- 58 files changed, 442 insertions(+), 225 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 src/storage.c create mode 100644 src/storage.h diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..3813ff8ac --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "files.associations": { + "zmalloc.h": "c" + } +} \ No newline at end of file diff --git a/src/Makefile b/src/Makefile index adf32d557..680e3866b 100644 --- a/src/Makefile +++ b/src/Makefile @@ -119,7 +119,7 @@ ifeq ($(uname_S),DragonFly) else # All the other OSes (notably Linux) FINAL_LDFLAGS+= -rdynamic - FINAL_LIBS+=-ldl -pthread -lrt + FINAL_LIBS+=-ldl -pthread -lrt -lmemkind endif endif endif @@ -164,11 +164,11 @@ endif REDIS_SERVER_NAME=redis-server REDIS_SENTINEL_NAME=redis-sentinel -REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o acl.o +REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o acl.o storage.o REDIS_CLI_NAME=redis-cli -REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o siphash.o crc16.o +REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o siphash.o crc16.o storage.o REDIS_BENCHMARK_NAME=redis-benchmark -REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o zmalloc.o redis-benchmark.o +REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o zmalloc.o redis-benchmark.o storage.o REDIS_CHECK_RDB_NAME=redis-check-rdb REDIS_CHECK_AOF_NAME=redis-check-aof diff --git a/src/acl.c b/src/acl.c index ecaae9268..b37b4719a 100644 --- a/src/acl.c +++ b/src/acl.c @@ -146,7 +146,7 @@ void ACLListFreeSds(void *item) { * If the user with such name already exists NULL is returned. */ user *ACLCreateUser(const char *name, size_t namelen) { if (raxFind(Users,(unsigned char*)name,namelen) != raxNotFound) return NULL; - user *u = zmalloc(sizeof(*u)); + user *u = zmalloc(sizeof(*u), MALLOC_LOCAL); u->name = sdsnewlen(name,namelen); u->flags = 0; u->allowed_subcommands = NULL; @@ -267,7 +267,7 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) { * this user, we have to allocate the subcommands array. */ if (u->allowed_subcommands == NULL) { u->allowed_subcommands = zcalloc(USER_COMMAND_BITS_COUNT * - sizeof(sds*)); + sizeof(sds*), MALLOC_LOCAL); } /* We also need to enlarge the allocation pointing to the diff --git a/src/adlist.c b/src/adlist.c index ec5f8bbf4..359262521 100644 --- a/src/adlist.c +++ b/src/adlist.c @@ -42,7 +42,7 @@ list *listCreate(void) { struct list *list; - if ((list = zmalloc(sizeof(*list))) == NULL) + if ((list = zmalloc(sizeof(*list), MALLOC_SHARED)) == NULL) return NULL; list->head = list->tail = NULL; list->len = 0; @@ -89,7 +89,7 @@ list *listAddNodeHead(list *list, void *value) { listNode *node; - if ((node = zmalloc(sizeof(*node))) == NULL) + if ((node = zmalloc(sizeof(*node), MALLOC_SHARED)) == NULL) return NULL; node->value = value; if (list->len == 0) { @@ -115,7 +115,7 @@ list *listAddNodeTail(list *list, void *value) { listNode *node; - if ((node = zmalloc(sizeof(*node))) == NULL) + if ((node = zmalloc(sizeof(*node), MALLOC_SHARED)) == NULL) return NULL; node->value = value; if (list->len == 0) { @@ -134,7 +134,7 @@ list *listAddNodeTail(list *list, void *value) list *listInsertNode(list *list, listNode *old_node, void *value, int after) { listNode *node; - if ((node = zmalloc(sizeof(*node))) == NULL) + if ((node = zmalloc(sizeof(*node), MALLOC_SHARED)) == NULL) return NULL; node->value = value; if (after) { @@ -187,7 +187,7 @@ listIter *listGetIterator(list *list, int direction) { listIter *iter; - if ((iter = zmalloc(sizeof(*iter))) == NULL) return NULL; + if ((iter = zmalloc(sizeof(*iter), MALLOC_SHARED)) == NULL) return NULL; if (direction == AL_START_HEAD) iter->next = list->head; else diff --git a/src/ae.c b/src/ae.c index 53629ef77..7f59aecf1 100644 --- a/src/ae.c +++ b/src/ae.c @@ -64,9 +64,9 @@ aeEventLoop *aeCreateEventLoop(int setsize) { aeEventLoop *eventLoop; int i; - if ((eventLoop = zmalloc(sizeof(*eventLoop))) == NULL) goto err; - eventLoop->events = zmalloc(sizeof(aeFileEvent)*setsize); - eventLoop->fired = zmalloc(sizeof(aeFiredEvent)*setsize); + if ((eventLoop = zmalloc(sizeof(*eventLoop), MALLOC_LOCAL)) == NULL) goto err; + eventLoop->events = zmalloc(sizeof(aeFileEvent)*setsize, MALLOC_LOCAL); + eventLoop->fired = zmalloc(sizeof(aeFiredEvent)*setsize, MALLOC_LOCAL); if (eventLoop->events == NULL || eventLoop->fired == NULL) goto err; eventLoop->setsize = setsize; eventLoop->lastTime = time(NULL); @@ -212,7 +212,7 @@ long long aeCreateTimeEvent(aeEventLoop *eventLoop, long long milliseconds, long long id = eventLoop->timeEventNextId++; aeTimeEvent *te; - te = zmalloc(sizeof(*te)); + te = zmalloc(sizeof(*te), MALLOC_LOCAL); if (te == NULL) return AE_ERR; te->id = id; aeAddMillisecondsToNow(milliseconds,&te->when_sec,&te->when_ms); diff --git a/src/ae_epoll.c b/src/ae_epoll.c index 410aac70d..298bcf6ce 100644 --- a/src/ae_epoll.c +++ b/src/ae_epoll.c @@ -37,10 +37,10 @@ typedef struct aeApiState { } aeApiState; static int aeApiCreate(aeEventLoop *eventLoop) { - aeApiState *state = zmalloc(sizeof(aeApiState)); + aeApiState *state = zmalloc(sizeof(aeApiState), MALLOC_LOCAL); if (!state) return -1; - state->events = zmalloc(sizeof(struct epoll_event)*eventLoop->setsize); + state->events = zmalloc(sizeof(struct epoll_event)*eventLoop->setsize, MALLOC_LOCAL); if (!state->events) { zfree(state); return -1; diff --git a/src/ae_evport.c b/src/ae_evport.c index 5c317becb..7b7fbe28e 100644 --- a/src/ae_evport.c +++ b/src/ae_evport.c @@ -74,7 +74,7 @@ typedef struct aeApiState { static int aeApiCreate(aeEventLoop *eventLoop) { int i; - aeApiState *state = zmalloc(sizeof(aeApiState)); + aeApiState *state = zmalloc(sizeof(aeApiState), MALLOC_LOCAL); if (!state) return -1; state->portfd = port_create(); diff --git a/src/ae_kqueue.c b/src/ae_kqueue.c index 6796f4ceb..36148bf58 100644 --- a/src/ae_kqueue.c +++ b/src/ae_kqueue.c @@ -42,7 +42,7 @@ static int aeApiCreate(aeEventLoop *eventLoop) { aeApiState *state = zmalloc(sizeof(aeApiState)); if (!state) return -1; - state->events = zmalloc(sizeof(struct kevent)*eventLoop->setsize); + state->events = zmalloc(sizeof(struct kevent)*eventLoop->setsize, MALLOC_LOCAL); if (!state->events) { zfree(state); return -1; diff --git a/src/ae_select.c b/src/ae_select.c index c039a8ea3..7e6746aae 100644 --- a/src/ae_select.c +++ b/src/ae_select.c @@ -40,7 +40,7 @@ typedef struct aeApiState { } aeApiState; static int aeApiCreate(aeEventLoop *eventLoop) { - aeApiState *state = zmalloc(sizeof(aeApiState)); + aeApiState *state = zmalloc(sizeof(aeApiState), MALLOC_LOCAL); if (!state) return -1; FD_ZERO(&state->rfds); diff --git a/src/aof.c b/src/aof.c index 46ae58324..ed9da54ef 100644 --- a/src/aof.c +++ b/src/aof.c @@ -143,7 +143,7 @@ void aofRewriteBufferAppend(unsigned char *s, unsigned long len) { if (len) { /* First block to allocate, or need another block. */ int numblocks; - block = zmalloc(sizeof(*block)); + block = zmalloc(sizeof(*block), MALLOC_LOCAL); block->free = AOF_RW_BUF_BLOCK_SIZE; block->used = 0; listAddNodeTail(server.aof_rewrite_buf_blocks,block); @@ -627,7 +627,7 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a /* In Redis commands are always executed in the context of a client, so in * order to load the append only file we need to create a fake client. */ struct client *createFakeClient(void) { - struct client *c = zmalloc(sizeof(*c)); + struct client *c = zmalloc(sizeof(*c), MALLOC_LOCAL); selectDb(c,0); c->fd = -1; @@ -752,7 +752,7 @@ int loadAppendOnlyFile(char *filename) { argc = atoi(buf+1); if (argc < 1) goto fmterr; - argv = zmalloc(sizeof(robj*)*argc); + argv = zmalloc(sizeof(robj*)*argc, MALLOC_LOCAL); fakeClient->argc = argc; fakeClient->argv = argv; diff --git a/src/bio.c b/src/bio.c index 2af684570..13662197e 100644 --- a/src/bio.c +++ b/src/bio.c @@ -129,7 +129,7 @@ void bioInit(void) { } void bioCreateBackgroundJob(int type, void *arg1, void *arg2, void *arg3) { - struct bio_job *job = zmalloc(sizeof(*job)); + struct bio_job *job = zmalloc(sizeof(*job), MALLOC_LOCAL); job->time = time(NULL); job->arg1 = arg1; diff --git a/src/bitops.c b/src/bitops.c index 8d03a7699..e2b53169a 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -621,9 +621,9 @@ void bitopCommand(client *c) { /* Lookup keys, and store pointers to the string objects into an array. */ numkeys = c->argc - 3; - src = zmalloc(sizeof(unsigned char*) * numkeys); - len = zmalloc(sizeof(long) * numkeys); - objects = zmalloc(sizeof(robj*) * numkeys); + src = zmalloc(sizeof(unsigned char*) * numkeys, MALLOC_LOCAL); + len = zmalloc(sizeof(long) * numkeys, MALLOC_LOCAL); + objects = zmalloc(sizeof(robj*) * numkeys, MALLOC_LOCAL); for (j = 0; j < numkeys; j++) { o = lookupKeyRead(c->db,c->argv[j+3]); /* Handle non-existing keys as empty strings. */ diff --git a/src/blocked.c b/src/blocked.c index f9e196626..24d674935 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -513,7 +513,7 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo * is NULL for lists and sorted sets, or the stream ID for streams. */ void *key_data = NULL; if (btype == BLOCKED_STREAM) { - key_data = zmalloc(sizeof(streamID)); + key_data = zmalloc(sizeof(streamID), MALLOC_SHARED); memcpy(key_data,ids+j,sizeof(streamID)); } @@ -596,7 +596,7 @@ void signalKeyAsReady(redisDb *db, robj *key) { if (dictFind(db->ready_keys,key) != NULL) return; /* Ok, we need to queue this key into server.ready_keys. */ - rl = zmalloc(sizeof(*rl)); + rl = zmalloc(sizeof(*rl), MALLOC_SHARED); rl->key = key; rl->db = db; incrRefCount(key); diff --git a/src/cluster.c b/src/cluster.c index 1a3a348b5..bd04646d0 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -119,7 +119,7 @@ int clusterLoadConfig(char *filename) { * * To simplify we allocate 1024+CLUSTER_SLOTS*128 bytes per line. */ maxline = 1024+CLUSTER_SLOTS*128; - line = zmalloc(maxline); + line = zmalloc(maxline, MALLOC_LOCAL); while(fgets(line,maxline,fp) != NULL) { int argc; sds *argv; @@ -429,7 +429,7 @@ void clusterUpdateMyselfFlags(void) { void clusterInit(void) { int saveconf = 0; - server.cluster = zmalloc(sizeof(clusterState)); + server.cluster = zmalloc(sizeof(clusterState), MALLOC_LOCAL); server.cluster->myself = NULL; server.cluster->currentEpoch = 0; server.cluster->state = CLUSTER_FAIL; @@ -587,7 +587,7 @@ void clusterReset(int hard) { * -------------------------------------------------------------------------- */ clusterLink *createClusterLink(clusterNode *node) { - clusterLink *link = zmalloc(sizeof(*link)); + clusterLink *link = zmalloc(sizeof(*link), MALLOC_LOCAL); link->ctime = mstime(); link->sndbuf = sdsempty(); link->rcvbuf = sdsempty(); @@ -692,7 +692,7 @@ unsigned int keyHashSlot(char *key, int keylen) { * The node is created and returned to the user, but it is not automatically * added to the nodes hash table. */ clusterNode *createClusterNode(char *nodename, int flags) { - clusterNode *node = zmalloc(sizeof(*node)); + clusterNode *node = zmalloc(sizeof(*node), MALLOC_LOCAL); if (nodename) memcpy(node->name, nodename, CLUSTER_NAMELEN); @@ -749,7 +749,7 @@ int clusterNodeAddFailureReport(clusterNode *failing, clusterNode *sender) { } /* Otherwise create a new report. */ - fr = zmalloc(sizeof(*fr)); + fr = zmalloc(sizeof(*fr), MALLOC_LOCAL); fr->node = sender; fr->time = mstime(); listAddNodeTail(l,fr); @@ -2401,7 +2401,7 @@ void clusterSendPing(clusterLink *link, int type) { /* Note: clusterBuildMessageHdr() expects the buffer to be always at least * sizeof(clusterMsg) or more. */ if (totlen < (int)sizeof(clusterMsg)) totlen = sizeof(clusterMsg); - buf = zcalloc(totlen); + buf = zcalloc(totlen, MALLOC_LOCAL); hdr = (clusterMsg*) buf; /* Populate the header. */ @@ -2538,7 +2538,7 @@ void clusterSendPublish(clusterLink *link, robj *channel, robj *message) { if (totlen < sizeof(buf)) { payload = buf; } else { - payload = zmalloc(totlen); + payload = zmalloc(totlen, MALLOC_LOCAL); memcpy(payload,hdr,sizeof(*hdr)); hdr = (clusterMsg*) payload; } @@ -2607,7 +2607,7 @@ void clusterSendModule(clusterLink *link, uint64_t module_id, uint8_t type, if (totlen < sizeof(buf)) { heapbuf = buf; } else { - heapbuf = zmalloc(totlen); + heapbuf = zmalloc(totlen, MALLOC_LOCAL); memcpy(heapbuf,hdr,sizeof(*hdr)); hdr = (clusterMsg*) heapbuf; } @@ -4277,7 +4277,7 @@ NULL /* CLUSTER ADDSLOTS [slot] ... */ /* CLUSTER DELSLOTS [slot] ... */ int j, slot; - unsigned char *slots = zmalloc(CLUSTER_SLOTS); + unsigned char *slots = zmalloc(CLUSTER_SLOTS, MALLOC_LOCAL); int del = !strcasecmp(c->argv[1]->ptr,"delslots"); memset(slots,0,CLUSTER_SLOTS); @@ -4546,7 +4546,7 @@ NULL unsigned int keys_in_slot = countKeysInSlot(slot); if (maxkeys > keys_in_slot) maxkeys = keys_in_slot; - keys = zmalloc(sizeof(robj*)*maxkeys); + keys = zmalloc(sizeof(robj*)*maxkeys, MALLOC_LOCAL); numkeys = getKeysInSlot(slot, keys, maxkeys); addReplyArrayLen(c,numkeys); for (j = 0; j < numkeys; j++) { @@ -5008,7 +5008,7 @@ migrateCachedSocket* migrateGetSocket(client *c, robj *host, robj *port, long ti } /* Add to the cache and return it to the caller. */ - cs = zmalloc(sizeof(*cs)); + cs = zmalloc(sizeof(*cs), MALLOC_LOCAL); cs->fd = fd; cs->last_dbid = -1; cs->last_use_time = server.unixtime; @@ -5256,7 +5256,7 @@ try_again: * to propagate the MIGRATE as a DEL command (if no COPY option was given). * We allocate num_keys+1 because the additional argument is for "DEL" * command name itself. */ - if (!copy) newargv = zmalloc(sizeof(robj*)*(num_keys+1)); + if (!copy) newargv = zmalloc(sizeof(robj*)*(num_keys+1), MALLOC_LOCAL); for (j = 0; j < num_keys; j++) { if (syncReadLine(cs->fd, buf2, sizeof(buf2), timeout) <= 0) { diff --git a/src/config.c b/src/config.c index 91bbdee72..be1c45235 100644 --- a/src/config.c +++ b/src/config.c @@ -159,8 +159,8 @@ void queueLoadModule(sds path, sds *argv, int argc) { int i; struct moduleLoadQueueEntry *loadmod; - loadmod = zmalloc(sizeof(struct moduleLoadQueueEntry)); - loadmod->argv = zmalloc(sizeof(robj*)*argc); + loadmod = zmalloc(sizeof(struct moduleLoadQueueEntry), MALLOC_LOCAL); + loadmod->argv = zmalloc(sizeof(robj*)*argc, MALLOC_LOCAL); loadmod->path = sdsnew(path); loadmod->argc = argc; for (i = 0; i < argc; i++) { @@ -1672,7 +1672,7 @@ void rewriteConfigMarkAsProcessed(struct rewriteConfigState *state, const char * * If the old file does not exist at all, an empty state is returned. */ struct rewriteConfigState *rewriteConfigReadOldFile(char *path) { FILE *fp = fopen(path,"r"); - struct rewriteConfigState *state = zmalloc(sizeof(*state)); + struct rewriteConfigState *state = zmalloc(sizeof(*state), MALLOC_LOCAL); char buf[CONFIG_MAX_LINE+1]; int linenum = -1; diff --git a/src/db.c b/src/db.c index f59ae8406..b44975b40 100644 --- a/src/db.c +++ b/src/db.c @@ -1200,7 +1200,7 @@ int *getKeysUsingCommandTable(struct redisCommand *cmd,robj **argv, int argc, in last = cmd->lastkey; if (last < 0) last = argc+last; - keys = zmalloc(sizeof(int)*((last - cmd->firstkey)+1)); + keys = zmalloc(sizeof(int)*((last - cmd->firstkey)+1), MALLOC_SHARED); for (j = cmd->firstkey; j <= last; j += cmd->keystep) { if (j >= argc) { /* Modules commands, and standard commands with a not fixed number @@ -1267,7 +1267,7 @@ int *zunionInterGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *nu /* Keys in z{union,inter}store come from two places: * argv[1] = storage key, * argv[3...n] = keys to intersect */ - keys = zmalloc(sizeof(int)*(num+1)); + keys = zmalloc(sizeof(int)*(num+1), MALLOC_SHARED); /* Add all key positions for argv[3...n] to keys[] */ for (i = 0; i < num; i++) keys[i] = 3+i; @@ -1293,7 +1293,7 @@ int *evalGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) return NULL; } - keys = zmalloc(sizeof(int)*num); + keys = zmalloc(sizeof(int)*num, MALLOC_SHARED); *numkeys = num; /* Add all key positions for argv[3...n] to keys[] */ @@ -1314,7 +1314,7 @@ int *sortGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) UNUSED(cmd); num = 0; - keys = zmalloc(sizeof(int)*2); /* Alloc 2 places for the worst case. */ + keys = zmalloc(sizeof(int)*2, MALLOC_SHARED); /* Alloc 2 places for the worst case. */ keys[num++] = 1; /* is always present. */ @@ -1372,7 +1372,7 @@ int *migrateGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkey } } - keys = zmalloc(sizeof(int)*num); + keys = zmalloc(sizeof(int)*num, MALLOC_SHARED); for (i = 0; i < num; i++) keys[i] = first+i; *numkeys = num; return keys; @@ -1405,7 +1405,7 @@ int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numk * argv[1] = key, * argv[5...n] = stored key if present */ - keys = zmalloc(sizeof(int) * num); + keys = zmalloc(sizeof(int) * num, MALLOC_SHARED); /* Add all key positions to keys[] */ keys[0] = 1; @@ -1454,7 +1454,7 @@ int *xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) num /= 2; /* We have half the keys as there are arguments because there are also the IDs, one per key. */ - keys = zmalloc(sizeof(int) * num); + keys = zmalloc(sizeof(int) * num, MALLOC_SHARED); for (i = streams_pos+1; i < argc-num; i++) keys[i-streams_pos-1] = i; *numkeys = num; return keys; @@ -1471,7 +1471,7 @@ void slotToKeyUpdateKey(robj *key, int add) { size_t keylen = sdslen(key->ptr); server.cluster->slots_keys_count[hashslot] += add ? 1 : -1; - if (keylen+2 > 64) indexed = zmalloc(keylen+2); + if (keylen+2 > 64) indexed = zmalloc(keylen+2, MALLOC_SHARED); indexed[0] = (hashslot >> 8) & 0xff; indexed[1] = hashslot & 0xff; memcpy(indexed+2,key->ptr,keylen); diff --git a/src/debug.c b/src/debug.c index a98bc61ad..d544d8832 100644 --- a/src/debug.c +++ b/src/debug.c @@ -345,7 +345,7 @@ NULL restartServer(flags,delay); addReplyError(c,"failed to restart the server. Check server logs."); } else if (!strcasecmp(c->argv[1]->ptr,"oom")) { - void *ptr = zmalloc(ULONG_MAX); /* Should trigger an out of memory. */ + void *ptr = zmalloc(ULONG_MAX, MALLOC_LOCAL); /* Should trigger an out of memory. */ zfree(ptr); addReply(c,shared.ok); } else if (!strcasecmp(c->argv[1]->ptr,"assert")) { diff --git a/src/defrag.c b/src/defrag.c index d67b6e253..7729c594c 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -72,8 +72,8 @@ void* activeDefragAlloc(void *ptr) { /* move this allocation to a new allocation. * make sure not to use the thread cache. so that we don't get back the same * pointers we try to free */ - size = zmalloc_size(ptr); - newptr = zmalloc_no_tcache(size); + size = zmalloc_size(ptr, MALLOC_LOCAL); + newptr = zmalloc_no_tcache(size, MALLOC_LOCAL); memcpy(newptr, ptr, size); zfree_no_tcache(ptr); return newptr; diff --git a/src/dict.c b/src/dict.c index 2cf9d4839..3560eb3d3 100644 --- a/src/dict.c +++ b/src/dict.c @@ -111,7 +111,7 @@ static void _dictReset(dictht *ht) dict *dictCreate(dictType *type, void *privDataPtr) { - dict *d = zmalloc(sizeof(*d)); + dict *d = zmalloc(sizeof(*d), MALLOC_SHARED); _dictInit(d,type,privDataPtr); return d; @@ -160,7 +160,7 @@ int dictExpand(dict *d, unsigned long size) /* Allocate the new hash table and initialize all pointers to NULL */ n.size = realsize; n.sizemask = realsize-1; - n.table = zcalloc(realsize*sizeof(dictEntry*)); + n.table = zcalloc(realsize*sizeof(dictEntry*), MALLOC_SHARED); n.used = 0; /* Is this the first initialization? If so it's not really a rehashing @@ -307,7 +307,7 @@ dictEntry *dictAddRaw(dict *d, void *key, dictEntry **existing) * system it is more likely that recently added entries are accessed * more frequently. */ ht = dictIsRehashing(d) ? &d->ht[1] : &d->ht[0]; - entry = zmalloc(sizeof(*entry)); + entry = zmalloc(sizeof(*entry), MALLOC_SHARED); entry->next = ht->table[index]; ht->table[index] = entry; ht->used++; @@ -541,7 +541,7 @@ long long dictFingerprint(dict *d) { dictIterator *dictGetIterator(dict *d) { - dictIterator *iter = zmalloc(sizeof(*iter)); + dictIterator *iter = zmalloc(sizeof(*iter), MALLOC_LOCAL); iter->d = d; iter->table = 0; diff --git a/src/evict.c b/src/evict.c index 773916ce8..cd033e62f 100644 --- a/src/evict.c +++ b/src/evict.c @@ -140,7 +140,7 @@ void evictionPoolAlloc(void) { struct evictionPoolEntry *ep; int j; - ep = zmalloc(sizeof(*ep)*EVPOOL_SIZE); + ep = zmalloc(sizeof(*ep)*EVPOOL_SIZE, MALLOC_LOCAL); for (j = 0; j < EVPOOL_SIZE; j++) { ep[j].idle = 0; ep[j].key = NULL; diff --git a/src/geo.c b/src/geo.c index 91a0421f5..1c792255e 100644 --- a/src/geo.c +++ b/src/geo.c @@ -51,7 +51,7 @@ int zslValueLteMax(double value, zrangespec *spec); /* Create a new array of geoPoints. */ geoArray *geoArrayCreate(void) { - geoArray *ga = zmalloc(sizeof(*ga)); + geoArray *ga = zmalloc(sizeof(*ga), MALLOC_SHARED); /* It gets allocated on first geoArrayAppend() call. */ ga->array = NULL; ga->buckets = 0; @@ -413,7 +413,7 @@ void geoaddCommand(client *c) { int elements = (c->argc - 2) / 3; int argc = 2+elements*2; /* ZADD key score ele ... */ - robj **argv = zcalloc(argc*sizeof(robj*)); + robj **argv = zcalloc(argc*sizeof(robj*), MALLOC_LOCAL); argv[0] = createRawStringObject("zadd",4); argv[1] = c->argv[1]; /* key */ incrRefCount(argv[1]); diff --git a/src/intset.c b/src/intset.c index 4445a5ca6..4cf15f4db 100644 --- a/src/intset.c +++ b/src/intset.c @@ -95,7 +95,7 @@ static void _intsetSet(intset *is, int pos, int64_t value) { /* Create an empty intset. */ intset *intsetNew(void) { - intset *is = zmalloc(sizeof(intset)); + intset *is = zmalloc(sizeof(intset), MALLOC_SHARED); is->encoding = intrev32ifbe(INTSET_ENC_INT16); is->length = 0; return is; diff --git a/src/latency.c b/src/latency.c index 33aa1245b..5b4610993 100644 --- a/src/latency.c +++ b/src/latency.c @@ -102,7 +102,7 @@ void latencyAddSample(char *event, mstime_t latency) { /* Create the time series if it does not exist. */ if (ts == NULL) { - ts = zmalloc(sizeof(*ts)); + ts = zmalloc(sizeof(*ts), MALLOC_SHARED); ts->idx = 0; ts->max = 0; memset(ts->samples,0,sizeof(ts->samples)); diff --git a/src/listpack_malloc.h b/src/listpack_malloc.h index 401ab6f74..4c0317528 100644 --- a/src/listpack_malloc.h +++ b/src/listpack_malloc.h @@ -39,7 +39,7 @@ #ifndef LISTPACK_ALLOC_H #define LISTPACK_ALLOC_H #include "zmalloc.h" -#define lp_malloc zmalloc +#define lp_malloc(size) zmalloc(size, MALLOC_SHARED) #define lp_realloc zrealloc #define lp_free zfree #endif diff --git a/src/lolwut5.c b/src/lolwut5.c index 8408b378d..b247cfb13 100644 --- a/src/lolwut5.c +++ b/src/lolwut5.c @@ -71,10 +71,10 @@ void lwTranslatePixelsGroup(int byte, char *output) { /* Allocate and return a new canvas of the specified size. */ lwCanvas *lwCreateCanvas(int width, int height) { - lwCanvas *canvas = zmalloc(sizeof(*canvas)); + lwCanvas *canvas = zmalloc(sizeof(*canvas), MALLOC_SHARED); canvas->width = width; canvas->height = height; - canvas->pixels = zmalloc(width*height); + canvas->pixels = zmalloc(width*height, MALLOC_SHARED); memset(canvas->pixels,0,width*height); return canvas; } diff --git a/src/module.c b/src/module.c index 8954fcdf0..b06493a1b 100644 --- a/src/module.c +++ b/src/module.c @@ -280,7 +280,7 @@ void RM_FreeDict(RedisModuleCtx *ctx, RedisModuleDict *d); * and in general is taken into account as memory allocated by Redis. * You should avoid using malloc(). */ void *RM_Alloc(size_t bytes) { - return zmalloc(bytes); + return zmalloc(bytes, MALLOC_LOCAL); } /* Use like calloc(). Memory allocated with this function is reported in @@ -288,7 +288,7 @@ void *RM_Alloc(size_t bytes) { * and in general is taken into account as memory allocated by Redis. * You should avoid using calloc() directly. */ void *RM_Calloc(size_t nmemb, size_t size) { - return zcalloc(nmemb*size); + return zcalloc(nmemb*size, MALLOC_LOCAL); } /* Use like realloc() for memory obtained with RedisModule_Alloc(). */ @@ -354,7 +354,7 @@ void *RM_PoolAlloc(RedisModuleCtx *ctx, size_t bytes) { if (left < bytes) { size_t blocksize = REDISMODULE_POOL_ALLOC_MIN_SIZE; if (blocksize < bytes) blocksize = bytes; - b = zmalloc(sizeof(*b) + blocksize); + b = zmalloc(sizeof(*b) + blocksize, MALLOC_LOCAL); b->size = blocksize; b->used = 0; b->next = ctx->pa_head; @@ -668,10 +668,10 @@ int RM_CreateCommand(RedisModuleCtx *ctx, const char *name, RedisModuleCmdFunc c * * Note that we use the Redis command table 'getkeys_proc' in order to * pass a reference to the command proxy structure. */ - cp = zmalloc(sizeof(*cp)); + cp = zmalloc(sizeof(*cp), MALLOC_LOCAL); cp->module = ctx->module; cp->func = cmdfunc; - cp->rediscmd = zmalloc(sizeof(*rediscmd)); + cp->rediscmd = zmalloc(sizeof(*rediscmd), MALLOC_LOCAL); cp->rediscmd->name = cmdname; cp->rediscmd->proc = RedisModuleCommandDispatcher; cp->rediscmd->arity = -1; @@ -695,7 +695,7 @@ void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int api RedisModule *module; if (ctx->module != NULL) return; - module = zmalloc(sizeof(*module)); + module = zmalloc(sizeof(*module), MALLOC_LOCAL); module->name = sdsnew((char*)name); module->ver = ver; module->apiver = apiver; @@ -1470,7 +1470,7 @@ void *RM_OpenKey(RedisModuleCtx *ctx, robj *keyname, int mode) { } /* Setup the key handle. */ - kp = zmalloc(sizeof(*kp)); + kp = zmalloc(sizeof(*kp), MALLOC_LOCAL); kp->ctx = ctx; kp->db = ctx->client->db; kp->key = keyname; @@ -2392,7 +2392,7 @@ int RM_HashGet(RedisModuleKey *key, int flags, ...) { * is processed as needed. Initially we just make sure to set the right * reply type, which is extremely cheap to do. */ RedisModuleCallReply *moduleCreateCallReplyFromProto(RedisModuleCtx *ctx, sds proto) { - RedisModuleCallReply *reply = zmalloc(sizeof(*reply)); + RedisModuleCallReply *reply = zmalloc(sizeof(*reply), MALLOC_LOCAL); reply->ctx = ctx; reply->proto = proto; reply->protolen = sdslen(proto); @@ -2482,7 +2482,7 @@ void moduleParseCallReply_Array(RedisModuleCallReply *reply) { return; } - reply->val.array = zmalloc(sizeof(RedisModuleCallReply)*arraylen); + reply->val.array = zmalloc(sizeof(RedisModuleCallReply)*arraylen, MALLOC_LOCAL); reply->len = arraylen; for (j = 0; j < arraylen; j++) { RedisModuleCallReply *ele = reply->val.array+j; @@ -2996,7 +2996,7 @@ moduleType *RM_CreateDataType(RedisModuleCtx *ctx, const char *name, int encver, moduleTypeFreeFunc free; } *tms = (struct typemethods*) typemethods_ptr; - moduleType *mt = zcalloc(sizeof(*mt)); + moduleType *mt = zcalloc(sizeof(*mt), MALLOC_LOCAL); mt->id = id; mt->module = ctx->module; mt->rdb_load = tms->rdb_load; @@ -3393,7 +3393,7 @@ void RM_EmitAOF(RedisModuleIO *io, const char *cmdname, const char *fmt, ...) { RedisModuleCtx *RM_GetContextFromIO(RedisModuleIO *io) { if (io->ctx) return io->ctx; /* Can't have more than one... */ RedisModuleCtx ctxtemplate = REDISMODULE_CTX_INIT; - io->ctx = zmalloc(sizeof(RedisModuleCtx)); + io->ctx = zmalloc(sizeof(RedisModuleCtx), MALLOC_LOCAL); *(io->ctx) = ctxtemplate; io->ctx->module = io->type->module; io->ctx->client = NULL; @@ -3530,7 +3530,7 @@ RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc int islua = c->flags & CLIENT_LUA; int ismulti = c->flags & CLIENT_MULTI; - c->bpop.module_blocked_handle = zmalloc(sizeof(RedisModuleBlockedClient)); + c->bpop.module_blocked_handle = zmalloc(sizeof(RedisModuleBlockedClient), MALLOC_LOCAL); RedisModuleBlockedClient *bc = c->bpop.module_blocked_handle; /* We need to handle the invalid operation of calling modules blocking @@ -3780,7 +3780,7 @@ int RM_BlockedClientDisconnected(RedisModuleCtx *ctx) { * TODO: thread safe contexts do not inherit the blocked client * selected database. */ RedisModuleCtx *RM_GetThreadSafeContext(RedisModuleBlockedClient *bc) { - RedisModuleCtx *ctx = zmalloc(sizeof(*ctx)); + RedisModuleCtx *ctx = zmalloc(sizeof(*ctx), MALLOC_LOCAL); RedisModuleCtx empty = REDISMODULE_CTX_INIT; memcpy(ctx,&empty,sizeof(empty)); if (bc) { @@ -3880,7 +3880,7 @@ void moduleReleaseGIL(void) { * See https://redis.io/topics/notifications for more information. */ int RM_SubscribeToKeyspaceEvents(RedisModuleCtx *ctx, int types, RedisModuleNotificationFunc callback) { - RedisModuleKeyspaceSubscriber *sub = zmalloc(sizeof(*sub)); + RedisModuleKeyspaceSubscriber *sub = zmalloc(sizeof(*sub), MALLOC_LOCAL); sub->module = ctx->module; sub->event_mask = types; sub->notify_callback = callback; @@ -4017,7 +4017,7 @@ void RM_RegisterClusterMessageReceiver(RedisModuleCtx *ctx, uint8_t type, RedisM /* Not found, let's add it. */ if (callback) { - r = zmalloc(sizeof(*r)); + r = zmalloc(sizeof(*r), MALLOC_LOCAL); r->module_id = module_id; r->module = ctx->module; r->callback = callback; @@ -4069,14 +4069,14 @@ char **RM_GetClusterNodesList(RedisModuleCtx *ctx, size_t *numnodes) { if (!server.cluster_enabled) return NULL; size_t count = dictSize(server.cluster->nodes); - char **ids = zmalloc((count+1)*REDISMODULE_NODE_ID_LEN); + char **ids = zmalloc((count+1)*REDISMODULE_NODE_ID_LEN, MALLOC_LOCAL); dictIterator *di = dictGetIterator(server.cluster->nodes); dictEntry *de; int j = 0; while((de = dictNext(di)) != NULL) { clusterNode *node = dictGetVal(de); if (node->flags & (CLUSTER_NODE_NOADDR|CLUSTER_NODE_HANDSHAKE)) continue; - ids[j] = zmalloc(REDISMODULE_NODE_ID_LEN); + ids[j] = zmalloc(REDISMODULE_NODE_ID_LEN, MALLOC_LOCAL); memcpy(ids[j],node->name,REDISMODULE_NODE_ID_LEN); j++; } @@ -4269,7 +4269,7 @@ int moduleTimerHandler(struct aeEventLoop *eventLoop, long long id, void *client * the specified function using `data` as argument. The returned timer ID can be * used to get information from the timer or to stop it before it fires. */ RedisModuleTimerID RM_CreateTimer(RedisModuleCtx *ctx, mstime_t period, RedisModuleTimerProc callback, void *data) { - RedisModuleTimer *timer = zmalloc(sizeof(*timer)); + RedisModuleTimer *timer = zmalloc(sizeof(*timer), MALLOC_LOCAL); timer->module = ctx->module; timer->callback = callback; timer->data = data; @@ -4367,7 +4367,7 @@ int RM_GetTimerInfo(RedisModuleCtx *ctx, RedisModuleTimerID id, uint64_t *remain * Next / Prev dictionary iterator calls. */ RedisModuleDict *RM_CreateDict(RedisModuleCtx *ctx) { - struct RedisModuleDict *d = zmalloc(sizeof(*d)); + struct RedisModuleDict *d = zmalloc(sizeof(*d), MALLOC_LOCAL); d->rax = raxNew(); if (ctx != NULL) autoMemoryAdd(ctx,REDISMODULE_AM_DICT,d); return d; @@ -4467,7 +4467,7 @@ int RM_DictDel(RedisModuleDict *d, RedisModuleString *key, void *oldval) { * REDISMODULE_ERR at the first call, otherwise they'll produce elements. */ RedisModuleDictIter *RM_DictIteratorStartC(RedisModuleDict *d, const char *op, void *key, size_t keylen) { - RedisModuleDictIter *di = zmalloc(sizeof(*di)); + RedisModuleDictIter *di = zmalloc(sizeof(*di), MALLOC_LOCAL); di->dict = d; raxStart(&di->ri,d->rax); raxSeek(&di->ri,op,key,keylen); diff --git a/src/multi.c b/src/multi.c index 71090d8ed..720ec90e2 100644 --- a/src/multi.c +++ b/src/multi.c @@ -63,7 +63,7 @@ void queueMultiCommand(client *c) { mc = c->mstate.commands+c->mstate.count; mc->cmd = c->cmd; mc->argc = c->argc; - mc->argv = zmalloc(sizeof(robj*)*c->argc); + mc->argv = zmalloc(sizeof(robj*)*c->argc, MALLOC_LOCAL); memcpy(mc->argv,c->argv,sizeof(robj*)*c->argc); for (j = 0; j < c->argc; j++) incrRefCount(mc->argv[j]); @@ -253,7 +253,7 @@ void watchForKey(client *c, robj *key) { } listAddNodeTail(clients,c); /* Add the new key to the list of keys watched by this client */ - wk = zmalloc(sizeof(*wk)); + wk = zmalloc(sizeof(*wk), MALLOC_SHARED); wk->key = key; wk->db = c->db; incrRefCount(key); diff --git a/src/networking.c b/src/networking.c index 9e62cb6be..b853da5bd 100644 --- a/src/networking.c +++ b/src/networking.c @@ -57,7 +57,7 @@ size_t getStringObjectSdsUsedMemory(robj *o) { /* Client.reply list dup and free methods. */ void *dupClientReplyValue(void *o) { clientReplyBlock *old = o; - clientReplyBlock *buf = zmalloc(sizeof(clientReplyBlock) + old->size); + clientReplyBlock *buf = zmalloc(sizeof(clientReplyBlock) + old->size, MALLOC_LOCAL); memcpy(buf, o, sizeof(clientReplyBlock) + old->size); return buf; } @@ -83,7 +83,7 @@ void linkClient(client *c) { } client *createClient(int fd) { - client *c = zmalloc(sizeof(client)); + client *c = zmalloc(sizeof(client), MALLOC_LOCAL); /* passing -1 as fd it is possible to create a non connected client. * This is useful since all the commands needs to be executed @@ -281,7 +281,7 @@ void _addReplyProtoToList(client *c, const char *s, size_t len) { /* Create a new node, make sure it is allocated to at * least PROTO_REPLY_CHUNK_BYTES */ size_t size = len < PROTO_REPLY_CHUNK_BYTES? PROTO_REPLY_CHUNK_BYTES: len; - tail = zmalloc(size + sizeof(clientReplyBlock)); + tail = zmalloc(size + sizeof(clientReplyBlock), MALLOC_LOCAL); /* take over the allocation's internal fragmentation */ tail->size = zmalloc_usable(tail) - sizeof(clientReplyBlock); tail->used = len; @@ -460,7 +460,7 @@ void setDeferredAggregateLen(client *c, void *node, long length, char prefix) { listDelNode(c->reply,ln); } else { /* Create a new node */ - clientReplyBlock *buf = zmalloc(lenstr_len + sizeof(clientReplyBlock)); + clientReplyBlock *buf = zmalloc(lenstr_len + sizeof(clientReplyBlock), MALLOC_LOCAL); /* Take over the allocation's internal fragmentation */ buf->size = zmalloc_usable(buf) - sizeof(clientReplyBlock); buf->used = lenstr_len; @@ -1333,7 +1333,7 @@ int processInlineBuffer(client *c) { /* Setup argv array on client structure */ if (argc) { if (c->argv) zfree(c->argv); - c->argv = zmalloc(sizeof(robj*)*argc); + c->argv = zmalloc(sizeof(robj*)*argc, MALLOC_LOCAL); } /* Create redis objects for all arguments. */ @@ -1345,7 +1345,7 @@ int processInlineBuffer(client *c) { sdsfree(argv[j]); } } - zfree(argv); + sds_free(argv); return C_OK; } @@ -1431,7 +1431,7 @@ int processMultibulkBuffer(client *c) { /* Setup argv array on client structure */ if (c->argv) zfree(c->argv); - c->argv = zmalloc(sizeof(robj*)*c->multibulklen); + c->argv = zmalloc(sizeof(robj*)*c->multibulklen, MALLOC_LOCAL); } serverAssertWithInfo(c,NULL,c->multibulklen > 0); @@ -2123,7 +2123,7 @@ void rewriteClientCommandVector(client *c, int argc, ...) { int j; robj **argv; /* The new argument vector */ - argv = zmalloc(sizeof(robj*)*argc); + argv = zmalloc(sizeof(robj*)*argc, MALLOC_LOCAL); va_start(ap,argc); for (j = 0; j < argc; j++) { robj *a; diff --git a/src/object.c b/src/object.c index ec0bd02ee..ddd50b553 100644 --- a/src/object.c +++ b/src/object.c @@ -39,7 +39,7 @@ /* ===================== Creation and parsing of objects ==================== */ robj *createObject(int type, void *ptr) { - robj *o = zmalloc(sizeof(*o)); + robj *o = salloc_obj(); //zmalloc(sizeof(*o), MALLOC_SHARED); o->type = type; o->encoding = OBJ_ENCODING_RAW; o->ptr = ptr; @@ -82,7 +82,7 @@ robj *createRawStringObject(const char *ptr, size_t len) { * an object where the sds string is actually an unmodifiable string * allocated in the same chunk as the object itself. */ robj *createEmbeddedStringObject(const char *ptr, size_t len) { - robj *o = zmalloc(sizeof(robj)+sizeof(struct sdshdr8)+len+1); + robj *o = zmalloc(sizeof(robj)+sizeof(struct sdshdr8)+len+1, MALLOC_SHARED); struct sdshdr8 *sh = (void*)(o+1); o->type = OBJ_STRING; @@ -247,7 +247,7 @@ robj *createHashObject(void) { } robj *createZsetObject(void) { - zset *zs = zmalloc(sizeof(*zs)); + zset *zs = zmalloc(sizeof(*zs), MALLOC_SHARED); robj *o; zs->dict = dictCreate(&zsetDictType,NULL); @@ -272,7 +272,7 @@ robj *createStreamObject(void) { } robj *createModuleObject(moduleType *mt, void *value) { - moduleValue *mv = zmalloc(sizeof(*mv)); + moduleValue *mv = zmalloc(sizeof(*mv), MALLOC_SHARED); mv->type = mt; mv->value = value; return createObject(OBJ_MODULE,mv); @@ -362,7 +362,10 @@ void decrRefCount(robj *o) { case OBJ_STREAM: freeStreamObject(o); break; default: serverPanic("Unknown object type"); break; } - zfree(o); + if (o->type == OBJ_STRING && o->encoding == OBJ_ENCODING_EMBSTR) + zfree(o); + else + sfree_obj(o); } else { if (o->refcount <= 0) serverPanic("decrRefCount against refcount <= 0"); if (o->refcount != OBJ_SHARED_REFCOUNT) o->refcount--; @@ -945,7 +948,7 @@ struct redisMemOverhead *getMemoryOverheadData(void) { size_t mem_total = 0; size_t mem = 0; size_t zmalloc_used = zmalloc_used_memory(); - struct redisMemOverhead *mh = zcalloc(sizeof(*mh)); + struct redisMemOverhead *mh = zcalloc(sizeof(*mh), MALLOC_LOCAL); mh->total_allocated = zmalloc_used; mh->startup_allocated = server.initial_memory_usage; diff --git a/src/pubsub.c b/src/pubsub.c index 994dd9734..a7ee67400 100644 --- a/src/pubsub.c +++ b/src/pubsub.c @@ -209,7 +209,7 @@ int pubsubSubscribePattern(client *c, robj *pattern) { pubsubPattern *pat; listAddNodeTail(c->pubsub_patterns,pattern); incrRefCount(pattern); - pat = zmalloc(sizeof(*pat)); + pat = zmalloc(sizeof(*pat), MALLOC_LOCAL); pat->pattern = getDecodedObject(pattern); pat->client = c; listAddNodeTail(server.pubsub_patterns,pat); diff --git a/src/quicklist.c b/src/quicklist.c index 7b5484116..ea0a2ae77 100644 --- a/src/quicklist.c +++ b/src/quicklist.c @@ -94,7 +94,7 @@ static const size_t optimization_level[] = {4096, 8192, 16384, 32768, 65536}; quicklist *quicklistCreate(void) { struct quicklist *quicklist; - quicklist = zmalloc(sizeof(*quicklist)); + quicklist = zmalloc(sizeof(*quicklist), MALLOC_SHARED); quicklist->head = quicklist->tail = NULL; quicklist->len = 0; quicklist->count = 0; @@ -137,7 +137,7 @@ quicklist *quicklistNew(int fill, int compress) { REDIS_STATIC quicklistNode *quicklistCreateNode(void) { quicklistNode *node; - node = zmalloc(sizeof(*node)); + node = zmalloc(sizeof(*node), MALLOC_SHARED); node->zl = NULL; node->count = 0; node->sz = 0; @@ -184,7 +184,7 @@ REDIS_STATIC int __quicklistCompressNode(quicklistNode *node) { if (node->sz < MIN_COMPRESS_BYTES) return 0; - quicklistLZF *lzf = zmalloc(sizeof(*lzf) + node->sz); + quicklistLZF *lzf = zmalloc(sizeof(*lzf) + node->sz, MALLOC_SHARED); /* Cancel if compression fails or doesn't compress small enough */ if (((lzf->sz = lzf_compress(node->zl, node->sz, lzf->compressed, @@ -217,7 +217,7 @@ REDIS_STATIC int __quicklistDecompressNode(quicklistNode *node) { node->attempted_compress = 0; #endif - void *decompressed = zmalloc(node->sz); + void *decompressed = zmalloc(node->sz, MALLOC_SHARED); quicklistLZF *lzf = (quicklistLZF *)node->zl; if (lzf_decompress(lzf->compressed, lzf->sz, decompressed, node->sz) == 0) { /* Someone requested decompress, but we can't decompress. Not good. */ @@ -799,7 +799,7 @@ REDIS_STATIC quicklistNode *_quicklistSplitNode(quicklistNode *node, int offset, size_t zl_sz = node->sz; quicklistNode *new_node = quicklistCreateNode(); - new_node->zl = zmalloc(zl_sz); + new_node->zl = zmalloc(zl_sz, MALLOC_SHARED); /* Copy original ziplist so we can split it */ memcpy(new_node->zl, node->zl, zl_sz); @@ -1048,7 +1048,7 @@ int quicklistCompare(unsigned char *p1, unsigned char *p2, int p2_len) { quicklistIter *quicklistGetIterator(const quicklist *quicklist, int direction) { quicklistIter *iter; - iter = zmalloc(sizeof(*iter)); + iter = zmalloc(sizeof(*iter), MALLOC_LOCAL); if (direction == AL_START_HEAD) { iter->current = quicklist->head; @@ -1195,10 +1195,10 @@ quicklist *quicklistDup(quicklist *orig) { if (current->encoding == QUICKLIST_NODE_ENCODING_LZF) { quicklistLZF *lzf = (quicklistLZF *)current->zl; size_t lzf_sz = sizeof(*lzf) + lzf->sz; - node->zl = zmalloc(lzf_sz); + node->zl = zmalloc(lzf_sz, MALLOC_SHARED); memcpy(node->zl, current->zl, lzf_sz); } else if (current->encoding == QUICKLIST_NODE_ENCODING_RAW) { - node->zl = zmalloc(current->sz); + node->zl = zmalloc(current->sz, MALLOC_SHARED); memcpy(node->zl, current->zl, current->sz); } @@ -1372,7 +1372,7 @@ int quicklistPopCustom(quicklist *quicklist, int where, unsigned char **data, REDIS_STATIC void *_quicklistSaver(unsigned char *data, unsigned int sz) { unsigned char *vstr; if (data) { - vstr = zmalloc(sz); + vstr = zmalloc(sz, MALLOC_SHARED); memcpy(vstr, data, sz); return vstr; } diff --git a/src/rax_malloc.h b/src/rax_malloc.h index 9295985c6..a5aea44d9 100644 --- a/src/rax_malloc.h +++ b/src/rax_malloc.h @@ -38,7 +38,7 @@ #ifndef RAX_ALLOC_H #define RAX_ALLOC_H #include "zmalloc.h" -#define rax_malloc zmalloc +#define rax_malloc(size) zmalloc(size, MALLOC_SHARED) #define rax_realloc zrealloc #define rax_free zfree #endif diff --git a/src/rdb.c b/src/rdb.c index 351b90774..32a3df1d5 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -32,6 +32,7 @@ #include "zipmap.h" #include "endianconv.h" #include "stream.h" +#include "storage.h" #include #include @@ -284,7 +285,7 @@ void *rdbLoadIntegerObject(rio *rdb, int enctype, int flags, size_t *lenptr) { char buf[LONG_STR_SIZE], *p; int len = ll2string(buf,sizeof(buf),val); if (lenptr) *lenptr = len; - p = plain ? zmalloc(len) : sdsnewlen(SDS_NOINIT,len); + p = plain ? zmalloc(len, MALLOC_SHARED) : sdsnewlen(SDS_NOINIT,len); memcpy(p,buf,len); return p; } else if (encode) { @@ -345,7 +346,7 @@ ssize_t rdbSaveLzfStringObject(rio *rdb, unsigned char *s, size_t len) { /* We require at least four bytes compression for this to be worth it */ if (len <= 4) return 0; outlen = len-4; - if ((out = zmalloc(outlen+1)) == NULL) return 0; + if ((out = zmalloc(outlen+1, MALLOC_LOCAL)) == NULL) return 0; comprlen = lzf_compress(s, len, out, outlen); if (comprlen == 0) { zfree(out); @@ -368,11 +369,11 @@ void *rdbLoadLzfStringObject(rio *rdb, int flags, size_t *lenptr) { if ((clen = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL; if ((len = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL; - if ((c = zmalloc(clen)) == NULL) goto err; + if ((c = zmalloc(clen, MALLOC_SHARED)) == NULL) goto err; /* Allocate our target according to the uncompressed size. */ if (plain) { - val = zmalloc(len); + val = zmalloc(len, MALLOC_SHARED); } else { val = sdsnewlen(SDS_NOINIT,len); } @@ -501,7 +502,7 @@ void *rdbGenericLoadStringObject(rio *rdb, int flags, size_t *lenptr) { if (len == RDB_LENERR) return NULL; if (plain || sds) { - void *buf = plain ? zmalloc(len) : sdsnewlen(SDS_NOINIT,len); + void *buf = plain ? zmalloc(len, MALLOC_SHARED) : sdsnewlen(SDS_NOINIT,len); if (lenptr) *lenptr = len; if (len && rioRead(rdb,buf,len) == 0) { if (plain) @@ -1290,10 +1291,12 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) { openChildInfoPipe(); start = ustime(); + handle_prefork(); if ((childpid = fork()) == 0) { int retval; /* Child */ + handle_postfork(0); closeListeningSockets(0); redisSetProcTitle("redis-rdb-bgsave"); retval = rdbSave(filename,rsi); @@ -1312,6 +1315,7 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) { exitFromChild((retval == C_OK) ? 0 : 1); } else { /* Parent */ + handle_postfork(childpid); server.stat_fork_time = ustime()-start; server.stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / server.stat_fork_time / (1024*1024*1024); /* GB per second. */ latencyAddSampleIfNeeded("fork",server.stat_fork_time/1000); @@ -2141,7 +2145,7 @@ void backgroundSaveDoneHandlerSocket(int exitcode, int bysignal) { * If the process returned an error, consider the list of slaves that * can continue to be empty, so that it's just a special case of the * normal code path. */ - ok_slaves = zmalloc(sizeof(uint64_t)); /* Make space for the count. */ + ok_slaves = zmalloc(sizeof(uint64_t), MALLOC_LOCAL); /* Make space for the count. */ ok_slaves[0] = 0; if (!bysignal && exitcode == 0) { int readlen = sizeof(uint64_t); @@ -2258,11 +2262,11 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { /* Collect the file descriptors of the slaves we want to transfer * the RDB to, which are i WAIT_BGSAVE_START state. */ - fds = zmalloc(sizeof(int)*listLength(server.slaves)); + fds = zmalloc(sizeof(int)*listLength(server.slaves), MALLOC_LOCAL); /* We also allocate an array of corresponding client IDs. This will * be useful for the child process in order to build the report * (sent via unix pipe) that will be sent to the parent. */ - clientids = zmalloc(sizeof(uint64_t)*listLength(server.slaves)); + clientids = zmalloc(sizeof(uint64_t)*listLength(server.slaves), MALLOC_LOCAL); numfds = 0; listRewind(server.slaves,&li); @@ -2326,7 +2330,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { * can match the report with a specific slave, and 'error' is * set to 0 if the replication process terminated with a success * or the error code if an error occurred. */ - void *msg = zmalloc(sizeof(uint64_t)*(1+2*numfds)); + void *msg = zmalloc(sizeof(uint64_t)*(1+2*numfds), MALLOC_LOCAL); uint64_t *len = msg; uint64_t *ids = len+1; int j, msglen; diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c index d30879dc4..716c492df 100644 --- a/src/redis-benchmark.c +++ b/src/redis-benchmark.c @@ -45,6 +45,7 @@ #include "hiredis.h" #include "adlist.h" #include "zmalloc.h" +#include "storage.h" #define UNUSED(V) ((void) V) #define RANDPTR_INITIAL_SIZE 8 @@ -315,7 +316,7 @@ static void writeHandler(aeEventLoop *el, int fd, void *privdata, int mask) { * Even when cloning another client, prefix commands are applied if needed.*/ static client createClient(char *cmd, size_t len, client from) { int j; - client c = zmalloc(sizeof(struct _client)); + client c = zmalloc(sizeof(struct _client), MALLOC_LOCAL); if (config.hostsocket == NULL) { c->context = redisConnectNonBlock(config.hostip,config.hostport); @@ -379,7 +380,7 @@ static client createClient(char *cmd, size_t len, client from) { if (from) { c->randlen = from->randlen; c->randfree = 0; - c->randptr = zmalloc(sizeof(char*)*c->randlen); + c->randptr = zmalloc(sizeof(char*)*c->randlen, MALLOC_LOCAL); /* copy the offsets. */ for (j = 0; j < (int)c->randlen; j++) { c->randptr[j] = c->obuf + (from->randptr[j]-from->obuf); @@ -391,7 +392,7 @@ static client createClient(char *cmd, size_t len, client from) { c->randlen = 0; c->randfree = RANDPTR_INITIAL_SIZE; - c->randptr = zmalloc(sizeof(char*)*c->randfree); + c->randptr = zmalloc(sizeof(char*)*c->randfree, MALLOC_LOCAL); while ((p = strstr(p,"__rand_int__")) != NULL) { if (c->randfree == 0) { c->randptr = zrealloc(c->randptr,sizeof(char*)*c->randlen*2); @@ -652,6 +653,8 @@ int main(int argc, const char **argv) { client c; + storage_init(); + srandom(time(NULL)); signal(SIGHUP, SIG_IGN); signal(SIGPIPE, SIG_IGN); @@ -684,7 +687,7 @@ int main(int argc, const char **argv) { argc -= i; argv += i; - config.latency = zmalloc(sizeof(long long)*config.requests); + config.latency = zmalloc(sizeof(long long)*config.requests, MALLOC_LOCAL); if (config.keepalive == 0) { printf("WARNING: keepalive disabled, you probably need 'echo 1 > /proc/sys/net/ipv4/tcp_tw_reuse' for Linux and 'sudo sysctl -w net.inet.tcp.msl=1000' for Mac OS X in order to use a lot of clients/requests\n"); @@ -716,7 +719,7 @@ int main(int argc, const char **argv) { } /* Run default benchmark suite. */ - data = zmalloc(config.datasize+1); + data = zmalloc(config.datasize+1, MALLOC_LOCAL); do { memset(data,'x',config.datasize); data[config.datasize] = '\0'; diff --git a/src/redis-check-aof.c b/src/redis-check-aof.c index c4d5a225e..a79adb946 100644 --- a/src/redis-check-aof.c +++ b/src/redis-check-aof.c @@ -82,7 +82,7 @@ int readString(FILE *fp, char** target) { /* Increase length to also consume \r\n */ len += 2; - *target = (char*)zmalloc(len); + *target = (char*)zmalloc(len, MALLOC_LOCAL); if (!readBytes(fp,*target,len)) { return 0; } diff --git a/src/redis-cli.c b/src/redis-cli.c index 93290e5ed..91deac496 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -55,6 +55,7 @@ #include "help.h" #include "anet.h" #include "ae.h" +#include "storage.h" #define UNUSED(V) ((void) V) @@ -477,11 +478,11 @@ static void cliInitHelp(void) { helpEntry tmp; helpEntriesLen = len = commandslen+groupslen; - helpEntries = zmalloc(sizeof(helpEntry)*len); + helpEntries = zmalloc(sizeof(helpEntry)*len, MALLOC_LOCAL); for (i = 0; i < groupslen; i++) { tmp.argc = 1; - tmp.argv = zmalloc(sizeof(sds)); + tmp.argv = zmalloc(sizeof(sds), MALLOC_LOCAL); tmp.argv[0] = sdscatprintf(sdsempty(),"@%s",commandGroups[i]); tmp.full = tmp.argv[0]; tmp.type = CLI_HELP_GROUP; @@ -532,13 +533,13 @@ static void cliIntegrateHelp(void) { helpEntry *new = helpEntries+(helpEntriesLen-1); new->argc = 1; - new->argv = zmalloc(sizeof(sds)); + new->argv = zmalloc(sizeof(sds), MALLOC_LOCAL); new->argv[0] = sdsnew(cmdname); new->full = new->argv[0]; new->type = CLI_HELP_COMMAND; sdstoupper(new->argv[0]); - struct commandHelp *ch = zmalloc(sizeof(*ch)); + struct commandHelp *ch = zmalloc(sizeof(*ch), MALLOC_LOCAL); ch->name = new->argv[0]; ch->params = sdsempty(); int args = llabs(entry->element[1]->integer); @@ -1171,7 +1172,7 @@ static int cliSendCommand(int argc, char **argv, long repeat) { } /* Setup argument length */ - argvlen = zmalloc(argc*sizeof(size_t)); + argvlen = zmalloc(argc*sizeof(size_t), MALLOC_LOCAL); for (j = 0; j < argc; j++) argvlen[j] = sdslen(argv[j]); @@ -1590,7 +1591,7 @@ static int confirmWithYes(char *msg) { /* Turn the plain C strings into Sds strings */ static char **convertToSds(int count, char** args) { int j; - char **sds = zmalloc(sizeof(char*)*count); + char **sds = zmalloc(sizeof(char*)*count, MALLOC_LOCAL); for(j = 0; j < count; j++) sds[j] = sdsnew(args[j]); @@ -1872,7 +1873,7 @@ static int evalMode(int argc, char **argv) { } /* Create our argument vector */ - argv2 = zmalloc(sizeof(sds)*(argc+3)); + argv2 = zmalloc(sizeof(sds)*(argc+3), MALLOC_LOCAL); argv2[0] = sdsnew("EVAL"); argv2[1] = script; for (j = 0; j < argc; j++) { @@ -2193,7 +2194,7 @@ static void freeClusterManager(void) { } static clusterManagerNode *clusterManagerNewNode(char *ip, int port) { - clusterManagerNode *node = zmalloc(sizeof(*node)); + clusterManagerNode *node = zmalloc(sizeof(*node), MALLOC_LOCAL); node->context = NULL; node->name = NULL; node->ip = ip; @@ -2228,7 +2229,7 @@ static int clusterManagerCheckRedisReply(clusterManagerNode *n, if (!r || (is_err = (r->type == REDIS_REPLY_ERROR))) { if (is_err) { if (err != NULL) { - *err = zmalloc((r->len + 1) * sizeof(char)); + *err = zmalloc((r->len + 1) * sizeof(char), MALLOC_LOCAL); strcpy(*err, r->str); } else CLUSTER_MANAGER_PRINT_REPLY_ERROR(n, r->str); } @@ -2374,7 +2375,7 @@ static redisReply *clusterManagerGetNodeRedisInfo(clusterManagerNode *node, if (info == NULL) return NULL; if (info->type == REDIS_REPLY_ERROR) { if (err != NULL) { - *err = zmalloc((info->len + 1) * sizeof(char)); + *err = zmalloc((info->len + 1) * sizeof(char), MALLOC_LOCAL); strcpy(*err, info->str); } freeReplyObject(info); @@ -2451,7 +2452,7 @@ static int clusterManagerGetAntiAffinityScore(clusterManagerNodeArray *ipnodes, int node_len = cluster_manager.nodes->len; clusterManagerNode **offending_p = NULL; if (offending != NULL) { - *offending = zcalloc(node_len * sizeof(clusterManagerNode*)); + *offending = zcalloc(node_len * sizeof(clusterManagerNode*), MALLOC_LOCAL); offending_p = *offending; } /* For each set of nodes in the same host, split by @@ -2541,7 +2542,7 @@ static void clusterManagerOptimizeAntiAffinity(clusterManagerNodeArray *ipnodes, clusterManagerNode *first = offenders[rand_idx], *second = NULL; clusterManagerNode **other_replicas = zcalloc((node_len - 1) * - sizeof(*other_replicas)); + sizeof(*other_replicas), MALLOC_LOCAL); int other_replicas_count = 0; listIter li; listNode *ln; @@ -2769,8 +2770,8 @@ static int clusterManagerAddSlots(clusterManagerNode *node, char**err) int success = 1; /* First two args are used for the command itself. */ int argc = node->slots_count + 2; - sds *argv = zmalloc(argc * sizeof(*argv)); - size_t *argvlen = zmalloc(argc * sizeof(*argvlen)); + sds *argv = zmalloc(argc * sizeof(*argv), MALLOC_LOCAL); + size_t *argvlen = zmalloc(argc * sizeof(*argvlen), MALLOC_LOCAL); argv[0] = "CLUSTER"; argv[1] = "ADDSLOTS"; argvlen[0] = 7; @@ -2869,7 +2870,7 @@ static int clusterManagerSetSlot(clusterManagerNode *node1, if (reply->type == REDIS_REPLY_ERROR) { success = 0; if (err != NULL) { - *err = zmalloc((reply->len + 1) * sizeof(char)); + *err = zmalloc((reply->len + 1) * sizeof(char), MALLOC_LOCAL); strcpy(*err, reply->str); } else CLUSTER_MANAGER_PRINT_REPLY_ERROR(node1, reply->str); goto cleanup; @@ -2983,8 +2984,8 @@ static int clusterManagerCompareKeysValues(clusterManagerNode *n1, { size_t i, argc = keys_reply->elements + 2; static const char *hash_zero = "0000000000000000000000000000000000000000"; - char **argv = zcalloc(argc * sizeof(char *)); - size_t *argv_len = zcalloc(argc * sizeof(size_t)); + char **argv = zcalloc(argc * sizeof(char *), MALLOC_LOCAL); + size_t *argv_len = zcalloc(argc * sizeof(size_t), MALLOC_LOCAL); argv[0] = "DEBUG"; argv_len[0] = 5; argv[1] = "DIGEST-VALUE"; @@ -3051,8 +3052,8 @@ static redisReply *clusterManagerMigrateKeysInReply(clusterManagerNode *source, if (config.auth) c += 2; size_t argc = c + reply->elements; size_t i, offset = 6; // Keys Offset - argv = zcalloc(argc * sizeof(char *)); - argv_len = zcalloc(argc * sizeof(size_t)); + argv = zcalloc(argc * sizeof(char *), MALLOC_LOCAL); + argv_len = zcalloc(argc * sizeof(size_t), MALLOC_LOCAL); char portstr[255]; char timeoutstr[255]; snprintf(portstr, 10, "%d", target->port); @@ -3130,7 +3131,7 @@ static int clusterManagerMigrateKeysInSlot(clusterManagerNode *source, if (reply->type == REDIS_REPLY_ERROR) { success = 0; if (err != NULL) { - *err = zmalloc((reply->len + 1) * sizeof(char)); + *err = zmalloc((reply->len + 1) * sizeof(char), MALLOC_LOCAL); strcpy(*err, reply->str); CLUSTER_MANAGER_PRINT_REPLY_ERROR(source, *err); } @@ -3142,7 +3143,7 @@ static int clusterManagerMigrateKeysInSlot(clusterManagerNode *source, freeReplyObject(reply); break; } - if (verbose) dots = zmalloc((count+1) * sizeof(char)); + if (verbose) dots = zmalloc((count+1) * sizeof(char), MALLOC_LOCAL); /* Calling MIGRATE command. */ migrate_reply = clusterManagerMigrateKeysInReply(source, target, reply, 0, timeout, @@ -3242,7 +3243,7 @@ static int clusterManagerMigrateKeysInSlot(clusterManagerNode *source, if (!success) { if (migrate_reply != NULL) { if (err) { - *err = zmalloc((migrate_reply->len + 1) * sizeof(char)); + *err = zmalloc((migrate_reply->len + 1) * sizeof(char), MALLOC_LOCAL); strcpy(*err, migrate_reply->str); } printf("\n"); @@ -3318,7 +3319,7 @@ static int clusterManagerMoveSlot(clusterManagerNode *source, if (r->type == REDIS_REPLY_ERROR) { success = 0; if (err != NULL) { - *err = zmalloc((r->len + 1) * sizeof(char)); + *err = zmalloc((r->len + 1) * sizeof(char), MALLOC_LOCAL); strcpy(*err, r->str); CLUSTER_MANAGER_PRINT_REPLY_ERROR(n, *err); } @@ -3347,7 +3348,7 @@ static int clusterManagerFlushNodeConfig(clusterManagerNode *node, char **err) { node->replicate); if (reply == NULL || (is_err = (reply->type == REDIS_REPLY_ERROR))) { if (is_err && err != NULL) { - *err = zmalloc((reply->len + 1) * sizeof(char)); + *err = zmalloc((reply->len + 1) * sizeof(char), MALLOC_LOCAL); strcpy(*err, reply->str); } success = 0; @@ -3715,7 +3716,7 @@ static sds clusterManagerGetConfigSignature(clusterManagerNode *node) { zrealloc(node_configs, (node_count * sizeof(char *))); /* Make room for '|' separators. */ tot_size += (sizeof(char) * (c - 1)); - char *cfg = zmalloc((sizeof(char) * tot_size) + 1); + char *cfg = zmalloc((sizeof(char) * tot_size) + 1, MALLOC_LOCAL); memcpy(cfg, nodename, name_len); char *sp = cfg + name_len; *(sp++) = ':'; @@ -4641,7 +4642,7 @@ static clusterManagerNode *clusterNodeForResharding(char *id, static list *clusterManagerComputeReshardTable(list *sources, int numslots) { list *moved = listCreate(); int src_count = listLength(sources), i = 0, tot_slots = 0, j; - clusterManagerNode **sorted = zmalloc(src_count * sizeof(*sorted)); + clusterManagerNode **sorted = zmalloc(src_count * sizeof(*sorted), MALLOC_LOCAL); listIter li; listNode *ln; listRewind(sources, &li); @@ -4662,7 +4663,7 @@ static list *clusterManagerComputeReshardTable(list *sources, int numslots) { int slot = node->slots[j]; if (!slot) continue; if (count >= max || (int)listLength(moved) >= numslots) break; - clusterManagerReshardTableItem *item = zmalloc(sizeof(*item)); + clusterManagerReshardTableItem *item = zmalloc(sizeof(*item), MALLOC_LOCAL); item->source = node; item->slot = j; listAddNodeTail(moved, item); @@ -4720,7 +4721,7 @@ static void clusterManagerLog(int level, const char* fmt, ...) { static void clusterManagerNodeArrayInit(clusterManagerNodeArray *array, int alloc_len) { - array->nodes = zcalloc(alloc_len * sizeof(clusterManagerNode*)); + array->nodes = zcalloc(alloc_len * sizeof(clusterManagerNode*), MALLOC_LOCAL); array->alloc = array->nodes; array->len = alloc_len; array->count = 0; @@ -4861,9 +4862,9 @@ static int clusterManagerCommandCreate(int argc, char **argv) { clusterManagerLogInfo(">>> Performing hash slots allocation " "on %d nodes...\n", node_len); int interleaved_len = 0, ip_count = 0; - clusterManagerNode **interleaved = zcalloc(node_len*sizeof(**interleaved)); - char **ips = zcalloc(node_len * sizeof(char*)); - clusterManagerNodeArray *ip_nodes = zcalloc(node_len * sizeof(*ip_nodes)); + clusterManagerNode **interleaved = zcalloc(node_len*sizeof(**interleaved), MALLOC_LOCAL); + char **ips = zcalloc(node_len * sizeof(char*), MALLOC_LOCAL); + clusterManagerNodeArray *ip_nodes = zcalloc(node_len * sizeof(*ip_nodes), MALLOC_LOCAL); listIter li; listNode *ln; listRewind(cluster_manager.nodes, &li); @@ -5500,7 +5501,7 @@ static int clusterManagerCommandRebalance(int argc, char **argv) { nodes_involved++; listAddNodeTail(involved, n); } - weightedNodes = zmalloc(nodes_involved * sizeof(clusterManagerNode *)); + weightedNodes = zmalloc(nodes_involved * sizeof(clusterManagerNode *), MALLOC_LOCAL); if (weightedNodes == NULL) goto cleanup; /* Check cluster, only proceed if it looks sane. */ clusterManagerCheckCluster(1); @@ -5845,7 +5846,7 @@ static int clusterManagerCommandCall(int argc, char **argv) { if (!clusterManagerLoadInfoFromNode(refnode, 0)) return 0; argc--; argv++; - size_t *argvlen = zmalloc(argc*sizeof(size_t)); + size_t *argvlen = zmalloc(argc*sizeof(size_t), MALLOC_LOCAL); clusterManagerLogInfo(">>> Calling"); for (i = 0; i < argc; i++) { argvlen[i] = strlen(argv[i]); @@ -6841,7 +6842,7 @@ static char *getInfoField(char *info, char *field) { n1 = strchr(p,'\r'); n2 = strchr(p,','); if (n2 && n2 < n1) n1 = n2; - result = zmalloc(sizeof(char)*(n1-p)+1); + result = zmalloc(sizeof(char)*(n1-p)+1, MALLOC_LOCAL); memcpy(result,p,(n1-p)); result[n1-p] = '\0'; return result; @@ -7175,6 +7176,7 @@ static void intrinsicLatencyMode(void) { int main(int argc, char **argv) { int firstarg; + storage_init(); config.hostip = sdsnew("127.0.0.1"); config.hostport = 6379; config.hostsocket = NULL; diff --git a/src/replication.c b/src/replication.c index 3bc42d62a..103af2691 100644 --- a/src/replication.c +++ b/src/replication.c @@ -76,7 +76,7 @@ char *replicationGetSlaveName(client *c) { void createReplicationBacklog(void) { serverAssert(server.repl_backlog == NULL); - server.repl_backlog = zmalloc(server.repl_backlog_size); + server.repl_backlog = zmalloc(server.repl_backlog_size, MALLOC_LOCAL); server.repl_backlog_histlen = 0; server.repl_backlog_idx = 0; @@ -105,7 +105,7 @@ void resizeReplicationBacklog(long long newsize) { * worse often we need to alloc additional space before freeing the * old buffer. */ zfree(server.repl_backlog); - server.repl_backlog = zmalloc(server.repl_backlog_size); + server.repl_backlog = zmalloc(server.repl_backlog_size, MALLOC_LOCAL); server.repl_backlog_histlen = 0; server.repl_backlog_idx = 0; /* Next byte we have is... the next since the buffer is empty. */ diff --git a/src/rio.c b/src/rio.c index c9c76b8f2..0f44b017d 100644 --- a/src/rio.c +++ b/src/rio.c @@ -267,8 +267,8 @@ void rioInitWithFdset(rio *r, int *fds, int numfds) { int j; *r = rioFdsetIO; - r->io.fdset.fds = zmalloc(sizeof(int)*numfds); - r->io.fdset.state = zmalloc(sizeof(int)*numfds); + r->io.fdset.fds = zmalloc(sizeof(int)*numfds, MALLOC_LOCAL); + r->io.fdset.state = zmalloc(sizeof(int)*numfds, MALLOC_LOCAL); memcpy(r->io.fdset.fds,fds,sizeof(int)*numfds); for (j = 0; j < numfds; j++) r->io.fdset.state[j] = 0; r->io.fdset.numfds = numfds; diff --git a/src/scripting.c b/src/scripting.c index f6df38400..3b3b996e0 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1866,7 +1866,7 @@ sds *ldbReplParseCommand(int *argcp) { if (*argcp <= 0 || *argcp > 1024) goto protoerr; /* Parse each argument. */ - argv = zmalloc(sizeof(sds)*(*argcp)); + argv = zmalloc(sizeof(sds)*(*argcp), MALLOC_LOCAL); argc = 0; while(argc < *argcp) { if (*p != '$') goto protoerr; diff --git a/src/sds.c b/src/sds.c index 330c955e8..a87173d94 100644 --- a/src/sds.c +++ b/src/sds.c @@ -96,7 +96,7 @@ sds sdsnewlen(const void *init, size_t initlen) { int hdrlen = sdsHdrSize(type); unsigned char *fp; /* flags pointer. */ - sh = s_malloc(hdrlen+initlen+1); + sh = s_malloc(hdrlen+initlen+1, MALLOC_SHARED); if (init==SDS_NOINIT) init = NULL; else if (!init) @@ -234,7 +234,7 @@ sds sdsMakeRoomFor(sds s, size_t addlen) { } else { /* Since the header size changes, need to move the string forward, * and can't use realloc */ - newsh = s_malloc(hdrlen+newlen+1); + newsh = s_malloc(hdrlen+newlen+1, MALLOC_SHARED); if (newsh == NULL) return NULL; memcpy((char*)newsh+hdrlen, s, len+1); s_free(sh); @@ -273,7 +273,7 @@ sds sdsRemoveFreeSpace(sds s) { if (newsh == NULL) return NULL; s = (char*)newsh+oldhdrlen; } else { - newsh = s_malloc(hdrlen+len+1); + newsh = s_malloc(hdrlen+len+1, MALLOC_SHARED); if (newsh == NULL) return NULL; memcpy((char*)newsh+hdrlen, s, len+1); s_free(sh); @@ -523,7 +523,7 @@ sds sdscatvprintf(sds s, const char *fmt, va_list ap) { /* We try to start using a static buffer for speed. * If not possible we revert to heap allocation. */ if (buflen > sizeof(staticbuf)) { - buf = s_malloc(buflen); + buf = s_malloc(buflen, MALLOC_SHARED); if (buf == NULL) return NULL; } else { buflen = sizeof(staticbuf); @@ -539,7 +539,7 @@ sds sdscatvprintf(sds s, const char *fmt, va_list ap) { if (buf[buflen-2] != '\0') { if (buf != staticbuf) s_free(buf); buflen *= 2; - buf = s_malloc(buflen); + buf = s_malloc(buflen, MALLOC_SHARED); if (buf == NULL) return NULL; continue; } @@ -816,7 +816,7 @@ sds *sdssplitlen(const char *s, ssize_t len, const char *sep, int seplen, int *c if (seplen < 1 || len < 0) return NULL; - tokens = s_malloc(sizeof(sds)*slots); + tokens = s_malloc(sizeof(sds)*slots, MALLOC_SHARED); if (tokens == NULL) return NULL; if (len == 0) { @@ -1044,7 +1044,7 @@ sds *sdssplitargs(const char *line, int *argc) { current = NULL; } else { /* Even on empty input string return something not NULL. */ - if (vector == NULL) vector = s_malloc(sizeof(void*)); + if (vector == NULL) vector = s_malloc(sizeof(void*), MALLOC_SHARED); return vector; } } @@ -1111,7 +1111,7 @@ sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen) { * the overhead of function calls. Here we define these wrappers only for * the programs SDS is linked to, if they want to touch the SDS internals * even if they use a different allocator. */ -void *sds_malloc(size_t size) { return s_malloc(size); } +void *sds_malloc(size_t size) { return s_malloc(size, MALLOC_SHARED); } void *sds_realloc(void *ptr, size_t size) { return s_realloc(ptr,size); } void sds_free(void *ptr) { s_free(ptr); } diff --git a/src/sdsalloc.h b/src/sdsalloc.h index 531d41929..cb814acce 100644 --- a/src/sdsalloc.h +++ b/src/sdsalloc.h @@ -37,6 +37,10 @@ * to use the default libc allocator). */ #include "zmalloc.h" -#define s_malloc zmalloc -#define s_realloc zrealloc -#define s_free zfree +#include "storage.h" +//#define s_malloc zmalloc +#define s_malloc salloc +//#define s_realloc zrealloc +#define s_realloc srealloc +//#define s_free zfree +#define s_free sfree diff --git a/src/sentinel.c b/src/sentinel.c index 4d03c9c12..7add1e744 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -341,7 +341,7 @@ static int redisAeAttach(aeEventLoop *loop, redisAsyncContext *ac) { return C_ERR; /* Create container for context and r/w events */ - e = (redisAeEvents*)zmalloc(sizeof(*e)); + e = (redisAeEvents*)zmalloc(sizeof(*e), MALLOC_LOCAL); e->context = ac; e->loop = loop; e->fd = c->fd; @@ -549,7 +549,7 @@ sentinelAddr *createSentinelAddr(char *hostname, int port) { errno = ENOENT; return NULL; } - sa = zmalloc(sizeof(*sa)); + sa = zmalloc(sizeof(*sa), MALLOC_LOCAL); sa->ip = sdsnew(ip); sa->port = port; return sa; @@ -559,7 +559,7 @@ sentinelAddr *createSentinelAddr(char *hostname, int port) { sentinelAddr *dupSentinelAddr(sentinelAddr *src) { sentinelAddr *sa; - sa = zmalloc(sizeof(*sa)); + sa = zmalloc(sizeof(*sa), MALLOC_LOCAL); sa->ip = sdsnew(src->ip); sa->port = src->port; return sa; @@ -703,10 +703,10 @@ void sentinelScheduleScriptExecution(char *path, ...) { va_end(ap); argv[0] = sdsnew(path); - sj = zmalloc(sizeof(*sj)); + sj = zmalloc(sizeof(*sj), MALLOC_LOCAL); sj->flags = SENTINEL_SCRIPT_NONE; sj->retry_num = 0; - sj->argv = zmalloc(sizeof(char*)*(argc+1)); + sj->argv = zmalloc(sizeof(char*)*(argc+1), MALLOC_LOCAL); sj->start_time = 0; sj->pid = 0; memcpy(sj->argv,argv,sizeof(char*)*(argc+1)); @@ -950,7 +950,7 @@ void sentinelCallClientReconfScript(sentinelRedisInstance *master, int role, cha /* Create a not yet connected link object. */ instanceLink *createInstanceLink(void) { - instanceLink *link = zmalloc(sizeof(*link)); + instanceLink *link = zmalloc(sizeof(*link), MALLOC_LOCAL); link->refcount = 1; link->disconnected = 1; @@ -1200,7 +1200,7 @@ sentinelRedisInstance *createSentinelRedisInstance(char *name, int flags, char * } /* Create the instance object. */ - ri = zmalloc(sizeof(*ri)); + ri = zmalloc(sizeof(*ri), MALLOC_LOCAL); /* Note that all the instances are started in the disconnected state, * the event loop will take care of connecting them. */ ri->flags = flags; @@ -4091,7 +4091,7 @@ int compareSlavesForPromotion(const void *a, const void *b) { sentinelRedisInstance *sentinelSelectSlave(sentinelRedisInstance *master) { sentinelRedisInstance **instance = - zmalloc(sizeof(instance[0])*dictSize(master->slaves)); + zmalloc(sizeof(instance[0])*dictSize(master->slaves), MALLOC_LOCAL); sentinelRedisInstance *selected = NULL; int instances = 0; dictIterator *di; diff --git a/src/server.c b/src/server.c index 1df63f65f..62b37dfa1 100644 --- a/src/server.c +++ b/src/server.c @@ -33,6 +33,7 @@ #include "bio.h" #include "latency.h" #include "atomicvar.h" +#include "storage.h" #include #include @@ -2729,7 +2730,7 @@ void initServer(void) { strerror(errno)); exit(1); } - server.db = zmalloc(sizeof(redisDb)*server.dbnum); + server.db = zmalloc(sizeof(redisDb)*server.dbnum, MALLOC_LOCAL); /* Open the TCP listening socket for the user commands. */ if (server.port != 0 && @@ -3066,7 +3067,7 @@ void alsoPropagate(struct redisCommand *cmd, int dbid, robj **argv, int argc, if (server.loading) return; /* No propagation during loading. */ - argvcopy = zmalloc(sizeof(robj*)*argc); + argvcopy = zmalloc(sizeof(robj*)*argc, MALLOC_LOCAL); for (j = 0; j < argc; j++) { argvcopy[j] = argv[j]; incrRefCount(argv[j]); @@ -4481,7 +4482,7 @@ void usage(void) { void redisAsciiArt(void) { #include "asciilogo.h" - char *buf = zmalloc(1024*16); + char *buf = zmalloc(1024*16, MALLOC_LOCAL); char *mode; if (server.cluster_enabled) mode = "cluster"; @@ -4739,6 +4740,8 @@ int main(int argc, char **argv) { struct timeval tv; int j; + storage_init(); + #ifdef REDIS_TEST if (argc == 3 && !strcasecmp(argv[1], "test")) { if (!strcasecmp(argv[2], "ziplist")) { @@ -4789,7 +4792,7 @@ int main(int argc, char **argv) { /* Store the executable path and arguments in a safe place in order * to be able to restart the server later. */ server.executable = getAbsolutePath(argv[0]); - server.exec_argv = zmalloc(sizeof(char*)*(argc+1)); + server.exec_argv = zmalloc(sizeof(char*)*(argc+1), MALLOC_LOCAL); server.exec_argv[argc] = NULL; for (j = 0; j < argc; j++) server.exec_argv[j] = zstrdup(argv[j]); diff --git a/src/slowlog.c b/src/slowlog.c index 1d715e39b..b6041f096 100644 --- a/src/slowlog.c +++ b/src/slowlog.c @@ -46,12 +46,12 @@ * Incrementing the ref count of all the objects retained is up to * this function. */ slowlogEntry *slowlogCreateEntry(client *c, robj **argv, int argc, long long duration) { - slowlogEntry *se = zmalloc(sizeof(*se)); + slowlogEntry *se = zmalloc(sizeof(*se), MALLOC_LOCAL); int j, slargc = argc; if (slargc > SLOWLOG_ENTRY_MAX_ARGC) slargc = SLOWLOG_ENTRY_MAX_ARGC; se->argc = slargc; - se->argv = zmalloc(sizeof(robj*)*slargc); + se->argv = zmalloc(sizeof(robj*)*slargc, MALLOC_LOCAL); for (j = 0; j < slargc; j++) { /* Logging too many arguments is a useless memory waste, so we stop * at SLOWLOG_ENTRY_MAX_ARGC, but use the last argument to specify diff --git a/src/sort.c b/src/sort.c index 8608cd8b3..7ae794501 100644 --- a/src/sort.c +++ b/src/sort.c @@ -36,7 +36,7 @@ zskiplistNode* zslGetElementByRank(zskiplist *zsl, unsigned long rank); redisSortOperation *createSortOperation(int type, robj *pattern) { - redisSortOperation *so = zmalloc(sizeof(*so)); + redisSortOperation *so = zmalloc(sizeof(*so), MALLOC_LOCAL); so->type = type; so->pattern = pattern; return so; @@ -340,7 +340,7 @@ void sortCommand(client *c) { } /* Load the sorting vector with all the objects to sort */ - vector = zmalloc(sizeof(redisSortObject)*vectorlen); + vector = zmalloc(sizeof(redisSortObject)*vectorlen, MALLOC_LOCAL); j = 0; if (sortval->type == OBJ_LIST && dontsort) { diff --git a/src/sparkline.c b/src/sparkline.c index 0a986883d..e18ce98fd 100644 --- a/src/sparkline.c +++ b/src/sparkline.c @@ -55,7 +55,7 @@ static int label_margin_top = 1; /* Create a new sequence. */ struct sequence *createSparklineSequence(void) { - struct sequence *seq = zmalloc(sizeof(*seq)); + struct sequence *seq = zmalloc(sizeof(*seq), MALLOC_LOCAL); seq->length = 0; seq->samples = NULL; return seq; @@ -99,7 +99,7 @@ sds sparklineRenderRange(sds output, struct sequence *seq, int rows, int offset, double relmax = seq->max - seq->min; int steps = charset_len*rows; int row = 0; - char *chars = zmalloc(len); + char *chars = zmalloc(len, MALLOC_LOCAL); int loop = 1; int opt_fill = flags & SPARKLINE_FILL; int opt_log = flags & SPARKLINE_LOG_SCALE; diff --git a/src/storage.c b/src/storage.c new file mode 100644 index 000000000..90009c2a8 --- /dev/null +++ b/src/storage.c @@ -0,0 +1,155 @@ +#include "server.h" + +#include +#include +#include +#include +#include +#include +#include +#include "storage.h" + +struct memkind *mkdisk = NULL; +static char *PMEM_DIR = "/mnt/btrfs_scratch/"; + + + +#define OBJECT_PAGE_BUFFER_SIZE 8192 //(size in objs) +struct object_page +{ + uint64_t allocmap[OBJECT_PAGE_BUFFER_SIZE/(8*sizeof(uint64_t))]; + robj rgobj[OBJECT_PAGE_BUFFER_SIZE]; + struct object_page *pnext; +}; +#define OBJ_PAGE_BITS_PER_WORD 64 +struct object_page *headObjpage = NULL; + +void storage_init() +{ + int errv = memkind_create_pmem(PMEM_DIR, 0, &mkdisk); + if (errv) + { + fprintf(stderr, "Memory pool creation failed: %d\n", errv); + exit(EXIT_FAILURE); + } + headObjpage = memkind_calloc(MEMKIND_HUGETLB, 1, sizeof(struct object_page)); +} + +int IdxAllocObject(struct object_page *page) +{ + for (size_t iword = 0; iword < OBJ_PAGE_BITS_PER_WORD; ++iword) + { + if ((page->allocmap[iword] + 1) != 0) + { + int ibit = 0; + uint64_t bitword = page->allocmap[iword]; + while (bitword & 1) + { + bitword >>= 1; + ++ibit; + } + page->allocmap[iword] |= 1 << ibit; + return (iword * OBJ_PAGE_BITS_PER_WORD) + ibit; + } + } + return -1; +} + +struct redisObject *salloc_obj() +{ + struct object_page *cur = headObjpage; + for (;;) + { + int idx = IdxAllocObject(cur); + if (idx >= 0) + { + return &cur->rgobj[idx]; + } + + if (cur->pnext == NULL) + { + cur->pnext = memkind_calloc(MEMKIND_HUGETLB, 1, sizeof(struct object_page)); + } + + cur = cur->pnext; + } +} +void sfree_obj(struct redisObject *obj) +{ + struct object_page *cur = headObjpage; + for (;;) + { + if (obj >= cur->rgobj && (obj < (cur->rgobj + OBJECT_PAGE_BUFFER_SIZE))) + { + // Its on this page + int idx = obj - cur->rgobj; + cur->allocmap[idx / OBJ_PAGE_BITS_PER_WORD] &= ~(1 << (idx % OBJ_PAGE_BITS_PER_WORD)); + break; + } + cur = cur->pnext; + } + return; +} + +void *salloc(size_t cb, enum MALLOC_CLASS class) +{ + switch (class) + { + case MALLOC_SHARED: + return memkind_malloc(mkdisk, cb); + default: + return memkind_malloc(MEMKIND_DEFAULT, cb); + } + return NULL; +} + +void *scalloc(size_t cb, size_t c, enum MALLOC_CLASS class) +{ + switch (class) + { + case MALLOC_SHARED: + return memkind_calloc(mkdisk, cb, c); + default: + return memkind_calloc(MEMKIND_DEFAULT, cb, c); + } + return NULL; +} + +void sfree(void *pv) +{ + memkind_free(NULL, pv); +} + +void *srealloc(void *pv, size_t cb) +{ + memkind_t kind = mkdisk; + return memkind_realloc(kind, pv, cb); +} + +int fdNew = -1; +void handle_prefork() +{ + memkind_tmpfile(PMEM_DIR, &fdNew); + if (ioctl(fdNew, FICLONE, memkind_fd(mkdisk)) == -1) + { + perror("failed to fork file"); + exit(EXIT_FAILURE); + } +} + +void handle_postfork(int pid) +{ + if (pid != 0) + { + // Parent, close fdNew + close(fdNew); + fdNew = -1; + } + else + { + int fdOriginal = memkind_fd(mkdisk); + memkind_pmem_remapfd(mkdisk, fdNew); + close(fdOriginal); + } + +} \ No newline at end of file diff --git a/src/storage.h b/src/storage.h new file mode 100644 index 000000000..ebb2d1c5e --- /dev/null +++ b/src/storage.h @@ -0,0 +1,23 @@ +#ifndef __STORAGE_H__ +#define __STORAGE_H__ + +enum MALLOC_CLASS +{ + MALLOC_LOCAL, + MALLOC_SHARED, +}; + +void storage_init(void); + +struct redisObject *salloc_obj(); +void sfree_obj(struct redisObject *obj); + +void *salloc(size_t cb, enum MALLOC_CLASS class); +void *scalloc(size_t cb, size_t c, enum MALLOC_CLASS class); +void sfree(void*); +void *srealloc(void *pv, size_t cb); + +void handle_prefork(); +void handle_postfork(int pid); + +#endif diff --git a/src/t_hash.c b/src/t_hash.c index d8aee6572..c94945a1f 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -322,7 +322,7 @@ unsigned long hashTypeLength(const robj *o) { } hashTypeIterator *hashTypeInitIterator(robj *subject) { - hashTypeIterator *hi = zmalloc(sizeof(hashTypeIterator)); + hashTypeIterator *hi = zmalloc(sizeof(hashTypeIterator), MALLOC_LOCAL); hi->subject = subject; hi->encoding = subject->encoding; diff --git a/src/t_list.c b/src/t_list.c index 451ffb4b5..143f672a4 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -82,7 +82,7 @@ unsigned long listTypeLength(const robj *subject) { /* Initialize an iterator at the specified index. */ listTypeIterator *listTypeInitIterator(robj *subject, long index, unsigned char direction) { - listTypeIterator *li = zmalloc(sizeof(listTypeIterator)); + listTypeIterator *li = zmalloc(sizeof(listTypeIterator), MALLOC_LOCAL); li->subject = subject; li->encoding = subject->encoding; li->direction = direction; diff --git a/src/t_set.c b/src/t_set.c index 61013dbcd..89b5cf70a 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -119,7 +119,7 @@ int setTypeIsMember(robj *subject, sds value) { } setTypeIterator *setTypeInitIterator(robj *subject) { - setTypeIterator *si = zmalloc(sizeof(setTypeIterator)); + setTypeIterator *si = zmalloc(sizeof(setTypeIterator), MALLOC_LOCAL); si->subject = subject; si->encoding = subject->encoding; if (si->encoding == OBJ_ENCODING_HT) { @@ -791,7 +791,7 @@ int qsortCompareSetsByRevCardinality(const void *s1, const void *s2) { void sinterGenericCommand(client *c, robj **setkeys, unsigned long setnum, robj *dstkey) { - robj **sets = zmalloc(sizeof(robj*)*setnum); + robj **sets = zmalloc(sizeof(robj*)*setnum, MALLOC_SHARED); setTypeIterator *si; robj *dstset = NULL; sds elesds; @@ -930,7 +930,7 @@ void sinterstoreCommand(client *c) { void sunionDiffGenericCommand(client *c, robj **setkeys, int setnum, robj *dstkey, int op) { - robj **sets = zmalloc(sizeof(robj*)*setnum); + robj **sets = zmalloc(sizeof(robj*)*setnum, MALLOC_SHARED); setTypeIterator *si; robj *dstset = NULL; sds ele; diff --git a/src/t_stream.c b/src/t_stream.c index 1a5acac42..5fb14628b 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -50,7 +50,7 @@ size_t streamReplyWithRangeFromConsumerPEL(client *c, stream *s, streamID *start /* Create a new stream data structure. */ stream *streamNew(void) { - stream *s = zmalloc(sizeof(*s)); + stream *s = zmalloc(sizeof(*s), MALLOC_SHARED); s->rax = raxNew(); s->length = 0; s->last_id.ms = 0; @@ -1402,8 +1402,8 @@ void xreadCommand(client *c) { /* Parse the IDs and resolve the group name. */ if (streams_count > STREAMID_STATIC_VECTOR_LEN) - ids = zmalloc(sizeof(streamID)*streams_count); - if (groupname) groups = zmalloc(sizeof(streamCG*)*streams_count); + ids = zmalloc(sizeof(streamID)*streams_count, MALLOC_SHARED); + if (groupname) groups = zmalloc(sizeof(streamCG*)*streams_count, MALLOC_SHARED); for (int i = streams_arg + streams_count; i < c->argc; i++) { /* Specifying "$" as last-known-id means that the client wants to be @@ -1595,7 +1595,7 @@ cleanup: /* Cleanup. */ * time to the current time. The NACK consumer will be set to the one * specified as argument of the function. */ streamNACK *streamCreateNACK(streamConsumer *consumer) { - streamNACK *nack = zmalloc(sizeof(*nack)); + streamNACK *nack = zmalloc(sizeof(*nack), MALLOC_SHARED); nack->delivery_time = mstime(); nack->delivery_count = 1; nack->consumer = consumer; @@ -1628,7 +1628,7 @@ streamCG *streamCreateCG(stream *s, char *name, size_t namelen, streamID *id) { if (raxFind(s->cgroups,(unsigned char*)name,namelen) != raxNotFound) return NULL; - streamCG *cg = zmalloc(sizeof(*cg)); + streamCG *cg = zmalloc(sizeof(*cg), MALLOC_SHARED); cg->pel = raxNew(); cg->consumers = raxNew(); cg->last_id = *id; @@ -1661,7 +1661,7 @@ streamConsumer *streamLookupConsumer(streamCG *cg, sds name, int create) { sdslen(name)); if (consumer == raxNotFound) { if (!create) return NULL; - consumer = zmalloc(sizeof(*consumer)); + consumer = zmalloc(sizeof(*consumer), MALLOC_SHARED); consumer->name = sdsdup(name); consumer->pel = raxNew(); raxInsert(cg->consumers,(unsigned char*)name,sdslen(name), diff --git a/src/t_zset.c b/src/t_zset.c index 0427ee887..daeabcf93 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -70,7 +70,7 @@ int zslLexValueLteMax(sds value, zlexrangespec *spec); * The SDS string 'ele' is referenced by the node after the call. */ zskiplistNode *zslCreateNode(int level, double score, sds ele) { zskiplistNode *zn = - zmalloc(sizeof(*zn)+level*sizeof(struct zskiplistLevel)); + zmalloc(sizeof(*zn)+level*sizeof(struct zskiplistLevel), MALLOC_SHARED); zn->score = score; zn->ele = ele; return zn; @@ -81,7 +81,7 @@ zskiplist *zslCreate(void) { int j; zskiplist *zsl; - zsl = zmalloc(sizeof(*zsl)); + zsl = zmalloc(sizeof(*zsl), MALLOC_SHARED); zsl->level = 1; zsl->length = 0; zsl->header = zslCreateNode(ZSKIPLIST_MAXLEVEL,0,NULL); @@ -1182,7 +1182,7 @@ void zsetConvert(robj *zobj, int encoding) { if (encoding != OBJ_ENCODING_SKIPLIST) serverPanic("Unknown target encoding"); - zs = zmalloc(sizeof(*zs)); + zs = zmalloc(sizeof(*zs), MALLOC_SHARED); zs->dict = dictCreate(&zsetDictType,NULL); zs->zsl = zslCreate(); @@ -1590,7 +1590,7 @@ void zaddGenericCommand(client *c, int flags) { /* Start parsing all the scores, we need to emit any syntax error * before executing additions to the sorted set, as the command should * either execute fully or nothing at all. */ - scores = zmalloc(sizeof(double)*elements); + scores = zmalloc(sizeof(double)*elements, MALLOC_SHARED); for (j = 0; j < elements; j++) { if (getDoubleFromObjectOrReply(c,c->argv[scoreidx+j*2],&scores[j],NULL) != C_OK) goto cleanup; @@ -2203,7 +2203,7 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int op) { } /* read keys to be used for input */ - src = zcalloc(sizeof(zsetopsrc) * setnum); + src = zcalloc(sizeof(zsetopsrc) * setnum, MALLOC_SHARED); for (i = 0, j = 3; i < setnum; i++, j++) { robj *obj = lookupKeyWrite(c->db,c->argv[j]); if (obj != NULL) { diff --git a/src/ziplist.c b/src/ziplist.c index 1579d1109..733e2e685 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -577,7 +577,7 @@ void zipEntry(unsigned char *p, zlentry *e) { /* Create a new empty ziplist. */ unsigned char *ziplistNew(void) { unsigned int bytes = ZIPLIST_HEADER_SIZE+1; - unsigned char *zl = zmalloc(bytes); + unsigned char *zl = zmalloc(bytes, MALLOC_SHARED); ZIPLIST_BYTES(zl) = intrev32ifbe(bytes); ZIPLIST_TAIL_OFFSET(zl) = intrev32ifbe(ZIPLIST_HEADER_SIZE); ZIPLIST_LENGTH(zl) = 0; diff --git a/src/zipmap.c b/src/zipmap.c index 22bfa1a46..88e7f8706 100644 --- a/src/zipmap.c +++ b/src/zipmap.c @@ -94,7 +94,7 @@ /* Create a new empty zipmap. */ unsigned char *zipmapNew(void) { - unsigned char *zm = zmalloc(2); + unsigned char *zm = zmalloc(2, MALLOC_SHARED); zm[0] = 0; /* Length */ zm[1] = ZIPMAP_END; diff --git a/src/zmalloc.c b/src/zmalloc.c index 2482f512b..7d307bad7 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -70,6 +70,16 @@ void zlibc_free(void *ptr) { #define mallocx(size,flags) je_mallocx(size,flags) #define dallocx(ptr,flags) je_dallocx(ptr,flags) #endif +#include "storage.h" +#undef malloc +#undef calloc +#undef realloc +#undef free +#define malloc(size, type) salloc(size, type) +#define calloc(count, size, type) scalloc(count, size, type) +#define realloc(ptr, size) srealloc(ptr, size) +#define free(ptr) sfree(ptr) +//#define zmalloc_size(ptr) (sizeof(ptr)) #define update_zmalloc_stat_alloc(__n) do { \ size_t _n = (__n); \ @@ -95,8 +105,8 @@ static void zmalloc_default_oom(size_t size) { static void (*zmalloc_oom_handler)(size_t) = zmalloc_default_oom; -void *zmalloc(size_t size) { - void *ptr = malloc(size+PREFIX_SIZE); +void *zmalloc(size_t size, enum MALLOC_CLASS class) { + void *ptr = malloc(size+PREFIX_SIZE, class); if (!ptr) zmalloc_oom_handler(size); #ifdef HAVE_MALLOC_SIZE @@ -127,8 +137,8 @@ void zfree_no_tcache(void *ptr) { } #endif -void *zcalloc(size_t size) { - void *ptr = calloc(1, size+PREFIX_SIZE); +void *zcalloc(size_t size, enum MALLOC_CLASS class) { + void *ptr = calloc(1, size+PREFIX_SIZE, class); if (!ptr) zmalloc_oom_handler(size); #ifdef HAVE_MALLOC_SIZE @@ -148,7 +158,7 @@ void *zrealloc(void *ptr, size_t size) { size_t oldsize; void *newptr; - if (ptr == NULL) return zmalloc(size); + if (ptr == NULL) return zmalloc(size, MALLOC_SHARED); #ifdef HAVE_MALLOC_SIZE oldsize = zmalloc_size(ptr); newptr = realloc(ptr,size); @@ -207,7 +217,7 @@ void zfree(void *ptr) { char *zstrdup(const char *s) { size_t l = strlen(s)+1; - char *p = zmalloc(l); + char *p = zmalloc(l, MALLOC_SHARED); memcpy(p,s,l); return p; diff --git a/src/zmalloc.h b/src/zmalloc.h index 6fb19b046..0692e64c2 100644 --- a/src/zmalloc.h +++ b/src/zmalloc.h @@ -35,7 +35,12 @@ #define __xstr(s) __str(s) #define __str(s) #s -#if defined(USE_TCMALLOC) +#if 1 + #define ZMALLOC_LIB ("custom") + #include "storage.h" + #undef USE_JEMALLOC + #define USE_MALLOC_CLASS 1 +#elif defined(USE_TCMALLOC) #define ZMALLOC_LIB ("tcmalloc-" __xstr(TC_VERSION_MAJOR) "." __xstr(TC_VERSION_MINOR)) #include #if (TC_VERSION_MAJOR == 1 && TC_VERSION_MINOR >= 6) || (TC_VERSION_MAJOR > 1) @@ -77,8 +82,8 @@ #define HAVE_DEFRAG #endif -void *zmalloc(size_t size); -void *zcalloc(size_t size); +void *zmalloc(size_t size, enum MALLOC_CLASS class); +void *zcalloc(size_t size, enum MALLOC_CLASS class); void *zrealloc(void *ptr, size_t size); void zfree(void *ptr); char *zstrdup(const char *s); diff --git a/utils/hashtable/rehashing.c b/utils/hashtable/rehashing.c index b57a9043a..ee4d10c2d 100644 --- a/utils/hashtable/rehashing.c +++ b/utils/hashtable/rehashing.c @@ -71,7 +71,7 @@ int sortPointers(const void *a, const void *b) { void stressGetKeys(dict *d, int times, int *perfect_run, int *approx_run) { int j; - dictEntry **des = zmalloc(sizeof(dictEntry*)*dictSize(d)); + dictEntry **des = zmalloc(sizeof(dictEntry*)*dictSize(d), MALLOC_SHARED); for (j = 0; j < times; j++) { int requested = rand() % (dictSize(d)+1); int returned = dictGetSomeKeys(d, des, requested); From eeca73efc50cf1f219e637117e1cd0cf19a52485 Mon Sep 17 00:00:00 2001 From: John Sully Date: Wed, 30 Jan 2019 01:15:10 -0500 Subject: [PATCH 003/101] crash fixes --- src/rdb.c | 3 -- src/sdsalloc.h | 9 ++-- src/storage.c | 134 ++++++++++++++++++++++++++++++++++--------------- src/storage.h | 7 +-- 4 files changed, 100 insertions(+), 53 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index 32a3df1d5..0374a3999 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1291,12 +1291,10 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) { openChildInfoPipe(); start = ustime(); - handle_prefork(); if ((childpid = fork()) == 0) { int retval; /* Child */ - handle_postfork(0); closeListeningSockets(0); redisSetProcTitle("redis-rdb-bgsave"); retval = rdbSave(filename,rsi); @@ -1315,7 +1313,6 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) { exitFromChild((retval == C_OK) ? 0 : 1); } else { /* Parent */ - handle_postfork(childpid); server.stat_fork_time = ustime()-start; server.stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / server.stat_fork_time / (1024*1024*1024); /* GB per second. */ latencyAddSampleIfNeeded("fork",server.stat_fork_time/1000); diff --git a/src/sdsalloc.h b/src/sdsalloc.h index cb814acce..207e83adb 100644 --- a/src/sdsalloc.h +++ b/src/sdsalloc.h @@ -38,9 +38,6 @@ #include "zmalloc.h" #include "storage.h" -//#define s_malloc zmalloc -#define s_malloc salloc -//#define s_realloc zrealloc -#define s_realloc srealloc -//#define s_free zfree -#define s_free sfree +#define s_malloc zmalloc +#define s_realloc zrealloc +#define s_free zfree diff --git a/src/storage.c b/src/storage.c index 90009c2a8..759c99840 100644 --- a/src/storage.c +++ b/src/storage.c @@ -12,30 +12,41 @@ struct memkind *mkdisk = NULL; static char *PMEM_DIR = "/mnt/btrfs_scratch/"; - +void handle_prefork(); +void handle_postfork_parent(); +void handle_postfork_child(); #define OBJECT_PAGE_BUFFER_SIZE 8192 //(size in objs) +#define OBJ_PAGE_BITS_PER_WORD 64 struct object_page { uint64_t allocmap[OBJECT_PAGE_BUFFER_SIZE/(8*sizeof(uint64_t))]; - robj rgobj[OBJECT_PAGE_BUFFER_SIZE]; struct object_page *pnext; + char rgb[]; }; -#define OBJ_PAGE_BITS_PER_WORD 64 -struct object_page *headObjpage = NULL; -void storage_init() +struct alloc_pool { - int errv = memkind_create_pmem(PMEM_DIR, 0, &mkdisk); - if (errv) - { - fprintf(stderr, "Memory pool creation failed: %d\n", errv); - exit(EXIT_FAILURE); - } - headObjpage = memkind_calloc(MEMKIND_HUGETLB, 1, sizeof(struct object_page)); -} + unsigned cbObject; + struct object_page *pobjpageHead; +}; -int IdxAllocObject(struct object_page *page) + +struct object_page *pool_allocate_page(int cbObject) +{ + size_t cb = (((size_t)cbObject) * OBJECT_PAGE_BUFFER_SIZE) + sizeof(struct object_page); + return scalloc(cb, 1, MALLOC_SHARED); +} +void pool_initialize(struct alloc_pool *ppool, int cbObject) +{ + if ((cbObject % 8) != 0) + { + cbObject += 8 - (cbObject % 8); + } + ppool->cbObject = cbObject; + ppool->pobjpageHead = pool_allocate_page(cbObject); +} +static int IdxAllocObject(struct object_page *page) { for (size_t iword = 0; iword < OBJ_PAGE_BITS_PER_WORD; ++iword) { @@ -48,49 +59,93 @@ int IdxAllocObject(struct object_page *page) bitword >>= 1; ++ibit; } - page->allocmap[iword] |= 1 << ibit; + page->allocmap[iword] |= 1ULL << ibit; return (iword * OBJ_PAGE_BITS_PER_WORD) + ibit; } } return -1; } - -struct redisObject *salloc_obj() +void *pool_alloc(struct alloc_pool *ppool) { - struct object_page *cur = headObjpage; + struct object_page *cur = ppool->pobjpageHead; for (;;) { int idx = IdxAllocObject(cur); if (idx >= 0) { - return &cur->rgobj[idx]; + return cur->rgb + (((size_t)ppool->cbObject) * idx); } if (cur->pnext == NULL) { - cur->pnext = memkind_calloc(MEMKIND_HUGETLB, 1, sizeof(struct object_page)); + cur->pnext = pool_allocate_page(ppool->cbObject); } cur = cur->pnext; } } -void sfree_obj(struct redisObject *obj) + +#pragma weak serverLog +void serverLog(int level, const char*fmt, ...){} + +void pool_free(struct alloc_pool *ppool, void *pv) { - struct object_page *cur = headObjpage; - for (;;) + struct object_page *cur = ppool->pobjpageHead; + char *obj = pv; + + for (;cur != NULL;) { - if (obj >= cur->rgobj && (obj < (cur->rgobj + OBJECT_PAGE_BUFFER_SIZE))) + if (obj >= cur->rgb && (obj < (cur->rgb + (OBJECT_PAGE_BUFFER_SIZE * ppool->cbObject)))) { // Its on this page - int idx = obj - cur->rgobj; - cur->allocmap[idx / OBJ_PAGE_BITS_PER_WORD] &= ~(1 << (idx % OBJ_PAGE_BITS_PER_WORD)); - break; + int idx = (obj - cur->rgb) / ppool->cbObject; + cur->allocmap[idx / OBJ_PAGE_BITS_PER_WORD] &= ~(1ULL << (idx % OBJ_PAGE_BITS_PER_WORD)); + return; } cur = cur->pnext; } + serverLog(LOG_CRIT, "obj not from pool"); + sfree(obj); // we don't know where it came from return; } +#define EMBSTR_ROBJ_SIZE (sizeof(robj)+sizeof(struct sdshdr8)+OBJ_ENCODING_EMBSTR_SIZE_LIMIT+1) +struct alloc_pool poolobj; +struct alloc_pool poolembstrobj; + +void storage_init() +{ + int errv = memkind_create_pmem(PMEM_DIR, 0, &mkdisk); + if (errv) + { + fprintf(stderr, "Memory pool creation failed: %d\n", errv); + exit(EXIT_FAILURE); + } + pool_initialize(&poolobj, sizeof(robj)); + pool_initialize(&poolembstrobj, EMBSTR_ROBJ_SIZE); + + pthread_atfork(handle_prefork, handle_postfork_parent, handle_postfork_child); +} + + + +struct redisObject *salloc_obj() +{ + return pool_alloc(&poolobj); +} +void sfree_obj(struct redisObject *obj) +{ + pool_free(&poolobj, obj); +} +struct redisObject *salloc_objembstr() +{ + return pool_alloc(&poolembstrobj); +} +void sfree_objembstr(robj *obj) +{ + pool_free(&poolembstrobj, obj); +} + void *salloc(size_t cb, enum MALLOC_CLASS class) { switch (class) @@ -137,19 +192,16 @@ void handle_prefork() } } -void handle_postfork(int pid) +void handle_postfork_parent() { - if (pid != 0) - { - // Parent, close fdNew - close(fdNew); - fdNew = -1; - } - else - { - int fdOriginal = memkind_fd(mkdisk); - memkind_pmem_remapfd(mkdisk, fdNew); - close(fdOriginal); - } - + // Parent, close fdNew + close(fdNew); + fdNew = -1; +} + +void handle_postfork_child() +{ + int fdOriginal = memkind_fd(mkdisk); + memkind_pmem_remapfd(mkdisk, fdNew); + close(fdOriginal); } \ No newline at end of file diff --git a/src/storage.h b/src/storage.h index ebb2d1c5e..882113cdc 100644 --- a/src/storage.h +++ b/src/storage.h @@ -1,6 +1,8 @@ #ifndef __STORAGE_H__ #define __STORAGE_H__ +#define OBJ_ENCODING_EMBSTR_SIZE_LIMIT 44 // Note: also defined in object.c - should always match + enum MALLOC_CLASS { MALLOC_LOCAL, @@ -11,13 +13,12 @@ void storage_init(void); struct redisObject *salloc_obj(); void sfree_obj(struct redisObject *obj); +struct redisObject *salloc_objembstr(); +void sfree_objembstr(struct redisObject *obj); void *salloc(size_t cb, enum MALLOC_CLASS class); void *scalloc(size_t cb, size_t c, enum MALLOC_CLASS class); void sfree(void*); void *srealloc(void *pv, size_t cb); -void handle_prefork(); -void handle_postfork(int pid); - #endif From 4a84628bf0676e134a952205442253c7f8e1b99a Mon Sep 17 00:00:00 2001 From: John Sully Date: Fri, 1 Feb 2019 13:54:59 -0500 Subject: [PATCH 004/101] Support configurable file backed scratch --- src/Makefile | 4 +-- src/config.c | 2 ++ src/object.c | 7 ++-- src/redis-benchmark.c | 2 +- src/redis-cli.c | 2 +- src/server.c | 2 +- src/storage-lite.c | 41 ++++++++++++++++++++++++ src/storage.c | 74 ++++++++++++++++++++++++++++++++----------- src/storage.h | 7 +--- 9 files changed, 107 insertions(+), 34 deletions(-) create mode 100644 src/storage-lite.c diff --git a/src/Makefile b/src/Makefile index 680e3866b..9820ffcc8 100644 --- a/src/Makefile +++ b/src/Makefile @@ -166,9 +166,9 @@ REDIS_SERVER_NAME=redis-server REDIS_SENTINEL_NAME=redis-sentinel REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o acl.o storage.o REDIS_CLI_NAME=redis-cli -REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o siphash.o crc16.o storage.o +REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o siphash.o crc16.o storage-lite.o REDIS_BENCHMARK_NAME=redis-benchmark -REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o zmalloc.o redis-benchmark.o storage.o +REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o zmalloc.o redis-benchmark.o storage-lite.o REDIS_CHECK_RDB_NAME=redis-check-rdb REDIS_CHECK_AOF_NAME=redis-check-aof diff --git a/src/config.c b/src/config.c index be1c45235..5f5fd6792 100644 --- a/src/config.c +++ b/src/config.c @@ -804,6 +804,8 @@ void loadServerConfigFromString(char *config) { err = sentinelHandleConfiguration(argv+1,argc-1); if (err) goto loaderr; } + } else if (!strcasecmp(argv[0],"scratch-file-path")) { + storage_init(argv[1]); } else { err = "Bad directive or wrong number of arguments"; goto loaderr; } diff --git a/src/object.c b/src/object.c index ddd50b553..a5da2b142 100644 --- a/src/object.c +++ b/src/object.c @@ -39,7 +39,7 @@ /* ===================== Creation and parsing of objects ==================== */ robj *createObject(int type, void *ptr) { - robj *o = salloc_obj(); //zmalloc(sizeof(*o), MALLOC_SHARED); + robj *o = zmalloc(sizeof(*o), MALLOC_SHARED); o->type = type; o->encoding = OBJ_ENCODING_RAW; o->ptr = ptr; @@ -362,10 +362,7 @@ void decrRefCount(robj *o) { case OBJ_STREAM: freeStreamObject(o); break; default: serverPanic("Unknown object type"); break; } - if (o->type == OBJ_STRING && o->encoding == OBJ_ENCODING_EMBSTR) - zfree(o); - else - sfree_obj(o); + zfree(o); } else { if (o->refcount <= 0) serverPanic("decrRefCount against refcount <= 0"); if (o->refcount != OBJ_SHARED_REFCOUNT) o->refcount--; diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c index 716c492df..5dffd46b6 100644 --- a/src/redis-benchmark.c +++ b/src/redis-benchmark.c @@ -653,7 +653,7 @@ int main(int argc, const char **argv) { client c; - storage_init(); + storage_init(NULL); srandom(time(NULL)); signal(SIGHUP, SIG_IGN); diff --git a/src/redis-cli.c b/src/redis-cli.c index 91deac496..abcfd2b78 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -7176,7 +7176,7 @@ static void intrinsicLatencyMode(void) { int main(int argc, char **argv) { int firstarg; - storage_init(); + storage_init(NULL); config.hostip = sdsnew("127.0.0.1"); config.hostport = 6379; config.hostsocket = NULL; diff --git a/src/server.c b/src/server.c index 62b37dfa1..1444100b7 100644 --- a/src/server.c +++ b/src/server.c @@ -4740,7 +4740,7 @@ int main(int argc, char **argv) { struct timeval tv; int j; - storage_init(); + storage_init(NULL); #ifdef REDIS_TEST if (argc == 3 && !strcasecmp(argv[1], "test")) { diff --git a/src/storage-lite.c b/src/storage-lite.c new file mode 100644 index 000000000..ed199ef45 --- /dev/null +++ b/src/storage-lite.c @@ -0,0 +1,41 @@ +#include +#include +#include +#include +#include +#include +#include +#include "storage.h" +#include + +// initialize the memory subsystem. +// NOTE: This may be called twice, first with NULL specifying we should use ram +// later, after the configuration file is loaded with a path to where we should +// place our temporary file. +void storage_init(const char *tmpfilePath) +{ + assert(tmpfilePath == NULL); + (void)tmpfilePath; +} + +void *salloc(size_t cb, enum MALLOC_CLASS class) +{ + (void)class; + return malloc(cb); +} + +void *scalloc(size_t cb, size_t c, enum MALLOC_CLASS class) +{ + (void)class; + return calloc(cb, c); +} + +void sfree(void *pv) +{ + free(pv); +} + +void *srealloc(void *pv, size_t cb) +{ + return realloc(pv, cb); +} diff --git a/src/storage.c b/src/storage.c index 759c99840..d6fcc1b8f 100644 --- a/src/storage.c +++ b/src/storage.c @@ -10,7 +10,7 @@ #include "storage.h" struct memkind *mkdisk = NULL; -static char *PMEM_DIR = "/mnt/btrfs_scratch/"; +static const char *PMEM_DIR = NULL; void handle_prefork(); void handle_postfork_parent(); @@ -85,9 +85,6 @@ void *pool_alloc(struct alloc_pool *ppool) } } -#pragma weak serverLog -void serverLog(int level, const char*fmt, ...){} - void pool_free(struct alloc_pool *ppool, void *pv) { struct object_page *cur = ppool->pobjpageHead; @@ -113,18 +110,62 @@ void pool_free(struct alloc_pool *ppool, void *pv) struct alloc_pool poolobj; struct alloc_pool poolembstrobj; -void storage_init() +int forkFile() { - int errv = memkind_create_pmem(PMEM_DIR, 0, &mkdisk); - if (errv) + int fdT; + memkind_tmpfile(PMEM_DIR, &fdT); + if (ioctl(fdT, FICLONE, memkind_fd(mkdisk)) == -1) { - fprintf(stderr, "Memory pool creation failed: %d\n", errv); - exit(EXIT_FAILURE); + return -1; } - pool_initialize(&poolobj, sizeof(robj)); - pool_initialize(&poolembstrobj, EMBSTR_ROBJ_SIZE); + return fdT; +} - pthread_atfork(handle_prefork, handle_postfork_parent, handle_postfork_child); +// initialize the memory subsystem. +// NOTE: This may be called twice, first with NULL specifying we should use ram +// later, after the configuration file is loaded with a path to where we should +// place our temporary file. +void storage_init(const char *tmpfilePath) +{ + if (tmpfilePath == NULL) + { + serverAssert(mkdisk == NULL); + mkdisk = MEMKIND_DEFAULT; + } + else + { + // First create the file + serverAssert(mkdisk == MEMKIND_DEFAULT); + PMEM_DIR = memkind_malloc(MEMKIND_DEFAULT, strlen(tmpfilePath)); + strcpy((char*)PMEM_DIR, tmpfilePath); + int errv = memkind_create_pmem(PMEM_DIR, 0, &mkdisk); + if (errv == MEMKIND_ERROR_INVALID) + { + serverLog(LOG_CRIT, "Memory pool creation failed: %s", strerror(errno)); + exit(EXIT_FAILURE); + } + else if (errv) + { + char msgbuf[1024]; + memkind_error_message(errv, msgbuf, 1024); + serverLog(LOG_CRIT, "Memory pool creation failed: %s", msgbuf); + exit(EXIT_FAILURE); + } + + // Next test if COW is working + int fdTest = forkFile(); + if (fdTest < 0) + { + serverLog(LOG_ERR, "Scratch file system does not support Copy on Write. To fix this scratch-file-path must point to a path on a filesystem which supports copy on write, such as btrfs."); + exit(EXIT_FAILURE); + } + close(fdTest); + + pool_initialize(&poolobj, sizeof(robj)); + pool_initialize(&poolembstrobj, EMBSTR_ROBJ_SIZE); + + pthread_atfork(handle_prefork, handle_postfork_parent, handle_postfork_child); + } } @@ -184,12 +225,9 @@ void *srealloc(void *pv, size_t cb) int fdNew = -1; void handle_prefork() { - memkind_tmpfile(PMEM_DIR, &fdNew); - if (ioctl(fdNew, FICLONE, memkind_fd(mkdisk)) == -1) - { - perror("failed to fork file"); - exit(EXIT_FAILURE); - } + fdNew = forkFile(); + if (fdNew < 0) + serverLog(LOG_ERR, "Failed to clone scratch file"); } void handle_postfork_parent() diff --git a/src/storage.h b/src/storage.h index 882113cdc..f35bc62e3 100644 --- a/src/storage.h +++ b/src/storage.h @@ -9,12 +9,7 @@ enum MALLOC_CLASS MALLOC_SHARED, }; -void storage_init(void); - -struct redisObject *salloc_obj(); -void sfree_obj(struct redisObject *obj); -struct redisObject *salloc_objembstr(); -void sfree_objembstr(struct redisObject *obj); +void storage_init(const char *tmpfilePath); void *salloc(size_t cb, enum MALLOC_CLASS class); void *scalloc(size_t cb, size_t c, enum MALLOC_CLASS class); From 0a443027b8f413ae51430b41ec31a47a9a207f12 Mon Sep 17 00:00:00 2001 From: John Sully Date: Fri, 1 Feb 2019 14:01:35 -0500 Subject: [PATCH 005/101] template config --- redis.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/redis.conf b/redis.conf index 93ab9a42e..9847c98f8 100644 --- a/redis.conf +++ b/redis.conf @@ -1376,3 +1376,7 @@ rdb-save-incremental-fsync yes # the main dictionary scan # active-defrag-max-scan-fields 1000 +# Path to directory for file backed scratchpad. The file backed scratchpad +# reduces memory requirements by storing rarely accessed data on disk +# instead of RAM. A temporary file will be created in this directory. +# scratch-file-path /tmp/ From e28d9adeee16be10a5c66fb4f932687ee6315574 Mon Sep 17 00:00:00 2001 From: John Sully Date: Fri, 1 Feb 2019 15:21:00 -0500 Subject: [PATCH 006/101] Properly integrate memkind --- deps/Makefile | 5 +++++ deps/memkind/Makefile | 12 ++++++++++++ src/Makefile | 10 ++++++++-- src/defrag.c | 4 ++-- src/zmalloc.c | 27 ++++++++++++++++----------- src/zmalloc.h | 6 +++--- 6 files changed, 46 insertions(+), 18 deletions(-) create mode 100644 deps/memkind/Makefile diff --git a/deps/Makefile b/deps/Makefile index eb35c1e1f..bacc8f48e 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -53,6 +53,11 @@ linenoise: .make-prerequisites .PHONY: linenoise +.PHONY: memkind +memkind: + cd memkind && $(MAKE) + + ifeq ($(uname_S),SunOS) # Make isinf() available LUA_CFLAGS= -D__C99FEATURES__=1 diff --git a/deps/memkind/Makefile b/deps/memkind/Makefile new file mode 100644 index 000000000..af75dfaa4 --- /dev/null +++ b/deps/memkind/Makefile @@ -0,0 +1,12 @@ +export JE_PREFIX=jemk_ + +.PHONY: all +all: + cd src; ./build_jemalloc.sh + cd src; ./build.sh + +.PHONY: clean +clean: + cd src; make clean + + diff --git a/src/Makefile b/src/Makefile index 9820ffcc8..966da199d 100644 --- a/src/Makefile +++ b/src/Makefile @@ -38,7 +38,7 @@ MALLOC=libc ifneq ($(uname_M),armv6l) ifneq ($(uname_M),armv7l) ifeq ($(uname_S),Linux) - MALLOC=jemalloc + MALLOC=memkind endif endif endif @@ -119,7 +119,7 @@ ifeq ($(uname_S),DragonFly) else # All the other OSes (notably Linux) FINAL_LDFLAGS+= -rdynamic - FINAL_LIBS+=-ldl -pthread -lrt -lmemkind + FINAL_LIBS+=-ldl -pthread -lrt endif endif endif @@ -145,6 +145,12 @@ ifeq ($(MALLOC),jemalloc) FINAL_LIBS := ../deps/jemalloc/lib/libjemalloc.a $(FINAL_LIBS) endif +ifeq ($(MALLOC),memkind) + DEPENDENCY_TARGETS+= memkind + FINAL_CFLAGS+= -DUSE_MEMKIND -I../deps/memkind/src/include + FINAL_LIBS := ../deps/memkind/src/.libs/libmemkind.a -lnuma $(FINAL_LIBS) +endif + REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS) REDIS_LD=$(QUIET_LINK)$(CC) $(FINAL_LDFLAGS) REDIS_INSTALL=$(QUIET_INSTALL)$(INSTALL) diff --git a/src/defrag.c b/src/defrag.c index 7729c594c..d67b6e253 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -72,8 +72,8 @@ void* activeDefragAlloc(void *ptr) { /* move this allocation to a new allocation. * make sure not to use the thread cache. so that we don't get back the same * pointers we try to free */ - size = zmalloc_size(ptr, MALLOC_LOCAL); - newptr = zmalloc_no_tcache(size, MALLOC_LOCAL); + size = zmalloc_size(ptr); + newptr = zmalloc_no_tcache(size); memcpy(newptr, ptr, size); zfree_no_tcache(ptr); return newptr; diff --git a/src/zmalloc.c b/src/zmalloc.c index 7d307bad7..3af72417f 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -57,7 +57,12 @@ void zlibc_free(void *ptr) { #endif /* Explicitly override malloc/free etc when using tcmalloc. */ -#if defined(USE_TCMALLOC) +#if defined(USE_MEMKIND) +#define malloc(size, type) salloc(size, type) +#define calloc(count, size, type) scalloc(count, size, type) +#define realloc(ptr, size) srealloc(ptr, size) +#define free(ptr) sfree(ptr) +#elif defined(USE_TCMALLOC) #define malloc(size) tc_malloc(size) #define calloc(count,size) tc_calloc(count,size) #define realloc(ptr,size) tc_realloc(ptr,size) @@ -70,16 +75,6 @@ void zlibc_free(void *ptr) { #define mallocx(size,flags) je_mallocx(size,flags) #define dallocx(ptr,flags) je_dallocx(ptr,flags) #endif -#include "storage.h" -#undef malloc -#undef calloc -#undef realloc -#undef free -#define malloc(size, type) salloc(size, type) -#define calloc(count, size, type) scalloc(count, size, type) -#define realloc(ptr, size) srealloc(ptr, size) -#define free(ptr) sfree(ptr) -//#define zmalloc_size(ptr) (sizeof(ptr)) #define update_zmalloc_stat_alloc(__n) do { \ size_t _n = (__n); \ @@ -106,7 +101,12 @@ static void zmalloc_default_oom(size_t size) { static void (*zmalloc_oom_handler)(size_t) = zmalloc_default_oom; void *zmalloc(size_t size, enum MALLOC_CLASS class) { +#ifdef USE_MEMKIND void *ptr = malloc(size+PREFIX_SIZE, class); +#else + (void)class; + void *ptr = malloc(size+PREFIX_SIZE); +#endif if (!ptr) zmalloc_oom_handler(size); #ifdef HAVE_MALLOC_SIZE @@ -138,7 +138,12 @@ void zfree_no_tcache(void *ptr) { #endif void *zcalloc(size_t size, enum MALLOC_CLASS class) { +#ifdef USE_MEMKIND void *ptr = calloc(1, size+PREFIX_SIZE, class); +#else + (void)class; + void *ptr = calloc(1, size+PREFIX_SIZE); +#endif if (!ptr) zmalloc_oom_handler(size); #ifdef HAVE_MALLOC_SIZE diff --git a/src/zmalloc.h b/src/zmalloc.h index 0692e64c2..2ad37b905 100644 --- a/src/zmalloc.h +++ b/src/zmalloc.h @@ -35,9 +35,9 @@ #define __xstr(s) __str(s) #define __str(s) #s -#if 1 - #define ZMALLOC_LIB ("custom") - #include "storage.h" +#include "storage.h" +#if defined(USE_MEMKIND) + #define ZMALLOC_LIB ("memkind") #undef USE_JEMALLOC #define USE_MALLOC_CLASS 1 #elif defined(USE_TCMALLOC) From 3e61388df39f46e664864211d23df2d354a20083 Mon Sep 17 00:00:00 2001 From: John Sully Date: Mon, 4 Feb 2019 16:56:13 -0500 Subject: [PATCH 007/101] complete malloc memory class work, and pass tests --- deps/memkind/src/include/memkind.h | 1 + deps/memkind/src/src/memkind.c | 15 ++++++++++ deps/memkind/src/src/memkind_arena.c | 2 +- src/acl.c | 2 +- src/ae.c | 4 +-- src/ae_epoll.c | 2 +- src/ae_kqueue.c | 2 +- src/bitops.c | 2 +- src/cluster.c | 6 ++-- src/config.c | 6 ++-- src/geo.c | 2 +- src/intset.c | 2 +- src/listpack_malloc.h | 2 +- src/module.c | 16 +++++------ src/multi.c | 2 +- src/networking.c | 2 +- src/object.c | 2 +- src/quicklist.c | 2 +- src/rax_malloc.h | 2 +- src/rdb.c | 2 +- src/redis-benchmark.c | 4 +-- src/redis-cli.c | 20 ++++++------- src/scripting.c | 2 +- src/sds.c | 10 +++---- src/sentinel.c | 4 +-- src/server.c | 4 +-- src/sparkline.c | 2 +- src/storage-lite.c | 12 ++++++-- src/storage.c | 43 +++++++++++++++++----------- src/storage.h | 5 ++-- src/ziplist.c | 4 +-- src/zipmap.c | 2 +- src/zmalloc.c | 8 +++--- src/zmalloc.h | 5 +++- 34 files changed, 120 insertions(+), 81 deletions(-) diff --git a/deps/memkind/src/include/memkind.h b/deps/memkind/src/include/memkind.h index 720d7a302..4a8046663 100644 --- a/deps/memkind/src/include/memkind.h +++ b/deps/memkind/src/include/memkind.h @@ -318,6 +318,7 @@ void memkind_free(memkind_t kind, void *ptr); int memkind_fd(struct memkind *kind); void memkind_pmem_remapfd(struct memkind *kind, int fdNew); int memkind_tmpfile(const char *dir, int *fd); +memkind_t memkind_get_kind(void *ptr); #ifdef __cplusplus } diff --git a/deps/memkind/src/src/memkind.c b/deps/memkind/src/src/memkind.c index 74d4b3c17..188eb6985 100644 --- a/deps/memkind/src/src/memkind.c +++ b/deps/memkind/src/src/memkind.c @@ -791,3 +791,18 @@ MEMKIND_EXPORT int memkind_get_kind_by_partition(int partition, return memkind_get_kind_by_partition_internal(partition, kind); } +int memkind_lookup_arena(void *ptr, unsigned int *arena); +MEMKIND_EXPORT memkind_t memkind_get_kind(void *ptr) +{ + unsigned arena; + int err = memkind_lookup_arena(ptr, &arena); + memkind_t kind = NULL; + if (MEMKIND_UNLIKELY(err)) + return NULL; + kind = get_kind_by_arena(arena); + + if (!kind) + return MEMKIND_DEFAULT; + + return kind; +} diff --git a/deps/memkind/src/src/memkind_arena.c b/deps/memkind/src/src/memkind_arena.c index 7919a5dc3..d93ee4282 100644 --- a/deps/memkind/src/src/memkind_arena.c +++ b/deps/memkind/src/src/memkind_arena.c @@ -435,7 +435,7 @@ static void tcache_finalize(void *args) } } -static inline int memkind_lookup_arena(void *ptr, unsigned int *arena) +int memkind_lookup_arena(void *ptr, unsigned int *arena) { size_t sz = sizeof(unsigned); unsigned temp_arena; diff --git a/src/acl.c b/src/acl.c index b37b4719a..6b6274098 100644 --- a/src/acl.c +++ b/src/acl.c @@ -286,7 +286,7 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) { /* Now we can make space for the new item (and the null term). */ items += 2; u->allowed_subcommands[id] = zrealloc(u->allowed_subcommands[id], - sizeof(sds)*items); + sizeof(sds)*items, MALLOC_LOCAL); u->allowed_subcommands[id][items-2] = sdsnew(sub); u->allowed_subcommands[id][items-1] = NULL; } diff --git a/src/ae.c b/src/ae.c index 7f59aecf1..80f1d8111 100644 --- a/src/ae.c +++ b/src/ae.c @@ -111,8 +111,8 @@ int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) { if (eventLoop->maxfd >= setsize) return AE_ERR; if (aeApiResize(eventLoop,setsize) == -1) return AE_ERR; - eventLoop->events = zrealloc(eventLoop->events,sizeof(aeFileEvent)*setsize); - eventLoop->fired = zrealloc(eventLoop->fired,sizeof(aeFiredEvent)*setsize); + eventLoop->events = zrealloc(eventLoop->events,sizeof(aeFileEvent)*setsize, MALLOC_LOCAL); + eventLoop->fired = zrealloc(eventLoop->fired,sizeof(aeFiredEvent)*setsize, MALLOC_LOCAL); eventLoop->setsize = setsize; /* Make sure that if we created new slots, they are initialized with diff --git a/src/ae_epoll.c b/src/ae_epoll.c index 298bcf6ce..d299c8c1b 100644 --- a/src/ae_epoll.c +++ b/src/ae_epoll.c @@ -58,7 +58,7 @@ static int aeApiCreate(aeEventLoop *eventLoop) { static int aeApiResize(aeEventLoop *eventLoop, int setsize) { aeApiState *state = eventLoop->apidata; - state->events = zrealloc(state->events, sizeof(struct epoll_event)*setsize); + state->events = zrealloc(state->events, sizeof(struct epoll_event)*setsize, MALLOC_LOCAL); return 0; } diff --git a/src/ae_kqueue.c b/src/ae_kqueue.c index 36148bf58..19ac9ffc1 100644 --- a/src/ae_kqueue.c +++ b/src/ae_kqueue.c @@ -60,7 +60,7 @@ static int aeApiCreate(aeEventLoop *eventLoop) { static int aeApiResize(aeEventLoop *eventLoop, int setsize) { aeApiState *state = eventLoop->apidata; - state->events = zrealloc(state->events, sizeof(struct kevent)*setsize); + state->events = zrealloc(state->events, sizeof(struct kevent)*setsize, MALLOC_LOCAL); return 0; } diff --git a/src/bitops.c b/src/bitops.c index e2b53169a..c35b2ca23 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -978,7 +978,7 @@ void bitfieldCommand(client *c) { } /* Populate the array of operations we'll process. */ - ops = zrealloc(ops,sizeof(*ops)*(numops+1)); + ops = zrealloc(ops,sizeof(*ops)*(numops+1), MALLOC_SHARED); ops[numops].offset = bitoffset; ops[numops].i64 = i64; ops[numops].opcode = opcode; diff --git a/src/cluster.c b/src/cluster.c index bd04646d0..5b3d9663e 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -842,7 +842,7 @@ int clusterNodeAddSlave(clusterNode *master, clusterNode *slave) { for (j = 0; j < master->numslaves; j++) if (master->slaves[j] == slave) return C_ERR; master->slaves = zrealloc(master->slaves, - sizeof(clusterNode*)*(master->numslaves+1)); + sizeof(clusterNode*)*(master->numslaves+1), MALLOC_LOCAL); master->slaves[master->numslaves] = slave; master->numslaves++; master->flags |= CLUSTER_NODE_MIGRATE_TO; @@ -5119,8 +5119,8 @@ void migrateCommand(client *c) { * the caller there was nothing to migrate. We don't return an error in * this case, since often this is due to a normal condition like the key * expiring in the meantime. */ - ov = zrealloc(ov,sizeof(robj*)*num_keys); - kv = zrealloc(kv,sizeof(robj*)*num_keys); + ov = zrealloc(ov,sizeof(robj*)*num_keys, MALLOC_LOCAL); + kv = zrealloc(kv,sizeof(robj*)*num_keys, MALLOC_LOCAL); int oi = 0; for (j = 0; j < num_keys; j++) { diff --git a/src/config.c b/src/config.c index 5f5fd6792..3e7733138 100644 --- a/src/config.c +++ b/src/config.c @@ -143,7 +143,7 @@ int yesnotoi(char *s) { } void appendServerSaveParams(time_t seconds, int changes) { - server.saveparams = zrealloc(server.saveparams,sizeof(struct saveparam)*(server.saveparamslen+1)); + server.saveparams = zrealloc(server.saveparams,sizeof(struct saveparam)*(server.saveparamslen+1), MALLOC_LOCAL); server.saveparams[server.saveparamslen].seconds = seconds; server.saveparams[server.saveparamslen].changes = changes; server.saveparamslen++; @@ -805,7 +805,7 @@ void loadServerConfigFromString(char *config) { if (err) goto loaderr; } } else if (!strcasecmp(argv[0],"scratch-file-path")) { - storage_init(argv[1]); + storage_init(argv[1], server.maxmemory); } else { err = "Bad directive or wrong number of arguments"; goto loaderr; } @@ -1642,7 +1642,7 @@ struct rewriteConfigState { /* Append the new line to the current configuration state. */ void rewriteConfigAppendLine(struct rewriteConfigState *state, sds line) { - state->lines = zrealloc(state->lines, sizeof(char*) * (state->numlines+1)); + state->lines = zrealloc(state->lines, sizeof(char*) * (state->numlines+1), MALLOC_LOCAL); state->lines[state->numlines++] = line; } diff --git a/src/geo.c b/src/geo.c index 1c792255e..8e89dbefb 100644 --- a/src/geo.c +++ b/src/geo.c @@ -64,7 +64,7 @@ geoArray *geoArrayCreate(void) { geoPoint *geoArrayAppend(geoArray *ga) { if (ga->used == ga->buckets) { ga->buckets = (ga->buckets == 0) ? 8 : ga->buckets*2; - ga->array = zrealloc(ga->array,sizeof(geoPoint)*ga->buckets); + ga->array = zrealloc(ga->array,sizeof(geoPoint)*ga->buckets, MALLOC_SHARED); } geoPoint *gp = ga->array+ga->used; ga->used++; diff --git a/src/intset.c b/src/intset.c index 4cf15f4db..328d50b19 100644 --- a/src/intset.c +++ b/src/intset.c @@ -104,7 +104,7 @@ intset *intsetNew(void) { /* Resize the intset */ static intset *intsetResize(intset *is, uint32_t len) { uint32_t size = len*intrev32ifbe(is->encoding); - is = zrealloc(is,sizeof(intset)+size); + is = zrealloc(is,sizeof(intset)+size, MALLOC_SHARED); return is; } diff --git a/src/listpack_malloc.h b/src/listpack_malloc.h index 4c0317528..314570997 100644 --- a/src/listpack_malloc.h +++ b/src/listpack_malloc.h @@ -40,6 +40,6 @@ #define LISTPACK_ALLOC_H #include "zmalloc.h" #define lp_malloc(size) zmalloc(size, MALLOC_SHARED) -#define lp_realloc zrealloc +#define lp_realloc(ptr, size) zrealloc(ptr, size, MALLOC_SHARED) #define lp_free zfree #endif diff --git a/src/module.c b/src/module.c index b06493a1b..e73494149 100644 --- a/src/module.c +++ b/src/module.c @@ -280,7 +280,7 @@ void RM_FreeDict(RedisModuleCtx *ctx, RedisModuleDict *d); * and in general is taken into account as memory allocated by Redis. * You should avoid using malloc(). */ void *RM_Alloc(size_t bytes) { - return zmalloc(bytes, MALLOC_LOCAL); + return zmalloc(bytes, MALLOC_SHARED); } /* Use like calloc(). Memory allocated with this function is reported in @@ -288,12 +288,12 @@ void *RM_Alloc(size_t bytes) { * and in general is taken into account as memory allocated by Redis. * You should avoid using calloc() directly. */ void *RM_Calloc(size_t nmemb, size_t size) { - return zcalloc(nmemb*size, MALLOC_LOCAL); + return zcalloc(nmemb*size, MALLOC_SHARED); } /* Use like realloc() for memory obtained with RedisModule_Alloc(). */ void* RM_Realloc(void *ptr, size_t bytes) { - return zrealloc(ptr,bytes); + return zrealloc(ptr,bytes, MALLOC_SHARED); } /* Use like free() for memory obtained by RedisModule_Alloc() and @@ -558,7 +558,7 @@ int RM_IsKeysPositionRequest(RedisModuleCtx *ctx) { void RM_KeyAtPos(RedisModuleCtx *ctx, int pos) { if (!(ctx->flags & REDISMODULE_CTX_KEYS_POS_REQUEST)) return; if (pos <= 0) return; - ctx->keys_pos = zrealloc(ctx->keys_pos,sizeof(int)*(ctx->keys_count+1)); + ctx->keys_pos = zrealloc(ctx->keys_pos,sizeof(int)*(ctx->keys_count+1), MALLOC_LOCAL); ctx->keys_pos[ctx->keys_count++] = pos; } @@ -735,7 +735,7 @@ void autoMemoryAdd(RedisModuleCtx *ctx, int type, void *ptr) { if (ctx->amqueue_used == ctx->amqueue_len) { ctx->amqueue_len *= 2; if (ctx->amqueue_len < 16) ctx->amqueue_len = 16; - ctx->amqueue = zrealloc(ctx->amqueue,sizeof(struct AutoMemEntry)*ctx->amqueue_len); + ctx->amqueue = zrealloc(ctx->amqueue,sizeof(struct AutoMemEntry)*ctx->amqueue_len, MALLOC_LOCAL); } ctx->amqueue[ctx->amqueue_used].type = type; ctx->amqueue[ctx->amqueue_used].ptr = ptr; @@ -1121,7 +1121,7 @@ int RM_ReplyWithArray(RedisModuleCtx *ctx, long len) { if (c == NULL) return REDISMODULE_OK; if (len == REDISMODULE_POSTPONED_ARRAY_LEN) { ctx->postponed_arrays = zrealloc(ctx->postponed_arrays,sizeof(void*)* - (ctx->postponed_arrays_count+1)); + (ctx->postponed_arrays_count+1), MALLOC_LOCAL); ctx->postponed_arrays[ctx->postponed_arrays_count] = addReplyDeferredLen(c); ctx->postponed_arrays_count++; @@ -2616,7 +2616,7 @@ robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int /* As a first guess to avoid useless reallocations, size argv to * hold one argument for each char specifier in 'fmt'. */ argv_size = strlen(fmt)+1; /* +1 because of the command name. */ - argv = zrealloc(argv,sizeof(robj*)*argv_size); + argv = zrealloc(argv,sizeof(robj*)*argv_size, MALLOC_LOCAL); /* Build the arguments vector based on the format specifier. */ argv[0] = createStringObject(cmdname,strlen(cmdname)); @@ -2648,7 +2648,7 @@ robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int * We resize by vector_len-1 elements, because we held * one element in argv for the vector already */ argv_size += vlen-1; - argv = zrealloc(argv,sizeof(robj*)*argv_size); + argv = zrealloc(argv,sizeof(robj*)*argv_size, MALLOC_LOCAL); size_t i = 0; for (i = 0; i < vlen; i++) { diff --git a/src/multi.c b/src/multi.c index 720ec90e2..6aff069a3 100644 --- a/src/multi.c +++ b/src/multi.c @@ -59,7 +59,7 @@ void queueMultiCommand(client *c) { int j; c->mstate.commands = zrealloc(c->mstate.commands, - sizeof(multiCmd)*(c->mstate.count+1)); + sizeof(multiCmd)*(c->mstate.count+1), MALLOC_LOCAL); mc = c->mstate.commands+c->mstate.count; mc->cmd = c->cmd; mc->argc = c->argc; diff --git a/src/networking.c b/src/networking.c index b853da5bd..d438e70b8 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2170,7 +2170,7 @@ void rewriteClientCommandArgument(client *c, int i, robj *newval) { robj *oldval; if (i >= c->argc) { - c->argv = zrealloc(c->argv,sizeof(robj*)*(i+1)); + c->argv = zrealloc(c->argv,sizeof(robj*)*(i+1), MALLOC_LOCAL); c->argc = i+1; c->argv[i] = NULL; } diff --git a/src/object.c b/src/object.c index a5da2b142..2d4095942 100644 --- a/src/object.c +++ b/src/object.c @@ -1035,7 +1035,7 @@ struct redisMemOverhead *getMemoryOverheadData(void) { if (keyscount==0) continue; mh->total_keys += keyscount; - mh->db = zrealloc(mh->db,sizeof(mh->db[0])*(mh->num_dbs+1)); + mh->db = zrealloc(mh->db,sizeof(mh->db[0])*(mh->num_dbs+1), MALLOC_LOCAL); mh->db[mh->num_dbs].dbid = j; mem = dictSize(db->dict) * sizeof(dictEntry) + diff --git a/src/quicklist.c b/src/quicklist.c index ea0a2ae77..02771ef00 100644 --- a/src/quicklist.c +++ b/src/quicklist.c @@ -194,7 +194,7 @@ REDIS_STATIC int __quicklistCompressNode(quicklistNode *node) { zfree(lzf); return 0; } - lzf = zrealloc(lzf, sizeof(*lzf) + lzf->sz); + lzf = zrealloc(lzf, sizeof(*lzf) + lzf->sz, MALLOC_SHARED); zfree(node->zl); node->zl = (unsigned char *)lzf; node->encoding = QUICKLIST_NODE_ENCODING_LZF; diff --git a/src/rax_malloc.h b/src/rax_malloc.h index a5aea44d9..81a5815ba 100644 --- a/src/rax_malloc.h +++ b/src/rax_malloc.h @@ -39,6 +39,6 @@ #define RAX_ALLOC_H #include "zmalloc.h" #define rax_malloc(size) zmalloc(size, MALLOC_SHARED) -#define rax_realloc zrealloc +#define rax_realloc(ptr, size) zrealloc(ptr, size, MALLOC_SHARED) #define rax_free zfree #endif diff --git a/src/rdb.c b/src/rdb.c index 0374a3999..4065ca11b 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -2154,7 +2154,7 @@ void backgroundSaveDoneHandlerSocket(int exitcode, int bysignal) { /* Make space for enough elements as specified by the first * uint64_t element in the array. */ - ok_slaves = zrealloc(ok_slaves,sizeof(uint64_t)+readlen); + ok_slaves = zrealloc(ok_slaves,sizeof(uint64_t)+readlen, MALLOC_LOCAL); if (readlen && read(server.rdb_pipe_read_result_from_child, ok_slaves+1, readlen) != readlen) diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c index 5dffd46b6..369bb0aa4 100644 --- a/src/redis-benchmark.c +++ b/src/redis-benchmark.c @@ -395,7 +395,7 @@ static client createClient(char *cmd, size_t len, client from) { c->randptr = zmalloc(sizeof(char*)*c->randfree, MALLOC_LOCAL); while ((p = strstr(p,"__rand_int__")) != NULL) { if (c->randfree == 0) { - c->randptr = zrealloc(c->randptr,sizeof(char*)*c->randlen*2); + c->randptr = zrealloc(c->randptr,sizeof(char*)*c->randlen*2, MALLOC_LOCAL); c->randfree += c->randlen; } c->randptr[c->randlen++] = p; @@ -653,7 +653,7 @@ int main(int argc, const char **argv) { client c; - storage_init(NULL); + storage_init(NULL, 0); srandom(time(NULL)); signal(SIGHUP, SIG_IGN); diff --git a/src/redis-cli.c b/src/redis-cli.c index abcfd2b78..d0dffb005 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -529,7 +529,7 @@ static void cliIntegrateHelp(void) { if (i != helpEntriesLen) continue; helpEntriesLen++; - helpEntries = zrealloc(helpEntries,sizeof(helpEntry)*helpEntriesLen); + helpEntries = zrealloc(helpEntries,sizeof(helpEntry)*helpEntriesLen, MALLOC_LOCAL); helpEntry *new = helpEntries+(helpEntriesLen-1); new->argc = 1; @@ -1815,7 +1815,7 @@ static void repl(void) { static int noninteractive(int argc, char **argv) { int retval = 0; if (config.stdinarg) { - argv = zrealloc(argv, (argc+1)*sizeof(char*)); + argv = zrealloc(argv, (argc+1)*sizeof(char*), MALLOC_LOCAL); argv[argc] = readArgFromStdin(); retval = issueCommand(argc+1, argv); } else { @@ -3456,7 +3456,7 @@ static int clusterManagerNodeLoadInfo(clusterManagerNode *node, int opts, sds dst = sdsnew(p); node->migrating_count += 2; node->migrating = zrealloc(node->migrating, - (node->migrating_count * sizeof(sds))); + (node->migrating_count * sizeof(sds)), MALLOC_LOCAL); node->migrating[node->migrating_count - 2] = slot; node->migrating[node->migrating_count - 1] = @@ -3470,7 +3470,7 @@ static int clusterManagerNodeLoadInfo(clusterManagerNode *node, int opts, sds src = sdsnew(p); node->importing_count += 2; node->importing = zrealloc(node->importing, - (node->importing_count * sizeof(sds))); + (node->importing_count * sizeof(sds)), MALLOC_LOCAL); node->importing[node->importing_count - 2] = slot; node->importing[node->importing_count - 1] = @@ -3704,7 +3704,7 @@ static sds clusterManagerGetConfigSignature(clusterManagerNode *node) { } else line = p; if (slotsdef[0] != '[') { c++; - slots = zrealloc(slots, (c * sizeof(char *))); + slots = zrealloc(slots, (c * sizeof(char *)), MALLOC_LOCAL); slots[c - 1] = slotsdef; } } @@ -3713,7 +3713,7 @@ static sds clusterManagerGetConfigSignature(clusterManagerNode *node) { qsort(slots, c, sizeof(char *), clusterManagerSlotCompare); node_count++; node_configs = - zrealloc(node_configs, (node_count * sizeof(char *))); + zrealloc(node_configs, (node_count * sizeof(char *)), MALLOC_LOCAL); /* Make room for '|' separators. */ tot_size += (sizeof(char) * (c - 1)); char *cfg = zmalloc((sizeof(char) * tot_size) + 1, MALLOC_LOCAL); @@ -6610,8 +6610,8 @@ static void findBigKeys(void) { /* Reallocate our type and size array if we need to */ if(keys->elements > arrsize) { - types = zrealloc(types, sizeof(int)*keys->elements); - sizes = zrealloc(sizes, sizeof(unsigned long long)*keys->elements); + types = zrealloc(types, sizeof(int)*keys->elements, MALLOC_LOCAL); + sizes = zrealloc(sizes, sizeof(unsigned long long)*keys->elements, MALLOC_LOCAL); if(!types || !sizes) { fprintf(stderr, "Failed to allocate storage for keys!\n"); @@ -6760,7 +6760,7 @@ static void findHotKeys(void) { /* Reallocate our freqs array if we need to */ if(keys->elements > arrsize) { - freqs = zrealloc(freqs, sizeof(unsigned long long)*keys->elements); + freqs = zrealloc(freqs, sizeof(unsigned long long)*keys->elements, MALLOC_LOCAL); if(!freqs) { fprintf(stderr, "Failed to allocate storage for keys!\n"); @@ -7176,7 +7176,7 @@ static void intrinsicLatencyMode(void) { int main(int argc, char **argv) { int firstarg; - storage_init(NULL); + storage_init(NULL, 0); config.hostip = sdsnew("127.0.0.1"); config.hostport = 6379; config.hostsocket = NULL; diff --git a/src/scripting.c b/src/scripting.c index 3b3b996e0..85da662e6 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -407,7 +407,7 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { /* Build the arguments vector */ if (argv_size < argc) { - argv = zrealloc(argv,sizeof(robj*)*argc); + argv = zrealloc(argv,sizeof(robj*)*argc, MALLOC_LOCAL); argv_size = argc; } diff --git a/src/sds.c b/src/sds.c index a87173d94..cc574bade 100644 --- a/src/sds.c +++ b/src/sds.c @@ -228,7 +228,7 @@ sds sdsMakeRoomFor(sds s, size_t addlen) { hdrlen = sdsHdrSize(type); if (oldtype==type) { - newsh = s_realloc(sh, hdrlen+newlen+1); + newsh = s_realloc(sh, hdrlen+newlen+1, MALLOC_SHARED); if (newsh == NULL) return NULL; s = (char*)newsh+hdrlen; } else { @@ -269,7 +269,7 @@ sds sdsRemoveFreeSpace(sds s) { * only if really needed. Otherwise if the change is huge, we manually * reallocate the string to use the different header type. */ if (oldtype==type || type > SDS_TYPE_8) { - newsh = s_realloc(sh, oldhdrlen+len+1); + newsh = s_realloc(sh, oldhdrlen+len+1, MALLOC_SHARED); if (newsh == NULL) return NULL; s = (char*)newsh+oldhdrlen; } else { @@ -829,7 +829,7 @@ sds *sdssplitlen(const char *s, ssize_t len, const char *sep, int seplen, int *c sds *newtokens; slots *= 2; - newtokens = s_realloc(tokens,sizeof(sds)*slots); + newtokens = s_realloc(tokens,sizeof(sds)*slots, MALLOC_SHARED); if (newtokens == NULL) goto cleanup; tokens = newtokens; } @@ -1038,7 +1038,7 @@ sds *sdssplitargs(const char *line, int *argc) { if (*p) p++; } /* add the token to the vector */ - vector = s_realloc(vector,((*argc)+1)*sizeof(char*)); + vector = s_realloc(vector,((*argc)+1)*sizeof(char*), MALLOC_SHARED); vector[*argc] = current; (*argc)++; current = NULL; @@ -1112,7 +1112,7 @@ sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen) { * the programs SDS is linked to, if they want to touch the SDS internals * even if they use a different allocator. */ void *sds_malloc(size_t size) { return s_malloc(size, MALLOC_SHARED); } -void *sds_realloc(void *ptr, size_t size) { return s_realloc(ptr,size); } +void *sds_realloc(void *ptr, size_t size) { return s_realloc(ptr,size, MALLOC_SHARED); } void sds_free(void *ptr) { s_free(ptr); } #if defined(SDS_TEST_MAIN) diff --git a/src/sentinel.c b/src/sentinel.c index 7add1e744..dc7da22e1 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1499,7 +1499,7 @@ int sentinelResetMasterAndChangeAddress(sentinelRedisInstance *master, char *ip, sentinelRedisInstance *slave = dictGetVal(de); if (sentinelAddrIsEqual(slave->addr,newaddr)) continue; - slaves = zrealloc(slaves,sizeof(sentinelAddr*)*(numslaves+1)); + slaves = zrealloc(slaves,sizeof(sentinelAddr*)*(numslaves+1), MALLOC_LOCAL); slaves[numslaves++] = createSentinelAddr(slave->addr->ip, slave->addr->port); } @@ -1509,7 +1509,7 @@ int sentinelResetMasterAndChangeAddress(sentinelRedisInstance *master, char *ip, * as a slave as well, so that we'll be able to sense / reconfigure * the old master. */ if (!sentinelAddrIsEqual(newaddr,master->addr)) { - slaves = zrealloc(slaves,sizeof(sentinelAddr*)*(numslaves+1)); + slaves = zrealloc(slaves,sizeof(sentinelAddr*)*(numslaves+1), MALLOC_LOCAL); slaves[numslaves++] = createSentinelAddr(master->addr->ip, master->addr->port); } diff --git a/src/server.c b/src/server.c index 1444100b7..ce00e4711 100644 --- a/src/server.c +++ b/src/server.c @@ -2973,7 +2973,7 @@ int redisOpArrayAppend(redisOpArray *oa, struct redisCommand *cmd, int dbid, { redisOp *op; - oa->ops = zrealloc(oa->ops,sizeof(redisOp)*(oa->numops+1)); + oa->ops = zrealloc(oa->ops,sizeof(redisOp)*(oa->numops+1), MALLOC_LOCAL); op = oa->ops+oa->numops; op->cmd = cmd; op->dbid = dbid; @@ -4740,7 +4740,7 @@ int main(int argc, char **argv) { struct timeval tv; int j; - storage_init(NULL); + storage_init(NULL, 0); #ifdef REDIS_TEST if (argc == 3 && !strcasecmp(argv[1], "test")) { diff --git a/src/sparkline.c b/src/sparkline.c index e18ce98fd..5ee6c7252 100644 --- a/src/sparkline.c +++ b/src/sparkline.c @@ -70,7 +70,7 @@ void sparklineSequenceAddSample(struct sequence *seq, double value, char *label) if (value < seq->min) seq->min = value; else if (value > seq->max) seq->max = value; } - seq->samples = zrealloc(seq->samples,sizeof(struct sample)*(seq->length+1)); + seq->samples = zrealloc(seq->samples,sizeof(struct sample)*(seq->length+1), MALLOC_SHARED); seq->samples[seq->length].value = value; seq->samples[seq->length].label = label; seq->length++; diff --git a/src/storage-lite.c b/src/storage-lite.c index ed199ef45..9ed51db1b 100644 --- a/src/storage-lite.c +++ b/src/storage-lite.c @@ -7,15 +7,17 @@ #include #include "storage.h" #include +#include // initialize the memory subsystem. // NOTE: This may be called twice, first with NULL specifying we should use ram // later, after the configuration file is loaded with a path to where we should // place our temporary file. -void storage_init(const char *tmpfilePath) +void storage_init(const char *tmpfilePath, size_t cbReserve) { assert(tmpfilePath == NULL); (void)tmpfilePath; + (void)cbReserve; } void *salloc(size_t cb, enum MALLOC_CLASS class) @@ -35,7 +37,13 @@ void sfree(void *pv) free(pv); } -void *srealloc(void *pv, size_t cb) +void *srealloc(void *pv, size_t cb, enum MALLOC_CLASS class) { + (void)class; return realloc(pv, cb); } + +size_t salloc_usable_size(void *ptr) +{ + return malloc_usable_size(ptr); +} \ No newline at end of file diff --git a/src/storage.c b/src/storage.c index d6fcc1b8f..72472fd9f 100644 --- a/src/storage.c +++ b/src/storage.c @@ -7,6 +7,7 @@ #include #include #include +#include #include "storage.h" struct memkind *mkdisk = NULL; @@ -125,7 +126,7 @@ int forkFile() // NOTE: This may be called twice, first with NULL specifying we should use ram // later, after the configuration file is loaded with a path to where we should // place our temporary file. -void storage_init(const char *tmpfilePath) +void storage_init(const char *tmpfilePath, size_t cbFileReserve) { if (tmpfilePath == NULL) { @@ -161,6 +162,11 @@ void storage_init(const char *tmpfilePath) } close(fdTest); + // Now lets make the file big + if (cbFileReserve == 0) + cbFileReserve = 1*1024*1024*1024; // 1 GB (enough to be interesting) + posix_fallocate64(memkind_fd(mkdisk), 0, cbFileReserve); + pool_initialize(&poolobj, sizeof(robj)); pool_initialize(&poolembstrobj, EMBSTR_ROBJ_SIZE); @@ -187,28 +193,34 @@ void sfree_objembstr(robj *obj) pool_free(&poolembstrobj, obj); } -void *salloc(size_t cb, enum MALLOC_CLASS class) +size_t salloc_usable_size(void *ptr) +{ + return memkind_malloc_usable_size(memkind_get_kind(ptr), ptr); +} + +static memkind_t kindFromClass(enum MALLOC_CLASS class) { switch (class) { case MALLOC_SHARED: - return memkind_malloc(mkdisk, cb); + return mkdisk; default: - return memkind_malloc(MEMKIND_DEFAULT, cb); + break; } - return NULL; + return MEMKIND_DEFAULT; +} + +void *salloc(size_t cb, enum MALLOC_CLASS class) +{ + if (cb == 0) + cb = 1; + + return memkind_malloc(kindFromClass(class), cb); } void *scalloc(size_t cb, size_t c, enum MALLOC_CLASS class) { - switch (class) - { - case MALLOC_SHARED: - return memkind_calloc(mkdisk, cb, c); - default: - return memkind_calloc(MEMKIND_DEFAULT, cb, c); - } - return NULL; + return memkind_calloc(kindFromClass(class), cb, c); } void sfree(void *pv) @@ -216,10 +228,9 @@ void sfree(void *pv) memkind_free(NULL, pv); } -void *srealloc(void *pv, size_t cb) +void *srealloc(void *pv, size_t cb, enum MALLOC_CLASS class) { - memkind_t kind = mkdisk; - return memkind_realloc(kind, pv, cb); + return memkind_realloc(kindFromClass(class), pv, cb); } int fdNew = -1; diff --git a/src/storage.h b/src/storage.h index f35bc62e3..06c3572a5 100644 --- a/src/storage.h +++ b/src/storage.h @@ -9,11 +9,12 @@ enum MALLOC_CLASS MALLOC_SHARED, }; -void storage_init(const char *tmpfilePath); +void storage_init(const char *tmpfilePath, size_t cbFileReserve); void *salloc(size_t cb, enum MALLOC_CLASS class); void *scalloc(size_t cb, size_t c, enum MALLOC_CLASS class); void sfree(void*); -void *srealloc(void *pv, size_t cb); +void *srealloc(void *pv, size_t cb, enum MALLOC_CLASS class); +size_t salloc_usable_size(void *ptr); #endif diff --git a/src/ziplist.c b/src/ziplist.c index 733e2e685..ff0422d21 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -587,7 +587,7 @@ unsigned char *ziplistNew(void) { /* Resize the ziplist. */ unsigned char *ziplistResize(unsigned char *zl, unsigned int len) { - zl = zrealloc(zl,len); + zl = zrealloc(zl,len, MALLOC_SHARED); ZIPLIST_BYTES(zl) = intrev32ifbe(len); zl[len-1] = ZIP_END; return zl; @@ -903,7 +903,7 @@ unsigned char *ziplistMerge(unsigned char **first, unsigned char **second) { size_t second_offset = intrev32ifbe(ZIPLIST_TAIL_OFFSET(*second)); /* Extend target to new zlbytes then append or prepend source. */ - target = zrealloc(target, zlbytes); + target = zrealloc(target, zlbytes, MALLOC_SHARED); if (append) { /* append == appending to target */ /* Copy source after target (copying over original [END]): diff --git a/src/zipmap.c b/src/zipmap.c index 88e7f8706..54c32f214 100644 --- a/src/zipmap.c +++ b/src/zipmap.c @@ -200,7 +200,7 @@ static unsigned int zipmapRawEntryLength(unsigned char *p) { } static inline unsigned char *zipmapResize(unsigned char *zm, unsigned int len) { - zm = zrealloc(zm, len); + zm = zrealloc(zm, len, MALLOC_SHARED); zm[len-1] = ZIPMAP_END; return zm; } diff --git a/src/zmalloc.c b/src/zmalloc.c index 3af72417f..35599ce0e 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -60,7 +60,7 @@ void zlibc_free(void *ptr) { #if defined(USE_MEMKIND) #define malloc(size, type) salloc(size, type) #define calloc(count, size, type) scalloc(count, size, type) -#define realloc(ptr, size) srealloc(ptr, size) +#define realloc(ptr, size, type) srealloc(ptr, size, type) #define free(ptr) sfree(ptr) #elif defined(USE_TCMALLOC) #define malloc(size) tc_malloc(size) @@ -156,17 +156,17 @@ void *zcalloc(size_t size, enum MALLOC_CLASS class) { #endif } -void *zrealloc(void *ptr, size_t size) { +void *zrealloc(void *ptr, size_t size, enum MALLOC_CLASS class) { #ifndef HAVE_MALLOC_SIZE void *realptr; #endif size_t oldsize; void *newptr; - if (ptr == NULL) return zmalloc(size, MALLOC_SHARED); + if (ptr == NULL) return zmalloc(size, class); #ifdef HAVE_MALLOC_SIZE oldsize = zmalloc_size(ptr); - newptr = realloc(ptr,size); + newptr = realloc(ptr,size, class); if (!newptr) zmalloc_oom_handler(size); update_zmalloc_stat_free(oldsize); diff --git a/src/zmalloc.h b/src/zmalloc.h index 2ad37b905..63ae7074a 100644 --- a/src/zmalloc.h +++ b/src/zmalloc.h @@ -36,10 +36,13 @@ #define __str(s) #s #include "storage.h" +#define USE_MEMKIND 1 #if defined(USE_MEMKIND) #define ZMALLOC_LIB ("memkind") #undef USE_JEMALLOC #define USE_MALLOC_CLASS 1 + #define HAVE_MALLOC_SIZE 1 + #define zmalloc_size(p) salloc_usable_size(p) #elif defined(USE_TCMALLOC) #define ZMALLOC_LIB ("tcmalloc-" __xstr(TC_VERSION_MAJOR) "." __xstr(TC_VERSION_MINOR)) #include @@ -84,7 +87,7 @@ void *zmalloc(size_t size, enum MALLOC_CLASS class); void *zcalloc(size_t size, enum MALLOC_CLASS class); -void *zrealloc(void *ptr, size_t size); +void *zrealloc(void *ptr, size_t size, enum MALLOC_CLASS class); void zfree(void *ptr); char *zstrdup(const char *s); size_t zmalloc_used_memory(void); From cd8145e3141c66717f9853426638d2ba1ce0a084 Mon Sep 17 00:00:00 2001 From: John Sully Date: Tue, 5 Feb 2019 23:36:40 -0500 Subject: [PATCH 008/101] Make main headers C++ safe, and change rdb to use file descriptor instead of FILE pointer --- .vscode/settings.json | 3 ++- src/Makefile | 13 ++++++++++--- src/acl.c | 8 ++++---- src/aof.c | 10 +++++----- src/cluster.c | 8 ++++---- src/db.c | 42 +++++++++++++++++++++--------------------- src/debug.c | 18 +++++++++--------- src/defrag.c | 20 ++++++++++---------- src/evict.c | 8 ++++---- src/fmacros.h | 4 ++++ src/geo.c | 2 +- src/intset.h | 8 +++++++- src/lazyfree.c | 10 +++++----- src/module.c | 40 ++++++++++++++++++++-------------------- src/multi.c | 2 +- src/networking.c | 4 ++-- src/object.c | 26 +++++++++++++------------- src/pubsub.c | 8 ++++---- src/quicklist.c | 14 +++++++------- src/quicklist.h | 12 +++++++++--- src/rax.h | 8 +++++++- src/rdb-s3.cpp | 13 +++++++++++++ src/rdb.c | 43 ++++++++++++++++++++++++++----------------- src/rdb.h | 1 + src/redis-check-rdb.c | 2 +- src/replication.c | 2 +- src/rio.c | 15 +++++++-------- src/rio.h | 4 ++-- src/sds.h | 20 +++++++++++++------- src/server.c | 21 +++++++++++---------- src/server.h | 19 ++++++++++--------- src/sort.c | 6 +++--- src/storage.h | 6 +++--- src/stream.h | 4 ++-- src/t_list.c | 4 ++-- src/t_stream.c | 32 ++++++++++++++++---------------- src/t_zset.c | 38 +++++++++++++++++++------------------- src/zmalloc.h | 6 +++--- 38 files changed, 282 insertions(+), 222 deletions(-) create mode 100644 src/rdb-s3.cpp diff --git a/.vscode/settings.json b/.vscode/settings.json index 3813ff8ac..6fac65a3d 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,6 @@ { "files.associations": { - "zmalloc.h": "c" + "zmalloc.h": "c", + "stat.h": "c" } } \ No newline at end of file diff --git a/src/Makefile b/src/Makefile index 966da199d..b2c7cc891 100644 --- a/src/Makefile +++ b/src/Makefile @@ -21,12 +21,13 @@ NODEPS:=clean distclean # Default settings STD=-std=c99 -pedantic -DREDIS_STATIC='' +CXX_STD=-std=c++14 -pedantic ifneq (,$(findstring clang,$(CC))) ifneq (,$(findstring FreeBSD,$(uname_S))) STD+=-Wno-c11-extensions endif endif -WARN=-Wall -W -Wno-missing-field-initializers +WARN=-Wall -Werror -W -Wno-missing-field-initializers OPT=$(OPTIMIZATION) PREFIX?=/usr/local @@ -73,6 +74,7 @@ endif -include .make-settings FINAL_CFLAGS=$(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(REDIS_CFLAGS) +FINAL_CXXFLAGS=$(CXX_STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(REDIS_CFLAGS) FINAL_LDFLAGS=$(LDFLAGS) $(REDIS_LDFLAGS) $(DEBUG) FINAL_LIBS=-lm DEBUG=-g -ggdb @@ -128,6 +130,7 @@ endif endif # Include paths to dependencies FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src +FINAL_CXXFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src ifeq ($(MALLOC),tcmalloc) FINAL_CFLAGS+= -DUSE_TCMALLOC @@ -152,7 +155,8 @@ ifeq ($(MALLOC),memkind) endif REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS) -REDIS_LD=$(QUIET_LINK)$(CC) $(FINAL_LDFLAGS) +REDIS_CXX=$(QUIET_CC)$(CC) $(FINAL_CXXFLAGS) +REDIS_LD=$(QUIET_LINK)$(CXX) $(FINAL_LDFLAGS) REDIS_INSTALL=$(QUIET_INSTALL)$(INSTALL) CCCOLOR="\033[34m" @@ -170,7 +174,7 @@ endif REDIS_SERVER_NAME=redis-server REDIS_SENTINEL_NAME=redis-sentinel -REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o acl.o storage.o +REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o acl.o storage.o rdb-s3.o REDIS_CLI_NAME=redis-cli REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o siphash.o crc16.o storage-lite.o REDIS_BENCHMARK_NAME=redis-benchmark @@ -253,6 +257,9 @@ dict-benchmark: dict.c zmalloc.c sds.c siphash.c %.o: %.c .make-prerequisites $(REDIS_CC) -c $< +%.o: %.cpp .make-prerequisites + $(REDIS_CXX) -c $< + clean: rm -rf $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov redis.info lcov-html Makefile.dep dict-benchmark diff --git a/src/acl.c b/src/acl.c index 6b6274098..a63b8696f 100644 --- a/src/acl.c +++ b/src/acl.c @@ -580,7 +580,7 @@ user *ACLGetUserByName(const char *name, size_t namelen) { * command, the second if the command is denied because the user is trying * to access keys that are not among the specified patterns. */ int ACLCheckCommandPerm(client *c) { - user *u = c->user; + user *u = c->puser; uint64_t id = c->cmd->id; /* If there is no associated user, the connection can run anything. */ @@ -615,7 +615,7 @@ int ACLCheckCommandPerm(client *c) { /* Check if the user can execute commands explicitly touching the keys * mentioned in the command arguments. */ - if (!(c->user->flags & USER_FLAG_ALLKEYS) && + if (!(c->puser->flags & USER_FLAG_ALLKEYS) && (c->cmd->getkeys_proc || c->cmd->firstkey)) { int numkeys; @@ -684,8 +684,8 @@ void aclCommand(client *c) { } addReply(c,shared.ok); } else if (!strcasecmp(sub,"whoami")) { - if (c->user != NULL) { - addReplyBulkCBuffer(c,c->user->name,sdslen(c->user->name)); + if (c->puser != NULL) { + addReplyBulkCBuffer(c,c->puser->name,sdslen(c->puser->name)); } else { addReplyNull(c); } diff --git a/src/aof.c b/src/aof.c index ed9da54ef..1a7ad4072 100644 --- a/src/aof.c +++ b/src/aof.c @@ -648,7 +648,7 @@ struct client *createFakeClient(void) { c->watched_keys = listCreate(); c->peerid = NULL; c->resp = 2; - c->user = NULL; + c->puser = NULL; listSetFreeMethod(c->reply,freeClientReplyValue); listSetDupMethod(c->reply,dupClientReplyValue); initClientMultiState(c); @@ -717,7 +717,7 @@ int loadAppendOnlyFile(char *filename) { serverLog(LL_NOTICE,"Reading RDB preamble from AOF file..."); if (fseek(fp,0,SEEK_SET) == -1) goto readerr; - rioInitWithFile(&rdb,fp); + rioInitWithFile(&rdb,fileno(fp)); if (rdbLoadRio(&rdb,NULL,1) != C_OK) { serverLog(LL_WARNING,"Error reading the RDB preamble of the AOF file, AOF loading aborted"); goto readerr; @@ -1017,7 +1017,7 @@ int rewriteSortedSetObject(rio *r, robj *key, robj *o) { } } else if (o->encoding == OBJ_ENCODING_SKIPLIST) { zset *zs = o->ptr; - dictIterator *di = dictGetIterator(zs->dict); + dictIterator *di = dictGetIterator(zs->pdict); dictEntry *de; while((de = dictNext(di)) != NULL) { @@ -1272,7 +1272,7 @@ int rewriteAppendOnlyFileRio(rio *aof) { for (j = 0; j < server.dbnum; j++) { char selectcmd[] = "*2\r\n$6\r\nSELECT\r\n"; redisDb *db = server.db+j; - dict *d = db->dict; + dict *d = db->pdict; if (dictSize(d) == 0) continue; di = dictGetSafeIterator(d); @@ -1361,7 +1361,7 @@ int rewriteAppendOnlyFile(char *filename) { } server.aof_child_diff = sdsempty(); - rioInitWithFile(&aof,fp); + rioInitWithFile(&aof,fileno(fp)); if (server.aof_rewrite_incremental_fsync) rioSetAutoSync(&aof,REDIS_AUTOSYNC_BYTES); diff --git a/src/cluster.c b/src/cluster.c index 5b3d9663e..11703a359 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -3888,7 +3888,7 @@ int verifyClusterConfigWithData(void) { /* Make sure we only have keys in DB0. */ for (j = 1; j < server.dbnum; j++) { - if (dictSize(server.db[j].dict)) return C_ERR; + if (dictSize(server.db[j].pdict)) return C_ERR; } /* Check that all the slots we see populated memory have a corresponding @@ -4264,7 +4264,7 @@ NULL clusterReplyMultiBulkSlots(c); } else if (!strcasecmp(c->argv[1]->ptr,"flushslots") && c->argc == 2) { /* CLUSTER FLUSHSLOTS */ - if (dictSize(server.db[0].dict) != 0) { + if (dictSize(server.db[0].pdict) != 0) { addReplyError(c,"DB must be empty to perform CLUSTER FLUSHSLOTS."); return; } @@ -4599,7 +4599,7 @@ NULL * slots nor keys to accept to replicate some other node. * Slaves can switch to another master without issues. */ if (nodeIsMaster(myself) && - (myself->numslots != 0 || dictSize(server.db[0].dict) != 0)) { + (myself->numslots != 0 || dictSize(server.db[0].pdict) != 0)) { addReplyError(c, "To set a master the node must be empty and " "without assigned slots."); @@ -4756,7 +4756,7 @@ NULL /* Slaves can be reset while containing data, but not master nodes * that must be empty. */ - if (nodeIsMaster(myself) && dictSize(c->db->dict) != 0) { + if (nodeIsMaster(myself) && dictSize(c->db->pdict) != 0) { addReplyError(c,"CLUSTER RESET can't be called with " "master nodes containing keys"); return; diff --git a/src/db.c b/src/db.c index b44975b40..6ad033e91 100644 --- a/src/db.c +++ b/src/db.c @@ -53,7 +53,7 @@ void updateLFU(robj *val) { * implementations that should instead rely on lookupKeyRead(), * lookupKeyWrite() and lookupKeyReadWithFlags(). */ robj *lookupKey(redisDb *db, robj *key, int flags) { - dictEntry *de = dictFind(db->dict,key->ptr); + dictEntry *de = dictFind(db->pdict,key->ptr); if (de) { robj *val = dictGetVal(de); @@ -172,7 +172,7 @@ robj *lookupKeyWriteOrReply(client *c, robj *key, robj *reply) { * The program is aborted if the key already exists. */ void dbAdd(redisDb *db, robj *key, robj *val) { sds copy = sdsdup(key->ptr); - int retval = dictAdd(db->dict, copy, val); + int retval = dictAdd(db->pdict, copy, val); serverAssertWithInfo(NULL,key,retval == DICT_OK); if (val->type == OBJ_LIST || @@ -187,7 +187,7 @@ void dbAdd(redisDb *db, robj *key, robj *val) { * * The program is aborted if the key was not already present. */ void dbOverwrite(redisDb *db, robj *key, robj *val) { - dictEntry *de = dictFind(db->dict,key->ptr); + dictEntry *de = dictFind(db->pdict,key->ptr); serverAssertWithInfo(NULL,key,de != NULL); dictEntry auxentry = *de; @@ -195,14 +195,14 @@ void dbOverwrite(redisDb *db, robj *key, robj *val) { if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) { val->lru = old->lru; } - dictSetVal(db->dict, de, val); + dictSetVal(db->pdict, de, val); if (server.lazyfree_lazy_server_del) { freeObjAsync(old); - dictSetVal(db->dict, &auxentry, NULL); + dictSetVal(db->pdict, &auxentry, NULL); } - dictFreeVal(db->dict, &auxentry); + dictFreeVal(db->pdict, &auxentry); } /* High level Set operation. This function can be used in order to set @@ -225,7 +225,7 @@ void setKey(redisDb *db, robj *key, robj *val) { } int dbExists(redisDb *db, robj *key) { - return dictFind(db->dict,key->ptr) != NULL; + return dictFind(db->pdict,key->ptr) != NULL; } /* Return a random key, in form of a Redis object. @@ -235,13 +235,13 @@ int dbExists(redisDb *db, robj *key) { robj *dbRandomKey(redisDb *db) { dictEntry *de; int maxtries = 100; - int allvolatile = dictSize(db->dict) == dictSize(db->expires); + int allvolatile = dictSize(db->pdict) == dictSize(db->expires); while(1) { sds key; robj *keyobj; - de = dictGetRandomKey(db->dict); + de = dictGetRandomKey(db->pdict); if (de == NULL) return NULL; key = dictGetKey(de); @@ -272,7 +272,7 @@ int dbSyncDelete(redisDb *db, robj *key) { /* Deleting an entry from the expires dict will not free the sds of * the key, because it is shared with the main dictionary. */ if (dictSize(db->expires) > 0) dictDelete(db->expires,key->ptr); - if (dictDelete(db->dict,key->ptr) == DICT_OK) { + if (dictDelete(db->pdict,key->ptr) == DICT_OK) { if (server.cluster_enabled) slotToKeyDel(key); return 1; } else { @@ -357,11 +357,11 @@ long long emptyDb(int dbnum, int flags, void(callback)(void*)) { } for (int j = startdb; j <= enddb; j++) { - removed += dictSize(server.db[j].dict); + removed += dictSize(server.db[j].pdict); if (async) { emptyDbAsync(&server.db[j]); } else { - dictEmpty(server.db[j].dict,callback); + dictEmpty(server.db[j].pdict,callback); dictEmpty(server.db[j].expires,callback); } } @@ -538,7 +538,7 @@ void keysCommand(client *c) { unsigned long numkeys = 0; void *replylen = addReplyDeferredLen(c); - di = dictGetSafeIterator(c->db->dict); + di = dictGetSafeIterator(c->db->pdict); allkeys = (pattern[0] == '*' && pattern[1] == '\0'); while((de = dictNext(di)) != NULL) { sds key = dictGetKey(de); @@ -677,7 +677,7 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { /* Handle the case of a hash table. */ ht = NULL; if (o == NULL) { - ht = c->db->dict; + ht = c->db->pdict; } else if (o->type == OBJ_SET && o->encoding == OBJ_ENCODING_HT) { ht = o->ptr; } else if (o->type == OBJ_HASH && o->encoding == OBJ_ENCODING_HT) { @@ -685,7 +685,7 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { count *= 2; /* We return key / value for this type. */ } else if (o->type == OBJ_ZSET && o->encoding == OBJ_ENCODING_SKIPLIST) { zset *zs = o->ptr; - ht = zs->dict; + ht = zs->pdict; count *= 2; /* We return key / value for this type. */ } @@ -803,7 +803,7 @@ void scanCommand(client *c) { } void dbsizeCommand(client *c) { - addReplyLongLong(c,dictSize(c->db->dict)); + addReplyLongLong(c,dictSize(c->db->pdict)); } void lastsaveCommand(client *c) { @@ -1004,11 +1004,11 @@ int dbSwapDatabases(int id1, int id2) { /* Swap hash tables. Note that we don't swap blocking_keys, * ready_keys and watched_keys, since we want clients to * remain in the same DB they were. */ - db1->dict = db2->dict; + db1->pdict = db2->pdict; db1->expires = db2->expires; db1->avg_ttl = db2->avg_ttl; - db2->dict = aux.dict; + db2->pdict = aux.pdict; db2->expires = aux.expires; db2->avg_ttl = aux.avg_ttl; @@ -1062,7 +1062,7 @@ void swapdbCommand(client *c) { int removeExpire(redisDb *db, robj *key) { /* An expire may only be removed if there is a corresponding entry in the * main dict. Otherwise, the key will never be freed. */ - serverAssertWithInfo(NULL,key,dictFind(db->dict,key->ptr) != NULL); + serverAssertWithInfo(NULL,key,dictFind(db->pdict,key->ptr) != NULL); return dictDelete(db->expires,key->ptr) == DICT_OK; } @@ -1074,7 +1074,7 @@ void setExpire(client *c, redisDb *db, robj *key, long long when) { dictEntry *kde, *de; /* Reuse the sds from the main dict in the expire dict */ - kde = dictFind(db->dict,key->ptr); + kde = dictFind(db->pdict,key->ptr); serverAssertWithInfo(NULL,key,kde != NULL); de = dictAddOrFind(db->expires,dictGetKey(kde)); dictSetSignedIntegerVal(de,when); @@ -1095,7 +1095,7 @@ long long getExpire(redisDb *db, robj *key) { /* The entry was found in the expire dict, this means it should also * be present in the main dict (safety check). */ - serverAssertWithInfo(NULL,key,dictFind(db->dict,key->ptr) != NULL); + serverAssertWithInfo(NULL,key,dictFind(db->pdict,key->ptr) != NULL); return dictGetSignedIntegerVal(de); } diff --git a/src/debug.c b/src/debug.c index d544d8832..f2f7ac462 100644 --- a/src/debug.c +++ b/src/debug.c @@ -179,7 +179,7 @@ void xorObjectDigest(redisDb *db, robj *keyobj, unsigned char *digest, robj *o) } } else if (o->encoding == OBJ_ENCODING_SKIPLIST) { zset *zs = o->ptr; - dictIterator *di = dictGetIterator(zs->dict); + dictIterator *di = dictGetIterator(zs->pdict); dictEntry *de; while((de = dictNext(di)) != NULL) { @@ -267,8 +267,8 @@ void computeDatasetDigest(unsigned char *final) { for (j = 0; j < server.dbnum; j++) { redisDb *db = server.db+j; - if (dictSize(db->dict) == 0) continue; - di = dictGetSafeIterator(db->dict); + if (dictSize(db->pdict) == 0) continue; + di = dictGetSafeIterator(db->pdict); /* hash the DB id, so the same dataset moved in a different * DB will lead to a different digest */ @@ -388,7 +388,7 @@ NULL robj *val; char *strenc; - if ((de = dictFind(c->db->dict,c->argv[2]->ptr)) == NULL) { + if ((de = dictFind(c->db->pdict,c->argv[2]->ptr)) == NULL) { addReply(c,shared.nokeyerr); return; } @@ -440,7 +440,7 @@ NULL robj *val; sds key; - if ((de = dictFind(c->db->dict,c->argv[2]->ptr)) == NULL) { + if ((de = dictFind(c->db->pdict,c->argv[2]->ptr)) == NULL) { addReply(c,shared.nokeyerr); return; } @@ -480,7 +480,7 @@ NULL if (getLongFromObjectOrReply(c, c->argv[2], &keys, NULL) != C_OK) return; - dictExpand(c->db->dict,keys); + dictExpand(c->db->pdict,keys); for (j = 0; j < keys; j++) { long valsize = 0; snprintf(buf,sizeof(buf),"%s:%lu", @@ -631,7 +631,7 @@ NULL } stats = sdscatprintf(stats,"[Dictionary HT]\n"); - dictGetStats(buf,sizeof(buf),server.db[dbid].dict); + dictGetStats(buf,sizeof(buf),server.db[dbid].pdict); stats = sdscat(stats,buf); stats = sdscatprintf(stats,"[Expires HT]\n"); @@ -651,7 +651,7 @@ NULL case OBJ_ENCODING_SKIPLIST: { zset *zs = o->ptr; - ht = zs->dict; + ht = zs->pdict; } break; case OBJ_ENCODING_HT: @@ -1192,7 +1192,7 @@ void logCurrentClient(void) { dictEntry *de; key = getDecodedObject(cc->argv[1]); - de = dictFind(cc->db->dict, key->ptr); + de = dictFind(cc->db->pdict, key->ptr); if (de) { val = dictGetVal(de); serverLog(LL_WARNING,"key '%s' found in DB containing the following object:", (char*)key->ptr); diff --git a/src/defrag.c b/src/defrag.c index d67b6e253..a6d64d065 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -256,7 +256,7 @@ long activeDefragZsetEntry(zset *zs, dictEntry *de) { defragged++, de->key = newsds; newscore = zslDefrag(zs->zsl, *(double*)dictGetVal(de), sdsele, newsds); if (newscore) { - dictSetVal(zs->dict, de, newscore); + dictSetVal(zs->pdict, de, newscore); defragged++; } return defragged; @@ -464,7 +464,7 @@ long scanLaterZset(robj *ob, unsigned long *cursor) { if (ob->type != OBJ_ZSET || ob->encoding != OBJ_ENCODING_SKIPLIST) return 0; zset *zs = (zset*)ob->ptr; - dict *d = zs->dict; + dict *d = zs->pdict; scanLaterZsetData data = {zs, 0}; *cursor = dictScan(d, *cursor, scanLaterZsetCallback, defragDictBucketCallback, &data); return data.defragged; @@ -539,20 +539,20 @@ long defragZsetSkiplist(redisDb *db, dictEntry *kde) { defragged++, zs->zsl = newzsl; if ((newheader = activeDefragAlloc(zs->zsl->header))) defragged++, zs->zsl->header = newheader; - if (dictSize(zs->dict) > server.active_defrag_max_scan_fields) + if (dictSize(zs->pdict) > server.active_defrag_max_scan_fields) defragLater(db, kde); else { - dictIterator *di = dictGetIterator(zs->dict); + dictIterator *di = dictGetIterator(zs->pdict); while((de = dictNext(di)) != NULL) { defragged += activeDefragZsetEntry(zs, de); } dictReleaseIterator(di); } /* handle the dict struct */ - if ((newdict = activeDefragAlloc(zs->dict))) - defragged++, zs->dict = newdict; + if ((newdict = activeDefragAlloc(zs->pdict))) + defragged++, zs->pdict = newdict; /* defrag the dict tables */ - defragged += dictDefragTables(zs->dict); + defragged += dictDefragTables(zs->pdict); return defragged; } @@ -775,7 +775,7 @@ long defragKey(redisDb *db, dictEntry *de) { /* Dirty code: * I can't search in db->expires for that key after i already released * the pointer it holds it won't be able to do the string compare */ - uint64_t hash = dictGetHash(db->dict, de->key); + uint64_t hash = dictGetHash(db->pdict, de->key); replaceSateliteDictKeyPtrAndOrDefragDictEntry(db->expires, keysds, newsds, hash, &defragged); } @@ -953,7 +953,7 @@ int defragLaterStep(redisDb *db, long long endtime) { } /* each time we enter this function we need to fetch the key from the dict again (if it still exists) */ - dictEntry *de = dictFind(db->dict, current_key); + dictEntry *de = dictFind(db->pdict, current_key); key_defragged = server.stat_active_defrag_hits; do { int quit = 0; @@ -1106,7 +1106,7 @@ void activeDefragCycle(void) { break; /* this will exit the function and we'll continue on the next cycle */ } - cursor = dictScan(db->dict, cursor, defragScanCallback, defragDictBucketCallback, db); + cursor = dictScan(db->pdict, cursor, defragScanCallback, defragDictBucketCallback, db); /* Once in 16 scan iterations, 512 pointer reallocations. or 64 keys * (if we have a lot of pointers in one hash bucket or rehasing), diff --git a/src/evict.c b/src/evict.c index cd033e62f..28cd73f6f 100644 --- a/src/evict.c +++ b/src/evict.c @@ -489,9 +489,9 @@ int freeMemoryIfNeeded(void) { for (i = 0; i < server.dbnum; i++) { db = server.db+i; dict = (server.maxmemory_policy & MAXMEMORY_FLAG_ALLKEYS) ? - db->dict : db->expires; + db->pdict : db->expires; if ((keys = dictSize(dict)) != 0) { - evictionPoolPopulate(i, dict, db->dict, pool); + evictionPoolPopulate(i, dict, db->pdict, pool); total_keys += keys; } } @@ -503,7 +503,7 @@ int freeMemoryIfNeeded(void) { bestdbid = pool[k].dbid; if (server.maxmemory_policy & MAXMEMORY_FLAG_ALLKEYS) { - de = dictFind(server.db[pool[k].dbid].dict, + de = dictFind(server.db[pool[k].dbid].pdict, pool[k].key); } else { de = dictFind(server.db[pool[k].dbid].expires, @@ -539,7 +539,7 @@ int freeMemoryIfNeeded(void) { j = (++next_db) % server.dbnum; db = server.db+j; dict = (server.maxmemory_policy == MAXMEMORY_ALLKEYS_RANDOM) ? - db->dict : db->expires; + db->pdict : db->expires; if (dictSize(dict) != 0) { de = dictGetRandomKey(dict); bestkey = dictGetKey(de); diff --git a/src/fmacros.h b/src/fmacros.h index 6e56c759d..3b1bc5eb8 100644 --- a/src/fmacros.h +++ b/src/fmacros.h @@ -33,9 +33,11 @@ #define _BSD_SOURCE #if defined(__linux__) +#ifndef __cplusplus #define _GNU_SOURCE #define _DEFAULT_SOURCE #endif +#endif #if defined(_AIX) #define _ALL_SOURCE @@ -55,7 +57,9 @@ #define _POSIX_C_SOURCE 199506L #endif +#ifndef _LARGEFILE_SOURCE #define _LARGEFILE_SOURCE +#endif #define _FILE_OFFSET_BITS 64 #endif diff --git a/src/geo.c b/src/geo.c index 8e89dbefb..dace5542a 100644 --- a/src/geo.c +++ b/src/geo.c @@ -651,7 +651,7 @@ void georadiusGeneric(client *c, int flags) { if (maxelelen < elelen) maxelelen = elelen; znode = zslInsert(zs->zsl,score,gp->member); - serverAssert(dictAdd(zs->dict,gp->member,&znode->score) == DICT_OK); + serverAssert(dictAdd(zs->pdict,gp->member,&znode->score) == DICT_OK); gp->member = NULL; } diff --git a/src/intset.h b/src/intset.h index 8119e6636..6849abff9 100644 --- a/src/intset.h +++ b/src/intset.h @@ -32,10 +32,16 @@ #define __INTSET_H #include +#ifdef __cplusplus +#define ZERO_LENGTH_ARRAY_LENGTH 1 +#else +#define ZERO_LENGTH_ARRAY_LENGTH +#endif + typedef struct intset { uint32_t encoding; uint32_t length; - int8_t contents[]; + int8_t contents[ZERO_LENGTH_ARRAY_LENGTH]; } intset; intset *intsetNew(void); diff --git a/src/lazyfree.c b/src/lazyfree.c index 3d3159c90..74c71beb8 100644 --- a/src/lazyfree.c +++ b/src/lazyfree.c @@ -59,7 +59,7 @@ int dbAsyncDelete(redisDb *db, robj *key) { /* If the value is composed of a few allocations, to free in a lazy way * is actually just slower... So under a certain limit we just free * the object synchronously. */ - dictEntry *de = dictUnlink(db->dict,key->ptr); + dictEntry *de = dictUnlink(db->pdict,key->ptr); if (de) { robj *val = dictGetVal(de); size_t free_effort = lazyfreeGetFreeEffort(val); @@ -75,14 +75,14 @@ int dbAsyncDelete(redisDb *db, robj *key) { if (free_effort > LAZYFREE_THRESHOLD && val->refcount == 1) { atomicIncr(lazyfree_objects,1); bioCreateBackgroundJob(BIO_LAZY_FREE,val,NULL,NULL); - dictSetVal(db->dict,de,NULL); + dictSetVal(db->pdict,de,NULL); } } /* Release the key-val pair, or just the key if we set the val * field to NULL in order to lazy free it later. */ if (de) { - dictFreeUnlinkedEntry(db->dict,de); + dictFreeUnlinkedEntry(db->pdict,de); if (server.cluster_enabled) slotToKeyDel(key); return 1; } else { @@ -105,8 +105,8 @@ void freeObjAsync(robj *o) { * create a new empty set of hash tables and scheduling the old ones for * lazy freeing. */ void emptyDbAsync(redisDb *db) { - dict *oldht1 = db->dict, *oldht2 = db->expires; - db->dict = dictCreate(&dbDictType,NULL); + dict *oldht1 = db->pdict, *oldht2 = db->expires; + db->pdict = dictCreate(&dbDictType,NULL); db->expires = dictCreate(&keyptrDictType,NULL); atomicIncr(lazyfree_objects,dictSize(oldht1)); bioCreateBackgroundJob(BIO_LAZY_FREE,NULL,oldht1,oldht2); diff --git a/src/module.c b/src/module.c index e73494149..b8428a5ff 100644 --- a/src/module.c +++ b/src/module.c @@ -3074,11 +3074,11 @@ void moduleRDBLoadError(RedisModuleIO *io) { void RM_SaveUnsigned(RedisModuleIO *io, uint64_t value) { if (io->error) return; /* Save opcode. */ - int retval = rdbSaveLen(io->rio, RDB_MODULE_OPCODE_UINT); + int retval = rdbSaveLen(io->prio, RDB_MODULE_OPCODE_UINT); if (retval == -1) goto saveerr; io->bytes += retval; /* Save value. */ - retval = rdbSaveLen(io->rio, value); + retval = rdbSaveLen(io->prio, value); if (retval == -1) goto saveerr; io->bytes += retval; return; @@ -3092,11 +3092,11 @@ saveerr: * new data types. */ uint64_t RM_LoadUnsigned(RedisModuleIO *io) { if (io->ver == 2) { - uint64_t opcode = rdbLoadLen(io->rio,NULL); + uint64_t opcode = rdbLoadLen(io->prio,NULL); if (opcode != RDB_MODULE_OPCODE_UINT) goto loaderr; } uint64_t value; - int retval = rdbLoadLenByRef(io->rio, NULL, &value); + int retval = rdbLoadLenByRef(io->prio, NULL, &value); if (retval == -1) goto loaderr; return value; @@ -3128,11 +3128,11 @@ int64_t RM_LoadSigned(RedisModuleIO *io) { void RM_SaveString(RedisModuleIO *io, RedisModuleString *s) { if (io->error) return; /* Save opcode. */ - ssize_t retval = rdbSaveLen(io->rio, RDB_MODULE_OPCODE_STRING); + ssize_t retval = rdbSaveLen(io->prio, RDB_MODULE_OPCODE_STRING); if (retval == -1) goto saveerr; io->bytes += retval; /* Save value. */ - retval = rdbSaveStringObject(io->rio, s); + retval = rdbSaveStringObject(io->prio, s); if (retval == -1) goto saveerr; io->bytes += retval; return; @@ -3146,11 +3146,11 @@ saveerr: void RM_SaveStringBuffer(RedisModuleIO *io, const char *str, size_t len) { if (io->error) return; /* Save opcode. */ - ssize_t retval = rdbSaveLen(io->rio, RDB_MODULE_OPCODE_STRING); + ssize_t retval = rdbSaveLen(io->prio, RDB_MODULE_OPCODE_STRING); if (retval == -1) goto saveerr; io->bytes += retval; /* Save value. */ - retval = rdbSaveRawString(io->rio, (unsigned char*)str,len); + retval = rdbSaveRawString(io->prio, (unsigned char*)str,len); if (retval == -1) goto saveerr; io->bytes += retval; return; @@ -3162,10 +3162,10 @@ saveerr: /* Implements RM_LoadString() and RM_LoadStringBuffer() */ void *moduleLoadString(RedisModuleIO *io, int plain, size_t *lenptr) { if (io->ver == 2) { - uint64_t opcode = rdbLoadLen(io->rio,NULL); + uint64_t opcode = rdbLoadLen(io->prio,NULL); if (opcode != RDB_MODULE_OPCODE_STRING) goto loaderr; } - void *s = rdbGenericLoadStringObject(io->rio, + void *s = rdbGenericLoadStringObject(io->prio, plain ? RDB_LOAD_PLAIN : RDB_LOAD_NONE, lenptr); if (s == NULL) goto loaderr; return s; @@ -3205,11 +3205,11 @@ char *RM_LoadStringBuffer(RedisModuleIO *io, size_t *lenptr) { void RM_SaveDouble(RedisModuleIO *io, double value) { if (io->error) return; /* Save opcode. */ - int retval = rdbSaveLen(io->rio, RDB_MODULE_OPCODE_DOUBLE); + int retval = rdbSaveLen(io->prio, RDB_MODULE_OPCODE_DOUBLE); if (retval == -1) goto saveerr; io->bytes += retval; /* Save value. */ - retval = rdbSaveBinaryDoubleValue(io->rio, value); + retval = rdbSaveBinaryDoubleValue(io->prio, value); if (retval == -1) goto saveerr; io->bytes += retval; return; @@ -3222,11 +3222,11 @@ saveerr: * double value saved by RedisModule_SaveDouble(). */ double RM_LoadDouble(RedisModuleIO *io) { if (io->ver == 2) { - uint64_t opcode = rdbLoadLen(io->rio,NULL); + uint64_t opcode = rdbLoadLen(io->prio,NULL); if (opcode != RDB_MODULE_OPCODE_DOUBLE) goto loaderr; } double value; - int retval = rdbLoadBinaryDoubleValue(io->rio, &value); + int retval = rdbLoadBinaryDoubleValue(io->prio, &value); if (retval == -1) goto loaderr; return value; @@ -3241,11 +3241,11 @@ loaderr: void RM_SaveFloat(RedisModuleIO *io, float value) { if (io->error) return; /* Save opcode. */ - int retval = rdbSaveLen(io->rio, RDB_MODULE_OPCODE_FLOAT); + int retval = rdbSaveLen(io->prio, RDB_MODULE_OPCODE_FLOAT); if (retval == -1) goto saveerr; io->bytes += retval; /* Save value. */ - retval = rdbSaveBinaryFloatValue(io->rio, value); + retval = rdbSaveBinaryFloatValue(io->prio, value); if (retval == -1) goto saveerr; io->bytes += retval; return; @@ -3258,11 +3258,11 @@ saveerr: * float value saved by RedisModule_SaveFloat(). */ float RM_LoadFloat(RedisModuleIO *io) { if (io->ver == 2) { - uint64_t opcode = rdbLoadLen(io->rio,NULL); + uint64_t opcode = rdbLoadLen(io->prio,NULL); if (opcode != RDB_MODULE_OPCODE_FLOAT) goto loaderr; } float value; - int retval = rdbLoadBinaryFloatValue(io->rio, &value); + int retval = rdbLoadBinaryFloatValue(io->prio, &value); if (retval == -1) goto loaderr; return value; @@ -3373,12 +3373,12 @@ void RM_EmitAOF(RedisModuleIO *io, const char *cmdname, const char *fmt, ...) { } /* Bulk count. */ - if (!io->error && rioWriteBulkCount(io->rio,'*',argc) == 0) + if (!io->error && rioWriteBulkCount(io->prio,'*',argc) == 0) io->error = 1; /* Arguments. */ for (j = 0; j < argc; j++) { - if (!io->error && rioWriteBulkObject(io->rio,argv[j]) == 0) + if (!io->error && rioWriteBulkObject(io->prio,argv[j]) == 0) io->error = 1; decrRefCount(argv[j]); } diff --git a/src/multi.c b/src/multi.c index 6aff069a3..aec913a0a 100644 --- a/src/multi.c +++ b/src/multi.c @@ -329,7 +329,7 @@ void touchWatchedKeysOnFlush(int dbid) { * key exists, mark the client as dirty, as the key will be * removed. */ if (dbid == -1 || wk->db->id == dbid) { - if (dictFind(wk->db->dict, wk->key->ptr) != NULL) + if (dictFind(wk->db->pdict, wk->key->ptr) != NULL) c->flags |= CLIENT_DIRTY_CAS; } } diff --git a/src/networking.c b/src/networking.c index d438e70b8..3b43707e2 100644 --- a/src/networking.c +++ b/src/networking.c @@ -119,7 +119,7 @@ client *createClient(int fd) { c->argc = 0; c->argv = NULL; c->cmd = c->lastcmd = NULL; - c->user = DefaultUser; + c->puser = DefaultUser; c->multibulklen = 0; c->bulklen = -1; c->sentlen = 0; @@ -127,7 +127,7 @@ client *createClient(int fd) { c->ctime = c->lastinteraction = server.unixtime; /* If the default user does not require authentication, the user is * directly authenticated. */ - c->authenticated = (c->user->flags & USER_FLAG_NOPASS) != 0; + c->authenticated = (c->puser->flags & USER_FLAG_NOPASS) != 0; c->replstate = REPL_STATE_NONE; c->repl_put_online_on_ack = 0; c->reploff = 0; diff --git a/src/object.c b/src/object.c index 2d4095942..028d0783a 100644 --- a/src/object.c +++ b/src/object.c @@ -250,7 +250,7 @@ robj *createZsetObject(void) { zset *zs = zmalloc(sizeof(*zs), MALLOC_SHARED); robj *o; - zs->dict = dictCreate(&zsetDictType,NULL); + zs->pdict = dictCreate(&zsetDictType,NULL); zs->zsl = zslCreate(); o = createObject(OBJ_ZSET,zs); o->encoding = OBJ_ENCODING_SKIPLIST; @@ -310,7 +310,7 @@ void freeZsetObject(robj *o) { switch (o->encoding) { case OBJ_ENCODING_SKIPLIST: zs = o->ptr; - dictRelease(zs->dict); + dictRelease(zs->pdict); zslFree(zs->zsl); zfree(zs); break; @@ -823,7 +823,7 @@ size_t objectComputeSize(robj *o, size_t sample_size) { if (o->encoding == OBJ_ENCODING_ZIPLIST) { asize = sizeof(*o)+(ziplistBlobLen(o->ptr)); } else if (o->encoding == OBJ_ENCODING_SKIPLIST) { - d = ((zset*)o->ptr)->dict; + d = ((zset*)o->ptr)->pdict; zskiplist *zsl = ((zset*)o->ptr)->zsl; zskiplistNode *znode = zsl->header->level[0].forward; asize = sizeof(*o)+sizeof(zset)+(sizeof(struct dictEntry*)*dictSlots(d)); @@ -859,14 +859,14 @@ size_t objectComputeSize(robj *o, size_t sample_size) { } else if (o->type == OBJ_STREAM) { stream *s = o->ptr; asize = sizeof(*o); - asize += streamRadixTreeMemoryUsage(s->rax); + asize += streamRadixTreeMemoryUsage(s->prax); /* Now we have to add the listpacks. The last listpack is often non * complete, so we estimate the size of the first N listpacks, and * use the average to compute the size of the first N-1 listpacks, and * finally add the real size of the last node. */ raxIterator ri; - raxStart(&ri,s->rax); + raxStart(&ri,s->prax); raxSeek(&ri,"^",NULL,0); size_t lpsize = 0, samples = 0; while(samples < sample_size && raxNext(&ri)) { @@ -874,11 +874,11 @@ size_t objectComputeSize(robj *o, size_t sample_size) { lpsize += lpBytes(lp); samples++; } - if (s->rax->numele <= samples) { + if (s->prax->numele <= samples) { asize += lpsize; } else { if (samples) lpsize /= samples; /* Compute the average. */ - asize += lpsize * (s->rax->numele-1); + asize += lpsize * (s->prax->numele-1); /* No need to check if seek succeeded, we enter this branch only * if there are a few elements in the radix tree. */ raxSeek(&ri,"$",NULL,0); @@ -1031,16 +1031,16 @@ struct redisMemOverhead *getMemoryOverheadData(void) { for (j = 0; j < server.dbnum; j++) { redisDb *db = server.db+j; - long long keyscount = dictSize(db->dict); + long long keyscount = dictSize(db->pdict); if (keyscount==0) continue; mh->total_keys += keyscount; mh->db = zrealloc(mh->db,sizeof(mh->db[0])*(mh->num_dbs+1), MALLOC_LOCAL); mh->db[mh->num_dbs].dbid = j; - mem = dictSize(db->dict) * sizeof(dictEntry) + - dictSlots(db->dict) * sizeof(dictEntry*) + - dictSize(db->dict) * sizeof(robj); + mem = dictSize(db->pdict) * sizeof(dictEntry) + + dictSlots(db->pdict) * sizeof(dictEntry*) + + dictSize(db->pdict) * sizeof(robj); mh->db[mh->num_dbs].overhead_ht_main = mem; mem_total+=mem; @@ -1222,7 +1222,7 @@ void objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle, robj *objectCommandLookup(client *c, robj *key) { dictEntry *de; - if ((de = dictFind(c->db->dict,key->ptr)) == NULL) return NULL; + if ((de = dictFind(c->db->pdict,key->ptr)) == NULL) return NULL; return (robj*) dictGetVal(de); } @@ -1315,7 +1315,7 @@ NULL return; } } - if ((de = dictFind(c->db->dict,c->argv[2]->ptr)) == NULL) { + if ((de = dictFind(c->db->pdict,c->argv[2]->ptr)) == NULL) { addReplyNull(c); return; } diff --git a/src/pubsub.c b/src/pubsub.c index a7ee67400..bade59d78 100644 --- a/src/pubsub.c +++ b/src/pubsub.c @@ -130,7 +130,7 @@ void freePubsubPattern(void *p) { int listMatchPubsubPattern(void *a, void *b) { pubsubPattern *pa = a, *pb = b; - return (pa->client == pb->client) && + return (pa->pclient == pb->pclient) && (equalStringObjects(pa->pattern,pb->pattern)); } @@ -211,7 +211,7 @@ int pubsubSubscribePattern(client *c, robj *pattern) { incrRefCount(pattern); pat = zmalloc(sizeof(*pat), MALLOC_LOCAL); pat->pattern = getDecodedObject(pattern); - pat->client = c; + pat->pclient = c; listAddNodeTail(server.pubsub_patterns,pat); } /* Notify the client */ @@ -230,7 +230,7 @@ int pubsubUnsubscribePattern(client *c, robj *pattern, int notify) { if ((ln = listSearchKey(c->pubsub_patterns,pattern)) != NULL) { retval = 1; listDelNode(c->pubsub_patterns,ln); - pat.client = c; + pat.pclient = c; pat.pattern = pattern; ln = listSearchKey(server.pubsub_patterns,&pat); listDelNode(server.pubsub_patterns,ln); @@ -309,7 +309,7 @@ int pubsubPublishMessage(robj *channel, robj *message) { (char*)channel->ptr, sdslen(channel->ptr),0)) { - addReplyPubsubPatMessage(pat->client, + addReplyPubsubPatMessage(pat->pclient, pat->pattern,channel,message); receivers++; } diff --git a/src/quicklist.c b/src/quicklist.c index 02771ef00..2f5eab215 100644 --- a/src/quicklist.c +++ b/src/quicklist.c @@ -75,7 +75,7 @@ static const size_t optimization_level[] = {4096, 8192, 16384, 32768, 65536}; do { \ (e)->zi = (e)->value = NULL; \ (e)->longval = -123456789; \ - (e)->quicklist = NULL; \ + (e)->qlist = NULL; \ (e)->node = NULL; \ (e)->offset = 123456789; \ (e)->sz = 0; \ @@ -634,7 +634,7 @@ REDIS_STATIC int quicklistDelIndex(quicklist *quicklist, quicklistNode *node, void quicklistDelEntry(quicklistIter *iter, quicklistEntry *entry) { quicklistNode *prev = entry->node->prev; quicklistNode *next = entry->node->next; - int deleted_node = quicklistDelIndex((quicklist *)entry->quicklist, + int deleted_node = quicklistDelIndex((quicklist *)entry->qlist, entry->node, &entry->zi); /* after delete, the zi is now invalid for any future usage. */ @@ -1059,7 +1059,7 @@ quicklistIter *quicklistGetIterator(const quicklist *quicklist, int direction) { } iter->direction = direction; - iter->quicklist = quicklist; + iter->qlist = quicklist; iter->zi = NULL; @@ -1088,7 +1088,7 @@ quicklistIter *quicklistGetIteratorAtIdx(const quicklist *quicklist, * If we still have a valid current node, then re-encode current node. */ void quicklistReleaseIterator(quicklistIter *iter) { if (iter->current) - quicklistCompress(iter->quicklist, iter->current); + quicklistCompress(iter->qlist, iter->current); zfree(iter); } @@ -1122,7 +1122,7 @@ int quicklistNext(quicklistIter *iter, quicklistEntry *entry) { return 0; } - entry->quicklist = iter->quicklist; + entry->qlist = iter->qlist; entry->node = iter->current; if (!iter->current) { @@ -1160,7 +1160,7 @@ int quicklistNext(quicklistIter *iter, quicklistEntry *entry) { } else { /* We ran out of ziplist entries. * Pick next node, update offset, then re-run retrieval. */ - quicklistCompress(iter->quicklist, iter->current); + quicklistCompress(iter->qlist, iter->current); if (iter->direction == AL_START_HEAD) { /* Forward traversal */ D("Jumping to start of next node"); @@ -1230,7 +1230,7 @@ int quicklistIndex(const quicklist *quicklist, const long long idx, int forward = idx < 0 ? 0 : 1; /* < 0 -> reverse, 0+ -> forward */ initEntry(entry); - entry->quicklist = quicklist; + entry->qlist = quicklist; if (!forward) { index = (-idx) - 1; diff --git a/src/quicklist.h b/src/quicklist.h index a7e27a2dd..165da0877 100644 --- a/src/quicklist.h +++ b/src/quicklist.h @@ -31,6 +31,12 @@ #ifndef __QUICKLIST_H__ #define __QUICKLIST_H__ +#ifdef __cplusplus +#define ZERO_LENGTH_ARRAY_LENGTH 1 +#else +#define ZERO_LENGTH_ARRAY_LENGTH +#endif + /* Node, quicklist, and Iterator are the only data structures used currently. */ /* quicklistNode is a 32 byte struct describing a ziplist for a quicklist. @@ -61,7 +67,7 @@ typedef struct quicklistNode { * When quicklistNode->zl is compressed, node->zl points to a quicklistLZF */ typedef struct quicklistLZF { unsigned int sz; /* LZF size in bytes*/ - char compressed[]; + char compressed[ZERO_LENGTH_ARRAY_LENGTH]; } quicklistLZF; /* quicklist is a 40 byte struct (on 64-bit systems) describing a quicklist. @@ -80,7 +86,7 @@ typedef struct quicklist { } quicklist; typedef struct quicklistIter { - const quicklist *quicklist; + const quicklist *qlist; quicklistNode *current; unsigned char *zi; long offset; /* offset in current ziplist */ @@ -88,7 +94,7 @@ typedef struct quicklistIter { } quicklistIter; typedef struct quicklistEntry { - const quicklist *quicklist; + const quicklist *qlist; quicklistNode *node; unsigned char *zi; unsigned char *value; diff --git a/src/rax.h b/src/rax.h index f2521d14a..737f1cbb1 100644 --- a/src/rax.h +++ b/src/rax.h @@ -31,6 +31,12 @@ #ifndef RAX_H #define RAX_H +#ifdef __cplusplus +#define ZERO_LENGTH_ARRAY_LENGTH 1 +#else +#define ZERO_LENGTH_ARRAY_LENGTH +#endif + #include /* Representation of a radix tree as implemented in this file, that contains @@ -127,7 +133,7 @@ typedef struct raxNode { * children, an additional value pointer is present (as you can see * in the representation above as "value-ptr" field). */ - unsigned char data[]; + unsigned char data[ZERO_LENGTH_ARRAY_LENGTH]; } raxNode; typedef struct rax { diff --git a/src/rdb-s3.cpp b/src/rdb-s3.cpp new file mode 100644 index 000000000..294498efe --- /dev/null +++ b/src/rdb-s3.cpp @@ -0,0 +1,13 @@ +extern "C" { +#include "rio.h" +#include "server.h" +} + +/* Save the DB on disk. Return C_ERR on error, C_OK on success. */ +extern "C" int rdbSaveS3(char *s3bucket, rdbSaveInfo *rsi) +{ + (void)s3bucket; + (void)rsi; + // NOP + return C_ERR; +} \ No newline at end of file diff --git a/src/rdb.c b/src/rdb.c index 4065ca11b..ab157c630 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -895,7 +895,7 @@ ssize_t rdbSaveObject(rio *rdb, robj *o) { } else if (o->type == OBJ_STREAM) { /* Store how many listpacks we have inside the radix tree. */ stream *s = o->ptr; - rax *rax = s->rax; + rax *rax = s->prax; if ((n = rdbSaveLen(rdb,raxSize(rax))) == -1) return -1; nwritten += n; @@ -1116,7 +1116,7 @@ int rdbSaveRio(rio *rdb, int *error, int flags, rdbSaveInfo *rsi) { for (j = 0; j < server.dbnum; j++) { redisDb *db = server.db+j; - dict *d = db->dict; + dict *d = db->pdict; if (dictSize(d) == 0) continue; di = dictGetSafeIterator(d); @@ -1129,7 +1129,7 @@ int rdbSaveRio(rio *rdb, int *error, int flags, rdbSaveInfo *rsi) { * However this does not limit the actual size of the DB to load since * these sizes are just hints to resize the hash tables. */ uint64_t db_size, expires_size; - db_size = dictSize(db->dict); + db_size = dictSize(db->pdict); expires_size = dictSize(db->expires); if (rdbSaveType(rdb,RDB_OPCODE_RESIZEDB) == -1) goto werr; if (rdbSaveLen(rdb,db_size) == -1) goto werr; @@ -1216,13 +1216,28 @@ werr: /* Write error. */ return C_ERR; } +int rdbSaveFd(int fd, rdbSaveInfo *rsi) +{ + int error = 0; + rio rdb; + + rioInitWithFile(&rdb,fd); + + if (server.rdb_save_incremental_fsync) + rioSetAutoSync(&rdb,REDIS_AUTOSYNC_BYTES); + + if (rdbSaveRio(&rdb,&error,RDB_SAVE_NONE,rsi) == C_ERR) { + errno = error; + return C_ERR; + } + return C_OK; +} + /* Save the DB on disk. Return C_ERR on error, C_OK on success. */ int rdbSave(char *filename, rdbSaveInfo *rsi) { char tmpfile[256]; char cwd[MAXPATHLEN]; /* Current working dir path for error messages. */ FILE *fp; - rio rdb; - int error = 0; snprintf(tmpfile,256,"temp-%d.rdb", (int) getpid()); fp = fopen(tmpfile,"w"); @@ -1237,13 +1252,7 @@ int rdbSave(char *filename, rdbSaveInfo *rsi) { return C_ERR; } - rioInitWithFile(&rdb,fp); - - if (server.rdb_save_incremental_fsync) - rioSetAutoSync(&rdb,REDIS_AUTOSYNC_BYTES); - - if (rdbSaveRio(&rdb,&error,RDB_SAVE_NONE,rsi) == C_ERR) { - errno = error; + if (rdbSaveFd(fileno(fp), rsi) == C_ERR){ goto werr; } @@ -1459,7 +1468,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { zs = o->ptr; if (zsetlen > DICT_HT_INITIAL_SIZE) - dictExpand(zs->dict,zsetlen); + dictExpand(zs->pdict,zsetlen); /* Load every single element of the sorted set. */ while(zsetlen--) { @@ -1480,7 +1489,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { if (sdslen(sdsele) > maxelelen) maxelelen = sdslen(sdsele); znode = zslInsert(zs->zsl,score,sdsele); - dictAdd(zs->dict,sdsele,&znode->score); + dictAdd(zs->pdict,sdsele,&znode->score); } /* Convert *after* loading, since sorted sets are not stored ordered. */ @@ -1667,7 +1676,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { } /* Insert the key in the radix tree. */ - int retval = raxInsert(s->rax, + int retval = raxInsert(s->prax, (unsigned char*)nodekey,sizeof(streamID),lp,NULL); sdsfree(nodekey); if (!retval) @@ -1928,7 +1937,7 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof) { goto eoferr; if ((expires_size = rdbLoadLen(rdb,NULL)) == RDB_LENERR) goto eoferr; - dictExpand(db->dict,db_size); + dictExpand(db->pdict,db_size); dictExpand(db->expires,expires_size); continue; /* Read next opcode. */ } else if (type == RDB_OPCODE_AUX) { @@ -2074,7 +2083,7 @@ int rdbLoad(char *filename, rdbSaveInfo *rsi) { if ((fp = fopen(filename,"r")) == NULL) return C_ERR; startLoading(fp); - rioInitWithFile(&rdb,fp); + rioInitWithFile(&rdb,fileno(fp)); retval = rdbLoadRio(&rdb,rsi,0); fclose(fp); stopLoading(); diff --git a/src/rdb.h b/src/rdb.h index 7b9486169..be1a7668e 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -140,6 +140,7 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi); int rdbSaveToSlavesSockets(rdbSaveInfo *rsi); void rdbRemoveTempFile(pid_t childpid); int rdbSave(char *filename, rdbSaveInfo *rsi); +int rdbSaveFd(int fd, rdbSaveInfo *rsi); ssize_t rdbSaveObject(rio *rdb, robj *o); size_t rdbSavedObjectLen(robj *o); robj *rdbLoadObject(int type, rio *rdb); diff --git a/src/redis-check-rdb.c b/src/redis-check-rdb.c index 8de1d8f48..93fd6cc5a 100644 --- a/src/redis-check-rdb.c +++ b/src/redis-check-rdb.c @@ -186,7 +186,7 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) { int closefile = (fp == NULL); if (fp == NULL && (fp = fopen(rdbfilename,"r")) == NULL) return 1; - rioInitWithFile(&rdb,fp); + rioInitWithFile(&rdb,fileno(fp)); rdbstate.rio = &rdb; rdb.update_cksum = rdbLoadProgressCallback; if (rioRead(&rdb,buf,9) == 0) goto eoferr; diff --git a/src/replication.c b/src/replication.c index 103af2691..ea433326e 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1080,7 +1080,7 @@ void replicationCreateMasterClient(int fd, int dbid) { server.master->authenticated = 1; server.master->reploff = server.master_initial_offset; server.master->read_reploff = server.master->reploff; - server.master->user = NULL; /* This client can do everything. */ + server.master->puser = NULL; /* This client can do everything. */ memcpy(server.master->replid, server.master_replid, sizeof(server.master_replid)); /* If master offset is set to -1, this master is old and is not diff --git a/src/rio.c b/src/rio.c index 0f44b017d..8dcb77721 100644 --- a/src/rio.c +++ b/src/rio.c @@ -109,14 +109,13 @@ void rioInitWithBuffer(rio *r, sds s) { static size_t rioFileWrite(rio *r, const void *buf, size_t len) { size_t retval; - retval = fwrite(buf,len,1,r->io.file.fp); + retval = write(r->io.file.fd,buf,len); r->io.file.buffered += len; if (r->io.file.autosync && r->io.file.buffered >= r->io.file.autosync) { - fflush(r->io.file.fp); - redis_fsync(fileno(r->io.file.fp)); + redis_fsync(r->io.file.fd); r->io.file.buffered = 0; } return retval; @@ -124,18 +123,18 @@ static size_t rioFileWrite(rio *r, const void *buf, size_t len) { /* Returns 1 or 0 for success/failure. */ static size_t rioFileRead(rio *r, void *buf, size_t len) { - return fread(buf,len,1,r->io.file.fp); + return read(r->io.file.fd,buf,len); } /* Returns read/write position in file. */ static off_t rioFileTell(rio *r) { - return ftello(r->io.file.fp); + return lseek(r->io.file.fd, 0, SEEK_CUR); } /* Flushes any buffer to target device if applicable. Returns 1 on success * and 0 on failures. */ static int rioFileFlush(rio *r) { - return (fflush(r->io.file.fp) == 0) ? 1 : 0; + return (fsync(r->io.file.fd) == 0) ? 1 : 0; } static const rio rioFileIO = { @@ -150,9 +149,9 @@ static const rio rioFileIO = { { { NULL, 0 } } /* union for io-specific vars */ }; -void rioInitWithFile(rio *r, FILE *fp) { +void rioInitWithFile(rio *r, int fd) { *r = rioFileIO; - r->io.file.fp = fp; + r->io.file.fd = fd; r->io.file.buffered = 0; r->io.file.autosync = 0; } diff --git a/src/rio.h b/src/rio.h index c996c54f6..f118556e1 100644 --- a/src/rio.h +++ b/src/rio.h @@ -69,7 +69,7 @@ struct _rio { } buffer; /* Stdio file pointer target. */ struct { - FILE *fp; + int fd; off_t buffered; /* Bytes written since last fsync. */ off_t autosync; /* fsync after 'autosync' bytes written. */ } file; @@ -124,7 +124,7 @@ static inline int rioFlush(rio *r) { return r->flush(r); } -void rioInitWithFile(rio *r, FILE *fp); +void rioInitWithFile(rio *r, int fd); void rioInitWithBuffer(rio *r, sds s); void rioInitWithFdset(rio *r, int *fds, int numfds); diff --git a/src/sds.h b/src/sds.h index 1bdb60dec..555a1d40c 100644 --- a/src/sds.h +++ b/src/sds.h @@ -34,43 +34,49 @@ #define __SDS_H #define SDS_MAX_PREALLOC (1024*1024) -const char *SDS_NOINIT; +extern const char *SDS_NOINIT; #include #include #include +#ifdef __cplusplus +#define ZERO_LENGTH_ARRAY_LENGTH 1 +#else +#define ZERO_LENGTH_ARRAY_LENGTH +#endif + typedef char *sds; /* Note: sdshdr5 is never used, we just access the flags byte directly. * However is here to document the layout of type 5 SDS strings. */ struct __attribute__ ((__packed__)) sdshdr5 { unsigned char flags; /* 3 lsb of type, and 5 msb of string length */ - char buf[]; + char buf[ZERO_LENGTH_ARRAY_LENGTH]; }; struct __attribute__ ((__packed__)) sdshdr8 { uint8_t len; /* used */ uint8_t alloc; /* excluding the header and null terminator */ unsigned char flags; /* 3 lsb of type, 5 unused bits */ - char buf[]; + char buf[ZERO_LENGTH_ARRAY_LENGTH]; }; struct __attribute__ ((__packed__)) sdshdr16 { uint16_t len; /* used */ uint16_t alloc; /* excluding the header and null terminator */ unsigned char flags; /* 3 lsb of type, 5 unused bits */ - char buf[]; + char buf[ZERO_LENGTH_ARRAY_LENGTH]; }; struct __attribute__ ((__packed__)) sdshdr32 { uint32_t len; /* used */ uint32_t alloc; /* excluding the header and null terminator */ unsigned char flags; /* 3 lsb of type, 5 unused bits */ - char buf[]; + char buf[ZERO_LENGTH_ARRAY_LENGTH]; }; struct __attribute__ ((__packed__)) sdshdr64 { uint64_t len; /* used */ uint64_t alloc; /* excluding the header and null terminator */ unsigned char flags; /* 3 lsb of type, 5 unused bits */ - char buf[]; + char buf[ZERO_LENGTH_ARRAY_LENGTH]; }; #define SDS_TYPE_5 0 @@ -80,7 +86,7 @@ struct __attribute__ ((__packed__)) sdshdr64 { #define SDS_TYPE_64 4 #define SDS_TYPE_MASK 7 #define SDS_TYPE_BITS 3 -#define SDS_HDR_VAR(T,s) struct sdshdr##T *sh = (void*)((s)-(sizeof(struct sdshdr##T))); +#define SDS_HDR_VAR(T,s) struct sdshdr##T *sh = (struct sdshdr##T *)(((void*)((s)-(sizeof(struct sdshdr##T))))); #define SDS_HDR(T,s) ((struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T)))) #define SDS_TYPE_5_LEN(f) ((f)>>SDS_TYPE_BITS) diff --git a/src/server.c b/src/server.c index ce00e4711..4c35f8c25 100644 --- a/src/server.c +++ b/src/server.c @@ -1283,7 +1283,7 @@ dictType zsetDictType = { NULL /* val destructor */ }; -/* Db->dict, keys are sds strings, vals are Redis objects. */ +/* db->pdict, keys are sds strings, vals are Redis objects. */ dictType dbDictType = { dictSdsHash, /* hash function */ NULL, /* key dup */ @@ -1414,8 +1414,8 @@ int htNeedsResize(dict *dict) { /* If the percentage of used slots in the HT reaches HASHTABLE_MIN_FILL * we resize the hash table to save memory */ void tryResizeHashTables(int dbid) { - if (htNeedsResize(server.db[dbid].dict)) - dictResize(server.db[dbid].dict); + if (htNeedsResize(server.db[dbid].pdict)) + dictResize(server.db[dbid].pdict); if (htNeedsResize(server.db[dbid].expires)) dictResize(server.db[dbid].expires); } @@ -1429,8 +1429,8 @@ void tryResizeHashTables(int dbid) { * is returned. */ int incrementallyRehash(int dbid) { /* Keys dictionary */ - if (dictIsRehashing(server.db[dbid].dict)) { - dictRehashMilliseconds(server.db[dbid].dict,1); + if (dictIsRehashing(server.db[dbid].pdict)) { + dictRehashMilliseconds(server.db[dbid].pdict,1); return 1; /* already used our millisecond for this loop... */ } /* Expires */ @@ -1856,8 +1856,8 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { for (j = 0; j < server.dbnum; j++) { long long size, used, vkeys; - size = dictSlots(server.db[j].dict); - used = dictSize(server.db[j].dict); + size = dictSlots(server.db[j].pdict); + used = dictSize(server.db[j].pdict); vkeys = dictSize(server.db[j].expires); if (used || vkeys) { serverLog(LL_VERBOSE,"DB %d: %lld keys (%lld volatile) in %lld slots HT.",j,used,vkeys,size); @@ -2267,6 +2267,7 @@ void initServerConfig(void) { server.aof_use_rdb_preamble = CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE; server.pidfile = NULL; server.rdb_filename = zstrdup(CONFIG_DEFAULT_RDB_FILENAME); + server.rdb_s3bucketpath = NULL; server.aof_filename = zstrdup(CONFIG_DEFAULT_AOF_FILENAME); server.rdb_compression = CONFIG_DEFAULT_RDB_COMPRESSION; server.rdb_checksum = CONFIG_DEFAULT_RDB_CHECKSUM; @@ -2757,7 +2758,7 @@ void initServer(void) { /* Create the Redis databases, and initialize other internal state. */ for (j = 0; j < server.dbnum; j++) { - server.db[j].dict = dictCreate(&dbDictType,NULL); + server.db[j].pdict = dictCreate(&dbDictType,NULL); server.db[j].expires = dictCreate(&keyptrDictType,NULL); server.db[j].blocking_keys = dictCreate(&keylistDictType,NULL); server.db[j].ready_keys = dictCreate(&objectKeyPointerValueDictType,NULL); @@ -3630,7 +3631,7 @@ void authCommand(client *c) { if (ACLCheckUserCredentials(username,password) == C_OK) { c->authenticated = 1; - c->user = ACLGetUserByName(username->ptr,sdslen(username->ptr)); + c->puser = ACLGetUserByName(username->ptr,sdslen(username->ptr)); addReply(c,shared.ok); } else { addReplyError(c,"-WRONGPASS invalid username-password pair"); @@ -4364,7 +4365,7 @@ sds genRedisInfoString(char *section) { for (j = 0; j < server.dbnum; j++) { long long keys, vkeys; - keys = dictSize(server.db[j].dict); + keys = dictSize(server.db[j].pdict); vkeys = dictSize(server.db[j].expires); if (keys || vkeys) { info = sdscatprintf(info, diff --git a/src/server.h b/src/server.h index 1e38b8ae6..8f10d6050 100644 --- a/src/server.h +++ b/src/server.h @@ -570,7 +570,7 @@ typedef struct moduleValue { * to care about error conditions. */ typedef struct RedisModuleIO { size_t bytes; /* Bytes read / written so far. */ - rio *rio; /* Rio stream. */ + rio *prio; /* Rio stream. */ moduleType *type; /* Module type doing the operation. */ int error; /* True if error condition happened. */ int ver; /* Module serialization version: 1 (old), @@ -581,7 +581,7 @@ typedef struct RedisModuleIO { /* Macro to initialize an IO context. Note that the 'ver' field is populated * inside rdb.c according to the version of the value to load. */ #define moduleInitIOContext(iovar,mtype,rioptr) do { \ - iovar.rio = rioptr; \ + iovar.prio = rioptr; \ iovar.type = mtype; \ iovar.bytes = 0; \ iovar.error = 0; \ @@ -652,14 +652,14 @@ struct evictionPoolEntry; /* Defined in evict.c */ * which is actually a linked list of blocks like that, that is: client->reply. */ typedef struct clientReplyBlock { size_t size, used; - char buf[]; + char buf[ZERO_LENGTH_ARRAY_LENGTH]; } clientReplyBlock; /* Redis database representation. There are multiple databases identified * by integers from 0 (the default database) up to the max configured * database. The database number is the 'id' field in the structure. */ typedef struct redisDb { - dict *dict; /* The keyspace for this DB */ + dict *pdict; /* The keyspace for this DB */ dict *expires; /* Timeout of keys with a timeout set */ dict *blocking_keys; /* Keys with clients waiting for data (BLPOP)*/ dict *ready_keys; /* Blocked keys that received a PUSH */ @@ -791,7 +791,7 @@ typedef struct client { int argc; /* Num of arguments of current command. */ robj **argv; /* Arguments of current command. */ struct redisCommand *cmd, *lastcmd; /* Last command executed. */ - user *user; /* User associated with this connection. If the + user *puser; /* User associated with this connection. If the user is set to NULL the connection can do anything (admin). */ int reqtype; /* Request protocol type: PROTO_REQ_* */ @@ -873,7 +873,7 @@ typedef struct zskiplistNode { struct zskiplistLevel { struct zskiplistNode *forward; unsigned long span; - } level[]; + } level[ZERO_LENGTH_ARRAY_LENGTH]; } zskiplistNode; typedef struct zskiplist { @@ -883,7 +883,7 @@ typedef struct zskiplist { } zskiplist; typedef struct zset { - dict *dict; + dict *pdict; zskiplist *zsl; } zset; @@ -1165,6 +1165,7 @@ struct redisServer { struct saveparam *saveparams; /* Save points array for RDB */ int saveparamslen; /* Number of saving points */ char *rdb_filename; /* Name of RDB file */ + char *rdb_s3bucketpath; /* Path for AWS S3 backup of RDB file */ int rdb_compression; /* Use compression in RDB? */ int rdb_checksum; /* Use RDB checksum? */ time_t lastsave; /* Unix time of last successful save */ @@ -1353,7 +1354,7 @@ struct redisServer { }; typedef struct pubsubPattern { - client *client; + client *pclient; robj *pattern; } pubsubPattern; @@ -1553,7 +1554,7 @@ void freeClientsInAsyncFreeQueue(void); void asyncCloseClientOnOutputBufferLimitReached(client *c); int getClientType(client *c); int getClientTypeByName(char *name); -char *getClientTypeName(int class); +char *getClientTypeName(int cclass); void flushSlavesOutputBuffers(void); void disconnectSlaves(void); int listenToPort(int port, int *fds, int *count); diff --git a/src/sort.c b/src/sort.c index 7ae794501..b16b9aeef 100644 --- a/src/sort.c +++ b/src/sort.c @@ -309,7 +309,7 @@ void sortCommand(client *c) { switch(sortval->type) { case OBJ_LIST: vectorlen = listTypeLength(sortval); break; case OBJ_SET: vectorlen = setTypeSize(sortval); break; - case OBJ_ZSET: vectorlen = dictSize(((zset*)sortval->ptr)->dict); break; + case OBJ_ZSET: vectorlen = dictSize(((zset*)sortval->ptr)->pdict); break; default: vectorlen = 0; serverPanic("Bad SORT type"); /* Avoid GCC warning */ } @@ -404,7 +404,7 @@ void sortCommand(client *c) { /* Check if starting point is trivial, before doing log(N) lookup. */ if (desc) { - long zsetlen = dictSize(((zset*)sortval->ptr)->dict); + long zsetlen = dictSize(((zset*)sortval->ptr)->pdict); ln = zsl->tail; if (start > 0) @@ -428,7 +428,7 @@ void sortCommand(client *c) { end -= start; start = 0; } else if (sortval->type == OBJ_ZSET) { - dict *set = ((zset*)sortval->ptr)->dict; + dict *set = ((zset*)sortval->ptr)->pdict; dictIterator *di; dictEntry *setele; sds sdsele; diff --git a/src/storage.h b/src/storage.h index 06c3572a5..8b21cd204 100644 --- a/src/storage.h +++ b/src/storage.h @@ -11,10 +11,10 @@ enum MALLOC_CLASS void storage_init(const char *tmpfilePath, size_t cbFileReserve); -void *salloc(size_t cb, enum MALLOC_CLASS class); -void *scalloc(size_t cb, size_t c, enum MALLOC_CLASS class); +void *salloc(size_t cb, enum MALLOC_CLASS mclass); +void *scalloc(size_t cb, size_t c, enum MALLOC_CLASS mclass); void sfree(void*); -void *srealloc(void *pv, size_t cb, enum MALLOC_CLASS class); +void *srealloc(void *pv, size_t cb, enum MALLOC_CLASS mclass); size_t salloc_usable_size(void *ptr); #endif diff --git a/src/stream.h b/src/stream.h index ef08753b5..01494e346 100644 --- a/src/stream.h +++ b/src/stream.h @@ -14,7 +14,7 @@ typedef struct streamID { } streamID; typedef struct stream { - rax *rax; /* The radix tree holding the stream. */ + rax *prax; /* The radix tree holding the stream. */ uint64_t length; /* Number of elements inside this stream. */ streamID last_id; /* Zero if there are yet no items. */ rax *cgroups; /* Consumer groups dictionary: name -> streamCG */ @@ -27,7 +27,7 @@ typedef struct stream { * rewriting code that also needs to iterate the stream to emit the XADD * commands. */ typedef struct streamIterator { - stream *stream; /* The stream we are iterating. */ + stream *pstream; /* The stream we are iterating. */ streamID master_id; /* ID of the master entry at listpack head. */ uint64_t master_fields_count; /* Master entries # of fields. */ unsigned char *master_fields_start; /* Master entries start in listpack. */ diff --git a/src/t_list.c b/src/t_list.c index 143f672a4..4858403bf 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -144,10 +144,10 @@ void listTypeInsert(listTypeEntry *entry, robj *value, int where) { sds str = value->ptr; size_t len = sdslen(str); if (where == LIST_TAIL) { - quicklistInsertAfter((quicklist *)entry->entry.quicklist, + quicklistInsertAfter((quicklist *)entry->entry.qlist, &entry->entry, str, len); } else if (where == LIST_HEAD) { - quicklistInsertBefore((quicklist *)entry->entry.quicklist, + quicklistInsertBefore((quicklist *)entry->entry.qlist, &entry->entry, str, len); } decrRefCount(value); diff --git a/src/t_stream.c b/src/t_stream.c index 5fb14628b..d87a7f3ee 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -51,7 +51,7 @@ size_t streamReplyWithRangeFromConsumerPEL(client *c, stream *s, streamID *start /* Create a new stream data structure. */ stream *streamNew(void) { stream *s = zmalloc(sizeof(*s), MALLOC_SHARED); - s->rax = raxNew(); + s->prax = raxNew(); s->length = 0; s->last_id.ms = 0; s->last_id.seq = 0; @@ -61,7 +61,7 @@ stream *streamNew(void) { /* Free a stream, including the listpacks stored inside the radix tree. */ void freeStream(stream *s) { - raxFreeWithCallback(s->rax,(void(*)(void*))lpFree); + raxFreeWithCallback(s->prax,(void(*)(void*))lpFree); if (s->cgroups) raxFreeWithCallback(s->cgroups,(void(*)(void*))streamFreeCG); zfree(s); @@ -179,7 +179,7 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_ /* Add the new entry. */ raxIterator ri; - raxStart(&ri,s->rax); + raxStart(&ri,s->prax); raxSeek(&ri,"$",NULL,0); size_t lp_bytes = 0; /* Total bytes in the tail listpack. */ @@ -265,7 +265,7 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_ lp = lpAppend(lp,(unsigned char*)field,sdslen(field)); } lp = lpAppendInteger(lp,0); /* Master entry zero terminator. */ - raxInsert(s->rax,(unsigned char*)&rax_key,sizeof(rax_key),lp,NULL); + raxInsert(s->prax,(unsigned char*)&rax_key,sizeof(rax_key),lp,NULL); /* The first entry we insert, has obviously the same fields of the * master entry. */ flags |= STREAM_ITEM_FLAG_SAMEFIELDS; @@ -350,7 +350,7 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_ /* Insert back into the tree in order to update the listpack pointer. */ if (ri.data != lp) - raxInsert(s->rax,(unsigned char*)&rax_key,sizeof(rax_key),lp,NULL); + raxInsert(s->prax,(unsigned char*)&rax_key,sizeof(rax_key),lp,NULL); s->length++; s->last_id = id; if (added_id) *added_id = id; @@ -375,7 +375,7 @@ int64_t streamTrimByLength(stream *s, size_t maxlen, int approx) { if (s->length <= maxlen) return 0; raxIterator ri; - raxStart(&ri,s->rax); + raxStart(&ri,s->prax); raxSeek(&ri,"^",NULL,0); int64_t deleted = 0; @@ -387,7 +387,7 @@ int64_t streamTrimByLength(stream *s, size_t maxlen, int approx) { * least maxlen elements. */ if (s->length - entries >= maxlen) { lpFree(lp); - raxRemove(s->rax,ri.key,ri.key_len,NULL); + raxRemove(s->prax,ri.key,ri.key_len,NULL); raxSeek(&ri,">=",ri.key,ri.key_len); s->length -= entries; deleted += entries; @@ -454,7 +454,7 @@ int64_t streamTrimByLength(stream *s, size_t maxlen, int approx) { } /* Update the listpack with the new pointer. */ - raxInsert(s->rax,ri.key,ri.key_len,lp,NULL); + raxInsert(s->prax,ri.key,ri.key_len,lp,NULL); break; /* If we are here, there was enough to delete in the current node, so no need to go to the next node. */ @@ -503,7 +503,7 @@ void streamIteratorStart(streamIterator *si, stream *s, streamID *start, streamI } /* Seek the correct node in the radix tree. */ - raxStart(&si->ri,s->rax); + raxStart(&si->ri,s->prax); if (!rev) { if (start && (start->ms || start->seq)) { raxSeek(&si->ri,"<=",(unsigned char*)si->start_key, @@ -521,7 +521,7 @@ void streamIteratorStart(streamIterator *si, stream *s, streamID *start, streamI raxSeek(&si->ri,"$",NULL,0); } } - si->stream = s; + si->pstream = s; si->lp = NULL; /* There is no current listpack right now. */ si->lp_ele = NULL; /* Current listpack cursor. */ si->rev = rev; /* Direction, if non-zero reversed, from end to start. */ @@ -718,7 +718,7 @@ void streamIteratorRemoveEntry(streamIterator *si, streamID *current) { /* If this is the last element in the listpack, we can remove the whole * node. */ lpFree(lp); - raxRemove(si->stream->rax,si->ri.key,si->ri.key_len,NULL); + raxRemove(si->pstream->prax,si->ri.key,si->ri.key_len,NULL); } else { /* In the base case we alter the counters of valid/deleted entries. */ lp = lpReplaceInteger(lp,&p,aux-1); @@ -728,11 +728,11 @@ void streamIteratorRemoveEntry(streamIterator *si, streamID *current) { /* Update the listpack with the new pointer. */ if (si->lp != lp) - raxInsert(si->stream->rax,si->ri.key,si->ri.key_len,lp,NULL); + raxInsert(si->pstream->prax,si->ri.key,si->ri.key_len,lp,NULL); } /* Update the number of entries counter. */ - si->stream->length--; + si->pstream->length--; /* Re-seek the iterator to fix the now messed up state. */ streamID start, end; @@ -744,7 +744,7 @@ void streamIteratorRemoveEntry(streamIterator *si, streamID *current) { streamDecodeID(si->end_key,&end); } streamIteratorStop(si); - streamIteratorStart(si,si->stream,&start,&end,si->rev); + streamIteratorStart(si,si->pstream,&start,&end,si->rev); /* TODO: perform a garbage collection here if the ration between * deleted and valid goes over a certain limit. */ @@ -2517,9 +2517,9 @@ NULL addReplyBulkCString(c,"length"); addReplyLongLong(c,s->length); addReplyBulkCString(c,"radix-tree-keys"); - addReplyLongLong(c,raxSize(s->rax)); + addReplyLongLong(c,raxSize(s->prax)); addReplyBulkCString(c,"radix-tree-nodes"); - addReplyLongLong(c,s->rax->numnodes); + addReplyLongLong(c,s->prax->numnodes); addReplyBulkCString(c,"groups"); addReplyLongLong(c,s->cgroups ? raxSize(s->cgroups) : 0); addReplyBulkCString(c,"last-generated-id"); diff --git a/src/t_zset.c b/src/t_zset.c index daeabcf93..af82d4d37 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -1183,7 +1183,7 @@ void zsetConvert(robj *zobj, int encoding) { serverPanic("Unknown target encoding"); zs = zmalloc(sizeof(*zs), MALLOC_SHARED); - zs->dict = dictCreate(&zsetDictType,NULL); + zs->pdict = dictCreate(&zsetDictType,NULL); zs->zsl = zslCreate(); eptr = ziplistIndex(zl,0); @@ -1200,7 +1200,7 @@ void zsetConvert(robj *zobj, int encoding) { ele = sdsnewlen((char*)vstr,vlen); node = zslInsert(zs->zsl,score,ele); - serverAssert(dictAdd(zs->dict,ele,&node->score) == DICT_OK); + serverAssert(dictAdd(zs->pdict,ele,&node->score) == DICT_OK); zzlNext(zl,&eptr,&sptr); } @@ -1216,7 +1216,7 @@ void zsetConvert(robj *zobj, int encoding) { /* Approach similar to zslFree(), since we want to free the skiplist at * the same time as creating the ziplist. */ zs = zobj->ptr; - dictRelease(zs->dict); + dictRelease(zs->pdict); node = zs->zsl->header->level[0].forward; zfree(zs->zsl->header); zfree(zs->zsl); @@ -1259,7 +1259,7 @@ int zsetScore(robj *zobj, sds member, double *score) { if (zzlFind(zobj->ptr, member, score) == NULL) return C_ERR; } else if (zobj->encoding == OBJ_ENCODING_SKIPLIST) { zset *zs = zobj->ptr; - dictEntry *de = dictFind(zs->dict, member); + dictEntry *de = dictFind(zs->pdict, member); if (de == NULL) return C_ERR; *score = *(double*)dictGetVal(de); } else { @@ -1373,7 +1373,7 @@ int zsetAdd(robj *zobj, double score, sds ele, int *flags, double *newscore) { zskiplistNode *znode; dictEntry *de; - de = dictFind(zs->dict,ele); + de = dictFind(zs->pdict,ele); if (de != NULL) { /* NX? Return, same element already exists. */ if (nx) { @@ -1405,7 +1405,7 @@ int zsetAdd(robj *zobj, double score, sds ele, int *flags, double *newscore) { } else if (!xx) { ele = sdsdup(ele); znode = zslInsert(zs->zsl,score,ele); - serverAssert(dictAdd(zs->dict,ele,&znode->score) == DICT_OK); + serverAssert(dictAdd(zs->pdict,ele,&znode->score) == DICT_OK); *flags |= ZADD_ADDED; if (newscore) *newscore = score; return 1; @@ -1434,7 +1434,7 @@ int zsetDel(robj *zobj, sds ele) { dictEntry *de; double score; - de = dictUnlink(zs->dict,ele); + de = dictUnlink(zs->pdict,ele); if (de != NULL) { /* Get the score in order to delete from the skiplist later. */ score = *(double*)dictGetVal(de); @@ -1444,13 +1444,13 @@ int zsetDel(robj *zobj, sds ele) { * actually releases the SDS string representing the element, * which is shared between the skiplist and the hash table, so * we need to delete from the skiplist as the final step. */ - dictFreeUnlinkedEntry(zs->dict,de); + dictFreeUnlinkedEntry(zs->pdict,de); /* Delete from skiplist. */ int retval = zslDelete(zs->zsl,score,ele,NULL); serverAssert(retval); - if (htNeedsResize(zs->dict)) dictResize(zs->dict); + if (htNeedsResize(zs->pdict)) dictResize(zs->pdict); return 1; } } else { @@ -1507,7 +1507,7 @@ long zsetRank(robj *zobj, sds ele, int reverse) { dictEntry *de; double score; - de = dictFind(zs->dict,ele); + de = dictFind(zs->pdict,ele); if (de != NULL) { score = *(double*)dictGetVal(de); rank = zslGetRank(zsl,score,ele); @@ -1758,17 +1758,17 @@ void zremrangeGenericCommand(client *c, int rangetype) { zset *zs = zobj->ptr; switch(rangetype) { case ZRANGE_RANK: - deleted = zslDeleteRangeByRank(zs->zsl,start+1,end+1,zs->dict); + deleted = zslDeleteRangeByRank(zs->zsl,start+1,end+1,zs->pdict); break; case ZRANGE_SCORE: - deleted = zslDeleteRangeByScore(zs->zsl,&range,zs->dict); + deleted = zslDeleteRangeByScore(zs->zsl,&range,zs->pdict); break; case ZRANGE_LEX: - deleted = zslDeleteRangeByLex(zs->zsl,&lexrange,zs->dict); + deleted = zslDeleteRangeByLex(zs->zsl,&lexrange,zs->pdict); break; } - if (htNeedsResize(zs->dict)) dictResize(zs->dict); - if (dictSize(zs->dict) == 0) { + if (htNeedsResize(zs->pdict)) dictResize(zs->pdict); + if (dictSize(zs->pdict) == 0) { dbDelete(c->db,key); keyremoved = 1; } @@ -2117,7 +2117,7 @@ int zuiFind(zsetopsrc *op, zsetopval *val, double *score) { } else if (op->encoding == OBJ_ENCODING_SKIPLIST) { zset *zs = op->subject->ptr; dictEntry *de; - if ((de = dictFind(zs->dict,val->ele)) != NULL) { + if ((de = dictFind(zs->pdict,val->ele)) != NULL) { *score = *(double*)dictGetVal(de); return 1; } else { @@ -2303,7 +2303,7 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int op) { if (j == setnum) { tmp = zuiNewSdsFromValue(&zval); znode = zslInsert(dstzset->zsl,score,tmp); - dictAdd(dstzset->dict,tmp,&znode->score); + dictAdd(dstzset->pdict,tmp,&znode->score); if (sdslen(tmp) > maxelelen) maxelelen = sdslen(tmp); } } @@ -2363,13 +2363,13 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int op) { /* We now are aware of the final size of the resulting sorted set, * let's resize the dictionary embedded inside the sorted set to the * right size, in order to save rehashing time. */ - dictExpand(dstzset->dict,dictSize(accumulator)); + dictExpand(dstzset->pdict,dictSize(accumulator)); while((de = dictNext(di)) != NULL) { sds ele = dictGetKey(de); score = dictGetDoubleVal(de); znode = zslInsert(dstzset->zsl,score,ele); - dictAdd(dstzset->dict,ele,&znode->score); + dictAdd(dstzset->pdict,ele,&znode->score); } dictReleaseIterator(di); dictRelease(accumulator); diff --git a/src/zmalloc.h b/src/zmalloc.h index 63ae7074a..979015c59 100644 --- a/src/zmalloc.h +++ b/src/zmalloc.h @@ -85,9 +85,9 @@ #define HAVE_DEFRAG #endif -void *zmalloc(size_t size, enum MALLOC_CLASS class); -void *zcalloc(size_t size, enum MALLOC_CLASS class); -void *zrealloc(void *ptr, size_t size, enum MALLOC_CLASS class); +void *zmalloc(size_t size, enum MALLOC_CLASS mclass); +void *zcalloc(size_t size, enum MALLOC_CLASS mclass); +void *zrealloc(void *ptr, size_t size, enum MALLOC_CLASS mclass); void zfree(void *ptr); char *zstrdup(const char *s); size_t zmalloc_used_memory(void); From 564dabd0a2e18dd4d7a518c922bbd5fe6244542c Mon Sep 17 00:00:00 2001 From: John Sully Date: Wed, 6 Feb 2019 01:06:48 -0500 Subject: [PATCH 009/101] Support AWS S3 saving via the s3 cli tools --- src/config.c | 3 +++ src/db.c | 2 +- src/debug.c | 2 +- src/rdb-s3.cpp | 26 +++++++++++++++++--------- src/rdb.c | 21 ++++++++++++++++----- src/rdb.h | 5 +++-- src/replication.c | 2 +- src/server.c | 8 ++++---- 8 files changed, 46 insertions(+), 23 deletions(-) diff --git a/src/config.c b/src/config.c index 0c4793327..4df08e47f 100644 --- a/src/config.c +++ b/src/config.c @@ -550,6 +550,9 @@ void loadServerConfigFromString(char *config) { } zfree(server.rdb_filename); server.rdb_filename = zstrdup(argv[1]); + } else if(!strcasecmp(argv[0],"db-s3-object") && argc == 2) { + zfree(server.rdb_s3bucketpath); + server.rdb_s3bucketpath = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"active-defrag-threshold-lower") && argc == 2) { server.active_defrag_threshold_lower = atoi(argv[1]); if (server.active_defrag_threshold_lower < 0 || diff --git a/src/db.c b/src/db.c index 6ad033e91..c3b8ffd80 100644 --- a/src/db.c +++ b/src/db.c @@ -455,7 +455,7 @@ void flushallCommand(client *c) { int saved_dirty = server.dirty; rdbSaveInfo rsi, *rsiptr; rsiptr = rdbPopulateSaveInfo(&rsi); - rdbSave(server.rdb_filename,rsiptr); + rdbSave(rsiptr); server.dirty = saved_dirty; } server.dirty++; diff --git a/src/debug.c b/src/debug.c index f2f7ac462..078282507 100644 --- a/src/debug.c +++ b/src/debug.c @@ -356,7 +356,7 @@ NULL } else if (!strcasecmp(c->argv[1]->ptr,"reload")) { rdbSaveInfo rsi, *rsiptr; rsiptr = rdbPopulateSaveInfo(&rsi); - if (rdbSave(server.rdb_filename,rsiptr) != C_OK) { + if (rdbSave(rsiptr) != C_OK) { addReply(c,shared.err); return; } diff --git a/src/rdb-s3.cpp b/src/rdb-s3.cpp index 695bdd25f..d6bf34ae0 100644 --- a/src/rdb-s3.cpp +++ b/src/rdb-s3.cpp @@ -8,6 +8,7 @@ extern "C" { /* Save the DB on disk. Return C_ERR on error, C_OK on success. */ extern "C" int rdbSaveS3(char *s3bucket, rdbSaveInfo *rsi) { + int status = EXIT_FAILURE; int fd[2]; if (pipe(fd) != 0) return C_ERR; @@ -23,21 +24,28 @@ extern "C" int rdbSaveS3(char *s3bucket, rdbSaveInfo *rsi) if (pid == 0) { // child process - dup2(fd[1], STDIN_FILENO); - execlp("aws", "s3", "cp", "-", s3bucket, nullptr); + dup2(fd[0], STDIN_FILENO); + close(fd[1]); + close(fd[0]); + execlp("aws", "aws", "s3", "cp", "-", s3bucket, nullptr); exit(EXIT_FAILURE); } else { + close(fd[0]); + if (rdbSaveFd(fd[1], rsi) != C_OK) + { + close(fd[1]); + return C_ERR; + } close(fd[1]); - rdbSaveFd(fd[0], rsi); - int status; waitpid(pid, &status, 0); } - close(fd[0]); - - - // NOP - return C_ERR; + if (status != EXIT_SUCCESS) + serverLog(LL_WARNING, "Failed to save DB to AWS S3"); + else + serverLog(LL_NOTICE,"DB saved on AWS S3"); + + return (status == EXIT_SUCCESS) ? C_OK : C_ERR; } \ No newline at end of file diff --git a/src/rdb.c b/src/rdb.c index ab157c630..16c3423a3 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1233,8 +1233,19 @@ int rdbSaveFd(int fd, rdbSaveInfo *rsi) return C_OK; } +int rdbSave(rdbSaveInfo *rsi) +{ + int err = C_OK; + if (server.rdb_filename != NULL) + err = rdbSaveFile(server.rdb_filename, rsi); + + if (err == C_OK && server.rdb_s3bucketpath != NULL) + err = rdbSaveS3(server.rdb_s3bucketpath, rsi); + return err; +} + /* Save the DB on disk. Return C_ERR on error, C_OK on success. */ -int rdbSave(char *filename, rdbSaveInfo *rsi) { +int rdbSaveFile(char *filename, rdbSaveInfo *rsi) { char tmpfile[256]; char cwd[MAXPATHLEN]; /* Current working dir path for error messages. */ FILE *fp; @@ -1289,7 +1300,7 @@ werr: return C_ERR; } -int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) { +int rdbSaveBackground(rdbSaveInfo *rsi) { pid_t childpid; long long start; @@ -1306,7 +1317,7 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) { /* Child */ closeListeningSockets(0); redisSetProcTitle("redis-rdb-bgsave"); - retval = rdbSave(filename,rsi); + retval = rdbSave(rsi); if (retval == C_OK) { size_t private_dirty = zmalloc_get_private_dirty(-1); @@ -2413,7 +2424,7 @@ void saveCommand(client *c) { } rdbSaveInfo rsi, *rsiptr; rsiptr = rdbPopulateSaveInfo(&rsi); - if (rdbSave(server.rdb_filename,rsiptr) == C_OK) { + if (rdbSave(rsiptr) == C_OK) { addReply(c,shared.ok); } else { addReply(c,shared.err); @@ -2450,7 +2461,7 @@ void bgsaveCommand(client *c) { "Use BGSAVE SCHEDULE in order to schedule a BGSAVE whenever " "possible."); } - } else if (rdbSaveBackground(server.rdb_filename,rsiptr) == C_OK) { + } else if (rdbSaveBackground(rsiptr) == C_OK) { addReplyStatus(c,"Background saving started"); } else { addReply(c,shared.err); diff --git a/src/rdb.h b/src/rdb.h index 65d78c531..2daa49984 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -136,10 +136,11 @@ int rdbLoadLenByRef(rio *rdb, int *isencoded, uint64_t *lenptr); int rdbSaveObjectType(rio *rdb, robj *o); int rdbLoadObjectType(rio *rdb); int rdbLoad(char *filename, rdbSaveInfo *rsi); -int rdbSaveBackground(char *filename, rdbSaveInfo *rsi); +int rdbSaveBackground(rdbSaveInfo *rsi); int rdbSaveToSlavesSockets(rdbSaveInfo *rsi); void rdbRemoveTempFile(pid_t childpid); -int rdbSave(char *filename, rdbSaveInfo *rsi); +int rdbSave(rdbSaveInfo *rsi); +int rdbSaveFile(char *filename, rdbSaveInfo *rsi); int rdbSaveFd(int fd, rdbSaveInfo *rsi); int rdbSaveS3(char *path, rdbSaveInfo *rsi); ssize_t rdbSaveObject(rio *rdb, robj *o); diff --git a/src/replication.c b/src/replication.c index ea433326e..fa0585638 100644 --- a/src/replication.c +++ b/src/replication.c @@ -577,7 +577,7 @@ int startBgsaveForReplication(int mincapa) { if (socket_target) retval = rdbSaveToSlavesSockets(rsiptr); else - retval = rdbSaveBackground(server.rdb_filename,rsiptr); + retval = rdbSaveBackground(rsiptr); } else { serverLog(LL_WARNING,"BGSAVE for replication: replication information not available, can't generate the RDB file right now. Try later."); retval = C_ERR; diff --git a/src/server.c b/src/server.c index 6fd066a1e..c03a6d9d7 100644 --- a/src/server.c +++ b/src/server.c @@ -1946,7 +1946,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { sp->changes, (int)sp->seconds); rdbSaveInfo rsi, *rsiptr; rsiptr = rdbPopulateSaveInfo(&rsi); - rdbSaveBackground(server.rdb_filename,rsiptr); + rdbSaveBackground(rsiptr); break; } } @@ -2019,7 +2019,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { { rdbSaveInfo rsi, *rsiptr; rsiptr = rdbPopulateSaveInfo(&rsi); - if (rdbSaveBackground(server.rdb_filename,rsiptr) == C_OK) + if (rdbSaveBackground(rsiptr) == C_OK) server.rdb_bgsave_scheduled = 0; } @@ -2266,7 +2266,7 @@ void initServerConfig(void) { server.aof_load_truncated = CONFIG_DEFAULT_AOF_LOAD_TRUNCATED; server.aof_use_rdb_preamble = CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE; server.pidfile = NULL; - server.rdb_filename = zstrdup(CONFIG_DEFAULT_RDB_FILENAME); + server.rdb_filename = NULL; server.rdb_s3bucketpath = NULL; server.aof_filename = zstrdup(CONFIG_DEFAULT_AOF_FILENAME); server.acl_filename = zstrdup(CONFIG_DEFAULT_ACL_FILENAME); @@ -3542,7 +3542,7 @@ int prepareForShutdown(int flags) { /* Snapshotting. Perform a SYNC SAVE and exit */ rdbSaveInfo rsi, *rsiptr; rsiptr = rdbPopulateSaveInfo(&rsi); - if (rdbSave(server.rdb_filename,rsiptr) != C_OK) { + if (rdbSave(rsiptr) != C_OK) { /* Ooops.. error saving! The best we can do is to continue * operating. Note that if there was a background saving process, * in the next cron() Redis will be notified that the background From 1050774a5b4ae85cd28b84cf95fa95b7b8ae980d Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 6 Feb 2019 12:39:11 +0100 Subject: [PATCH 010/101] ACL: initial design for ACLLoadFromFile() function. --- src/acl.c | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/src/acl.c b/src/acl.c index db2f291c6..1a721e628 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1009,6 +1009,85 @@ int ACLLoadConfiguredUsers(void) { return C_OK; } +/* This function loads the ACL from the specified filename: every line + * is validated and shold be either empty or in the format used to specify + * users in the redis.conf configuration or in the ACL file, that is: + * + * user ... rules ... + * + * Note that this function considers comments starting with '#' as errors + * because the ACL file is meant to be rewritten, and comments would be + * lost after the rewrite. Yet empty lines are allowed to avoid being too + * strict. + * + * One important part of implementing ACL LOAD, that uses this function, is + * to avoid ending with broken rules if the ACL file is invalid for some + * reason, so the function will attempt to validate the rules before loading + * each user. For every line that will be found broken the function will + * collect an error message. All the valid lines will be correctly processed. + * + * At the end of the process, if no errors were found in the whole file then + * NULL is returned. Otherwise an SDS string describing in a single line + * a description of all the issues found is returned. */ +sds ACLLoadFromFile(const char *filename) { + FILE *fp; + char buf[1024]; + + /* Open the ACL file. */ + if ((fp = fopen(filename,"r")) == NULL) { + sds errors = sdscatprintf(sdsempty(), + "Error loading ACLs, opening file '%s': %s", + filename, strerror(errno)); + return errors; + } + + /* Load the whole file as a single string in memory. */ + sds acls = sdsempty(); + while(fgets(buf,CONFIG_MAX_LINE+1,fp) != NULL) + acls = sdscat(acls,buf); + fclose(fp); + + /* Split the file into lines and attempt to load each line. */ + int totlines; + sds *lines, errors = sdsempty(); + lines = sdssplitlen(acls,strlen(acls),"\n",1,&totlines); + + for (int i = 0; i < totlines; i++) { + sds *argv; + int argc; + int linenum = i+1; + + lines[i] = sdstrim(lines[i]," \t\r\n"); + + /* Skip blank lines */ + if (lines[i][0] == '\0') continue; + + /* Split into arguments */ + argv = sdssplitargs(lines[i],&argc); + if (argv == NULL) { + errors = sdscatprintf(errors, + "%d: unbalanced quotes in acl line.", + linenum); + continue; + } + + /* Skip this line if the resulting command vector is empty. */ + if (argc == 0) { + sdsfreesplitres(argv,argc); + continue; + } + + /* Try to process the line. */ + } + + sdsfreesplitres(lines,totlines); + if (sdslen(errors) == 0) { + sdsfree(errors); + errors = NULL; + } + return errors; +} + /* ============================================================================= * ACL related commands * ==========================================================================*/ From 1922f590d2f3132a78bf02b7fab9c484f9eca2c7 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 6 Feb 2019 16:19:17 +0100 Subject: [PATCH 011/101] ACL: refactoring creation of unlinked users. --- src/acl.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/acl.c b/src/acl.c index 1a721e628..e19c26eb3 100644 --- a/src/acl.c +++ b/src/acl.c @@ -185,6 +185,23 @@ user *ACLCreateUser(const char *name, size_t namelen) { return u; } +/* This function should be called when we need an unlinked "fake" user + * we can use in order to validate ACL rules or for other similar reasons. + * The user will not get linked to the Users radix tree. The returned + * user should be released with ACLFreeUser() as usually. */ +user *ACLCreateUnlinkedUser(void) { + char username[64]; + for (int j = 0; ; j++) { + snprintf(username,sizeof(username),"__fakeuser:%d__",j); + user *fakeuser = ACLCreateUser(username,strlen(username)); + if (fakeuser == NULL) continue; + int retval = raxRemove(Users,(unsigned char*) username, + strlen(username),NULL); + serverAssert(retval != 0); + return fakeuser; + } +} + /* Release the memory used by the user structure. Note that this function * will not remove the user from the Users global radix tree. */ void ACLFreeUser(user *u) { @@ -944,11 +961,7 @@ int ACLAppendUserForLoading(sds *argv, int argc, int *argc_err) { /* Try to apply the user rules in a fake user to see if they * are actually valid. */ - char *funame = "__fakeuser__"; - user *fakeuser = ACLCreateUser(funame,strlen(funame)); - serverAssert(fakeuser != NULL); - int retval = raxRemove(Users,(unsigned char*) funame,strlen(funame),NULL); - serverAssert(retval != 0); + user *fakeuser = ACLCreateUnlinkedUser(); for (int j = 2; j < argc; j++) { if (ACLSetUser(fakeuser,argv[j],sdslen(argv[j])) == C_ERR) { From dc8605f944c49ca3aee4e8a59c967a2ab2fd4db0 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 6 Feb 2019 16:44:55 +0100 Subject: [PATCH 012/101] ACL: now ACLLoadFromFile() validates against fake user. --- src/acl.c | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/src/acl.c b/src/acl.c index e19c26eb3..2fbc564ab 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1065,6 +1065,10 @@ sds ACLLoadFromFile(const char *filename) { sds *lines, errors = sdsempty(); lines = sdssplitlen(acls,strlen(acls),"\n",1,&totlines); + /* We need a fake user to validate the rules before making changes + * to the real user mentioned in the ACL line. */ + user *fakeuser = ACLCreateUnlinkedUser(); + for (int i = 0; i < totlines; i++) { sds *argv; int argc; @@ -1079,8 +1083,8 @@ sds ACLLoadFromFile(const char *filename) { argv = sdssplitargs(lines[i],&argc); if (argv == NULL) { errors = sdscatprintf(errors, - "%d: unbalanced quotes in acl line.", - linenum); + "%d: unbalanced quotes in acl line. ", + linenum); continue; } @@ -1090,15 +1094,40 @@ sds ACLLoadFromFile(const char *filename) { continue; } - /* Try to process the line. */ + /* The line should start with the "user" keyword. */ + if (strcmp(argv[0],"user")) { + errors = sdscatprintf(errors, + "%d: line should start with user keyword. ", + linenum); + continue; + } + + /* Try to process the line using the fake user to validate iif + * the rules are able to apply cleanly. */ + ACLSetUser(fakeuser,"reset",-1); + int j; + for (j = 2; j < argc; j++) { + if (ACLSetUser(fakeuser,argv[j],sdslen(argv[j])) != C_OK) { + char *errmsg = ACLSetUserStringError(); + errors = sdscatprintf(errors, + "%d: error in ACL: %s. ", + linenum, errmsg); + continue; + } + } + if (j != argc) continue; /* Error in ACL rules, don't apply. */ + + /* We can finally lookup the user and apply the rule. */ } + ACLFreeUser(fakeuser); sdsfreesplitres(lines,totlines); if (sdslen(errors) == 0) { sdsfree(errors); - errors = NULL; + return NULL; + } else { + return sdstrim(errors," "); } - return errors; } /* ============================================================================= From a980f6168e55fbd2ddcb22c249f8e62257c4a01d Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 7 Feb 2019 12:04:25 +0100 Subject: [PATCH 013/101] ACL: fix and complete ACLLoadFromFile() loading step. --- src/acl.c | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/acl.c b/src/acl.c index 2fbc564ab..1c6e9a0c5 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1095,10 +1095,12 @@ sds ACLLoadFromFile(const char *filename) { } /* The line should start with the "user" keyword. */ - if (strcmp(argv[0],"user")) { + if (strcmp(argv[0],"user") || argc < 2) { errors = sdscatprintf(errors, - "%d: line should start with user keyword. ", + "%d: line should start with user keyword followed " + "by the username. ", linenum); + sdsfreesplitres(argv,argc); continue; } @@ -1115,9 +1117,26 @@ sds ACLLoadFromFile(const char *filename) { continue; } } - if (j != argc) continue; /* Error in ACL rules, don't apply. */ + if (j != argc) { + sdsfreesplitres(argv,argc); + continue; /* Error in ACL rules, don't apply. */ + } - /* We can finally lookup the user and apply the rule. */ + /* We can finally lookup the user and apply the rule. If the + * user already exists we always reset it to start. */ + user *u = ACLCreateUser(argv[1],sdslen(argv[1])); + if (!u) { + u = ACLGetUserByName(argv[1],sdslen(argv[1])); + serverAssert(u != NULL); + ACLSetUser(u,"reset",-1); + } + + /* Note that the same rules already applied to the fake user, so + * we just assert that everything goess well: it should. */ + for (j = 2; j < argc; j++) + serverAssert(ACLSetUser(fakeuser,argv[j],sdslen(argv[j]) == C_OK); + + sdsfreesplitres(argv,argc); } ACLFreeUser(fakeuser); From dfb8b08f30bb48e97264b6cafcad12c1d136c84b Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 7 Feb 2019 12:20:30 +0100 Subject: [PATCH 014/101] ACL: implement LOAD subcommand plus some minor rafactoring. --- src/acl.c | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/src/acl.c b/src/acl.c index 1c6e9a0c5..d6b22f7c7 100644 --- a/src/acl.c +++ b/src/acl.c @@ -758,10 +758,9 @@ sds ACLDefaultUserFirstPassword(void) { return listNodeValue(first); } -/* Initialization of the ACL subsystem. */ -void ACLInit(void) { - Users = raxNew(); - UsersToLoad = listCreate(); +/* Initialize the default user, that will always exist for all the process + * lifetime. */ +void ACLInitDefaultUser(void) { DefaultUser = ACLCreateUser("default",7); ACLSetUser(DefaultUser,"+@all",-1); ACLSetUser(DefaultUser,"~*",-1); @@ -769,6 +768,13 @@ void ACLInit(void) { ACLSetUser(DefaultUser,"nopass",-1); } +/* Initialization of the ACL subsystem. */ +void ACLInit(void) { + Users = raxNew(); + UsersToLoad = listCreate(); + ACLInitDefaultUser(); +} + /* Check the username and password pair and return C_OK if they are valid, * otherwise C_ERR is returned and errno is set to: * @@ -1134,7 +1140,7 @@ sds ACLLoadFromFile(const char *filename) { /* Note that the same rules already applied to the fake user, so * we just assert that everything goess well: it should. */ for (j = 2; j < argc; j++) - serverAssert(ACLSetUser(fakeuser,argv[j],sdslen(argv[j]) == C_OK); + serverAssert(ACLSetUser(fakeuser,argv[j],sdslen(argv[j])) == C_OK); sdsfreesplitres(argv,argc); } @@ -1297,6 +1303,19 @@ void aclCommand(client *c) { } else { addReplyNull(c); } + } else if (!strcasecmp(sub,"load")) { + if (server.acl_filename[0] == '\0') { + addReplyError(c,"This Redis instance is not configured to use an ACL file. You may want to specify users via the ACL SETUSER command and then issue a CONFIG REWRITE (assuming you have a Redis configuration file set) in order to store users in the Redis configuration."); + return; + } else { + sds errors = ACLLoadFromFile(server.acl_filename); + if (errors == NULL) { + addReply(c,shared.ok); + } else { + addReplyError(c,errors); + sdsfree(errors); + } + } } else if (!strcasecmp(sub,"help")) { const char *help[] = { "LIST -- Show user details in config file format.", From f7b86d2b8fa41f285f5bfeb45538bd1c7e5825ff Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 7 Feb 2019 12:57:21 +0100 Subject: [PATCH 015/101] ACL: WIP: preserve the old config on loading errors. --- src/acl.c | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 2 deletions(-) diff --git a/src/acl.c b/src/acl.c index d6b22f7c7..45c875de9 100644 --- a/src/acl.c +++ b/src/acl.c @@ -90,6 +90,7 @@ struct ACLUserFlag { void ACLResetSubcommandsForCommand(user *u, unsigned long id); void ACLResetSubcommands(user *u); +void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub); /* ============================================================================= * Helper functions for the rest of the ACL implementation @@ -163,6 +164,11 @@ void ACLListFreeSds(void *item) { sdsfree(item); } +/* Method to duplicate list elements from ACL users password/ptterns lists. */ +void *ACLListDupSds(void *item) { + return sdsdup(item); +} + /* Create a new user with the specified name, store it in the list * of users (the Users global radix tree), and returns a reference to * the structure representing the user. @@ -178,8 +184,10 @@ user *ACLCreateUser(const char *name, size_t namelen) { u->patterns = listCreate(); listSetMatchMethod(u->passwords,ACLListMatchSds); listSetFreeMethod(u->passwords,ACLListFreeSds); + listSetDupMethod(u->passwords,ACLListDupSds); listSetMatchMethod(u->patterns,ACLListMatchSds); listSetFreeMethod(u->patterns,ACLListFreeSds); + listSetDupMethod(u->patterns,ACLListDupSds); memset(u->allowed_commands,0,sizeof(u->allowed_commands)); raxInsert(Users,(unsigned char*)name,namelen,u,NULL); return u; @@ -212,6 +220,38 @@ void ACLFreeUser(user *u) { zfree(u); } +/* Copy the user ACL rules from the source user 'src' to the destination + * user 'dst' so that at the end of the process they'll have exactly the + * same rules (but the names will continue to be the original ones). */ +void ACLCopyUser(user *dst, user *src) { + listRelease(dst->passwords); + listRelease(dst->patterns); + dst->passwords = listDup(src->passwords); + dst->patterns = listDup(src->patterns); + memcpy(dst->allowed_commands,src->allowed_commands, + sizeof(dst->allowed_commands)); + dst->flags = src->flags; + ACLResetSubcommands(dst); + /* Copy the allowed subcommands array of array of SDS strings. */ + if (src->allowed_subcommands) { + for (int j = 0; j < USER_COMMAND_BITS_COUNT; j++) { + if (src->allowed_subcommands[j]) { + for (int i = 0; src->allowed_subcommands[j][i]; i++) + { + ACLAddAllowedSubcommand(dst, j, + src->allowed_subcommands[j][i]); + } + } + } + } +} + +/* Free all the users registered in the radix tree 'users' and free the + * radix tree itself. */ +void ACLFreeUsersSet(rax *users) { + /* TODO */ +} + /* Given a command ID, this function set by reference 'word' and 'bit' * so that user->allowed_commands[word] will address the right word * where the corresponding bit for the provided ID is stored, and @@ -1043,7 +1083,10 @@ int ACLLoadConfiguredUsers(void) { * to avoid ending with broken rules if the ACL file is invalid for some * reason, so the function will attempt to validate the rules before loading * each user. For every line that will be found broken the function will - * collect an error message. All the valid lines will be correctly processed. + * collect an error message. + * + * IMPORTANT: If there is at least a single error, nothing will be loaded + * and the rules will remain exactly as they were. * * At the end of the process, if no errors were found in the whole file then * NULL is returned. Otherwise an SDS string describing in a single line @@ -1075,6 +1118,14 @@ sds ACLLoadFromFile(const char *filename) { * to the real user mentioned in the ACL line. */ user *fakeuser = ACLCreateUnlinkedUser(); + /* We do all the loading in a fresh insteance of the Users radix tree, + * so if there are errors loading the ACL file we can rollback to the + * old version. */ + rax *old_users = Users; + Users = raxNew(); + ACLInitDefaultUser(); + + /* Load each line of the file. */ for (int i = 0; i < totlines; i++) { sds *argv; int argc; @@ -1147,11 +1198,24 @@ sds ACLLoadFromFile(const char *filename) { ACLFreeUser(fakeuser); sdsfreesplitres(lines,totlines); + + /* Chec if we found errors and react accordingly. */ if (sdslen(errors) == 0) { + /* The default user pointer is referenced in different places: instead + * of replacing such occurrences it is much simpler to copy the new + * default user configuration in the old one. */ + user *new = ACLGetUserByName("default",7); + ACLCopyUser(DefaultUser,new); + ACLFreeUser(new); + raxInsert(Users,(unsigned char*)"default",7,DefaultUser,NULL); + sdsfree(errors); return NULL; } else { - return sdstrim(errors," "); + ACLFreeUsersSet(Users); + Users = old_users; + errors = sdscat(errors,"WARNING: ACL errors detected, no change to the previously active ACL rules was performed"); + return errors; } } From 61c7688b75aaf840a3273ad9685d5676bdbed52b Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 7 Feb 2019 16:20:42 +0100 Subject: [PATCH 016/101] ACL: fix a few ACLLoadFromFile() errors and finish ACLFreeUsersSet(). --- src/acl.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/acl.c b/src/acl.c index 45c875de9..3133bfecf 100644 --- a/src/acl.c +++ b/src/acl.c @@ -249,7 +249,7 @@ void ACLCopyUser(user *dst, user *src) { /* Free all the users registered in the radix tree 'users' and free the * radix tree itself. */ void ACLFreeUsersSet(rax *users) { - /* TODO */ + raxFreeWithCallback(users,(void(*)(void*))ACLFreeUser); } /* Given a command ID, this function set by reference 'word' and 'bit' @@ -1208,7 +1208,8 @@ sds ACLLoadFromFile(const char *filename) { ACLCopyUser(DefaultUser,new); ACLFreeUser(new); raxInsert(Users,(unsigned char*)"default",7,DefaultUser,NULL); - + raxRemove(old_users,(unsigned char*)"default",7,NULL); + ACLFreeUsersSet(old_users); sdsfree(errors); return NULL; } else { From 735cb69f12f720382ef695925f0083b4aa772788 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 7 Feb 2019 16:47:14 +0100 Subject: [PATCH 017/101] ACL: add assertion and fix comment typo. --- src/acl.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/acl.c b/src/acl.c index 3133bfecf..12253f6cf 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1199,12 +1199,13 @@ sds ACLLoadFromFile(const char *filename) { ACLFreeUser(fakeuser); sdsfreesplitres(lines,totlines); - /* Chec if we found errors and react accordingly. */ + /* Check if we found errors and react accordingly. */ if (sdslen(errors) == 0) { /* The default user pointer is referenced in different places: instead * of replacing such occurrences it is much simpler to copy the new * default user configuration in the old one. */ user *new = ACLGetUserByName("default",7); + serverAssert(new != NULL); ACLCopyUser(DefaultUser,new); ACLFreeUser(new); raxInsert(Users,(unsigned char*)"default",7,DefaultUser,NULL); From 68d127d7b6856d7680e04bff999453b74d69dd9b Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 7 Feb 2019 16:53:35 +0100 Subject: [PATCH 018/101] ACL: fix fgets wrong buffer size. --- src/acl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/acl.c b/src/acl.c index 12253f6cf..b17036df7 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1105,7 +1105,7 @@ sds ACLLoadFromFile(const char *filename) { /* Load the whole file as a single string in memory. */ sds acls = sdsempty(); - while(fgets(buf,CONFIG_MAX_LINE+1,fp) != NULL) + while(fgets(buf,sizeof(buf),fp) != NULL) acls = sdscat(acls,buf); fclose(fp); From 434489abb7d033f3b27ba28697402fe75ae385c0 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 7 Feb 2019 17:00:35 +0100 Subject: [PATCH 019/101] ACL: ACLLoadFromFile(), restore DefaultUser global. --- src/acl.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/acl.c b/src/acl.c index b17036df7..4ab660e9f 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1122,6 +1122,7 @@ sds ACLLoadFromFile(const char *filename) { * so if there are errors loading the ACL file we can rollback to the * old version. */ rax *old_users = Users; + user *old_default_user = DefaultUser; Users = raxNew(); ACLInitDefaultUser(); @@ -1198,6 +1199,7 @@ sds ACLLoadFromFile(const char *filename) { ACLFreeUser(fakeuser); sdsfreesplitres(lines,totlines); + DefaultUser = old_default_user; /* This pointer must never change. */ /* Check if we found errors and react accordingly. */ if (sdslen(errors) == 0) { From b1db13d8fa807b155f5bfaa25a002286af9a4ea2 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 7 Feb 2019 17:07:35 +0100 Subject: [PATCH 020/101] ACL: ACLLoadFromFile(): several errors fixed to make it work. --- src/acl.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/acl.c b/src/acl.c index 4ab660e9f..fecd33e8a 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1141,8 +1141,8 @@ sds ACLLoadFromFile(const char *filename) { argv = sdssplitargs(lines[i],&argc); if (argv == NULL) { errors = sdscatprintf(errors, - "%d: unbalanced quotes in acl line. ", - linenum); + "%s:%d: unbalanced quotes in acl line. ", + server.acl_filename, linenum); continue; } @@ -1155,8 +1155,8 @@ sds ACLLoadFromFile(const char *filename) { /* The line should start with the "user" keyword. */ if (strcmp(argv[0],"user") || argc < 2) { errors = sdscatprintf(errors, - "%d: line should start with user keyword followed " - "by the username. ", + "%s:%d should start with user keyword followed " + "by the username. ", server.acl_filename, linenum); sdsfreesplitres(argv,argc); continue; @@ -1170,14 +1170,18 @@ sds ACLLoadFromFile(const char *filename) { if (ACLSetUser(fakeuser,argv[j],sdslen(argv[j])) != C_OK) { char *errmsg = ACLSetUserStringError(); errors = sdscatprintf(errors, - "%d: error in ACL: %s. ", - linenum, errmsg); + "%s:%d: %s. ", + server.acl_filename, linenum, errmsg); continue; } } - if (j != argc) { + + /* Apply the rule to the new users set only if so far there + * are no errors, otherwise it's useless since we are going + * to discard the new users set anyway. */ + if (sdslen(errors) != 0) { sdsfreesplitres(argv,argc); - continue; /* Error in ACL rules, don't apply. */ + continue; } /* We can finally lookup the user and apply the rule. If the @@ -1192,7 +1196,7 @@ sds ACLLoadFromFile(const char *filename) { /* Note that the same rules already applied to the fake user, so * we just assert that everything goess well: it should. */ for (j = 2; j < argc; j++) - serverAssert(ACLSetUser(fakeuser,argv[j],sdslen(argv[j])) == C_OK); + serverAssert(ACLSetUser(u,argv[j],sdslen(argv[j])) == C_OK); sdsfreesplitres(argv,argc); } From 87ce87e68c7e307bfe4e7fb2dc3d4d5499bef4f4 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 7 Feb 2019 17:20:03 +0100 Subject: [PATCH 021/101] ACL: load ACL file at startup. Prevent silly configurations. --- src/acl.c | 33 +++++++++++++++++++++++++++++++++ src/server.c | 6 +----- src/server.h | 1 + 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/src/acl.c b/src/acl.c index fecd33e8a..4ae5830bd 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1227,6 +1227,39 @@ sds ACLLoadFromFile(const char *filename) { } } +/* This function is called once the server is already running, modules are + * loaded, and we are ready to start, in order to load the ACLs either from + * the pending list of users defined in redis.conf, or from the ACL file. + * The function will just exit with an error if the user is trying to mix + * both the loading methods. */ +void ACLLoadUsersAtStartup(void) { + if (server.acl_filename[0] != '\0' && listLength(UsersToLoad) != 0) { + serverLog(LL_WARNING, + "Configuring Redis with users defined in redis.conf and at " + "the same setting an ACL file path is invalid. This setup " + "is very likely to lead to configuration errors and security " + "holes, please define either an ACL file or declare users " + "directly in your redis.conf, but not both."); + exit(1); + } + + if (ACLLoadConfiguredUsers() == C_ERR) { + serverLog(LL_WARNING, + "Critical error while loading ACLs. Exiting."); + exit(1); + } + + if (server.acl_filename[0] != '\0') { + sds errors = ACLLoadFromFile(server.acl_filename); + if (errors) { + serverLog(LL_WARNING, + "Aborting Redis startup because of ACL errors: %s", errors); + sdsfree(errors); + exit(1); + } + } +} + /* ============================================================================= * ACL related commands * ==========================================================================*/ diff --git a/src/server.c b/src/server.c index de84e430e..c257d0573 100644 --- a/src/server.c +++ b/src/server.c @@ -4908,11 +4908,7 @@ int main(int argc, char **argv) { linuxMemoryWarnings(); #endif moduleLoadFromQueue(); - if (ACLLoadConfiguredUsers() == C_ERR) { - serverLog(LL_WARNING, - "Critical error while loading ACLs. Exiting."); - exit(1); - } + ACLLoadUsersAtStartup(); loadDataFromDisk(); if (server.cluster_enabled) { if (verifyClusterConfigWithData() == C_ERR) { diff --git a/src/server.h b/src/server.h index d2c6aa1e0..59f7cbe10 100644 --- a/src/server.h +++ b/src/server.h @@ -1746,6 +1746,7 @@ int ACLAppendUserForLoading(sds *argv, int argc, int *argc_err); char *ACLSetUserStringError(void); int ACLLoadConfiguredUsers(void); sds ACLDescribeUser(user *u); +void ACLLoadUsersAtStartup(void); /* Sorted sets data type */ From 4a7062f9bd77ca75af37454bf5a77d86b8924ff6 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 8 Feb 2019 09:52:07 +0100 Subject: [PATCH 022/101] ACL: some documentation inside redis.conf. --- redis.conf | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/redis.conf b/redis.conf index d1ced7eb3..71214b620 100644 --- a/redis.conf +++ b/redis.conf @@ -501,6 +501,94 @@ replica-priority 100 # can be easily a long string from /dev/urandom or whatever, so by using a # long and unguessable password no brute force attack will be possible. +# Redis ACL users are defined in the following format: +# +# user ... acl rules ... +# +# For example: +# +# user worker +@list +@connection ~jobs:* on >ffa9203c493aa99 +# +# The special username "default" is used for new connections. If this user +# has the "nopass" rule, then new connections will be immediately authenticated +# as the "default" user without the need of any password provided via the +# AUTH command. Otherwise if the "default" user is not flagged with "nopass" +# the connections will start in not authenticated state, and will require +# AUTH (or the HELLO command AUTH option) in order to be authenticated and +# start to work. +# +# The ACL rules that describe what an user can do are the following: +# +# on Enable the user: it is possible to authenticate as this user. +# off Disable the user: it's no longer possible to authenticate +# with this user, however the already authenticated connections +# will still work. +# + Allow the execution of that command +# - Disallow the execution of that command +# +@ Allow the execution of all the commands in such category +# with valid categories are like @admin, @set, @sortedset, ... +# and so forth, see the full list in the server.c file where +# the Redis command table is described and defined. +# The special category @all means all the commands, but currently +# present in the server, and that will be loaded in the future +# via modules. +# +|subcommand Allow a specific subcommand of an otherwise +# disabled command. Note that this form is not +# allowed as negative like -DEBUG|SEGFAULT, but +# only additive starting with "+". +# allcommands Alias for +@all. Note that it implies the ability to execute +# all the future commands loaded via the modules system. +# nocommands Alias for -@all. +# ~ Add a pattern of keys that can be mentioned as part of +# commands. For instance ~* allows all the keys. The pattern +# is a glob-style pattern like the one of KEYS. +# It is possible to specify multiple patterns. +# allkeys Alias for ~* +# resetkeys Flush the list of allowed keys patterns. +# > Add this passowrd to the list of valid password for the user. +# For example >mypass will add "mypass" to the list. +# This directive clears the "nopass" flag (see later). +# < Remove this password from the list of valid passwords. +# nopass All the set passwords of the user are removed, and the user +# is flagged as requiring no password: it means that every +# password will work against this user. If this directive is +# used for the default user, every new connection will be +# immediately authenticated with the default user without +# any explicit AUTH command required. Note that the "resetpass" +# directive will clear this condition. +# resetpass Flush the list of allowed passwords. Moreover removes the +# "nopass" status. After "resetpass" the user has no associated +# passwords and there is no way to authenticate without adding +# some password (or setting it as "nopass" later). +# reset Performs the following actions: resetpass, resetkeys, off, +# -@all. The user returns to the same state it has immediately +# after its creation. +# +# ACL rules can be specified in any order: for instance you can start with +# passwords, then flags, or key patterns. However note that the additive +# and subtractive rules will CHANGE MEANING depending on the ordering. +# For instance see the following example: +# +# user alice on +@all -DEBUG ~* >somepassword +# +# This will allow "alice" to use all the commands with the exception of the +# DEBUG command, since +@all added all the commands to the set of the commands +# alice can use, and later DEBUG was removed. However if we invert the order +# of two ACL rules the result will be different: +# +# user alice on -DEBUG +@all ~* >somepassword +# +# Now DEBUG was removed when alice had yet no commands in the set of allowed +# commands, later all the commands are added, so the user will be able to +# execute everything. +# +# Basically ACL rules are processed left-to-right. +# +# For more information about ACL configuration please refer to +# the Redis web site at https://redis.io/topics/acl + +# Using an external ACL file +# # Instead of configuring users here in this file, it is possible to use # a stand-alone file just listing users. The two methods cannot be mixed: # if you configure users here and at the same time you activate the exteranl From 66fd5e058f7353cc45936957deafb37e4d4f2e60 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 8 Feb 2019 11:50:39 +0100 Subject: [PATCH 023/101] ACL: ignore modules commands when adding categories. We can't trust modules commands flagging, so module commands must be always explicitly added, with the exception of +@all that will include everything. However something like +@readonly should not include command from modules that may be potentially dangerous: our categories must be safe and reliable and modules may not be like that. --- src/acl.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/acl.c b/src/acl.c index 4ae5830bd..159a3507a 100644 --- a/src/acl.c +++ b/src/acl.c @@ -313,6 +313,7 @@ int ACLSetUserCommandBitsForCategory(user *u, const char *category, int value) { dictEntry *de; while ((de = dictNext(di)) != NULL) { struct redisCommand *cmd = dictGetVal(de); + if (cmd->flags & CMD_MODULE) continue; /* Ignore modules commands. */ if (cmd->flags & cflag) { ACLSetUserCommandBit(u,cmd->id,value); ACLResetSubcommandsForCommand(u,cmd->id); From fcd5ff1a768b9451bac1dafd92d72086a167a410 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 8 Feb 2019 12:38:41 +0100 Subject: [PATCH 024/101] ACL: add arity check in ACL command where missing. --- src/acl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/acl.c b/src/acl.c index 159a3507a..2e2bd24a5 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1403,13 +1403,13 @@ void aclCommand(client *c) { } } raxStop(&ri); - } else if (!strcasecmp(sub,"whoami")) { + } else if (!strcasecmp(sub,"whoami") && c->argc == 2) { if (c->user != NULL) { addReplyBulkCBuffer(c,c->user->name,sdslen(c->user->name)); } else { addReplyNull(c); } - } else if (!strcasecmp(sub,"load")) { + } else if (!strcasecmp(sub,"load") && c->argc == 2) { if (server.acl_filename[0] == '\0') { addReplyError(c,"This Redis instance is not configured to use an ACL file. You may want to specify users via the ACL SETUSER command and then issue a CONFIG REWRITE (assuming you have a Redis configuration file set) in order to store users in the Redis configuration."); return; From 48423054ead6b6f6c183efd9eac0a6e8fca3de05 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 8 Feb 2019 12:40:42 +0100 Subject: [PATCH 025/101] ACL: add command fingerprint for CAT subcommand. --- src/acl.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/acl.c b/src/acl.c index 2e2bd24a5..3d73e0843 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1268,7 +1268,9 @@ void ACLLoadUsersAtStartup(void) { /* ACL -- show and modify the configuration of ACL users. * ACL HELP * ACL LIST - * ACL SETUSER ... user attribs ... + * ACL USERS + * ACL CAT [] + * ACL SETUSER ... acl rules ... * ACL DELUSER * ACL GETUSER */ @@ -1429,6 +1431,8 @@ void aclCommand(client *c) { "SETUSER [attribs ...] -- Create or modify a user.", "GETUSER -- Get the user details.", "DELUSER -- Delete a user.", +"CAT -- List available categories.", +"CAT -- List commands inside category.", "WHOAMI -- Return the current connection username.", NULL }; From cedd67eaff8982c69f5b581badc6ed86ef5fe3d1 Mon Sep 17 00:00:00 2001 From: John Sully Date: Fri, 8 Feb 2019 17:56:32 -0500 Subject: [PATCH 026/101] remove Redis branding --- .gitignore | 7 +++++++ src/Makefile | 24 ++++++++++++------------ src/asciilogo.h | 26 +++++++++----------------- 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/.gitignore b/.gitignore index a188cfc82..2b2e15eba 100644 --- a/.gitignore +++ b/.gitignore @@ -3,12 +3,19 @@ *.log dump.rdb redis-benchmark +keydb-benchmark redis-check-aof +keydb-check-aof redis-check-rdb +keydb-check-rdb redis-check-dump +keydb-check-dump redis-cli +keydb-cli redis-sentinel +keydb-sentinel redis-server +keydb-server doc-tools release misc/* diff --git a/src/Makefile b/src/Makefile index b2c7cc891..8997fa81b 100644 --- a/src/Makefile +++ b/src/Makefile @@ -172,15 +172,15 @@ QUIET_LINK = @printf ' %b %b\n' $(LINKCOLOR)LINK$(ENDCOLOR) $(BINCOLOR)$@$(EN QUIET_INSTALL = @printf ' %b %b\n' $(LINKCOLOR)INSTALL$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) 1>&2; endif -REDIS_SERVER_NAME=redis-server -REDIS_SENTINEL_NAME=redis-sentinel +REDIS_SERVER_NAME=keydb-server +REDIS_SENTINEL_NAME=keydb-sentinel REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o acl.o storage.o rdb-s3.o -REDIS_CLI_NAME=redis-cli +REDIS_CLI_NAME=keydb-cli REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o siphash.o crc16.o storage-lite.o -REDIS_BENCHMARK_NAME=redis-benchmark +REDIS_BENCHMARK_NAME=keydb-benchmark REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o zmalloc.o redis-benchmark.o storage-lite.o -REDIS_CHECK_RDB_NAME=redis-check-rdb -REDIS_CHECK_AOF_NAME=redis-check-aof +REDIS_CHECK_RDB_NAME=keydb-check-rdb +REDIS_CHECK_AOF_NAME=keydb-check-aof all: $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME) @echo "" @@ -224,27 +224,27 @@ ifneq ($(strip $(PREV_FINAL_LDFLAGS)), $(strip $(FINAL_LDFLAGS))) .make-prerequisites: persist-settings endif -# redis-server +# keydb-server $(REDIS_SERVER_NAME): $(REDIS_SERVER_OBJ) $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/lua/src/liblua.a $(FINAL_LIBS) -# redis-sentinel +# keydb-sentinel $(REDIS_SENTINEL_NAME): $(REDIS_SERVER_NAME) $(REDIS_INSTALL) $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) -# redis-check-rdb +# keydb-check-rdb $(REDIS_CHECK_RDB_NAME): $(REDIS_SERVER_NAME) $(REDIS_INSTALL) $(REDIS_SERVER_NAME) $(REDIS_CHECK_RDB_NAME) -# redis-check-aof +# keydb-check-aof $(REDIS_CHECK_AOF_NAME): $(REDIS_SERVER_NAME) $(REDIS_INSTALL) $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) -# redis-cli +# keydb-cli $(REDIS_CLI_NAME): $(REDIS_CLI_OBJ) $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/linenoise/linenoise.o $(FINAL_LIBS) -# redis-benchmark +# keydb-benchmark $(REDIS_BENCHMARK_NAME): $(REDIS_BENCHMARK_OBJ) $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a $(FINAL_LIBS) diff --git a/src/asciilogo.h b/src/asciilogo.h index 83c538b54..514902001 100644 --- a/src/asciilogo.h +++ b/src/asciilogo.h @@ -28,20 +28,12 @@ */ char *ascii_logo = -" _._ \n" -" _.-``__ ''-._ \n" -" _.-`` `. `_. ''-._ Redis %s (%s/%d) %s bit\n" -" .-`` .-```. ```\\/ _.,_ ''-._ \n" -" ( ' , .-` | `, ) Running in %s mode\n" -" |`-._`-...-` __...-.``-._|'` _.-'| Port: %d\n" -" | `-._ `._ / _.-' | PID: %ld\n" -" `-._ `-._ `-./ _.-' _.-' \n" -" |`-._`-._ `-.__.-' _.-'_.-'| \n" -" | `-._`-._ _.-'_.-' | http://redis.io \n" -" `-._ `-._`-.__.-'_.-' _.-' \n" -" |`-._`-._ `-.__.-' _.-'_.-'| \n" -" | `-._`-._ _.-'_.-' | \n" -" `-._ `-._`-.__.-'_.-' _.-' \n" -" `-._ `-.__.-' _.-' \n" -" `-._ _.-' \n" -" `-.__.-' \n\n"; +" \n" +" \n" +" KeyDB %s (%s/%d) %s bit\n" +" \n" +" Running in %s mode\n" +" Port: %d\n" +" PID: %ld\n" +" \n" +" \n\n"; From 5420f93de92852da4e608e7110c8f4bec90b9abe Mon Sep 17 00:00:00 2001 From: John Sully Date: Sat, 9 Feb 2019 07:56:45 -0500 Subject: [PATCH 027/101] Fix bug where we try to load a database with no name --- src/server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index c03a6d9d7..e9eae14e0 100644 --- a/src/server.c +++ b/src/server.c @@ -4590,7 +4590,7 @@ void loadDataFromDisk(void) { if (server.aof_state == AOF_ON) { if (loadAppendOnlyFile(server.aof_filename) == C_OK) serverLog(LL_NOTICE,"DB loaded from append only file: %.3f seconds",(float)(ustime()-start)/1000000); - } else { + } else if (server.rdb_filename != NULL) { rdbSaveInfo rsi = RDB_SAVE_INFO_INIT; if (rdbLoad(server.rdb_filename,&rsi) == C_OK) { serverLog(LL_NOTICE,"DB loaded from disk: %.3f seconds", From dac121346216924cb1116cb0fe9863f4e416ceee Mon Sep 17 00:00:00 2001 From: John Sully Date: Sat, 9 Feb 2019 10:11:46 -0500 Subject: [PATCH 028/101] complete rebranding with tests passing --- README.md | 12 +++---- deps/hiredis/Makefile | 4 +-- redis.conf | 8 ++--- sentinel.conf | 8 ++--- src/aof.c | 6 ++-- src/cluster.c | 6 ++-- src/debug.c | 2 +- src/latency.c | 2 +- src/rdb.c | 4 +-- src/redis-benchmark.c | 14 ++++---- src/redis-check-rdb.c | 2 +- src/redis-cli.c | 34 +++++++++---------- src/redis-trib.rb | 18 +++++----- src/replication.c | 2 +- src/server.c | 34 +++++++++---------- src/server.h | 2 +- tests/cluster/tests/04-resharding.tcl | 2 +- .../cluster/tests/12-replica-migration-2.tcl | 4 +-- tests/cluster/tests/helpers/onlydots.tcl | 4 +-- tests/instances.tcl | 8 ++--- tests/integration/aof-race.tcl | 2 +- tests/integration/aof.tcl | 6 ++-- tests/integration/psync2-reg.tcl | 4 +-- tests/integration/redis-cli.tcl | 4 +-- tests/sentinel/tests/07-down-conditions.tcl | 2 +- tests/support/server.tcl | 6 ++-- tests/support/test.tcl | 2 +- tests/unit/wait.tcl | 4 +-- utils/cluster_fail_time.tcl | 8 ++--- utils/create-cluster/README | 2 +- utils/create-cluster/create-cluster | 12 +++---- utils/generate-command-help.rb | 2 +- utils/install_server.sh | 8 ++--- utils/redis-copy.rb | 16 ++++----- utils/redis-sha1.rb | 2 +- utils/redis_init_script | 4 +-- utils/releasetools/01_create_tarball.sh | 4 +-- utils/releasetools/02_upload_tarball.sh | 2 +- utils/releasetools/03_test_release.sh | 6 ++-- utils/releasetools/04_release_hash.sh | 10 +++--- utils/speed-regression.tcl | 12 +++---- 41 files changed, 147 insertions(+), 147 deletions(-) diff --git a/README.md b/README.md index 2b4eeb19b..0940f6ce3 100644 --- a/README.md +++ b/README.md @@ -108,19 +108,19 @@ Running Redis To run Redis with the default configuration just type: % cd src - % ./redis-server + % ./keydb-server If you want to provide your redis.conf, you have to run it using an additional parameter (the path of the configuration file): % cd src - % ./redis-server /path/to/redis.conf + % ./keydb-server /path/to/redis.conf It is possible to alter the Redis configuration by passing parameters directly as options using the command line. Examples: - % ./redis-server --port 9999 --replicaof 127.0.0.1 6379 - % ./redis-server /etc/redis/6379.conf --loglevel debug + % ./keydb-server --port 9999 --replicaof 127.0.0.1 6379 + % ./keydb-server /etc/redis/6379.conf --loglevel debug All the options in redis.conf are also supported as options using the command line, with exactly the same name. @@ -128,11 +128,11 @@ line, with exactly the same name. Playing with Redis ------------------ -You can use redis-cli to play with Redis. Start a redis-server instance, +You can use keydb-cli to play with Redis. Start a keydb-server instance, then in another terminal try the following: % cd src - % ./redis-cli + % ./keydb-cli redis> ping PONG redis> set foo bar diff --git a/deps/hiredis/Makefile b/deps/hiredis/Makefile index 06ca99468..aac8aa06c 100644 --- a/deps/hiredis/Makefile +++ b/deps/hiredis/Makefile @@ -23,9 +23,9 @@ INSTALL_INCLUDE_PATH= $(DESTDIR)$(PREFIX)/$(INCLUDE_PATH) INSTALL_LIBRARY_PATH= $(DESTDIR)$(PREFIX)/$(LIBRARY_PATH) INSTALL_PKGCONF_PATH= $(INSTALL_LIBRARY_PATH)/$(PKGCONF_PATH) -# redis-server configuration used for testing +# keydb-server configuration used for testing REDIS_PORT=56379 -REDIS_SERVER=redis-server +REDIS_SERVER=keydb-server define REDIS_TEST_CONFIG daemonize yes pidfile /tmp/hiredis-test-redis.pid diff --git a/redis.conf b/redis.conf index 18eda41b9..2d0de9610 100644 --- a/redis.conf +++ b/redis.conf @@ -3,7 +3,7 @@ # Note that in order to read the configuration file, Redis must be # started with the file path as first argument: # -# ./redis-server /path/to/redis.conf +# ./keydb-server /path/to/redis.conf # Note on units: when memory size is needed, it is possible to specify # it in the usual form of 1k 5GB 4M and so forth: @@ -804,7 +804,7 @@ auto-aof-rewrite-min-size 64mb # the Redis server starts emitting a log to inform the user of the event. # Otherwise if the option is set to no, the server aborts with an error # and refuses to start. When the option is set to no, the user requires -# to fix the AOF file using the "redis-check-aof" utility before to restart +# to fix the AOF file using the "keydb-check-aof" utility before to restart # the server. # # Note that if the AOF file will be found to be corrupted in the middle @@ -1313,8 +1313,8 @@ rdb-save-incremental-fsync yes # # NOTE: The above table was obtained by running the following commands: # -# redis-benchmark -n 1000000 incr foo -# redis-cli object freq foo +# keydb-benchmark -n 1000000 incr foo +# keydb-cli object freq foo # # NOTE 2: The counter initial value is 5 in order to give new objects a chance # to accumulate hits. diff --git a/sentinel.conf b/sentinel.conf index bc9a705ac..d8f009f42 100644 --- a/sentinel.conf +++ b/sentinel.conf @@ -21,14 +21,14 @@ port 26379 # By default Redis Sentinel does not run as a daemon. Use 'yes' if you need it. -# Note that Redis will write a pid file in /var/run/redis-sentinel.pid when +# Note that Redis will write a pid file in /var/run/keydb-sentinel.pid when # daemonized. daemonize no # When running daemonized, Redis Sentinel writes a pid file in -# /var/run/redis-sentinel.pid by default. You can specify a custom pid file +# /var/run/keydb-sentinel.pid by default. You can specify a custom pid file # location here. -pidfile /var/run/redis-sentinel.pid +pidfile /var/run/keydb-sentinel.pid # Specify the log file name. Also the empty string can be used to force # Sentinel to log on the standard output. Note that if you use standard @@ -64,7 +64,7 @@ logfile "" # unmounting filesystems. dir /tmp -# sentinel monitor +# sentinel monitor # # Tells Sentinel to monitor this master, and to consider it in O_DOWN # (Objectively Down) state only if at least sentinels agree. diff --git a/src/aof.c b/src/aof.c index 1a7ad4072..d548c72b8 100644 --- a/src/aof.c +++ b/src/aof.c @@ -867,12 +867,12 @@ uxeof: /* Unexpected AOF end of file. */ } } if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */ - serverLog(LL_WARNING,"Unexpected end of file reading the append only file. You can: 1) Make a backup of your AOF file, then use ./redis-check-aof --fix . 2) Alternatively you can set the 'aof-load-truncated' configuration option to yes and restart the server."); + serverLog(LL_WARNING,"Unexpected end of file reading the append only file. You can: 1) Make a backup of your AOF file, then use ./keydb-check-aof --fix . 2) Alternatively you can set the 'aof-load-truncated' configuration option to yes and restart the server."); exit(1); fmterr: /* Format error. */ if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */ - serverLog(LL_WARNING,"Bad file format reading the append only file: make a backup of your AOF file, then use ./redis-check-aof --fix "); + serverLog(LL_WARNING,"Bad file format reading the append only file: make a backup of your AOF file, then use ./keydb-check-aof --fix "); exit(1); } @@ -1546,7 +1546,7 @@ int rewriteAppendOnlyFileBackground(void) { /* Child */ closeListeningSockets(0); - redisSetProcTitle("redis-aof-rewrite"); + redisSetProcTitle("keydb-aof-rewrite"); snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid()); if (rewriteAppendOnlyFile(tmpfile) == C_OK) { size_t private_dirty = zmalloc_get_private_dirty(-1); diff --git a/src/cluster.c b/src/cluster.c index 11703a359..c4c0d814c 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -1104,7 +1104,7 @@ void clusterHandleConfigEpochCollision(clusterNode *sender) { * about the node we want to remove, we don't re-add it before some time. * * Currently the CLUSTER_BLACKLIST_TTL is set to 1 minute, this means - * that redis-trib has 60 seconds to send CLUSTER FORGET messages to nodes + * that keydb-trib has 60 seconds to send CLUSTER FORGET messages to nodes * in the cluster without dealing with the problem of other nodes re-adding * back the node to nodes we already sent the FORGET command to. * @@ -1556,7 +1556,7 @@ void clusterUpdateSlotsConfigWith(clusterNode *sender, uint64_t senderConfigEpoc if (server.cluster->slots[j] == sender) continue; /* The slot is in importing state, it should be modified only - * manually via redis-trib (example: a resharding is in progress + * manually via keydb-trib (example: a resharding is in progress * and the migrating side slot was already closed and is advertising * a new config. We still want the slot to be closed manually). */ if (server.cluster->importing_slots_from[j]) continue; @@ -3861,7 +3861,7 @@ void clusterUpdateState(void) { * B) If according to our config other nodes are already in charge for * this lots, we set the slots as IMPORTING from our point of view * in order to justify we have those slots, and in order to make - * redis-trib aware of the issue, so that it can try to fix it. + * keydb-trib aware of the issue, so that it can try to fix it. * 2) If we find data in a DB different than DB0 we return C_ERR to * signal the caller it should quit the server with an error message * or take other actions. diff --git a/src/debug.c b/src/debug.c index 078282507..36942e61e 100644 --- a/src/debug.c +++ b/src/debug.c @@ -1383,7 +1383,7 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) { "\n=== REDIS BUG REPORT END. Make sure to include from START to END. ===\n\n" " Please report the crash by opening an issue on github:\n\n" " http://github.com/antirez/redis/issues\n\n" -" Suspect RAM error? Use redis-server --test-memory to verify it.\n\n" +" Suspect RAM error? Use keydb-server --test-memory to verify it.\n\n" ); /* free(messages); Don't call free() with possibly corrupted memory. */ diff --git a/src/latency.c b/src/latency.c index 5b4610993..ddb235c13 100644 --- a/src/latency.c +++ b/src/latency.c @@ -418,7 +418,7 @@ sds createLatencyReport(void) { " 1) Lower the system load.\n" " 2) Use a computer / VM just for Redis if you are running other softawre in the same system.\n" " 3) Check if you have a \"noisy neighbour\" problem.\n" - " 4) Check with 'redis-cli --intrinsic-latency 100' what is the intrinsic latency in your system.\n" + " 4) Check with 'keydb-cli --intrinsic-latency 100' what is the intrinsic latency in your system.\n" " 5) Check if the problem is allocator-related by recompiling Redis with MALLOC=libc, if you are using Jemalloc. However this may create fragmentation problems.\n"); } diff --git a/src/rdb.c b/src/rdb.c index 16c3423a3..d2ddef56a 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1316,7 +1316,7 @@ int rdbSaveBackground(rdbSaveInfo *rsi) { /* Child */ closeListeningSockets(0); - redisSetProcTitle("redis-rdb-bgsave"); + redisSetProcTitle("keydb-rdb-bgsave"); retval = rdbSave(rsi); if (retval == C_OK) { size_t private_dirty = zmalloc_get_private_dirty(-1); @@ -2314,7 +2314,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { zfree(fds); closeListeningSockets(0); - redisSetProcTitle("redis-rdb-to-slaves"); + redisSetProcTitle("keydb-rdb-to-slaves"); retval = rdbSaveRioWithEOFMark(&slave_sockets,NULL,rsi); if (retval == C_OK && rioFlush(&slave_sockets) == 0) diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c index 369bb0aa4..ae61936da 100644 --- a/src/redis-benchmark.c +++ b/src/redis-benchmark.c @@ -566,7 +566,7 @@ invalid: usage: printf( -"Usage: redis-benchmark [-h ] [-p ] [-c ] [-n ] [-k ]\n\n" +"Usage: keydb-benchmark [-h ] [-p ] [-c ] [-n ] [-k ]\n\n" " -h Server hostname (default 127.0.0.1)\n" " -p Server port (default 6379)\n" " -s Server socket (overrides host and port)\n" @@ -593,17 +593,17 @@ usage: " -I Idle mode. Just open N idle connections and wait.\n\n" "Examples:\n\n" " Run the benchmark with the default configuration against 127.0.0.1:6379:\n" -" $ redis-benchmark\n\n" +" $ keydb-benchmark\n\n" " Use 20 parallel clients, for a total of 100k requests, against 192.168.1.1:\n" -" $ redis-benchmark -h 192.168.1.1 -p 6379 -n 100000 -c 20\n\n" +" $ keydb-benchmark -h 192.168.1.1 -p 6379 -n 100000 -c 20\n\n" " Fill 127.0.0.1:6379 with about 1 million keys only using the SET test:\n" -" $ redis-benchmark -t set -n 1000000 -r 100000000\n\n" +" $ keydb-benchmark -t set -n 1000000 -r 100000000\n\n" " Benchmark 127.0.0.1:6379 for a few commands producing CSV output:\n" -" $ redis-benchmark -t ping,set,get -n 100000 --csv\n\n" +" $ keydb-benchmark -t ping,set,get -n 100000 --csv\n\n" " Benchmark a specific command line:\n" -" $ redis-benchmark -r 10000 -n 10000 eval 'return redis.call(\"ping\")' 0\n\n" +" $ keydb-benchmark -r 10000 -n 10000 eval 'return redis.call(\"ping\")' 0\n\n" " Fill a list with 10000 random elements:\n" -" $ redis-benchmark -r 10000 -n 10000 lpush mylist __rand_int__\n\n" +" $ keydb-benchmark -r 10000 -n 10000 lpush mylist __rand_int__\n\n" " On user specified command lines __rand_int__ is replaced with a random integer\n" " with a range of values selected by the -r option.\n" ); diff --git a/src/redis-check-rdb.c b/src/redis-check-rdb.c index 93fd6cc5a..b85974857 100644 --- a/src/redis-check-rdb.c +++ b/src/redis-check-rdb.c @@ -328,7 +328,7 @@ err: } /* RDB check main: called form redis.c when Redis is executed with the - * redis-check-rdb alias, on during RDB loading errors. + * keydb-check-rdb alias, on during RDB loading errors. * * The function works in two ways: can be called with argc/argv as a * standalone executable, or called with a non NULL 'fp' argument if we diff --git a/src/redis-cli.c b/src/redis-cli.c index d0dffb005..03ead55f2 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -573,14 +573,14 @@ static void cliOutputCommandHelp(struct commandHelp *help, int group) { static void cliOutputGenericHelp(void) { sds version = cliVersion(); printf( - "redis-cli %s\n" + "keydb-cli %s\n" "To get help about Redis commands type:\n" " \"help @\" to get a list of commands in \n" " \"help \" for help on \n" " \"help \" to get a list of possible help topics\n" " \"quit\" to exit\n" "\n" - "To set redis-cli preferences:\n" + "To set keydb-cli preferences:\n" " \":set hints\" enable online hints\n" " \":set nohints\" disable online hints\n" "Set your preferences in ~/.redisclirc\n", @@ -1426,7 +1426,7 @@ static int parseOptions(int argc, char **argv) { CLUSTER_MANAGER_CMD_FLAG_CHECK_OWNERS; } else if (!strcmp(argv[i],"-v") || !strcmp(argv[i], "--version")) { sds version = cliVersion(); - printf("redis-cli %s\n", version); + printf("keydb-cli %s\n", version); sdsfree(version); exit(0); } else if (CLUSTER_MANAGER_MODE() && argv[i][0] != '-') { @@ -1494,9 +1494,9 @@ static sds readArgFromStdin(void) { static void usage(void) { sds version = cliVersion(); fprintf(stderr, -"redis-cli %s\n" +"keydb-cli %s\n" "\n" -"Usage: redis-cli [OPTIONS] [cmd [arg [arg ...]]]\n" +"Usage: keydb-cli [OPTIONS] [cmd [arg [arg ...]]]\n" " -h Server hostname (default: 127.0.0.1).\n" " -p Server port (default: 6379).\n" " -s Server socket (overrides hostname and port).\n" @@ -1562,16 +1562,16 @@ static void usage(void) { " Use --cluster help to list all available cluster manager commands.\n" "\n" "Examples:\n" -" cat /etc/passwd | redis-cli -x set mypasswd\n" -" redis-cli get mypasswd\n" -" redis-cli -r 100 lpush mylist x\n" -" redis-cli -r 100 -i 1 info | grep used_memory_human:\n" -" redis-cli --eval myscript.lua key1 key2 , arg1 arg2 arg3\n" -" redis-cli --scan --pattern '*:12345*'\n" +" cat /etc/passwd | keydb-cli -x set mypasswd\n" +" keydb-cli get mypasswd\n" +" keydb-cli -r 100 lpush mylist x\n" +" keydb-cli -r 100 -i 1 info | grep used_memory_human:\n" +" keydb-cli --eval myscript.lua key1 key2 , arg1 arg2 arg3\n" +" keydb-cli --scan --pattern '*:12345*'\n" "\n" " (Note: when using --eval the comma separates KEYS[] from ARGV[] items)\n" "\n" -"When no command is given, redis-cli starts in interactive mode.\n" +"When no command is given, keydb-cli starts in interactive mode.\n" "Type \"help\" in interactive mode for information on available commands\n" "and settings.\n" "\n"); @@ -1656,12 +1656,12 @@ void cliSetPreferences(char **argv, int argc, int interactive) { if (!strcasecmp(argv[1],"hints")) pref.hints = 1; else if (!strcasecmp(argv[1],"nohints")) pref.hints = 0; else { - printf("%sunknown redis-cli preference '%s'\n", + printf("%sunknown keydb-cli preference '%s'\n", interactive ? "" : ".redisclirc: ", argv[1]); } } else { - printf("%sunknown redis-cli internal command '%s'\n", + printf("%sunknown keydb-cli internal command '%s'\n", interactive ? "" : ".redisclirc: ", argv[0]); } @@ -1733,7 +1733,7 @@ static void repl(void) { repeat = strtol(argv[0], &endptr, 10); if (argc > 1 && *endptr == '\0') { if (errno == ERANGE || errno == EINVAL || repeat <= 0) { - fputs("Invalid redis-cli repeat command option value.\n", stdout); + fputs("Invalid keydb-cli repeat command option value.\n", stdout); sdsfreesplitres(argv, argc); linenoiseFree(line); continue; @@ -4398,7 +4398,7 @@ static int clusterManagerFixOpenSlot(int slot) { } else { unhandled_case: success = 0; - clusterManagerLogErr("[ERR] Sorry, redis-cli can't fix this slot " + clusterManagerLogErr("[ERR] Sorry, keydb-cli can't fix this slot " "yet (work in progress). Slot is set as " "migrating in %s, as importing in %s, " "owner is %s:%d\n", migrating_str, @@ -4784,7 +4784,7 @@ static void clusterManagerPrintNotClusterNodeError(clusterManagerNode *node, clusterManagerLogErr("[ERR] Node %s:%d %s\n", node->ip, node->port, msg); } -/* Execute redis-cli in Cluster Manager mode */ +/* Execute keydb-cli in Cluster Manager mode */ static void clusterManagerMode(clusterManagerCommandProc *proc) { int argc = config.cluster_manager_command.argc; char **argv = config.cluster_manager_command.argv; diff --git a/src/redis-trib.rb b/src/redis-trib.rb index b1af83069..510434556 100755 --- a/src/redis-trib.rb +++ b/src/redis-trib.rb @@ -84,7 +84,7 @@ def parse_options(cmd) end def command_example(cmd, args, opts) - cmd = "redis-cli --cluster #{cmd}" + cmd = "keydb-cli --cluster #{cmd}" args.each{|a| a = a.to_s a = a.inspect if a[' '] @@ -104,26 +104,26 @@ end $command = ARGV.shift $opts, $args = parse_options($command) if $command -puts "WARNING: redis-trib.rb is not longer available!".yellow -puts "You should use #{'redis-cli'.bold} instead." +puts "WARNING: keydb-trib.rb is not longer available!".yellow +puts "You should use #{'keydb-cli'.bold} instead." puts '' -puts "All commands and features belonging to redis-trib.rb "+ - "have been moved\nto redis-cli." -puts "In order to use them you should call redis-cli with the #{'--cluster'.bold}" +puts "All commands and features belonging to keydb-trib.rb "+ + "have been moved\nto keydb-cli." +puts "In order to use them you should call keydb-cli with the #{'--cluster'.bold}" puts "option followed by the subcommand name, arguments and options." puts '' puts "Use the following syntax:" -puts "redis-cli --cluster SUBCOMMAND [ARGUMENTS] [OPTIONS]".bold +puts "keydb-cli --cluster SUBCOMMAND [ARGUMENTS] [OPTIONS]".bold puts '' puts "Example:" if $command example = command_example $command, $args, $opts else - example = "redis-cli --cluster info 127.0.0.1:7000" + example = "keydb-cli --cluster info 127.0.0.1:7000" end puts example.bold puts '' puts "To get help about all subcommands, type:" -puts "redis-cli --cluster help".bold +puts "keydb-cli --cluster help".bold puts '' exit 1 diff --git a/src/replication.c b/src/replication.c index fa0585638..8bd5cfa85 100644 --- a/src/replication.c +++ b/src/replication.c @@ -671,7 +671,7 @@ void syncCommand(client *c) { } } else { /* If a slave uses SYNC, we are dealing with an old implementation - * of the replication protocol (like redis-cli --slave). Flag the client + * of the replication protocol (like keydb-cli --slave). Flag the client * so that we don't expect to receive REPLCONF ACK feedbacks. */ c->flags |= CLIENT_PRE_PSYNC; } diff --git a/src/server.c b/src/server.c index e9eae14e0..0a46fca9b 100644 --- a/src/server.c +++ b/src/server.c @@ -4466,19 +4466,19 @@ void version(void) { } void usage(void) { - fprintf(stderr,"Usage: ./redis-server [/path/to/redis.conf] [options]\n"); - fprintf(stderr," ./redis-server - (read config from stdin)\n"); - fprintf(stderr," ./redis-server -v or --version\n"); - fprintf(stderr," ./redis-server -h or --help\n"); - fprintf(stderr," ./redis-server --test-memory \n\n"); + fprintf(stderr,"Usage: ./keydb-server [/path/to/redis.conf] [options]\n"); + fprintf(stderr," ./keydb-server - (read config from stdin)\n"); + fprintf(stderr," ./keydb-server -v or --version\n"); + fprintf(stderr," ./keydb-server -h or --help\n"); + fprintf(stderr," ./keydb-server --test-memory \n\n"); fprintf(stderr,"Examples:\n"); - fprintf(stderr," ./redis-server (run the server with default conf)\n"); - fprintf(stderr," ./redis-server /etc/redis/6379.conf\n"); - fprintf(stderr," ./redis-server --port 7777\n"); - fprintf(stderr," ./redis-server --port 7777 --replicaof 127.0.0.1 8888\n"); - fprintf(stderr," ./redis-server /etc/myredis.conf --loglevel verbose\n\n"); + fprintf(stderr," ./keydb-server (run the server with default conf)\n"); + fprintf(stderr," ./keydb-server /etc/redis/6379.conf\n"); + fprintf(stderr," ./keydb-server --port 7777\n"); + fprintf(stderr," ./keydb-server --port 7777 --replicaof 127.0.0.1 8888\n"); + fprintf(stderr," ./keydb-server /etc/myredis.conf --loglevel verbose\n\n"); fprintf(stderr,"Sentinel mode:\n"); - fprintf(stderr," ./redis-server /etc/sentinel.conf --sentinel\n"); + fprintf(stderr," ./keydb-server /etc/sentinel.conf --sentinel\n"); exit(1); } @@ -4574,11 +4574,11 @@ void setupSignalHandlers(void) { void memtest(size_t megabytes, int passes); /* Returns 1 if there is --sentinel among the arguments or if - * argv[0] contains "redis-sentinel". */ + * argv[0] contains "keydb-sentinel". */ int checkForSentinelMode(int argc, char **argv) { int j; - if (strstr(argv[0],"redis-sentinel") != NULL) return 1; + if (strstr(argv[0],"keydb-sentinel") != NULL) return 1; for (j = 1; j < argc; j++) if (!strcmp(argv[j],"--sentinel")) return 1; return 0; @@ -4806,12 +4806,12 @@ int main(int argc, char **argv) { initSentinel(); } - /* Check if we need to start in redis-check-rdb/aof mode. We just execute + /* Check if we need to start in keydb-check-rdb/aof mode. We just execute * the program main. However the program is part of the Redis executable * so that we can easily execute an RDB check on loading errors. */ - if (strstr(argv[0],"redis-check-rdb") != NULL) + if (strstr(argv[0],"keydb-check-rdb") != NULL) redis_check_rdb_main(argc,argv,NULL); - else if (strstr(argv[0],"redis-check-aof") != NULL) + else if (strstr(argv[0],"keydb-check-aof") != NULL) redis_check_aof_main(argc,argv); if (argc >= 2) { @@ -4830,7 +4830,7 @@ int main(int argc, char **argv) { exit(0); } else { fprintf(stderr,"Please specify the amount of memory to test in megabytes.\n"); - fprintf(stderr,"Example: ./redis-server --test-memory 4096\n\n"); + fprintf(stderr,"Example: ./keydb-server --test-memory 4096\n\n"); exit(1); } } diff --git a/src/server.h b/src/server.h index 3044b86a4..1ee6a1f8c 100644 --- a/src/server.h +++ b/src/server.h @@ -1994,7 +1994,7 @@ void sentinelTimer(void); char *sentinelHandleConfiguration(char **argv, int argc); void sentinelIsRunning(void); -/* redis-check-rdb & aof */ +/* keydb-check-rdb & aof */ int redis_check_rdb(char *rdbfilename, FILE *fp); int redis_check_rdb_main(int argc, char **argv, FILE *fp); int redis_check_aof_main(int argc, char **argv); diff --git a/tests/cluster/tests/04-resharding.tcl b/tests/cluster/tests/04-resharding.tcl index 68fba135e..255d886e4 100644 --- a/tests/cluster/tests/04-resharding.tcl +++ b/tests/cluster/tests/04-resharding.tcl @@ -73,7 +73,7 @@ test "Cluster consistency during live resharding" { flush stdout set target [dict get [get_myself [randomInt 5]] id] set tribpid [lindex [exec \ - ../../../src/redis-cli --cluster reshard \ + ../../../src/keydb-cli --cluster reshard \ 127.0.0.1:[get_instance_attrib redis 0 port] \ --cluster-from all \ --cluster-to $target \ diff --git a/tests/cluster/tests/12-replica-migration-2.tcl b/tests/cluster/tests/12-replica-migration-2.tcl index 3d8b7b04b..85de18464 100644 --- a/tests/cluster/tests/12-replica-migration-2.tcl +++ b/tests/cluster/tests/12-replica-migration-2.tcl @@ -31,7 +31,7 @@ test "Each master should have at least two replicas attached" { set master0_id [dict get [get_myself 0] id] test "Resharding all the master #0 slots away from it" { set output [exec \ - ../../../src/redis-cli --cluster rebalance \ + ../../../src/keydb-cli --cluster rebalance \ 127.0.0.1:[get_instance_attrib redis 0 port] \ --cluster-weight ${master0_id}=0 >@ stdout ] } @@ -49,7 +49,7 @@ test "Resharding back some slot to master #0" { # new resharding. after 10000 set output [exec \ - ../../../src/redis-cli --cluster rebalance \ + ../../../src/keydb-cli --cluster rebalance \ 127.0.0.1:[get_instance_attrib redis 0 port] \ --cluster-weight ${master0_id}=.01 \ --cluster-use-empty-masters >@ stdout] diff --git a/tests/cluster/tests/helpers/onlydots.tcl b/tests/cluster/tests/helpers/onlydots.tcl index 4a6d1aee0..7fec31448 100644 --- a/tests/cluster/tests/helpers/onlydots.tcl +++ b/tests/cluster/tests/helpers/onlydots.tcl @@ -1,7 +1,7 @@ # Read the standard input and only shows dots in the output, filtering out # all the other characters. Designed to avoid bufferization so that when -# we get the output of redis-trib and want to show just the dots, we'll see -# the dots as soon as redis-trib will output them. +# we get the output of keydb-trib and want to show just the dots, we'll see +# the dots as soon as keydb-trib will output them. fconfigure stdin -buffering none diff --git a/tests/instances.tcl b/tests/instances.tcl index 357b34818..b349df7c3 100644 --- a/tests/instances.tcl +++ b/tests/instances.tcl @@ -38,9 +38,9 @@ if {[catch {cd tmp}]} { # the provided configuration file. Returns the PID of the process. proc exec_instance {type cfgfile} { if {$type eq "redis"} { - set prgname redis-server + set prgname keydb-server } elseif {$type eq "sentinel"} { - set prgname redis-sentinel + set prgname keydb-sentinel } else { error "Unknown instance type." } @@ -179,7 +179,7 @@ proc pause_on_error {} { set cmd [lindex $argv 0] if {$cmd eq {continue}} { break - } elseif {$cmd eq {show-redis-logs}} { + } elseif {$cmd eq {show-keydb-logs}} { set count 10 if {[lindex $argv 1] ne {}} {set count [lindex $argv 1]} foreach_redis_id id { @@ -230,7 +230,7 @@ proc pause_on_error {} { } elseif {$cmd eq {help}} { puts "ls List Sentinel and Redis instances." puts "show-sentinel-logs \[N\] Show latest N lines of logs." - puts "show-redis-logs \[N\] Show latest N lines of logs." + puts "show-keydb-logs \[N\] Show latest N lines of logs." puts "S cmd ... arg Call command in Sentinel ." puts "R cmd ... arg Call command in Redis ." puts "SI Show Sentinel INFO ." diff --git a/tests/integration/aof-race.tcl b/tests/integration/aof-race.tcl index fb8d71083..e408fda8e 100644 --- a/tests/integration/aof-race.tcl +++ b/tests/integration/aof-race.tcl @@ -14,7 +14,7 @@ tags {"aof"} { # was subsequently appended to the new AOF, resulting in duplicate commands. start_server_aof [list dir $server_path] { set client [redis [srv host] [srv port]] - set bench [open "|src/redis-benchmark -q -p [srv port] -c 20 -n 20000 incr foo" "r+"] + set bench [open "|src/keydb-benchmark -q -p [srv port] -c 20 -n 20000 incr foo" "r+"] after 100 # Benchmark should be running by now: start background rewrite diff --git a/tests/integration/aof.tcl b/tests/integration/aof.tcl index e397faeeb..277b0d3df 100644 --- a/tests/integration/aof.tcl +++ b/tests/integration/aof.tcl @@ -150,16 +150,16 @@ tags {"aof"} { } } - ## Test that redis-check-aof indeed sees this AOF is not valid + ## Test that keydb-check-aof indeed sees this AOF is not valid test "Short read: Utility should confirm the AOF is not valid" { catch { - exec src/redis-check-aof $aof_path + exec src/keydb-check-aof $aof_path } result assert_match "*not valid*" $result } test "Short read: Utility should be able to fix the AOF" { - set result [exec src/redis-check-aof --fix $aof_path << "y\n"] + set result [exec src/keydb-check-aof --fix $aof_path << "y\n"] assert_match "*Successfully truncated AOF*" $result } diff --git a/tests/integration/psync2-reg.tcl b/tests/integration/psync2-reg.tcl index 3d408368e..6e8b1403c 100644 --- a/tests/integration/psync2-reg.tcl +++ b/tests/integration/psync2-reg.tcl @@ -1,7 +1,7 @@ # Issue 3899 regression test. # We create a chain of three instances: master -> slave -> slave2 # and continuously break the link while traffic is generated by -# redis-benchmark. At the end we check that the data is the same +# keydb-benchmark. At the end we check that the data is the same # everywhere. start_server {tags {"psync2"}} { @@ -36,7 +36,7 @@ start_server {} { } set cycle_start_time [clock milliseconds] - set bench_pid [exec src/redis-benchmark -p $R_port(0) -n 10000000 -r 1000 incr __rand_int__ > /dev/null &] + set bench_pid [exec src/keydb-benchmark -p $R_port(0) -n 10000000 -r 1000 incr __rand_int__ > /dev/null &] while 1 { set elapsed [expr {[clock milliseconds]-$cycle_start_time}] if {$elapsed > $duration*1000} break diff --git a/tests/integration/redis-cli.tcl b/tests/integration/redis-cli.tcl index 40e4222e3..d6dea8fb3 100644 --- a/tests/integration/redis-cli.tcl +++ b/tests/integration/redis-cli.tcl @@ -1,7 +1,7 @@ start_server {tags {"cli"}} { proc open_cli {} { set ::env(TERM) dumb - set fd [open [format "|src/redis-cli -p %d -n 9" [srv port]] "r+"] + set fd [open [format "|src/keydb-cli -p %d -n 9" [srv port]] "r+"] fconfigure $fd -buffering none fconfigure $fd -blocking false fconfigure $fd -translation binary @@ -54,7 +54,7 @@ start_server {tags {"cli"}} { } proc _run_cli {opts args} { - set cmd [format "src/redis-cli -p %d -n 9 $args" [srv port]] + set cmd [format "src/keydb-cli -p %d -n 9 $args" [srv port]] foreach {key value} $opts { if {$key eq "pipe"} { set cmd "sh -c \"$value | $cmd\"" diff --git a/tests/sentinel/tests/07-down-conditions.tcl b/tests/sentinel/tests/07-down-conditions.tcl index fb2993b6f..e4076ccbc 100644 --- a/tests/sentinel/tests/07-down-conditions.tcl +++ b/tests/sentinel/tests/07-down-conditions.tcl @@ -28,7 +28,7 @@ test "Crash the majority of Sentinels to prevent failovers for this unit" { test "SDOWN is triggered by non-responding but not crashed instance" { lassign [S 4 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] host port ensure_master_up - exec ../../../src/redis-cli -h $host -p $port debug sleep 10 > /dev/null & + exec ../../../src/keydb-cli -h $host -p $port debug sleep 10 > /dev/null & ensure_master_down ensure_master_up } diff --git a/tests/support/server.tcl b/tests/support/server.tcl index 0edb25d8a..659fe60f4 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -212,11 +212,11 @@ proc start_server {options {code undefined}} { set stderr [format "%s/%s" [dict get $config "dir"] "stderr"] if {$::valgrind} { - set pid [exec valgrind --track-origins=yes --suppressions=src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full src/redis-server $config_file > $stdout 2> $stderr &] + set pid [exec valgrind --track-origins=yes --suppressions=src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full src/keydb-server $config_file > $stdout 2> $stderr &] } elseif ($::stack_logging) { - set pid [exec /usr/bin/env MallocStackLogging=1 MallocLogFile=/tmp/malloc_log.txt src/redis-server $config_file > $stdout 2> $stderr &] + set pid [exec /usr/bin/env MallocStackLogging=1 MallocLogFile=/tmp/malloc_log.txt src/keydb-server $config_file > $stdout 2> $stderr &] } else { - set pid [exec src/redis-server $config_file > $stdout 2> $stderr &] + set pid [exec src/keydb-server $config_file > $stdout 2> $stderr &] } # Tell the test server about this new instance. diff --git a/tests/support/test.tcl b/tests/support/test.tcl index 6f02f2f12..6849cb876 100644 --- a/tests/support/test.tcl +++ b/tests/support/test.tcl @@ -139,7 +139,7 @@ proc test {name code {okpattern undefined}} { } if {$::traceleaks} { - set output [exec leaks redis-server] + set output [exec leaks keydb-server] if {![string match {*0 leaks*} $output]} { send_data_packet $::test_server_fd err "Detected a memory leak in test '$name': $output" } diff --git a/tests/unit/wait.tcl b/tests/unit/wait.tcl index e2f5d2942..b978c1757 100644 --- a/tests/unit/wait.tcl +++ b/tests/unit/wait.tcl @@ -31,8 +31,8 @@ start_server {} { } test {WAIT should not acknowledge 1 additional copy if slave is blocked} { - exec src/redis-cli -h $slave_host -p $slave_port debug sleep 5 > /dev/null 2> /dev/null & - after 1000 ;# Give redis-cli the time to execute the command. + exec src/keydb-cli -h $slave_host -p $slave_port debug sleep 5 > /dev/null 2> /dev/null & + after 1000 ;# Give keydb-cli the time to execute the command. $master set foo 0 $master incr foo $master incr foo diff --git a/utils/cluster_fail_time.tcl b/utils/cluster_fail_time.tcl index 87399495f..c15765346 100644 --- a/utils/cluster_fail_time.tcl +++ b/utils/cluster_fail_time.tcl @@ -15,11 +15,11 @@ proc avg vector { set samples {} while 1 { - exec redis-cli -p $::fail_port debug sleep $::sleep_time > /dev/null & + exec keydb-cli -p $::fail_port debug sleep $::sleep_time > /dev/null & # Wait for fail? to appear. while 1 { - set output [exec redis-cli -p $::other_port cluster nodes] + set output [exec keydb-cli -p $::other_port cluster nodes] if {[string match {*fail\?*} $output]} break after 100 } @@ -29,7 +29,7 @@ while 1 { # Wait for fail? to disappear. while 1 { - set output [exec redis-cli -p $::other_port cluster nodes] + set output [exec keydb-cli -p $::other_port cluster nodes] if {![string match {*fail\?*} $output]} break after 100 } @@ -43,7 +43,7 @@ while 1 { puts "AVG([llength $samples]): [avg $samples]" # Wait for the instance to be available again. - exec redis-cli -p $::fail_port ping + exec keydb-cli -p $::fail_port ping # Wait for the fail flag to be cleared. after 2000 diff --git a/utils/create-cluster/README b/utils/create-cluster/README index e682f6dc9..c1234c643 100644 --- a/utils/create-cluster/README +++ b/utils/create-cluster/README @@ -15,7 +15,7 @@ To create a cluster, follow these steps: 1. Edit create-cluster and change the start / end port, depending on the number of instances you want to create. 2. Use "./create-cluster start" in order to run the instances. -3. Use "./create-cluster create" in order to execute redis-cli --cluster create, so that +3. Use "./create-cluster create" in order to execute keydb-cli --cluster create, so that an actual Redis cluster will be created. 4. Now you are ready to play with the cluster. AOF files and logs for each instances are created in the current directory. diff --git a/utils/create-cluster/create-cluster b/utils/create-cluster/create-cluster index 468f924a4..ed3d620d7 100755 --- a/utils/create-cluster/create-cluster +++ b/utils/create-cluster/create-cluster @@ -22,7 +22,7 @@ then while [ $((PORT < ENDPORT)) != "0" ]; do PORT=$((PORT+1)) echo "Starting $PORT" - ../../src/redis-server --port $PORT --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes + ../../src/keydb-server --port $PORT --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes done exit 0 fi @@ -34,7 +34,7 @@ then PORT=$((PORT+1)) HOSTS="$HOSTS 127.0.0.1:$PORT" done - ../../src/redis-cli --cluster create $HOSTS --cluster-replicas $REPLICAS + ../../src/keydb-cli --cluster create $HOSTS --cluster-replicas $REPLICAS exit 0 fi @@ -43,7 +43,7 @@ then while [ $((PORT < ENDPORT)) != "0" ]; do PORT=$((PORT+1)) echo "Stopping $PORT" - ../../src/redis-cli -p $PORT shutdown nosave + ../../src/keydb-cli -p $PORT shutdown nosave done exit 0 fi @@ -54,7 +54,7 @@ then while [ 1 ]; do clear date - ../../src/redis-cli -p $PORT cluster nodes | head -30 + ../../src/keydb-cli -p $PORT cluster nodes | head -30 sleep 1 done exit 0 @@ -72,7 +72,7 @@ if [ "$1" == "call" ] then while [ $((PORT < ENDPORT)) != "0" ]; do PORT=$((PORT+1)) - ../../src/redis-cli -p $PORT $2 $3 $4 $5 $6 $7 $8 $9 + ../../src/keydb-cli -p $PORT $2 $3 $4 $5 $6 $7 $8 $9 done exit 0 fi @@ -94,7 +94,7 @@ fi echo "Usage: $0 [start|create|stop|watch|tail|clean]" echo "start -- Launch Redis Cluster instances." -echo "create -- Create a cluster using redis-cli --cluster create." +echo "create -- Create a cluster using keydb-cli --cluster create." echo "stop -- Stop Redis Cluster instances." echo "watch -- Show CLUSTER NODES output (first 30 lines) of first node." echo "tail -- Run tail -f of instance at base port + ID." diff --git a/utils/generate-command-help.rb b/utils/generate-command-help.rb index 29acef69d..2e46d1ed8 100755 --- a/utils/generate-command-help.rb +++ b/utils/generate-command-help.rb @@ -53,7 +53,7 @@ def commands require "json" require "uri" - url = URI.parse "https://raw.githubusercontent.com/antirez/redis-doc/master/commands.json" + url = URI.parse "https://raw.githubusercontent.com/antirez/keydb-doc/master/commands.json" client = Net::HTTP.new url.host, url.port client.use_ssl = true response = client.get url.path diff --git a/utils/install_server.sh b/utils/install_server.sh index 7eb341417..9934bca92 100755 --- a/utils/install_server.sh +++ b/utils/install_server.sh @@ -37,7 +37,7 @@ # REDIS_CONFIG_FILE=/etc/redis/1234.conf \ # REDIS_LOG_FILE=/var/log/redis_1234.log \ # REDIS_DATA_DIR=/var/lib/redis/1234 \ -# REDIS_EXECUTABLE=`command -v redis-server` ./utils/install_server.sh +# REDIS_EXECUTABLE=`command -v keydb-server` ./utils/install_server.sh # # This generates a redis config file and an /etc/init.d script, and installs them. # @@ -116,7 +116,7 @@ fi if [ ! -x "$REDIS_EXECUTABLE" ] ; then _MANUAL_EXECUTION=true #get the redis executable path - _REDIS_EXECUTABLE=`command -v redis-server` + _REDIS_EXECUTABLE=`command -v keydb-server` read -p "Please select the redis executable path [$_REDIS_EXECUTABLE] " REDIS_EXECUTABLE if [ ! -x "$REDIS_EXECUTABLE" ] ; then REDIS_EXECUTABLE=$_REDIS_EXECUTABLE @@ -129,9 +129,9 @@ if [ ! -x "$REDIS_EXECUTABLE" ] ; then fi #check the default for redis cli -CLI_EXEC=`command -v redis-cli` +CLI_EXEC=`command -v keydb-cli` if [ -z "$CLI_EXEC" ] ; then - CLI_EXEC=`dirname $REDIS_EXECUTABLE`"/redis-cli" + CLI_EXEC=`dirname $REDIS_EXECUTABLE`"/keydb-cli" fi echo "Selected config:" diff --git a/utils/redis-copy.rb b/utils/redis-copy.rb index 7c5c52dd6..aa9e797b5 100644 --- a/utils/redis-copy.rb +++ b/utils/redis-copy.rb @@ -1,27 +1,27 @@ -# redis-copy.rb - Copyright (C) 2009-2010 Salvatore Sanfilippo +# keydb-copy.rb - Copyright (C) 2009-2010 Salvatore Sanfilippo # BSD license, See the COPYING file for more information. # # Copy the whole dataset from one Redis instance to another one # # WARNING: this utility is deprecated and serves as a legacy adapter -# for the more-robust redis-copy gem. +# for the more-robust keydb-copy gem. require 'shellwords' def redisCopy(opts={}) src = "#{opts[:srchost]}:#{opts[:srcport]}" dst = "#{opts[:dsthost]}:#{opts[:dstport]}" - `redis-copy #{src.shellescape} #{dst.shellescape}` + `keydb-copy #{src.shellescape} #{dst.shellescape}` rescue Errno::ENOENT - $stderr.puts 'This utility requires the redis-copy executable', - 'from the redis-copy gem on https://rubygems.org', - 'To install it, run `gem install redis-copy`.' + $stderr.puts 'This utility requires the keydb-copy executable', + 'from the keydb-copy gem on https://rubygems.org', + 'To install it, run `gem install keydb-copy`.' exit 1 end -$stderr.puts "This utility is deprecated. Use the redis-copy gem instead." +$stderr.puts "This utility is deprecated. Use the keydb-copy gem instead." if ARGV.length != 4 - puts "Usage: redis-copy.rb " + puts "Usage: keydb-copy.rb " exit 1 end puts "WARNING: it's up to you to FLUSHDB the destination host before to continue, press any key when ready." diff --git a/utils/redis-sha1.rb b/utils/redis-sha1.rb index 24498e25a..5746d7b9c 100644 --- a/utils/redis-sha1.rb +++ b/utils/redis-sha1.rb @@ -1,4 +1,4 @@ -# redis-sha1.rb - Copyright (C) 2009 Salvatore Sanfilippo +# keydb-sha1.rb - Copyright (C) 2009 Salvatore Sanfilippo # BSD license, See the COPYING file for more information. # # Performs the SHA1 sum of the whole datset. diff --git a/utils/redis_init_script b/utils/redis_init_script index 006db87e5..bee5545ef 100755 --- a/utils/redis_init_script +++ b/utils/redis_init_script @@ -12,8 +12,8 @@ ### END INIT INFO REDISPORT=6379 -EXEC=/usr/local/bin/redis-server -CLIEXEC=/usr/local/bin/redis-cli +EXEC=/usr/local/bin/keydb-server +CLIEXEC=/usr/local/bin/keydb-cli PIDFILE=/var/run/redis_${REDISPORT}.pid CONF="/etc/redis/${REDISPORT}.conf" diff --git a/utils/releasetools/01_create_tarball.sh b/utils/releasetools/01_create_tarball.sh index 54bca8c04..dd55f8f49 100755 --- a/utils/releasetools/01_create_tarball.sh +++ b/utils/releasetools/01_create_tarball.sh @@ -6,10 +6,10 @@ then fi TAG=$1 -TARNAME="redis-${TAG}.tar" +TARNAME="keydb-${TAG}.tar" echo "Generating /tmp/${TARNAME}" cd ~/hack/redis -git archive $TAG --prefix redis-${TAG}/ > /tmp/$TARNAME || exit 1 +git archive $TAG --prefix keydb-${TAG}/ > /tmp/$TARNAME || exit 1 echo "Gizipping the archive" rm -f /tmp/$TARNAME.gz gzip -9 /tmp/$TARNAME diff --git a/utils/releasetools/02_upload_tarball.sh b/utils/releasetools/02_upload_tarball.sh index ed7065388..17d36662d 100755 --- a/utils/releasetools/02_upload_tarball.sh +++ b/utils/releasetools/02_upload_tarball.sh @@ -1,6 +1,6 @@ #!/bin/bash echo "Uploading..." -scp /tmp/redis-${1}.tar.gz antirez@antirez.com:/var/virtual/download.redis.io/httpdocs/releases/ +scp /tmp/keydb-${1}.tar.gz antirez@antirez.com:/var/virtual/download.redis.io/httpdocs/releases/ echo "Updating web site... (press any key if it is a stable release, or Ctrl+C)" read x ssh antirez@antirez.com "cd /var/virtual/download.redis.io/httpdocs; ./update.sh ${1}" diff --git a/utils/releasetools/03_test_release.sh b/utils/releasetools/03_test_release.sh index 3dfdcd6a3..52e4a75c4 100755 --- a/utils/releasetools/03_test_release.sh +++ b/utils/releasetools/03_test_release.sh @@ -6,7 +6,7 @@ then fi TAG=$1 -TARNAME="redis-${TAG}.tar.gz" +TARNAME="keydb-${TAG}.tar.gz" DOWNLOADURL="http://download.redis.io/releases/${TARNAME}" ssh antirez@metal "export TERM=xterm; @@ -14,10 +14,10 @@ ssh antirez@metal "export TERM=xterm; rm -rf test_release_tmp_dir; cd test_release_tmp_dir; rm -f $TARNAME; - rm -rf redis-${TAG}; + rm -rf keydb-${TAG}; wget $DOWNLOADURL; tar xvzf $TARNAME; - cd redis-${TAG}; + cd keydb-${TAG}; make; ./runtest; ./runtest-sentinel; diff --git a/utils/releasetools/04_release_hash.sh b/utils/releasetools/04_release_hash.sh index 9d5c6ad4b..5c7a946a5 100755 --- a/utils/releasetools/04_release_hash.sh +++ b/utils/releasetools/04_release_hash.sh @@ -1,8 +1,8 @@ #!/bin/bash -SHA=$(curl -s http://download.redis.io/releases/redis-${1}.tar.gz | shasum -a 256 | cut -f 1 -d' ') -ENTRY="hash redis-${1}.tar.gz sha256 $SHA http://download.redis.io/releases/redis-${1}.tar.gz" -echo $ENTRY >> ~/hack/redis-hashes/README -vi ~/hack/redis-hashes/README +SHA=$(curl -s http://download.redis.io/releases/keydb-${1}.tar.gz | shasum -a 256 | cut -f 1 -d' ') +ENTRY="hash keydb-${1}.tar.gz sha256 $SHA http://download.redis.io/releases/keydb-${1}.tar.gz" +echo $ENTRY >> ~/hack/keydb-hashes/README +vi ~/hack/keydb-hashes/README echo "Press any key to commit, Ctrl-C to abort)." read yes -(cd ~/hack/redis-hashes; git commit -a -m "${1} hash."; git push) +(cd ~/hack/keydb-hashes; git commit -a -m "${1} hash."; git push) diff --git a/utils/speed-regression.tcl b/utils/speed-regression.tcl index 86a7d8d86..8d5220c75 100755 --- a/utils/speed-regression.tcl +++ b/utils/speed-regression.tcl @@ -20,15 +20,15 @@ proc run-tests branches { exec -ignorestderr make 2> /dev/null if {$branch_id == 0} { - puts " copy redis-benchmark from unstable to /tmp..." - exec -ignorestderr cp ./redis-benchmark /tmp + puts " copy keydb-benchmark from unstable to /tmp..." + exec -ignorestderr cp ./keydb-benchmark /tmp incr branch_id continue } # Start the Redis server - puts " starting the server... [exec ./redis-server -v]" - set pids [exec echo "port $::port\nloglevel warning\n" | ./redis-server - > /dev/null 2> /dev/null &] + puts " starting the server... [exec ./keydb-server -v]" + set pids [exec echo "port $::port\nloglevel warning\n" | ./keydb-server - > /dev/null 2> /dev/null &] puts " pids: $pids" after 1000 puts " running the benchmark" @@ -38,7 +38,7 @@ proc run-tests branches { puts " redis INFO shows version: [lindex [split $i] 0]" $r close - set output [exec /tmp/redis-benchmark -n $::requests -t $::tests -d $::datasize --csv -p $::port] + set output [exec /tmp/keydb-benchmark -n $::requests -t $::tests -d $::datasize --csv -p $::port] lappend runs $b $output puts " killing server..." catch {exec kill -9 [lindex $pids 0]} @@ -83,7 +83,7 @@ proc combine-results {results} { } proc main {} { - # Note: the first branch is only used in order to get the redis-benchmark + # Note: the first branch is only used in order to get the keydb-benchmark # executable. Tests are performed starting from the second branch. set branches { slowset 2.2.0 2.4.0 unstable slowset From f6736cd713aa040684d7e9d14211d3cea45ae58c Mon Sep 17 00:00:00 2001 From: John Sully Date: Sat, 9 Feb 2019 13:04:18 -0500 Subject: [PATCH 029/101] Reduce memory usage for in place strings by 8 bytes --- README.md | 2 +- src/acl.c | 22 +++--- src/aof.c | 32 ++++----- src/bitops.c | 40 +++++------ src/blocked.c | 6 +- src/cluster.c | 158 +++++++++++++++++++++--------------------- src/config.c | 64 ++++++++--------- src/db.c | 84 +++++++++++----------- src/debug.c | 114 +++++++++++++++--------------- src/expire.c | 6 +- src/geo.c | 20 +++--- src/hyperloglog.c | 58 ++++++++-------- src/latency.c | 20 +++--- src/lazyfree.c | 12 ++-- src/module.c | 92 ++++++++++++------------ src/multi.c | 2 +- src/networking.c | 86 +++++++++++------------ src/notify.c | 4 +- src/object.c | 145 +++++++++++++++++++------------------- src/pubsub.c | 20 +++--- src/rdb.c | 102 +++++++++++++-------------- src/redis-check-rdb.c | 4 +- src/replication.c | 44 ++++++------ src/scripting.c | 56 +++++++-------- src/sentinel.c | 98 +++++++++++++------------- src/server.c | 43 ++++++------ src/server.h | 11 ++- src/slowlog.c | 16 ++--- src/sort.c | 38 +++++----- src/t_hash.c | 62 ++++++++--------- src/t_list.c | 34 ++++----- src/t_set.c | 60 ++++++++-------- src/t_stream.c | 102 +++++++++++++-------------- src/t_string.c | 22 +++--- src/t_zset.c | 152 ++++++++++++++++++++-------------------- 35 files changed, 919 insertions(+), 912 deletions(-) diff --git a/README.md b/README.md index 0940f6ce3..352630b7d 100644 --- a/README.md +++ b/README.md @@ -420,7 +420,7 @@ Anatomy of a Redis command All the Redis commands are defined in the following way: void foobarCommand(client *c) { - printf("%s",c->argv[1]->ptr); /* Do something with the argument. */ + printf("%s",ptrFromObj(c->argv[1])); /* Do something with the argument. */ addReply(c,shared.ok); /* Reply something to the client. */ } diff --git a/src/acl.c b/src/acl.c index 50d91aa54..42cd0c734 100644 --- a/src/acl.c +++ b/src/acl.c @@ -759,7 +759,7 @@ void ACLInit(void) { * ENONENT: if the specified user does not exist at all. */ int ACLCheckUserCredentials(robj *username, robj *password) { - user *u = ACLGetUserByName(username->ptr,sdslen(username->ptr)); + user *u = ACLGetUserByName(ptrFromObj(username),sdslen(ptrFromObj(username))); if (u == NULL) { errno = ENOENT; return C_ERR; @@ -781,7 +781,7 @@ int ACLCheckUserCredentials(robj *username, robj *password) { listRewind(u->passwords,&li); while((ln = listNext(&li))) { sds thispass = listNodeValue(ln); - if (!time_independent_strcmp(password->ptr, thispass)) + if (!time_independent_strcmp(ptrFromObj(password), thispass)) return C_OK; } @@ -868,7 +868,7 @@ int ACLCheckCommandPerm(client *c) { while (1) { if (u->allowed_subcommands[id][subid] == NULL) return ACL_DENIED_CMD; - if (!strcasecmp(c->argv[1]->ptr, + if (!strcasecmp(ptrFromObj(c->argv[1]), u->allowed_subcommands[id][subid])) break; /* Subcommand match found. Stop here. */ subid++; @@ -894,8 +894,8 @@ int ACLCheckCommandPerm(client *c) { sds pattern = listNodeValue(ln); size_t plen = sdslen(pattern); int idx = keyidx[j]; - if (stringmatchlen(pattern,plen,c->argv[idx]->ptr, - sdslen(c->argv[idx]->ptr),0)) + if (stringmatchlen(pattern,plen,ptrFromObj(c->argv[idx]), + sdslen(ptrFromObj(c->argv[idx])),0)) { match = 1; break; @@ -1021,18 +1021,18 @@ int ACLLoadConfiguredUsers(void) { * ACL GETUSER */ void aclCommand(client *c) { - char *sub = c->argv[1]->ptr; + char *sub = ptrFromObj(c->argv[1]); if (!strcasecmp(sub,"setuser") && c->argc >= 3) { - sds username = c->argv[2]->ptr; + sds username = ptrFromObj(c->argv[2]); user *u = ACLGetUserByName(username,sdslen(username)); if (!u) u = ACLCreateUser(username,sdslen(username)); serverAssert(u != NULL); for (int j = 3; j < c->argc; j++) { - if (ACLSetUser(u,c->argv[j]->ptr,sdslen(c->argv[j]->ptr)) != C_OK) { + if (ACLSetUser(u,ptrFromObj(c->argv[j]),sdslen(ptrFromObj(c->argv[j]))) != C_OK) { char *errmsg = ACLSetUserStringError(); addReplyErrorFormat(c, "Error in ACL SETUSER modifier '%s': %s", - (char*)c->argv[j]->ptr, errmsg); + (char*)ptrFromObj(c->argv[j]), errmsg); return; } } @@ -1040,7 +1040,7 @@ void aclCommand(client *c) { } else if (!strcasecmp(sub,"deluser") && c->argc >= 3) { int deleted = 0; for (int j = 2; j < c->argc; j++) { - sds username = c->argv[j]->ptr; + sds username = ptrFromObj(c->argv[j]); if (!strcmp(username,"default")) { addReplyError(c,"The 'default' user cannot be removed"); return; @@ -1076,7 +1076,7 @@ void aclCommand(client *c) { } addReplyLongLong(c,deleted); } else if (!strcasecmp(sub,"getuser") && c->argc == 3) { - user *u = ACLGetUserByName(c->argv[2]->ptr,sdslen(c->argv[2]->ptr)); + user *u = ACLGetUserByName(ptrFromObj(c->argv[2]),sdslen(ptrFromObj(c->argv[2]))); if (u == NULL) { addReplyNull(c); return; diff --git a/src/aof.c b/src/aof.c index d548c72b8..c71f88aa6 100644 --- a/src/aof.c +++ b/src/aof.c @@ -506,11 +506,11 @@ sds catAppendOnlyGenericCommand(sds dst, int argc, robj **argv) { for (j = 0; j < argc; j++) { o = getDecodedObject(argv[j]); buf[0] = '$'; - len = 1+ll2string(buf+1,sizeof(buf)-1,sdslen(o->ptr)); + len = 1+ll2string(buf+1,sizeof(buf)-1,sdslen(ptrFromObj(o))); buf[len++] = '\r'; buf[len++] = '\n'; dst = sdscatlen(dst,buf,len); - dst = sdscatlen(dst,o->ptr,sdslen(o->ptr)); + dst = sdscatlen(dst,ptrFromObj(o),sdslen(ptrFromObj(o))); dst = sdscatlen(dst,"\r\n",2); decrRefCount(o); } @@ -530,7 +530,7 @@ sds catAppendOnlyExpireAtCommand(sds buf, struct redisCommand *cmd, robj *key, r /* Make sure we can use strtoll */ seconds = getDecodedObject(seconds); - when = strtoll(seconds->ptr,NULL,10); + when = strtoll(ptrFromObj(seconds),NULL,10); /* Convert argument into milliseconds for EXPIRE, SETEX, EXPIREAT */ if (cmd->proc == expireCommand || cmd->proc == setexCommand || cmd->proc == expireatCommand) @@ -587,8 +587,8 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a /* Translate SET [EX seconds][PX milliseconds] to SET and PEXPIREAT */ buf = catAppendOnlyGenericCommand(buf,3,argv); for (i = 3; i < argc; i ++) { - if (!strcasecmp(argv[i]->ptr, "ex")) exarg = argv[i+1]; - if (!strcasecmp(argv[i]->ptr, "px")) pxarg = argv[i+1]; + if (!strcasecmp(ptrFromObj(argv[i]), "ex")) exarg = argv[i+1]; + if (!strcasecmp(ptrFromObj(argv[i]), "px")) pxarg = argv[i+1]; } serverAssert(!(exarg && pxarg)); if (exarg) @@ -780,11 +780,11 @@ int loadAppendOnlyFile(char *filename) { } /* Command lookup */ - cmd = lookupCommand(argv[0]->ptr); + cmd = lookupCommand(ptrFromObj(argv[0])); if (!cmd) { serverLog(LL_WARNING, "Unknown command '%s' reading the append only file", - (char*)argv[0]->ptr); + (char*)ptrFromObj(argv[0])); exit(1); } @@ -886,9 +886,9 @@ int rioWriteBulkObject(rio *r, robj *obj) { /* Avoid using getDecodedObject to help copy-on-write (we are often * in a child process when this function is called). */ if (obj->encoding == OBJ_ENCODING_INT) { - return rioWriteBulkLongLong(r,(long)obj->ptr); + return rioWriteBulkLongLong(r,(long)obj->m_ptr); } else if (sdsEncodedObject(obj)) { - return rioWriteBulkString(r,obj->ptr,sdslen(obj->ptr)); + return rioWriteBulkString(r,ptrFromObj(obj),sdslen(ptrFromObj(obj))); } else { serverPanic("Unknown string encoding"); } @@ -900,7 +900,7 @@ int rewriteListObject(rio *r, robj *key, robj *o) { long long count = 0, items = listTypeLength(o); if (o->encoding == OBJ_ENCODING_QUICKLIST) { - quicklist *list = o->ptr; + quicklist *list = ptrFromObj(o); quicklistIter *li = quicklistGetIterator(list, AL_START_HEAD); quicklistEntry entry; @@ -937,7 +937,7 @@ int rewriteSetObject(rio *r, robj *key, robj *o) { int ii = 0; int64_t llval; - while(intsetGet(o->ptr,ii++,&llval)) { + while(intsetGet(ptrFromObj(o),ii++,&llval)) { if (count == 0) { int cmd_items = (items > AOF_REWRITE_ITEMS_PER_CMD) ? AOF_REWRITE_ITEMS_PER_CMD : items; @@ -951,7 +951,7 @@ int rewriteSetObject(rio *r, robj *key, robj *o) { items--; } } else if (o->encoding == OBJ_ENCODING_HT) { - dictIterator *di = dictGetIterator(o->ptr); + dictIterator *di = dictGetIterator(ptrFromObj(o)); dictEntry *de; while((de = dictNext(di)) != NULL) { @@ -981,7 +981,7 @@ int rewriteSortedSetObject(rio *r, robj *key, robj *o) { long long count = 0, items = zsetLength(o); if (o->encoding == OBJ_ENCODING_ZIPLIST) { - unsigned char *zl = o->ptr; + unsigned char *zl = ptrFromObj(o); unsigned char *eptr, *sptr; unsigned char *vstr; unsigned int vlen; @@ -1016,7 +1016,7 @@ int rewriteSortedSetObject(rio *r, robj *key, robj *o) { items--; } } else if (o->encoding == OBJ_ENCODING_SKIPLIST) { - zset *zs = o->ptr; + zset *zs = ptrFromObj(o); dictIterator *di = dictGetIterator(zs->pdict); dictEntry *de; @@ -1137,7 +1137,7 @@ int rioWriteStreamPendingEntry(rio *r, robj *key, const char *groupname, size_t /* Emit the commands needed to rebuild a stream object. * The function returns 0 on error, 1 on success. */ int rewriteStreamObject(rio *r, robj *key, robj *o) { - stream *s = o->ptr; + stream *s = ptrFromObj(o); streamIterator si; streamIteratorStart(&si,s,NULL,NULL,0); streamID id; @@ -1237,7 +1237,7 @@ int rewriteStreamObject(rio *r, robj *key, robj *o) { * The function returns 0 on error, 1 on success. */ int rewriteModuleObject(rio *r, robj *key, robj *o) { RedisModuleIO io; - moduleValue *mv = o->ptr; + moduleValue *mv = ptrFromObj(o); moduleType *mt = mv->type; moduleInitIOContext(io,mt,r); mt->aof_rewrite(&io,key,mv->value); diff --git a/src/bitops.c b/src/bitops.c index c35b2ca23..b473d9b97 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -411,7 +411,7 @@ void printBits(unsigned char *p, unsigned long count) { int getBitOffsetFromArgument(client *c, robj *o, size_t *offset, int hash, int bits) { long long loffset; char *err = "bit offset is not an integer or out of range"; - char *p = o->ptr; + char *p = ptrFromObj(o); size_t plen = sdslen(p); int usehash = 0; @@ -445,7 +445,7 @@ int getBitOffsetFromArgument(client *c, robj *o, size_t *offset, int hash, int b * * On error C_ERR is returned and an error is sent to the client. */ int getBitfieldTypeFromArgument(client *c, robj *o, int *sign, int *bits) { - char *p = o->ptr; + char *p = ptrFromObj(o); char *err = "Invalid bitfield type. Use something like i16 u8. Note that u64 is not supported but i64 is."; long long llbits; @@ -485,7 +485,7 @@ robj *lookupStringForBitCommand(client *c, size_t maxbit) { } else { if (checkType(c,o,OBJ_STRING)) return NULL; o = dbUnshareStringValue(c->db,c->argv[1],o); - o->ptr = sdsgrowzero(o->ptr,byte+1); + o->m_ptr = sdsgrowzero(ptrFromObj(o),byte+1); } return o; } @@ -511,10 +511,10 @@ unsigned char *getObjectReadOnlyString(robj *o, long *len, char *llbuf) { * array if our string was integer encoded. */ if (o && o->encoding == OBJ_ENCODING_INT) { p = (unsigned char*) llbuf; - if (len) *len = ll2string(llbuf,LONG_STR_SIZE,(long)o->ptr); + if (len) *len = ll2string(llbuf,LONG_STR_SIZE,(long)ptrFromObj(o)); } else if (o) { - p = (unsigned char*) o->ptr; - if (len) *len = sdslen(o->ptr); + p = (unsigned char*) ptrFromObj(o); + if (len) *len = sdslen(ptrFromObj(o)); } else { if (len) *len = 0; } @@ -546,14 +546,14 @@ void setbitCommand(client *c) { /* Get current values */ byte = bitoffset >> 3; - byteval = ((uint8_t*)o->ptr)[byte]; + byteval = ((uint8_t*)ptrFromObj(o))[byte]; bit = 7 - (bitoffset & 0x7); bitval = byteval & (1 << bit); /* Update byte with new bit value and return original value */ byteval &= ~(1 << bit); byteval |= ((on & 0x1) << bit); - ((uint8_t*)o->ptr)[byte] = byteval; + ((uint8_t*)ptrFromObj(o))[byte] = byteval; signalModifiedKey(c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_STRING,"setbit",c->argv[1],c->db->id); server.dirty++; @@ -577,10 +577,10 @@ void getbitCommand(client *c) { byte = bitoffset >> 3; bit = 7 - (bitoffset & 0x7); if (sdsEncodedObject(o)) { - if (byte < sdslen(o->ptr)) - bitval = ((uint8_t*)o->ptr)[byte] & (1 << bit); + if (byte < sdslen(ptrFromObj(o))) + bitval = ((uint8_t*)ptrFromObj(o))[byte] & (1 << bit); } else { - if (byte < (size_t)ll2string(llbuf,sizeof(llbuf),(long)o->ptr)) + if (byte < (size_t)ll2string(llbuf,sizeof(llbuf),(long)ptrFromObj(o))) bitval = llbuf[byte] & (1 << bit); } @@ -589,7 +589,7 @@ void getbitCommand(client *c) { /* BITOP op_name target_key src_key1 src_key2 src_key3 ... src_keyN */ void bitopCommand(client *c) { - char *opname = c->argv[1]->ptr; + char *opname = ptrFromObj(c->argv[1]); robj *o, *targetkey = c->argv[2]; unsigned long op, j, numkeys; robj **objects; /* Array of source objects. */ @@ -647,8 +647,8 @@ void bitopCommand(client *c) { return; } objects[j] = getDecodedObject(o); - src[j] = objects[j]->ptr; - len[j] = sdslen(objects[j]->ptr); + src[j] = ptrFromObj(objects[j]); + len[j] = sdslen(ptrFromObj(objects[j])); if (len[j] > maxlen) maxlen = len[j]; if (j == 0 || len[j] < minlen) minlen = len[j]; } @@ -922,7 +922,7 @@ void bitfieldCommand(client *c) { for (j = 2; j < c->argc; j++) { int remargs = c->argc-j-1; /* Remaining args other than current. */ - char *subcmd = c->argv[j]->ptr; /* Current command name. */ + char *subcmd = ptrFromObj(c->argv[j]); /* Current command name. */ int opcode; /* Current operation code. */ long long i64 = 0; /* Signed SET value. */ int sign = 0; /* Signed or unsigned type? */ @@ -935,7 +935,7 @@ void bitfieldCommand(client *c) { else if (!strcasecmp(subcmd,"incrby") && remargs >= 3) opcode = BITFIELDOP_INCRBY; else if (!strcasecmp(subcmd,"overflow") && remargs >= 1) { - char *owtypename = c->argv[j+1]->ptr; + char *owtypename = ptrFromObj(c->argv[j+1]); j++; if (!strcasecmp(owtypename,"wrap")) owtype = BFOVERFLOW_WRAP; @@ -1023,7 +1023,7 @@ void bitfieldCommand(client *c) { int64_t oldval, newval, wrapped, retval; int overflow; - oldval = getSignedBitfield(o->ptr,thisop->offset, + oldval = getSignedBitfield(ptrFromObj(o),thisop->offset, thisop->bits); if (thisop->opcode == BITFIELDOP_INCRBY) { @@ -1044,7 +1044,7 @@ void bitfieldCommand(client *c) { * NULL to signal the condition. */ if (!(overflow && thisop->owtype == BFOVERFLOW_FAIL)) { addReplyLongLong(c,retval); - setSignedBitfield(o->ptr,thisop->offset, + setSignedBitfield(ptrFromObj(o),thisop->offset, thisop->bits,newval); } else { addReplyNull(c); @@ -1053,7 +1053,7 @@ void bitfieldCommand(client *c) { uint64_t oldval, newval, wrapped, retval; int overflow; - oldval = getUnsignedBitfield(o->ptr,thisop->offset, + oldval = getUnsignedBitfield(ptrFromObj(o),thisop->offset, thisop->bits); if (thisop->opcode == BITFIELDOP_INCRBY) { @@ -1073,7 +1073,7 @@ void bitfieldCommand(client *c) { * NULL to signal the condition. */ if (!(overflow && thisop->owtype == BFOVERFLOW_FAIL)) { addReplyLongLong(c,retval); - setUnsignedBitfield(o->ptr,thisop->offset, + setUnsignedBitfield(ptrFromObj(o),thisop->offset, thisop->bits,newval); } else { addReplyNull(c); diff --git a/src/blocked.c b/src/blocked.c index 24d674935..2ac57b5db 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -374,7 +374,7 @@ void handleClientsBlockedOnKeys(void) { /* Serve clients blocked on stream key. */ else if (o != NULL && o->type == OBJ_STREAM) { dictEntry *de = dictFind(rl->db->blocking_keys,rl->key); - stream *s = o->ptr; + stream *s = ptrFromObj(o); /* We need to provide the new data arrived on the stream * to all the clients that are waiting for an offset smaller @@ -403,7 +403,7 @@ void handleClientsBlockedOnKeys(void) { streamCG *group = NULL; if (receiver->bpop.xread_group) { group = streamLookupCG(s, - receiver->bpop.xread_group->ptr); + ptrFromObj(receiver->bpop.xread_group)); /* If the group was not found, send an error * to the consumer. */ if (!group) { @@ -427,7 +427,7 @@ void handleClientsBlockedOnKeys(void) { if (group) { consumer = streamLookupConsumer(group, - receiver->bpop.xread_consumer->ptr, + ptrFromObj(receiver->bpop.xread_consumer), 1); noack = receiver->bpop.xread_group_noack; } diff --git a/src/cluster.c b/src/cluster.c index c4c0d814c..11eb170fc 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -2523,8 +2523,8 @@ void clusterSendPublish(clusterLink *link, robj *channel, robj *message) { channel = getDecodedObject(channel); message = getDecodedObject(message); - channel_len = sdslen(channel->ptr); - message_len = sdslen(message->ptr); + channel_len = sdslen(ptrFromObj(channel)); + message_len = sdslen(ptrFromObj(message)); clusterBuildMessageHdr(hdr,CLUSTERMSG_TYPE_PUBLISH); totlen = sizeof(clusterMsg)-sizeof(union clusterMsgData); @@ -2542,9 +2542,9 @@ void clusterSendPublish(clusterLink *link, robj *channel, robj *message) { memcpy(payload,hdr,sizeof(*hdr)); hdr = (clusterMsg*) payload; } - memcpy(hdr->data.publish.msg.bulk_data,channel->ptr,sdslen(channel->ptr)); - memcpy(hdr->data.publish.msg.bulk_data+sdslen(channel->ptr), - message->ptr,sdslen(message->ptr)); + memcpy(hdr->data.publish.msg.bulk_data,ptrFromObj(channel),sdslen(ptrFromObj(channel))); + memcpy(hdr->data.publish.msg.bulk_data+sdslen(ptrFromObj(channel)), + ptrFromObj(message),sdslen(ptrFromObj(message))); if (link) clusterSendMessage(link,payload,totlen); @@ -4193,7 +4193,7 @@ void clusterCommand(client *c) { return; } - if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) { + if (c->argc == 2 && !strcasecmp(ptrFromObj(c->argv[1]),"help")) { const char *help[] = { "ADDSLOTS [slot ...] -- Assign slots to current node.", "BUMPEPOCH -- Advance the cluster config epoch.", @@ -4220,35 +4220,35 @@ void clusterCommand(client *c) { NULL }; addReplyHelp(c, help); - } else if (!strcasecmp(c->argv[1]->ptr,"meet") && (c->argc == 4 || c->argc == 5)) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"meet") && (c->argc == 4 || c->argc == 5)) { /* CLUSTER MEET [cport] */ long long port, cport; if (getLongLongFromObject(c->argv[3], &port) != C_OK) { addReplyErrorFormat(c,"Invalid TCP base port specified: %s", - (char*)c->argv[3]->ptr); + (char*)ptrFromObj(c->argv[3])); return; } if (c->argc == 5) { if (getLongLongFromObject(c->argv[4], &cport) != C_OK) { addReplyErrorFormat(c,"Invalid TCP bus port specified: %s", - (char*)c->argv[4]->ptr); + (char*)ptrFromObj(c->argv[4])); return; } } else { cport = port + CLUSTER_PORT_INCR; } - if (clusterStartHandshake(c->argv[2]->ptr,port,cport) == 0 && + if (clusterStartHandshake(ptrFromObj(c->argv[2]),port,cport) == 0 && errno == EINVAL) { addReplyErrorFormat(c,"Invalid node address specified: %s:%s", - (char*)c->argv[2]->ptr, (char*)c->argv[3]->ptr); + (char*)ptrFromObj(c->argv[2]), (char*)ptrFromObj(c->argv[3])); } else { addReply(c,shared.ok); } - } else if (!strcasecmp(c->argv[1]->ptr,"nodes") && c->argc == 2) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"nodes") && c->argc == 2) { /* CLUSTER NODES */ robj *o; sds ci = clusterGenNodesDescription(0); @@ -4256,13 +4256,13 @@ NULL o = createObject(OBJ_STRING,ci); addReplyBulk(c,o); decrRefCount(o); - } else if (!strcasecmp(c->argv[1]->ptr,"myid") && c->argc == 2) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"myid") && c->argc == 2) { /* CLUSTER MYID */ addReplyBulkCBuffer(c,myself->name, CLUSTER_NAMELEN); - } else if (!strcasecmp(c->argv[1]->ptr,"slots") && c->argc == 2) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"slots") && c->argc == 2) { /* CLUSTER SLOTS */ clusterReplyMultiBulkSlots(c); - } else if (!strcasecmp(c->argv[1]->ptr,"flushslots") && c->argc == 2) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"flushslots") && c->argc == 2) { /* CLUSTER FLUSHSLOTS */ if (dictSize(server.db[0].pdict) != 0) { addReplyError(c,"DB must be empty to perform CLUSTER FLUSHSLOTS."); @@ -4271,14 +4271,14 @@ NULL clusterDelNodeSlots(myself); clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE|CLUSTER_TODO_SAVE_CONFIG); addReply(c,shared.ok); - } else if ((!strcasecmp(c->argv[1]->ptr,"addslots") || - !strcasecmp(c->argv[1]->ptr,"delslots")) && c->argc >= 3) + } else if ((!strcasecmp(ptrFromObj(c->argv[1]),"addslots") || + !strcasecmp(ptrFromObj(c->argv[1]),"delslots")) && c->argc >= 3) { /* CLUSTER ADDSLOTS [slot] ... */ /* CLUSTER DELSLOTS [slot] ... */ int j, slot; unsigned char *slots = zmalloc(CLUSTER_SLOTS, MALLOC_LOCAL); - int del = !strcasecmp(c->argv[1]->ptr,"delslots"); + int del = !strcasecmp(ptrFromObj(c->argv[1]),"delslots"); memset(slots,0,CLUSTER_SLOTS); /* Check that all the arguments are parseable and that all the @@ -4321,7 +4321,7 @@ NULL zfree(slots); clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE|CLUSTER_TODO_SAVE_CONFIG); addReply(c,shared.ok); - } else if (!strcasecmp(c->argv[1]->ptr,"setslot") && c->argc >= 4) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"setslot") && c->argc >= 4) { /* SETSLOT 10 MIGRATING */ /* SETSLOT 10 IMPORTING */ /* SETSLOT 10 STABLE */ @@ -4336,40 +4336,40 @@ NULL if ((slot = getSlotOrReply(c,c->argv[2])) == -1) return; - if (!strcasecmp(c->argv[3]->ptr,"migrating") && c->argc == 5) { + if (!strcasecmp(ptrFromObj(c->argv[3]),"migrating") && c->argc == 5) { if (server.cluster->slots[slot] != myself) { addReplyErrorFormat(c,"I'm not the owner of hash slot %u",slot); return; } - if ((n = clusterLookupNode(c->argv[4]->ptr)) == NULL) { + if ((n = clusterLookupNode(ptrFromObj(c->argv[4]))) == NULL) { addReplyErrorFormat(c,"I don't know about node %s", - (char*)c->argv[4]->ptr); + (char*)ptrFromObj(c->argv[4])); return; } server.cluster->migrating_slots_to[slot] = n; - } else if (!strcasecmp(c->argv[3]->ptr,"importing") && c->argc == 5) { + } else if (!strcasecmp(ptrFromObj(c->argv[3]),"importing") && c->argc == 5) { if (server.cluster->slots[slot] == myself) { addReplyErrorFormat(c, "I'm already the owner of hash slot %u",slot); return; } - if ((n = clusterLookupNode(c->argv[4]->ptr)) == NULL) { + if ((n = clusterLookupNode(ptrFromObj(c->argv[4]))) == NULL) { addReplyErrorFormat(c,"I don't know about node %s", - (char*)c->argv[4]->ptr); + (char*)ptrFromObj(c->argv[4])); return; } server.cluster->importing_slots_from[slot] = n; - } else if (!strcasecmp(c->argv[3]->ptr,"stable") && c->argc == 4) { + } else if (!strcasecmp(ptrFromObj(c->argv[3]),"stable") && c->argc == 4) { /* CLUSTER SETSLOT STABLE */ server.cluster->importing_slots_from[slot] = NULL; server.cluster->migrating_slots_to[slot] = NULL; - } else if (!strcasecmp(c->argv[3]->ptr,"node") && c->argc == 5) { + } else if (!strcasecmp(ptrFromObj(c->argv[3]),"node") && c->argc == 5) { /* CLUSTER SETSLOT NODE */ - clusterNode *n = clusterLookupNode(c->argv[4]->ptr); + clusterNode *n = clusterLookupNode(ptrFromObj(c->argv[4])); if (!n) { addReplyErrorFormat(c,"Unknown node %s", - (char*)c->argv[4]->ptr); + (char*)ptrFromObj(c->argv[4])); return; } /* If this hash slot was served by 'myself' before to switch @@ -4418,14 +4418,14 @@ NULL } clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG|CLUSTER_TODO_UPDATE_STATE); addReply(c,shared.ok); - } else if (!strcasecmp(c->argv[1]->ptr,"bumpepoch") && c->argc == 2) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"bumpepoch") && c->argc == 2) { /* CLUSTER BUMPEPOCH */ int retval = clusterBumpConfigEpochWithoutConsensus(); sds reply = sdscatprintf(sdsempty(),"+%s %llu\r\n", (retval == C_OK) ? "BUMPED" : "STILL", (unsigned long long) myself->configEpoch); addReplySds(c,reply); - } else if (!strcasecmp(c->argv[1]->ptr,"info") && c->argc == 2) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"info") && c->argc == 2) { /* CLUSTER INFO */ char *statestr[] = {"ok","fail","needhelp"}; int slots_assigned = 0, slots_ok = 0, slots_pfail = 0, slots_fail = 0; @@ -4501,7 +4501,7 @@ NULL (unsigned long)sdslen(info))); addReplySds(c,info); addReply(c,shared.crlf); - } else if (!strcasecmp(c->argv[1]->ptr,"saveconfig") && c->argc == 2) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"saveconfig") && c->argc == 2) { int retval = clusterSaveConfig(1); if (retval == 0) @@ -4509,12 +4509,12 @@ NULL else addReplyErrorFormat(c,"error saving the cluster node config: %s", strerror(errno)); - } else if (!strcasecmp(c->argv[1]->ptr,"keyslot") && c->argc == 3) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"keyslot") && c->argc == 3) { /* CLUSTER KEYSLOT */ - sds key = c->argv[2]->ptr; + sds key = ptrFromObj(c->argv[2]); addReplyLongLong(c,keyHashSlot(key,sdslen(key))); - } else if (!strcasecmp(c->argv[1]->ptr,"countkeysinslot") && c->argc == 3) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"countkeysinslot") && c->argc == 3) { /* CLUSTER COUNTKEYSINSLOT */ long long slot; @@ -4525,7 +4525,7 @@ NULL return; } addReplyLongLong(c,countKeysInSlot(slot)); - } else if (!strcasecmp(c->argv[1]->ptr,"getkeysinslot") && c->argc == 4) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"getkeysinslot") && c->argc == 4) { /* CLUSTER GETKEYSINSLOT */ long long maxkeys, slot; unsigned int numkeys, j; @@ -4554,12 +4554,12 @@ NULL decrRefCount(keys[j]); } zfree(keys); - } else if (!strcasecmp(c->argv[1]->ptr,"forget") && c->argc == 3) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"forget") && c->argc == 3) { /* CLUSTER FORGET */ - clusterNode *n = clusterLookupNode(c->argv[2]->ptr); + clusterNode *n = clusterLookupNode(ptrFromObj(c->argv[2])); if (!n) { - addReplyErrorFormat(c,"Unknown node %s", (char*)c->argv[2]->ptr); + addReplyErrorFormat(c,"Unknown node %s", (char*)ptrFromObj(c->argv[2])); return; } else if (n == myself) { addReplyError(c,"I tried hard but I can't forget myself..."); @@ -4573,13 +4573,13 @@ NULL clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE| CLUSTER_TODO_SAVE_CONFIG); addReply(c,shared.ok); - } else if (!strcasecmp(c->argv[1]->ptr,"replicate") && c->argc == 3) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"replicate") && c->argc == 3) { /* CLUSTER REPLICATE */ - clusterNode *n = clusterLookupNode(c->argv[2]->ptr); + clusterNode *n = clusterLookupNode(ptrFromObj(c->argv[2])); /* Lookup the specified node in our table. */ if (!n) { - addReplyErrorFormat(c,"Unknown node %s", (char*)c->argv[2]->ptr); + addReplyErrorFormat(c,"Unknown node %s", (char*)ptrFromObj(c->argv[2])); return; } @@ -4610,15 +4610,15 @@ NULL clusterSetMaster(n); clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE|CLUSTER_TODO_SAVE_CONFIG); addReply(c,shared.ok); - } else if ((!strcasecmp(c->argv[1]->ptr,"slaves") || - !strcasecmp(c->argv[1]->ptr,"replicas")) && c->argc == 3) { + } else if ((!strcasecmp(ptrFromObj(c->argv[1]),"slaves") || + !strcasecmp(ptrFromObj(c->argv[1]),"replicas")) && c->argc == 3) { /* CLUSTER SLAVES */ - clusterNode *n = clusterLookupNode(c->argv[2]->ptr); + clusterNode *n = clusterLookupNode(ptrFromObj(c->argv[2])); int j; /* Lookup the specified node in our table. */ if (!n) { - addReplyErrorFormat(c,"Unknown node %s", (char*)c->argv[2]->ptr); + addReplyErrorFormat(c,"Unknown node %s", (char*)ptrFromObj(c->argv[2])); return; } @@ -4633,28 +4633,28 @@ NULL addReplyBulkCString(c,ni); sdsfree(ni); } - } else if (!strcasecmp(c->argv[1]->ptr,"count-failure-reports") && + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"count-failure-reports") && c->argc == 3) { /* CLUSTER COUNT-FAILURE-REPORTS */ - clusterNode *n = clusterLookupNode(c->argv[2]->ptr); + clusterNode *n = clusterLookupNode(ptrFromObj(c->argv[2])); if (!n) { - addReplyErrorFormat(c,"Unknown node %s", (char*)c->argv[2]->ptr); + addReplyErrorFormat(c,"Unknown node %s", (char*)ptrFromObj(c->argv[2])); return; } else { addReplyLongLong(c,clusterNodeFailureReportsCount(n)); } - } else if (!strcasecmp(c->argv[1]->ptr,"failover") && + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"failover") && (c->argc == 2 || c->argc == 3)) { /* CLUSTER FAILOVER [FORCE|TAKEOVER] */ int force = 0, takeover = 0; if (c->argc == 3) { - if (!strcasecmp(c->argv[2]->ptr,"force")) { + if (!strcasecmp(ptrFromObj(c->argv[2]),"force")) { force = 1; - } else if (!strcasecmp(c->argv[2]->ptr,"takeover")) { + } else if (!strcasecmp(ptrFromObj(c->argv[2]),"takeover")) { takeover = 1; force = 1; /* Takeover also implies force. */ } else { @@ -4700,7 +4700,7 @@ NULL clusterSendMFStart(myself->slaveof); } addReply(c,shared.ok); - } else if (!strcasecmp(c->argv[1]->ptr,"set-config-epoch") && c->argc == 3) + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"set-config-epoch") && c->argc == 3) { /* CLUSTER SET-CONFIG-EPOCH * @@ -4736,7 +4736,7 @@ NULL CLUSTER_TODO_SAVE_CONFIG); addReply(c,shared.ok); } - } else if (!strcasecmp(c->argv[1]->ptr,"reset") && + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"reset") && (c->argc == 2 || c->argc == 3)) { /* CLUSTER RESET [SOFT|HARD] */ @@ -4744,9 +4744,9 @@ NULL /* Parse soft/hard argument. Default is soft. */ if (c->argc == 3) { - if (!strcasecmp(c->argv[2]->ptr,"hard")) { + if (!strcasecmp(ptrFromObj(c->argv[2]),"hard")) { hard = 1; - } else if (!strcasecmp(c->argv[2]->ptr,"soft")) { + } else if (!strcasecmp(ptrFromObj(c->argv[2]),"soft")) { hard = 0; } else { addReply(c,shared.syntaxerr); @@ -4860,11 +4860,11 @@ void restoreCommand(client *c) { /* Parse additional options */ for (j = 4; j < c->argc; j++) { int additional = c->argc-j-1; - if (!strcasecmp(c->argv[j]->ptr,"replace")) { + if (!strcasecmp(ptrFromObj(c->argv[j]),"replace")) { replace = 1; - } else if (!strcasecmp(c->argv[j]->ptr,"absttl")) { + } else if (!strcasecmp(ptrFromObj(c->argv[j]),"absttl")) { absttl = 1; - } else if (!strcasecmp(c->argv[j]->ptr,"idletime") && additional >= 1 && + } else if (!strcasecmp(ptrFromObj(c->argv[j]),"idletime") && additional >= 1 && lfu_freq == -1) { if (getLongLongFromObjectOrReply(c,c->argv[j+1],&lru_idle,NULL) @@ -4875,7 +4875,7 @@ void restoreCommand(client *c) { } lru_clock = LRU_CLOCK(); j++; /* Consume additional arg. */ - } else if (!strcasecmp(c->argv[j]->ptr,"freq") && additional >= 1 && + } else if (!strcasecmp(ptrFromObj(c->argv[j]),"freq") && additional >= 1 && lru_idle == -1) { if (getLongLongFromObjectOrReply(c,c->argv[j+1],&lfu_freq,NULL) @@ -4906,13 +4906,13 @@ void restoreCommand(client *c) { } /* Verify RDB version and data checksum. */ - if (verifyDumpPayload(c->argv[3]->ptr,sdslen(c->argv[3]->ptr)) == C_ERR) + if (verifyDumpPayload(ptrFromObj(c->argv[3]),sdslen(ptrFromObj(c->argv[3]))) == C_ERR) { addReplyError(c,"DUMP payload version or checksum are wrong"); return; } - rioInitWithBuffer(&payload,c->argv[3]->ptr); + rioInitWithBuffer(&payload,ptrFromObj(c->argv[3])); if (((type = rdbLoadObjectType(&payload)) == -1) || ((obj = rdbLoadObject(type,&payload)) == NULL)) { @@ -4967,9 +4967,9 @@ migrateCachedSocket* migrateGetSocket(client *c, robj *host, robj *port, long ti migrateCachedSocket *cs; /* Check if we have an already cached socket for this ip:port pair. */ - name = sdscatlen(name,host->ptr,sdslen(host->ptr)); + name = sdscatlen(name,ptrFromObj(host),sdslen(ptrFromObj(host))); name = sdscatlen(name,":",1); - name = sdscatlen(name,port->ptr,sdslen(port->ptr)); + name = sdscatlen(name,ptrFromObj(port),sdslen(ptrFromObj(port))); cs = dictFetchValue(server.migrate_cached_sockets,name); if (cs) { sdsfree(name); @@ -4988,8 +4988,8 @@ migrateCachedSocket* migrateGetSocket(client *c, robj *host, robj *port, long ti } /* Create the socket */ - fd = anetTcpNonBlockConnect(server.neterr,c->argv[1]->ptr, - atoi(c->argv[2]->ptr)); + fd = anetTcpNonBlockConnect(server.neterr,ptrFromObj(c->argv[1]), + atoi(ptrFromObj(c->argv[2]))); if (fd == -1) { sdsfree(name); addReplyErrorFormat(c,"Can't connect to target node: %s", @@ -5021,9 +5021,9 @@ void migrateCloseSocket(robj *host, robj *port) { sds name = sdsempty(); migrateCachedSocket *cs; - name = sdscatlen(name,host->ptr,sdslen(host->ptr)); + name = sdscatlen(name,ptrFromObj(host),sdslen(ptrFromObj(host))); name = sdscatlen(name,":",1); - name = sdscatlen(name,port->ptr,sdslen(port->ptr)); + name = sdscatlen(name,ptrFromObj(port),sdslen(ptrFromObj(port))); cs = dictFetchValue(server.migrate_cached_sockets,name); if (!cs) { sdsfree(name); @@ -5079,19 +5079,19 @@ void migrateCommand(client *c) { /* Parse additional options */ for (j = 6; j < c->argc; j++) { int moreargs = j < c->argc-1; - if (!strcasecmp(c->argv[j]->ptr,"copy")) { + if (!strcasecmp(ptrFromObj(c->argv[j]),"copy")) { copy = 1; - } else if (!strcasecmp(c->argv[j]->ptr,"replace")) { + } else if (!strcasecmp(ptrFromObj(c->argv[j]),"replace")) { replace = 1; - } else if (!strcasecmp(c->argv[j]->ptr,"auth")) { + } else if (!strcasecmp(ptrFromObj(c->argv[j]),"auth")) { if (!moreargs) { addReply(c,shared.syntaxerr); return; } j++; - password = c->argv[j]->ptr; - } else if (!strcasecmp(c->argv[j]->ptr,"keys")) { - if (sdslen(c->argv[3]->ptr) != 0) { + password = ptrFromObj(c->argv[j]); + } else if (!strcasecmp(ptrFromObj(c->argv[j]),"keys")) { + if (sdslen(ptrFromObj(c->argv[3])) != 0) { addReplyError(c, "When using MIGRATE KEYS option, the key argument" " must be set to the empty string"); @@ -5196,8 +5196,8 @@ try_again: else serverAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,"RESTORE",7)); serverAssertWithInfo(c,NULL,sdsEncodedObject(kv[j])); - serverAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,kv[j]->ptr, - sdslen(kv[j]->ptr))); + serverAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,ptrFromObj(kv[j]), + sdslen(ptrFromObj(kv[j])))); serverAssertWithInfo(c,NULL,rioWriteBulkLongLong(&cmd,ttl)); /* Emit the payload argument, that is the serialized object using @@ -5500,8 +5500,8 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in keyindex = getKeysFromCommand(mcmd,margv,margc,&numkeys); for (j = 0; j < numkeys; j++) { robj *thiskey = margv[keyindex[j]]; - int thisslot = keyHashSlot((char*)thiskey->ptr, - sdslen(thiskey->ptr)); + int thisslot = keyHashSlot((char*)ptrFromObj(thiskey), + sdslen(ptrFromObj(thiskey))); if (firstkey == NULL) { /* This is the first key we see. Check what is the slot @@ -5681,7 +5681,7 @@ int clusterRedirectBlockedClientIfNeeded(client *c) { di = dictGetIterator(c->bpop.keys); if ((de = dictNext(di)) != NULL) { robj *key = dictGetKey(de); - int slot = keyHashSlot((char*)key->ptr, sdslen(key->ptr)); + int slot = keyHashSlot((char*)ptrFromObj(key), sdslen(ptrFromObj(key))); clusterNode *node = server.cluster->slots[slot]; /* We send an error and unblock the client if: diff --git a/src/config.c b/src/config.c index 4df08e47f..7e9b19d76 100644 --- a/src/config.c +++ b/src/config.c @@ -889,36 +889,36 @@ void loadServerConfig(char *filename, char *options) { *----------------------------------------------------------------------------*/ #define config_set_bool_field(_name,_var) \ - } else if (!strcasecmp(c->argv[2]->ptr,_name)) { \ - int yn = yesnotoi(o->ptr); \ + } else if (!strcasecmp(ptrFromObj(c->argv[2]),_name)) { \ + int yn = yesnotoi(ptrFromObj(o)); \ if (yn == -1) goto badfmt; \ _var = yn; #define config_set_numerical_field(_name,_var,min,max) \ - } else if (!strcasecmp(c->argv[2]->ptr,_name)) { \ + } else if (!strcasecmp(ptrFromObj(c->argv[2]),_name)) { \ if (getLongLongFromObject(o,&ll) == C_ERR) goto badfmt; \ if (min != LLONG_MIN && ll < min) goto badfmt; \ if (max != LLONG_MAX && ll > max) goto badfmt; \ _var = ll; #define config_set_memory_field(_name,_var) \ - } else if (!strcasecmp(c->argv[2]->ptr,_name)) { \ - ll = memtoll(o->ptr,&err); \ + } else if (!strcasecmp(ptrFromObj(c->argv[2]),_name)) { \ + ll = memtoll(ptrFromObj(o),&err); \ if (err || ll < 0) goto badfmt; \ _var = ll; #define config_set_enum_field(_name,_var,_enumvar) \ - } else if (!strcasecmp(c->argv[2]->ptr,_name)) { \ - int enumval = configEnumGetValue(_enumvar,o->ptr); \ + } else if (!strcasecmp(ptrFromObj(c->argv[2]),_name)) { \ + int enumval = configEnumGetValue(_enumvar,ptrFromObj(o)); \ if (enumval == INT_MIN) goto badfmt; \ _var = enumval; #define config_set_special_field(_name) \ - } else if (!strcasecmp(c->argv[2]->ptr,_name)) { + } else if (!strcasecmp(ptrFromObj(c->argv[2]),_name)) { #define config_set_special_field_with_alias(_name1,_name2) \ - } else if (!strcasecmp(c->argv[2]->ptr,_name1) || \ - !strcasecmp(c->argv[2]->ptr,_name2)) { + } else if (!strcasecmp(ptrFromObj(c->argv[2]),_name1) || \ + !strcasecmp(ptrFromObj(c->argv[2]),_name2)) { #define config_set_else } else @@ -934,26 +934,26 @@ void configSetCommand(client *c) { /* Special fields that can't be handled with general macros. */ config_set_special_field("dbfilename") { - if (!pathIsBaseName(o->ptr)) { + if (!pathIsBaseName(ptrFromObj(o))) { addReplyError(c, "dbfilename can't be a path, just a filename"); return; } zfree(server.rdb_filename); - server.rdb_filename = zstrdup(o->ptr); + server.rdb_filename = zstrdup(ptrFromObj(o)); } config_set_special_field("requirepass") { - if (sdslen(o->ptr) > CONFIG_AUTHPASS_MAX_LEN) goto badfmt; + if (sdslen(ptrFromObj(o)) > CONFIG_AUTHPASS_MAX_LEN) goto badfmt; /* The old "requirepass" directive just translates to setting * a password to the default user. */ ACLSetUser(DefaultUser,"resetpass",-1); - sds aclop = sdscatprintf(sdsempty(),">%s",(char*)o->ptr); + sds aclop = sdscatprintf(sdsempty(),">%s",(char*)ptrFromObj(o)); ACLSetUser(DefaultUser,aclop,sdslen(aclop)); sdsfree(aclop); } config_set_special_field("masterauth") { zfree(server.masterauth); - server.masterauth = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; + server.masterauth = ((char*)ptrFromObj(o))[0] ? zstrdup(ptrFromObj(o)) : NULL; } config_set_special_field("cluster-announce-ip") { zfree(server.cluster_announce_ip); - server.cluster_announce_ip = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; + server.cluster_announce_ip = ((char*)ptrFromObj(o))[0] ? zstrdup(ptrFromObj(o)) : NULL; } config_set_special_field("maxclients") { int orig_value = server.maxclients; @@ -981,7 +981,7 @@ void configSetCommand(client *c) { } } } config_set_special_field("appendonly") { - int enable = yesnotoi(o->ptr); + int enable = yesnotoi(ptrFromObj(o)); if (enable == -1) goto badfmt; if (enable == 0 && server.aof_state != AOF_OFF) { @@ -995,7 +995,7 @@ void configSetCommand(client *c) { } } config_set_special_field("save") { int vlen, j; - sds *v = sdssplitlen(o->ptr,sdslen(o->ptr)," ",1,&vlen); + sds *v = sdssplitlen(ptrFromObj(o),sdslen(ptrFromObj(o))," ",1,&vlen); /* Perform sanity check before setting the new config: * - Even number of args @@ -1028,13 +1028,13 @@ void configSetCommand(client *c) { } sdsfreesplitres(v,vlen); } config_set_special_field("dir") { - if (chdir((char*)o->ptr) == -1) { + if (chdir((char*)ptrFromObj(o)) == -1) { addReplyErrorFormat(c,"Changing directory: %s", strerror(errno)); return; } } config_set_special_field("client-output-buffer-limit") { int vlen, j; - sds *v = sdssplitlen(o->ptr,sdslen(o->ptr)," ",1,&vlen); + sds *v = sdssplitlen(ptrFromObj(o),sdslen(ptrFromObj(o))," ",1,&vlen); /* We need a multiple of 4: */ if (vlen % 4) { @@ -1079,7 +1079,7 @@ void configSetCommand(client *c) { } sdsfreesplitres(v,vlen); } config_set_special_field("notify-keyspace-events") { - int flags = keyspaceEventsStringToFlags(o->ptr); + int flags = keyspaceEventsStringToFlags(ptrFromObj(o)); if (flags == -1) goto badfmt; server.notify_keyspace_events = flags; @@ -1087,7 +1087,7 @@ void configSetCommand(client *c) { "replica-announce-ip") { zfree(server.slave_announce_ip); - server.slave_announce_ip = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; + server.slave_announce_ip = ((char*)ptrFromObj(o))[0] ? zstrdup(ptrFromObj(o)) : NULL; /* Boolean fields. * config_set_bool_field(name,var). */ @@ -1298,7 +1298,7 @@ void configSetCommand(client *c) { /* Everyhing else is an error... */ } config_set_else { addReplyErrorFormat(c,"Unsupported CONFIG parameter: %s", - (char*)c->argv[2]->ptr); + (char*)ptrFromObj(c->argv[2])); return; } @@ -1308,8 +1308,8 @@ void configSetCommand(client *c) { badfmt: /* Bad format errors */ addReplyErrorFormat(c,"Invalid argument '%s' for CONFIG SET '%s'", - (char*)o->ptr, - (char*)c->argv[2]->ptr); + (char*)ptrFromObj(o), + (char*)ptrFromObj(c->argv[2])); } /*----------------------------------------------------------------------------- @@ -1352,7 +1352,7 @@ badfmt: /* Bad format errors */ void configGetCommand(client *c) { robj *o = c->argv[2]; void *replylen = addReplyDeferredLen(c); - char *pattern = o->ptr; + char *pattern = ptrFromObj(o); char buf[128]; int matches = 0; serverAssertWithInfo(c,o,sdsEncodedObject(o)); @@ -2335,12 +2335,12 @@ int rewriteConfig(char *path) { void configCommand(client *c) { /* Only allow CONFIG GET while loading. */ - if (server.loading && strcasecmp(c->argv[1]->ptr,"get")) { + if (server.loading && strcasecmp(ptrFromObj(c->argv[1]),"get")) { addReplyError(c,"Only CONFIG GET is allowed during loading"); return; } - if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) { + if (c->argc == 2 && !strcasecmp(ptrFromObj(c->argv[1]),"help")) { const char *help[] = { "GET -- Return parameters matching the glob-like and their values.", "SET -- Set parameter to value.", @@ -2349,15 +2349,15 @@ void configCommand(client *c) { NULL }; addReplyHelp(c, help); - } else if (!strcasecmp(c->argv[1]->ptr,"set") && c->argc == 4) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"set") && c->argc == 4) { configSetCommand(c); - } else if (!strcasecmp(c->argv[1]->ptr,"get") && c->argc == 3) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"get") && c->argc == 3) { configGetCommand(c); - } else if (!strcasecmp(c->argv[1]->ptr,"resetstat") && c->argc == 2) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"resetstat") && c->argc == 2) { resetServerStats(); resetCommandTableStats(); addReply(c,shared.ok); - } else if (!strcasecmp(c->argv[1]->ptr,"rewrite") && c->argc == 2) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"rewrite") && c->argc == 2) { if (server.configfile == NULL) { addReplyError(c,"The server is running without a config file"); return; diff --git a/src/db.c b/src/db.c index c3b8ffd80..c8553c985 100644 --- a/src/db.c +++ b/src/db.c @@ -53,7 +53,7 @@ void updateLFU(robj *val) { * implementations that should instead rely on lookupKeyRead(), * lookupKeyWrite() and lookupKeyReadWithFlags(). */ robj *lookupKey(redisDb *db, robj *key, int flags) { - dictEntry *de = dictFind(db->pdict,key->ptr); + dictEntry *de = dictFind(db->pdict,ptrFromObj(key)); if (de) { robj *val = dictGetVal(de); @@ -171,7 +171,7 @@ robj *lookupKeyWriteOrReply(client *c, robj *key, robj *reply) { * * The program is aborted if the key already exists. */ void dbAdd(redisDb *db, robj *key, robj *val) { - sds copy = sdsdup(key->ptr); + sds copy = sdsdup(ptrFromObj(key)); int retval = dictAdd(db->pdict, copy, val); serverAssertWithInfo(NULL,key,retval == DICT_OK); @@ -187,7 +187,7 @@ void dbAdd(redisDb *db, robj *key, robj *val) { * * The program is aborted if the key was not already present. */ void dbOverwrite(redisDb *db, robj *key, robj *val) { - dictEntry *de = dictFind(db->pdict,key->ptr); + dictEntry *de = dictFind(db->pdict,ptrFromObj(key)); serverAssertWithInfo(NULL,key,de != NULL); dictEntry auxentry = *de; @@ -225,7 +225,7 @@ void setKey(redisDb *db, robj *key, robj *val) { } int dbExists(redisDb *db, robj *key) { - return dictFind(db->pdict,key->ptr) != NULL; + return dictFind(db->pdict,ptrFromObj(key)) != NULL; } /* Return a random key, in form of a Redis object. @@ -271,8 +271,8 @@ robj *dbRandomKey(redisDb *db) { int dbSyncDelete(redisDb *db, robj *key) { /* Deleting an entry from the expires dict will not free the sds of * the key, because it is shared with the main dictionary. */ - if (dictSize(db->expires) > 0) dictDelete(db->expires,key->ptr); - if (dictDelete(db->pdict,key->ptr) == DICT_OK) { + if (dictSize(db->expires) > 0) dictDelete(db->expires,ptrFromObj(key)); + if (dictDelete(db->pdict,ptrFromObj(key)) == DICT_OK) { if (server.cluster_enabled) slotToKeyDel(key); return 1; } else { @@ -318,7 +318,7 @@ robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o) { serverAssert(o->type == OBJ_STRING); if (o->refcount != 1 || o->encoding != OBJ_ENCODING_RAW) { robj *decoded = getDecodedObject(o); - o = createRawStringObject(decoded->ptr, sdslen(decoded->ptr)); + o = createRawStringObject(ptrFromObj(decoded), sdslen(ptrFromObj(decoded))); decrRefCount(decoded); dbOverwrite(db,key,o); } @@ -415,7 +415,7 @@ void signalFlushedDb(int dbid) { int getFlushCommandFlags(client *c, int *flags) { /* Parse the optional ASYNC option. */ if (c->argc > 1) { - if (c->argc > 2 || strcasecmp(c->argv[1]->ptr,"async")) { + if (c->argc > 2 || strcasecmp(ptrFromObj(c->argv[1]),"async")) { addReply(c,shared.syntaxerr); return C_ERR; } @@ -533,7 +533,7 @@ void randomkeyCommand(client *c) { void keysCommand(client *c) { dictIterator *di; dictEntry *de; - sds pattern = c->argv[1]->ptr; + sds pattern = ptrFromObj(c->argv[1]); int plen = sdslen(pattern), allkeys; unsigned long numkeys = 0; void *replylen = addReplyDeferredLen(c); @@ -598,8 +598,8 @@ int parseScanCursorOrReply(client *c, robj *o, unsigned long *cursor) { /* Use strtoul() because we need an *unsigned* long, so * getLongLongFromObject() does not cover the whole cursor space. */ errno = 0; - *cursor = strtoul(o->ptr, &eptr, 10); - if (isspace(((char*)o->ptr)[0]) || eptr[0] != '\0' || errno == ERANGE) + *cursor = strtoul(ptrFromObj(o), &eptr, 10); + if (isspace(((char*)ptrFromObj(o))[0]) || eptr[0] != '\0' || errno == ERANGE) { addReplyError(c, "invalid cursor"); return C_ERR; @@ -638,7 +638,7 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { /* Step 1: Parse options. */ while (i < c->argc) { j = c->argc - i; - if (!strcasecmp(c->argv[i]->ptr, "count") && j >= 2) { + if (!strcasecmp(ptrFromObj(c->argv[i]), "count") && j >= 2) { if (getLongFromObjectOrReply(c, c->argv[i+1], &count, NULL) != C_OK) { @@ -651,8 +651,8 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { } i += 2; - } else if (!strcasecmp(c->argv[i]->ptr, "match") && j >= 2) { - pat = c->argv[i+1]->ptr; + } else if (!strcasecmp(ptrFromObj(c->argv[i]), "match") && j >= 2) { + pat = ptrFromObj(c->argv[i+1]); patlen = sdslen(pat); /* The pattern always matches if it is exactly "*", so it is @@ -679,12 +679,12 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { if (o == NULL) { ht = c->db->pdict; } else if (o->type == OBJ_SET && o->encoding == OBJ_ENCODING_HT) { - ht = o->ptr; + ht = ptrFromObj(o); } else if (o->type == OBJ_HASH && o->encoding == OBJ_ENCODING_HT) { - ht = o->ptr; + ht = ptrFromObj(o); count *= 2; /* We return key / value for this type. */ } else if (o->type == OBJ_ZSET && o->encoding == OBJ_ENCODING_SKIPLIST) { - zset *zs = o->ptr; + zset *zs = ptrFromObj(o); ht = zs->pdict; count *= 2; /* We return key / value for this type. */ } @@ -711,11 +711,11 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { int pos = 0; int64_t ll; - while(intsetGet(o->ptr,pos++,&ll)) + while(intsetGet(ptrFromObj(o),pos++,&ll)) listAddNodeTail(keys,createStringObjectFromLongLong(ll)); cursor = 0; } else if (o->type == OBJ_HASH || o->type == OBJ_ZSET) { - unsigned char *p = ziplistIndex(o->ptr,0); + unsigned char *p = ziplistIndex(ptrFromObj(o),0); unsigned char *vstr; unsigned int vlen; long long vll; @@ -725,7 +725,7 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { listAddNodeTail(keys, (vstr != NULL) ? createStringObject((char*)vstr,vlen) : createStringObjectFromLongLong(vll)); - p = ziplistNext(o->ptr,p); + p = ziplistNext(ptrFromObj(o),p); } cursor = 0; } else { @@ -742,14 +742,14 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { /* Filter element if it does not match the pattern. */ if (!filter && use_pattern) { if (sdsEncodedObject(kobj)) { - if (!stringmatchlen(pat, patlen, kobj->ptr, sdslen(kobj->ptr), 0)) + if (!stringmatchlen(pat, patlen, ptrFromObj(kobj), sdslen(ptrFromObj(kobj)), 0)) filter = 1; } else { char buf[LONG_STR_SIZE]; int len; serverAssert(kobj->encoding == OBJ_ENCODING_INT); - len = ll2string(buf,sizeof(buf),(long)kobj->ptr); + len = ll2string(buf,sizeof(buf),(long)ptrFromObj(kobj)); if (!stringmatchlen(pat, patlen, buf, len, 0)) filter = 1; } } @@ -826,7 +826,7 @@ void typeCommand(client *c) { case OBJ_HASH: type = "hash"; break; case OBJ_STREAM: type = "stream"; break; case OBJ_MODULE: { - moduleValue *mv = o->ptr; + moduleValue *mv = ptrFromObj(o); type = mv->type->name; }; break; default: type = "unknown"; break; @@ -842,9 +842,9 @@ void shutdownCommand(client *c) { addReply(c,shared.syntaxerr); return; } else if (c->argc == 2) { - if (!strcasecmp(c->argv[1]->ptr,"nosave")) { + if (!strcasecmp(ptrFromObj(c->argv[1]),"nosave")) { flags |= SHUTDOWN_NOSAVE; - } else if (!strcasecmp(c->argv[1]->ptr,"save")) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"save")) { flags |= SHUTDOWN_SAVE; } else { addReply(c,shared.syntaxerr); @@ -870,7 +870,7 @@ void renameGenericCommand(client *c, int nx) { /* When source and dest key is the same, no operation is performed, * if the key exists, however we still return an error on unexisting key. */ - if (sdscmp(c->argv[1]->ptr,c->argv[2]->ptr) == 0) samekey = 1; + if (sdscmp(ptrFromObj(c->argv[1]),ptrFromObj(c->argv[2])) == 0) samekey = 1; if ((o = lookupKeyWriteOrReply(c,c->argv[1],shared.nokeyerr)) == NULL) return; @@ -1062,8 +1062,8 @@ void swapdbCommand(client *c) { int removeExpire(redisDb *db, robj *key) { /* An expire may only be removed if there is a corresponding entry in the * main dict. Otherwise, the key will never be freed. */ - serverAssertWithInfo(NULL,key,dictFind(db->pdict,key->ptr) != NULL); - return dictDelete(db->expires,key->ptr) == DICT_OK; + serverAssertWithInfo(NULL,key,dictFind(db->pdict,ptrFromObj(key)) != NULL); + return dictDelete(db->expires,ptrFromObj(key)) == DICT_OK; } /* Set an expire to the specified key. If the expire is set in the context @@ -1074,7 +1074,7 @@ void setExpire(client *c, redisDb *db, robj *key, long long when) { dictEntry *kde, *de; /* Reuse the sds from the main dict in the expire dict */ - kde = dictFind(db->pdict,key->ptr); + kde = dictFind(db->pdict,ptrFromObj(key)); serverAssertWithInfo(NULL,key,kde != NULL); de = dictAddOrFind(db->expires,dictGetKey(kde)); dictSetSignedIntegerVal(de,when); @@ -1091,11 +1091,11 @@ long long getExpire(redisDb *db, robj *key) { /* No expire? return ASAP */ if (dictSize(db->expires) == 0 || - (de = dictFind(db->expires,key->ptr)) == NULL) return -1; + (de = dictFind(db->expires,ptrFromObj(key))) == NULL) return -1; /* The entry was found in the expire dict, this means it should also * be present in the main dict (safety check). */ - serverAssertWithInfo(NULL,key,dictFind(db->pdict,key->ptr) != NULL); + serverAssertWithInfo(NULL,key,dictFind(db->pdict,ptrFromObj(key)) != NULL); return dictGetSignedIntegerVal(de); } @@ -1256,7 +1256,7 @@ int *zunionInterGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *nu int i, num, *keys; UNUSED(cmd); - num = atoi(argv[2]->ptr); + num = atoi(ptrFromObj(argv[2])); /* Sanity check. Don't return any key if the command is going to * reply with syntax error. */ if (num < 1 || num > (argc-3)) { @@ -1285,7 +1285,7 @@ int *evalGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) int i, num, *keys; UNUSED(cmd); - num = atoi(argv[2]->ptr); + num = atoi(ptrFromObj(argv[2])); /* Sanity check. Don't return any key if the command is going to * reply with syntax error. */ if (num <= 0 || num > (argc-3)) { @@ -1334,10 +1334,10 @@ int *sortGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) for (i = 2; i < argc; i++) { for (j = 0; skiplist[j].name != NULL; j++) { - if (!strcasecmp(argv[i]->ptr,skiplist[j].name)) { + if (!strcasecmp(ptrFromObj(argv[i]),skiplist[j].name)) { i += skiplist[j].skip; break; - } else if (!strcasecmp(argv[i]->ptr,"store") && i+1 < argc) { + } else if (!strcasecmp(ptrFromObj(argv[i]),"store") && i+1 < argc) { /* Note: we don't increment "num" here and continue the loop * to be sure to process the *last* "STORE" option if multiple * ones are provided. This is same behavior as SORT. */ @@ -1362,8 +1362,8 @@ int *migrateGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkey /* But check for the extended one with the KEYS option. */ if (argc > 6) { for (i = 6; i < argc; i++) { - if (!strcasecmp(argv[i]->ptr,"keys") && - sdslen(argv[3]->ptr) == 0) + if (!strcasecmp(ptrFromObj(argv[i]),"keys") && + sdslen(ptrFromObj(argv[3])) == 0) { first = i+1; num = argc-first; @@ -1389,7 +1389,7 @@ int *georadiusGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numk /* Check for the presence of the stored key in the command */ int stored_key = -1; for (i = 5; i < argc; i++) { - char *arg = argv[i]->ptr; + char *arg = ptrFromObj(argv[i]); /* For the case when user specifies both "store" and "storedist" options, the * second key specified would override the first key. This behavior is kept * the same as in georadiusCommand method. @@ -1428,7 +1428,7 @@ int *xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) * name of the stream key. */ int streams_pos = -1; for (i = 1; i < argc; i++) { - char *arg = argv[i]->ptr; + char *arg = ptrFromObj(argv[i]); if (!strcasecmp(arg, "block")) { i++; /* Skip option argument. */ } else if (!strcasecmp(arg, "count")) { @@ -1465,16 +1465,16 @@ int *xreadGetKeys(struct redisCommand *cmd, robj **argv, int argc, int *numkeys) * while rehashing the cluster and in other conditions when we need to * understand if we have keys for a given hash slot. */ void slotToKeyUpdateKey(robj *key, int add) { - unsigned int hashslot = keyHashSlot(key->ptr,sdslen(key->ptr)); + unsigned int hashslot = keyHashSlot(ptrFromObj(key),sdslen(ptrFromObj(key))); unsigned char buf[64]; unsigned char *indexed = buf; - size_t keylen = sdslen(key->ptr); + size_t keylen = sdslen(ptrFromObj(key)); server.cluster->slots_keys_count[hashslot] += add ? 1 : -1; if (keylen+2 > 64) indexed = zmalloc(keylen+2, MALLOC_SHARED); indexed[0] = (hashslot >> 8) & 0xff; indexed[1] = hashslot & 0xff; - memcpy(indexed+2,key->ptr,keylen); + memcpy(indexed+2,ptrFromObj(key),keylen); if (add) { raxInsert(server.cluster->slots_to_keys,indexed,keylen+2,NULL,NULL); } else { diff --git a/src/debug.c b/src/debug.c index 36942e61e..51e5f39f5 100644 --- a/src/debug.c +++ b/src/debug.c @@ -76,7 +76,7 @@ void xorDigest(unsigned char *digest, void *ptr, size_t len) { void xorStringObjectDigest(unsigned char *digest, robj *o) { o = getDecodedObject(o); - xorDigest(digest,o->ptr,sdslen(o->ptr)); + xorDigest(digest,ptrFromObj(o),sdslen(ptrFromObj(o))); decrRefCount(o); } @@ -106,7 +106,7 @@ void mixDigest(unsigned char *digest, void *ptr, size_t len) { void mixStringObjectDigest(unsigned char *digest, robj *o) { o = getDecodedObject(o); - mixDigest(digest,o->ptr,sdslen(o->ptr)); + mixDigest(digest,ptrFromObj(o),sdslen(ptrFromObj(o))); decrRefCount(o); } @@ -148,7 +148,7 @@ void xorObjectDigest(redisDb *db, robj *keyobj, unsigned char *digest, robj *o) unsigned char eledigest[20]; if (o->encoding == OBJ_ENCODING_ZIPLIST) { - unsigned char *zl = o->ptr; + unsigned char *zl = ptrFromObj(o); unsigned char *eptr, *sptr; unsigned char *vstr; unsigned int vlen; @@ -178,7 +178,7 @@ void xorObjectDigest(redisDb *db, robj *keyobj, unsigned char *digest, robj *o) zzlNext(zl,&eptr,&sptr); } } else if (o->encoding == OBJ_ENCODING_SKIPLIST) { - zset *zs = o->ptr; + zset *zs = ptrFromObj(o); dictIterator *di = dictGetIterator(zs->pdict); dictEntry *de; @@ -214,7 +214,7 @@ void xorObjectDigest(redisDb *db, robj *keyobj, unsigned char *digest, robj *o) hashTypeReleaseIterator(hi); } else if (o->type == OBJ_STREAM) { streamIterator si; - streamIteratorStart(&si,o->ptr,NULL,NULL,0); + streamIteratorStart(&si,ptrFromObj(o),NULL,NULL,0); streamID id; int64_t numfields; @@ -235,7 +235,7 @@ void xorObjectDigest(redisDb *db, robj *keyobj, unsigned char *digest, robj *o) streamIteratorStop(&si); } else if (o->type == OBJ_MODULE) { RedisModuleDigest md; - moduleValue *mv = o->ptr; + moduleValue *mv = ptrFromObj(o); moduleType *mt = mv->type; moduleInitDigestContext(md); if (mt->digest) { @@ -298,7 +298,7 @@ void computeDatasetDigest(unsigned char *final) { } void debugCommand(client *c) { - if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) { + if (c->argc == 2 && !strcasecmp(ptrFromObj(c->argv[1]),"help")) { const char *help[] = { "ASSERT -- Crash by assertion failed.", "CHANGE-REPL-ID -- Change the replication IDs of the instance. Dangerous, should be used only for testing the replication subsystem.", @@ -326,12 +326,12 @@ void debugCommand(client *c) { NULL }; addReplyHelp(c, help); - } else if (!strcasecmp(c->argv[1]->ptr,"segfault")) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"segfault")) { *((char*)-1) = 'x'; - } else if (!strcasecmp(c->argv[1]->ptr,"panic")) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"panic")) { serverPanic("DEBUG PANIC called at Unix time %ld", time(NULL)); - } else if (!strcasecmp(c->argv[1]->ptr,"restart") || - !strcasecmp(c->argv[1]->ptr,"crash-and-recover")) + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"restart") || + !strcasecmp(ptrFromObj(c->argv[1]),"crash-and-recover")) { long long delay = 0; if (c->argc >= 3) { @@ -339,21 +339,21 @@ NULL != C_OK) return; if (delay < 0) delay = 0; } - int flags = !strcasecmp(c->argv[1]->ptr,"restart") ? + int flags = !strcasecmp(ptrFromObj(c->argv[1]),"restart") ? (RESTART_SERVER_GRACEFULLY|RESTART_SERVER_CONFIG_REWRITE) : RESTART_SERVER_NONE; restartServer(flags,delay); addReplyError(c,"failed to restart the server. Check server logs."); - } else if (!strcasecmp(c->argv[1]->ptr,"oom")) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"oom")) { void *ptr = zmalloc(ULONG_MAX, MALLOC_LOCAL); /* Should trigger an out of memory. */ zfree(ptr); addReply(c,shared.ok); - } else if (!strcasecmp(c->argv[1]->ptr,"assert")) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"assert")) { serverAssertWithInfo(c,c->argv[0],1 == 2); - } else if (!strcasecmp(c->argv[1]->ptr,"log") && c->argc == 3) { - serverLog(LL_WARNING, "DEBUG LOG: %s", (char*)c->argv[2]->ptr); + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"log") && c->argc == 3) { + serverLog(LL_WARNING, "DEBUG LOG: %s", (char*)ptrFromObj(c->argv[2])); addReply(c,shared.ok); - } else if (!strcasecmp(c->argv[1]->ptr,"reload")) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"reload")) { rdbSaveInfo rsi, *rsiptr; rsiptr = rdbPopulateSaveInfo(&rsi); if (rdbSave(rsiptr) != C_OK) { @@ -370,7 +370,7 @@ NULL } serverLog(LL_WARNING,"DB reloaded by DEBUG RELOAD"); addReply(c,shared.ok); - } else if (!strcasecmp(c->argv[1]->ptr,"loadaof")) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"loadaof")) { if (server.aof_state != AOF_OFF) flushAppendOnlyFile(1); emptyDb(-1,EMPTYDB_NO_FLAGS,NULL); protectClient(c); @@ -383,12 +383,12 @@ NULL server.dirty = 0; /* Prevent AOF / replication */ serverLog(LL_WARNING,"Append Only File loaded by DEBUG LOADAOF"); addReply(c,shared.ok); - } else if (!strcasecmp(c->argv[1]->ptr,"object") && c->argc == 3) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"object") && c->argc == 3) { dictEntry *de; robj *val; char *strenc; - if ((de = dictFind(c->db->pdict,c->argv[2]->ptr)) == NULL) { + if ((de = dictFind(c->db->pdict,ptrFromObj(c->argv[2]))) == NULL) { addReply(c,shared.nokeyerr); return; } @@ -399,7 +399,7 @@ NULL if (val->encoding == OBJ_ENCODING_QUICKLIST) { char *nextra = extra; int remaining = sizeof(extra); - quicklist *ql = val->ptr; + quicklist *ql = val->m_ptr; /* Add number of quicklist nodes */ int used = snprintf(nextra, remaining, " ql_nodes:%lu", ql->len); nextra += used; @@ -435,12 +435,12 @@ NULL (void*)val, val->refcount, strenc, rdbSavedObjectLen(val), val->lru, estimateObjectIdleTime(val)/1000, extra); - } else if (!strcasecmp(c->argv[1]->ptr,"sdslen") && c->argc == 3) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"sdslen") && c->argc == 3) { dictEntry *de; robj *val; sds key; - if ((de = dictFind(c->db->pdict,c->argv[2]->ptr)) == NULL) { + if ((de = dictFind(c->db->pdict,ptrFromObj(c->argv[2]))) == NULL) { addReply(c,shared.nokeyerr); return; } @@ -456,11 +456,11 @@ NULL (long long) sdslen(key), (long long) sdsavail(key), (long long) sdsZmallocSize(key), - (long long) sdslen(val->ptr), - (long long) sdsavail(val->ptr), + (long long) sdslen(ptrFromObj(val)), + (long long) sdsavail(ptrFromObj(val)), (long long) getStringObjectSdsUsedMemory(val)); } - } else if (!strcasecmp(c->argv[1]->ptr,"ziplist") && c->argc == 3) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"ziplist") && c->argc == 3) { robj *o; if ((o = objectCommandLookupOrReply(c,c->argv[2],shared.nokeyerr)) @@ -469,10 +469,10 @@ NULL if (o->encoding != OBJ_ENCODING_ZIPLIST) { addReplyError(c,"Not an sds encoded string."); } else { - ziplistRepr(o->ptr); + ziplistRepr(ptrFromObj(o)); addReplyStatus(c,"Ziplist structure printed on stdout"); } - } else if (!strcasecmp(c->argv[1]->ptr,"populate") && + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"populate") && c->argc >= 3 && c->argc <= 5) { long keys, j; robj *key, *val; @@ -484,7 +484,7 @@ NULL for (j = 0; j < keys; j++) { long valsize = 0; snprintf(buf,sizeof(buf),"%s:%lu", - (c->argc == 3) ? "key" : (char*)c->argv[3]->ptr, j); + (c->argc == 3) ? "key" : (char*)ptrFromObj(c->argv[3]), j); key = createStringObject(buf,strlen(buf)); if (c->argc == 5) if (getLongFromObjectOrReply(c, c->argv[4], &valsize, NULL) != C_OK) @@ -499,14 +499,14 @@ NULL else { int buflen = strlen(buf); val = createStringObject(NULL,valsize); - memcpy(val->ptr, buf, valsize<=buflen? valsize: buflen); + memcpy(ptrFromObj(val), buf, valsize<=buflen? valsize: buflen); } dbAdd(c->db,key,val); signalModifiedKey(c->db,key); decrRefCount(key); } addReply(c,shared.ok); - } else if (!strcasecmp(c->argv[1]->ptr,"digest") && c->argc == 2) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"digest") && c->argc == 2) { /* DEBUG DIGEST (form without keys specified) */ unsigned char digest[20]; sds d = sdsempty(); @@ -515,7 +515,7 @@ NULL for (int i = 0; i < 20; i++) d = sdscatprintf(d, "%02x",digest[i]); addReplyStatus(c,d); sdsfree(d); - } else if (!strcasecmp(c->argv[1]->ptr,"digest-value") && c->argc >= 2) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"digest-value") && c->argc >= 2) { /* DEBUG DIGEST-VALUE key key key ... key. */ addReplyArrayLen(c,c->argc-2); for (int j = 2; j < c->argc; j++) { @@ -529,10 +529,10 @@ NULL addReplyStatus(c,d); sdsfree(d); } - } else if (!strcasecmp(c->argv[1]->ptr,"protocol") && c->argc == 3) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"protocol") && c->argc == 3) { /* DEBUG PROTOCOL [string|integer|double|bignum|null|array|set|map| * attrib|push|verbatim|true|false|state|err|bloberr] */ - char *name = c->argv[2]->ptr; + char *name = ptrFromObj(c->argv[2]); if (!strcasecmp(name,"string")) { addReplyBulkCString(c,"Hello World"); } else if (!strcasecmp(name,"integer")) { @@ -581,8 +581,8 @@ NULL } else { addReplyError(c,"Wrong protocol type name. Please use one of the following: string|integer|double|bignum|null|array|set|map|attrib|push|verbatim|true|false|state|err|bloberr"); } - } else if (!strcasecmp(c->argv[1]->ptr,"sleep") && c->argc == 3) { - double dtime = strtod(c->argv[2]->ptr,NULL); + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"sleep") && c->argc == 3) { + double dtime = strtod(ptrFromObj(c->argv[2]),NULL); long long utime = dtime*1000000; struct timespec tv; @@ -590,24 +590,24 @@ NULL tv.tv_nsec = (utime % 1000000) * 1000; nanosleep(&tv, NULL); addReply(c,shared.ok); - } else if (!strcasecmp(c->argv[1]->ptr,"set-active-expire") && + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"set-active-expire") && c->argc == 3) { - server.active_expire_enabled = atoi(c->argv[2]->ptr); + server.active_expire_enabled = atoi(ptrFromObj(c->argv[2])); addReply(c,shared.ok); - } else if (!strcasecmp(c->argv[1]->ptr,"lua-always-replicate-commands") && + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"lua-always-replicate-commands") && c->argc == 3) { - server.lua_always_replicate_commands = atoi(c->argv[2]->ptr); + server.lua_always_replicate_commands = atoi(ptrFromObj(c->argv[2])); addReply(c,shared.ok); - } else if (!strcasecmp(c->argv[1]->ptr,"error") && c->argc == 3) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"error") && c->argc == 3) { sds errstr = sdsnewlen("-",1); - errstr = sdscatsds(errstr,c->argv[2]->ptr); + errstr = sdscatsds(errstr,ptrFromObj(c->argv[2])); errstr = sdsmapchars(errstr,"\n\r"," ",2); /* no newlines in errors. */ errstr = sdscatlen(errstr,"\r\n",2); addReplySds(c,errstr); - } else if (!strcasecmp(c->argv[1]->ptr,"structsize") && c->argc == 2) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"structsize") && c->argc == 2) { sds sizes = sdsempty(); sizes = sdscatprintf(sizes,"bits:%d ",(sizeof(void*) == 8)?64:32); sizes = sdscatprintf(sizes,"robj:%d ",(int)sizeof(robj)); @@ -618,7 +618,7 @@ NULL sizes = sdscatprintf(sizes,"sdshdr32:%d ",(int)sizeof(struct sdshdr32)); sizes = sdscatprintf(sizes,"sdshdr64:%d ",(int)sizeof(struct sdshdr64)); addReplyBulkSds(c,sizes); - } else if (!strcasecmp(c->argv[1]->ptr,"htstats") && c->argc == 3) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"htstats") && c->argc == 3) { long dbid; sds stats = sdsempty(); char buf[4096]; @@ -639,7 +639,7 @@ NULL stats = sdscat(stats,buf); addReplyBulkSds(c,stats); - } else if (!strcasecmp(c->argv[1]->ptr,"htstats-key") && c->argc == 3) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"htstats-key") && c->argc == 3) { robj *o; dict *ht = NULL; @@ -650,12 +650,12 @@ NULL switch (o->encoding) { case OBJ_ENCODING_SKIPLIST: { - zset *zs = o->ptr; + zset *zs = ptrFromObj(o); ht = zs->pdict; } break; case OBJ_ENCODING_HT: - ht = o->ptr; + ht = ptrFromObj(o); break; } @@ -667,12 +667,12 @@ NULL dictGetStats(buf,sizeof(buf),ht); addReplyBulkCString(c,buf); } - } else if (!strcasecmp(c->argv[1]->ptr,"change-repl-id") && c->argc == 2) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"change-repl-id") && c->argc == 2) { serverLog(LL_WARNING,"Changing replication IDs after receiving DEBUG change-repl-id"); changeReplicationId(); clearReplicationId2(); addReply(c,shared.ok); - } else if (!strcasecmp(c->argv[1]->ptr,"stringmatch-test") && c->argc == 2) + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"stringmatch-test") && c->argc == 2) { stringmatchlen_fuzz_test(); addReplyStatus(c,"Apparently Redis did not crash: test passed"); @@ -710,7 +710,7 @@ void _serverAssertPrintClientInfo(const client *c) { char *arg; if (c->argv[j]->type == OBJ_STRING && sdsEncodedObject(c->argv[j])) { - arg = (char*) c->argv[j]->ptr; + arg = (char*) ptrFromObj(c->argv[j]); } else { snprintf(buf,sizeof(buf),"Object type: %u, encoding: %u", c->argv[j]->type, c->argv[j]->encoding); @@ -726,9 +726,9 @@ void serverLogObjectDebugInfo(const robj *o) { serverLog(LL_WARNING,"Object encoding: %d", o->encoding); serverLog(LL_WARNING,"Object refcount: %d", o->refcount); if (o->type == OBJ_STRING && sdsEncodedObject(o)) { - serverLog(LL_WARNING,"Object raw string len: %zu", sdslen(o->ptr)); - if (sdslen(o->ptr) < 4096) { - sds repr = sdscatrepr(sdsempty(),o->ptr,sdslen(o->ptr)); + serverLog(LL_WARNING,"Object raw string len: %zu", sdslen(ptrFromObj(o))); + if (sdslen(ptrFromObj(o)) < 4096) { + sds repr = sdscatrepr(sdsempty(),ptrFromObj(o),sdslen(ptrFromObj(o))); serverLog(LL_WARNING,"Object raw string content: %s", repr); sdsfree(repr); } @@ -741,7 +741,7 @@ void serverLogObjectDebugInfo(const robj *o) { } else if (o->type == OBJ_ZSET) { serverLog(LL_WARNING,"Sorted set size: %d", (int) zsetLength(o)); if (o->encoding == OBJ_ENCODING_SKIPLIST) - serverLog(LL_WARNING,"Skiplist level: %d", (int) ((const zset*)o->ptr)->zsl->level); + serverLog(LL_WARNING,"Skiplist level: %d", (int) ((const zset*)ptrFromObj(o))->zsl->level); } } @@ -1182,7 +1182,7 @@ void logCurrentClient(void) { decoded = getDecodedObject(cc->argv[j]); serverLog(LL_WARNING|LL_RAW,"argv[%d]: '%s'\n", j, - (char*)decoded->ptr); + (char*)ptrFromObj(decoded)); decrRefCount(decoded); } /* Check if the first argument, usually a key, is found inside the @@ -1192,10 +1192,10 @@ void logCurrentClient(void) { dictEntry *de; key = getDecodedObject(cc->argv[1]); - de = dictFind(cc->db->pdict, key->ptr); + de = dictFind(cc->db->pdict, ptrFromObj(key)); if (de) { val = dictGetVal(de); - serverLog(LL_WARNING,"key '%s' found in DB containing the following object:", (char*)key->ptr); + serverLog(LL_WARNING,"key '%s' found in DB containing the following object:", (char*)ptrFromObj(key)); serverLogObjectDebugInfo(val); } decrRefCount(key); diff --git a/src/expire.c b/src/expire.c index 0b92ee3fe..20acd7f45 100644 --- a/src/expire.c +++ b/src/expire.c @@ -355,13 +355,13 @@ void rememberSlaveKeyWithExpire(redisDb *db, robj *key) { } if (db->id > 63) return; - dictEntry *de = dictAddOrFind(slaveKeysWithExpire,key->ptr); + dictEntry *de = dictAddOrFind(slaveKeysWithExpire,ptrFromObj(key)); /* If the entry was just created, set it to a copy of the SDS string * representing the key: we don't want to need to take those keys * in sync with the main DB. The keys will be removed by expireSlaveKeys() * as it scans to find keys to remove. */ - if (de->key == key->ptr) { - de->key = sdsdup(key->ptr); + if (de->key == ptrFromObj(key)) { + de->key = sdsdup(ptrFromObj(key)); dictSetUnsignedIntegerVal(de,0); } diff --git a/src/geo.c b/src/geo.c index dace5542a..f04e83f5e 100644 --- a/src/geo.c +++ b/src/geo.c @@ -113,7 +113,7 @@ int extractLongLatOrReply(client *c, robj **argv, double *xy) { int longLatFromMember(robj *zobj, robj *member, double *xy) { double score = 0; - if (zsetScore(zobj, member->ptr, &score) == C_ERR) return C_ERR; + if (zsetScore(zobj, ptrFromObj(member), &score) == C_ERR) return C_ERR; if (!decodeGeohash(score, xy)) return C_ERR; return C_OK; } @@ -125,7 +125,7 @@ int longLatFromMember(robj *zobj, robj *member, double *xy) { * If the unit is not valid, an error is reported to the client, and a value * less than zero is returned. */ double extractUnitOrReply(client *c, robj *unit) { - char *u = unit->ptr; + char *u = ptrFromObj(unit); if (!strcmp(u, "m")) { return 1; @@ -230,7 +230,7 @@ int geoGetPointsInRange(robj *zobj, double min, double max, double lon, double l sds member; if (zobj->encoding == OBJ_ENCODING_ZIPLIST) { - unsigned char *zl = zobj->ptr; + unsigned char *zl = zobj->m_ptr; unsigned char *eptr, *sptr; unsigned char *vstr = NULL; unsigned int vlen = 0; @@ -259,7 +259,7 @@ int geoGetPointsInRange(robj *zobj, double min, double max, double lon, double l zzlNext(zl, &eptr, &sptr); } } else if (zobj->encoding == OBJ_ENCODING_SKIPLIST) { - zset *zs = zobj->ptr; + zset *zs = zobj->m_ptr; zskiplist *zsl = zs->zsl; zskiplistNode *ln; @@ -504,7 +504,7 @@ void georadiusGeneric(client *c, int flags) { if (c->argc > base_args) { int remaining = c->argc - base_args; for (int i = 0; i < remaining; i++) { - char *arg = c->argv[base_args + i]->ptr; + char *arg = ptrFromObj(c->argv[base_args + i]); if (!strcasecmp(arg, "withdist")) { withdist = 1; } else if (!strcasecmp(arg, "withhash")) { @@ -639,7 +639,7 @@ void georadiusGeneric(client *c, int flags) { if (returned_items) { zobj = createZsetObject(); - zs = zobj->ptr; + zs = zobj->m_ptr; } for (i = 0; i < returned_items; i++) { @@ -709,7 +709,7 @@ void geohashCommand(client *c) { addReplyArrayLen(c,c->argc-2); for (j = 2; j < c->argc; j++) { double score; - if (!zobj || zsetScore(zobj, c->argv[j]->ptr, &score) == C_ERR) { + if (!zobj || zsetScore(zobj, ptrFromObj(c->argv[j]), &score) == C_ERR) { addReplyNull(c); } else { /* The internal format we use for geocoding is a bit different @@ -762,7 +762,7 @@ void geoposCommand(client *c) { addReplyArrayLen(c,c->argc-2); for (j = 2; j < c->argc; j++) { double score; - if (!zobj || zsetScore(zobj, c->argv[j]->ptr, &score) == C_ERR) { + if (!zobj || zsetScore(zobj, ptrFromObj(c->argv[j]), &score) == C_ERR) { addReplyNullArray(c); } else { /* Decode... */ @@ -802,8 +802,8 @@ void geodistCommand(client *c) { /* Get the scores. We need both otherwise NULL is returned. */ double score1, score2, xyxy[4]; - if (zsetScore(zobj, c->argv[2]->ptr, &score1) == C_ERR || - zsetScore(zobj, c->argv[3]->ptr, &score2) == C_ERR) + if (zsetScore(zobj, ptrFromObj(c->argv[2]), &score1) == C_ERR || + zsetScore(zobj, ptrFromObj(c->argv[3]), &score2) == C_ERR) { addReplyNull(c); return; diff --git a/src/hyperloglog.c b/src/hyperloglog.c index fc21ea006..16ff7c679 100644 --- a/src/hyperloglog.c +++ b/src/hyperloglog.c @@ -582,7 +582,7 @@ void hllDenseRegHisto(uint8_t *registers, int* reghisto) { * The function returns C_OK if the sparse representation was valid, * otherwise C_ERR is returned if the representation was corrupted. */ int hllSparseToDense(robj *o) { - sds sparse = o->ptr, dense; + sds sparse = ptrFromObj(o), dense; struct hllhdr *hdr, *oldhdr = (struct hllhdr*)sparse; int idx = 0, runlen, regval; uint8_t *p = (uint8_t*)sparse, *end = p+sdslen(sparse); @@ -630,8 +630,8 @@ int hllSparseToDense(robj *o) { } /* Free the old representation and set the new one. */ - sdsfree(o->ptr); - o->ptr = dense; + sdsfree(ptrFromObj(o)); + o->m_ptr = dense; return C_OK; } @@ -665,12 +665,12 @@ int hllSparseSet(robj *o, long index, uint8_t count) { * into XZERO-VAL-XZERO). Make sure there is enough space right now * so that the pointers we take during the execution of the function * will be valid all the time. */ - o->ptr = sdsMakeRoomFor(o->ptr,3); + o->m_ptr = sdsMakeRoomFor(ptrFromObj(o),3); /* Step 1: we need to locate the opcode we need to modify to check * if a value update is actually needed. */ - sparse = p = ((uint8_t*)o->ptr) + HLL_HDR_SIZE; - end = p + sdslen(o->ptr) - HLL_HDR_SIZE; + sparse = p = ((uint8_t*)ptrFromObj(o)) + HLL_HDR_SIZE; + end = p + sdslen(ptrFromObj(o)) - HLL_HDR_SIZE; first = 0; prev = NULL; /* Points to previous opcode at the end of the loop. */ @@ -828,9 +828,9 @@ int hllSparseSet(robj *o, long index, uint8_t count) { int deltalen = seqlen-oldlen; if (deltalen > 0 && - sdslen(o->ptr)+deltalen > server.hll_sparse_max_bytes) goto promote; + sdslen(ptrFromObj(o))+deltalen > server.hll_sparse_max_bytes) goto promote; if (deltalen && next) memmove(next+deltalen,next,end-next); - sdsIncrLen(o->ptr,deltalen); + sdsIncrLen(ptrFromObj(o),deltalen); memcpy(p,seq,seqlen); end += deltalen; @@ -860,7 +860,7 @@ updated: if (len <= HLL_SPARSE_VAL_MAX_LEN) { HLL_SPARSE_VAL_SET(p+1,v1,len); memmove(p,p+1,end-p); - sdsIncrLen(o->ptr,-1); + sdsIncrLen(ptrFromObj(o),-1); end--; /* After a merge we reiterate without incrementing 'p' * in order to try to merge the just merged value with @@ -873,13 +873,13 @@ updated: } /* Invalidate the cached cardinality. */ - hdr = o->ptr; + hdr = ptrFromObj(o); HLL_INVALIDATE_CACHE(hdr); return 1; promote: /* Promote to dense representation. */ if (hllSparseToDense(o) == C_ERR) return -1; /* Corrupted HLL. */ - hdr = o->ptr; + hdr = ptrFromObj(o); /* We need to call hllDenseAdd() to perform the operation after the * conversion. However the result must be 1, since if we need to @@ -1043,7 +1043,7 @@ uint64_t hllCount(struct hllhdr *hdr, int *invalid) { /* Call hllDenseAdd() or hllSparseAdd() according to the HLL encoding. */ int hllAdd(robj *o, unsigned char *ele, size_t elesize) { - struct hllhdr *hdr = o->ptr; + struct hllhdr *hdr = ptrFromObj(o); switch(hdr->encoding) { case HLL_DENSE: return hllDenseAdd(hdr->registers,ele,elesize); case HLL_SPARSE: return hllSparseAdd(o,ele,elesize); @@ -1060,7 +1060,7 @@ int hllAdd(robj *o, unsigned char *ele, size_t elesize) { * If the HyperLogLog is sparse and is found to be invalid, C_ERR * is returned, otherwise the function always succeeds. */ int hllMerge(uint8_t *max, robj *hll) { - struct hllhdr *hdr = hll->ptr; + struct hllhdr *hdr = ptrFromObj(hll); int i; if (hdr->encoding == HLL_DENSE) { @@ -1071,7 +1071,7 @@ int hllMerge(uint8_t *max, robj *hll) { if (val > max[i]) max[i] = val; } } else { - uint8_t *p = hll->ptr, *end = p + sdslen(hll->ptr); + uint8_t *p = ptrFromObj(hll), *end = p + sdslen(ptrFromObj(hll)); long runlen, regval; p += HLL_HDR_SIZE; @@ -1130,7 +1130,7 @@ robj *createHLLObject(void) { /* Create the actual object. */ o = createObject(OBJ_STRING,s); - hdr = o->ptr; + hdr = ptrFromObj(o); memcpy(hdr->magic,"HYLL",4); hdr->encoding = HLL_SPARSE; return o; @@ -1148,7 +1148,7 @@ int isHLLObjectOrReply(client *c, robj *o) { if (!sdsEncodedObject(o)) goto invalid; if (stringObjectLen(o) < sizeof(*hdr)) goto invalid; - hdr = o->ptr; + hdr = ptrFromObj(o); /* Magic should be "HYLL". */ if (hdr->magic[0] != 'H' || hdr->magic[1] != 'Y' || @@ -1189,8 +1189,8 @@ void pfaddCommand(client *c) { } /* Perform the low level ADD operation for every element. */ for (j = 2; j < c->argc; j++) { - int retval = hllAdd(o, (unsigned char*)c->argv[j]->ptr, - sdslen(c->argv[j]->ptr)); + int retval = hllAdd(o, (unsigned char*)ptrFromObj(c->argv[j]), + sdslen(ptrFromObj(c->argv[j]))); switch(retval) { case 1: updated++; @@ -1200,7 +1200,7 @@ void pfaddCommand(client *c) { return; } } - hdr = o->ptr; + hdr = ptrFromObj(o); if (updated) { signalModifiedKey(c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_STRING,"pfadd",c->argv[1],c->db->id); @@ -1262,7 +1262,7 @@ void pfcountCommand(client *c) { o = dbUnshareStringValue(c->db,c->argv[1],o); /* Check if the cached cardinality is valid. */ - hdr = o->ptr; + hdr = ptrFromObj(o); if (HLL_VALID_CACHE(hdr)) { /* Just return the cached value. */ card = (uint64_t)hdr->card[0]; @@ -1319,7 +1319,7 @@ void pfmergeCommand(client *c) { /* If at least one involved HLL is dense, use the dense representation * as target ASAP to save time and avoid the conversion step. */ - hdr = o->ptr; + hdr = ptrFromObj(o); if (hdr->encoding == HLL_DENSE) use_dense = 1; /* Merge with this HLL with our 'max' HHL by setting max[i] @@ -1356,13 +1356,13 @@ void pfmergeCommand(client *c) { * invalidate the cached value. */ for (j = 0; j < HLL_REGISTERS; j++) { if (max[j] == 0) continue; - hdr = o->ptr; + hdr = ptrFromObj(o); switch(hdr->encoding) { case HLL_DENSE: hllDenseSet(hdr->registers,j,max[j]); break; case HLL_SPARSE: hllSparseSet(o,j,max[j]); break; } } - hdr = o->ptr; /* o->ptr may be different now, as a side effect of + hdr = ptrFromObj(o); /* ptrFromObj(o) may be different now, as a side effect of last hllSparseSet() call. */ HLL_INVALIDATE_CACHE(hdr); @@ -1438,7 +1438,7 @@ void pfselftestCommand(client *c) { /* Make sure that for small cardinalities we use sparse * encoding. */ if (j == checkpoint && j < server.hll_sparse_max_bytes/2) { - hdr2 = o->ptr; + hdr2 = ptrFromObj(o); if (hdr2->encoding != HLL_SPARSE) { addReplyError(c, "TESTFAILED sparse encoding not used"); goto cleanup; @@ -1446,7 +1446,7 @@ void pfselftestCommand(client *c) { } /* Check that dense and sparse representations agree. */ - if (j == checkpoint && hllCount(hdr,NULL) != hllCount(o->ptr,NULL)) { + if (j == checkpoint && hllCount(hdr,NULL) != hllCount(ptrFromObj(o),NULL)) { addReplyError(c, "TESTFAILED dense/sparse disagree"); goto cleanup; } @@ -1485,7 +1485,7 @@ cleanup: /* PFDEBUG ... args ... * Different debugging related operations about the HLL implementation. */ void pfdebugCommand(client *c) { - char *cmd = c->argv[1]->ptr; + char *cmd = ptrFromObj(c->argv[1]); struct hllhdr *hdr; robj *o; int j; @@ -1497,7 +1497,7 @@ void pfdebugCommand(client *c) { } if (isHLLObjectOrReply(c,o) != C_OK) return; o = dbUnshareStringValue(c->db,c->argv[2],o); - hdr = o->ptr; + hdr = ptrFromObj(o); /* PFDEBUG GETREG */ if (!strcasecmp(cmd,"getreg")) { @@ -1511,7 +1511,7 @@ void pfdebugCommand(client *c) { server.dirty++; /* Force propagation on encoding change. */ } - hdr = o->ptr; + hdr = ptrFromObj(o); addReplyArrayLen(c,HLL_REGISTERS); for (j = 0; j < HLL_REGISTERS; j++) { uint8_t val; @@ -1524,7 +1524,7 @@ void pfdebugCommand(client *c) { else if (!strcasecmp(cmd,"decode")) { if (c->argc != 3) goto arityerr; - uint8_t *p = o->ptr, *end = p+sdslen(o->ptr); + uint8_t *p = ptrFromObj(o), *end = p+sdslen(ptrFromObj(o)); sds decoded = sdsempty(); if (hdr->encoding != HLL_SPARSE) { diff --git a/src/latency.c b/src/latency.c index ddb235c13..278ef4d29 100644 --- a/src/latency.c +++ b/src/latency.c @@ -579,21 +579,21 @@ NULL }; struct latencyTimeSeries *ts; - if (!strcasecmp(c->argv[1]->ptr,"history") && c->argc == 3) { + if (!strcasecmp(ptrFromObj(c->argv[1]),"history") && c->argc == 3) { /* LATENCY HISTORY */ - ts = dictFetchValue(server.latency_events,c->argv[2]->ptr); + ts = dictFetchValue(server.latency_events,ptrFromObj(c->argv[2])); if (ts == NULL) { addReplyArrayLen(c,0); } else { latencyCommandReplyWithSamples(c,ts); } - } else if (!strcasecmp(c->argv[1]->ptr,"graph") && c->argc == 3) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"graph") && c->argc == 3) { /* LATENCY GRAPH */ sds graph; dictEntry *de; char *event; - de = dictFind(server.latency_events,c->argv[2]->ptr); + de = dictFind(server.latency_events,ptrFromObj(c->argv[2])); if (de == NULL) goto nodataerr; ts = dictGetVal(de); event = dictGetKey(de); @@ -601,16 +601,16 @@ NULL graph = latencyCommandGenSparkeline(event,ts); addReplyBulkCString(c,graph); sdsfree(graph); - } else if (!strcasecmp(c->argv[1]->ptr,"latest") && c->argc == 2) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"latest") && c->argc == 2) { /* LATENCY LATEST */ latencyCommandReplyWithLatestEvents(c); - } else if (!strcasecmp(c->argv[1]->ptr,"doctor") && c->argc == 2) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"doctor") && c->argc == 2) { /* LATENCY DOCTOR */ sds report = createLatencyReport(); addReplyBulkCBuffer(c,report,sdslen(report)); sdsfree(report); - } else if (!strcasecmp(c->argv[1]->ptr,"reset") && c->argc >= 2) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"reset") && c->argc >= 2) { /* LATENCY RESET */ if (c->argc == 2) { addReplyLongLong(c,latencyResetEvent(NULL)); @@ -618,10 +618,10 @@ NULL int j, resets = 0; for (j = 2; j < c->argc; j++) - resets += latencyResetEvent(c->argv[j]->ptr); + resets += latencyResetEvent(ptrFromObj(c->argv[j])); addReplyLongLong(c,resets); } - } else if (!strcasecmp(c->argv[1]->ptr,"help") && c->argc >= 2) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"help") && c->argc >= 2) { addReplyHelp(c, help); } else { addReplySubcommandSyntaxError(c); @@ -632,6 +632,6 @@ nodataerr: /* Common error when the user asks for an event we have no latency * information about. */ addReplyErrorFormat(c, - "No samples available for event '%s'", (char*) c->argv[2]->ptr); + "No samples available for event '%s'", (char*) ptrFromObj(c->argv[2])); } diff --git a/src/lazyfree.c b/src/lazyfree.c index 74c71beb8..6ea21768e 100644 --- a/src/lazyfree.c +++ b/src/lazyfree.c @@ -30,16 +30,16 @@ size_t lazyfreeGetPendingObjectsCount(void) { * representing the list. */ size_t lazyfreeGetFreeEffort(robj *obj) { if (obj->type == OBJ_LIST) { - quicklist *ql = obj->ptr; + quicklist *ql = ptrFromObj(obj); return ql->len; } else if (obj->type == OBJ_SET && obj->encoding == OBJ_ENCODING_HT) { - dict *ht = obj->ptr; + dict *ht = ptrFromObj(obj); return dictSize(ht); } else if (obj->type == OBJ_ZSET && obj->encoding == OBJ_ENCODING_SKIPLIST){ - zset *zs = obj->ptr; + zset *zs = ptrFromObj(obj); return zs->zsl->length; } else if (obj->type == OBJ_HASH && obj->encoding == OBJ_ENCODING_HT) { - dict *ht = obj->ptr; + dict *ht = ptrFromObj(obj); return dictSize(ht); } else { return 1; /* Everything else is a single allocation. */ @@ -54,12 +54,12 @@ size_t lazyfreeGetFreeEffort(robj *obj) { int dbAsyncDelete(redisDb *db, robj *key) { /* Deleting an entry from the expires dict will not free the sds of * the key, because it is shared with the main dictionary. */ - if (dictSize(db->expires) > 0) dictDelete(db->expires,key->ptr); + if (dictSize(db->expires) > 0) dictDelete(db->expires,ptrFromObj(key)); /* If the value is composed of a few allocations, to free in a lazy way * is actually just slower... So under a certain limit we just free * the object synchronously. */ - dictEntry *de = dictUnlink(db->pdict,key->ptr); + dictEntry *de = dictUnlink(db->pdict,ptrFromObj(key)); if (de) { robj *val = dictGetVal(de); size_t free_effort = lazyfreeGetFreeEffort(val); diff --git a/src/module.c b/src/module.c index e5af4ba0e..45fa4e293 100644 --- a/src/module.c +++ b/src/module.c @@ -392,7 +392,7 @@ int moduleCreateEmptyKey(RedisModuleKey *key, int type) { switch(type) { case REDISMODULE_KEYTYPE_LIST: obj = createQuicklistObject(); - quicklistSetOptions(obj->ptr, server.list_max_ziplist_size, + quicklistSetOptions(obj->m_ptr, server.list_max_ziplist_size, server.list_compress_depth); break; case REDISMODULE_KEYTYPE_ZSET: @@ -938,8 +938,8 @@ const char *RM_StringPtrLen(const RedisModuleString *str, size_t *len) { if (len) *len = strlen(errmsg); return errmsg; } - if (len) *len = sdslen(str->ptr); - return str->ptr; + if (len) *len = sdslen(ptrFromObj(str)); + return ptrFromObj(str); } /* -------------------------------------------------------------------------- @@ -951,7 +951,7 @@ const char *RM_StringPtrLen(const RedisModuleString *str, size_t *len) { * as a valid, strict long long (no spaces before/after), REDISMODULE_ERR * is returned. */ int RM_StringToLongLong(const RedisModuleString *str, long long *ll) { - return string2ll(str->ptr,sdslen(str->ptr),ll) ? REDISMODULE_OK : + return string2ll(ptrFromObj(str),sdslen(ptrFromObj(str)),ll) ? REDISMODULE_OK : REDISMODULE_ERR; } @@ -983,11 +983,11 @@ RedisModuleString *moduleAssertUnsharedString(RedisModuleString *str) { if (str->encoding == OBJ_ENCODING_EMBSTR) { /* Note: here we "leak" the additional allocation that was * used in order to store the embedded string in the object. */ - str->ptr = sdsnewlen(str->ptr,sdslen(str->ptr)); + str->m_ptr = sdsnewlen(ptrFromObj(str),sdslen(ptrFromObj(str))); str->encoding = OBJ_ENCODING_RAW; } else if (str->encoding == OBJ_ENCODING_INT) { /* Convert the string from integer to raw encoding. */ - str->ptr = sdsfromlonglong((long)str->ptr); + str->m_ptr = sdsfromlonglong((long)str->m_ptr); str->encoding = OBJ_ENCODING_RAW; } return str; @@ -1000,7 +1000,7 @@ int RM_StringAppendBuffer(RedisModuleCtx *ctx, RedisModuleString *str, const cha UNUSED(ctx); str = moduleAssertUnsharedString(str); if (str == NULL) return REDISMODULE_ERR; - str->ptr = sdscatlen(str->ptr,buf,len); + str->m_ptr = sdscatlen(str->m_ptr,buf,len); return REDISMODULE_OK; } @@ -1024,7 +1024,7 @@ int RM_StringAppendBuffer(RedisModuleCtx *ctx, RedisModuleString *str, const cha int RM_WrongArity(RedisModuleCtx *ctx) { addReplyErrorFormat(ctx->client, "wrong number of arguments for '%s' command", - (char*)ctx->client->argv[0]->ptr); + (char*)ptrFromObj(ctx->client->argv[0])); return REDISMODULE_OK; } @@ -1650,8 +1650,8 @@ char *RM_StringDMA(RedisModuleKey *key, size_t *len, int mode) { if ((mode & REDISMODULE_WRITE) || key->value->encoding != OBJ_ENCODING_RAW) key->value = dbUnshareStringValue(key->db, key->key, key->value); - *len = sdslen(key->value->ptr); - return key->value->ptr; + *len = sdslen(ptrFromObj(key->value)); + return ptrFromObj(key->value); } /* If the string is open for writing and is of string type, resize it, padding @@ -1684,14 +1684,14 @@ int RM_StringTruncate(RedisModuleKey *key, size_t newlen) { } else { /* Unshare and resize. */ key->value = dbUnshareStringValue(key->db, key->key, key->value); - size_t curlen = sdslen(key->value->ptr); + size_t curlen = sdslen(ptrFromObj(key->value)); if (newlen > curlen) { - key->value->ptr = sdsgrowzero(key->value->ptr,newlen); + key->value->m_ptr = sdsgrowzero(ptrFromObj(key->value),newlen); } else if (newlen < curlen) { - sdsrange(key->value->ptr,0,newlen-1); + sdsrange(ptrFromObj(key->value),0,newlen-1); /* If the string is too wasteful, reallocate it. */ - if (sdslen(key->value->ptr) < sdsavail(key->value->ptr)) - key->value->ptr = sdsRemoveFreeSpace(key->value->ptr); + if (sdslen(ptrFromObj(key->value)) < sdsavail(ptrFromObj(key->value))) + key->value->m_ptr = sdsRemoveFreeSpace(ptrFromObj(key->value)); } } return REDISMODULE_OK; @@ -1790,7 +1790,7 @@ int RM_ZsetAdd(RedisModuleKey *key, double score, RedisModuleString *ele, int *f if (key->value && key->value->type != OBJ_ZSET) return REDISMODULE_ERR; if (key->value == NULL) moduleCreateEmptyKey(key,REDISMODULE_KEYTYPE_ZSET); if (flagsptr) flags = RM_ZsetAddFlagsToCoreFlags(*flagsptr); - if (zsetAdd(key->value,score,ele->ptr,&flags,NULL) == 0) { + if (zsetAdd(key->value,score,ptrFromObj(ele),&flags,NULL) == 0) { if (flagsptr) *flagsptr = 0; return REDISMODULE_ERR; } @@ -1818,7 +1818,7 @@ int RM_ZsetIncrby(RedisModuleKey *key, double score, RedisModuleString *ele, int if (key->value == NULL) moduleCreateEmptyKey(key,REDISMODULE_KEYTYPE_ZSET); if (flagsptr) flags = RM_ZsetAddFlagsToCoreFlags(*flagsptr); flags |= ZADD_INCR; - if (zsetAdd(key->value,score,ele->ptr,&flags,newscore) == 0) { + if (zsetAdd(key->value,score,ptrFromObj(ele),&flags,newscore) == 0) { if (flagsptr) *flagsptr = 0; return REDISMODULE_ERR; } @@ -1852,7 +1852,7 @@ int RM_ZsetIncrby(RedisModuleKey *key, double score, RedisModuleString *ele, int int RM_ZsetRem(RedisModuleKey *key, RedisModuleString *ele, int *deleted) { if (!(key->mode & REDISMODULE_WRITE)) return REDISMODULE_ERR; if (key->value && key->value->type != OBJ_ZSET) return REDISMODULE_ERR; - if (key->value != NULL && zsetDel(key->value,ele->ptr)) { + if (key->value != NULL && zsetDel(key->value,ptrFromObj(ele))) { if (deleted) *deleted = 1; } else { if (deleted) *deleted = 0; @@ -1871,7 +1871,7 @@ int RM_ZsetRem(RedisModuleKey *key, RedisModuleString *ele, int *deleted) { int RM_ZsetScore(RedisModuleKey *key, RedisModuleString *ele, double *score) { if (key->value == NULL) return REDISMODULE_ERR; if (key->value->type != OBJ_ZSET) return REDISMODULE_ERR; - if (zsetScore(key->value,ele->ptr,score) == C_ERR) return REDISMODULE_ERR; + if (zsetScore(key->value,ptrFromObj(ele),score) == C_ERR) return REDISMODULE_ERR; return REDISMODULE_OK; } @@ -1923,10 +1923,10 @@ int zsetInitScoreRange(RedisModuleKey *key, double min, double max, int minex, i zrs->maxex = maxex; if (key->value->encoding == OBJ_ENCODING_ZIPLIST) { - key->zcurrent = first ? zzlFirstInRange(key->value->ptr,zrs) : - zzlLastInRange(key->value->ptr,zrs); + key->zcurrent = first ? zzlFirstInRange(ptrFromObj(key->value),zrs) : + zzlLastInRange(ptrFromObj(key->value),zrs); } else if (key->value->encoding == OBJ_ENCODING_SKIPLIST) { - zset *zs = key->value->ptr; + zset *zs = ptrFromObj(key->value); zskiplist *zsl = zs->zsl; key->zcurrent = first ? zslFirstInRange(zsl,zrs) : zslLastInRange(zsl,zrs); @@ -1987,10 +1987,10 @@ int zsetInitLexRange(RedisModuleKey *key, RedisModuleString *min, RedisModuleStr key->ztype = REDISMODULE_ZSET_RANGE_LEX; if (key->value->encoding == OBJ_ENCODING_ZIPLIST) { - key->zcurrent = first ? zzlFirstInLexRange(key->value->ptr,zlrs) : - zzlLastInLexRange(key->value->ptr,zlrs); + key->zcurrent = first ? zzlFirstInLexRange(ptrFromObj(key->value),zlrs) : + zzlLastInLexRange(ptrFromObj(key->value),zlrs); } else if (key->value->encoding == OBJ_ENCODING_SKIPLIST) { - zset *zs = key->value->ptr; + zset *zs = ptrFromObj(key->value); zskiplist *zsl = zs->zsl; key->zcurrent = first ? zslFirstInLexRange(zsl,zlrs) : zslLastInLexRange(zsl,zlrs); @@ -2036,7 +2036,7 @@ RedisModuleString *RM_ZsetRangeCurrentElement(RedisModuleKey *key, double *score eptr = key->zcurrent; sds ele = ziplistGetObject(eptr); if (score) { - sptr = ziplistNext(key->value->ptr,eptr); + sptr = ziplistNext(ptrFromObj(key->value),eptr); *score = zzlGetScore(sptr); } str = createObject(OBJ_STRING,ele); @@ -2058,7 +2058,7 @@ int RM_ZsetRangeNext(RedisModuleKey *key) { if (!key->ztype || !key->zcurrent) return 0; /* No active iterator. */ if (key->value->encoding == OBJ_ENCODING_ZIPLIST) { - unsigned char *zl = key->value->ptr; + unsigned char *zl = ptrFromObj(key->value); unsigned char *eptr = key->zcurrent; unsigned char *next; next = ziplistNext(zl,eptr); /* Skip element. */ @@ -2121,7 +2121,7 @@ int RM_ZsetRangePrev(RedisModuleKey *key) { if (!key->ztype || !key->zcurrent) return 0; /* No active iterator. */ if (key->value->encoding == OBJ_ENCODING_ZIPLIST) { - unsigned char *zl = key->value->ptr; + unsigned char *zl = ptrFromObj(key->value); unsigned char *eptr = key->zcurrent; unsigned char *prev; prev = ziplistPrev(zl,eptr); /* Go back to previous score. */ @@ -2256,7 +2256,7 @@ int RM_HashSet(RedisModuleKey *key, int flags, ...) { /* Handle XX and NX */ if (flags & (REDISMODULE_HASH_XX|REDISMODULE_HASH_NX)) { - int exists = hashTypeExists(key->value, field->ptr); + int exists = hashTypeExists(key->value, ptrFromObj(field)); if (((flags & REDISMODULE_HASH_XX) && !exists) || ((flags & REDISMODULE_HASH_NX) && exists)) { @@ -2267,7 +2267,7 @@ int RM_HashSet(RedisModuleKey *key, int flags, ...) { /* Handle deletion if value is REDISMODULE_HASH_DELETE. */ if (value == REDISMODULE_HASH_DELETE) { - updated += hashTypeDelete(key->value, field->ptr); + updated += hashTypeDelete(key->value, ptrFromObj(field)); if (flags & REDISMODULE_HASH_CFIELDS) decrRefCount(field); continue; } @@ -2281,12 +2281,12 @@ int RM_HashSet(RedisModuleKey *key, int flags, ...) { robj *argv[2] = {field,value}; hashTypeTryConversion(key->value,argv,0,1); - updated += hashTypeSet(key->value, field->ptr, value->ptr, low_flags); + updated += hashTypeSet(key->value, ptrFromObj(field), ptrFromObj(value), low_flags); /* If CFIELDS is active, SDS string ownership is now of hashTypeSet(), * however we still have to release the 'field' object shell. */ if (flags & REDISMODULE_HASH_CFIELDS) { - field->ptr = NULL; /* Prevent the SDS string from being freed. */ + field->m_ptr = NULL; /* Prevent the SDS string from being freed. */ decrRefCount(field); } } @@ -2358,13 +2358,13 @@ int RM_HashGet(RedisModuleKey *key, int flags, ...) { if (flags & REDISMODULE_HASH_EXISTS) { existsptr = va_arg(ap,int*); if (key->value) - *existsptr = hashTypeExists(key->value,field->ptr); + *existsptr = hashTypeExists(key->value,ptrFromObj(field)); else *existsptr = 0; } else { valueptr = va_arg(ap,RedisModuleString**); if (key->value) { - *valueptr = hashTypeGetValueObject(key->value,field->ptr); + *valueptr = hashTypeGetValueObject(key->value,ptrFromObj(field)); if (*valueptr) { robj *decoded = getDecodedObject(*valueptr); decrRefCount(*valueptr); @@ -3035,7 +3035,7 @@ moduleType *RM_ModuleTypeGetType(RedisModuleKey *key) { if (key == NULL || key->value == NULL || RM_KeyType(key) != REDISMODULE_KEYTYPE_MODULE) return NULL; - moduleValue *mv = key->value->ptr; + moduleValue *mv = ptrFromObj(key->value); return mv->type; } @@ -3049,7 +3049,7 @@ void *RM_ModuleTypeGetValue(RedisModuleKey *key) { if (key == NULL || key->value == NULL || RM_KeyType(key) != REDISMODULE_KEYTYPE_MODULE) return NULL; - moduleValue *mv = key->value->ptr; + moduleValue *mv = ptrFromObj(key->value); return mv->value; } @@ -4407,12 +4407,12 @@ int RM_DictReplaceC(RedisModuleDict *d, void *key, size_t keylen, void *ptr) { /* Like RedisModule_DictSetC() but takes the key as a RedisModuleString. */ int RM_DictSet(RedisModuleDict *d, RedisModuleString *key, void *ptr) { - return RM_DictSetC(d,key->ptr,sdslen(key->ptr),ptr); + return RM_DictSetC(d,ptrFromObj(key),sdslen(ptrFromObj(key)),ptr); } /* Like RedisModule_DictReplaceC() but takes the key as a RedisModuleString. */ int RM_DictReplace(RedisModuleDict *d, RedisModuleString *key, void *ptr) { - return RM_DictReplaceC(d,key->ptr,sdslen(key->ptr),ptr); + return RM_DictReplaceC(d,ptrFromObj(key),sdslen(ptrFromObj(key)),ptr); } /* Return the value stored at the specified key. The function returns NULL @@ -4428,7 +4428,7 @@ void *RM_DictGetC(RedisModuleDict *d, void *key, size_t keylen, int *nokey) { /* Like RedisModule_DictGetC() but takes the key as a RedisModuleString. */ void *RM_DictGet(RedisModuleDict *d, RedisModuleString *key, int *nokey) { - return RM_DictGetC(d,key->ptr,sdslen(key->ptr),nokey); + return RM_DictGetC(d,ptrFromObj(key),sdslen(ptrFromObj(key)),nokey); } /* Remove the specified key from the dictionary, returning REDISMODULE_OK if @@ -4445,7 +4445,7 @@ int RM_DictDelC(RedisModuleDict *d, void *key, size_t keylen, void *oldval) { /* Like RedisModule_DictDelC() but gets the key as a RedisModuleString. */ int RM_DictDel(RedisModuleDict *d, RedisModuleString *key, void *oldval) { - return RM_DictDelC(d,key->ptr,sdslen(key->ptr),oldval); + return RM_DictDelC(d,ptrFromObj(key),sdslen(ptrFromObj(key)),oldval); } /* Return an interator, setup in order to start iterating from the specified @@ -4479,7 +4479,7 @@ RedisModuleDictIter *RM_DictIteratorStartC(RedisModuleDict *d, const char *op, v /* Exactly like RedisModule_DictIteratorStartC, but the key is passed as a * RedisModuleString. */ RedisModuleDictIter *RM_DictIteratorStart(RedisModuleDict *d, const char *op, RedisModuleString *key) { - return RM_DictIteratorStartC(d,op,key->ptr,sdslen(key->ptr)); + return RM_DictIteratorStartC(d,op,ptrFromObj(key),sdslen(ptrFromObj(key))); } /* Release the iterator created with RedisModule_DictIteratorStart(). This call @@ -4503,7 +4503,7 @@ int RM_DictIteratorReseekC(RedisModuleDictIter *di, const char *op, void *key, s /* Like RedisModule_DictIteratorReseekC() but takes the key as as a * RedisModuleString. */ int RM_DictIteratorReseek(RedisModuleDictIter *di, const char *op, RedisModuleString *key) { - return RM_DictIteratorReseekC(di,op,key->ptr,sdslen(key->ptr)); + return RM_DictIteratorReseekC(di,op,ptrFromObj(key),sdslen(ptrFromObj(key))); } /* Return the current item of the dictionary iterator 'di' and steps to the @@ -4594,7 +4594,7 @@ int RM_DictCompareC(RedisModuleDictIter *di, const char *op, void *key, size_t k * iterator key as a RedisModuleString. */ int RM_DictCompare(RedisModuleDictIter *di, const char *op, RedisModuleString *key) { if (raxEOF(&di->ri)) return REDISMODULE_ERR; - int res = raxCompare(&di->ri,op,key->ptr,sdslen(key->ptr)); + int res = raxCompare(&di->ri,op,ptrFromObj(key),sdslen(ptrFromObj(key))); return res ? REDISMODULE_OK : REDISMODULE_ERR; } @@ -4844,7 +4844,7 @@ void addReplyLoadedModules(client *c) { * * MODULE LOAD [args...] */ void moduleCommand(client *c) { - char *subcmd = c->argv[1]->ptr; + char *subcmd = ptrFromObj(c->argv[1]); if (c->argc == 2 && !strcasecmp(subcmd,"help")) { const char *help[] = { "LIST -- Return a list of loaded modules.", @@ -4863,13 +4863,13 @@ NULL argv = &c->argv[3]; } - if (moduleLoad(c->argv[2]->ptr,(void **)argv,argc) == C_OK) + if (moduleLoad(ptrFromObj(c->argv[2]),(void **)argv,argc) == C_OK) addReply(c,shared.ok); else addReplyError(c, "Error loading the extension. Please check the server logs."); } else if (!strcasecmp(subcmd,"unload") && c->argc == 3) { - if (moduleUnload(c->argv[2]->ptr) == C_OK) + if (moduleUnload(ptrFromObj(c->argv[2])) == C_OK) addReply(c,shared.ok); else { char *errmsg; diff --git a/src/multi.c b/src/multi.c index aec913a0a..6d722b8af 100644 --- a/src/multi.c +++ b/src/multi.c @@ -329,7 +329,7 @@ void touchWatchedKeysOnFlush(int dbid) { * key exists, mark the client as dirty, as the key will be * removed. */ if (dbid == -1 || wk->db->id == dbid) { - if (dictFind(wk->db->pdict, wk->key->ptr) != NULL) + if (dictFind(wk->db->pdict, ptrFromObj(wk->key)) != NULL) c->flags |= CLIENT_DIRTY_CAS; } } diff --git a/src/networking.c b/src/networking.c index 3b43707e2..1c917af2a 100644 --- a/src/networking.c +++ b/src/networking.c @@ -48,7 +48,7 @@ size_t sdsZmallocSize(sds s) { size_t getStringObjectSdsUsedMemory(robj *o) { serverAssertWithInfo(NULL,o,o->type == OBJ_STRING); switch(o->encoding) { - case OBJ_ENCODING_RAW: return sdsZmallocSize(o->ptr); + case OBJ_ENCODING_RAW: return sdsZmallocSize(ptrFromObj(o)); case OBJ_ENCODING_EMBSTR: return zmalloc_size(o)-sizeof(robj); default: return 0; /* Just integer encoding for now. */ } @@ -302,14 +302,14 @@ void addReply(client *c, robj *obj) { if (prepareClientToWrite(c) != C_OK) return; if (sdsEncodedObject(obj)) { - if (_addReplyToBuffer(c,obj->ptr,sdslen(obj->ptr)) != C_OK) - _addReplyProtoToList(c,obj->ptr,sdslen(obj->ptr)); + if (_addReplyToBuffer(c,ptrFromObj(obj),sdslen(ptrFromObj(obj))) != C_OK) + _addReplyProtoToList(c,ptrFromObj(obj),sdslen(ptrFromObj(obj))); } else if (obj->encoding == OBJ_ENCODING_INT) { /* For integer encoded strings we just convert it into a string * using our optimized function, and attach the resulting string * to the output buffer. */ char buf[32]; - size_t len = ll2string(buf,sizeof(buf),(long)obj->ptr); + size_t len = ll2string(buf,sizeof(buf),(long)ptrFromObj(obj)); if (_addReplyToBuffer(c,buf,len) != C_OK) _addReplyProtoToList(c,buf,len); } else { @@ -639,9 +639,9 @@ void addReplyBulkLen(client *c, robj *obj) { size_t len; if (sdsEncodedObject(obj)) { - len = sdslen(obj->ptr); + len = sdslen(ptrFromObj(obj)); } else { - long n = (long)obj->ptr; + long n = (long)ptrFromObj(obj); /* Compute how many bytes will take this integer as a radix 10 string */ len = 1; @@ -733,7 +733,7 @@ void addReplyVerbatim(client *c, const char *s, size_t len, const char *ext) { * subcommands in response to the 'help' subcommand. The help array * is terminated by NULL sentinel. */ void addReplyHelp(client *c, const char **help) { - sds cmd = sdsnew((char*) c->argv[0]->ptr); + sds cmd = sdsnew((char*) ptrFromObj(c->argv[0])); void *blenp = addReplyDeferredLen(c); int blen = 0; @@ -752,11 +752,11 @@ void addReplyHelp(client *c, const char **help) { * This function is typically invoked by from commands that support * subcommands in response to an unknown subcommand or argument error. */ void addReplySubcommandSyntaxError(client *c) { - sds cmd = sdsnew((char*) c->argv[0]->ptr); + sds cmd = sdsnew((char*) ptrFromObj(c->argv[0])); sdstoupper(cmd); addReplyErrorFormat(c, "Unknown subcommand or wrong number of arguments for '%s'. Try %s HELP.", - (char*)c->argv[1]->ptr,cmd); + (char*)ptrFromObj(c->argv[1]),cmd); sdsfree(cmd); } @@ -1785,7 +1785,7 @@ sds catClientInfoString(sds s, client *client) { (unsigned long long) client->id, getClientPeerId(client), client->fd, - client->name ? (char*)client->name->ptr : "", + client->name ? (char*)ptrFromObj(client->name) : "", (long long)(server.unixtime - client->ctime), (long long)(server.unixtime - client->lastinteraction), flags, @@ -1823,7 +1823,7 @@ void clientCommand(client *c) { listIter li; client *client; - if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) { + if (c->argc == 2 && !strcasecmp(ptrFromObj(c->argv[1]),"help")) { const char *help[] = { "id -- Return the ID of the current connection.", "getname -- Return the name of the current connection.", @@ -1841,17 +1841,17 @@ void clientCommand(client *c) { NULL }; addReplyHelp(c, help); - } else if (!strcasecmp(c->argv[1]->ptr,"id") && c->argc == 2) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"id") && c->argc == 2) { /* CLIENT ID */ addReplyLongLong(c,c->id); - } else if (!strcasecmp(c->argv[1]->ptr,"list")) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"list")) { /* CLIENT LIST */ int type = -1; - if (c->argc == 4 && !strcasecmp(c->argv[2]->ptr,"type")) { - type = getClientTypeByName(c->argv[3]->ptr); + if (c->argc == 4 && !strcasecmp(ptrFromObj(c->argv[2]),"type")) { + type = getClientTypeByName(ptrFromObj(c->argv[3])); if (type == -1) { addReplyErrorFormat(c,"Unknown client type '%s'", - (char*) c->argv[3]->ptr); + (char*) ptrFromObj(c->argv[3])); return; } } else if (c->argc != 2) { @@ -1861,21 +1861,21 @@ NULL sds o = getAllClientsInfoString(type); addReplyBulkCBuffer(c,o,sdslen(o)); sdsfree(o); - } else if (!strcasecmp(c->argv[1]->ptr,"reply") && c->argc == 3) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"reply") && c->argc == 3) { /* CLIENT REPLY ON|OFF|SKIP */ - if (!strcasecmp(c->argv[2]->ptr,"on")) { + if (!strcasecmp(ptrFromObj(c->argv[2]),"on")) { c->flags &= ~(CLIENT_REPLY_SKIP|CLIENT_REPLY_OFF); addReply(c,shared.ok); - } else if (!strcasecmp(c->argv[2]->ptr,"off")) { + } else if (!strcasecmp(ptrFromObj(c->argv[2]),"off")) { c->flags |= CLIENT_REPLY_OFF; - } else if (!strcasecmp(c->argv[2]->ptr,"skip")) { + } else if (!strcasecmp(ptrFromObj(c->argv[2]),"skip")) { if (!(c->flags & CLIENT_REPLY_OFF)) c->flags |= CLIENT_REPLY_SKIP_NEXT; } else { addReply(c,shared.syntaxerr); return; } - } else if (!strcasecmp(c->argv[1]->ptr,"kill")) { + } else if (!strcasecmp(ptrFromObj(c->argv[1]),"kill")) { /* CLIENT KILL * CLIENT KILL