From ad41a7c40430bc043c2b95556197af2862a2bab5 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Sun, 16 Nov 2014 13:03:54 -0500 Subject: [PATCH 01/29] Add addReplyBulkSds() function Refactor a common pattern into one function so we don't end up with copy/paste programming. --- src/cluster.c | 5 +---- src/networking.c | 8 ++++++++ src/redis.c | 6 +----- src/redis.h | 1 + src/sentinel.c | 5 +---- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index 6fc028627..a308fc507 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -3886,10 +3886,7 @@ void clusterCommand(redisClient *c) { server.cluster->stats_bus_messages_sent, server.cluster->stats_bus_messages_received ); - addReplySds(c,sdscatprintf(sdsempty(),"$%lu\r\n", - (unsigned long)sdslen(info))); - addReplySds(c,info); - addReply(c,shared.crlf); + addReplyBulkSds(c, info); } else if (!strcasecmp(c->argv[1]->ptr,"saveconfig") && c->argc == 2) { int retval = clusterSaveConfig(1); diff --git a/src/networking.c b/src/networking.c index e4890588a..607d225fd 100644 --- a/src/networking.c +++ b/src/networking.c @@ -525,6 +525,14 @@ void addReplyBulkCBuffer(redisClient *c, void *p, size_t len) { addReply(c,shared.crlf); } +/* Add sds to reply (takes ownership of sds and frees it) */ +void addReplyBulkSds(redisClient *c, sds s) { + addReplySds(c,sdscatfmt(sdsempty(),"$%u\r\n", + (unsigned long)sdslen(s))); + addReplySds(c,s); + addReply(c,shared.crlf); +} + /* Add a C nul term string as bulk reply */ void addReplyBulkCString(redisClient *c, char *s) { if (s == NULL) { diff --git a/src/redis.c b/src/redis.c index d68065512..9077dd5e6 100644 --- a/src/redis.c +++ b/src/redis.c @@ -3058,11 +3058,7 @@ void infoCommand(redisClient *c) { addReply(c,shared.syntaxerr); return; } - sds info = genRedisInfoString(section); - addReplySds(c,sdscatprintf(sdsempty(),"$%lu\r\n", - (unsigned long)sdslen(info))); - addReplySds(c,info); - addReply(c,shared.crlf); + addReplyBulkSds(c, genRedisInfoString(section)); } void monitorCommand(redisClient *c) { diff --git a/src/redis.h b/src/redis.h index d60eaf34a..87415967d 100644 --- a/src/redis.h +++ b/src/redis.h @@ -1043,6 +1043,7 @@ void addReplyBulkCBuffer(redisClient *c, void *p, size_t len); void addReplyBulkLongLong(redisClient *c, long long ll); void addReply(redisClient *c, robj *obj); void addReplySds(redisClient *c, sds s); +void addReplyBulkSds(redisClient *c, sds s); void addReplyError(redisClient *c, char *err); void addReplyStatus(redisClient *c, char *status); void addReplyDouble(redisClient *c, double d); diff --git a/src/sentinel.c b/src/sentinel.c index 01c811813..33d970e57 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2908,10 +2908,7 @@ void sentinelInfoCommand(redisClient *c) { dictReleaseIterator(di); } - addReplySds(c,sdscatprintf(sdsempty(),"$%lu\r\n", - (unsigned long)sdslen(info))); - addReplySds(c,info); - addReply(c,shared.crlf); + addReplyBulkSds(c, info); } /* Implements Sentinel verison of the ROLE command. The output is From 27937c2821a85efd4245fcbac3ebed782129f5d7 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Sun, 16 Nov 2014 13:05:48 -0500 Subject: [PATCH 02/29] Add DEBUG JEMALLOC INFO Uses jemalloc function malloc_stats_print() to return stats about what jemalloc has allocated internally. --- src/debug.c | 18 ++++++++++++++++++ src/redis-cli.c | 3 +++ 2 files changed, 21 insertions(+) diff --git a/src/debug.c b/src/debug.c index caf95ec58..c464644cd 100644 --- a/src/debug.c +++ b/src/debug.c @@ -252,6 +252,12 @@ void computeDatasetDigest(unsigned char *final) { } } +void inputCatSds(void *result, const char *str) { + /* result is actually a (sds *), so re-cast it here */ + sds *info = (sds *)result; + *info = sdscat(*info, str); +} + void debugCommand(redisClient *c) { if (!strcasecmp(c->argv[1]->ptr,"segfault")) { *((char*)-1) = 'x'; @@ -379,6 +385,18 @@ void debugCommand(redisClient *c) { errstr = sdsmapchars(errstr,"\n\r"," ",2); /* no newlines in errors. */ errstr = sdscatlen(errstr,"\r\n",2); addReplySds(c,errstr); + } else if (!strcasecmp(c->argv[1]->ptr,"jemalloc") && c->argc == 3) { +#if defined(USE_JEMALLOC) + if (!strcasecmp(c->argv[2]->ptr, "info")) { + sds info = sdsempty(); + je_malloc_stats_print(inputCatSds, &info, NULL); + addReplyBulkSds(c, info); + } else { + addReplyErrorFormat(c, "Valid jemalloc debug fields: info"); + } +#else + addReplyErrorFormat(c, "jemalloc support not available"); +#endif } else { addReplyErrorFormat(c, "Unknown DEBUG subcommand or wrong number of arguments for '%s'", (char*)c->argv[1]->ptr); diff --git a/src/redis-cli.c b/src/redis-cli.c index 2a703ad7f..3c1458742 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -629,6 +629,9 @@ static int cliSendCommand(int argc, char **argv, int repeat) { output_raw = 0; if (!strcasecmp(command,"info") || + (argc == 3 && !strcasecmp(command,"debug") && + (!strcasecmp(argv[1],"jemalloc") && + !strcasecmp(argv[2],"info"))) || (argc == 2 && !strcasecmp(command,"cluster") && (!strcasecmp(argv[1],"nodes") || !strcasecmp(argv[1],"info"))) || From 8380655e85f0afd1f0afc99b464717cb97002b7a Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Sat, 8 Nov 2014 10:16:54 -0500 Subject: [PATCH 03/29] Remove ziplist compiler warnings Only happen when compiled with the test define. --- src/ziplist.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/ziplist.c b/src/ziplist.c index 64a22adfc..41d9cb20c 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -1028,10 +1028,12 @@ void pop(unsigned char *zl, int where) { else printf("Pop tail: "); - if (vstr) + if (vstr) { if (vlen && fwrite(vstr,vlen,1,stdout) == 0) perror("fwrite"); - else + } + else { printf("%lld", vlong); + } printf("\n"); ziplistDeleteRange(zl,-1,1); @@ -1368,7 +1370,7 @@ int main(int argc, char **argv) { { char v[3][257]; zlentry e[3]; - int i; + size_t i; for (i = 0; i < (sizeof(v)/sizeof(v[0])); i++) { memset(v[i], 'a' + i, sizeof(v[0])); @@ -1464,7 +1466,7 @@ int main(int argc, char **argv) { for (i = 0; i < 20000; i++) { zl = ziplistNew(); ref = listCreate(); - listSetFreeMethod(ref,sdsfree); + listSetFreeMethod(ref,(void (*)(void*))sdsfree); len = rand() % 256; /* Create lists */ From 8febcffdc597566f1e307c0534014b2bdf687c02 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Wed, 12 Nov 2014 21:58:57 -0500 Subject: [PATCH 04/29] Allow all code tests to run using Redis args Previously, many files had individual main() functions for testing, but each required being compiled with their own testing flags. That gets difficult when you have 8 different flags you need to set just to run all tests (plus, some test files required other files to be compiled aaginst them, and it seems some didn't build at all without including the rest of Redis). Now all individual test main() funcions are renamed to a test function for the file itself and one global REDIS_TEST define enables testing across the entire codebase. Tests can now be run with: - `./redis-server test ` e.g. ./redis-server test ziplist If REDIS_TEST is not defined, then no tests get included and no tests are included in the final redis-server binary. --- src/crc64.c | 8 ++++++-- src/crc64.h | 4 ++++ src/endianconv.c | 8 ++++++-- src/endianconv.h | 4 ++++ src/intset.c | 42 ++++++++++++++++++++++++------------------ src/intset.h | 4 ++++ src/redis.c | 24 ++++++++++++++++++++++++ src/redis.h | 6 ++++++ src/sds.c | 15 ++++++++++++--- src/sds.h | 4 ++++ src/sha1.c | 11 ++++++----- src/sha1.h | 7 +++++++ src/util.c | 12 ++++++++---- src/util.h | 4 ++++ src/ziplist.c | 19 +++++++++---------- src/ziplist.h | 4 ++++ src/zipmap.c | 10 +++++++--- src/zipmap.h | 4 ++++ 18 files changed, 143 insertions(+), 47 deletions(-) diff --git a/src/crc64.c b/src/crc64.c index ecdba90e0..f1f764922 100644 --- a/src/crc64.c +++ b/src/crc64.c @@ -181,9 +181,13 @@ uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) { } /* Test main */ -#ifdef TEST_MAIN +#ifdef REDIS_TEST #include -int main(void) { + +#define UNUSED(x) (void)(x) +int crc64Test(int argc, char *argv[]) { + UNUSED(argc); + UNUSED(argv); printf("e9c6d914c4b8d9ca == %016llx\n", (unsigned long long) crc64(0,(unsigned char*)"123456789",9)); return 0; diff --git a/src/crc64.h b/src/crc64.h index ab375d3f4..c9fca519d 100644 --- a/src/crc64.h +++ b/src/crc64.h @@ -5,4 +5,8 @@ uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l); +#ifdef REDIS_TEST +int crc64Test(int argc, char *argv[]); +#endif + #endif diff --git a/src/endianconv.c b/src/endianconv.c index 9adf09c1f..f3b0b4730 100644 --- a/src/endianconv.c +++ b/src/endianconv.c @@ -101,12 +101,16 @@ uint64_t intrev64(uint64_t v) { return v; } -#ifdef TESTMAIN +#ifdef REDIS_TEST #include -int main(void) { +#define UNUSED(x) (void)(x) +int endianconvTest(int argc, char *argv[]) { char buf[32]; + UNUSED(argc); + UNUSED(argv); + sprintf(buf,"ciaoroma"); memrev16(buf); printf("%s\n", buf); diff --git a/src/endianconv.h b/src/endianconv.h index d93cd99ba..08f553136 100644 --- a/src/endianconv.h +++ b/src/endianconv.h @@ -71,4 +71,8 @@ uint64_t intrev64(uint64_t v); #define ntohu64(v) intrev64(v) #endif +#ifdef REDIS_TEST +int endianconvTest(int argc, char *argv[]); +#endif + #endif diff --git a/src/intset.c b/src/intset.c index 92c4ecd6e..762bd48c8 100644 --- a/src/intset.c +++ b/src/intset.c @@ -365,44 +365,46 @@ size_t intsetBlobLen(intset *is) { return sizeof(intset)+intrev32ifbe(is->length)*intrev32ifbe(is->encoding); } -#ifdef INTSET_TEST_MAIN +#ifdef REDIS_TEST #include +#include -void intsetRepr(intset *is) { - int i; - for (i = 0; i < intrev32ifbe(is->length); i++) { +#if 0 +static void intsetRepr(intset *is) { + for (uint32_t i = 0; i < intrev32ifbe(is->length); i++) { printf("%lld\n", (uint64_t)_intsetGet(is,i)); } printf("\n"); } -void error(char *err) { +static void error(char *err) { printf("%s\n", err); exit(1); } +#endif -void ok(void) { +static void ok(void) { printf("OK\n"); } -long long usec(void) { +static long long usec(void) { struct timeval tv; gettimeofday(&tv,NULL); return (((long long)tv.tv_sec)*1000000)+tv.tv_usec; } #define assert(_e) ((_e)?(void)0:(_assert(#_e,__FILE__,__LINE__),exit(1))) -void _assert(char *estr, char *file, int line) { +static void _assert(char *estr, char *file, int line) { printf("\n\n=== ASSERTION FAILED ===\n"); printf("==> %s:%d '%s' is not true\n",file,line,estr); } -intset *createSet(int bits, int size) { +static intset *createSet(int bits, int size) { uint64_t mask = (1< 32) { value = (rand()*rand()) & mask; } else { @@ -413,10 +415,8 @@ intset *createSet(int bits, int size) { return is; } -void checkConsistency(intset *is) { - int i; - - for (i = 0; i < (intrev32ifbe(is->length)-1); i++) { +static void checkConsistency(intset *is) { + for (uint32_t i = 0; i < (intrev32ifbe(is->length)-1); i++) { uint32_t encoding = intrev32ifbe(is->encoding); if (encoding == INTSET_ENC_INT16) { @@ -432,11 +432,15 @@ void checkConsistency(intset *is) { } } -int main(int argc, char **argv) { +#define UNUSED(x) (void)(x) +int intsetTest(int argc, char **argv) { uint8_t success; int i; intset *is; - sranddev(); + srand(time(NULL)); + + UNUSED(argc); + UNUSED(argv); printf("Value encodings: "); { assert(_intsetValueEncoding(-32768) == INTSET_ENC_INT16); @@ -464,7 +468,7 @@ int main(int argc, char **argv) { } printf("Large number of random adds: "); { - int inserts = 0; + uint32_t inserts = 0; is = intsetNew(); for (i = 0; i < 1024; i++) { is = intsetAdd(is,rand()%0x800,&success); @@ -566,5 +570,7 @@ int main(int argc, char **argv) { checkConsistency(is); ok(); } + + return 0; } #endif diff --git a/src/intset.h b/src/intset.h index 51a512753..7550df303 100644 --- a/src/intset.h +++ b/src/intset.h @@ -48,4 +48,8 @@ uint8_t intsetGet(intset *is, uint32_t pos, int64_t *value); uint32_t intsetLen(intset *is); size_t intsetBlobLen(intset *is); +#ifdef REDIS_TEST +int intsetTest(int argc, char *argv[]); +#endif + #endif // __INTSET_H diff --git a/src/redis.c b/src/redis.c index 9077dd5e6..01912b79b 100644 --- a/src/redis.c +++ b/src/redis.c @@ -3655,6 +3655,30 @@ int redisIsSupervised(void) { int main(int argc, char **argv) { struct timeval tv; +#ifdef REDIS_TEST + if (argc == 3 && !strcasecmp(argv[1], "test")) { + if (!strcasecmp(argv[2], "ziplist")) { + return ziplistTest(argc, argv); + } else if (!strcasecmp(argv[2], "intset")) { + return intsetTest(argc, argv); + } else if (!strcasecmp(argv[2], "zipmap")) { + return zipmapTest(argc, argv); + } else if (!strcasecmp(argv[2], "sha1test")) { + return sha1Test(argc, argv); + } else if (!strcasecmp(argv[2], "util")) { + return utilTest(argc, argv); + } else if (!strcasecmp(argv[2], "sds")) { + return sdsTest(argc, argv); + } else if (!strcasecmp(argv[2], "endianconv")) { + return endianconvTest(argc, argv); + } else if (!strcasecmp(argv[2], "crc64")) { + return crc64Test(argc, argv); + } + + return -1; /* test not found */ + } +#endif + /* We need to initialize our libraries, and the server configuration. */ #ifdef INIT_SETPROCTITLE_REPLACEMENT spt_init(argc, argv); diff --git a/src/redis.h b/src/redis.h index 87415967d..1720c24bd 100644 --- a/src/redis.h +++ b/src/redis.h @@ -66,6 +66,12 @@ typedef long long mstime_t; /* millisecond time type. */ #include "latency.h" /* Latency monitor API */ #include "sparkline.h" /* ASII graphs API */ +/* Following includes allow test functions to be called from Redis main() */ +#include "zipmap.h" +#include "sha1.h" +#include "endianconv.h" +#include "crc64.h" + /* Error codes */ #define REDIS_OK 0 #define REDIS_ERR -1 diff --git a/src/sds.c b/src/sds.c index 1df1043ed..05ee0ad56 100644 --- a/src/sds.c +++ b/src/sds.c @@ -962,12 +962,15 @@ sds sdsjoin(char **argv, int argc, char *sep) { return join; } -#ifdef SDS_TEST_MAIN +#if defined(REDIS_TEST) || defined(SDS_TEST_MAIN) #include #include "testhelp.h" #include "limits.h" -int main(void) { +#define UNUSED(x) (void)(x) +int sdsTest(int argc, char *argv[]) { + UNUSED(argc); + UNUSED(argv); { struct sdshdr *sh; sds x = sdsnew("foo"), y; @@ -1092,7 +1095,7 @@ int main(void) { memcmp(y,"\"\\a\\n\\x00foo\\r\"",15) == 0) { - int oldfree; + unsigned int oldfree; sdsfree(x); x = sdsnew("0"); @@ -1113,3 +1116,9 @@ int main(void) { return 0; } #endif + +#ifdef SDS_TEST_MAIN +int main(void) { + return sdsTest(); +} +#endif diff --git a/src/sds.h b/src/sds.h index 37aaf7a28..93dd4f28e 100644 --- a/src/sds.h +++ b/src/sds.h @@ -98,4 +98,8 @@ void sdsIncrLen(sds s, int incr); sds sdsRemoveFreeSpace(sds s); size_t sdsAllocSize(sds s); +#ifdef REDIS_TEST +int sdsTest(int argc, char *argv[]); +#endif + #endif diff --git a/src/sha1.c b/src/sha1.c index 59e6f461d..199545df4 100644 --- a/src/sha1.c +++ b/src/sha1.c @@ -199,16 +199,19 @@ void SHA1Final(unsigned char digest[20], SHA1_CTX* context) } /* ================ end of sha1.c ================ */ -#if 0 +#ifdef REDIS_TEST #define BUFSIZE 4096 -int -main(int argc, char **argv) +#define UNUSED(x) (void)(x) +int sha1Test(int argc, char **argv) { SHA1_CTX ctx; unsigned char hash[20], buf[BUFSIZE]; int i; + UNUSED(argc); + UNUSED(argv); + for(i=0;i -void test_string2ll(void) { +static void test_string2ll(void) { char buf[32]; long long v; @@ -587,7 +587,7 @@ void test_string2ll(void) { assert(string2ll(buf,strlen(buf),&v) == 0); } -void test_string2l(void) { +static void test_string2l(void) { char buf[32]; long v; @@ -636,7 +636,11 @@ void test_string2l(void) { #endif } -int main(int argc, char **argv) { +#define UNUSED(x) (void)(x) +int utilTest(int argc, char **argv) { + UNUSED(argc); + UNUSED(argv); + test_string2ll(); test_string2l(); return 0; diff --git a/src/util.h b/src/util.h index b3667cd6f..666042c9b 100644 --- a/src/util.h +++ b/src/util.h @@ -42,4 +42,8 @@ int d2string(char *buf, size_t len, double value); sds getAbsolutePath(char *filename); int pathIsBaseName(char *path); +#ifdef REDIS_TEST +int utilTest(int argc, char **argv); +#endif + #endif diff --git a/src/ziplist.c b/src/ziplist.c index 41d9cb20c..3ff44c5b8 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -952,14 +952,14 @@ void ziplistRepr(unsigned char *zl) { printf("{end}\n\n"); } -#ifdef ZIPLIST_TEST_MAIN +#ifdef REDIS_TEST #include #include "adlist.h" #include "sds.h" #define debug(f, ...) { if (DEBUG) printf(f, __VA_ARGS__); } -unsigned char *createList() { +static unsigned char *createList() { unsigned char *zl = ziplistNew(); zl = ziplistPush(zl, (unsigned char*)"foo", 3, ZIPLIST_TAIL); zl = ziplistPush(zl, (unsigned char*)"quux", 4, ZIPLIST_TAIL); @@ -968,7 +968,7 @@ unsigned char *createList() { return zl; } -unsigned char *createIntList() { +static unsigned char *createIntList() { unsigned char *zl = ziplistNew(); char buf[32]; @@ -987,13 +987,13 @@ unsigned char *createIntList() { return zl; } -long long usec(void) { +static long long usec(void) { struct timeval tv; gettimeofday(&tv,NULL); return (((long long)tv.tv_sec)*1000000)+tv.tv_usec; } -void stress(int pos, int num, int maxsize, int dnum) { +static void stress(int pos, int num, int maxsize, int dnum) { int i,j,k; unsigned char *zl; char posstr[2][5] = { "HEAD", "TAIL" }; @@ -1016,7 +1016,7 @@ void stress(int pos, int num, int maxsize, int dnum) { } } -void pop(unsigned char *zl, int where) { +static void pop(unsigned char *zl, int where) { unsigned char *p, *vstr; unsigned int vlen; long long vlong; @@ -1043,7 +1043,7 @@ void pop(unsigned char *zl, int where) { } } -int randstring(char *target, unsigned int min, unsigned int max) { +static int randstring(char *target, unsigned int min, unsigned int max) { int p = 0; int len = min+rand()%(max-min+1); int minval, maxval; @@ -1069,7 +1069,7 @@ int randstring(char *target, unsigned int min, unsigned int max) { return len; } -void verify(unsigned char *zl, zlentry *e) { +static void verify(unsigned char *zl, zlentry *e) { int i; int len = ziplistLen(zl); zlentry _e; @@ -1085,7 +1085,7 @@ void verify(unsigned char *zl, zlentry *e) { } } -int main(int argc, char **argv) { +int ziplistTest(int argc, char **argv) { unsigned char *zl, *p; unsigned char *entry; unsigned int elen; @@ -1534,5 +1534,4 @@ int main(int argc, char **argv) { return 0; } - #endif diff --git a/src/ziplist.h b/src/ziplist.h index b29c34167..bc27006aa 100644 --- a/src/ziplist.h +++ b/src/ziplist.h @@ -44,3 +44,7 @@ unsigned int ziplistCompare(unsigned char *p, unsigned char *s, unsigned int sle unsigned char *ziplistFind(unsigned char *p, unsigned char *vstr, unsigned int vlen, unsigned int skip); unsigned int ziplistLen(unsigned char *zl); size_t ziplistBlobLen(unsigned char *zl); + +#ifdef REDIS_TEST +int ziplistTest(int argc, char *argv[]); +#endif diff --git a/src/zipmap.c b/src/zipmap.c index 384b76bba..22bfa1a46 100644 --- a/src/zipmap.c +++ b/src/zipmap.c @@ -370,8 +370,8 @@ size_t zipmapBlobLen(unsigned char *zm) { return totlen; } -#ifdef ZIPMAP_TEST_MAIN -void zipmapRepr(unsigned char *p) { +#ifdef REDIS_TEST +static void zipmapRepr(unsigned char *p) { unsigned int l; printf("{status %u}",*p++); @@ -404,9 +404,13 @@ void zipmapRepr(unsigned char *p) { printf("\n"); } -int main(void) { +#define UNUSED(x) (void)(x) +int zipmapTest(int argc, char *argv[]) { unsigned char *zm; + UNUSED(argc); + UNUSED(argv); + zm = zipmapNew(); zm = zipmapSet(zm,(unsigned char*) "name",4, (unsigned char*) "foo",3,NULL); diff --git a/src/zipmap.h b/src/zipmap.h index 9cf1b2484..ac588f05a 100644 --- a/src/zipmap.h +++ b/src/zipmap.h @@ -46,4 +46,8 @@ unsigned int zipmapLen(unsigned char *zm); size_t zipmapBlobLen(unsigned char *zm); void zipmapRepr(unsigned char *p); +#ifdef REDIS_TEST +int zipmapTest(int argc, char *argv[]); +#endif + #endif From 9b343678d2edaaecb10aaa314f78e41b25534f44 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Thu, 20 Nov 2014 13:00:08 -0500 Subject: [PATCH 05/29] Add simple ll2string() tests --- src/util.c | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/util.c b/src/util.c index bd158ae90..d69721bf4 100644 --- a/src/util.c +++ b/src/util.c @@ -636,6 +636,47 @@ static void test_string2l(void) { #endif } +static void test_ll2string(void) { + char buf[32]; + long long v; + int sz; + + v = 0; + sz = ll2string(buf, sizeof buf, v); + assert(sz == 1); + assert(!strcmp(buf, "0")); + + v = -1; + sz = ll2string(buf, sizeof buf, v); + assert(sz == 2); + assert(!strcmp(buf, "-1")); + + v = 99; + sz = ll2string(buf, sizeof buf, v); + assert(sz == 2); + assert(!strcmp(buf, "99")); + + v = -99; + sz = ll2string(buf, sizeof buf, v); + assert(sz == 3); + assert(!strcmp(buf, "-99")); + + v = -2147483648; + sz = ll2string(buf, sizeof buf, v); + assert(sz == 11); + assert(!strcmp(buf, "-2147483648")); + + v = LLONG_MIN; + sz = ll2string(buf, sizeof buf, v); + assert(sz == 20); + assert(!strcmp(buf, "-9223372036854775808")); + + v = LLONG_MAX; + sz = ll2string(buf, sizeof buf, v); + assert(sz == 19); + assert(!strcmp(buf, "9223372036854775807")); +} + #define UNUSED(x) (void)(x) int utilTest(int argc, char **argv) { UNUSED(argc); @@ -643,6 +684,7 @@ int utilTest(int argc, char **argv) { test_string2ll(); test_string2l(); + test_ll2string(); return 0; } #endif From d01d4ddcbb62a19e841e4505930acd1986547dbf Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Thu, 13 Nov 2014 15:12:08 -0500 Subject: [PATCH 06/29] Allow forcing non-jemalloc build --- src/Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Makefile b/src/Makefile index 3d34685ba..231761f5e 100644 --- a/src/Makefile +++ b/src/Makefile @@ -46,6 +46,10 @@ ifeq ($(USE_JEMALLOC),yes) MALLOC=jemalloc endif +ifeq ($(USE_JEMALLOC),no) + MALLOC=libc +endif + # Override default settings if possible -include .make-settings From fae53dea3333af7e5db1b26d7b7d4d5cea5a6d70 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Thu, 13 Nov 2014 22:17:57 -0500 Subject: [PATCH 07/29] Fix how zipEntry returns values zipEntry was returning a struct, but that caused some problems with tests under 32 bit builds. The tests run better if we operate on structs allocated in the caller without worrying about copying on return. --- src/ziplist.c | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/src/ziplist.c b/src/ziplist.c index 3ff44c5b8..8975e6305 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -404,14 +404,12 @@ static int64_t zipLoadInteger(unsigned char *p, unsigned char encoding) { } /* Return a struct with all information about an entry. */ -static zlentry zipEntry(unsigned char *p) { - zlentry e; +static void zipEntry(unsigned char *p, zlentry *e) { - ZIP_DECODE_PREVLEN(p, e.prevrawlensize, e.prevrawlen); - ZIP_DECODE_LENGTH(p + e.prevrawlensize, e.encoding, e.lensize, e.len); - e.headersize = e.prevrawlensize + e.lensize; - e.p = p; - return e; + ZIP_DECODE_PREVLEN(p, e->prevrawlensize, e->prevrawlen); + ZIP_DECODE_LENGTH(p + e->prevrawlensize, e->encoding, e->lensize, e->len); + e->headersize = e->prevrawlensize + e->lensize; + e->p = p; } /* Create a new empty ziplist. */ @@ -460,13 +458,13 @@ static unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p zlentry cur, next; while (p[0] != ZIP_END) { - cur = zipEntry(p); + zipEntry(p, &cur); rawlen = cur.headersize + cur.len; rawlensize = zipPrevEncodeLength(NULL,rawlen); /* Abort if there is no next entry. */ if (p[rawlen] == ZIP_END) break; - next = zipEntry(p+rawlen); + zipEntry(p+rawlen, &next); /* Abort when "prevlen" has not changed. */ if (next.prevrawlen == rawlen) break; @@ -521,7 +519,7 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsig int nextdiff = 0; zlentry first, tail; - first = zipEntry(p); + zipEntry(p, &first); for (i = 0; p[0] != ZIP_END && i < num; i++) { p += zipRawEntryLength(p); deleted++; @@ -545,7 +543,7 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsig /* When the tail contains more than one entry, we need to take * "nextdiff" in account as well. Otherwise, a change in the * size of prevlen doesn't have an effect on the *tail* offset. */ - tail = zipEntry(p); + zipEntry(p, &tail); if (p[tail.headersize+tail.len] != ZIP_END) { ZIPLIST_TAIL_OFFSET(zl) = intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+nextdiff); @@ -635,7 +633,7 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig /* When the tail contains more than one entry, we need to take * "nextdiff" in account as well. Otherwise, a change in the * size of prevlen doesn't have an effect on the *tail* offset. */ - tail = zipEntry(p+reqlen); + zipEntry(p+reqlen, &tail); if (p[reqlen+tail.headersize+tail.len] != ZIP_END) { ZIPLIST_TAIL_OFFSET(zl) = intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+nextdiff); @@ -748,7 +746,7 @@ unsigned int ziplistGet(unsigned char *p, unsigned char **sstr, unsigned int *sl if (p == NULL || p[0] == ZIP_END) return 0; if (sstr) *sstr = NULL; - entry = zipEntry(p); + zipEntry(p, &entry); if (ZIP_IS_STR(entry.encoding)) { if (sstr) { *slen = entry.len; @@ -796,7 +794,7 @@ unsigned int ziplistCompare(unsigned char *p, unsigned char *sstr, unsigned int long long zval, sval; if (p[0] == ZIP_END) return 0; - entry = zipEntry(p); + zipEntry(p, &entry); if (ZIP_IS_STR(entry.encoding)) { /* Raw compare */ if (entry.len == slen) { @@ -913,7 +911,7 @@ void ziplistRepr(unsigned char *zl) { intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))); p = ZIPLIST_ENTRY_HEAD(zl); while(*p != ZIP_END) { - entry = zipEntry(p); + zipEntry(p, &entry); printf( "{" "addr 0x%08lx, " @@ -1076,10 +1074,10 @@ static void verify(unsigned char *zl, zlentry *e) { for (i = 0; i < len; i++) { memset(&e[i], 0, sizeof(zlentry)); - e[i] = zipEntry(ziplistIndex(zl, i)); + zipEntry(ziplistIndex(zl, i), &e[i]); memset(&_e, 0, sizeof(zlentry)); - _e = zipEntry(ziplistIndex(zl, -len+i)); + zipEntry(ziplistIndex(zl, -len+i), &_e); assert(memcmp(&e[i], &_e, sizeof(zlentry)) == 0); } From 53b1ee34ddba1446d81f2ce07c402db271a94674 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Thu, 13 Nov 2014 16:12:09 -0500 Subject: [PATCH 08/29] Fix ziplistDeleteRange index parameter It's valid to delete from negative offsets, so we *don't* want unsigned arguments here. --- src/ziplist.c | 2 +- src/ziplist.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ziplist.c b/src/ziplist.c index 8975e6305..7e8823f5a 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -781,7 +781,7 @@ unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p) { } /* Delete a range of entries from the ziplist. */ -unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigned int num) { +unsigned char *ziplistDeleteRange(unsigned char *zl, int index, unsigned int num) { unsigned char *p = ziplistIndex(zl,index); return (p == NULL) ? zl : __ziplistDelete(zl,p,num); } diff --git a/src/ziplist.h b/src/ziplist.h index bc27006aa..ef0e27140 100644 --- a/src/ziplist.h +++ b/src/ziplist.h @@ -39,7 +39,7 @@ unsigned char *ziplistPrev(unsigned char *zl, unsigned char *p); unsigned int ziplistGet(unsigned char *p, unsigned char **sval, unsigned int *slen, long long *lval); unsigned char *ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen); unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p); -unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigned int num); +unsigned char *ziplistDeleteRange(unsigned char *zl, int index, unsigned int num); unsigned int ziplistCompare(unsigned char *p, unsigned char *s, unsigned int slen); unsigned char *ziplistFind(unsigned char *p, unsigned char *vstr, unsigned int vlen, unsigned int skip); unsigned int ziplistLen(unsigned char *zl); From 1dfcd75ae36f5f7bea86893c391ea863cd32dee9 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Thu, 13 Nov 2014 16:12:44 -0500 Subject: [PATCH 09/29] Fix ziplist test for pop() The previous test wasn't returning the new ziplist, so the test was invalid. Now the test works properly. These problems were simultaenously discovered in #2154 and that PR also had an additional fix we included here. --- src/ziplist.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ziplist.c b/src/ziplist.c index 7e8823f5a..5ee71d31b 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -1014,7 +1014,7 @@ static void stress(int pos, int num, int maxsize, int dnum) { } } -static void pop(unsigned char *zl, int where) { +static unsigned char *pop(unsigned char *zl, int where) { unsigned char *p, *vstr; unsigned int vlen; long long vlong; @@ -1034,7 +1034,7 @@ static void pop(unsigned char *zl, int where) { } printf("\n"); - ziplistDeleteRange(zl,-1,1); + return ziplistDelete(zl,&p); } else { printf("ERROR: Could not pop\n"); exit(1); @@ -1099,16 +1099,16 @@ int ziplistTest(int argc, char **argv) { zl = createList(); ziplistRepr(zl); - pop(zl,ZIPLIST_TAIL); + zl = pop(zl,ZIPLIST_TAIL); ziplistRepr(zl); - pop(zl,ZIPLIST_HEAD); + zl = pop(zl,ZIPLIST_HEAD); ziplistRepr(zl); - pop(zl,ZIPLIST_TAIL); + zl = pop(zl,ZIPLIST_TAIL); ziplistRepr(zl); - pop(zl,ZIPLIST_TAIL); + zl = pop(zl,ZIPLIST_TAIL); ziplistRepr(zl); printf("Get element at index 3:\n"); From 9b786b124d6a547b87700114acdb9a617521a4bf Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Thu, 13 Nov 2014 16:21:27 -0500 Subject: [PATCH 10/29] Cleanup ziplist valgrind warnings Valgrind can't detect 'memset' initializes things, so let's statically initialize them to remove some unnecessary warnings. --- src/ziplist.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/ziplist.c b/src/ziplist.c index 5ee71d31b..111f2a9a7 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -162,6 +162,13 @@ typedef struct zlentry { unsigned char *p; } zlentry; +#define ZIPLIST_ENTRY_ZERO(zle) { \ + (zle)->prevrawlensize = (zle)->prevrawlen = 0; \ + (zle)->lensize = (zle)->len = (zle)->headersize = 0; \ + (zle)->encoding = 0; \ + (zle)->p = NULL; \ +} + /* Extract the encoding from the byte pointed by 'ptr' and set it into * 'encoding'. */ #define ZIP_ENTRY_ENCODING(ptr, encoding) do { \ @@ -1068,11 +1075,12 @@ static int randstring(char *target, unsigned int min, unsigned int max) { } static void verify(unsigned char *zl, zlentry *e) { - int i; int len = ziplistLen(zl); zlentry _e; - for (i = 0; i < len; i++) { + ZIPLIST_ENTRY_ZERO(&_e); + + for (int i = 0; i < len; i++) { memset(&e[i], 0, sizeof(zlentry)); zipEntry(ziplistIndex(zl, i), &e[i]); @@ -1347,7 +1355,7 @@ int ziplistTest(int argc, char **argv) { printf("Regression test for >255 byte strings:\n"); { - char v1[257],v2[257]; + char v1[257] = {0}, v2[257] = {0}; memset(v1,'x',256); memset(v2,'y',256); zl = ziplistNew(); @@ -1366,8 +1374,9 @@ int ziplistTest(int argc, char **argv) { printf("Regression test deleting next to last entries:\n"); { - char v[3][257]; - zlentry e[3]; + char v[3][257] = {{0}}; + zlentry e[3] = {{.prevrawlensize = 0, .prevrawlen = 0, .lensize = 0, + .len = 0, .headersize = 0, .encoding = 0, .p = NULL}}; size_t i; for (i = 0; i < (sizeof(v)/sizeof(v[0])); i++) { From d956d809acb848aec0f6524e1337d274a635980d Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Thu, 13 Nov 2014 23:35:10 -0500 Subject: [PATCH 11/29] Fix three simple clang analyzer warnings --- src/rdb.c | 2 +- src/sentinel.c | 2 +- src/t_zset.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index 3dd69f289..325e5a62f 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -720,7 +720,7 @@ int rdbSave(char *filename) { char tmpfile[256]; FILE *fp; rio rdb; - int error; + int error = 0; snprintf(tmpfile,256,"temp-%d.rdb", (int) getpid()); fp = fopen(tmpfile,"w"); diff --git a/src/sentinel.c b/src/sentinel.c index 33d970e57..c693a5862 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -577,7 +577,7 @@ void sentinelEvent(int level, char *type, sentinelRedisInstance *ri, if (level == REDIS_WARNING && ri != NULL) { sentinelRedisInstance *master = (ri->flags & SRI_MASTER) ? ri : ri->master; - if (master->notification_script) { + if (master && master->notification_script) { sentinelScheduleScriptExecution(master->notification_script, type,msg,NULL); } diff --git a/src/t_zset.c b/src/t_zset.c index d3c7214bd..64418c9b4 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -1382,7 +1382,7 @@ void zremrangeGenericCommand(redisClient *c, int rangetype) { robj *key = c->argv[1]; robj *zobj; int keyremoved = 0; - unsigned long deleted; + unsigned long deleted = 0; zrangespec range; zlexrangespec lexrange; long start, end, llen; From 5e362b84ab8b769bf2738daea97b45a375d223f1 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Thu, 13 Nov 2014 14:11:47 -0500 Subject: [PATCH 12/29] Add quicklist implementation This replaces individual ziplist vs. linkedlist representations for Redis list operations. Big thanks for all the reviews and feedback from everybody in https://github.com/antirez/redis/pull/2143 --- src/Makefile | 2 +- src/aof.c | 43 +- src/object.c | 16 +- src/quicklist.c | 2155 ++++++++++++++++++++++++++++++++++++ src/quicklist.h | 120 ++ src/rdb.c | 65 +- src/redis.c | 2 + src/redis.h | 12 +- src/sort.c | 5 +- src/t_list.c | 356 ++---- tests/support/test.tcl | 7 +- tests/unit/aofrw.tcl | 4 +- tests/unit/sort.tcl | 10 +- tests/unit/type/list-2.tcl | 8 +- tests/unit/type/list-3.tcl | 2 +- tests/unit/type/list.tcl | 168 +-- 16 files changed, 2495 insertions(+), 480 deletions(-) create mode 100644 src/quicklist.c create mode 100644 src/quicklist.h diff --git a/src/Makefile b/src/Makefile index 231761f5e..16172b25f 100644 --- a/src/Makefile +++ b/src/Makefile @@ -117,7 +117,7 @@ endif REDIS_SERVER_NAME=redis-server REDIS_SENTINEL_NAME=redis-sentinel -REDIS_SERVER_OBJ=adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o +REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o REDIS_CLI_NAME=redis-cli REDIS_CLI_OBJ=anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o REDIS_BENCHMARK_NAME=redis-benchmark diff --git a/src/aof.c b/src/aof.c index 0af519bfa..f5a90a12c 100644 --- a/src/aof.c +++ b/src/aof.c @@ -770,52 +770,29 @@ int rioWriteBulkObject(rio *r, robj *obj) { int rewriteListObject(rio *r, robj *key, robj *o) { long long count = 0, items = listTypeLength(o); - if (o->encoding == REDIS_ENCODING_ZIPLIST) { - unsigned char *zl = o->ptr; - unsigned char *p = ziplistIndex(zl,0); - unsigned char *vstr; - unsigned int vlen; - long long vlong; + if (o->encoding == REDIS_ENCODING_QUICKLIST) { + quicklist *list = o->ptr; + quicklistIter *li = quicklistGetIterator(list, AL_START_HEAD); + quicklistEntry entry; - while(ziplistGet(p,&vstr,&vlen,&vlong)) { + while (quicklistNext(li,&entry)) { if (count == 0) { int cmd_items = (items > REDIS_AOF_REWRITE_ITEMS_PER_CMD) ? REDIS_AOF_REWRITE_ITEMS_PER_CMD : items; - if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0; if (rioWriteBulkString(r,"RPUSH",5) == 0) return 0; if (rioWriteBulkObject(r,key) == 0) return 0; } - if (vstr) { - if (rioWriteBulkString(r,(char*)vstr,vlen) == 0) return 0; + + if (entry.value) { + if (rioWriteBulkString(r,(char*)entry.value,entry.sz) == 0) return 0; } else { - if (rioWriteBulkLongLong(r,vlong) == 0) return 0; + if (rioWriteBulkLongLong(r,entry.longval) == 0) return 0; } - p = ziplistNext(zl,p); - if (++count == REDIS_AOF_REWRITE_ITEMS_PER_CMD) count = 0; - items--; - } - } else if (o->encoding == REDIS_ENCODING_LINKEDLIST) { - list *list = o->ptr; - listNode *ln; - listIter li; - - listRewind(list,&li); - while((ln = listNext(&li))) { - robj *eleobj = listNodeValue(ln); - - if (count == 0) { - int cmd_items = (items > REDIS_AOF_REWRITE_ITEMS_PER_CMD) ? - REDIS_AOF_REWRITE_ITEMS_PER_CMD : items; - - if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0; - if (rioWriteBulkString(r,"RPUSH",5) == 0) return 0; - if (rioWriteBulkObject(r,key) == 0) return 0; - } - if (rioWriteBulkObject(r,eleobj) == 0) return 0; if (++count == REDIS_AOF_REWRITE_ITEMS_PER_CMD) count = 0; items--; } + quicklistReleaseIterator(li); } else { redisPanic("Unknown list encoding"); } diff --git a/src/object.c b/src/object.c index 11c77c3c3..f75421ee8 100644 --- a/src/object.c +++ b/src/object.c @@ -180,11 +180,10 @@ robj *dupStringObject(robj *o) { } } -robj *createListObject(void) { - list *l = listCreate(); +robj *createQuicklistObject(void) { + quicklist *l = quicklistCreate(); robj *o = createObject(REDIS_LIST,l); - listSetFreeMethod(l,decrRefCountVoid); - o->encoding = REDIS_ENCODING_LINKEDLIST; + o->encoding = REDIS_ENCODING_QUICKLIST; return o; } @@ -242,11 +241,8 @@ void freeStringObject(robj *o) { void freeListObject(robj *o) { switch (o->encoding) { - case REDIS_ENCODING_LINKEDLIST: - listRelease((list*) o->ptr); - break; - case REDIS_ENCODING_ZIPLIST: - zfree(o->ptr); + case REDIS_ENCODING_QUICKLIST: + quicklistRelease(o->ptr); break; default: redisPanic("Unknown list encoding type"); @@ -678,7 +674,7 @@ char *strEncoding(int encoding) { case REDIS_ENCODING_RAW: return "raw"; case REDIS_ENCODING_INT: return "int"; case REDIS_ENCODING_HT: return "hashtable"; - case REDIS_ENCODING_LINKEDLIST: return "linkedlist"; + case REDIS_ENCODING_QUICKLIST: return "quicklist"; case REDIS_ENCODING_ZIPLIST: return "ziplist"; case REDIS_ENCODING_INTSET: return "intset"; case REDIS_ENCODING_SKIPLIST: return "skiplist"; diff --git a/src/quicklist.c b/src/quicklist.c new file mode 100644 index 000000000..415af36c9 --- /dev/null +++ b/src/quicklist.c @@ -0,0 +1,2155 @@ +/* quicklist.c - A doubly linked list of ziplists + * + * Copyright (c) 2014, Matt Stancliff + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must start the above copyright notice, + * this quicklist of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this quicklist of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include /* for memcpy */ +#include "quicklist.h" +#include "zmalloc.h" +#include "ziplist.h" +#include "util.h" /* for ll2string */ + +#if defined(REDIS_TEST) || defined(REDIS_TEST_VERBOSE) +#include /* for printf (debug printing), snprintf (genstr) */ +#endif + +/* If not verbose testing, remove all debug printing. */ +#ifndef REDIS_TEST_VERBOSE +#define D(...) +#else +#define D(...) \ + do { \ + printf("%s:%s:%d:\t", __FILE__, __FUNCTION__, __LINE__); \ + printf(__VA_ARGS__); \ + printf("\n"); \ + } while (0); +#endif + +/* Simple way to give quicklistEntry structs default values with one call. */ +#define initEntry(e) \ + do { \ + (e)->zi = (e)->value = NULL; \ + (e)->longval = -123456789; \ + (e)->quicklist = NULL; \ + (e)->node = NULL; \ + (e)->offset = 123456789; \ + (e)->sz = 0; \ + } while (0) + +/* Create a new quicklist. + * Free with quicklistRelease(). + * + * On error, NULL is returned. Otherwise the pointer to the new quicklist. */ +quicklist *quicklistCreate(void) { + struct quicklist *quicklist; + + if ((quicklist = zmalloc(sizeof(*quicklist))) == NULL) + return NULL; + quicklist->head = quicklist->tail = NULL; + quicklist->len = 0; + quicklist->count = 0; + return quicklist; +} + +static quicklistNode *quicklistCreateNode(void) { + quicklistNode *node; + if ((node = zmalloc(sizeof(*node))) == NULL) + return NULL; + node->zl = NULL; + node->count = 0; + node->next = node->prev = NULL; + return node; +} + +/* Return cached quicklist count */ +unsigned int quicklistCount(quicklist *ql) { return ql->count; } + +/* Free entire quicklist. */ +void quicklistRelease(quicklist *quicklist) { + unsigned long len; + quicklistNode *current, *next; + + current = quicklist->head; + len = quicklist->len; + while (len--) { + next = current->next; + + zfree(current->zl); + quicklist->count -= current->count; + + zfree(current); + + quicklist->len--; + current = next; + } + zfree(quicklist); +} + +/* Insert 'new_node' after 'old_node' if 'after' is 1. + * Insert 'new_node' before 'old_node' if 'after' is 0. */ +static void __quicklistInsertNode(quicklist *quicklist, quicklistNode *old_node, + quicklistNode *new_node, int after) { + if (after) { + new_node->prev = old_node; + if (old_node) { + new_node->next = old_node->next; + if (old_node->next) + old_node->next->prev = new_node; + old_node->next = new_node; + } + if (quicklist->tail == old_node) + quicklist->tail = new_node; + } else { + new_node->next = old_node; + if (old_node) { + new_node->prev = old_node->prev; + if (old_node->prev) + old_node->prev->next = new_node; + old_node->prev = new_node; + } + if (quicklist->head == old_node) + quicklist->head = new_node; + } + /* If this insert creates the first element in this quicklist, we + * need to initialize head/tail too. */ + if (quicklist->len == 0) { + quicklist->head = quicklist->tail = new_node; + } + quicklist->len++; +} + +/* Wrappers for node inserting around existing node. */ +static void _quicklistInsertNodeBefore(quicklist *quicklist, + quicklistNode *old_node, + quicklistNode *new_node) { + __quicklistInsertNode(quicklist, old_node, new_node, 0); +} + +static void _quicklistInsertNodeAfter(quicklist *quicklist, + quicklistNode *old_node, + quicklistNode *new_node) { + __quicklistInsertNode(quicklist, old_node, new_node, 1); +} + +/* Add new entry to head of quicklist. + * + * Returns 'quicklist' argument. */ +quicklist *quicklistPushHead(quicklist *quicklist, const size_t fill, + void *value, size_t sz) { + if (quicklist->head && quicklist->head->count < fill) { + quicklist->head->zl = + ziplistPush(quicklist->head->zl, value, sz, ZIPLIST_HEAD); + } else { + quicklistNode *node = quicklistCreateNode(); + node->zl = ziplistPush(ziplistNew(), value, sz, ZIPLIST_HEAD); + + _quicklistInsertNodeBefore(quicklist, quicklist->head, node); + } + quicklist->count++; + quicklist->head->count++; + return quicklist; +} + +/* Add new node to tail of quicklist. + * + * Returns 'quicklist' argument. */ +quicklist *quicklistPushTail(quicklist *quicklist, const size_t fill, + void *value, size_t sz) { + if (quicklist->tail && quicklist->tail->count < fill) { + quicklist->tail->zl = + ziplistPush(quicklist->tail->zl, value, sz, ZIPLIST_TAIL); + } else { + quicklistNode *node = quicklistCreateNode(); + node->zl = ziplistPush(ziplistNew(), value, sz, ZIPLIST_TAIL); + + _quicklistInsertNodeAfter(quicklist, quicklist->tail, node); + } + quicklist->count++; + quicklist->tail->count++; + return quicklist; +} + +/* Create new node consisting of a pre-formed ziplist. + * Used for loading RDBs where entire ziplists have been stored + * to be retrieved later. */ +void quicklistAppendZiplist(quicklist *quicklist, unsigned char *zl) { + quicklistNode *node = quicklistCreateNode(); + + node->zl = zl; + node->count = ziplistLen(node->zl); + + _quicklistInsertNodeAfter(quicklist, quicklist->tail, node); + quicklist->count += node->count; +} + +/* Append all values of ziplist 'zl' individually into 'quicklist'. + * + * This allows us to restore old RDB ziplists into new quicklists + * with smaller ziplist sizes than the saved RDB ziplist. + * + * Returns 'quicklist' argument. Frees passed-in ziplist 'zl' */ +quicklist *quicklistAppendValuesFromZiplist(quicklist *quicklist, + const size_t fill, + unsigned char *zl) { + unsigned char *value; + unsigned int sz; + long long longval; + char longstr[32] = { 0 }; + + unsigned char *p = ziplistIndex(zl, 0); + while (ziplistGet(p, &value, &sz, &longval)) { + if (!value) { + /* Write the longval as a string so we can re-add it */ + sz = ll2string(longstr, sizeof(longstr), longval); + value = (unsigned char *)longstr; + } + quicklistPushTail(quicklist, fill, value, sz); + p = ziplistNext(zl, p); + } + zfree(zl); + return quicklist; +} + +/* Create new (potentially multi-node) quicklist from a single existing ziplist. + * + * Returns new quicklist. Frees passed-in ziplist 'zl'. */ +quicklist *quicklistCreateFromZiplist(size_t fill, unsigned char *zl) { + return quicklistAppendValuesFromZiplist(quicklistCreate(), fill, zl); +} + +#define quicklistDeleteIfEmpty(ql, n) \ + do { \ + if ((n)->count == 0) { \ + __quicklistDelNode((ql), (n)); \ + } \ + } while (0) + +static void __quicklistDelNode(quicklist *quicklist, quicklistNode *node) { + if (node->next) + node->next->prev = node->prev; + if (node->prev) + node->prev->next = node->next; + + if (node == quicklist->tail) + quicklist->tail = node->prev; + + if (node == quicklist->head) + quicklist->head = node->next; + + quicklist->count -= node->count; + + zfree(node->zl); + zfree(node); + quicklist->len--; +} + +/* Delete one entry from list given the node for the entry and a pointer + * to the entry in the node. + * + * Returns 1 if the entire node was deleted, 0 if node still exists. + * Also updates in/out param 'p' with the next offset in the ziplist. */ +static int quicklistDelIndex(quicklist *quicklist, quicklistNode *node, + unsigned char **p) { + int gone = 0; + node->zl = ziplistDelete(node->zl, p); + node->count--; + if (node->count == 0) + gone = 1; + quicklist->count--; + quicklistDeleteIfEmpty(quicklist, node); + /* If we deleted all the nodes, our returned pointer is no longer valid */ + return gone ? 1 : 0; +} + +/* Delete one element represented by 'entry' + * + * 'entry' stores enough metadata to delete the proper position in + * the correct ziplist in the correct quicklist node. */ +void quicklistDelEntry(quicklistIter *iter, quicklistEntry *entry) { + quicklistNode *prev = entry->node->prev; + quicklistNode *next = entry->node->next; + int deleted_node = quicklistDelIndex((quicklist *)entry->quicklist, + entry->node, &entry->zi); + + /* after delete, the zi is now invalid for any future usage. */ + iter->zi = NULL; + + /* If current node is deleted, we must update iterator node and offset. */ + if (deleted_node) { + if (iter->direction == AL_START_HEAD) { + iter->current = next; + iter->offset = 0; + } else if (iter->direction == AL_START_TAIL) { + iter->current = prev; + iter->offset = -1; + } + } + /* else if (!deleted_node), no changes needed. + * we already reset iter->zi above, and the existing iter->offset + * doesn't move again because: + * - [1, 2, 3] => delete offset 1 => [1, 3]: next element still offset 1 + * - [1, 2, 3] => delete offset 0 => [2, 3]: next element still offset 0 + * if we deleted the last element at offet N and now + * length of this ziplist is N-1, the next call into + * quicklistNext() will jump to the next node. */ +} + +/* Replace quicklist entry at offset 'index' by 'data' with length 'sz'. + * + * Returns 1 if replace happened. + * Returns 0 if replace failed and no changes happened. */ +int quicklistReplaceAtIndex(quicklist *quicklist, long index, void *data, + int sz) { + quicklistEntry entry; + if (quicklistIndex(quicklist, index, &entry)) { + entry.node->zl = ziplistDelete(entry.node->zl, &entry.zi); + entry.node->zl = ziplistInsert(entry.node->zl, entry.zi, data, sz); + return 1; + } else { + return 0; + } +} + +/* Given two nodes, try to merge their ziplists. + * + * This helps us not have a quicklist with 3 element ziplists if + * our fill factor can handle much higher levels. + * + * Note: 'a' must be to the LEFT of 'b'. + * + * After calling this function, both 'a' and 'b' should be considered + * unusable. The return value from this function must be used + * instead of re-using any of the quicklistNode input arguments. + * + * Returns the input node picked to merge against or NULL if + * merging was not possible. */ +static quicklistNode *_quicklistZiplistMerge(quicklist *quicklist, + quicklistNode *a, + quicklistNode *b) { + /* Merge into node with largest initial count */ + quicklistNode *target = a->count > b->count ? a : b; + + if (a->count == 0 || b->count == 0) + return NULL; + + D("Requested merge (a,b) (%u, %u) and picked target %u", a->count, b->count, + target->count); + + int where; + unsigned char *p = NULL; + if (target == a) { + /* If target is node a, we append node b to node a, in-order */ + where = ZIPLIST_TAIL; + p = ziplistIndex(b->zl, 0); + D("WILL TRAVERSE B WITH LENGTH: %u, %u", b->count, ziplistLen(b->zl)); + } else { + /* If target b, we prepend node a to node b, in reverse order of a */ + where = ZIPLIST_HEAD; + p = ziplistIndex(a->zl, -1); + D("WILL TRAVERSE A WITH LENGTH: %u, %u", a->count, ziplistLen(a->zl)); + } + + unsigned char *val; + unsigned int sz; + long long longval; + char lv[32] = { 0 }; + /* NOTE: We could potentially create a built-in ziplist operation + * allowing direct merging of two ziplists. It would be more memory + * efficient (one big realloc instead of incremental), but it's more + * complex than using the existing ziplist API to read/push as below. */ + while (ziplistGet(p, &val, &sz, &longval)) { + if (!val) { + sz = ll2string(lv, sizeof(lv), longval); + val = (unsigned char *)lv; + } + target->zl = ziplistPush(target->zl, val, sz, where); + if (target == a) { + p = ziplistNext(b->zl, p); + b->count--; + a->count++; + } else { + p = ziplistPrev(a->zl, p); + a->count--; + b->count++; + } + D("Loop A: %u, B: %u", a->count, b->count); + } + + /* At this point, target is populated and not-target needs + * to be free'd and removed from the quicklist. */ + if (target == a) { + D("Deleting node B with current count: %d", b->count); + __quicklistDelNode(quicklist, b); + } else if (target == b) { + D("Deleting node A with current count: %d", a->count); + __quicklistDelNode(quicklist, a); + } + return target; +} + +/* Attempt to merge ziplists within two nodes on either side of 'center'. + * + * We attempt to merge: + * - (center->prev->prev, center->prev) + * - (center->next, center->next->next) + * - (center->prev, center) + * - (center, center->next) + */ +static void _quicklistMergeNodes(quicklist *quicklist, const size_t fill, + quicklistNode *center) { + quicklistNode *prev, *prev_prev, *next, *next_next, *target; + prev = prev_prev = next = next_next = target = NULL; + + if (center->prev) { + prev = center->prev; + if (center->prev->prev) + prev_prev = center->prev->prev; + } + + if (center->next) { + next = center->next; + if (center->next->next) + next_next = center->next->next; + } + + /* Try to merge prev_prev and prev */ + if (prev && prev_prev && (prev->count + prev_prev->count) <= fill) { + _quicklistZiplistMerge(quicklist, prev_prev, prev); + prev_prev = prev = NULL; /* they could have moved, invalidate them. */ + } + + /* Try to merge next and next_next */ + if (next && next_next && (next->count + next_next->count) <= fill) { + _quicklistZiplistMerge(quicklist, next, next_next); + next = next_next = NULL; /* they could have moved, invalidate them. */ + } + + /* Try to merge center node and previous node */ + if (center->prev && (center->count + center->prev->count) <= fill) { + target = _quicklistZiplistMerge(quicklist, center->prev, center); + center = NULL; /* center could have been deleted, invalidate it. */ + } else { + /* else, we didn't merge here, but target needs to be valid below. */ + target = center; + } + + /* Use result of center merge (or original) to merge with next node. */ + if (target && target->next && + (target->count + target->next->count) <= fill) { + _quicklistZiplistMerge(quicklist, target, target->next); + } +} + +/* Split 'node' into two parts, parameterized by 'offset' and 'after'. + * + * The 'after' argument controls which quicklistNode gets returned. + * If 'after'==1, returned node has elements after 'offset'. + * input node keeps elements up to 'offset', including 'offset'. + * If 'after'==0, returned node has elements up to 'offset', including 'offset'. + * input node keeps elements after 'offset'. + * + * If 'after'==1, returned node will have elements _after_ 'offset'. + * The returned node will have elements [OFFSET+1, END]. + * The input node keeps elements [0, OFFSET]. + * + * If 'after'==0, returned node will keep elements up to and including 'offset'. + * The returned node will have elements [0, OFFSET]. + * The input node keeps elements [OFFSET+1, END]. + * + * The input node keeps all elements not taken by the returned node. + * + * Returns newly created node or NULL if split not possible. */ +static quicklistNode *_quicklistSplitNode(quicklistNode *node, int offset, + int after) { + size_t zl_sz = ziplistBlobLen(node->zl); + + quicklistNode *new_node = quicklistCreateNode(); + if (!new_node) + return NULL; + + new_node->zl = zmalloc(zl_sz); + if (!new_node->zl) { + zfree(new_node); + return NULL; + } + + /* Copy original ziplist so we can split it */ + memcpy(new_node->zl, node->zl, zl_sz); + + /* -1 here means "continue deleting until the list ends" */ + int orig_start = after ? offset + 1 : 0; + int orig_extent = after ? -1 : offset; + int new_start = after ? 0 : offset; + int new_extent = after ? offset + 1 : -1; + + D("After %d (%d); ranges: [%d, %d], [%d, %d]", after, offset, orig_start, + orig_extent, new_start, new_extent); + + node->zl = ziplistDeleteRange(node->zl, orig_start, orig_extent); + node->count = ziplistLen(node->zl); + + new_node->zl = ziplistDeleteRange(new_node->zl, new_start, new_extent); + new_node->count = ziplistLen(new_node->zl); + + D("After split lengths: orig (%d), new (%d)", node->count, new_node->count); + return new_node; +} + +/* Insert a new entry before or after existing entry 'entry'. + * + * If after==1, the new value is inserted after 'entry', otherwise + * the new value is inserted before 'entry'. */ +static void _quicklistInsert(quicklist *quicklist, const size_t fill, + quicklistEntry *entry, void *value, + const size_t sz, int after) { + int full = 0, at_tail = 0, at_head = 0, full_next = 0, full_prev = 0; + quicklistNode *node = entry->node; + quicklistNode *new_node = NULL; + + if (!node) { + /* we have no reference node, so let's create only node in the list */ + D("No node given!"); + new_node = quicklistCreateNode(); + new_node->zl = ziplistPush(ziplistNew(), value, sz, ZIPLIST_HEAD); + __quicklistInsertNode(quicklist, NULL, new_node, after); + new_node->count++; + quicklist->count++; + return; + } + + /* Populate accounting flags for easier boolean checks later */ + if (node->count >= fill) { + D("Current node is full with count %d with requested fill %lu", + node->count, fill); + full = 1; + } + + if (after && (ziplistNext(node->zl, entry->zi) == NULL)) { + D("At Tail of current ziplist"); + at_tail = 1; + if (node->next && node->next->count >= fill) { + D("Next node is full too."); + full_next = 1; + } + } + + if (!after && (ziplistPrev(node->zl, entry->zi) == NULL)) { + D("At Head"); + at_head = 1; + if (node->prev && node->prev->count >= fill) { + D("Prev node is full too."); + full_prev = 1; + } + } + + /* Now determine where and how to insert the new element */ + if (!full && after) { + D("Not full, inserting after current position."); + unsigned char *next = ziplistNext(node->zl, entry->zi); + if (next == NULL) { + node->zl = ziplistPush(node->zl, value, sz, ZIPLIST_TAIL); + } else { + node->zl = ziplistInsert(node->zl, next, value, sz); + } + node->count++; + } else if (!full && !after) { + D("Not full, inserting before current position."); + node->zl = ziplistInsert(node->zl, entry->zi, value, sz); + node->count++; + } else if (full && at_tail && node->next && !full_next && after) { + /* If we are: at tail, next has free space, and inserting after: + * - insert entry at head of next node. */ + D("Full and tail, but next isn't full; inserting next node head"); + new_node = node->next; + new_node->zl = ziplistPush(new_node->zl, value, sz, ZIPLIST_HEAD); + new_node->count++; + } else if (full && at_head && node->prev && !full_prev && !after) { + /* If we are: at head, previous has free space, and inserting before: + * - insert entry at tail of previous node. */ + D("Full and head, but prev isn't full, inserting prev node tail"); + new_node = node->prev; + new_node->zl = ziplistPush(new_node->zl, value, sz, ZIPLIST_TAIL); + new_node->count++; + } else if (full && ((at_tail && node->next && full_next && after) || + (at_head && node->prev && full_prev && !after))) { + /* If we are: full, and our prev/next is full, then: + * - create new node and attach to quicklist */ + D("\tprovisioning new node..."); + new_node = quicklistCreateNode(); + new_node->zl = ziplistPush(ziplistNew(), value, sz, ZIPLIST_HEAD); + new_node->count++; + __quicklistInsertNode(quicklist, node, new_node, after); + } else if (full) { + /* else, node is full we need to split it. */ + /* covers both after and !after cases */ + D("\tsplitting node..."); + new_node = _quicklistSplitNode(node, entry->offset, after); + new_node->zl = ziplistPush(new_node->zl, value, sz, + after ? ZIPLIST_HEAD : ZIPLIST_TAIL); + new_node->count++; + __quicklistInsertNode(quicklist, node, new_node, after); + _quicklistMergeNodes(quicklist, fill, node); + } + + quicklist->count++; +} + +void quicklistInsertBefore(quicklist *quicklist, const size_t fill, + quicklistEntry *entry, void *value, + const size_t sz) { + _quicklistInsert(quicklist, fill, entry, value, sz, 0); +} + +void quicklistInsertAfter(quicklist *quicklist, const size_t fill, + quicklistEntry *entry, void *value, const size_t sz) { + _quicklistInsert(quicklist, fill, entry, value, sz, 1); +} + +/* Delete a range of elements from the quicklist. + * + * elements may span across multiple quicklistNodes, so we + * have to be careful about tracking where we start and end. + * + * Returns 1 if entries were deleted, 0 if nothing was deleted. */ +int quicklistDelRange(quicklist *quicklist, const long start, + const long count) { + if (count <= 0) + return 0; + + unsigned long extent = count; /* range is inclusive of start position */ + + if (start >= 0 && extent > (quicklist->count - start)) { + /* if requesting delete more elements than exist, limit to list size. */ + extent = quicklist->count - start; + } else if (start < 0 && extent > (unsigned long)(-start)) { + /* else, if at negative offset, limit max size to rest of list. */ + extent = -start; /* c.f. LREM -29 29; just delete until end. */ + } + + quicklistEntry entry; + if (!quicklistIndex(quicklist, start, &entry)) + return 0; + + D("Quicklist delete request for start %ld, count %ld, extent: %ld", start, + count, extent); + quicklistNode *node = entry.node; + + /* iterate over next nodes until everything is deleted. */ + while (extent) { + quicklistNode *next = node->next; + + unsigned long del; + int delete_entire_node = 0; + if (entry.offset == 0 && extent >= node->count) { + /* If we are deleting more than the count of this node, we + * can just delete the entire node without ziplist math. */ + delete_entire_node = 1; + del = node->count; + } else if (entry.offset >= 0 && extent >= node->count) { + /* If deleting more nodes after this one, calculate delete based + * on size of current node. */ + del = node->count - entry.offset; + } else if (entry.offset < 0) { + /* If offset is negative, we are in the first run of this loop + * and we are deleting the entire range + * from this start offset to end of list. Since the Negative + * offset is the number of elements until the tail of the list, + * just use it directly as the deletion count. */ + del = -entry.offset; + + /* If the positive offset is greater than the remaining extent, + * we only delete the remaining extent, not the entire offset. + */ + if (del > extent) + del = extent; + } else { + /* else, we are deleting less than the extent of this node, so + * use extent directly. */ + del = extent; + } + + D("[%ld]: asking to del: %ld because offset: %d; (ENTIRE NODE: %d), " + "node count: %u", + extent, del, entry.offset, delete_entire_node, node->count); + + if (delete_entire_node) { + __quicklistDelNode(quicklist, node); + } else { + node->zl = ziplistDeleteRange(node->zl, entry.offset, del); + node->count -= del; + quicklist->count -= del; + quicklistDeleteIfEmpty(quicklist, node); + } + + extent -= del; + + node = next; + + entry.offset = 0; + } + return 1; +} + +/* Passthrough to ziplistCompare() */ +int quicklistCompare(unsigned char *p1, unsigned char *p2, int p2_len) { + return ziplistCompare(p1, p2, p2_len); +} + +/* Returns a quicklist iterator 'iter'. After the initialization every + * call to quicklistNext() will return the next element of the quicklist. */ +quicklistIter *quicklistGetIterator(const quicklist *quicklist, int direction) { + quicklistIter *iter; + + if ((iter = zmalloc(sizeof(*iter))) == NULL) + return NULL; + + if (direction == AL_START_HEAD) { + iter->current = quicklist->head; + iter->offset = 0; + } else if (direction == AL_START_TAIL) { + iter->current = quicklist->tail; + iter->offset = -1; + } + + iter->direction = direction; + iter->quicklist = quicklist; + + iter->zi = NULL; + + return iter; +} + +/* Initialize an iterator at a specific offset 'idx' and make the iterator + * return nodes in 'direction' direction. */ +quicklistIter *quicklistGetIteratorAtIdx(const quicklist *quicklist, + const int direction, + const long long idx) { + quicklistEntry entry; + + if (quicklistIndex(quicklist, idx, &entry)) { + quicklistIter *base = quicklistGetIterator(quicklist, direction); + base->zi = NULL; + base->current = entry.node; + base->offset = entry.offset; + return base; + } else { + return NULL; + } +} + +/* Release iterator */ +void quicklistReleaseIterator(quicklistIter *iter) { zfree(iter); } + +/* Get next element in iterator. + * + * Note: You must NOT insert into the list while iterating over it. + * You *may* delete from the list while iterating using the + * quicklistDelEntry() function. + * If you insert into the quicklist while iterating, you should + * re-create the iterator after your addition. + * + * iter = quicklistGetIterator(quicklist,); + * quicklistEntry entry; + * while (quicklistNext(iter, &entry)) { + * if (entry.value) + * [[ use entry.value with entry.sz ]] + * else + * [[ use entry.longval ]] + * } + * + * Populates 'entry' with values for this iteration. + * Returns 0 when iteration is complete or if iteration not possible. + * If return value is 0, the contents of 'entry' are not valid. + */ +int quicklistNext(quicklistIter *iter, quicklistEntry *entry) { + initEntry(entry); + + if (!iter) { + D("Returning because no iter!"); + return 0; + } + + entry->quicklist = iter->quicklist; + entry->node = iter->current; + + if (!iter->current) { + D("Returning because current node is NULL") + return 0; + } + + unsigned char *(*nextFn)(unsigned char *, unsigned char *) = NULL; + int offset_update = 0; + + if (!iter->zi) { + /* If !zi, use current index. */ + iter->zi = ziplistIndex(iter->current->zl, iter->offset); + } else { + /* else, use existing iterator offset and get prev/next as necessary. */ + if (iter->direction == AL_START_HEAD) { + nextFn = ziplistNext; + offset_update = 1; + } else if (iter->direction == AL_START_TAIL) { + nextFn = ziplistPrev; + offset_update = -1; + } + iter->zi = nextFn(iter->current->zl, iter->zi); + iter->offset += offset_update; + } + + entry->zi = iter->zi; + entry->offset = iter->offset; + + if (iter->zi) { + /* Populate value from existing ziplist position */ + ziplistGet(entry->zi, &entry->value, &entry->sz, &entry->longval); + return 1; + } else { + /* We ran out of ziplist entries. + * Pick next node, update offset, then re-run retrieval. */ + if (iter->direction == AL_START_HEAD) { + /* Forward traversal */ + D("Jumping to start of next node"); + iter->current = iter->current->next; + iter->offset = 0; + } else if (iter->direction == AL_START_TAIL) { + /* Reverse traversal */ + D("Jumping to end of previous node"); + iter->current = iter->current->prev; + iter->offset = -1; + } + iter->zi = NULL; + return quicklistNext(iter, entry); + } +} + +/* Duplicate the quicklist. On out of memory NULL is returned. + * On success a copy of the original quicklist is returned. + * + * The original quicklist both on success or error is never modified. + * + * Returns newly allocated quicklist. */ +quicklist *quicklistDup(quicklist *orig) { + quicklist *copy; + int failure = 0; + + if ((copy = quicklistCreate()) == NULL) + return NULL; + + for (quicklistNode *current = orig->head; current; + current = current->next) { + quicklistNode *node = quicklistCreateNode(); + + size_t ziplen = ziplistBlobLen(current->zl); + if ((node->zl = zmalloc(ziplen)) == NULL) { + failure = 1; + break; + } + memcpy(node->zl, current->zl, ziplen); + + node->count = current->count; + copy->count += node->count; + + _quicklistInsertNodeAfter(copy, copy->tail, node); + } + + /* copy->count must equal orig->count here */ + + if (failure) { + quicklistRelease(copy); + return NULL; + } + + return copy; +} + +/* Populate 'entry' with the element at the specified zero-based index + * where 0 is the head, 1 is the element next to head + * and so on. Negative integers are used in order to count + * from the tail, -1 is the last element, -2 the penultimate + * and so on. If the index is out of range 0 is returned. + * + * Returns 1 if element found + * Returns 0 if element not found */ +int quicklistIndex(const quicklist *quicklist, const long long idx, + quicklistEntry *entry) { + quicklistNode *n; + unsigned long long accum = 0; + unsigned long long index; + int forward = idx < 0 ? 0 : 1; /* < 0 -> reverse, 0+ -> forward */ + + initEntry(entry); + entry->quicklist = quicklist; + + if (!forward) { + index = (-idx) - 1; + n = quicklist->tail; + } else { + index = idx; + n = quicklist->head; + } + + if (index >= quicklist->count) + return 0; + + while (n) { + if ((accum + n->count) > index) { + break; + } else { + D("Skipping over (%p) %u at accum %lld", (void *)n, n->count, + accum); + accum += n->count; + n = forward ? n->next : n->prev; + } + } + + if (!n) + return 0; + + D("Found node: %p at accum %llu, idx %llu, sub+ %llu, sub- %llu", (void *)n, + accum, index, index - accum, (-index) - 1 + accum); + + entry->node = n; + if (forward) { + /* forward = normal head-to-tail offset. */ + entry->offset = index - accum; + } else { + /* reverse = need negative offset for tail-to-head, so undo + * the result of the original if (index < 0) above. */ + entry->offset = (-index) - 1 + accum; + } + + entry->zi = ziplistIndex(entry->node->zl, entry->offset); + ziplistGet(entry->zi, &entry->value, &entry->sz, &entry->longval); + return 1; +} + +/* Rotate quicklist by moving the tail element to the head. */ +void quicklistRotate(quicklist *quicklist, const size_t fill) { + if (quicklist->count <= 1) + return; + + /* First, get the tail entry */ + quicklistNode *tail = quicklist->tail; + unsigned char *p = ziplistIndex(tail->zl, -1); + unsigned char *value; + long long longval; + unsigned int sz; + char longstr[32] = { 0 }; + ziplistGet(p, &value, &sz, &longval); + + /* If value found is NULL, then ziplistGet populated longval instead */ + if (!value) { + /* Write the longval as a string so we can re-add it */ + sz = ll2string(longstr, sizeof(longstr), longval); + value = (unsigned char *)longstr; + } + + /* Add tail entry to head (must happen before tail is deleted). */ + quicklistPushHead(quicklist, fill, value, sz); + + /* If quicklist has only one node, the head ziplist is also the + * tail ziplist and PushHead() could have reallocated our single ziplist, + * which would make our pre-existing 'p' unusable. */ + if (quicklist->len == 1) + p = ziplistIndex(tail->zl, -1); + + /* Remove tail entry. */ + quicklistDelIndex(quicklist, tail, &p); +} + +/* pop from quicklist and return result in 'data' ptr. Value of 'data' + * is the return value of 'saver' function pointer if the data is NOT a number. + * + * If the quicklist element is a long long, then the return value is returned in + * 'sval'. + * + * Return value of 0 means no elements available. + * Return value of 1 means check 'data' and 'sval' for values. + * If 'data' is set, use 'data' and 'sz'. Otherwise, use 'sval'. */ +int quicklistPopCustom(quicklist *quicklist, int where, unsigned char **data, + unsigned int *sz, long long *sval, + void *(*saver)(unsigned char *data, unsigned int sz)) { + unsigned char *p; + unsigned char *vstr; + unsigned int vlen; + long long vlong; + int pos = (where == QUICKLIST_HEAD) ? 0 : -1; + + if (quicklist->count == 0) + return 0; + + if (data) + *data = NULL; + if (sz) + *sz = 0; + if (sval) + *sval = -123456789; + + quicklistNode *node; + if (where == QUICKLIST_HEAD && quicklist->head) { + node = quicklist->head; + } else if (where == QUICKLIST_TAIL && quicklist->tail) { + node = quicklist->tail; + } else { + return 0; + } + + p = ziplistIndex(node->zl, pos); + if (ziplistGet(p, &vstr, &vlen, &vlong)) { + if (vstr) { + if (data) + *data = saver(vstr, vlen); + if (sz) + *sz = vlen; + } else { + if (data) + *data = NULL; + if (sval) + *sval = vlong; + } + quicklistDelIndex(quicklist, node, &p); + return 1; + } + return 0; +} + +/* Return a malloc'd copy of data passed in */ +static void *_quicklistSaver(unsigned char *data, unsigned int sz) { + unsigned char *vstr; + if (data) { + if ((vstr = zmalloc(sz)) == NULL) + return 0; + memcpy(data, vstr, sz); + return vstr; + } + return NULL; +} + +/* Default pop function + * + * Returns malloc'd value from quicklist */ +int quicklistPop(quicklist *quicklist, int where, unsigned char **data, + unsigned int *sz, long long *slong) { + unsigned char *vstr; + unsigned int vlen; + long long vlong; + if (quicklist->count == 0) + return 0; + int ret = quicklistPopCustom(quicklist, where, &vstr, &vlen, &vlong, + _quicklistSaver); + if (data) + *data = vstr; + if (slong) + *slong = vlong; + if (sz) + *sz = vlen; + return ret; +} + +/* Wrapper to allow argument-based switching between HEAD/TAIL pop */ +void quicklistPush(quicklist *quicklist, const size_t fill, void *value, + const size_t sz, int where) { + if (where == QUICKLIST_HEAD) { + quicklistPushHead(quicklist, fill, value, sz); + } else if (where == QUICKLIST_TAIL) { + quicklistPushTail(quicklist, fill, value, sz); + } +} + +/* The rest of this file is test cases and test helpers. */ +#ifdef REDIS_TEST +#include + +#define assert(_e) \ + do { \ + if (!(_e)) { \ + printf("\n\n=== ASSERTION FAILED ===\n"); \ + printf("==> %s:%d '%s' is not true\n", __FILE__, __LINE__, #_e); \ + err++; \ + } \ + } while (0) + +#define yell(str, ...) printf("ERROR! " str "\n\n", __VA_ARGS__) + +#define OK printf("\tOK\n") + +#define ERROR \ + do { \ + printf("\tERROR!\n"); \ + err++; \ + } while (0) + +#define ERR(x, ...) \ + do { \ + printf("%s:%s:%d:\t", __FILE__, __FUNCTION__, __LINE__); \ + printf("ERROR! " x "\n", __VA_ARGS__); \ + err++; \ + } while (0) + +#define TEST(name) printf("test — %s\n", name); +#define TEST_DESC(name, ...) printf("test — " name "\n", __VA_ARGS__); + +#define QL_TEST_VERBOSE 0 + +#define UNUSED(x) (void)(x) +static void ql_info(quicklist *ql) { +#if QL_TEST_VERBOSE + printf("Container length: %lu\n", ql->len); + printf("Container size: %lu\n", ql->count); + if (ql->head) + printf("\t(zsize head: %d)\n", ziplistLen(ql->head->zl)); + if (ql->tail) + printf("\t(zsize tail: %d)\n", ziplistLen(ql->tail->zl)); + printf("\n"); +#else + UNUSED(ql); +#endif +} + +/* Iterate over an entire quicklist. + * Print the list if 'print' == 1. + * + * Returns physical count of elements found by iterating over the list. */ +static int _itrprintr(quicklist *ql, int print, int forward) { + quicklistIter *iter = + quicklistGetIterator(ql, forward ? AL_START_HEAD : AL_START_TAIL); + quicklistEntry entry; + int i = 0; + int p = 0; + quicklistNode *prev = NULL; + while (quicklistNext(iter, &entry)) { + if (entry.node != prev) { + /* Count the number of list nodes too */ + p++; + prev = entry.node; + } + if (print) { + printf("[%3d (%2d)]: [%.*s] (%lld)\n", i, p, entry.sz, + (char *)entry.value, entry.longval); + } + i++; + } + quicklistReleaseIterator(iter); + return i; +} +static int itrprintr(quicklist *ql, int print) { + return _itrprintr(ql, print, 1); +} + +static int itrprintr_rev(quicklist *ql, int print) { + return _itrprintr(ql, print, 0); +} + +#define ql_verify(a, b, c, d, e) \ + do { \ + err += _ql_verify((a), (b), (c), (d), (e)); \ + } while (0) + +/* Verify list metadata matches physical list contents. */ +static int _ql_verify(quicklist *ql, uint32_t len, uint32_t count, + uint32_t head_count, uint32_t tail_count) { + int ok = 1; + + ql_info(ql); + if (len != ql->len) { + yell("quicklist length wrong: expected %d, got %lu", len, ql->len); + ok = 0; + } + + if (count != ql->count) { + yell("quicklist count wrong: expected %d, got %lu", count, ql->count); + ok = 0; + } + + int loopr = itrprintr(ql, 0); + if (loopr != (int)ql->count) { + yell("quicklist cached count not match actual count: expected %lu, got " + "%d", + ql->count, loopr); + ok = 0; + } + + int rloopr = itrprintr_rev(ql, 0); + if (loopr != rloopr) { + yell("quicklist has different forward count than reverse count! " + "Forward count is %d, reverse count is %d.", + loopr, rloopr); + ok = 0; + } + + if (ql->len == 0 && ok) { + OK; + return !ok; + } + + if (head_count != ql->head->count && + head_count != ziplistLen(ql->head->zl)) { + yell("quicklist head count wrong: expected %d, " + "got cached %d vs. actual %d", + head_count, ql->head->count, ziplistLen(ql->head->zl)); + ok = 0; + } + + if (tail_count != ql->tail->count && + tail_count != ziplistLen(ql->tail->zl)) { + yell("quicklist tail count wrong: expected %d, " + "got cached %u vs. actual %d", + tail_count, ql->tail->count, ziplistLen(ql->tail->zl)); + ok = 0; + } + if (ok) + OK; + return !ok; +} + +/* Generate new string concatenating integer i against string 'prefix' */ +static char *genstr(char *prefix, int i) { + static char result[64] = { 0 }; + snprintf(result, sizeof(result), "%s%d", prefix, i); + return result; +} + +/* Test fill cap */ +#define F 32 +/* main test, but callable from other files */ +int quicklistTest(int argc, char *argv[]) { + unsigned int err = 0; + + UNUSED(argc); + UNUSED(argv); + + TEST("create list") { + quicklist *ql = quicklistCreate(); + ql_verify(ql, 0, 0, 0, 0); + quicklistRelease(ql); + } + + TEST("add to tail of empty list") { + quicklist *ql = quicklistCreate(); + quicklistPushTail(ql, F, "hello", 6); + /* 1 for head and 1 for tail beacuse 1 node = head = tail */ + ql_verify(ql, 1, 1, 1, 1); + quicklistRelease(ql); + } + + TEST("add to head of empty list") { + quicklist *ql = quicklistCreate(); + quicklistPushHead(ql, F, "hello", 6); + /* 1 for head and 1 for tail beacuse 1 node = head = tail */ + ql_verify(ql, 1, 1, 1, 1); + quicklistRelease(ql); + } + + for (size_t f = 0; f < 32; f++) { + TEST_DESC("add to tail 5x at fill %lu", f) { + quicklist *ql = quicklistCreate(); + for (int i = 0; i < 5; i++) + quicklistPushTail(ql, f, genstr("hello", i), 32); + if (ql->count != 5) + ERROR; + if (f == 32) + ql_verify(ql, 1, 5, 5, 5); + quicklistRelease(ql); + } + } + + for (size_t f = 0; f < 32; f++) { + TEST_DESC("add to head 5x at fill %lu", f) { + quicklist *ql = quicklistCreate(); + for (int i = 0; i < 5; i++) + quicklistPushHead(ql, f, genstr("hello", i), 32); + if (ql->count != 5) + ERROR; + if (f == 32) + ql_verify(ql, 1, 5, 5, 5); + quicklistRelease(ql); + } + } + + for (size_t f = 0; f < 512; f++) { + TEST_DESC("add to tail 500x at fill %lu", f) { + quicklist *ql = quicklistCreate(); + for (int i = 0; i < 500; i++) + quicklistPushTail(ql, f, genstr("hello", i), 32); + if (ql->count != 500) + ERROR; + if (f == 32) + ql_verify(ql, 16, 500, 32, 20); + quicklistRelease(ql); + } + } + + for (size_t f = 0; f < 512; f++) { + TEST_DESC("add to head 500x at fill %lu", f) { + quicklist *ql = quicklistCreate(); + for (int i = 0; i < 500; i++) + quicklistPushHead(ql, f, genstr("hello", i), 32); + if (ql->count != 500) + ERROR; + if (f == 32) + ql_verify(ql, 16, 500, 20, 32); + quicklistRelease(ql); + } + } + + TEST("rotate empty") { + quicklist *ql = quicklistCreate(); + quicklistRotate(ql, F); + ql_verify(ql, 0, 0, 0, 0); + quicklistRelease(ql); + } + + for (size_t f = 0; f < 32; f++) { + TEST("rotate one val once") { + quicklist *ql = quicklistCreate(); + quicklistPushHead(ql, F, "hello", 6); + quicklistRotate(ql, F); + ql_verify(ql, 1, 1, 1, 1); + quicklistRelease(ql); + } + } + + for (size_t f = 0; f < 1024; f++) { + TEST_DESC("rotate 500 val 5000 times at fill %lu", f) { + quicklist *ql = quicklistCreate(); + quicklistPushHead(ql, f, "900", 3); + quicklistPushHead(ql, f, "7000", 4); + quicklistPushHead(ql, f, "-1200", 5); + quicklistPushHead(ql, f, "42", 2); + for (int i = 0; i < 500; i++) + quicklistPushHead(ql, f, genstr("hello", i), 32); + ql_info(ql); + for (int i = 0; i < 5000; i++) { + ql_info(ql); + quicklistRotate(ql, f); + } + if (f == 1) + ql_verify(ql, 504, 504, 1, 1); + else if (f == 2) + ql_verify(ql, 252, 504, 2, 2); + else if (f == 32) + ql_verify(ql, 16, 504, 32, 24); + quicklistRelease(ql); + } + } + + TEST("pop empty") { + quicklist *ql = quicklistCreate(); + quicklistPop(ql, QUICKLIST_HEAD, NULL, NULL, NULL); + ql_verify(ql, 0, 0, 0, 0); + quicklistRelease(ql); + } + + TEST("pop 1 string from 1") { + quicklist *ql = quicklistCreate(); + quicklistPushHead(ql, F, genstr("hello", 331), 32); + unsigned char *data; + unsigned int sz; + long long lv; + ql_info(ql); + quicklistPop(ql, QUICKLIST_HEAD, &data, &sz, &lv); + assert(data != NULL); + assert(sz == 32); + zfree(data); + ql_verify(ql, 0, 0, 0, 0); + quicklistRelease(ql); + } + + TEST("pop head 1 number from 1") { + quicklist *ql = quicklistCreate(); + quicklistPushHead(ql, F, "55513", 5); + unsigned char *data; + unsigned int sz; + long long lv; + ql_info(ql); + quicklistPop(ql, QUICKLIST_HEAD, &data, &sz, &lv); + assert(data == NULL); + assert(lv == 55513); + ql_verify(ql, 0, 0, 0, 0); + quicklistRelease(ql); + } + + TEST("pop head 500 from 500") { + quicklist *ql = quicklistCreate(); + for (int i = 0; i < 500; i++) + quicklistPushHead(ql, F, genstr("hello", i), 32); + ql_info(ql); + for (int i = 0; i < 500; i++) { + unsigned char *data; + unsigned int sz; + long long lv; + int ret = quicklistPop(ql, QUICKLIST_HEAD, &data, &sz, &lv); + assert(ret == 1); + assert(data != NULL); + assert(sz == 32); + zfree(data); + } + ql_verify(ql, 0, 0, 0, 0); + quicklistRelease(ql); + } + + TEST("pop head 5000 from 500") { + quicklist *ql = quicklistCreate(); + for (int i = 0; i < 500; i++) + quicklistPushHead(ql, F, genstr("hello", i), 32); + for (int i = 0; i < 5000; i++) { + unsigned char *data; + unsigned int sz; + long long lv; + int ret = quicklistPop(ql, QUICKLIST_HEAD, &data, &sz, &lv); + if (i < 500) { + assert(ret == 1); + assert(data != NULL); + assert(sz == 32); + zfree(data); + } else { + assert(ret == 0); + } + } + ql_verify(ql, 0, 0, 0, 0); + quicklistRelease(ql); + } + + TEST("iterate forward over 500 list") { + quicklist *ql = quicklistCreate(); + for (int i = 0; i < 500; i++) + quicklistPushHead(ql, F, genstr("hello", i), 32); + quicklistIter *iter = quicklistGetIterator(ql, AL_START_HEAD); + quicklistEntry entry; + int i = 499, count = 0; + while (quicklistNext(iter, &entry)) { + char *h = genstr("hello", i); + if (strcmp((char *)entry.value, h)) + ERR("value [%s] didn't match [%s] at position %d", entry.value, + h, i); + i--; + count++; + } + if (count != 500) + ERR("Didn't iterate over exactly 500 elements (%d)", i); + ql_verify(ql, 16, 500, 20, 32); + quicklistReleaseIterator(iter); + quicklistRelease(ql); + } + + TEST("iterate reverse over 500 list") { + quicklist *ql = quicklistCreate(); + for (int i = 0; i < 500; i++) + quicklistPushHead(ql, F, genstr("hello", i), 32); + quicklistIter *iter = quicklistGetIterator(ql, AL_START_TAIL); + quicklistEntry entry; + int i = 0; + while (quicklistNext(iter, &entry)) { + char *h = genstr("hello", i); + if (strcmp((char *)entry.value, h)) + ERR("value [%s] didn't match [%s] at position %d", entry.value, + h, i); + i++; + } + if (i != 500) + ERR("Didn't iterate over exactly 500 elements (%d)", i); + ql_verify(ql, 16, 500, 20, 32); + quicklistReleaseIterator(iter); + quicklistRelease(ql); + } + + TEST("insert before with 0 elements") { + quicklist *ql = quicklistCreate(); + quicklistEntry entry; + quicklistIndex(ql, 0, &entry); + quicklistInsertBefore(ql, F, &entry, "abc", 4); + ql_verify(ql, 1, 1, 1, 1); + quicklistRelease(ql); + } + + TEST("insert after with 0 elements") { + quicklist *ql = quicklistCreate(); + quicklistEntry entry; + quicklistIndex(ql, 0, &entry); + quicklistInsertAfter(ql, F, &entry, "abc", 4); + ql_verify(ql, 1, 1, 1, 1); + quicklistRelease(ql); + } + + TEST("insert after 1 element") { + quicklist *ql = quicklistCreate(); + quicklistPushHead(ql, F, "hello", 6); + quicklistEntry entry; + quicklistIndex(ql, 0, &entry); + quicklistInsertAfter(ql, F, &entry, "abc", 4); + ql_verify(ql, 1, 2, 2, 2); + quicklistRelease(ql); + } + + TEST("insert before 1 element") { + quicklist *ql = quicklistCreate(); + quicklistPushHead(ql, F, "hello", 6); + quicklistEntry entry; + quicklistIndex(ql, 0, &entry); + quicklistInsertAfter(ql, F, &entry, "abc", 4); + ql_verify(ql, 1, 2, 2, 2); + quicklistRelease(ql); + } + + for (size_t f = 0; f < 12; f++) { + TEST_DESC("insert once in elements while iterating at fill %lu\n", f) { + quicklist *ql = quicklistCreate(); + quicklistPushTail(ql, f, "abc", 3); + quicklistPushTail(ql, 1, "def", 3); /* force to unique node */ + quicklistPushTail(ql, f, "bob", 3); /* force to reset for +3 */ + quicklistPushTail(ql, f, "foo", 3); + quicklistPushTail(ql, f, "zoo", 3); + + itrprintr(ql, 1); + /* insert "bar" before "bob" while iterating over list. */ + quicklistIter *iter = quicklistGetIterator(ql, AL_START_HEAD); + quicklistEntry entry; + while (quicklistNext(iter, &entry)) { + if (!strncmp((char *)entry.value, "bob", 3)) { + /* Insert as fill = 1 so it spills into new node. */ + quicklistInsertBefore(ql, f, &entry, "bar", 3); + /* NOTE! You can't continue iterating after an insert into + * the list. You *must* re-create your iterator again if + * you want to traverse all entires. */ + break; + } + } + itrprintr(ql, 1); + + /* verify results */ + quicklistIndex(ql, 0, &entry); + if (strncmp((char *)entry.value, "abc", 3)) + ERR("Value 0 didn't match, instead got: %.*s", entry.sz, + entry.value); + quicklistIndex(ql, 1, &entry); + if (strncmp((char *)entry.value, "def", 3)) + ERR("Value 1 didn't match, instead got: %.*s", entry.sz, + entry.value); + quicklistIndex(ql, 2, &entry); + if (strncmp((char *)entry.value, "bar", 3)) + ERR("Value 2 didn't match, instead got: %.*s", entry.sz, + entry.value); + quicklistIndex(ql, 3, &entry); + if (strncmp((char *)entry.value, "bob", 3)) + ERR("Value 3 didn't match, instead got: %.*s", entry.sz, + entry.value); + quicklistIndex(ql, 4, &entry); + if (strncmp((char *)entry.value, "foo", 3)) + ERR("Value 4 didn't match, instead got: %.*s", entry.sz, + entry.value); + quicklistIndex(ql, 5, &entry); + if (strncmp((char *)entry.value, "zoo", 3)) + ERR("Value 5 didn't match, instead got: %.*s", entry.sz, + entry.value); + quicklistReleaseIterator(iter); + quicklistRelease(ql); + } + } + + for (size_t f = 0; f < 1024; f++) { + TEST_DESC("insert [before] 250 new in middle of 500 elements at fill" + " %ld", + f) { + quicklist *ql = quicklistCreate(); + for (int i = 0; i < 500; i++) + quicklistPushTail(ql, f, genstr("hello", i), 32); + for (int i = 0; i < 250; i++) { + quicklistEntry entry; + quicklistIndex(ql, 250, &entry); + quicklistInsertBefore(ql, f, &entry, genstr("abc", i), 32); + } + if (f == 32) + ql_verify(ql, 25, 750, 32, 20); + quicklistRelease(ql); + } + } + + for (size_t f = 0; f < 1024; f++) { + TEST_DESC( + "insert [after] 250 new in middle of 500 elements at fill %ld", f) { + quicklist *ql = quicklistCreate(); + for (int i = 0; i < 500; i++) + quicklistPushHead(ql, f, genstr("hello", i), 32); + for (int i = 0; i < 250; i++) { + quicklistEntry entry; + quicklistIndex(ql, 250, &entry); + quicklistInsertAfter(ql, f, &entry, genstr("abc", i), 32); + } + + if (ql->count != 750) + ERR("List size not 750, but rather %ld", ql->count); + + if (f == 32) + ql_verify(ql, 26, 750, 20, 32); + quicklistRelease(ql); + } + } + + TEST("duplicate empty list") { + quicklist *ql = quicklistCreate(); + ql_verify(ql, 0, 0, 0, 0); + quicklist *copy = quicklistDup(ql); + ql_verify(copy, 0, 0, 0, 0); + quicklistRelease(ql); + quicklistRelease(copy); + } + + TEST("duplicate list of 1 element") { + quicklist *ql = quicklistCreate(); + quicklistPushHead(ql, F, genstr("hello", 3), 32); + ql_verify(ql, 1, 1, 1, 1); + quicklist *copy = quicklistDup(ql); + ql_verify(copy, 1, 1, 1, 1); + quicklistRelease(ql); + quicklistRelease(copy); + } + + TEST("duplicate list of 500") { + quicklist *ql = quicklistCreate(); + for (int i = 0; i < 500; i++) + quicklistPushHead(ql, F, genstr("hello", i), 32); + ql_verify(ql, 16, 500, 20, 32); + + quicklist *copy = quicklistDup(ql); + ql_verify(copy, 16, 500, 20, 32); + quicklistRelease(ql); + quicklistRelease(copy); + } + + for (size_t f = 0; f < 512; f++) { + TEST_DESC("index 1,200 from 500 list at fill %lu", f) { + quicklist *ql = quicklistCreate(); + for (int i = 0; i < 500; i++) + quicklistPushTail(ql, f, genstr("hello", i + 1), 32); + quicklistEntry entry; + quicklistIndex(ql, 1, &entry); + if (!strcmp((char *)entry.value, "hello2")) + OK; + else + ERR("Value: %s", entry.value); + quicklistIndex(ql, 200, &entry); + if (!strcmp((char *)entry.value, "hello201")) + OK; + else + ERR("Value: %s", entry.value); + quicklistRelease(ql); + } + + TEST_DESC("index -1,-2 from 500 list at fill %lu", f) { + quicklist *ql = quicklistCreate(); + for (int i = 0; i < 500; i++) + quicklistPushTail(ql, f, genstr("hello", i + 1), 32); + quicklistEntry entry; + quicklistIndex(ql, -1, &entry); + if (!strcmp((char *)entry.value, "hello500")) + OK; + else + ERR("Value: %s", entry.value); + quicklistIndex(ql, -2, &entry); + if (!strcmp((char *)entry.value, "hello499")) + OK; + else + ERR("Value: %s", entry.value); + quicklistRelease(ql); + } + + TEST_DESC("index -100 from 500 list at fill %lu", f) { + quicklist *ql = quicklistCreate(); + for (int i = 0; i < 500; i++) + quicklistPushTail(ql, f, genstr("hello", i + 1), 32); + quicklistEntry entry; + quicklistIndex(ql, -100, &entry); + if (!strcmp((char *)entry.value, "hello401")) + OK; + else + ERR("Value: %s", entry.value); + quicklistRelease(ql); + } + + TEST_DESC("index too big +1 from 50 list at fill %lu", f) { + quicklist *ql = quicklistCreate(); + for (int i = 0; i < 50; i++) + quicklistPushTail(ql, f, genstr("hello", i + 1), 32); + quicklistEntry entry; + if (quicklistIndex(ql, 50, &entry)) + ERR("Index found at 50 with 50 list: %.*s", entry.sz, + entry.value); + else + OK; + quicklistRelease(ql); + } + } + + TEST("delete range empty list") { + quicklist *ql = quicklistCreate(); + quicklistDelRange(ql, 5, 20); + ql_verify(ql, 0, 0, 0, 0); + quicklistRelease(ql); + } + + TEST("delete range of entire node in list of one node") { + quicklist *ql = quicklistCreate(); + for (int i = 0; i < 32; i++) + quicklistPushHead(ql, F, genstr("hello", i), 32); + ql_verify(ql, 1, 32, 32, 32); + quicklistDelRange(ql, 0, 32); + ql_verify(ql, 0, 0, 0, 0); + quicklistRelease(ql); + } + + TEST("delete range of entire node with overflow counts") { + quicklist *ql = quicklistCreate(); + for (int i = 0; i < 32; i++) + quicklistPushHead(ql, F, genstr("hello", i), 32); + ql_verify(ql, 1, 32, 32, 32); + quicklistDelRange(ql, 0, 128); + ql_verify(ql, 0, 0, 0, 0); + quicklistRelease(ql); + } + + TEST("delete middle 100 of 500 list") { + quicklist *ql = quicklistCreate(); + for (int i = 0; i < 500; i++) + quicklistPushTail(ql, F, genstr("hello", i + 1), 32); + ql_verify(ql, 16, 500, 32, 20); + quicklistDelRange(ql, 200, 100); + ql_verify(ql, 14, 400, 32, 20); + quicklistRelease(ql); + } + + TEST("delete negative 1 from 500 list") { + quicklist *ql = quicklistCreate(); + for (int i = 0; i < 500; i++) + quicklistPushTail(ql, F, genstr("hello", i + 1), 32); + ql_verify(ql, 16, 500, 32, 20); + quicklistDelRange(ql, -1, 1); + ql_verify(ql, 16, 499, 32, 19); + quicklistRelease(ql); + } + + TEST("delete negative 1 from 500 list with overflow counts") { + quicklist *ql = quicklistCreate(); + for (int i = 0; i < 500; i++) + quicklistPushTail(ql, F, genstr("hello", i + 1), 32); + ql_verify(ql, 16, 500, 32, 20); + quicklistDelRange(ql, -1, 128); + ql_verify(ql, 16, 499, 32, 19); + quicklistRelease(ql); + } + + TEST("delete negative 100 from 500 list") { + quicklist *ql = quicklistCreate(); + for (int i = 0; i < 500; i++) + quicklistPushTail(ql, F, genstr("hello", i + 1), 32); + quicklistDelRange(ql, -100, 100); + ql_verify(ql, 13, 400, 32, 16); + quicklistRelease(ql); + } + + TEST("delete -10 count 5 from 50 list") { + quicklist *ql = quicklistCreate(); + for (int i = 0; i < 50; i++) + quicklistPushTail(ql, F, genstr("hello", i + 1), 32); + ql_verify(ql, 2, 50, 32, 18); + quicklistDelRange(ql, -10, 5); + ql_verify(ql, 2, 45, 32, 13); + quicklistRelease(ql); + } + + TEST("numbers only list read") { + quicklist *ql = quicklistCreate(); + quicklistPushTail(ql, F, "1111", 4); + quicklistPushTail(ql, F, "2222", 4); + quicklistPushTail(ql, F, "3333", 4); + quicklistPushTail(ql, F, "4444", 4); + ql_verify(ql, 1, 4, 4, 4); + quicklistEntry entry; + quicklistIndex(ql, 0, &entry); + if (entry.longval != 1111) + ERR("Not 1111, %lld", entry.longval); + quicklistIndex(ql, 1, &entry); + if (entry.longval != 2222) + ERR("Not 2222, %lld", entry.longval); + quicklistIndex(ql, 2, &entry); + if (entry.longval != 3333) + ERR("Not 3333, %lld", entry.longval); + quicklistIndex(ql, 3, &entry); + if (entry.longval != 4444) + ERR("Not 4444, %lld", entry.longval); + if (quicklistIndex(ql, 4, &entry)) + ERR("Index past elements: %lld", entry.longval); + quicklistIndex(ql, -1, &entry); + if (entry.longval != 4444) + ERR("Not 4444 (reverse), %lld", entry.longval); + quicklistIndex(ql, -2, &entry); + if (entry.longval != 3333) + ERR("Not 3333 (reverse), %lld", entry.longval); + quicklistIndex(ql, -3, &entry); + if (entry.longval != 2222) + ERR("Not 2222 (reverse), %lld", entry.longval); + quicklistIndex(ql, -4, &entry); + if (entry.longval != 1111) + ERR("Not 1111 (reverse), %lld", entry.longval); + if (quicklistIndex(ql, -5, &entry)) + ERR("Index past elements (reverse), %lld", entry.longval); + quicklistRelease(ql); + } + + TEST("numbers larger list read") { + quicklist *ql = quicklistCreate(); + char num[32]; + long long nums[5000]; + for (int i = 0; i < 5000; i++) { + nums[i] = -5157318210846258176 + i; + int sz = ll2string(num, sizeof(num), nums[i]); + quicklistPushTail(ql, F, num, sz); + } + quicklistPushTail(ql, F, "xxxxxxxxxxxxxxxxxxxx", 20); + quicklistEntry entry; + for (int i = 0; i < 5000; i++) { + quicklistIndex(ql, i, &entry); + if (entry.longval != nums[i]) + ERR("[%d] Not longval %lld but rather %lld", i, nums[i], + entry.longval); + entry.longval = 0xdeadbeef; + } + quicklistIndex(ql, 5000, &entry); + if (strncmp((char *)entry.value, "xxxxxxxxxxxxxxxxxxxx", 20)) + ERR("String val not match: %s", entry.value); + ql_verify(ql, 157, 5001, 32, 9); + quicklistRelease(ql); + } + + TEST("numbers larger list read B") { + quicklist *ql = quicklistCreate(); + quicklistPushTail(ql, F, "99", 2); + quicklistPushTail(ql, F, "98", 2); + quicklistPushTail(ql, F, "xxxxxxxxxxxxxxxxxxxx", 20); + quicklistPushTail(ql, F, "96", 2); + quicklistPushTail(ql, F, "95", 2); + quicklistReplaceAtIndex(ql, 1, "foo", 3); + quicklistReplaceAtIndex(ql, -1, "bar", 3); + quicklistRelease(ql); + OK; + } + + for (size_t f = 0; f < 16; f++) { + TEST_DESC("lrem test at fill %lu", f) { + quicklist *ql = quicklistCreate(); + char *words[] = { "abc", "foo", "bar", "foobar", "foobared", + "zap", "bar", "test", "foo" }; + char *result[] = { "abc", "foo", "foobar", "foobared", + "zap", "test", "foo" }; + char *resultB[] = { "abc", "foo", "foobar", + "foobared", "zap", "test" }; + for (int i = 0; i < 9; i++) + quicklistPushTail(ql, f, words[i], strlen(words[i])); + + /* lrem 0 bar */ + quicklistIter *iter = quicklistGetIterator(ql, AL_START_HEAD); + quicklistEntry entry; + int i = 0; + while (quicklistNext(iter, &entry)) { + if (quicklistCompare(entry.zi, (unsigned char *)"bar", 3)) { + quicklistDelEntry(iter, &entry); + } + i++; + } + quicklistReleaseIterator(iter); + + /* check result of lrem 0 bar */ + iter = quicklistGetIterator(ql, AL_START_HEAD); + i = 0; + int ok = 1; + while (quicklistNext(iter, &entry)) { + /* Result must be: abc, foo, foobar, foobared, zap, test, foo */ + if (strncmp((char *)entry.value, result[i], entry.sz)) { + ERR("No match at position %d, got %.*s instead of %s", i, + entry.sz, entry.value, result[i]); + ok = 0; + } + i++; + } + quicklistReleaseIterator(iter); + + quicklistPushTail(ql, f, "foo", 3); + + /* lrem -2 foo */ + iter = quicklistGetIterator(ql, AL_START_TAIL); + i = 0; + int del = 2; + while (quicklistNext(iter, &entry)) { + if (quicklistCompare(entry.zi, (unsigned char *)"foo", 3)) { + quicklistDelEntry(iter, &entry); + del--; + } + if (!del) + break; + i++; + } + quicklistReleaseIterator(iter); + + /* check result of lrem -2 foo */ + /* (we're ignoring the '2' part and still deleting all foo because + * we only have two foo) */ + iter = quicklistGetIterator(ql, AL_START_TAIL); + i = 0; + size_t resB = sizeof(resultB) / sizeof(*resultB); + while (quicklistNext(iter, &entry)) { + /* Result must be: abc, foo, foobar, foobared, zap, test, foo */ + if (strncmp((char *)entry.value, resultB[resB - 1 - i], + entry.sz)) { + ERR("No match at position %d, got %.*s instead of %s", i, + entry.sz, entry.value, resultB[resB - 1 - i]); + ok = 0; + } + i++; + } + + quicklistReleaseIterator(iter); + /* final result of all tests */ + if (ok) + OK; + quicklistRelease(ql); + } + } + + for (size_t f = 0; f < 16; f++) { + TEST_DESC("iterate reverse + delete at fill %lu", f) { + quicklist *ql = quicklistCreate(); + quicklistPushTail(ql, f, "abc", 3); + quicklistPushTail(ql, f, "def", 3); + quicklistPushTail(ql, f, "hij", 3); + quicklistPushTail(ql, f, "jkl", 3); + quicklistPushTail(ql, f, "oop", 3); + + quicklistEntry entry; + quicklistIter *iter = quicklistGetIterator(ql, AL_START_TAIL); + int i = 0; + while (quicklistNext(iter, &entry)) { + if (quicklistCompare(entry.zi, (unsigned char *)"hij", 3)) { + quicklistDelEntry(iter, &entry); + } + i++; + } + quicklistReleaseIterator(iter); + + if (i != 5) + ERR("Didn't iterate 5 times, iterated %d times.", i); + + /* Check results after deletion of "hij" */ + iter = quicklistGetIterator(ql, AL_START_HEAD); + i = 0; + char *vals[] = { "abc", "def", "jkl", "oop" }; + while (quicklistNext(iter, &entry)) { + if (!quicklistCompare(entry.zi, (unsigned char *)vals[i], 3)) { + ERR("Value at %d didn't match %s\n", i, vals[i]); + } + i++; + } + quicklistReleaseIterator(iter); + quicklistRelease(ql); + } + } + + for (size_t f = 0; f < 800; f++) { + TEST_DESC("iterator at index test at fill %lu", f) { + quicklist *ql = quicklistCreate(); + char num[32]; + long long nums[5000]; + for (int i = 0; i < 760; i++) { + nums[i] = -5157318210846258176 + i; + int sz = ll2string(num, sizeof(num), nums[i]); + quicklistPushTail(ql, f, num, sz); + } + + quicklistEntry entry; + quicklistIter *iter = + quicklistGetIteratorAtIdx(ql, AL_START_HEAD, 437); + int i = 437; + while (quicklistNext(iter, &entry)) { + if (entry.longval != nums[i]) + ERR("Expected %lld, but got %lld", entry.longval, nums[i]); + i++; + } + quicklistReleaseIterator(iter); + quicklistRelease(ql); + } + } + + for (size_t f = 0; f < 40; f++) { + TEST_DESC("ltrim test A at fill %lu", f) { + quicklist *ql = quicklistCreate(); + char num[32]; + long long nums[5000]; + for (int i = 0; i < 32; i++) { + nums[i] = -5157318210846258176 + i; + int sz = ll2string(num, sizeof(num), nums[i]); + quicklistPushTail(ql, f, num, sz); + } + if (f == 32) + ql_verify(ql, 1, 32, 32, 32); + /* ltrim 25 53 (keep [25,32] inclusive = 7 remaining) */ + quicklistDelRange(ql, 0, 25); + quicklistDelRange(ql, 0, 0); + quicklistEntry entry; + for (int i = 0; i < 7; i++) { + quicklistIndex(ql, i, &entry); + if (entry.longval != nums[25 + i]) + ERR("Deleted invalid range! Expected %lld but got %lld", + entry.longval, nums[25 + i]); + } + if (f == 32) + ql_verify(ql, 1, 7, 7, 7); + quicklistRelease(ql); + } + } + + for (size_t f = 0; f < 40; f++) { + TEST_DESC("ltrim test B at fill %lu", f) { + quicklist *ql = quicklistCreate(); + char num[32]; + long long nums[5000]; + for (int i = 0; i < 33; i++) { + nums[i] = i; + int sz = ll2string(num, sizeof(num), nums[i]); + quicklistPushTail(ql, f, num, sz); + } + if (f == 32) + ql_verify(ql, 2, 33, 32, 1); + /* ltrim 5 16 (keep [5,16] inclusive = 12 remaining) */ + quicklistDelRange(ql, 0, 5); + quicklistDelRange(ql, -16, 16); + if (f == 32) + ql_verify(ql, 1, 12, 12, 12); + quicklistEntry entry; + quicklistIndex(ql, 0, &entry); + if (entry.longval != 5) + ERR("A: longval not 5, but %lld", entry.longval); + else + OK; + quicklistIndex(ql, -1, &entry); + if (entry.longval != 16) + ERR("B! got instead: %lld", entry.longval); + else + OK; + quicklistPushTail(ql, f, "bobobob", 7); + quicklistIndex(ql, -1, &entry); + if (strncmp((char *)entry.value, "bobobob", 7)) + ERR("Tail doesn't match bobobob, it's %.*s instead", entry.sz, + entry.value); + for (int i = 0; i < 12; i++) { + quicklistIndex(ql, i, &entry); + if (entry.longval != nums[5 + i]) + ERR("Deleted invalid range! Expected %lld but got %lld", + entry.longval, nums[5 + i]); + } + quicklistRelease(ql); + } + } + + for (size_t f = 0; f < 40; f++) { + TEST_DESC("ltrim test C at fill %lu", f) { + quicklist *ql = quicklistCreate(); + char num[32]; + long long nums[5000]; + for (int i = 0; i < 33; i++) { + nums[i] = -5157318210846258176 + i; + int sz = ll2string(num, sizeof(num), nums[i]); + quicklistPushTail(ql, f, num, sz); + } + if (f == 32) + ql_verify(ql, 2, 33, 32, 1); + /* ltrim 3 3 (keep [3,3] inclusive = 1 remaining) */ + quicklistDelRange(ql, 0, 3); + quicklistDelRange(ql, -29, 4000); /* make sure not loop forever */ + if (f == 32) + ql_verify(ql, 1, 1, 1, 1); + quicklistEntry entry; + quicklistIndex(ql, 0, &entry); + if (entry.longval != -5157318210846258173) + ERROR; + else + OK; + quicklistRelease(ql); + } + } + + for (size_t f = 0; f < 40; f++) { + TEST_DESC("ltrim test D at fill %lu", f) { + quicklist *ql = quicklistCreate(); + char num[32]; + long long nums[5000]; + for (int i = 0; i < 33; i++) { + nums[i] = -5157318210846258176 + i; + int sz = ll2string(num, sizeof(num), nums[i]); + quicklistPushTail(ql, f, num, sz); + } + if (f == 32) + ql_verify(ql, 2, 33, 32, 1); + quicklistDelRange(ql, -12, 3); + if (ql->count != 30) + ERR("Didn't delete exactly three elements! Count is: %lu", + ql->count); + quicklistRelease(ql); + } + } + + for (size_t f = 0; f < 72; f++) { + TEST_DESC("create quicklist from ziplist at fill %lu", f) { + unsigned char *zl = ziplistNew(); + long long nums[32]; + char num[32]; + for (int i = 0; i < 33; i++) { + nums[i] = -5157318210846258176 + i; + int sz = ll2string(num, sizeof(num), nums[i]); + zl = ziplistPush(zl, (unsigned char *)num, sz, ZIPLIST_TAIL); + } + for (int i = 0; i < 33; i++) { + zl = ziplistPush(zl, (unsigned char *)genstr("hello", i), 32, + ZIPLIST_TAIL); + } + quicklist *ql = quicklistCreateFromZiplist(f, zl); + if (f == 1) + ql_verify(ql, 66, 66, 1, 1); + else if (f == 32) + ql_verify(ql, 3, 66, 32, 2); + else if (f == 66) + ql_verify(ql, 1, 66, 66, 66); + quicklistRelease(ql); + } + } + + if (!err) + printf("ALL TESTS PASSED!\n"); + else + ERR("Sorry, not all tests passed! In fact, %d tests failed.", err); + + return err; +} +#endif diff --git a/src/quicklist.h b/src/quicklist.h new file mode 100644 index 000000000..93b38d880 --- /dev/null +++ b/src/quicklist.h @@ -0,0 +1,120 @@ +/* quicklist.h - A generic doubly linked quicklist implementation + * + * Copyright (c) 2014, Matt Stancliff + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this quicklist of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this quicklist of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __QUICKLIST_H__ +#define __QUICKLIST_H__ + +/* Node, quicklist, and Iterator are the only data structures used currently. */ + +typedef struct quicklistNode { + struct quicklistNode *prev; + struct quicklistNode *next; + unsigned char *zl; + unsigned int count; /* cached count of items in ziplist */ +} quicklistNode; + +typedef struct quicklist { + quicklistNode *head; + quicklistNode *tail; + unsigned long len; /* number of quicklistNodes */ + unsigned long count; /* total count of all entries in all ziplists */ +} quicklist; + +typedef struct quicklistIter { + const quicklist *quicklist; + quicklistNode *current; + unsigned char *zi; + long offset; /* offset in current ziplist */ + int direction; +} quicklistIter; + +typedef struct quicklistEntry { + const quicklist *quicklist; + quicklistNode *node; + unsigned char *zi; + unsigned char *value; + unsigned int sz; + long long longval; + int offset; +} quicklistEntry; + +#define QUICKLIST_HEAD 0 +#define QUICKLIST_TAIL -1 + +/* Prototypes */ +quicklist *quicklistCreate(void); +void quicklistRelease(quicklist *quicklist); +quicklist *quicklistPushHead(quicklist *quicklist, const size_t fill, + void *value, const size_t sz); +quicklist *quicklistPushTail(quicklist *quicklist, const size_t fill, + void *value, const size_t sz); +void quicklistPush(quicklist *quicklist, const size_t fill, void *value, + const size_t sz, int where); +void quicklistAppendZiplist(quicklist *quicklist, unsigned char *zl); +quicklist *quicklistAppendValuesFromZiplist(quicklist *quicklist, + const size_t fill, + unsigned char *zl); +quicklist *quicklistCreateFromZiplist(size_t fill, unsigned char *zl); +void quicklistInsertAfter(quicklist *quicklist, const size_t fill, + quicklistEntry *node, void *value, const size_t sz); +void quicklistInsertBefore(quicklist *quicklist, const size_t fill, + quicklistEntry *node, void *value, const size_t sz); +void quicklistDelEntry(quicklistIter *iter, quicklistEntry *entry); +int quicklistReplaceAtIndex(quicklist *quicklist, long index, void *data, + int sz); +int quicklistDelRange(quicklist *quicklist, const long start, const long stop); +quicklistIter *quicklistGetIterator(const quicklist *quicklist, int direction); +quicklistIter *quicklistGetIteratorAtIdx(const quicklist *quicklist, + int direction, const long long idx); +int quicklistNext(quicklistIter *iter, quicklistEntry *node); +void quicklistReleaseIterator(quicklistIter *iter); +quicklist *quicklistDup(quicklist *orig); +int quicklistIndex(const quicklist *quicklist, const long long index, + quicklistEntry *entry); +void quicklistRewind(quicklist *quicklist, quicklistIter *li); +void quicklistRewindTail(quicklist *quicklist, quicklistIter *li); +void quicklistRotate(quicklist *quicklist, const size_t fill); +int quicklistPopCustom(quicklist *quicklist, int where, unsigned char **data, + unsigned int *sz, long long *sval, + void *(*saver)(unsigned char *data, unsigned int sz)); +int quicklistPop(quicklist *quicklist, int where, unsigned char **data, + unsigned int *sz, long long *slong); +unsigned int quicklistCount(quicklist *ql); +int quicklistCompare(unsigned char *p1, unsigned char *p2, int p2_len); + +#ifdef REDIS_TEST +int quicklistTest(int argc, char *argv[]); +#endif + +/* Directions for iterators */ +#define AL_START_HEAD 0 +#define AL_START_TAIL 1 + +#endif /* __QUICKLIST_H__ */ diff --git a/src/rdb.c b/src/rdb.c index 325e5a62f..70989897a 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -433,9 +433,7 @@ int rdbSaveObjectType(rio *rdb, robj *o) { case REDIS_STRING: return rdbSaveType(rdb,REDIS_RDB_TYPE_STRING); case REDIS_LIST: - if (o->encoding == REDIS_ENCODING_ZIPLIST) - return rdbSaveType(rdb,REDIS_RDB_TYPE_LIST_ZIPLIST); - else if (o->encoding == REDIS_ENCODING_LINKEDLIST) + if (o->encoding == REDIS_ENCODING_QUICKLIST) return rdbSaveType(rdb,REDIS_RDB_TYPE_LIST); else redisPanic("Unknown list encoding"); @@ -477,7 +475,7 @@ int rdbLoadObjectType(rio *rdb) { /* Save a Redis object. Returns -1 on error, number of bytes written on success. */ int rdbSaveObject(rio *rdb, robj *o) { - int n, nwritten = 0; + int n = 0, nwritten = 0; if (o->type == REDIS_STRING) { /* Save a string value */ @@ -485,25 +483,23 @@ int rdbSaveObject(rio *rdb, robj *o) { nwritten += n; } else if (o->type == REDIS_LIST) { /* Save a list value */ - if (o->encoding == REDIS_ENCODING_ZIPLIST) { - size_t l = ziplistBlobLen((unsigned char*)o->ptr); + if (o->encoding == REDIS_ENCODING_QUICKLIST) { + quicklist *list = o->ptr; + quicklistIter *li = quicklistGetIterator(list, AL_START_HEAD); + quicklistEntry entry; - if ((n = rdbSaveRawString(rdb,o->ptr,l)) == -1) return -1; - nwritten += n; - } else if (o->encoding == REDIS_ENCODING_LINKEDLIST) { - list *list = o->ptr; - listIter li; - listNode *ln; - - if ((n = rdbSaveLen(rdb,listLength(list))) == -1) return -1; + if ((n = rdbSaveLen(rdb,quicklistCount(list))) == -1) return -1; nwritten += n; - listRewind(list,&li); - while((ln = listNext(&li))) { - robj *eleobj = listNodeValue(ln); - if ((n = rdbSaveStringObject(rdb,eleobj)) == -1) return -1; + while (quicklistNext(li,&entry)) { + if (entry.value) { + if ((n = rdbSaveRawString(rdb,entry.value,entry.sz)) == -1) return -1; + } else { + if ((n = rdbSaveLongLongAsStringObject(rdb,entry.longval)) == -1) return -1; + } nwritten += n; } + quicklistReleaseIterator(li); } else { redisPanic("Unknown list encoding"); } @@ -831,33 +827,17 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { /* Read list value */ if ((len = rdbLoadLen(rdb,NULL)) == REDIS_RDB_LENERR) return NULL; - /* Use a real list when there are too many entries */ - if (len > server.list_max_ziplist_entries) { - o = createListObject(); - } else { - o = createZiplistObject(); - } + o = createQuicklistObject(); /* Load every single element of the list */ while(len--) { if ((ele = rdbLoadEncodedStringObject(rdb)) == NULL) return NULL; - - /* If we are using a ziplist and the value is too big, convert - * the object to a real list. */ - if (o->encoding == REDIS_ENCODING_ZIPLIST && - sdsEncodedObject(ele) && - sdslen(ele->ptr) > server.list_max_ziplist_value) - listTypeConvert(o,REDIS_ENCODING_LINKEDLIST); - - if (o->encoding == REDIS_ENCODING_ZIPLIST) { - dec = getDecodedObject(ele); - o->ptr = ziplistPush(o->ptr,dec->ptr,sdslen(dec->ptr),REDIS_TAIL); - decrRefCount(dec); - decrRefCount(ele); - } else { - ele = tryObjectEncoding(ele); - listAddNodeTail(o->ptr,ele); - } + dec = getDecodedObject(ele); + size_t len = sdslen(dec->ptr); + size_t zlen = server.list_max_ziplist_entries; + o->ptr = quicklistPushTail(o->ptr, zlen, dec->ptr, len); + decrRefCount(dec); + decrRefCount(ele); } } else if (rdbtype == REDIS_RDB_TYPE_SET) { /* Read list/set value */ @@ -1048,8 +1028,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { case REDIS_RDB_TYPE_LIST_ZIPLIST: o->type = REDIS_LIST; o->encoding = REDIS_ENCODING_ZIPLIST; - if (ziplistLen(o->ptr) > server.list_max_ziplist_entries) - listTypeConvert(o,REDIS_ENCODING_LINKEDLIST); + listTypeConvert(o,REDIS_ENCODING_QUICKLIST); break; case REDIS_RDB_TYPE_SET_INTSET: o->type = REDIS_SET; diff --git a/src/redis.c b/src/redis.c index 01912b79b..ee88eddb5 100644 --- a/src/redis.c +++ b/src/redis.c @@ -3659,6 +3659,8 @@ int main(int argc, char **argv) { if (argc == 3 && !strcasecmp(argv[1], "test")) { if (!strcasecmp(argv[2], "ziplist")) { return ziplistTest(argc, argv); + } else if (!strcasecmp(argv[2], "quicklist")) { + quicklistTest(argc, argv); } else if (!strcasecmp(argv[2], "intset")) { return intsetTest(argc, argv); } else if (!strcasecmp(argv[2], "zipmap")) { diff --git a/src/redis.h b/src/redis.h index 1720c24bd..38c072478 100644 --- a/src/redis.h +++ b/src/redis.h @@ -65,6 +65,7 @@ typedef long long mstime_t; /* millisecond time type. */ #include "util.h" /* Misc functions useful in many places */ #include "latency.h" /* Latency monitor API */ #include "sparkline.h" /* ASII graphs API */ +#include "quicklist.h" /* Following includes allow test functions to be called from Redis main() */ #include "zipmap.h" @@ -204,6 +205,7 @@ typedef long long mstime_t; /* millisecond time type. */ #define REDIS_ENCODING_INTSET 6 /* Encoded as intset */ #define REDIS_ENCODING_SKIPLIST 7 /* Encoded as skiplist */ #define REDIS_ENCODING_EMBSTR 8 /* Embedded sds string encoding */ +#define REDIS_ENCODING_QUICKLIST 9 /* Encoded as linked list of ziplists */ /* Defines related to the dump file format. To store 32 bits lengths for short * keys requires a lot of space, so we check the most significant 2 bits of @@ -964,15 +966,13 @@ typedef struct { robj *subject; unsigned char encoding; unsigned char direction; /* Iteration direction */ - unsigned char *zi; - listNode *ln; + quicklistIter *iter; } listTypeIterator; /* Structure for an entry while iterating over a list. */ typedef struct { listTypeIterator *li; - unsigned char *zi; /* Entry in ziplist */ - listNode *ln; /* Entry in linked list */ + quicklistEntry entry; /* Entry in quicklist */ } listTypeEntry; /* Structure to hold set iteration abstraction. */ @@ -1099,7 +1099,7 @@ int listTypeNext(listTypeIterator *li, listTypeEntry *entry); robj *listTypeGet(listTypeEntry *entry); void listTypeInsert(listTypeEntry *entry, robj *value, int where); int listTypeEqual(listTypeEntry *entry, robj *o); -void listTypeDelete(listTypeEntry *entry); +void listTypeDelete(listTypeIterator *iter, listTypeEntry *entry); void listTypeConvert(robj *subject, int enc); void unblockClientWaitingData(redisClient *c); void handleClientsBlockedOnLists(void); @@ -1137,7 +1137,7 @@ robj *getDecodedObject(robj *o); size_t stringObjectLen(robj *o); robj *createStringObjectFromLongLong(long long value); robj *createStringObjectFromLongDouble(long double value, int humanfriendly); -robj *createListObject(void); +robj *createQuicklistObject(void); robj *createZiplistObject(void); robj *createSetObject(void); robj *createIntsetObject(void); diff --git a/src/sort.c b/src/sort.c index 2b3276448..74b27cb67 100644 --- a/src/sort.c +++ b/src/sort.c @@ -220,7 +220,7 @@ void sortCommand(redisClient *c) { if (sortval) incrRefCount(sortval); else - sortval = createListObject(); + sortval = createQuicklistObject(); /* The SORT command has an SQL-alike syntax, parse it */ while(j < c->argc) { @@ -420,6 +420,7 @@ void sortCommand(redisClient *c) { } else { redisPanic("Unknown type"); } + printf("j: %d; vectorlen: %d\n", j, vectorlen); redisAssertWithInfo(c,sortval,j == vectorlen); /* Now it's time to load the right scores in the sorting vector */ @@ -509,7 +510,7 @@ void sortCommand(redisClient *c) { } } } else { - robj *sobj = createZiplistObject(); + robj *sobj = createQuicklistObject(); /* STORE option specified, set the sorting result as a List object */ for (j = start; j <= end; j++) { diff --git a/src/t_list.c b/src/t_list.c index fc27331f5..b40e0089a 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -33,75 +33,40 @@ * List API *----------------------------------------------------------------------------*/ -/* Check the argument length to see if it requires us to convert the ziplist - * to a real list. Only check raw-encoded objects because integer encoded - * objects are never too long. */ -void listTypeTryConversion(robj *subject, robj *value) { - if (subject->encoding != REDIS_ENCODING_ZIPLIST) return; - if (sdsEncodedObject(value) && - sdslen(value->ptr) > server.list_max_ziplist_value) - listTypeConvert(subject,REDIS_ENCODING_LINKEDLIST); -} - /* The function pushes an element to the specified list object 'subject', * at head or tail position as specified by 'where'. * * There is no need for the caller to increment the refcount of 'value' as * the function takes care of it if needed. */ void listTypePush(robj *subject, robj *value, int where) { - /* Check if we need to convert the ziplist */ - listTypeTryConversion(subject,value); - if (subject->encoding == REDIS_ENCODING_ZIPLIST && - ziplistLen(subject->ptr) >= server.list_max_ziplist_entries) - listTypeConvert(subject,REDIS_ENCODING_LINKEDLIST); - - if (subject->encoding == REDIS_ENCODING_ZIPLIST) { - int pos = (where == REDIS_HEAD) ? ZIPLIST_HEAD : ZIPLIST_TAIL; + if (subject->encoding == REDIS_ENCODING_QUICKLIST) { + int pos = (where == REDIS_HEAD) ? QUICKLIST_HEAD : QUICKLIST_TAIL; value = getDecodedObject(value); - subject->ptr = ziplistPush(subject->ptr,value->ptr,sdslen(value->ptr),pos); + size_t len = sdslen(value->ptr); + size_t zlen = server.list_max_ziplist_entries; + /* If this value is greater than our allowed values, create it in + * an isolated ziplist */ + quicklistPush(subject->ptr, zlen, value->ptr, len, pos); decrRefCount(value); - } else if (subject->encoding == REDIS_ENCODING_LINKEDLIST) { - if (where == REDIS_HEAD) { - listAddNodeHead(subject->ptr,value); - } else { - listAddNodeTail(subject->ptr,value); - } - incrRefCount(value); } else { redisPanic("Unknown list encoding"); } } +void *listPopSaver(unsigned char *data, unsigned int sz) { + return createStringObject((char*)data,sz); +} + robj *listTypePop(robj *subject, int where) { + long long vlong; robj *value = NULL; - if (subject->encoding == REDIS_ENCODING_ZIPLIST) { - unsigned char *p; - unsigned char *vstr; - unsigned int vlen; - long long vlong; - int pos = (where == REDIS_HEAD) ? 0 : -1; - p = ziplistIndex(subject->ptr,pos); - if (ziplistGet(p,&vstr,&vlen,&vlong)) { - if (vstr) { - value = createStringObject((char*)vstr,vlen); - } else { + + int ql_where = where == REDIS_HEAD ? QUICKLIST_HEAD : QUICKLIST_TAIL; + if (subject->encoding == REDIS_ENCODING_QUICKLIST) { + if (quicklistPopCustom(subject->ptr, ql_where, (unsigned char **)&value, + NULL, &vlong, listPopSaver)) { + if (!value) value = createStringObjectFromLongLong(vlong); - } - /* We only need to delete an element when it exists */ - subject->ptr = ziplistDelete(subject->ptr,&p); - } - } else if (subject->encoding == REDIS_ENCODING_LINKEDLIST) { - list *list = subject->ptr; - listNode *ln; - if (where == REDIS_HEAD) { - ln = listFirst(list); - } else { - ln = listLast(list); - } - if (ln != NULL) { - value = listNodeValue(ln); - incrRefCount(value); - listDelNode(list,ln); } } else { redisPanic("Unknown list encoding"); @@ -110,25 +75,28 @@ robj *listTypePop(robj *subject, int where) { } unsigned long listTypeLength(robj *subject) { - if (subject->encoding == REDIS_ENCODING_ZIPLIST) { - return ziplistLen(subject->ptr); - } else if (subject->encoding == REDIS_ENCODING_LINKEDLIST) { - return listLength((list*)subject->ptr); + if (subject->encoding == REDIS_ENCODING_QUICKLIST) { + return quicklistCount(subject->ptr); } else { redisPanic("Unknown list encoding"); } } /* Initialize an iterator at the specified index. */ -listTypeIterator *listTypeInitIterator(robj *subject, long index, unsigned char direction) { +listTypeIterator *listTypeInitIterator(robj *subject, long index, + unsigned char direction) { listTypeIterator *li = zmalloc(sizeof(listTypeIterator)); li->subject = subject; li->encoding = subject->encoding; li->direction = direction; - if (li->encoding == REDIS_ENCODING_ZIPLIST) { - li->zi = ziplistIndex(subject->ptr,index); - } else if (li->encoding == REDIS_ENCODING_LINKEDLIST) { - li->ln = listIndex(subject->ptr,index); + li->iter = NULL; + /* REDIS_HEAD means start at TAIL and move *towards* head. + * REDIS_TAIL means start at HEAD and move *towards tail. */ + int iter_direction = + direction == REDIS_HEAD ? AL_START_TAIL : AL_START_HEAD; + if (li->encoding == REDIS_ENCODING_QUICKLIST) { + li->iter = quicklistGetIteratorAtIdx(li->subject->ptr, + iter_direction, index); } else { redisPanic("Unknown list encoding"); } @@ -137,6 +105,7 @@ listTypeIterator *listTypeInitIterator(robj *subject, long index, unsigned char /* Clean up the iterator. */ void listTypeReleaseIterator(listTypeIterator *li) { + zfree(li->iter); zfree(li); } @@ -148,24 +117,8 @@ int listTypeNext(listTypeIterator *li, listTypeEntry *entry) { redisAssert(li->subject->encoding == li->encoding); entry->li = li; - if (li->encoding == REDIS_ENCODING_ZIPLIST) { - entry->zi = li->zi; - if (entry->zi != NULL) { - if (li->direction == REDIS_TAIL) - li->zi = ziplistNext(li->subject->ptr,li->zi); - else - li->zi = ziplistPrev(li->subject->ptr,li->zi); - return 1; - } - } else if (li->encoding == REDIS_ENCODING_LINKEDLIST) { - entry->ln = li->ln; - if (entry->ln != NULL) { - if (li->direction == REDIS_TAIL) - li->ln = li->ln->next; - else - li->ln = li->ln->prev; - return 1; - } + if (li->encoding == REDIS_ENCODING_QUICKLIST) { + return quicklistNext(li->iter, &entry->entry); } else { redisPanic("Unknown list encoding"); } @@ -174,24 +127,14 @@ int listTypeNext(listTypeIterator *li, listTypeEntry *entry) { /* Return entry or NULL at the current position of the iterator. */ robj *listTypeGet(listTypeEntry *entry) { - listTypeIterator *li = entry->li; robj *value = NULL; - if (li->encoding == REDIS_ENCODING_ZIPLIST) { - unsigned char *vstr; - unsigned int vlen; - long long vlong; - redisAssert(entry->zi != NULL); - if (ziplistGet(entry->zi,&vstr,&vlen,&vlong)) { - if (vstr) { - value = createStringObject((char*)vstr,vlen); - } else { - value = createStringObjectFromLongLong(vlong); - } + if (entry->li->encoding == REDIS_ENCODING_QUICKLIST) { + if (entry->entry.value) { + value = createStringObject((char *)entry->entry.value, + entry->entry.sz); + } else { + value = createStringObjectFromLongLong(entry->entry.longval); } - } else if (li->encoding == REDIS_ENCODING_LINKEDLIST) { - redisAssert(entry->ln != NULL); - value = listNodeValue(entry->ln); - incrRefCount(value); } else { redisPanic("Unknown list encoding"); } @@ -199,30 +142,19 @@ robj *listTypeGet(listTypeEntry *entry) { } void listTypeInsert(listTypeEntry *entry, robj *value, int where) { - robj *subject = entry->li->subject; - if (entry->li->encoding == REDIS_ENCODING_ZIPLIST) { + if (entry->li->encoding == REDIS_ENCODING_QUICKLIST) { value = getDecodedObject(value); + sds str = value->ptr; + size_t len = sdslen(str); + size_t zlen = server.list_max_ziplist_entries; if (where == REDIS_TAIL) { - unsigned char *next = ziplistNext(subject->ptr,entry->zi); - - /* When we insert after the current element, but the current element - * is the tail of the list, we need to do a push. */ - if (next == NULL) { - subject->ptr = ziplistPush(subject->ptr,value->ptr,sdslen(value->ptr),REDIS_TAIL); - } else { - subject->ptr = ziplistInsert(subject->ptr,next,value->ptr,sdslen(value->ptr)); - } - } else { - subject->ptr = ziplistInsert(subject->ptr,entry->zi,value->ptr,sdslen(value->ptr)); + quicklistInsertAfter((quicklist *)entry->entry.quicklist, zlen, + &entry->entry, str, len); + } else if (where == REDIS_HEAD) { + quicklistInsertBefore((quicklist *)entry->entry.quicklist, zlen, + &entry->entry, str, len); } decrRefCount(value); - } else if (entry->li->encoding == REDIS_ENCODING_LINKEDLIST) { - if (where == REDIS_TAIL) { - listInsertNode(subject->ptr,entry->ln,value,AL_START_TAIL); - } else { - listInsertNode(subject->ptr,entry->ln,value,AL_START_HEAD); - } - incrRefCount(value); } else { redisPanic("Unknown list encoding"); } @@ -230,59 +162,33 @@ void listTypeInsert(listTypeEntry *entry, robj *value, int where) { /* Compare the given object with the entry at the current position. */ int listTypeEqual(listTypeEntry *entry, robj *o) { - listTypeIterator *li = entry->li; - if (li->encoding == REDIS_ENCODING_ZIPLIST) { + if (entry->li->encoding == REDIS_ENCODING_QUICKLIST) { redisAssertWithInfo(NULL,o,sdsEncodedObject(o)); - return ziplistCompare(entry->zi,o->ptr,sdslen(o->ptr)); - } else if (li->encoding == REDIS_ENCODING_LINKEDLIST) { - return equalStringObjects(o,listNodeValue(entry->ln)); + return quicklistCompare(entry->entry.zi,o->ptr,sdslen(o->ptr)); } else { redisPanic("Unknown list encoding"); } } /* Delete the element pointed to. */ -void listTypeDelete(listTypeEntry *entry) { - listTypeIterator *li = entry->li; - if (li->encoding == REDIS_ENCODING_ZIPLIST) { - unsigned char *p = entry->zi; - li->subject->ptr = ziplistDelete(li->subject->ptr,&p); - - /* Update position of the iterator depending on the direction */ - if (li->direction == REDIS_TAIL) - li->zi = p; - else - li->zi = ziplistPrev(li->subject->ptr,p); - } else if (entry->li->encoding == REDIS_ENCODING_LINKEDLIST) { - listNode *next; - if (li->direction == REDIS_TAIL) - next = entry->ln->next; - else - next = entry->ln->prev; - listDelNode(li->subject->ptr,entry->ln); - li->ln = next; +void listTypeDelete(listTypeIterator *iter, listTypeEntry *entry) { + if (entry->li->encoding == REDIS_ENCODING_QUICKLIST) { + quicklistDelEntry(iter->iter, &entry->entry); } else { redisPanic("Unknown list encoding"); } } +/* Create a quicklist from a single ziplist */ void listTypeConvert(robj *subject, int enc) { - listTypeIterator *li; - listTypeEntry entry; - redisAssertWithInfo(NULL,subject,subject->type == REDIS_LIST); + redisAssertWithInfo(NULL,subject,subject->type==REDIS_LIST); + redisAssertWithInfo(NULL,subject,subject->encoding==REDIS_ENCODING_ZIPLIST); - if (enc == REDIS_ENCODING_LINKEDLIST) { - list *l = listCreate(); - listSetFreeMethod(l,decrRefCountVoid); + if (enc == REDIS_ENCODING_QUICKLIST) { + size_t zlen = server.list_max_ziplist_entries; - /* listTypeGet returns a robj with incremented refcount */ - li = listTypeInitIterator(subject,0,REDIS_TAIL); - while (listTypeNext(li,&entry)) listAddNodeTail(l,listTypeGet(&entry)); - listTypeReleaseIterator(li); - - subject->encoding = REDIS_ENCODING_LINKEDLIST; - zfree(subject->ptr); - subject->ptr = l; + subject->encoding = REDIS_ENCODING_QUICKLIST; + subject->ptr = quicklistCreateFromZiplist(zlen, subject->ptr); } else { redisPanic("Unsupported list conversion"); } @@ -304,7 +210,7 @@ void pushGenericCommand(redisClient *c, int where) { for (j = 2; j < c->argc; j++) { c->argv[j] = tryObjectEncoding(c->argv[j]); if (!lobj) { - lobj = createZiplistObject(); + lobj = createQuicklistObject(); dbAdd(c->db,c->argv[1],lobj); } listTypePush(lobj,c->argv[j],where); @@ -338,13 +244,6 @@ void pushxGenericCommand(redisClient *c, robj *refval, robj *val, int where) { checkType(c,subject,REDIS_LIST)) return; if (refval != NULL) { - /* We're not sure if this value can be inserted yet, but we cannot - * convert the list inside the iterator. We don't want to loop over - * the list twice (once to see if the value can be inserted and once - * to do the actual insert), so we assume this value can be inserted - * and convert the ziplist to a regular list if necessary. */ - listTypeTryConversion(subject,val); - /* Seek refval from head to tail */ iter = listTypeInitIterator(subject,0,REDIS_TAIL); while (listTypeNext(iter,&entry)) { @@ -357,10 +256,6 @@ void pushxGenericCommand(redisClient *c, robj *refval, robj *val, int where) { listTypeReleaseIterator(iter); if (inserted) { - /* Check if the length exceeds the ziplist length threshold. */ - if (subject->encoding == REDIS_ENCODING_ZIPLIST && - ziplistLen(subject->ptr) > server.list_max_ziplist_entries) - listTypeConvert(subject,REDIS_ENCODING_LINKEDLIST); signalModifiedKey(c->db,c->argv[1]); notifyKeyspaceEvent(REDIS_NOTIFY_LIST,"linsert", c->argv[1],c->db->id); @@ -418,31 +313,19 @@ void lindexCommand(redisClient *c) { if ((getLongFromObjectOrReply(c, c->argv[2], &index, NULL) != REDIS_OK)) return; - if (o->encoding == REDIS_ENCODING_ZIPLIST) { - unsigned char *p; - unsigned char *vstr; - unsigned int vlen; - long long vlong; - p = ziplistIndex(o->ptr,index); - if (ziplistGet(p,&vstr,&vlen,&vlong)) { - if (vstr) { - value = createStringObject((char*)vstr,vlen); + if (o->encoding == REDIS_ENCODING_QUICKLIST) { + quicklistEntry entry; + if (quicklistIndex(o->ptr, index, &entry)) { + if (entry.value) { + value = createStringObject((char*)entry.value,entry.sz); } else { - value = createStringObjectFromLongLong(vlong); + value = createStringObjectFromLongLong(entry.longval); } addReplyBulk(c,value); decrRefCount(value); } else { addReply(c,shared.nullbulk); } - } else if (o->encoding == REDIS_ENCODING_LINKEDLIST) { - listNode *ln = listIndex(o->ptr,index); - if (ln != NULL) { - value = listNodeValue(ln); - addReplyBulk(c,value); - } else { - addReply(c,shared.nullbulk); - } } else { redisPanic("Unknown list encoding"); } @@ -452,35 +335,18 @@ void lsetCommand(redisClient *c) { robj *o = lookupKeyWriteOrReply(c,c->argv[1],shared.nokeyerr); if (o == NULL || checkType(c,o,REDIS_LIST)) return; long index; - robj *value = (c->argv[3] = tryObjectEncoding(c->argv[3])); + robj *value = c->argv[3]; if ((getLongFromObjectOrReply(c, c->argv[2], &index, NULL) != REDIS_OK)) return; - listTypeTryConversion(o,value); - if (o->encoding == REDIS_ENCODING_ZIPLIST) { - unsigned char *p, *zl = o->ptr; - p = ziplistIndex(zl,index); - if (p == NULL) { + if (o->encoding == REDIS_ENCODING_QUICKLIST) { + quicklist *ql = o->ptr; + int replaced = quicklistReplaceAtIndex(ql, index, + value->ptr, sdslen(value->ptr)); + if (!replaced) { addReply(c,shared.outofrangeerr); } else { - o->ptr = ziplistDelete(o->ptr,&p); - value = getDecodedObject(value); - o->ptr = ziplistInsert(o->ptr,p,value->ptr,sdslen(value->ptr)); - decrRefCount(value); - addReply(c,shared.ok); - signalModifiedKey(c->db,c->argv[1]); - notifyKeyspaceEvent(REDIS_NOTIFY_LIST,"lset",c->argv[1],c->db->id); - server.dirty++; - } - } else if (o->encoding == REDIS_ENCODING_LINKEDLIST) { - listNode *ln = listIndex(o->ptr,index); - if (ln == NULL) { - addReply(c,shared.outofrangeerr); - } else { - decrRefCount((robj*)listNodeValue(ln)); - listNodeValue(ln) = value; - incrRefCount(value); addReply(c,shared.ok); signalModifiedKey(c->db,c->argv[1]); notifyKeyspaceEvent(REDIS_NOTIFY_LIST,"lset",c->argv[1],c->db->id); @@ -549,43 +415,28 @@ void lrangeCommand(redisClient *c) { /* Return the result in form of a multi-bulk reply */ addReplyMultiBulkLen(c,rangelen); - if (o->encoding == REDIS_ENCODING_ZIPLIST) { - unsigned char *p = ziplistIndex(o->ptr,start); - unsigned char *vstr; - unsigned int vlen; - long long vlong; + if (o->encoding == REDIS_ENCODING_QUICKLIST) { + listTypeIterator *iter = listTypeInitIterator(o, start, REDIS_TAIL); while(rangelen--) { - ziplistGet(p,&vstr,&vlen,&vlong); - if (vstr) { - addReplyBulkCBuffer(c,vstr,vlen); + listTypeEntry entry; + listTypeNext(iter, &entry); + quicklistEntry *qe = &entry.entry; + if (qe->value) { + addReplyBulkCBuffer(c,qe->value,qe->sz); } else { - addReplyBulkLongLong(c,vlong); + addReplyBulkLongLong(c,qe->longval); } - p = ziplistNext(o->ptr,p); - } - } else if (o->encoding == REDIS_ENCODING_LINKEDLIST) { - listNode *ln; - - /* If we are nearest to the end of the list, reach the element - * starting from tail and going backward, as it is faster. */ - if (start > llen/2) start -= llen; - ln = listIndex(o->ptr,start); - - while(rangelen--) { - addReplyBulk(c,ln->value); - ln = ln->next; } + listTypeReleaseIterator(iter); } else { - redisPanic("List encoding is not LINKEDLIST nor ZIPLIST!"); + redisPanic("List encoding is not QUICKLIST!"); } } void ltrimCommand(redisClient *c) { robj *o; - long start, end, llen, j, ltrim, rtrim; - list *list; - listNode *ln; + long start, end, llen, ltrim, rtrim; if ((getLongFromObjectOrReply(c, c->argv[2], &start, NULL) != REDIS_OK) || (getLongFromObjectOrReply(c, c->argv[3], &end, NULL) != REDIS_OK)) return; @@ -612,19 +463,9 @@ void ltrimCommand(redisClient *c) { } /* Remove list elements to perform the trim */ - if (o->encoding == REDIS_ENCODING_ZIPLIST) { - o->ptr = ziplistDeleteRange(o->ptr,0,ltrim); - o->ptr = ziplistDeleteRange(o->ptr,-rtrim,rtrim); - } else if (o->encoding == REDIS_ENCODING_LINKEDLIST) { - list = o->ptr; - for (j = 0; j < ltrim; j++) { - ln = listFirst(list); - listDelNode(list,ln); - } - for (j = 0; j < rtrim; j++) { - ln = listLast(list); - listDelNode(list,ln); - } + if (o->encoding == REDIS_ENCODING_QUICKLIST) { + quicklistDelRange(o->ptr,0,ltrim); + quicklistDelRange(o->ptr,-rtrim,rtrim); } else { redisPanic("Unknown list encoding"); } @@ -641,10 +482,9 @@ void ltrimCommand(redisClient *c) { void lremCommand(redisClient *c) { robj *subject, *obj; - obj = c->argv[3] = tryObjectEncoding(c->argv[3]); + obj = c->argv[3]; long toremove; long removed = 0; - listTypeEntry entry; if ((getLongFromObjectOrReply(c, c->argv[2], &toremove, NULL) != REDIS_OK)) return; @@ -652,10 +492,6 @@ void lremCommand(redisClient *c) { subject = lookupKeyWriteOrReply(c,c->argv[1],shared.czero); if (subject == NULL || checkType(c,subject,REDIS_LIST)) return; - /* Make sure obj is raw when we're dealing with a ziplist */ - if (subject->encoding == REDIS_ENCODING_ZIPLIST) - obj = getDecodedObject(obj); - listTypeIterator *li; if (toremove < 0) { toremove = -toremove; @@ -664,9 +500,10 @@ void lremCommand(redisClient *c) { li = listTypeInitIterator(subject,0,REDIS_TAIL); } + listTypeEntry entry; while (listTypeNext(li,&entry)) { if (listTypeEqual(&entry,obj)) { - listTypeDelete(&entry); + listTypeDelete(li, &entry); server.dirty++; removed++; if (toremove && removed == toremove) break; @@ -674,11 +511,10 @@ void lremCommand(redisClient *c) { } listTypeReleaseIterator(li); - /* Clean up raw encoded object */ - if (subject->encoding == REDIS_ENCODING_ZIPLIST) - decrRefCount(obj); + if (listTypeLength(subject) == 0) { + dbDelete(c->db,c->argv[1]); + } - if (listTypeLength(subject) == 0) dbDelete(c->db,c->argv[1]); addReplyLongLong(c,removed); if (removed) signalModifiedKey(c->db,c->argv[1]); } @@ -702,7 +538,7 @@ void lremCommand(redisClient *c) { void rpoplpushHandlePush(redisClient *c, robj *dstkey, robj *dstobj, robj *value) { /* Create the list if the key does not exist */ if (!dstobj) { - dstobj = createZiplistObject(); + dstobj = createQuicklistObject(); dbAdd(c->db,dstkey,dstobj); } signalModifiedKey(c->db,dstkey); @@ -1010,7 +846,9 @@ void handleClientsBlockedOnLists(void) { } } - if (listTypeLength(o) == 0) dbDelete(rl->db,rl->key); + if (listTypeLength(o) == 0) { + dbDelete(rl->db,rl->key); + } /* We don't call signalModifiedKey() as it was already called * when an element was pushed on the list. */ } diff --git a/tests/support/test.tcl b/tests/support/test.tcl index 7d390cc47..31371c567 100644 --- a/tests/support/test.tcl +++ b/tests/support/test.tcl @@ -19,9 +19,12 @@ proc assert_match {pattern value} { } } -proc assert_equal {expected value} { +proc assert_equal {expected value {detail ""}} { if {$expected ne $value} { - error "assertion:Expected '$value' to be equal to '$expected'" + if {$detail ne ""} { + set detail " (detail: $detail)" + } + error "assertion:Expected '$value' to be equal to '$expected'$detail" } } diff --git a/tests/unit/aofrw.tcl b/tests/unit/aofrw.tcl index a2d74168f..4fdbdc6c6 100644 --- a/tests/unit/aofrw.tcl +++ b/tests/unit/aofrw.tcl @@ -77,10 +77,10 @@ start_server {tags {"aofrw"}} { } foreach d {string int} { - foreach e {ziplist linkedlist} { + foreach e {quicklist} { test "AOF rewrite of list with $e encoding, $d data" { r flushall - if {$e eq {ziplist}} {set len 10} else {set len 1000} + set len 1000 for {set j 0} {$j < $len} {incr j} { if {$d eq {string}} { set data [randstring 0 16 alpha] diff --git a/tests/unit/sort.tcl b/tests/unit/sort.tcl index 8ebd75965..5ae48d450 100644 --- a/tests/unit/sort.tcl +++ b/tests/unit/sort.tcl @@ -36,9 +36,9 @@ start_server { } foreach {num cmd enc title} { - 16 lpush ziplist "Ziplist" - 1000 lpush linkedlist "Linked list" - 10000 lpush linkedlist "Big Linked list" + 16 lpush quicklist "Old Ziplist" + 1000 lpush quicklist "Old Linked list" + 10000 lpush quicklist "Old Big Linked list" 16 sadd intset "Intset" 1000 sadd hashtable "Hash table" 10000 sadd hashtable "Big Hash table" @@ -85,14 +85,14 @@ start_server { r sort tosort BY weight_* store sort-res assert_equal $result [r lrange sort-res 0 -1] assert_equal 16 [r llen sort-res] - assert_encoding ziplist sort-res + assert_encoding quicklist sort-res } test "SORT BY hash field STORE" { r sort tosort BY wobj_*->weight store sort-res assert_equal $result [r lrange sort-res 0 -1] assert_equal 16 [r llen sort-res] - assert_encoding ziplist sort-res + assert_encoding quicklist sort-res } test "SORT extracts STORE correctly" { diff --git a/tests/unit/type/list-2.tcl b/tests/unit/type/list-2.tcl index bf6a055eb..d25873912 100644 --- a/tests/unit/type/list-2.tcl +++ b/tests/unit/type/list-2.tcl @@ -2,7 +2,7 @@ start_server { tags {"list"} overrides { "list-max-ziplist-value" 16 - "list-max-ziplist-entries" 256 + "list-max-ziplist-entries" 4 } } { source "tests/unit/type/list-common.tcl" @@ -28,14 +28,18 @@ start_server { for {set i 0} {$i < 1000} {incr i} { set min [expr {int(rand()*$startlen)}] set max [expr {$min+int(rand()*$startlen)}] + set before_len [llength $mylist] + set before_len_r [r llen mylist] set mylist [lrange $mylist $min $max] r ltrim mylist $min $max - assert_equal $mylist [r lrange mylist 0 -1] + assert_equal $mylist [r lrange mylist 0 -1] "failed trim" + set starting [r llen mylist] for {set j [r llen mylist]} {$j < $startlen} {incr j} { set str [randomInt 9223372036854775807] r rpush mylist $str lappend mylist $str + assert_equal $mylist [r lrange mylist 0 -1] "failed append match" } } } diff --git a/tests/unit/type/list-3.tcl b/tests/unit/type/list-3.tcl index 94f9a0b79..81ef9b3e5 100644 --- a/tests/unit/type/list-3.tcl +++ b/tests/unit/type/list-3.tcl @@ -2,7 +2,7 @@ start_server { tags {list ziplist} overrides { "list-max-ziplist-value" 200000 - "list-max-ziplist-entries" 256 + "list-max-ziplist-entries" 16 } } { test {Explicit regression for a list bug} { diff --git a/tests/unit/type/list.tcl b/tests/unit/type/list.tcl index c8e26602b..0c0c07761 100644 --- a/tests/unit/type/list.tcl +++ b/tests/unit/type/list.tcl @@ -2,24 +2,24 @@ start_server { tags {"list"} overrides { "list-max-ziplist-value" 16 - "list-max-ziplist-entries" 256 + "list-max-ziplist-entries" 5 } } { source "tests/unit/type/list-common.tcl" test {LPUSH, RPUSH, LLENGTH, LINDEX, LPOP - ziplist} { # first lpush then rpush - assert_equal 1 [r lpush myziplist1 a] - assert_equal 2 [r rpush myziplist1 b] - assert_equal 3 [r rpush myziplist1 c] + assert_equal 1 [r lpush myziplist1 aa] + assert_equal 2 [r rpush myziplist1 bb] + assert_equal 3 [r rpush myziplist1 cc] assert_equal 3 [r llen myziplist1] - assert_equal a [r lindex myziplist1 0] - assert_equal b [r lindex myziplist1 1] - assert_equal c [r lindex myziplist1 2] + assert_equal aa [r lindex myziplist1 0] + assert_equal bb [r lindex myziplist1 1] + assert_equal cc [r lindex myziplist1 2] assert_equal {} [r lindex myziplist2 3] - assert_equal c [r rpop myziplist1] - assert_equal a [r lpop myziplist1] - assert_encoding ziplist myziplist1 + assert_equal cc [r rpop myziplist1] + assert_equal aa [r lpop myziplist1] + assert_encoding quicklist myziplist1 # first rpush then lpush assert_equal 1 [r rpush myziplist2 a] @@ -32,13 +32,13 @@ start_server { assert_equal {} [r lindex myziplist2 3] assert_equal a [r rpop myziplist2] assert_equal c [r lpop myziplist2] - assert_encoding ziplist myziplist2 + assert_encoding quicklist myziplist2 } test {LPUSH, RPUSH, LLENGTH, LINDEX, LPOP - regular list} { # first lpush then rpush assert_equal 1 [r lpush mylist1 $largevalue(linkedlist)] - assert_encoding linkedlist mylist1 + assert_encoding quicklist mylist1 assert_equal 2 [r rpush mylist1 b] assert_equal 3 [r rpush mylist1 c] assert_equal 3 [r llen mylist1] @@ -51,7 +51,7 @@ start_server { # first rpush then lpush assert_equal 1 [r rpush mylist2 $largevalue(linkedlist)] - assert_encoding linkedlist mylist2 + assert_encoding quicklist mylist2 assert_equal 2 [r lpush mylist2 b] assert_equal 3 [r lpush mylist2 c] assert_equal 3 [r llen mylist2] @@ -74,34 +74,22 @@ start_server { assert_equal {d c b a 0 1 2 3} [r lrange mylist 0 -1] } - test {DEL a list - ziplist} { - assert_equal 1 [r del myziplist2] - assert_equal 0 [r exists myziplist2] - assert_equal 0 [r llen myziplist2] - } - - test {DEL a list - regular list} { + test {DEL a list} { assert_equal 1 [r del mylist2] assert_equal 0 [r exists mylist2] assert_equal 0 [r llen mylist2] } - proc create_ziplist {key entries} { + proc create_list {key entries} { r del $key foreach entry $entries { r rpush $key $entry } - assert_encoding ziplist $key - } - - proc create_linkedlist {key entries} { - r del $key - foreach entry $entries { r rpush $key $entry } - assert_encoding linkedlist $key + assert_encoding quicklist $key } foreach {type large} [array get largevalue] { test "BLPOP, BRPOP: single existing list - $type" { set rd [redis_deferring_client] - create_$type blist "a b $large c d" + create_list blist "a b $large c d" $rd blpop blist 1 assert_equal {blist a} [$rd read] @@ -116,8 +104,8 @@ start_server { test "BLPOP, BRPOP: multiple existing lists - $type" { set rd [redis_deferring_client] - create_$type blist1 "a $large c" - create_$type blist2 "d $large f" + create_list blist1 "a $large c" + create_list blist2 "d $large f" $rd blpop blist1 blist2 1 assert_equal {blist1 a} [$rd read] @@ -137,7 +125,7 @@ start_server { test "BLPOP, BRPOP: second list has an entry - $type" { set rd [redis_deferring_client] r del blist1 - create_$type blist2 "d $large f" + create_list blist2 "d $large f" $rd blpop blist1 blist2 1 assert_equal {blist2 d} [$rd read] @@ -151,7 +139,7 @@ start_server { r del target set rd [redis_deferring_client] - create_$type blist "a b $large c d" + create_list blist "a b $large c d" $rd brpoplpush blist target 1 assert_equal d [$rd read] @@ -517,28 +505,28 @@ start_server { foreach {type large} [array get largevalue] { test "LPUSHX, RPUSHX - $type" { - create_$type xlist "$large c" + create_list xlist "$large c" assert_equal 3 [r rpushx xlist d] assert_equal 4 [r lpushx xlist a] assert_equal "a $large c d" [r lrange xlist 0 -1] } test "LINSERT - $type" { - create_$type xlist "a $large c d" - assert_equal 5 [r linsert xlist before c zz] - assert_equal "a $large zz c d" [r lrange xlist 0 10] - assert_equal 6 [r linsert xlist after c yy] - assert_equal "a $large zz c yy d" [r lrange xlist 0 10] - assert_equal 7 [r linsert xlist after d dd] - assert_equal -1 [r linsert xlist after bad ddd] - assert_equal "a $large zz c yy d dd" [r lrange xlist 0 10] - assert_equal 8 [r linsert xlist before a aa] - assert_equal -1 [r linsert xlist before bad aaa] - assert_equal "aa a $large zz c yy d dd" [r lrange xlist 0 10] + create_list xlist "a $large c d" + assert_equal 5 [r linsert xlist before c zz] "before c" + assert_equal "a $large zz c d" [r lrange xlist 0 10] "lrangeA" + assert_equal 6 [r linsert xlist after c yy] "after c" + assert_equal "a $large zz c yy d" [r lrange xlist 0 10] "lrangeB" + assert_equal 7 [r linsert xlist after d dd] "after d" + assert_equal -1 [r linsert xlist after bad ddd] "after bad" + assert_equal "a $large zz c yy d dd" [r lrange xlist 0 10] "lrangeC" + assert_equal 8 [r linsert xlist before a aa] "before a" + assert_equal -1 [r linsert xlist before bad aaa] "before bad" + assert_equal "aa a $large zz c yy d dd" [r lrange xlist 0 10] "lrangeD" # check inserting integer encoded value - assert_equal 9 [r linsert xlist before aa 42] - assert_equal 42 [r lrange xlist 0 0] + assert_equal 9 [r linsert xlist before aa 42] "before aa" + assert_equal 42 [r lrange xlist 0 0] "lrangeE" } } @@ -547,55 +535,7 @@ start_server { set e } {*ERR*syntax*error*} - test {LPUSHX, RPUSHX convert from ziplist to list} { - set large $largevalue(linkedlist) - - # convert when a large value is pushed - create_ziplist xlist a - assert_equal 2 [r rpushx xlist $large] - assert_encoding linkedlist xlist - create_ziplist xlist a - assert_equal 2 [r lpushx xlist $large] - assert_encoding linkedlist xlist - - # convert when the length threshold is exceeded - create_ziplist xlist [lrepeat 256 a] - assert_equal 257 [r rpushx xlist b] - assert_encoding linkedlist xlist - create_ziplist xlist [lrepeat 256 a] - assert_equal 257 [r lpushx xlist b] - assert_encoding linkedlist xlist - } - - test {LINSERT convert from ziplist to list} { - set large $largevalue(linkedlist) - - # convert when a large value is inserted - create_ziplist xlist a - assert_equal 2 [r linsert xlist before a $large] - assert_encoding linkedlist xlist - create_ziplist xlist a - assert_equal 2 [r linsert xlist after a $large] - assert_encoding linkedlist xlist - - # convert when the length threshold is exceeded - create_ziplist xlist [lrepeat 256 a] - assert_equal 257 [r linsert xlist before a a] - assert_encoding linkedlist xlist - create_ziplist xlist [lrepeat 256 a] - assert_equal 257 [r linsert xlist after a a] - assert_encoding linkedlist xlist - - # don't convert when the value could not be inserted - create_ziplist xlist [lrepeat 256 a] - assert_equal -1 [r linsert xlist before foo a] - assert_encoding ziplist xlist - create_ziplist xlist [lrepeat 256 a] - assert_equal -1 [r linsert xlist after foo a] - assert_encoding ziplist xlist - } - - foreach {type num} {ziplist 250 linkedlist 500} { + foreach {type num} {quicklist 250 quicklist 500} { proc check_numbered_list_consistency {key} { set len [r llen $key] for {set i 0} {$i < $len} {incr i} { @@ -664,16 +604,16 @@ start_server { foreach {type large} [array get largevalue] { test "RPOPLPUSH base case - $type" { r del mylist1 mylist2 - create_$type mylist1 "a $large c d" + create_list mylist1 "a $large c d" assert_equal d [r rpoplpush mylist1 mylist2] assert_equal c [r rpoplpush mylist1 mylist2] assert_equal "a $large" [r lrange mylist1 0 -1] assert_equal "c d" [r lrange mylist2 0 -1] - assert_encoding ziplist mylist2 + assert_encoding quicklist mylist2 } test "RPOPLPUSH with the same list as src and dst - $type" { - create_$type mylist "a $large c" + create_list mylist "a $large c" assert_equal "a $large c" [r lrange mylist 0 -1] assert_equal c [r rpoplpush mylist mylist] assert_equal "c a $large" [r lrange mylist 0 -1] @@ -681,8 +621,8 @@ start_server { foreach {othertype otherlarge} [array get largevalue] { test "RPOPLPUSH with $type source and existing target $othertype" { - create_$type srclist "a b c $large" - create_$othertype dstlist "$otherlarge" + create_list srclist "a b c $large" + create_list dstlist "$otherlarge" assert_equal $large [r rpoplpush srclist dstlist] assert_equal c [r rpoplpush srclist dstlist] assert_equal "a b" [r lrange srclist 0 -1] @@ -691,7 +631,7 @@ start_server { # When we rpoplpush'ed a large value, dstlist should be # converted to the same encoding as srclist. if {$type eq "linkedlist"} { - assert_encoding linkedlist dstlist + assert_encoding quicklist dstlist } } } @@ -713,7 +653,7 @@ start_server { } test {RPOPLPUSH against non list dst key} { - create_ziplist srclist {a b c d} + create_list srclist {a b c d} r set dstlist x assert_error WRONGTYPE* {r rpoplpush srclist dstlist} assert_type string dstlist @@ -727,7 +667,7 @@ start_server { foreach {type large} [array get largevalue] { test "Basic LPOP/RPOP - $type" { - create_$type mylist "$large 1 2" + create_list mylist "$large 1 2" assert_equal $large [r lpop mylist] assert_equal 2 [r rpop mylist] assert_equal 1 [r lpop mylist] @@ -745,7 +685,7 @@ start_server { assert_error WRONGTYPE* {r rpop notalist} } - foreach {type num} {ziplist 250 linkedlist 500} { + foreach {type num} {quicklist 250 quicklist 500} { test "Mass RPOP/LPOP - $type" { r del mylist set sum1 0 @@ -765,24 +705,24 @@ start_server { foreach {type large} [array get largevalue] { test "LRANGE basics - $type" { - create_$type mylist "$large 1 2 3 4 5 6 7 8 9" + create_list mylist "$large 1 2 3 4 5 6 7 8 9" assert_equal {1 2 3 4 5 6 7 8} [r lrange mylist 1 -2] assert_equal {7 8 9} [r lrange mylist -3 -1] assert_equal {4} [r lrange mylist 4 4] } test "LRANGE inverted indexes - $type" { - create_$type mylist "$large 1 2 3 4 5 6 7 8 9" + create_list mylist "$large 1 2 3 4 5 6 7 8 9" assert_equal {} [r lrange mylist 6 2] } test "LRANGE out of range indexes including the full list - $type" { - create_$type mylist "$large 1 2 3" + create_list mylist "$large 1 2 3" assert_equal "$large 1 2 3" [r lrange mylist -1000 1000] } test "LRANGE out of range negative end index - $type" { - create_$type mylist "$large 1 2 3" + create_list mylist "$large 1 2 3" assert_equal $large [r lrange mylist 0 -4] assert_equal {} [r lrange mylist 0 -5] } @@ -796,7 +736,7 @@ start_server { proc trim_list {type min max} { upvar 1 large large r del mylist - create_$type mylist "1 2 3 4 $large" + create_list mylist "1 2 3 4 $large" r ltrim mylist $min $max r lrange mylist 0 -1 } @@ -825,7 +765,7 @@ start_server { foreach {type large} [array get largevalue] { test "LSET - $type" { - create_$type mylist "99 98 $large 96 95" + create_list mylist "99 98 $large 96 95" r lset mylist 1 foo r lset mylist -1 bar assert_equal "99 foo $large 96 bar" [r lrange mylist 0 -1] @@ -847,7 +787,7 @@ start_server { foreach {type e} [array get largevalue] { test "LREM remove all the occurrences - $type" { - create_$type mylist "$e foo bar foobar foobared zap bar test foo" + create_list mylist "$e foo bar foobar foobared zap bar test foo" assert_equal 2 [r lrem mylist 0 bar] assert_equal "$e foo foobar foobared zap test foo" [r lrange mylist 0 -1] } @@ -863,7 +803,7 @@ start_server { } test "LREM starting from tail with negative count - $type" { - create_$type mylist "$e foo bar foobar foobared zap bar test foo foo" + create_list mylist "$e foo bar foobar foobared zap bar test foo foo" assert_equal 1 [r lrem mylist -1 bar] assert_equal "$e foo bar foobar foobared zap test foo foo" [r lrange mylist 0 -1] } @@ -874,7 +814,7 @@ start_server { } test "LREM deleting objects that may be int encoded - $type" { - create_$type myotherlist "$e 1 2 3" + create_list myotherlist "$e 1 2 3" assert_equal 1 [r lrem myotherlist 1 2] assert_equal 3 [r llen myotherlist] } From 9d2dc0249c5ef99586710d711e1381c4178aeb39 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Fri, 21 Nov 2014 14:52:10 -0500 Subject: [PATCH 13/29] Add ziplistMerge() This started out as #2158 by sunheehnus, but I kept rewriting it until I could understand things more easily and get a few more correctness guarantees out of the readability flow. The original commit created and returned a new ziplist with the contents of both input ziplists, but I prefer to grow one of the input ziplists and destroy the other one. So, instead of malloc+copy as in #2158, the merge now reallocs one of the existing ziplists and copies the other ziplist into the new space. Also added merge test cases to ziplistTest() --- src/quicklist.c | 71 +++++------------- src/ziplist.c | 192 ++++++++++++++++++++++++++++++++++++++++++++++++ src/ziplist.h | 1 + 3 files changed, 210 insertions(+), 54 deletions(-) diff --git a/src/quicklist.c b/src/quicklist.c index 415af36c9..cb2b7bdaf 100644 --- a/src/quicklist.c +++ b/src/quicklist.c @@ -351,65 +351,28 @@ int quicklistReplaceAtIndex(quicklist *quicklist, long index, void *data, static quicklistNode *_quicklistZiplistMerge(quicklist *quicklist, quicklistNode *a, quicklistNode *b) { - /* Merge into node with largest initial count */ - quicklistNode *target = a->count > b->count ? a : b; + D("Requested merge (a,b) (%u, %u)", a->count, b->count); - if (a->count == 0 || b->count == 0) - return NULL; + if ((ziplistMerge(&a->zl, &b->zl))) { + /* We merged ziplists! Now remove the unused quicklistNode. */ + quicklistNode *keep = NULL, *nokeep = NULL; + if (!a->zl) { + nokeep = a; + keep = b; + } else if (!b->zl) { + nokeep = b; + keep = a; + } + keep->count = ziplistLen(keep->zl); - D("Requested merge (a,b) (%u, %u) and picked target %u", a->count, b->count, - target->count); + nokeep->count = 0; + __quicklistDelNode(quicklist, nokeep); - int where; - unsigned char *p = NULL; - if (target == a) { - /* If target is node a, we append node b to node a, in-order */ - where = ZIPLIST_TAIL; - p = ziplistIndex(b->zl, 0); - D("WILL TRAVERSE B WITH LENGTH: %u, %u", b->count, ziplistLen(b->zl)); + return keep; } else { - /* If target b, we prepend node a to node b, in reverse order of a */ - where = ZIPLIST_HEAD; - p = ziplistIndex(a->zl, -1); - D("WILL TRAVERSE A WITH LENGTH: %u, %u", a->count, ziplistLen(a->zl)); + /* else, the merge returned NULL and nothing changed. */ + return NULL; } - - unsigned char *val; - unsigned int sz; - long long longval; - char lv[32] = { 0 }; - /* NOTE: We could potentially create a built-in ziplist operation - * allowing direct merging of two ziplists. It would be more memory - * efficient (one big realloc instead of incremental), but it's more - * complex than using the existing ziplist API to read/push as below. */ - while (ziplistGet(p, &val, &sz, &longval)) { - if (!val) { - sz = ll2string(lv, sizeof(lv), longval); - val = (unsigned char *)lv; - } - target->zl = ziplistPush(target->zl, val, sz, where); - if (target == a) { - p = ziplistNext(b->zl, p); - b->count--; - a->count++; - } else { - p = ziplistPrev(a->zl, p); - a->count--; - b->count++; - } - D("Loop A: %u, B: %u", a->count, b->count); - } - - /* At this point, target is populated and not-target needs - * to be free'd and removed from the quicklist. */ - if (target == a) { - D("Deleting node B with current count: %d", b->count); - __quicklistDelNode(quicklist, b); - } else if (target == b) { - D("Deleting node A with current count: %d", a->count); - __quicklistDelNode(quicklist, a); - } - return target; } /* Attempt to merge ziplists within two nodes on either side of 'center'. diff --git a/src/ziplist.c b/src/ziplist.c index 111f2a9a7..07bf657fe 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -143,6 +143,7 @@ #define ZIPLIST_TAIL_OFFSET(zl) (*((uint32_t*)((zl)+sizeof(uint32_t)))) #define ZIPLIST_LENGTH(zl) (*((uint16_t*)((zl)+sizeof(uint32_t)*2))) #define ZIPLIST_HEADER_SIZE (sizeof(uint32_t)*2+sizeof(uint16_t)) +#define ZIPLIST_END_SIZE (sizeof(uint8_t)) #define ZIPLIST_ENTRY_HEAD(zl) ((zl)+ZIPLIST_HEADER_SIZE) #define ZIPLIST_ENTRY_TAIL(zl) ((zl)+intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))) #define ZIPLIST_ENTRY_END(zl) ((zl)+intrev32ifbe(ZIPLIST_BYTES(zl))-1) @@ -176,6 +177,8 @@ typedef struct zlentry { if ((encoding) < ZIP_STR_MASK) (encoding) &= ZIP_STR_MASK; \ } while(0) +void ziplistRepr(unsigned char *zl); + /* Return bytes needed to store integer encoded by 'encoding' */ static unsigned int zipIntSize(unsigned char encoding) { switch(encoding) { @@ -670,6 +673,121 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig return zl; } +/* Merge ziplists 'first' and 'second' by appending 'second' to 'first'. + * + * NOTE: The larger ziplist is reallocated to contain the new merged ziplist. + * Either 'first' or 'second' can be used for the result. The parameter not + * used will be free'd and set to NULL. + * + * After calling this function, the input parameters are no longer valid since + * they are changed and free'd in-place. + * + * The result ziplist is the contents of 'first' followed by 'second'. + * + * On failure: returns NULL if the merge is impossible. + * On success: returns the merged ziplist (which is expanded version of either + * 'first' or 'second', also frees the other unused input ziplist, and sets the + * input ziplist argument equal to newly reallocated ziplist return value. */ +unsigned char *ziplistMerge(unsigned char **first, unsigned char **second) { + /* If any params are null, we can't merge, so NULL. */ + if (first == NULL || *first == NULL || second == NULL || *second == NULL) + return NULL; + + /* Can't merge same list into itself. */ + if (*first == *second) + return NULL; + + size_t first_bytes = intrev32ifbe(ZIPLIST_BYTES(*first)); + size_t first_len = intrev16ifbe(ZIPLIST_LENGTH(*first)); + + size_t second_bytes = intrev32ifbe(ZIPLIST_BYTES(*second)); + size_t second_len = intrev16ifbe(ZIPLIST_LENGTH(*second)); + + int append; + unsigned char *source, *target; + size_t target_bytes, source_bytes; + /* Pick the largest ziplist so we can resize easily in-place. + * We must also track if we are now appending or prepending to + * the target ziplist. */ + if (first_len >= second_len) { + /* retain first, append second to first. */ + target = *first; + target_bytes = first_bytes; + source = *second; + source_bytes = second_bytes; + append = 1; + } else { + /* else, retain second, prepend first to second. */ + target = *second; + target_bytes = second_bytes; + source = *first; + source_bytes = first_bytes; + append = 0; + } + + /* Calculate final bytes (subtract one pair of metadata) */ + size_t zlbytes = first_bytes + second_bytes - + ZIPLIST_HEADER_SIZE - ZIPLIST_END_SIZE; + size_t zllength = first_len + second_len; + + /* Combined zl length should be limited within UINT16_MAX */ + zllength = zllength < UINT16_MAX ? zllength : UINT16_MAX; + + /* Save offset positions before we start ripping memory apart. */ + size_t first_offset = intrev32ifbe(ZIPLIST_TAIL_OFFSET(*first)); + size_t second_offset = intrev32ifbe(ZIPLIST_TAIL_OFFSET(*second)); + + /* Extend target to new zlbytes then append or prepend source. */ + target = zrealloc(target, zlbytes); + if (append) { + /* append == appending to target */ + /* Copy source after target (copying over original [END]): + * [TARGET - END, SOURCE - HEADER] */ + memcpy(target + target_bytes - ZIPLIST_END_SIZE, + source + ZIPLIST_HEADER_SIZE, + source_bytes - ZIPLIST_HEADER_SIZE); + } else { + /* !append == prepending to target */ + /* Move target *contents* exactly size of (source - [END]), + * then copy source into vacataed space (source - [END]): + * [SOURCE - END, TARGET - HEADER] */ + memmove(target + source_bytes - ZIPLIST_END_SIZE, + target + ZIPLIST_HEADER_SIZE, + target_bytes - ZIPLIST_HEADER_SIZE); + memcpy(target, source, source_bytes - ZIPLIST_END_SIZE); + } + + /* Update header metadata. */ + ZIPLIST_BYTES(target) = intrev32ifbe(zlbytes); + ZIPLIST_LENGTH(target) = intrev16ifbe(zllength); + /* New tail offset is: + * + N bytes of first ziplist + * - 1 byte for [END] of first ziplist + * + M bytes for the offset of the original tail of the second ziplist + * - J bytes for HEADER because second_offset keeps no header. */ + ZIPLIST_TAIL_OFFSET(target) = intrev32ifbe( + (first_bytes - ZIPLIST_END_SIZE) + + (second_offset - ZIPLIST_HEADER_SIZE)); + + /* __ziplistCascadeUpdate just fixes the prev length values until it finds a + * correct prev length value (then it assumes the rest of the list is okay). + * We tell CascadeUpdate to start at the first ziplist's tail element to fix + * the merge seam. */ + target = __ziplistCascadeUpdate(target, target+first_offset); + + /* Now free and NULL out what we didn't realloc */ + if (append) { + zfree(*second); + *second = NULL; + *first = target; + } else { + zfree(*first); + *first = NULL; + *second = target; + } + return target; +} + unsigned char *ziplistPush(unsigned char *zl, unsigned char *s, unsigned int slen, int where) { unsigned char *p; p = (where == ZIPLIST_HEAD) ? ZIPLIST_ENTRY_HEAD(zl) : ZIPLIST_ENTRY_END(zl); @@ -1456,6 +1574,80 @@ int ziplistTest(int argc, char **argv) { printf("SUCCESS\n\n"); } + printf("Merge test:\n"); + { + /* create list gives us: [hello, foo, quux, 1024] */ + zl = createList(); + unsigned char *zl2 = createList(); + + unsigned char *zl3 = ziplistNew(); + unsigned char *zl4 = ziplistNew(); + + if (ziplistMerge(&zl4, &zl4)) { + printf("ERROR: Allowed merging of one ziplist into itself.\n"); + return 1; + } + + /* Merge two empty ziplists, get empty result back. */ + zl4 = ziplistMerge(&zl3, &zl4); + ziplistRepr(zl4); + if (ziplistLen(zl4)) { + printf("ERROR: Merging two empty ziplists created entries.\n"); + return 1; + } + zfree(zl4); + + zl2 = ziplistMerge(&zl, &zl2); + /* merge gives us: [hello, foo, quux, 1024, hello, foo, quux, 1024] */ + ziplistRepr(zl2); + + if (ziplistLen(zl2) != 8) { + printf("ERROR: Merged length not 8, but: %u\n", ziplistLen(zl2)); + return 1; + } + + p = ziplistIndex(zl2,0); + if (!ziplistCompare(p,(unsigned char*)"hello",5)) { + printf("ERROR: not \"hello\"\n"); + return 1; + } + if (ziplistCompare(p,(unsigned char*)"hella",5)) { + printf("ERROR: \"hella\"\n"); + return 1; + } + + p = ziplistIndex(zl2,3); + if (!ziplistCompare(p,(unsigned char*)"1024",4)) { + printf("ERROR: not \"1024\"\n"); + return 1; + } + if (ziplistCompare(p,(unsigned char*)"1025",4)) { + printf("ERROR: \"1025\"\n"); + return 1; + } + + p = ziplistIndex(zl2,4); + if (!ziplistCompare(p,(unsigned char*)"hello",5)) { + printf("ERROR: not \"hello\"\n"); + return 1; + } + if (ziplistCompare(p,(unsigned char*)"hella",5)) { + printf("ERROR: \"hella\"\n"); + return 1; + } + + p = ziplistIndex(zl2,7); + if (!ziplistCompare(p,(unsigned char*)"1024",4)) { + printf("ERROR: not \"1024\"\n"); + return 1; + } + if (ziplistCompare(p,(unsigned char*)"1025",4)) { + printf("ERROR: \"1025\"\n"); + return 1; + } + printf("SUCCESS\n\n"); + } + printf("Stress with random payloads of different encoding:\n"); { int i,j,len,where; diff --git a/src/ziplist.h b/src/ziplist.h index ef0e27140..e92b5e783 100644 --- a/src/ziplist.h +++ b/src/ziplist.h @@ -32,6 +32,7 @@ #define ZIPLIST_TAIL 1 unsigned char *ziplistNew(void); +unsigned char *ziplistMerge(unsigned char **first, unsigned char **second); unsigned char *ziplistPush(unsigned char *zl, unsigned char *s, unsigned int slen, int where); unsigned char *ziplistIndex(unsigned char *zl, int index); unsigned char *ziplistNext(unsigned char *zl, unsigned char *p); From 0f15eb183b13108b08141d164edabae5b4bbef99 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Fri, 21 Nov 2014 21:03:25 -0500 Subject: [PATCH 14/29] Free ziplist test lists during tests Freeing our test lists helps keep valgrind output clean --- src/ziplist.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/ziplist.c b/src/ziplist.c index 07bf657fe..7428d30e9 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -1222,6 +1222,8 @@ int ziplistTest(int argc, char **argv) { zl = createIntList(); ziplistRepr(zl); + zfree(zl); + zl = createList(); ziplistRepr(zl); @@ -1237,6 +1239,8 @@ int ziplistTest(int argc, char **argv) { zl = pop(zl,ZIPLIST_TAIL); ziplistRepr(zl); + zfree(zl); + printf("Get element at index 3:\n"); { zl = createList(); @@ -1252,6 +1256,7 @@ int ziplistTest(int argc, char **argv) { printf("%lld\n", value); } printf("\n"); + zfree(zl); } printf("Get element at index 4 (out of range):\n"); @@ -1265,6 +1270,7 @@ int ziplistTest(int argc, char **argv) { return 1; } printf("\n"); + zfree(zl); } printf("Get element at index -1 (last element):\n"); @@ -1282,6 +1288,7 @@ int ziplistTest(int argc, char **argv) { printf("%lld\n", value); } printf("\n"); + zfree(zl); } printf("Get element at index -4 (first element):\n"); @@ -1299,6 +1306,7 @@ int ziplistTest(int argc, char **argv) { printf("%lld\n", value); } printf("\n"); + zfree(zl); } printf("Get element at index -5 (reverse out of range):\n"); @@ -1312,6 +1320,7 @@ int ziplistTest(int argc, char **argv) { return 1; } printf("\n"); + zfree(zl); } printf("Iterate list from 0 to end:\n"); @@ -1329,6 +1338,7 @@ int ziplistTest(int argc, char **argv) { printf("\n"); } printf("\n"); + zfree(zl); } printf("Iterate list from 1 to end:\n"); @@ -1346,6 +1356,7 @@ int ziplistTest(int argc, char **argv) { printf("\n"); } printf("\n"); + zfree(zl); } printf("Iterate list from 2 to end:\n"); @@ -1363,6 +1374,7 @@ int ziplistTest(int argc, char **argv) { printf("\n"); } printf("\n"); + zfree(zl); } printf("Iterate starting out of range:\n"); @@ -1375,6 +1387,7 @@ int ziplistTest(int argc, char **argv) { printf("ERROR\n"); } printf("\n"); + zfree(zl); } printf("Iterate from back to front:\n"); @@ -1392,6 +1405,7 @@ int ziplistTest(int argc, char **argv) { printf("\n"); } printf("\n"); + zfree(zl); } printf("Iterate from back to front, deleting all items:\n"); @@ -1410,6 +1424,7 @@ int ziplistTest(int argc, char **argv) { printf("\n"); } printf("\n"); + zfree(zl); } printf("Delete inclusive range 0,0:\n"); @@ -1417,6 +1432,7 @@ int ziplistTest(int argc, char **argv) { zl = createList(); zl = ziplistDeleteRange(zl, 0, 1); ziplistRepr(zl); + zfree(zl); } printf("Delete inclusive range 0,1:\n"); @@ -1424,6 +1440,7 @@ int ziplistTest(int argc, char **argv) { zl = createList(); zl = ziplistDeleteRange(zl, 0, 2); ziplistRepr(zl); + zfree(zl); } printf("Delete inclusive range 1,2:\n"); @@ -1431,6 +1448,7 @@ int ziplistTest(int argc, char **argv) { zl = createList(); zl = ziplistDeleteRange(zl, 1, 2); ziplistRepr(zl); + zfree(zl); } printf("Delete with start index out of range:\n"); @@ -1438,6 +1456,7 @@ int ziplistTest(int argc, char **argv) { zl = createList(); zl = ziplistDeleteRange(zl, 5, 1); ziplistRepr(zl); + zfree(zl); } printf("Delete with num overflow:\n"); @@ -1445,6 +1464,7 @@ int ziplistTest(int argc, char **argv) { zl = createList(); zl = ziplistDeleteRange(zl, 1, 5); ziplistRepr(zl); + zfree(zl); } printf("Delete foo while iterating:\n"); @@ -1469,6 +1489,7 @@ int ziplistTest(int argc, char **argv) { } printf("\n"); ziplistRepr(zl); + zfree(zl); } printf("Regression test for >255 byte strings:\n"); @@ -1488,6 +1509,7 @@ int ziplistTest(int argc, char **argv) { assert(ziplistGet(p,&entry,&elen,&value)); assert(strncmp(v2,(char*)entry,elen) == 0); printf("SUCCESS\n\n"); + zfree(zl); } printf("Regression test deleting next to last entries:\n"); @@ -1526,6 +1548,7 @@ int ziplistTest(int argc, char **argv) { assert(e[1].prevrawlensize == 5); printf("SUCCESS\n\n"); + zfree(zl); } printf("Create long list and check indices:\n"); @@ -1547,6 +1570,7 @@ int ziplistTest(int argc, char **argv) { assert(999-i == value); } printf("SUCCESS\n\n"); + zfree(zl); } printf("Compare strings with ziplist entries:\n"); @@ -1572,6 +1596,7 @@ int ziplistTest(int argc, char **argv) { return 1; } printf("SUCCESS\n\n"); + zfree(zl); } printf("Merge test:\n"); @@ -1646,6 +1671,7 @@ int ziplistTest(int argc, char **argv) { return 1; } printf("SUCCESS\n\n"); + zfree(zl); } printf("Stress with random payloads of different encoding:\n"); From 60a9418ed9adaff7844113b89fd4a4c01100f0b9 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Tue, 25 Nov 2014 12:19:58 -0500 Subject: [PATCH 15/29] redis-benchmark: Add RPUSH and RPOP tests --- src/redis-benchmark.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c index 199203812..7567e0181 100644 --- a/src/redis-benchmark.c +++ b/src/redis-benchmark.c @@ -738,12 +738,24 @@ int main(int argc, const char **argv) { free(cmd); } + if (test_is_selected("rpush")) { + len = redisFormatCommand(&cmd,"RPUSH mylist %s",data); + benchmark("RPUSH",cmd,len); + free(cmd); + } + if (test_is_selected("lpop")) { len = redisFormatCommand(&cmd,"LPOP mylist"); benchmark("LPOP",cmd,len); free(cmd); } + if (test_is_selected("rpop")) { + len = redisFormatCommand(&cmd,"RPOP mylist"); + benchmark("RPOP",cmd,len); + free(cmd); + } + if (test_is_selected("sadd")) { len = redisFormatCommand(&cmd, "SADD myset element:__rand_int__"); From c6bf20c2a7423f464210dd19dd59073a6bb846a2 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Wed, 26 Nov 2014 13:42:10 -0500 Subject: [PATCH 16/29] Add adaptive quicklist fill factor Fill factor now has two options: - negative (1-5) for size-based ziplist filling - positive for length-based ziplist filling with implicit size cap. Negative offsets define ziplist size limits of: -1: 4k -2: 8k -3: 16k -4: 32k -5: 64k Positive offsets now automatically limit their max size to 8k. Any elements larger than 8k will be in individual nodes. Positive ziplist fill factors will keep adding elements to a ziplist until one of: - ziplist has FILL number of elements - or - - ziplist grows above our ziplist max size (currently 8k) When using positive fill factors, if you insert a large element (over 8k), that element will automatically allocate an individual quicklist node with one element and no other elements will be in the same ziplist inside that quicklist node. When using negative fill factors, elements up to the size limit can be added to one quicklist node. If an element is added larger than the max ziplist size, that element will be allocated an individual ziplist in a new quicklist node. Tests also updated to start testing at fill factor -5. --- src/quicklist.c | 225 ++++++++++++++++++++++++++++++++++-------------- src/quicklist.h | 22 ++--- 2 files changed, 173 insertions(+), 74 deletions(-) diff --git a/src/quicklist.c b/src/quicklist.c index cb2b7bdaf..2d711049d 100644 --- a/src/quicklist.c +++ b/src/quicklist.c @@ -38,6 +38,11 @@ #include /* for printf (debug printing), snprintf (genstr) */ #endif +/* Optimization levels for size-based filling */ +static const size_t optimization_level[] = { 4096, 8192, 16384, 32768, 65536 }; + +#define SIZE_SAFETY_LIMIT 8192 + /* If not verbose testing, remove all debug printing. */ #ifndef REDIS_TEST_VERBOSE #define D(...) @@ -82,6 +87,7 @@ static quicklistNode *quicklistCreateNode(void) { return NULL; node->zl = NULL; node->count = 0; + node->sz = 0; node->next = node->prev = NULL; return node; } @@ -156,12 +162,86 @@ static void _quicklistInsertNodeAfter(quicklist *quicklist, __quicklistInsertNode(quicklist, old_node, new_node, 1); } +static int _quicklistNodeSizeMeetsOptimizationRequirement(const size_t sz, + const int fill) { + if (fill >= 0) + return 0; + + size_t offset = (-fill) - 1; + if (offset < (sizeof(optimization_level) / sizeof(*optimization_level))) { + if (sz <= optimization_level[offset]) { + return 1; + } else { + return 0; + } + } else { + return 0; + } +} + +#define sizeMeetsSafetyLimit(sz) ((sz) <= SIZE_SAFETY_LIMIT) + +static int _quicklistNodeAllowInsert(const quicklistNode *node, const int fill, + const size_t sz) { + if (!node) + return 0; + + int ziplist_overhead; + /* size of previous offset */ + if (sz < 254) + ziplist_overhead = 1; + else + ziplist_overhead = 5; + + /* size of forward offset */ + if (sz < 64) + ziplist_overhead += 1; + else if (sz < 16384) + ziplist_overhead += 2; + else + ziplist_overhead += 5; + + /* new_sz overestimates if 'sz' encodes to an integer type */ + unsigned int new_sz = node->sz + sz + ziplist_overhead; + if (_quicklistNodeSizeMeetsOptimizationRequirement(new_sz, fill)) + return 1; + else if (!sizeMeetsSafetyLimit(new_sz)) + return 0; + else if ((int)node->count < fill) + return 1; + else + return 0; +} + +static int _quicklistNodeAllowMerge(const quicklistNode *a, + const quicklistNode *b, const int fill) { + if (!a || !b) + return 0; + + /* approximate merged ziplist size (- 11 to remove one ziplist + * header/trailer) */ + unsigned int merge_sz = a->sz + b->sz - 11; + if (_quicklistNodeSizeMeetsOptimizationRequirement(merge_sz, fill)) + return 1; + else if (!sizeMeetsSafetyLimit(merge_sz)) + return 0; + else if ((int)(a->count + b->count) <= fill) + return 1; + else + return 0; +} + +#define quicklistNodeUpdateSz(node) \ + do { \ + (node)->sz = ziplistBlobLen((node)->zl); \ + } while (0) + /* Add new entry to head of quicklist. * * Returns 'quicklist' argument. */ -quicklist *quicklistPushHead(quicklist *quicklist, const size_t fill, - void *value, size_t sz) { - if (quicklist->head && quicklist->head->count < fill) { +quicklist *quicklistPushHead(quicklist *quicklist, const int fill, void *value, + size_t sz) { + if (_quicklistNodeAllowInsert(quicklist->head, fill, sz)) { quicklist->head->zl = ziplistPush(quicklist->head->zl, value, sz, ZIPLIST_HEAD); } else { @@ -172,15 +252,16 @@ quicklist *quicklistPushHead(quicklist *quicklist, const size_t fill, } quicklist->count++; quicklist->head->count++; + quicklistNodeUpdateSz(quicklist->head); return quicklist; } /* Add new node to tail of quicklist. * * Returns 'quicklist' argument. */ -quicklist *quicklistPushTail(quicklist *quicklist, const size_t fill, - void *value, size_t sz) { - if (quicklist->tail && quicklist->tail->count < fill) { +quicklist *quicklistPushTail(quicklist *quicklist, const int fill, void *value, + size_t sz) { + if (_quicklistNodeAllowInsert(quicklist->tail, fill, sz)) { quicklist->tail->zl = ziplistPush(quicklist->tail->zl, value, sz, ZIPLIST_TAIL); } else { @@ -191,6 +272,7 @@ quicklist *quicklistPushTail(quicklist *quicklist, const size_t fill, } quicklist->count++; quicklist->tail->count++; + quicklistNodeUpdateSz(quicklist->tail); return quicklist; } @@ -202,6 +284,7 @@ void quicklistAppendZiplist(quicklist *quicklist, unsigned char *zl) { node->zl = zl; node->count = ziplistLen(node->zl); + node->sz = ziplistBlobLen(zl); _quicklistInsertNodeAfter(quicklist, quicklist->tail, node); quicklist->count += node->count; @@ -214,8 +297,7 @@ void quicklistAppendZiplist(quicklist *quicklist, unsigned char *zl) { * * Returns 'quicklist' argument. Frees passed-in ziplist 'zl' */ quicklist *quicklistAppendValuesFromZiplist(quicklist *quicklist, - const size_t fill, - unsigned char *zl) { + const int fill, unsigned char *zl) { unsigned char *value; unsigned int sz; long long longval; @@ -238,7 +320,7 @@ quicklist *quicklistAppendValuesFromZiplist(quicklist *quicklist, /* Create new (potentially multi-node) quicklist from a single existing ziplist. * * Returns new quicklist. Frees passed-in ziplist 'zl'. */ -quicklist *quicklistCreateFromZiplist(size_t fill, unsigned char *zl) { +quicklist *quicklistCreateFromZiplist(int fill, unsigned char *zl) { return quicklistAppendValuesFromZiplist(quicklistCreate(), fill, zl); } @@ -278,8 +360,11 @@ static int quicklistDelIndex(quicklist *quicklist, quicklistNode *node, int gone = 0; node->zl = ziplistDelete(node->zl, p); node->count--; - if (node->count == 0) + if (node->count == 0) { gone = 1; + } else { + quicklistNodeUpdateSz(node); + } quicklist->count--; quicklistDeleteIfEmpty(quicklist, node); /* If we deleted all the nodes, our returned pointer is no longer valid */ @@ -364,6 +449,7 @@ static quicklistNode *_quicklistZiplistMerge(quicklist *quicklist, keep = a; } keep->count = ziplistLen(keep->zl); + quicklistNodeUpdateSz(keep); nokeep->count = 0; __quicklistDelNode(quicklist, nokeep); @@ -383,7 +469,7 @@ static quicklistNode *_quicklistZiplistMerge(quicklist *quicklist, * - (center->prev, center) * - (center, center->next) */ -static void _quicklistMergeNodes(quicklist *quicklist, const size_t fill, +static void _quicklistMergeNodes(quicklist *quicklist, const int fill, quicklistNode *center) { quicklistNode *prev, *prev_prev, *next, *next_next, *target; prev = prev_prev = next = next_next = target = NULL; @@ -401,19 +487,19 @@ static void _quicklistMergeNodes(quicklist *quicklist, const size_t fill, } /* Try to merge prev_prev and prev */ - if (prev && prev_prev && (prev->count + prev_prev->count) <= fill) { + if (_quicklistNodeAllowMerge(prev, prev_prev, fill)) { _quicklistZiplistMerge(quicklist, prev_prev, prev); prev_prev = prev = NULL; /* they could have moved, invalidate them. */ } /* Try to merge next and next_next */ - if (next && next_next && (next->count + next_next->count) <= fill) { + if (_quicklistNodeAllowMerge(next, next_next, fill)) { _quicklistZiplistMerge(quicklist, next, next_next); next = next_next = NULL; /* they could have moved, invalidate them. */ } /* Try to merge center node and previous node */ - if (center->prev && (center->count + center->prev->count) <= fill) { + if (_quicklistNodeAllowMerge(center, center->prev, fill)) { target = _quicklistZiplistMerge(quicklist, center->prev, center); center = NULL; /* center could have been deleted, invalidate it. */ } else { @@ -422,8 +508,7 @@ static void _quicklistMergeNodes(quicklist *quicklist, const size_t fill, } /* Use result of center merge (or original) to merge with next node. */ - if (target && target->next && - (target->count + target->next->count) <= fill) { + if (_quicklistNodeAllowMerge(target, target->next, fill)) { _quicklistZiplistMerge(quicklist, target, target->next); } } @@ -475,9 +560,11 @@ static quicklistNode *_quicklistSplitNode(quicklistNode *node, int offset, node->zl = ziplistDeleteRange(node->zl, orig_start, orig_extent); node->count = ziplistLen(node->zl); + quicklistNodeUpdateSz(node); new_node->zl = ziplistDeleteRange(new_node->zl, new_start, new_extent); new_node->count = ziplistLen(new_node->zl); + quicklistNodeUpdateSz(new_node); D("After split lengths: orig (%d), new (%d)", node->count, new_node->count); return new_node; @@ -487,7 +574,7 @@ static quicklistNode *_quicklistSplitNode(quicklistNode *node, int offset, * * If after==1, the new value is inserted after 'entry', otherwise * the new value is inserted before 'entry'. */ -static void _quicklistInsert(quicklist *quicklist, const size_t fill, +static void _quicklistInsert(quicklist *quicklist, const int fill, quicklistEntry *entry, void *value, const size_t sz, int after) { int full = 0, at_tail = 0, at_head = 0, full_next = 0, full_prev = 0; @@ -506,7 +593,7 @@ static void _quicklistInsert(quicklist *quicklist, const size_t fill, } /* Populate accounting flags for easier boolean checks later */ - if (node->count >= fill) { + if (!_quicklistNodeAllowInsert(node, fill, sz)) { D("Current node is full with count %d with requested fill %lu", node->count, fill); full = 1; @@ -515,7 +602,7 @@ static void _quicklistInsert(quicklist *quicklist, const size_t fill, if (after && (ziplistNext(node->zl, entry->zi) == NULL)) { D("At Tail of current ziplist"); at_tail = 1; - if (node->next && node->next->count >= fill) { + if (!_quicklistNodeAllowInsert(node->next, fill, sz)) { D("Next node is full too."); full_next = 1; } @@ -524,7 +611,7 @@ static void _quicklistInsert(quicklist *quicklist, const size_t fill, if (!after && (ziplistPrev(node->zl, entry->zi) == NULL)) { D("At Head"); at_head = 1; - if (node->prev && node->prev->count >= fill) { + if (!_quicklistNodeAllowInsert(node->prev, fill, sz)) { D("Prev node is full too."); full_prev = 1; } @@ -540,10 +627,12 @@ static void _quicklistInsert(quicklist *quicklist, const size_t fill, node->zl = ziplistInsert(node->zl, next, value, sz); } node->count++; + quicklistNodeUpdateSz(node); } else if (!full && !after) { D("Not full, inserting before current position."); node->zl = ziplistInsert(node->zl, entry->zi, value, sz); node->count++; + quicklistNodeUpdateSz(node); } else if (full && at_tail && node->next && !full_next && after) { /* If we are: at tail, next has free space, and inserting after: * - insert entry at head of next node. */ @@ -551,6 +640,7 @@ static void _quicklistInsert(quicklist *quicklist, const size_t fill, new_node = node->next; new_node->zl = ziplistPush(new_node->zl, value, sz, ZIPLIST_HEAD); new_node->count++; + quicklistNodeUpdateSz(new_node); } else if (full && at_head && node->prev && !full_prev && !after) { /* If we are: at head, previous has free space, and inserting before: * - insert entry at tail of previous node. */ @@ -558,6 +648,7 @@ static void _quicklistInsert(quicklist *quicklist, const size_t fill, new_node = node->prev; new_node->zl = ziplistPush(new_node->zl, value, sz, ZIPLIST_TAIL); new_node->count++; + quicklistNodeUpdateSz(new_node); } else if (full && ((at_tail && node->next && full_next && after) || (at_head && node->prev && full_prev && !after))) { /* If we are: full, and our prev/next is full, then: @@ -566,6 +657,7 @@ static void _quicklistInsert(quicklist *quicklist, const size_t fill, new_node = quicklistCreateNode(); new_node->zl = ziplistPush(ziplistNew(), value, sz, ZIPLIST_HEAD); new_node->count++; + quicklistNodeUpdateSz(new_node); __quicklistInsertNode(quicklist, node, new_node, after); } else if (full) { /* else, node is full we need to split it. */ @@ -575,6 +667,7 @@ static void _quicklistInsert(quicklist *quicklist, const size_t fill, new_node->zl = ziplistPush(new_node->zl, value, sz, after ? ZIPLIST_HEAD : ZIPLIST_TAIL); new_node->count++; + quicklistNodeUpdateSz(new_node); __quicklistInsertNode(quicklist, node, new_node, after); _quicklistMergeNodes(quicklist, fill, node); } @@ -582,13 +675,13 @@ static void _quicklistInsert(quicklist *quicklist, const size_t fill, quicklist->count++; } -void quicklistInsertBefore(quicklist *quicklist, const size_t fill, +void quicklistInsertBefore(quicklist *quicklist, const int fill, quicklistEntry *entry, void *value, const size_t sz) { _quicklistInsert(quicklist, fill, entry, value, sz, 0); } -void quicklistInsertAfter(quicklist *quicklist, const size_t fill, +void quicklistInsertAfter(quicklist *quicklist, const int fill, quicklistEntry *entry, void *value, const size_t sz) { _quicklistInsert(quicklist, fill, entry, value, sz, 1); } @@ -665,6 +758,7 @@ int quicklistDelRange(quicklist *quicklist, const long start, } else { node->zl = ziplistDeleteRange(node->zl, entry.offset, del); node->count -= del; + quicklistNodeUpdateSz(node); quicklist->count -= del; quicklistDeleteIfEmpty(quicklist, node); } @@ -836,6 +930,7 @@ quicklist *quicklistDup(quicklist *orig) { node->count = current->count; copy->count += node->count; + node->sz = current->sz; _quicklistInsertNodeAfter(copy, copy->tail, node); } @@ -912,7 +1007,7 @@ int quicklistIndex(const quicklist *quicklist, const long long idx, } /* Rotate quicklist by moving the tail element to the head. */ -void quicklistRotate(quicklist *quicklist, const size_t fill) { +void quicklistRotate(quicklist *quicklist, const int fill) { if (quicklist->count <= 1) return; @@ -1035,7 +1130,7 @@ int quicklistPop(quicklist *quicklist, int where, unsigned char **data, } /* Wrapper to allow argument-based switching between HEAD/TAIL pop */ -void quicklistPush(quicklist *quicklist, const size_t fill, void *value, +void quicklistPush(quicklist *quicklist, const int fill, void *value, const size_t sz, int where) { if (where == QUICKLIST_HEAD) { quicklistPushHead(quicklist, fill, value, sz); @@ -1202,6 +1297,10 @@ static char *genstr(char *prefix, int i) { /* main test, but callable from other files */ int quicklistTest(int argc, char *argv[]) { unsigned int err = 0; + int optimize_start = + -(int)(sizeof(optimization_level) / sizeof(*optimization_level)); + + printf("Starting optimization offset at: %d\n", optimize_start); UNUSED(argc); UNUSED(argv); @@ -1228,8 +1327,8 @@ int quicklistTest(int argc, char *argv[]) { quicklistRelease(ql); } - for (size_t f = 0; f < 32; f++) { - TEST_DESC("add to tail 5x at fill %lu", f) { + for (int f = optimize_start; f < 32; f++) { + TEST_DESC("add to tail 5x at fill %d", f) { quicklist *ql = quicklistCreate(); for (int i = 0; i < 5; i++) quicklistPushTail(ql, f, genstr("hello", i), 32); @@ -1241,8 +1340,8 @@ int quicklistTest(int argc, char *argv[]) { } } - for (size_t f = 0; f < 32; f++) { - TEST_DESC("add to head 5x at fill %lu", f) { + for (int f = optimize_start; f < 32; f++) { + TEST_DESC("add to head 5x at fill %d", f) { quicklist *ql = quicklistCreate(); for (int i = 0; i < 5; i++) quicklistPushHead(ql, f, genstr("hello", i), 32); @@ -1254,8 +1353,8 @@ int quicklistTest(int argc, char *argv[]) { } } - for (size_t f = 0; f < 512; f++) { - TEST_DESC("add to tail 500x at fill %lu", f) { + for (int f = optimize_start; f < 512; f++) { + TEST_DESC("add to tail 500x at fill %d", f) { quicklist *ql = quicklistCreate(); for (int i = 0; i < 500; i++) quicklistPushTail(ql, f, genstr("hello", i), 32); @@ -1267,8 +1366,8 @@ int quicklistTest(int argc, char *argv[]) { } } - for (size_t f = 0; f < 512; f++) { - TEST_DESC("add to head 500x at fill %lu", f) { + for (int f = optimize_start; f < 512; f++) { + TEST_DESC("add to head 500x at fill %d", f) { quicklist *ql = quicklistCreate(); for (int i = 0; i < 500; i++) quicklistPushHead(ql, f, genstr("hello", i), 32); @@ -1287,7 +1386,7 @@ int quicklistTest(int argc, char *argv[]) { quicklistRelease(ql); } - for (size_t f = 0; f < 32; f++) { + for (int f = optimize_start; f < 32; f++) { TEST("rotate one val once") { quicklist *ql = quicklistCreate(); quicklistPushHead(ql, F, "hello", 6); @@ -1297,8 +1396,8 @@ int quicklistTest(int argc, char *argv[]) { } } - for (size_t f = 0; f < 1024; f++) { - TEST_DESC("rotate 500 val 5000 times at fill %lu", f) { + for (int f = optimize_start; f < 1024; f++) { + TEST_DESC("rotate 500 val 5000 times at fill %d", f) { quicklist *ql = quicklistCreate(); quicklistPushHead(ql, f, "900", 3); quicklistPushHead(ql, f, "7000", 4); @@ -1479,8 +1578,8 @@ int quicklistTest(int argc, char *argv[]) { quicklistRelease(ql); } - for (size_t f = 0; f < 12; f++) { - TEST_DESC("insert once in elements while iterating at fill %lu\n", f) { + for (int f = optimize_start; f < 12; f++) { + TEST_DESC("insert once in elements while iterating at fill %d\n", f) { quicklist *ql = quicklistCreate(); quicklistPushTail(ql, f, "abc", 3); quicklistPushTail(ql, 1, "def", 3); /* force to unique node */ @@ -1534,9 +1633,9 @@ int quicklistTest(int argc, char *argv[]) { } } - for (size_t f = 0; f < 1024; f++) { + for (int f = optimize_start; f < 1024; f++) { TEST_DESC("insert [before] 250 new in middle of 500 elements at fill" - " %ld", + " %d", f) { quicklist *ql = quicklistCreate(); for (int i = 0; i < 500; i++) @@ -1552,9 +1651,9 @@ int quicklistTest(int argc, char *argv[]) { } } - for (size_t f = 0; f < 1024; f++) { - TEST_DESC( - "insert [after] 250 new in middle of 500 elements at fill %ld", f) { + for (int f = optimize_start; f < 1024; f++) { + TEST_DESC("insert [after] 250 new in middle of 500 elements at fill %d", + f) { quicklist *ql = quicklistCreate(); for (int i = 0; i < 500; i++) quicklistPushHead(ql, f, genstr("hello", i), 32); @@ -1604,8 +1703,8 @@ int quicklistTest(int argc, char *argv[]) { quicklistRelease(copy); } - for (size_t f = 0; f < 512; f++) { - TEST_DESC("index 1,200 from 500 list at fill %lu", f) { + for (int f = optimize_start; f < 512; f++) { + TEST_DESC("index 1,200 from 500 list at fill %d", f) { quicklist *ql = quicklistCreate(); for (int i = 0; i < 500; i++) quicklistPushTail(ql, f, genstr("hello", i + 1), 32); @@ -1623,7 +1722,7 @@ int quicklistTest(int argc, char *argv[]) { quicklistRelease(ql); } - TEST_DESC("index -1,-2 from 500 list at fill %lu", f) { + TEST_DESC("index -1,-2 from 500 list at fill %d", f) { quicklist *ql = quicklistCreate(); for (int i = 0; i < 500; i++) quicklistPushTail(ql, f, genstr("hello", i + 1), 32); @@ -1641,7 +1740,7 @@ int quicklistTest(int argc, char *argv[]) { quicklistRelease(ql); } - TEST_DESC("index -100 from 500 list at fill %lu", f) { + TEST_DESC("index -100 from 500 list at fill %d", f) { quicklist *ql = quicklistCreate(); for (int i = 0; i < 500; i++) quicklistPushTail(ql, f, genstr("hello", i + 1), 32); @@ -1654,7 +1753,7 @@ int quicklistTest(int argc, char *argv[]) { quicklistRelease(ql); } - TEST_DESC("index too big +1 from 50 list at fill %lu", f) { + TEST_DESC("index too big +1 from 50 list at fill %d", f) { quicklist *ql = quicklistCreate(); for (int i = 0; i < 50; i++) quicklistPushTail(ql, f, genstr("hello", i + 1), 32); @@ -1821,8 +1920,8 @@ int quicklistTest(int argc, char *argv[]) { OK; } - for (size_t f = 0; f < 16; f++) { - TEST_DESC("lrem test at fill %lu", f) { + for (int f = optimize_start; f < 16; f++) { + TEST_DESC("lrem test at fill %d", f) { quicklist *ql = quicklistCreate(); char *words[] = { "abc", "foo", "bar", "foobar", "foobared", "zap", "bar", "test", "foo" }; @@ -1902,8 +2001,8 @@ int quicklistTest(int argc, char *argv[]) { } } - for (size_t f = 0; f < 16; f++) { - TEST_DESC("iterate reverse + delete at fill %lu", f) { + for (int f = optimize_start; f < 16; f++) { + TEST_DESC("iterate reverse + delete at fill %d", f) { quicklist *ql = quicklistCreate(); quicklistPushTail(ql, f, "abc", 3); quicklistPushTail(ql, f, "def", 3); @@ -1940,8 +2039,8 @@ int quicklistTest(int argc, char *argv[]) { } } - for (size_t f = 0; f < 800; f++) { - TEST_DESC("iterator at index test at fill %lu", f) { + for (int f = optimize_start; f < 800; f++) { + TEST_DESC("iterator at index test at fill %d", f) { quicklist *ql = quicklistCreate(); char num[32]; long long nums[5000]; @@ -1965,8 +2064,8 @@ int quicklistTest(int argc, char *argv[]) { } } - for (size_t f = 0; f < 40; f++) { - TEST_DESC("ltrim test A at fill %lu", f) { + for (int f = optimize_start; f < 40; f++) { + TEST_DESC("ltrim test A at fill %d", f) { quicklist *ql = quicklistCreate(); char num[32]; long long nums[5000]; @@ -1993,8 +2092,8 @@ int quicklistTest(int argc, char *argv[]) { } } - for (size_t f = 0; f < 40; f++) { - TEST_DESC("ltrim test B at fill %lu", f) { + for (int f = optimize_start; f < 40; f++) { + TEST_DESC("ltrim test B at fill %d", f) { quicklist *ql = quicklistCreate(); char num[32]; long long nums[5000]; @@ -2036,8 +2135,8 @@ int quicklistTest(int argc, char *argv[]) { } } - for (size_t f = 0; f < 40; f++) { - TEST_DESC("ltrim test C at fill %lu", f) { + for (int f = optimize_start; f < 40; f++) { + TEST_DESC("ltrim test C at fill %d", f) { quicklist *ql = quicklistCreate(); char num[32]; long long nums[5000]; @@ -2063,8 +2162,8 @@ int quicklistTest(int argc, char *argv[]) { } } - for (size_t f = 0; f < 40; f++) { - TEST_DESC("ltrim test D at fill %lu", f) { + for (int f = optimize_start; f < 40; f++) { + TEST_DESC("ltrim test D at fill %d", f) { quicklist *ql = quicklistCreate(); char num[32]; long long nums[5000]; @@ -2083,8 +2182,8 @@ int quicklistTest(int argc, char *argv[]) { } } - for (size_t f = 0; f < 72; f++) { - TEST_DESC("create quicklist from ziplist at fill %lu", f) { + for (int f = optimize_start; f < 72; f++) { + TEST_DESC("create quicklist from ziplist at fill %d", f) { unsigned char *zl = ziplistNew(); long long nums[32]; char num[32]; diff --git a/src/quicklist.h b/src/quicklist.h index 93b38d880..6f21f789d 100644 --- a/src/quicklist.h +++ b/src/quicklist.h @@ -38,6 +38,7 @@ typedef struct quicklistNode { struct quicklistNode *next; unsigned char *zl; unsigned int count; /* cached count of items in ziplist */ + unsigned int sz; /* ziplist size in bytes */ } quicklistNode; typedef struct quicklist { @@ -71,20 +72,19 @@ typedef struct quicklistEntry { /* Prototypes */ quicklist *quicklistCreate(void); void quicklistRelease(quicklist *quicklist); -quicklist *quicklistPushHead(quicklist *quicklist, const size_t fill, - void *value, const size_t sz); -quicklist *quicklistPushTail(quicklist *quicklist, const size_t fill, - void *value, const size_t sz); -void quicklistPush(quicklist *quicklist, const size_t fill, void *value, +quicklist *quicklistPushHead(quicklist *quicklist, const int fill, void *value, + const size_t sz); +quicklist *quicklistPushTail(quicklist *quicklist, const int fill, void *value, + const size_t sz); +void quicklistPush(quicklist *quicklist, const int fill, void *value, const size_t sz, int where); void quicklistAppendZiplist(quicklist *quicklist, unsigned char *zl); quicklist *quicklistAppendValuesFromZiplist(quicklist *quicklist, - const size_t fill, - unsigned char *zl); -quicklist *quicklistCreateFromZiplist(size_t fill, unsigned char *zl); -void quicklistInsertAfter(quicklist *quicklist, const size_t fill, + const int fill, unsigned char *zl); +quicklist *quicklistCreateFromZiplist(int fill, unsigned char *zl); +void quicklistInsertAfter(quicklist *quicklist, const int fill, quicklistEntry *node, void *value, const size_t sz); -void quicklistInsertBefore(quicklist *quicklist, const size_t fill, +void quicklistInsertBefore(quicklist *quicklist, const int fill, quicklistEntry *node, void *value, const size_t sz); void quicklistDelEntry(quicklistIter *iter, quicklistEntry *entry); int quicklistReplaceAtIndex(quicklist *quicklist, long index, void *data, @@ -100,7 +100,7 @@ int quicklistIndex(const quicklist *quicklist, const long long index, quicklistEntry *entry); void quicklistRewind(quicklist *quicklist, quicklistIter *li); void quicklistRewindTail(quicklist *quicklist, quicklistIter *li); -void quicklistRotate(quicklist *quicklist, const size_t fill); +void quicklistRotate(quicklist *quicklist, const int fill); int quicklistPopCustom(quicklist *quicklist, int where, unsigned char **data, unsigned int *sz, long long *sval, void *(*saver)(unsigned char *data, unsigned int sz)); From e1619772dbac6567462db4fac524e8df7e2556da Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Wed, 10 Dec 2014 15:40:34 -0500 Subject: [PATCH 17/29] Add sdsnative() Use the existing memory space for an SDS to convert it to a regular character buffer so we don't need to allocate duplicate space just to extract a usable buffer for native operations. --- src/sds.c | 11 +++++++++++ src/sds.h | 1 + 2 files changed, 12 insertions(+) diff --git a/src/sds.c b/src/sds.c index 05ee0ad56..3626dd524 100644 --- a/src/sds.c +++ b/src/sds.c @@ -88,6 +88,17 @@ void sdsfree(sds s) { zfree(s-sizeof(struct sdshdr)); } +/* Remove sds header so we can use buffer as malloc'd byte array. + * Returns the contents of 's' usable as a full malloc'd C string. */ +char *sdsnative(sds s) { + if (!s) return NULL; + + size_t len = sdslen(s); + char *base = s-sizeof(struct sdshdr); + memmove(base, s, len); + return zrealloc(base, len); +} + /* Set the sds string length to the length as obtained with strlen(), so * considering as content only up to the first null term character. * diff --git a/src/sds.h b/src/sds.h index 93dd4f28e..756ae0b53 100644 --- a/src/sds.h +++ b/src/sds.h @@ -60,6 +60,7 @@ sds sdsempty(void); size_t sdslen(const sds s); sds sdsdup(const sds s); void sdsfree(sds s); +char *sdsnative(sds s); size_t sdsavail(const sds s); sds sdsgrowzero(sds s, size_t len); sds sdscatlen(sds s, const void *t, size_t len); From 127c15e2b2429c602797e56d8f7e1fdec312c68f Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Wed, 10 Dec 2014 15:46:24 -0500 Subject: [PATCH 18/29] Convert RDB ziplist loading to sdsnative() This saves us an unnecessary zmalloc, memcpy, and two frees. --- src/rdb.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index 70989897a..14d6dea81 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -981,13 +981,9 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { rdbtype == REDIS_RDB_TYPE_ZSET_ZIPLIST || rdbtype == REDIS_RDB_TYPE_HASH_ZIPLIST) { - robj *aux = rdbLoadStringObject(rdb); - - if (aux == NULL) return NULL; - o = createObject(REDIS_STRING,NULL); /* string is just placeholder */ - o->ptr = zmalloc(sdslen(aux->ptr)); - memcpy(o->ptr,aux->ptr,sdslen(aux->ptr)); - decrRefCount(aux); + o = rdbLoadStringObject(rdb); + if (o == NULL) return NULL; + o->ptr = sdsnative(o->ptr); /* Fix the object encoding, and make sure to convert the encoded * data type into the base type if accordingly to the current From 101b3a6e42e84e5cb423ef413225d8b8d8cc0bbc Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Wed, 10 Dec 2014 13:53:12 -0500 Subject: [PATCH 19/29] Convert quicklist RDB to store ziplist nodes Turns out it's a huge improvement during save/reload/migrate/restore because, with compression enabled, we're compressing 4k or 8k chunks of data consisting of multiple elements in one ziplist instead of compressing series of smaller individual elements. --- src/rdb.c | 32 +++++++++++++++++++------------- src/rdb.h | 7 +++++-- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index 14d6dea81..fe9964635 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -434,7 +434,7 @@ int rdbSaveObjectType(rio *rdb, robj *o) { return rdbSaveType(rdb,REDIS_RDB_TYPE_STRING); case REDIS_LIST: if (o->encoding == REDIS_ENCODING_QUICKLIST) - return rdbSaveType(rdb,REDIS_RDB_TYPE_LIST); + return rdbSaveType(rdb,REDIS_RDB_TYPE_LIST_QUICKLIST); else redisPanic("Unknown list encoding"); case REDIS_SET: @@ -484,22 +484,16 @@ int rdbSaveObject(rio *rdb, robj *o) { } else if (o->type == REDIS_LIST) { /* Save a list value */ if (o->encoding == REDIS_ENCODING_QUICKLIST) { - quicklist *list = o->ptr; - quicklistIter *li = quicklistGetIterator(list, AL_START_HEAD); - quicklistEntry entry; + quicklist *ql = o->ptr; + quicklistNode *node = ql->head; - if ((n = rdbSaveLen(rdb,quicklistCount(list))) == -1) return -1; + if ((n = rdbSaveLen(rdb,ql->len)) == -1) return -1; nwritten += n; - while (quicklistNext(li,&entry)) { - if (entry.value) { - if ((n = rdbSaveRawString(rdb,entry.value,entry.sz)) == -1) return -1; - } else { - if ((n = rdbSaveLongLongAsStringObject(rdb,entry.longval)) == -1) return -1; - } + do { + if ((n = rdbSaveRawString(rdb,node->zl,node->sz)) == -1) return -1; nwritten += n; - } - quicklistReleaseIterator(li); + } while ((node = node->next)); } else { redisPanic("Unknown list encoding"); } @@ -974,7 +968,19 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { /* All pairs should be read by now */ redisAssert(len == 0); + } else if (rdbtype == REDIS_RDB_TYPE_LIST_QUICKLIST) { + if ((len = rdbLoadLen(rdb,NULL)) == REDIS_RDB_LENERR) return NULL; + o = createQuicklistObject(); + while (len--) { + if ((ele = rdbLoadStringObject(rdb)) == NULL) return NULL; + /* 'ele' contains a sds of the ziplist, but we need to extract + * the actual ziplist for future usage. We must copy the + * sds contents to a new buffer. */ + unsigned char *zl = (unsigned char *)sdsnative(ele->ptr); + zfree(ele); /* free robj container since we keep the ziplist */ + quicklistAppendZiplist(o->ptr, zl); + } } else if (rdbtype == REDIS_RDB_TYPE_HASH_ZIPMAP || rdbtype == REDIS_RDB_TYPE_LIST_ZIPLIST || rdbtype == REDIS_RDB_TYPE_SET_INTSET || diff --git a/src/rdb.h b/src/rdb.h index eb40d4993..55b3d261c 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -38,7 +38,7 @@ /* The current RDB version. When the format changes in a way that is no longer * backward compatible this number gets incremented. */ -#define REDIS_RDB_VERSION 6 +#define REDIS_RDB_VERSION 7 /* Defines related to the dump file format. To store 32 bits lengths for short * keys requires a lot of space, so we check the most significant 2 bits of @@ -74,6 +74,7 @@ #define REDIS_RDB_TYPE_SET 2 #define REDIS_RDB_TYPE_ZSET 3 #define REDIS_RDB_TYPE_HASH 4 +/* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */ /* Object types for encoded objects. */ #define REDIS_RDB_TYPE_HASH_ZIPMAP 9 @@ -81,9 +82,11 @@ #define REDIS_RDB_TYPE_SET_INTSET 11 #define REDIS_RDB_TYPE_ZSET_ZIPLIST 12 #define REDIS_RDB_TYPE_HASH_ZIPLIST 13 +#define REDIS_RDB_TYPE_LIST_QUICKLIST 14 +/* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */ /* Test if a type is an object type. */ -#define rdbIsObjectType(t) ((t >= 0 && t <= 4) || (t >= 9 && t <= 13)) +#define rdbIsObjectType(t) ((t >= 0 && t <= 4) || (t >= 9 && t <= 14)) /* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */ #define REDIS_RDB_OPCODE_EXPIRETIME_MS 252 From e0d94a7b017a6600332e94e95799d8af9bda1210 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Wed, 10 Dec 2014 16:08:54 -0500 Subject: [PATCH 20/29] Increase test size for migrating large values Previously, the old test ran 5,000 loops and used about 500k. With quicklist, storing those same 5,000 loops takes up 24k, so the "large value check" failed! This increases the test to 20,000 loops which makes the object dump 96k. --- tests/unit/dump.tcl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/dump.tcl b/tests/unit/dump.tcl index d39204f9f..5af53db8d 100644 --- a/tests/unit/dump.tcl +++ b/tests/unit/dump.tcl @@ -157,7 +157,7 @@ start_server {tags {"dump"}} { test {MIGRATE can correctly transfer large values} { set first [srv 0 client] r del key - for {set j 0} {$j < 5000} {incr j} { + for {set j 0} {$j < 40000} {incr j} { r rpush key 1 2 3 4 5 6 7 8 9 10 r rpush key "item 1" "item 2" "item 3" "item 4" "item 5" \ "item 6" "item 7" "item 8" "item 9" "item 10" @@ -175,7 +175,7 @@ start_server {tags {"dump"}} { assert {[$first exists key] == 0} assert {[$second exists key] == 1} assert {[$second ttl key] == -1} - assert {[$second llen key] == 5000*20} + assert {[$second llen key] == 40000*20} } } From 8d7021892ec79bfea3628bcc2999512d9a757a21 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Wed, 10 Dec 2014 20:37:15 -0500 Subject: [PATCH 21/29] Remove malloc failure checks We trust zmalloc to kill the whole process on memory failure --- src/quicklist.c | 40 ++++++++-------------------------------- 1 file changed, 8 insertions(+), 32 deletions(-) diff --git a/src/quicklist.c b/src/quicklist.c index 2d711049d..27ec4e02e 100644 --- a/src/quicklist.c +++ b/src/quicklist.c @@ -67,14 +67,11 @@ static const size_t optimization_level[] = { 4096, 8192, 16384, 32768, 65536 }; } while (0) /* Create a new quicklist. - * Free with quicklistRelease(). - * - * On error, NULL is returned. Otherwise the pointer to the new quicklist. */ + * Free with quicklistRelease(). */ quicklist *quicklistCreate(void) { struct quicklist *quicklist; - if ((quicklist = zmalloc(sizeof(*quicklist))) == NULL) - return NULL; + quicklist = zmalloc(sizeof(*quicklist)); quicklist->head = quicklist->tail = NULL; quicklist->len = 0; quicklist->count = 0; @@ -83,8 +80,7 @@ quicklist *quicklistCreate(void) { static quicklistNode *quicklistCreateNode(void) { quicklistNode *node; - if ((node = zmalloc(sizeof(*node))) == NULL) - return NULL; + node = zmalloc(sizeof(*node)); node->zl = NULL; node->count = 0; node->sz = 0; @@ -537,14 +533,7 @@ static quicklistNode *_quicklistSplitNode(quicklistNode *node, int offset, size_t zl_sz = ziplistBlobLen(node->zl); quicklistNode *new_node = quicklistCreateNode(); - if (!new_node) - return NULL; - new_node->zl = zmalloc(zl_sz); - if (!new_node->zl) { - zfree(new_node); - return NULL; - } /* Copy original ziplist so we can split it */ memcpy(new_node->zl, node->zl, zl_sz); @@ -782,8 +771,7 @@ int quicklistCompare(unsigned char *p1, unsigned char *p2, int p2_len) { quicklistIter *quicklistGetIterator(const quicklist *quicklist, int direction) { quicklistIter *iter; - if ((iter = zmalloc(sizeof(*iter))) == NULL) - return NULL; + iter = zmalloc(sizeof(*iter)); if (direction == AL_START_HEAD) { iter->current = quicklist->head; @@ -904,7 +892,7 @@ int quicklistNext(quicklistIter *iter, quicklistEntry *entry) { } } -/* Duplicate the quicklist. On out of memory NULL is returned. +/* Duplicate the quicklist. * On success a copy of the original quicklist is returned. * * The original quicklist both on success or error is never modified. @@ -912,20 +900,15 @@ int quicklistNext(quicklistIter *iter, quicklistEntry *entry) { * Returns newly allocated quicklist. */ quicklist *quicklistDup(quicklist *orig) { quicklist *copy; - int failure = 0; - if ((copy = quicklistCreate()) == NULL) - return NULL; + copy = quicklistCreate(); for (quicklistNode *current = orig->head; current; current = current->next) { quicklistNode *node = quicklistCreateNode(); size_t ziplen = ziplistBlobLen(current->zl); - if ((node->zl = zmalloc(ziplen)) == NULL) { - failure = 1; - break; - } + node->zl = zmalloc(ziplen); memcpy(node->zl, current->zl, ziplen); node->count = current->count; @@ -936,12 +919,6 @@ quicklist *quicklistDup(quicklist *orig) { } /* copy->count must equal orig->count here */ - - if (failure) { - quicklistRelease(copy); - return NULL; - } - return copy; } @@ -1100,8 +1077,7 @@ int quicklistPopCustom(quicklist *quicklist, int where, unsigned char **data, static void *_quicklistSaver(unsigned char *data, unsigned int sz) { unsigned char *vstr; if (data) { - if ((vstr = zmalloc(sz)) == NULL) - return 0; + vstr = zmalloc(sz); memcpy(data, vstr, sz); return vstr; } From 5127e3998058983351b6c0b94d1341f9d646c9cc Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Wed, 10 Dec 2014 22:54:19 -0500 Subject: [PATCH 22/29] Add quicklist info to DEBUG OBJECT Added field 'ql_nodes' and 'ql_avg_per_node'. ql_nodes is the number of quicklist nodes in the quicklist. ql_avg_node is the average fill level in each quicklist node. (LLEN / QL_NODES) Sample output: 127.0.0.1:6379> DEBUG object b Value at:0x7fa42bf2fed0 refcount:1 encoding:quicklist serializedlength:18489 lru:8983768 lru_seconds_idle:3 ql_nodes:430 ql_avg_per_node:511.73 127.0.0.1:6379> llen b (integer) 220044 --- src/debug.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/debug.c b/src/debug.c index c464644cd..e1761df3f 100644 --- a/src/debug.c +++ b/src/debug.c @@ -301,13 +301,25 @@ void debugCommand(redisClient *c) { val = dictGetVal(de); strenc = strEncoding(val->encoding); + char extra[128] = {0}; + if (val->encoding == REDIS_ENCODING_QUICKLIST) { + char *nextra = extra; + int remaining = sizeof(extra); + quicklist *ql = val->ptr; + double avg = (double)ql->count/ql->len; + int used = snprintf(nextra, remaining, " ql_nodes:%lu", ql->len); + nextra += used; + remaining -= used; + snprintf(nextra, remaining, " ql_avg_node:%.2f", avg); + } + addReplyStatusFormat(c, "Value at:%p refcount:%d " "encoding:%s serializedlength:%lld " - "lru:%d lru_seconds_idle:%llu", + "lru:%d lru_seconds_idle:%llu%s", (void*)val, val->refcount, strenc, (long long) rdbSavedObjectLen(val), - val->lru, estimateObjectIdleTime(val)/1000); + val->lru, estimateObjectIdleTime(val)/1000, extra); } else if (!strcasecmp(c->argv[1]->ptr,"sdslen") && c->argc == 3) { dictEntry *de; robj *val; From abdd1414a896c407c23a8f4165cfd6f027cf2b60 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Wed, 10 Dec 2014 21:26:31 -0500 Subject: [PATCH 23/29] Allow compression of interior quicklist nodes Let user set how many nodes to *not* compress. We can specify a compression "depth" of how many nodes to leave uncompressed on each end of the quicklist. Depth 0 = disable compression. Depth 1 = only leave head/tail uncompressed. - (read as: "skip 1 node on each end of the list before compressing") Depth 2 = leave head, head->next, tail->prev, tail uncompressed. - ("skip 2 nodes on each end of the list before compressing") Depth 3 = Depth 2 + head->next->next + tail->prev->prev - ("skip 3 nodes...") etc. This also: - updates RDB storage to use native quicklist compression (if node is already compressed) instead of uncompressing, generating the RDB string, then re-compressing the quicklist node. - internalizes the "fill" parameter for the quicklist so we don't need to pass it to _every_ function. Now it's just a property of the list. - allows a runtime-configurable compression option, so we can expose a compresion parameter in the configuration file if people want to trade slight request-per-second performance for up to 90%+ memory savings in some situations. - updates the quicklist tests to do multiple passes: 200k+ tests now. --- src/debug.c | 2 +- src/quicklist.c | 2250 ++++++++++++++++++++++++++++------------------- src/quicklist.h | 83 +- src/rdb.c | 62 +- src/t_list.c | 16 +- 5 files changed, 1451 insertions(+), 962 deletions(-) diff --git a/src/debug.c b/src/debug.c index e1761df3f..d11e3f037 100644 --- a/src/debug.c +++ b/src/debug.c @@ -307,7 +307,7 @@ void debugCommand(redisClient *c) { int remaining = sizeof(extra); quicklist *ql = val->ptr; double avg = (double)ql->count/ql->len; - int used = snprintf(nextra, remaining, " ql_nodes:%lu", ql->len); + int used = snprintf(nextra, remaining, " ql_nodes:%u", ql->len); nextra += used; remaining -= used; snprintf(nextra, remaining, " ql_avg_node:%.2f", avg); diff --git a/src/quicklist.c b/src/quicklist.c index 27ec4e02e..92ce50649 100644 --- a/src/quicklist.c +++ b/src/quicklist.c @@ -33,6 +33,7 @@ #include "zmalloc.h" #include "ziplist.h" #include "util.h" /* for ll2string */ +#include "lzf.h" #if defined(REDIS_TEST) || defined(REDIS_TEST_VERBOSE) #include /* for printf (debug printing), snprintf (genstr) */ @@ -41,8 +42,18 @@ /* Optimization levels for size-based filling */ static const size_t optimization_level[] = { 4096, 8192, 16384, 32768, 65536 }; +/* Maximum size in bytes of any multi-element ziplist. + * Larger values will live in their own isolated ziplists. */ #define SIZE_SAFETY_LIMIT 8192 +/* Minimum ziplist size in bytes for attempting compression. */ +#define MIN_COMPRESS_BYTES 48 + +/* Minimum size reduction in bytes to store compressed quicklistNode data. + * This also prevents us from storing compression if the compression + * resulted in a larger size than the original data. */ +#define MIN_COMPRESS_IMPROVE 8 + /* If not verbose testing, remove all debug printing. */ #ifndef REDIS_TEST_VERBOSE #define D(...) @@ -75,6 +86,40 @@ quicklist *quicklistCreate(void) { quicklist->head = quicklist->tail = NULL; quicklist->len = 0; quicklist->count = 0; + quicklist->compress = 0; + quicklist->fill = -2; + return quicklist; +} + +#define COMPRESS_MAX (1 << 16) +void quicklistSetCompressDepth(quicklist *quicklist, int compress) { + if (compress > COMPRESS_MAX) { + compress = COMPRESS_MAX; + } else if (compress < 0) { + compress = 0; + } + quicklist->compress = compress; +} + +#define FILL_MAX (1 << 15) +void quicklistSetFill(quicklist *quicklist, int fill) { + if (fill > FILL_MAX) { + fill = FILL_MAX; + } else if (fill < -5) { + fill = -5; + } + quicklist->fill = fill; +} + +void quicklistSetOptions(quicklist *quicklist, int fill, int depth) { + quicklistSetFill(quicklist, fill); + quicklistSetCompressDepth(quicklist, depth); +} + +/* Create a new quicklist with some default parameters. */ +quicklist *quicklistNew(int fill, int compress) { + quicklist *quicklist = quicklistCreate(); + quicklistSetOptions(quicklist, fill, compress); return quicklist; } @@ -85,6 +130,9 @@ static quicklistNode *quicklistCreateNode(void) { node->count = 0; node->sz = 0; node->next = node->prev = NULL; + node->encoding = QUICKLIST_NODE_ENCODING_RAW; + node->container = QUICKLIST_NODE_CONTAINER_ZIPLIST; + node->recompress = 0; return node; } @@ -112,8 +160,183 @@ void quicklistRelease(quicklist *quicklist) { zfree(quicklist); } +/* Compress the ziplist in 'node' and update encoding details. + * Returns 1 if ziplist compressed successfully. + * Returns 0 if compression failed or if ziplist too small to compress. */ +static int __quicklistCompressNode(quicklistNode *node) { +#ifdef REDIS_TEST + node->attempted_compress = 1; +#endif + + /* Don't bother compressing small values */ + if (node->sz < MIN_COMPRESS_BYTES) + return 0; + + quicklistLZF *lzf = zmalloc(sizeof(*lzf) + node->sz); + + /* Cancel if compression fails or doesn't compress small enough */ + if (((lzf->sz = lzf_compress(node->zl, node->sz, lzf->compressed, + node->sz)) == 0) || + lzf->sz + MIN_COMPRESS_IMPROVE >= node->sz) { + /* lzf_compress aborts/rejects compression if value not compressable. */ + zfree(lzf); + return 0; + } + lzf = zrealloc(lzf, sizeof(*lzf) + lzf->sz); + zfree(node->zl); + node->zl = (unsigned char *)lzf; + node->encoding = QUICKLIST_NODE_ENCODING_LZF; + node->recompress = 0; + return 1; +} + +/* Compress only uncompressed nodes. */ +#define quicklistCompressNode(_node) \ + do { \ + if ((_node) && (_node)->encoding == QUICKLIST_NODE_ENCODING_RAW) { \ + __quicklistCompressNode((_node)); \ + } \ + } while (0) + +/* Uncompress the ziplist in 'node' and update encoding details. + * Returns 1 on successful decode, 0 on failure to decode. */ +static int __quicklistDecompressNode(quicklistNode *node) { +#ifdef REDIS_TEST + node->attempted_compress = 0; +#endif + + void *decompressed = zmalloc(node->sz); + quicklistLZF *lzf = (quicklistLZF *)node->zl; + if (lzf_decompress(lzf->compressed, lzf->sz, decompressed, node->sz) == 0) { + /* Someone requested decompress, but we can't decompress. Not good. */ + zfree(decompressed); + return 0; + } + zfree(lzf); + node->zl = decompressed; + node->encoding = QUICKLIST_NODE_ENCODING_RAW; + return 1; +} + +/* Decompress only compressed nodes. */ +#define quicklistDecompressNode(_node) \ + do { \ + if ((_node) && (_node)->encoding == QUICKLIST_NODE_ENCODING_LZF) { \ + __quicklistDecompressNode((_node)); \ + } \ + } while (0) + +/* Force node to not be immediately re-compresable */ +#define quicklistDecompressNodeForUse(_node) \ + do { \ + if ((_node) && (_node)->encoding == QUICKLIST_NODE_ENCODING_LZF) { \ + __quicklistDecompressNode((_node)); \ + (_node)->recompress = 1; \ + } \ + } while (0) + +/* Extract the raw LZF data from this quicklistNode. + * Pointer to LZF data is assigned to '*data'. + * Return value is the length of compressed LZF data. */ +size_t quicklistGetLzf(const quicklistNode *node, void **data) { + quicklistLZF *lzf = (quicklistLZF *)node->zl; + *data = lzf->compressed; + return lzf->sz; +} + +#define quicklistAllowsCompression(_ql) ((_ql)->compress != 0) + +/* Force 'quicklist' to meet compression guidelines set by compress depth. + * The only way to guarantee interior nodes get compressed is to iterate + * to our "interior" compress depth then compress the next node we find. + * If compress depth is larger than the entire list, we return immediately. */ +static void __quicklistCompress(const quicklist *quicklist, + quicklistNode *node) { + /* If length is less than our compress depth (from both sides), + * we can't compress anything. */ + if (!quicklistAllowsCompression(quicklist) || + quicklist->len < (unsigned int)(quicklist->compress * 2)) + return; + +#if 0 + /* Optimized cases for small depth counts */ + if (quicklist->compress == 1) { + quicklistNode *h = quicklist->head, *t = quicklist->tail; + quicklistDecompressNode(h); + quicklistDecompressNode(t); + if (h != node && t != node) + quicklistCompressNode(node); + return; + } else if (quicklist->compress == 2) { + quicklistNode *h = quicklist->head, *hn = h->next, *hnn = hn->next; + quicklistNode *t = quicklist->tail, *tp = t->prev, *tpp = tp->prev; + quicklistDecompressNode(h); + quicklistDecompressNode(hn); + quicklistDecompressNode(t); + quicklistDecompressNode(tp); + if (h != node && hn != node && t != node && tp != node) { + quicklistCompressNode(node); + } + if (hnn != t) { + quicklistCompressNode(hnn); + } + if (tpp != h) { + quicklistCompressNode(tpp); + } + return; + } +#endif + + /* Iterate until we reach compress depth for both sides of the list.a + * Note: because we do length checks at the *top* of this function, + * we can skip explicit null checks below. Everything exists. */ + quicklistNode *forward = quicklist->head; + quicklistNode *reverse = quicklist->tail; + int depth = 0; + int in_depth = 0; + while (depth++ < quicklist->compress) { + quicklistDecompressNode(forward); + quicklistDecompressNode(reverse); + + if (forward == node || reverse == node) + in_depth = 1; + + if (forward == reverse) + return; + + forward = forward->next; + reverse = reverse->prev; + } + + if (!in_depth) + quicklistCompressNode(node); + + if (depth > 2) { + /* At this point, forward and reverse are one node beyond depth */ + quicklistCompressNode(forward); + quicklistCompressNode(reverse); + } +} + +#define quicklistCompress(_ql, _node) \ + do { \ + if ((_node)->recompress) \ + quicklistCompressNode((_node)); \ + else \ + __quicklistCompress((_ql), (_node)); \ + } while (0) + +/* If we previously used quicklistDecompressNodeForUse(), just recompress. */ +#define quicklistRecompressOnly(_ql, _node) \ + do { \ + if ((_node)->recompress) \ + quicklistCompressNode((_node)); \ + } while (0) + /* Insert 'new_node' after 'old_node' if 'after' is 1. - * Insert 'new_node' before 'old_node' if 'after' is 0. */ + * Insert 'new_node' before 'old_node' if 'after' is 0. + * Note: 'new_node' is *always* uncompressed, so if we assign it to + * head or tail, we do not need to uncompress it. */ static void __quicklistInsertNode(quicklist *quicklist, quicklistNode *old_node, quicklistNode *new_node, int after) { if (after) { @@ -137,11 +360,14 @@ static void __quicklistInsertNode(quicklist *quicklist, quicklistNode *old_node, if (quicklist->head == old_node) quicklist->head = new_node; } - /* If this insert creates the first element in this quicklist, we - * need to initialize head/tail too. */ + /* If this insert creates the only element so far, initialize head/tail. */ if (quicklist->len == 0) { quicklist->head = quicklist->tail = new_node; } + + if (old_node) + quicklistCompress(quicklist, old_node); + quicklist->len++; } @@ -232,44 +458,48 @@ static int _quicklistNodeAllowMerge(const quicklistNode *a, (node)->sz = ziplistBlobLen((node)->zl); \ } while (0) -/* Add new entry to head of quicklist. +/* Add new entry to head node of quicklist. * - * Returns 'quicklist' argument. */ -quicklist *quicklistPushHead(quicklist *quicklist, const int fill, void *value, - size_t sz) { - if (_quicklistNodeAllowInsert(quicklist->head, fill, sz)) { + * Returns 0 if used existing head. + * Returns 1 if new head created. */ +int quicklistPushHead(quicklist *quicklist, void *value, size_t sz) { + quicklistNode *orig_head = quicklist->head; + if (_quicklistNodeAllowInsert(quicklist->head, quicklist->fill, sz)) { quicklist->head->zl = ziplistPush(quicklist->head->zl, value, sz, ZIPLIST_HEAD); + quicklistNodeUpdateSz(quicklist->head); } else { quicklistNode *node = quicklistCreateNode(); node->zl = ziplistPush(ziplistNew(), value, sz, ZIPLIST_HEAD); + quicklistNodeUpdateSz(node); _quicklistInsertNodeBefore(quicklist, quicklist->head, node); } quicklist->count++; quicklist->head->count++; - quicklistNodeUpdateSz(quicklist->head); - return quicklist; + return (orig_head != quicklist->head); } -/* Add new node to tail of quicklist. +/* Add new entry to tail node of quicklist. * - * Returns 'quicklist' argument. */ -quicklist *quicklistPushTail(quicklist *quicklist, const int fill, void *value, - size_t sz) { - if (_quicklistNodeAllowInsert(quicklist->tail, fill, sz)) { + * Returns 0 if used existing tail. + * Returns 1 if new tail created. */ +int quicklistPushTail(quicklist *quicklist, void *value, size_t sz) { + quicklistNode *orig_tail = quicklist->tail; + if (_quicklistNodeAllowInsert(quicklist->tail, quicklist->fill, sz)) { quicklist->tail->zl = ziplistPush(quicklist->tail->zl, value, sz, ZIPLIST_TAIL); + quicklistNodeUpdateSz(quicklist->tail); } else { quicklistNode *node = quicklistCreateNode(); node->zl = ziplistPush(ziplistNew(), value, sz, ZIPLIST_TAIL); + quicklistNodeUpdateSz(node); _quicklistInsertNodeAfter(quicklist, quicklist->tail, node); } quicklist->count++; quicklist->tail->count++; - quicklistNodeUpdateSz(quicklist->tail); - return quicklist; + return (orig_tail != quicklist->tail); } /* Create new node consisting of a pre-formed ziplist. @@ -293,7 +523,7 @@ void quicklistAppendZiplist(quicklist *quicklist, unsigned char *zl) { * * Returns 'quicklist' argument. Frees passed-in ziplist 'zl' */ quicklist *quicklistAppendValuesFromZiplist(quicklist *quicklist, - const int fill, unsigned char *zl) { + unsigned char *zl) { unsigned char *value; unsigned int sz; long long longval; @@ -306,7 +536,7 @@ quicklist *quicklistAppendValuesFromZiplist(quicklist *quicklist, sz = ll2string(longstr, sizeof(longstr), longval); value = (unsigned char *)longstr; } - quicklistPushTail(quicklist, fill, value, sz); + quicklistPushTail(quicklist, value, sz); p = ziplistNext(zl, p); } zfree(zl); @@ -316,14 +546,16 @@ quicklist *quicklistAppendValuesFromZiplist(quicklist *quicklist, /* Create new (potentially multi-node) quicklist from a single existing ziplist. * * Returns new quicklist. Frees passed-in ziplist 'zl'. */ -quicklist *quicklistCreateFromZiplist(int fill, unsigned char *zl) { - return quicklistAppendValuesFromZiplist(quicklistCreate(), fill, zl); +quicklist *quicklistCreateFromZiplist(int fill, int compress, + unsigned char *zl) { + return quicklistAppendValuesFromZiplist(quicklistNew(fill, compress), zl); } #define quicklistDeleteIfEmpty(ql, n) \ do { \ if ((n)->count == 0) { \ __quicklistDelNode((ql), (n)); \ + (n) = NULL; \ } \ } while (0) @@ -333,11 +565,17 @@ static void __quicklistDelNode(quicklist *quicklist, quicklistNode *node) { if (node->prev) node->prev->next = node->next; - if (node == quicklist->tail) + if (node == quicklist->tail) { quicklist->tail = node->prev; + } - if (node == quicklist->head) + if (node == quicklist->head) { quicklist->head = node->next; + } + + /* If we deleted a node within our compress depth, we + * now have compressed nodes needing to be decompressed. */ + __quicklistCompress(quicklist, NULL); quicklist->count -= node->count; @@ -349,21 +587,25 @@ static void __quicklistDelNode(quicklist *quicklist, quicklistNode *node) { /* Delete one entry from list given the node for the entry and a pointer * to the entry in the node. * + * Note: quicklistDelIndex() *requires* uncompressed nodes because you + * already had to get *p from an uncompressed node somewhere. + * * Returns 1 if the entire node was deleted, 0 if node still exists. * Also updates in/out param 'p' with the next offset in the ziplist. */ static int quicklistDelIndex(quicklist *quicklist, quicklistNode *node, unsigned char **p) { int gone = 0; + node->zl = ziplistDelete(node->zl, p); node->count--; if (node->count == 0) { gone = 1; + __quicklistDelNode(quicklist, node); } else { quicklistNodeUpdateSz(node); } quicklist->count--; - quicklistDeleteIfEmpty(quicklist, node); - /* If we deleted all the nodes, our returned pointer is no longer valid */ + /* If we deleted the node, the original node is no longer valid */ return gone ? 1 : 0; } @@ -408,8 +650,10 @@ int quicklistReplaceAtIndex(quicklist *quicklist, long index, void *data, int sz) { quicklistEntry entry; if (quicklistIndex(quicklist, index, &entry)) { + // quicklistDecompressNode(entry.node); entry.node->zl = ziplistDelete(entry.node->zl, &entry.zi); entry.node->zl = ziplistInsert(entry.node->zl, entry.zi, data, sz); + quicklistCompress(quicklist, entry.node); return 1; } else { return 0; @@ -434,6 +678,8 @@ static quicklistNode *_quicklistZiplistMerge(quicklist *quicklist, quicklistNode *b) { D("Requested merge (a,b) (%u, %u)", a->count, b->count); + quicklistDecompressNode(a); + quicklistDecompressNode(b); if ((ziplistMerge(&a->zl, &b->zl))) { /* We merged ziplists! Now remove the unused quicklistNode. */ quicklistNode *keep = NULL, *nokeep = NULL; @@ -449,7 +695,7 @@ static quicklistNode *_quicklistZiplistMerge(quicklist *quicklist, nokeep->count = 0; __quicklistDelNode(quicklist, nokeep); - + quicklistCompress(quicklist, keep); return keep; } else { /* else, the merge returned NULL and nothing changed. */ @@ -465,8 +711,8 @@ static quicklistNode *_quicklistZiplistMerge(quicklist *quicklist, * - (center->prev, center) * - (center, center->next) */ -static void _quicklistMergeNodes(quicklist *quicklist, const int fill, - quicklistNode *center) { +static void _quicklistMergeNodes(quicklist *quicklist, quicklistNode *center) { + int fill = quicklist->fill; quicklistNode *prev, *prev_prev, *next, *next_next, *target; prev = prev_prev = next = next_next = target = NULL; @@ -530,7 +776,7 @@ static void _quicklistMergeNodes(quicklist *quicklist, const int fill, * Returns newly created node or NULL if split not possible. */ static quicklistNode *_quicklistSplitNode(quicklistNode *node, int offset, int after) { - size_t zl_sz = ziplistBlobLen(node->zl); + size_t zl_sz = node->sz; quicklistNode *new_node = quicklistCreateNode(); new_node->zl = zmalloc(zl_sz); @@ -563,10 +809,10 @@ static quicklistNode *_quicklistSplitNode(quicklistNode *node, int offset, * * If after==1, the new value is inserted after 'entry', otherwise * the new value is inserted before 'entry'. */ -static void _quicklistInsert(quicklist *quicklist, const int fill, - quicklistEntry *entry, void *value, - const size_t sz, int after) { +static void _quicklistInsert(quicklist *quicklist, quicklistEntry *entry, + void *value, const size_t sz, int after) { int full = 0, at_tail = 0, at_head = 0, full_next = 0, full_prev = 0; + int fill = quicklist->fill; quicklistNode *node = entry->node; quicklistNode *new_node = NULL; @@ -588,7 +834,7 @@ static void _quicklistInsert(quicklist *quicklist, const int fill, full = 1; } - if (after && (ziplistNext(node->zl, entry->zi) == NULL)) { + if (after && (entry->offset == node->count)) { D("At Tail of current ziplist"); at_tail = 1; if (!_quicklistNodeAllowInsert(node->next, fill, sz)) { @@ -597,7 +843,7 @@ static void _quicklistInsert(quicklist *quicklist, const int fill, } } - if (!after && (ziplistPrev(node->zl, entry->zi) == NULL)) { + if (!after && (entry->offset == 0)) { D("At Head"); at_head = 1; if (!_quicklistNodeAllowInsert(node->prev, fill, sz)) { @@ -609,6 +855,7 @@ static void _quicklistInsert(quicklist *quicklist, const int fill, /* Now determine where and how to insert the new element */ if (!full && after) { D("Not full, inserting after current position."); + quicklistDecompressNodeForUse(node); unsigned char *next = ziplistNext(node->zl, entry->zi); if (next == NULL) { node->zl = ziplistPush(node->zl, value, sz, ZIPLIST_TAIL); @@ -617,27 +864,34 @@ static void _quicklistInsert(quicklist *quicklist, const int fill, } node->count++; quicklistNodeUpdateSz(node); + quicklistRecompressOnly(quicklist, node); } else if (!full && !after) { D("Not full, inserting before current position."); + quicklistDecompressNodeForUse(node); node->zl = ziplistInsert(node->zl, entry->zi, value, sz); node->count++; quicklistNodeUpdateSz(node); + quicklistRecompressOnly(quicklist, node); } else if (full && at_tail && node->next && !full_next && after) { /* If we are: at tail, next has free space, and inserting after: * - insert entry at head of next node. */ D("Full and tail, but next isn't full; inserting next node head"); new_node = node->next; + quicklistDecompressNodeForUse(new_node); new_node->zl = ziplistPush(new_node->zl, value, sz, ZIPLIST_HEAD); new_node->count++; quicklistNodeUpdateSz(new_node); + quicklistRecompressOnly(quicklist, new_node); } else if (full && at_head && node->prev && !full_prev && !after) { /* If we are: at head, previous has free space, and inserting before: * - insert entry at tail of previous node. */ D("Full and head, but prev isn't full, inserting prev node tail"); new_node = node->prev; + quicklistDecompressNodeForUse(new_node); new_node->zl = ziplistPush(new_node->zl, value, sz, ZIPLIST_TAIL); new_node->count++; quicklistNodeUpdateSz(new_node); + quicklistRecompressOnly(quicklist, new_node); } else if (full && ((at_tail && node->next && full_next && after) || (at_head && node->prev && full_prev && !after))) { /* If we are: full, and our prev/next is full, then: @@ -652,27 +906,27 @@ static void _quicklistInsert(quicklist *quicklist, const int fill, /* else, node is full we need to split it. */ /* covers both after and !after cases */ D("\tsplitting node..."); + quicklistDecompressNodeForUse(node); new_node = _quicklistSplitNode(node, entry->offset, after); new_node->zl = ziplistPush(new_node->zl, value, sz, after ? ZIPLIST_HEAD : ZIPLIST_TAIL); new_node->count++; quicklistNodeUpdateSz(new_node); __quicklistInsertNode(quicklist, node, new_node, after); - _quicklistMergeNodes(quicklist, fill, node); + _quicklistMergeNodes(quicklist, node); } quicklist->count++; } -void quicklistInsertBefore(quicklist *quicklist, const int fill, - quicklistEntry *entry, void *value, - const size_t sz) { - _quicklistInsert(quicklist, fill, entry, value, sz, 0); +void quicklistInsertBefore(quicklist *quicklist, quicklistEntry *entry, + void *value, const size_t sz) { + _quicklistInsert(quicklist, entry, value, sz, 0); } -void quicklistInsertAfter(quicklist *quicklist, const int fill, - quicklistEntry *entry, void *value, const size_t sz) { - _quicklistInsert(quicklist, fill, entry, value, sz, 1); +void quicklistInsertAfter(quicklist *quicklist, quicklistEntry *entry, + void *value, const size_t sz) { + _quicklistInsert(quicklist, entry, value, sz, 1); } /* Delete a range of elements from the quicklist. @@ -745,11 +999,14 @@ int quicklistDelRange(quicklist *quicklist, const long start, if (delete_entire_node) { __quicklistDelNode(quicklist, node); } else { + quicklistDecompressNodeForUse(node); node->zl = ziplistDeleteRange(node->zl, entry.offset, del); - node->count -= del; quicklistNodeUpdateSz(node); + node->count -= del; quicklist->count -= del; quicklistDeleteIfEmpty(quicklist, node); + if (node) + quicklistRecompressOnly(quicklist, node); } extent -= del; @@ -807,8 +1064,14 @@ quicklistIter *quicklistGetIteratorAtIdx(const quicklist *quicklist, } } -/* Release iterator */ -void quicklistReleaseIterator(quicklistIter *iter) { zfree(iter); } +/* Release iterator. + * If we still have a valid current node, then re-encode current node. */ +void quicklistReleaseIterator(quicklistIter *iter) { + if (iter->current) + quicklistCompress(iter->quicklist, iter->current); + + zfree(iter); +} /* Get next element in iterator. * @@ -852,6 +1115,7 @@ int quicklistNext(quicklistIter *iter, quicklistEntry *entry) { if (!iter->zi) { /* If !zi, use current index. */ + quicklistDecompressNodeForUse(iter->current); iter->zi = ziplistIndex(iter->current->zl, iter->offset); } else { /* else, use existing iterator offset and get prev/next as necessary. */ @@ -876,6 +1140,7 @@ int quicklistNext(quicklistIter *iter, quicklistEntry *entry) { } else { /* We ran out of ziplist entries. * Pick next node, update offset, then re-run retrieval. */ + quicklistCompress(iter->quicklist, iter->current); if (iter->direction == AL_START_HEAD) { /* Forward traversal */ D("Jumping to start of next node"); @@ -901,19 +1166,26 @@ int quicklistNext(quicklistIter *iter, quicklistEntry *entry) { quicklist *quicklistDup(quicklist *orig) { quicklist *copy; - copy = quicklistCreate(); + copy = quicklistNew(orig->fill, orig->compress); for (quicklistNode *current = orig->head; current; current = current->next) { quicklistNode *node = quicklistCreateNode(); - size_t ziplen = ziplistBlobLen(current->zl); - node->zl = zmalloc(ziplen); - memcpy(node->zl, current->zl, ziplen); + if (node->encoding == QUICKLIST_NODE_ENCODING_LZF) { + quicklistLZF *lzf = (quicklistLZF *)node->zl; + size_t lzf_sz = sizeof(*lzf) + lzf->sz; + node->zl = zmalloc(lzf_sz); + memcpy(node->zl, current->zl, lzf_sz); + } else if (node->encoding == QUICKLIST_NODE_ENCODING_RAW) { + node->zl = zmalloc(current->sz); + memcpy(node->zl, current->zl, current->sz); + } node->count = current->count; copy->count += node->count; node->sz = current->sz; + node->encoding = current->encoding; _quicklistInsertNodeAfter(copy, copy->tail, node); } @@ -978,19 +1250,20 @@ int quicklistIndex(const quicklist *quicklist, const long long idx, entry->offset = (-index) - 1 + accum; } + quicklistDecompressNodeForUse(entry->node); entry->zi = ziplistIndex(entry->node->zl, entry->offset); ziplistGet(entry->zi, &entry->value, &entry->sz, &entry->longval); + // quicklistCompress(quicklist, entry->node); return 1; } /* Rotate quicklist by moving the tail element to the head. */ -void quicklistRotate(quicklist *quicklist, const int fill) { +void quicklistRotate(quicklist *quicklist) { if (quicklist->count <= 1) return; /* First, get the tail entry */ - quicklistNode *tail = quicklist->tail; - unsigned char *p = ziplistIndex(tail->zl, -1); + unsigned char *p = ziplistIndex(quicklist->tail->zl, -1); unsigned char *value; long long longval; unsigned int sz; @@ -1005,16 +1278,17 @@ void quicklistRotate(quicklist *quicklist, const int fill) { } /* Add tail entry to head (must happen before tail is deleted). */ - quicklistPushHead(quicklist, fill, value, sz); + quicklistPushHead(quicklist, value, sz); /* If quicklist has only one node, the head ziplist is also the * tail ziplist and PushHead() could have reallocated our single ziplist, * which would make our pre-existing 'p' unusable. */ - if (quicklist->len == 1) - p = ziplistIndex(tail->zl, -1); + if (quicklist->len == 1) { + p = ziplistIndex(quicklist->tail->zl, -1); + } /* Remove tail entry. */ - quicklistDelIndex(quicklist, tail, &p); + quicklistDelIndex(quicklist, quicklist->tail, &p); } /* pop from quicklist and return result in 'data' ptr. Value of 'data' @@ -1106,18 +1380,19 @@ int quicklistPop(quicklist *quicklist, int where, unsigned char **data, } /* Wrapper to allow argument-based switching between HEAD/TAIL pop */ -void quicklistPush(quicklist *quicklist, const int fill, void *value, - const size_t sz, int where) { +void quicklistPush(quicklist *quicklist, void *value, const size_t sz, + int where) { if (where == QUICKLIST_HEAD) { - quicklistPushHead(quicklist, fill, value, sz); + quicklistPushHead(quicklist, value, sz); } else if (where == QUICKLIST_TAIL) { - quicklistPushTail(quicklist, fill, value, sz); + quicklistPushTail(quicklist, value, sz); } } /* The rest of this file is test cases and test helpers. */ #ifdef REDIS_TEST #include +#include #define assert(_e) \ do { \ @@ -1165,6 +1440,20 @@ static void ql_info(quicklist *ql) { #endif } +/* Return the UNIX time in microseconds */ +static long long ustime(void) { + struct timeval tv; + long long ust; + + gettimeofday(&tv, NULL); + ust = ((long long)tv.tv_sec) * 1000000; + ust += tv.tv_usec; + return ust; +} + +/* Return the UNIX time in milliseconds */ +static long long mstime(void) { return ustime() / 1000; } + /* Iterate over an entire quicklist. * Print the list if 'print' == 1. * @@ -1207,17 +1496,17 @@ static int itrprintr_rev(quicklist *ql, int print) { /* Verify list metadata matches physical list contents. */ static int _ql_verify(quicklist *ql, uint32_t len, uint32_t count, uint32_t head_count, uint32_t tail_count) { - int ok = 1; + int errors = 0; ql_info(ql); if (len != ql->len) { - yell("quicklist length wrong: expected %d, got %lu", len, ql->len); - ok = 0; + yell("quicklist length wrong: expected %d, got %u", len, ql->len); + errors++; } if (count != ql->count) { yell("quicklist count wrong: expected %d, got %lu", count, ql->count); - ok = 0; + errors++; } int loopr = itrprintr(ql, 0); @@ -1225,7 +1514,7 @@ static int _ql_verify(quicklist *ql, uint32_t len, uint32_t count, yell("quicklist cached count not match actual count: expected %lu, got " "%d", ql->count, loopr); - ok = 0; + errors++; } int rloopr = itrprintr_rev(ql, 0); @@ -1233,32 +1522,62 @@ static int _ql_verify(quicklist *ql, uint32_t len, uint32_t count, yell("quicklist has different forward count than reverse count! " "Forward count is %d, reverse count is %d.", loopr, rloopr); - ok = 0; + errors++; } - if (ql->len == 0 && ok) { + if (ql->len == 0 && !errors) { OK; - return !ok; + return errors; } - if (head_count != ql->head->count && + if (ql->head && head_count != ql->head->count && head_count != ziplistLen(ql->head->zl)) { yell("quicklist head count wrong: expected %d, " "got cached %d vs. actual %d", head_count, ql->head->count, ziplistLen(ql->head->zl)); - ok = 0; + errors++; } - if (tail_count != ql->tail->count && + if (ql->tail && tail_count != ql->tail->count && tail_count != ziplistLen(ql->tail->zl)) { yell("quicklist tail count wrong: expected %d, " "got cached %u vs. actual %d", tail_count, ql->tail->count, ziplistLen(ql->tail->zl)); - ok = 0; + errors++; } - if (ok) + + if (quicklistAllowsCompression(ql)) { + quicklistNode *node = ql->head; + unsigned int low_raw = ql->compress; + unsigned int high_raw = ql->len - ql->compress; + + for (unsigned int at = 0; at < ql->len; at++, node = node->next) { + if (node && (at < low_raw || at >= high_raw)) { + if (node->encoding != QUICKLIST_NODE_ENCODING_RAW) { + yell("Incorrect compression: node %d is " + "compressed at depth %d ((%u, %u); total " + "nodes: %u; size: %u; recompress: %d)", + at, ql->compress, low_raw, high_raw, ql->len, node->sz, + node->recompress); + errors++; + } + } else { + if (node->encoding != QUICKLIST_NODE_ENCODING_LZF && + !node->attempted_compress) { + yell("Incorrect non-compression: node %d is NOT " + "compressed at depth %d ((%u, %u); total " + "nodes: %u; size: %u; recompress: %d; attempted: %d)", + at, ql->compress, low_raw, high_raw, ql->len, node->sz, + node->recompress, node->attempted_compress); + errors++; + } + } + } + } + + if (!errors) OK; - return !ok; + return errors; } /* Generate new string concatenating integer i against string 'prefix' */ @@ -1268,920 +1587,1027 @@ static char *genstr(char *prefix, int i) { return result; } -/* Test fill cap */ -#define F 32 /* main test, but callable from other files */ int quicklistTest(int argc, char *argv[]) { + UNUSED(argc); + UNUSED(argv); + unsigned int err = 0; int optimize_start = -(int)(sizeof(optimization_level) / sizeof(*optimization_level)); printf("Starting optimization offset at: %d\n", optimize_start); - UNUSED(argc); - UNUSED(argv); + int options[] = { 0, 1, 2, 3, 4, 5, 6, 10 }; + size_t option_count = sizeof(options) / sizeof(*options); + long long runtime[option_count]; - TEST("create list") { - quicklist *ql = quicklistCreate(); - ql_verify(ql, 0, 0, 0, 0); - quicklistRelease(ql); - } + for (int _i = 0; _i < (int)option_count; _i++) { + printf("Testing Option %d\n", options[_i]); + long long start = mstime(); - TEST("add to tail of empty list") { - quicklist *ql = quicklistCreate(); - quicklistPushTail(ql, F, "hello", 6); - /* 1 for head and 1 for tail beacuse 1 node = head = tail */ - ql_verify(ql, 1, 1, 1, 1); - quicklistRelease(ql); - } - - TEST("add to head of empty list") { - quicklist *ql = quicklistCreate(); - quicklistPushHead(ql, F, "hello", 6); - /* 1 for head and 1 for tail beacuse 1 node = head = tail */ - ql_verify(ql, 1, 1, 1, 1); - quicklistRelease(ql); - } - - for (int f = optimize_start; f < 32; f++) { - TEST_DESC("add to tail 5x at fill %d", f) { - quicklist *ql = quicklistCreate(); - for (int i = 0; i < 5; i++) - quicklistPushTail(ql, f, genstr("hello", i), 32); - if (ql->count != 5) - ERROR; - if (f == 32) - ql_verify(ql, 1, 5, 5, 5); + TEST("create list") { + quicklist *ql = quicklistNew(-2, options[_i]); + ql_verify(ql, 0, 0, 0, 0); quicklistRelease(ql); } - } - for (int f = optimize_start; f < 32; f++) { - TEST_DESC("add to head 5x at fill %d", f) { - quicklist *ql = quicklistCreate(); - for (int i = 0; i < 5; i++) - quicklistPushHead(ql, f, genstr("hello", i), 32); - if (ql->count != 5) - ERROR; - if (f == 32) - ql_verify(ql, 1, 5, 5, 5); - quicklistRelease(ql); - } - } - - for (int f = optimize_start; f < 512; f++) { - TEST_DESC("add to tail 500x at fill %d", f) { - quicklist *ql = quicklistCreate(); - for (int i = 0; i < 500; i++) - quicklistPushTail(ql, f, genstr("hello", i), 32); - if (ql->count != 500) - ERROR; - if (f == 32) - ql_verify(ql, 16, 500, 32, 20); - quicklistRelease(ql); - } - } - - for (int f = optimize_start; f < 512; f++) { - TEST_DESC("add to head 500x at fill %d", f) { - quicklist *ql = quicklistCreate(); - for (int i = 0; i < 500; i++) - quicklistPushHead(ql, f, genstr("hello", i), 32); - if (ql->count != 500) - ERROR; - if (f == 32) - ql_verify(ql, 16, 500, 20, 32); - quicklistRelease(ql); - } - } - - TEST("rotate empty") { - quicklist *ql = quicklistCreate(); - quicklistRotate(ql, F); - ql_verify(ql, 0, 0, 0, 0); - quicklistRelease(ql); - } - - for (int f = optimize_start; f < 32; f++) { - TEST("rotate one val once") { - quicklist *ql = quicklistCreate(); - quicklistPushHead(ql, F, "hello", 6); - quicklistRotate(ql, F); + TEST("add to tail of empty list") { + quicklist *ql = quicklistNew(-2, options[_i]); + quicklistPushTail(ql, "hello", 6); + /* 1 for head and 1 for tail beacuse 1 node = head = tail */ ql_verify(ql, 1, 1, 1, 1); quicklistRelease(ql); } - } - for (int f = optimize_start; f < 1024; f++) { - TEST_DESC("rotate 500 val 5000 times at fill %d", f) { - quicklist *ql = quicklistCreate(); - quicklistPushHead(ql, f, "900", 3); - quicklistPushHead(ql, f, "7000", 4); - quicklistPushHead(ql, f, "-1200", 5); - quicklistPushHead(ql, f, "42", 2); - for (int i = 0; i < 500; i++) - quicklistPushHead(ql, f, genstr("hello", i), 32); - ql_info(ql); - for (int i = 0; i < 5000; i++) { - ql_info(ql); - quicklistRotate(ql, f); - } - if (f == 1) - ql_verify(ql, 504, 504, 1, 1); - else if (f == 2) - ql_verify(ql, 252, 504, 2, 2); - else if (f == 32) - ql_verify(ql, 16, 504, 32, 24); + TEST("add to head of empty list") { + quicklist *ql = quicklistNew(-2, options[_i]); + quicklistPushHead(ql, "hello", 6); + /* 1 for head and 1 for tail beacuse 1 node = head = tail */ + ql_verify(ql, 1, 1, 1, 1); quicklistRelease(ql); } - } - TEST("pop empty") { - quicklist *ql = quicklistCreate(); - quicklistPop(ql, QUICKLIST_HEAD, NULL, NULL, NULL); - ql_verify(ql, 0, 0, 0, 0); - quicklistRelease(ql); - } + for (int f = optimize_start; f < 32; f++) { + TEST_DESC("add to tail 5x at fill %d at compress %d", f, + options[_i]) { + quicklist *ql = quicklistNew(f, options[_i]); + for (int i = 0; i < 5; i++) + quicklistPushTail(ql, genstr("hello", i), 32); + if (ql->count != 5) + ERROR; + if (f == 32) + ql_verify(ql, 1, 5, 5, 5); + quicklistRelease(ql); + } + } - TEST("pop 1 string from 1") { - quicklist *ql = quicklistCreate(); - quicklistPushHead(ql, F, genstr("hello", 331), 32); - unsigned char *data; - unsigned int sz; - long long lv; - ql_info(ql); - quicklistPop(ql, QUICKLIST_HEAD, &data, &sz, &lv); - assert(data != NULL); - assert(sz == 32); - zfree(data); - ql_verify(ql, 0, 0, 0, 0); - quicklistRelease(ql); - } + for (int f = optimize_start; f < 32; f++) { + TEST_DESC("add to head 5x at fill %d at compress %d", f, + options[_i]) { + quicklist *ql = quicklistNew(f, options[_i]); + for (int i = 0; i < 5; i++) + quicklistPushHead(ql, genstr("hello", i), 32); + if (ql->count != 5) + ERROR; + if (f == 32) + ql_verify(ql, 1, 5, 5, 5); + quicklistRelease(ql); + } + } - TEST("pop head 1 number from 1") { - quicklist *ql = quicklistCreate(); - quicklistPushHead(ql, F, "55513", 5); - unsigned char *data; - unsigned int sz; - long long lv; - ql_info(ql); - quicklistPop(ql, QUICKLIST_HEAD, &data, &sz, &lv); - assert(data == NULL); - assert(lv == 55513); - ql_verify(ql, 0, 0, 0, 0); - quicklistRelease(ql); - } + for (int f = optimize_start; f < 512; f++) { + TEST_DESC("add to tail 500x at fill %d at compress %d", f, + options[_i]) { + quicklist *ql = quicklistNew(f, options[_i]); + for (int i = 0; i < 500; i++) + quicklistPushTail(ql, genstr("hello", i), 64); + if (ql->count != 500) + ERROR; + if (f == 32) + ql_verify(ql, 16, 500, 32, 20); + quicklistRelease(ql); + } + } - TEST("pop head 500 from 500") { - quicklist *ql = quicklistCreate(); - for (int i = 0; i < 500; i++) - quicklistPushHead(ql, F, genstr("hello", i), 32); - ql_info(ql); - for (int i = 0; i < 500; i++) { + for (int f = optimize_start; f < 512; f++) { + TEST_DESC("add to head 500x at fill %d at compress %d", f, + options[_i]) { + quicklist *ql = quicklistNew(f, options[_i]); + for (int i = 0; i < 500; i++) + quicklistPushHead(ql, genstr("hello", i), 32); + if (ql->count != 500) + ERROR; + if (f == 32) + ql_verify(ql, 16, 500, 20, 32); + quicklistRelease(ql); + } + } + + TEST("rotate empty") { + quicklist *ql = quicklistNew(-2, options[_i]); + quicklistRotate(ql); + ql_verify(ql, 0, 0, 0, 0); + quicklistRelease(ql); + } + + for (int f = optimize_start; f < 32; f++) { + TEST("rotate one val once") { + quicklist *ql = quicklistNew(f, options[_i]); + quicklistPushHead(ql, "hello", 6); + quicklistRotate(ql); + /* Ignore compression verify because ziplist is + * too small to compress. */ + ql_verify(ql, 1, 1, 1, 1); + quicklistRelease(ql); + } + } + + for (int f = optimize_start; f < 3; f++) { + TEST_DESC("rotate 500 val 5000 times at fill %d at compress %d", f, + options[_i]) { + quicklist *ql = quicklistNew(f, options[_i]); + quicklistPushHead(ql, "900", 3); + quicklistPushHead(ql, "7000", 4); + quicklistPushHead(ql, "-1200", 5); + quicklistPushHead(ql, "42", 2); + for (int i = 0; i < 500; i++) + quicklistPushHead(ql, genstr("hello", i), 64); + ql_info(ql); + for (int i = 0; i < 5000; i++) { + ql_info(ql); + quicklistRotate(ql); + } + if (f == 1) + ql_verify(ql, 504, 504, 1, 1); + else if (f == 2) + ql_verify(ql, 252, 504, 2, 2); + else if (f == 32) + ql_verify(ql, 16, 504, 32, 24); + quicklistRelease(ql); + } + } + + TEST("pop empty") { + quicklist *ql = quicklistNew(-2, options[_i]); + quicklistPop(ql, QUICKLIST_HEAD, NULL, NULL, NULL); + ql_verify(ql, 0, 0, 0, 0); + quicklistRelease(ql); + } + + TEST("pop 1 string from 1") { + quicklist *ql = quicklistNew(-2, options[_i]); + quicklistPushHead(ql, genstr("hello", 331), 32); unsigned char *data; unsigned int sz; long long lv; - int ret = quicklistPop(ql, QUICKLIST_HEAD, &data, &sz, &lv); - assert(ret == 1); + ql_info(ql); + quicklistPop(ql, QUICKLIST_HEAD, &data, &sz, &lv); assert(data != NULL); assert(sz == 32); zfree(data); + ql_verify(ql, 0, 0, 0, 0); + quicklistRelease(ql); } - ql_verify(ql, 0, 0, 0, 0); - quicklistRelease(ql); - } - TEST("pop head 5000 from 500") { - quicklist *ql = quicklistCreate(); - for (int i = 0; i < 500; i++) - quicklistPushHead(ql, F, genstr("hello", i), 32); - for (int i = 0; i < 5000; i++) { + TEST("pop head 1 number from 1") { + quicklist *ql = quicklistNew(-2, options[_i]); + quicklistPushHead(ql, "55513", 5); unsigned char *data; unsigned int sz; long long lv; - int ret = quicklistPop(ql, QUICKLIST_HEAD, &data, &sz, &lv); - if (i < 500) { + ql_info(ql); + quicklistPop(ql, QUICKLIST_HEAD, &data, &sz, &lv); + assert(data == NULL); + assert(lv == 55513); + ql_verify(ql, 0, 0, 0, 0); + quicklistRelease(ql); + } + + TEST("pop head 500 from 500") { + quicklist *ql = quicklistNew(-2, options[_i]); + for (int i = 0; i < 500; i++) + quicklistPushHead(ql, genstr("hello", i), 32); + ql_info(ql); + for (int i = 0; i < 500; i++) { + unsigned char *data; + unsigned int sz; + long long lv; + int ret = quicklistPop(ql, QUICKLIST_HEAD, &data, &sz, &lv); assert(ret == 1); assert(data != NULL); assert(sz == 32); zfree(data); - } else { - assert(ret == 0); } + ql_verify(ql, 0, 0, 0, 0); + quicklistRelease(ql); } - ql_verify(ql, 0, 0, 0, 0); - quicklistRelease(ql); - } - TEST("iterate forward over 500 list") { - quicklist *ql = quicklistCreate(); - for (int i = 0; i < 500; i++) - quicklistPushHead(ql, F, genstr("hello", i), 32); - quicklistIter *iter = quicklistGetIterator(ql, AL_START_HEAD); - quicklistEntry entry; - int i = 499, count = 0; - while (quicklistNext(iter, &entry)) { - char *h = genstr("hello", i); - if (strcmp((char *)entry.value, h)) - ERR("value [%s] didn't match [%s] at position %d", entry.value, - h, i); - i--; - count++; + TEST("pop head 5000 from 500") { + quicklist *ql = quicklistNew(-2, options[_i]); + for (int i = 0; i < 500; i++) + quicklistPushHead(ql, genstr("hello", i), 32); + for (int i = 0; i < 5000; i++) { + unsigned char *data; + unsigned int sz; + long long lv; + int ret = quicklistPop(ql, QUICKLIST_HEAD, &data, &sz, &lv); + if (i < 500) { + assert(ret == 1); + assert(data != NULL); + assert(sz == 32); + zfree(data); + } else { + assert(ret == 0); + } + } + ql_verify(ql, 0, 0, 0, 0); + quicklistRelease(ql); } - if (count != 500) - ERR("Didn't iterate over exactly 500 elements (%d)", i); - ql_verify(ql, 16, 500, 20, 32); - quicklistReleaseIterator(iter); - quicklistRelease(ql); - } - TEST("iterate reverse over 500 list") { - quicklist *ql = quicklistCreate(); - for (int i = 0; i < 500; i++) - quicklistPushHead(ql, F, genstr("hello", i), 32); - quicklistIter *iter = quicklistGetIterator(ql, AL_START_TAIL); - quicklistEntry entry; - int i = 0; - while (quicklistNext(iter, &entry)) { - char *h = genstr("hello", i); - if (strcmp((char *)entry.value, h)) - ERR("value [%s] didn't match [%s] at position %d", entry.value, - h, i); - i++; - } - if (i != 500) - ERR("Didn't iterate over exactly 500 elements (%d)", i); - ql_verify(ql, 16, 500, 20, 32); - quicklistReleaseIterator(iter); - quicklistRelease(ql); - } - - TEST("insert before with 0 elements") { - quicklist *ql = quicklistCreate(); - quicklistEntry entry; - quicklistIndex(ql, 0, &entry); - quicklistInsertBefore(ql, F, &entry, "abc", 4); - ql_verify(ql, 1, 1, 1, 1); - quicklistRelease(ql); - } - - TEST("insert after with 0 elements") { - quicklist *ql = quicklistCreate(); - quicklistEntry entry; - quicklistIndex(ql, 0, &entry); - quicklistInsertAfter(ql, F, &entry, "abc", 4); - ql_verify(ql, 1, 1, 1, 1); - quicklistRelease(ql); - } - - TEST("insert after 1 element") { - quicklist *ql = quicklistCreate(); - quicklistPushHead(ql, F, "hello", 6); - quicklistEntry entry; - quicklistIndex(ql, 0, &entry); - quicklistInsertAfter(ql, F, &entry, "abc", 4); - ql_verify(ql, 1, 2, 2, 2); - quicklistRelease(ql); - } - - TEST("insert before 1 element") { - quicklist *ql = quicklistCreate(); - quicklistPushHead(ql, F, "hello", 6); - quicklistEntry entry; - quicklistIndex(ql, 0, &entry); - quicklistInsertAfter(ql, F, &entry, "abc", 4); - ql_verify(ql, 1, 2, 2, 2); - quicklistRelease(ql); - } - - for (int f = optimize_start; f < 12; f++) { - TEST_DESC("insert once in elements while iterating at fill %d\n", f) { - quicklist *ql = quicklistCreate(); - quicklistPushTail(ql, f, "abc", 3); - quicklistPushTail(ql, 1, "def", 3); /* force to unique node */ - quicklistPushTail(ql, f, "bob", 3); /* force to reset for +3 */ - quicklistPushTail(ql, f, "foo", 3); - quicklistPushTail(ql, f, "zoo", 3); - - itrprintr(ql, 1); - /* insert "bar" before "bob" while iterating over list. */ + TEST("iterate forward over 500 list") { + quicklist *ql = quicklistNew(-2, options[_i]); + quicklistSetFill(ql, 32); + for (int i = 0; i < 500; i++) + quicklistPushHead(ql, genstr("hello", i), 32); quicklistIter *iter = quicklistGetIterator(ql, AL_START_HEAD); quicklistEntry entry; + int i = 499, count = 0; while (quicklistNext(iter, &entry)) { - if (!strncmp((char *)entry.value, "bob", 3)) { - /* Insert as fill = 1 so it spills into new node. */ - quicklistInsertBefore(ql, f, &entry, "bar", 3); - /* NOTE! You can't continue iterating after an insert into - * the list. You *must* re-create your iterator again if - * you want to traverse all entires. */ - break; - } + char *h = genstr("hello", i); + if (strcmp((char *)entry.value, h)) + ERR("value [%s] didn't match [%s] at position %d", + entry.value, h, i); + i--; + count++; } - itrprintr(ql, 1); - - /* verify results */ - quicklistIndex(ql, 0, &entry); - if (strncmp((char *)entry.value, "abc", 3)) - ERR("Value 0 didn't match, instead got: %.*s", entry.sz, - entry.value); - quicklistIndex(ql, 1, &entry); - if (strncmp((char *)entry.value, "def", 3)) - ERR("Value 1 didn't match, instead got: %.*s", entry.sz, - entry.value); - quicklistIndex(ql, 2, &entry); - if (strncmp((char *)entry.value, "bar", 3)) - ERR("Value 2 didn't match, instead got: %.*s", entry.sz, - entry.value); - quicklistIndex(ql, 3, &entry); - if (strncmp((char *)entry.value, "bob", 3)) - ERR("Value 3 didn't match, instead got: %.*s", entry.sz, - entry.value); - quicklistIndex(ql, 4, &entry); - if (strncmp((char *)entry.value, "foo", 3)) - ERR("Value 4 didn't match, instead got: %.*s", entry.sz, - entry.value); - quicklistIndex(ql, 5, &entry); - if (strncmp((char *)entry.value, "zoo", 3)) - ERR("Value 5 didn't match, instead got: %.*s", entry.sz, - entry.value); + if (count != 500) + ERR("Didn't iterate over exactly 500 elements (%d)", i); + ql_verify(ql, 16, 500, 20, 32); quicklistReleaseIterator(iter); quicklistRelease(ql); } - } - for (int f = optimize_start; f < 1024; f++) { - TEST_DESC("insert [before] 250 new in middle of 500 elements at fill" - " %d", - f) { - quicklist *ql = quicklistCreate(); + TEST("iterate reverse over 500 list") { + quicklist *ql = quicklistNew(-2, options[_i]); + quicklistSetFill(ql, 32); for (int i = 0; i < 500; i++) - quicklistPushTail(ql, f, genstr("hello", i), 32); - for (int i = 0; i < 250; i++) { - quicklistEntry entry; - quicklistIndex(ql, 250, &entry); - quicklistInsertBefore(ql, f, &entry, genstr("abc", i), 32); - } - if (f == 32) - ql_verify(ql, 25, 750, 32, 20); - quicklistRelease(ql); - } - } - - for (int f = optimize_start; f < 1024; f++) { - TEST_DESC("insert [after] 250 new in middle of 500 elements at fill %d", - f) { - quicklist *ql = quicklistCreate(); - for (int i = 0; i < 500; i++) - quicklistPushHead(ql, f, genstr("hello", i), 32); - for (int i = 0; i < 250; i++) { - quicklistEntry entry; - quicklistIndex(ql, 250, &entry); - quicklistInsertAfter(ql, f, &entry, genstr("abc", i), 32); - } - - if (ql->count != 750) - ERR("List size not 750, but rather %ld", ql->count); - - if (f == 32) - ql_verify(ql, 26, 750, 20, 32); - quicklistRelease(ql); - } - } - - TEST("duplicate empty list") { - quicklist *ql = quicklistCreate(); - ql_verify(ql, 0, 0, 0, 0); - quicklist *copy = quicklistDup(ql); - ql_verify(copy, 0, 0, 0, 0); - quicklistRelease(ql); - quicklistRelease(copy); - } - - TEST("duplicate list of 1 element") { - quicklist *ql = quicklistCreate(); - quicklistPushHead(ql, F, genstr("hello", 3), 32); - ql_verify(ql, 1, 1, 1, 1); - quicklist *copy = quicklistDup(ql); - ql_verify(copy, 1, 1, 1, 1); - quicklistRelease(ql); - quicklistRelease(copy); - } - - TEST("duplicate list of 500") { - quicklist *ql = quicklistCreate(); - for (int i = 0; i < 500; i++) - quicklistPushHead(ql, F, genstr("hello", i), 32); - ql_verify(ql, 16, 500, 20, 32); - - quicklist *copy = quicklistDup(ql); - ql_verify(copy, 16, 500, 20, 32); - quicklistRelease(ql); - quicklistRelease(copy); - } - - for (int f = optimize_start; f < 512; f++) { - TEST_DESC("index 1,200 from 500 list at fill %d", f) { - quicklist *ql = quicklistCreate(); - for (int i = 0; i < 500; i++) - quicklistPushTail(ql, f, genstr("hello", i + 1), 32); - quicklistEntry entry; - quicklistIndex(ql, 1, &entry); - if (!strcmp((char *)entry.value, "hello2")) - OK; - else - ERR("Value: %s", entry.value); - quicklistIndex(ql, 200, &entry); - if (!strcmp((char *)entry.value, "hello201")) - OK; - else - ERR("Value: %s", entry.value); - quicklistRelease(ql); - } - - TEST_DESC("index -1,-2 from 500 list at fill %d", f) { - quicklist *ql = quicklistCreate(); - for (int i = 0; i < 500; i++) - quicklistPushTail(ql, f, genstr("hello", i + 1), 32); - quicklistEntry entry; - quicklistIndex(ql, -1, &entry); - if (!strcmp((char *)entry.value, "hello500")) - OK; - else - ERR("Value: %s", entry.value); - quicklistIndex(ql, -2, &entry); - if (!strcmp((char *)entry.value, "hello499")) - OK; - else - ERR("Value: %s", entry.value); - quicklistRelease(ql); - } - - TEST_DESC("index -100 from 500 list at fill %d", f) { - quicklist *ql = quicklistCreate(); - for (int i = 0; i < 500; i++) - quicklistPushTail(ql, f, genstr("hello", i + 1), 32); - quicklistEntry entry; - quicklistIndex(ql, -100, &entry); - if (!strcmp((char *)entry.value, "hello401")) - OK; - else - ERR("Value: %s", entry.value); - quicklistRelease(ql); - } - - TEST_DESC("index too big +1 from 50 list at fill %d", f) { - quicklist *ql = quicklistCreate(); - for (int i = 0; i < 50; i++) - quicklistPushTail(ql, f, genstr("hello", i + 1), 32); - quicklistEntry entry; - if (quicklistIndex(ql, 50, &entry)) - ERR("Index found at 50 with 50 list: %.*s", entry.sz, - entry.value); - else - OK; - quicklistRelease(ql); - } - } - - TEST("delete range empty list") { - quicklist *ql = quicklistCreate(); - quicklistDelRange(ql, 5, 20); - ql_verify(ql, 0, 0, 0, 0); - quicklistRelease(ql); - } - - TEST("delete range of entire node in list of one node") { - quicklist *ql = quicklistCreate(); - for (int i = 0; i < 32; i++) - quicklistPushHead(ql, F, genstr("hello", i), 32); - ql_verify(ql, 1, 32, 32, 32); - quicklistDelRange(ql, 0, 32); - ql_verify(ql, 0, 0, 0, 0); - quicklistRelease(ql); - } - - TEST("delete range of entire node with overflow counts") { - quicklist *ql = quicklistCreate(); - for (int i = 0; i < 32; i++) - quicklistPushHead(ql, F, genstr("hello", i), 32); - ql_verify(ql, 1, 32, 32, 32); - quicklistDelRange(ql, 0, 128); - ql_verify(ql, 0, 0, 0, 0); - quicklistRelease(ql); - } - - TEST("delete middle 100 of 500 list") { - quicklist *ql = quicklistCreate(); - for (int i = 0; i < 500; i++) - quicklistPushTail(ql, F, genstr("hello", i + 1), 32); - ql_verify(ql, 16, 500, 32, 20); - quicklistDelRange(ql, 200, 100); - ql_verify(ql, 14, 400, 32, 20); - quicklistRelease(ql); - } - - TEST("delete negative 1 from 500 list") { - quicklist *ql = quicklistCreate(); - for (int i = 0; i < 500; i++) - quicklistPushTail(ql, F, genstr("hello", i + 1), 32); - ql_verify(ql, 16, 500, 32, 20); - quicklistDelRange(ql, -1, 1); - ql_verify(ql, 16, 499, 32, 19); - quicklistRelease(ql); - } - - TEST("delete negative 1 from 500 list with overflow counts") { - quicklist *ql = quicklistCreate(); - for (int i = 0; i < 500; i++) - quicklistPushTail(ql, F, genstr("hello", i + 1), 32); - ql_verify(ql, 16, 500, 32, 20); - quicklistDelRange(ql, -1, 128); - ql_verify(ql, 16, 499, 32, 19); - quicklistRelease(ql); - } - - TEST("delete negative 100 from 500 list") { - quicklist *ql = quicklistCreate(); - for (int i = 0; i < 500; i++) - quicklistPushTail(ql, F, genstr("hello", i + 1), 32); - quicklistDelRange(ql, -100, 100); - ql_verify(ql, 13, 400, 32, 16); - quicklistRelease(ql); - } - - TEST("delete -10 count 5 from 50 list") { - quicklist *ql = quicklistCreate(); - for (int i = 0; i < 50; i++) - quicklistPushTail(ql, F, genstr("hello", i + 1), 32); - ql_verify(ql, 2, 50, 32, 18); - quicklistDelRange(ql, -10, 5); - ql_verify(ql, 2, 45, 32, 13); - quicklistRelease(ql); - } - - TEST("numbers only list read") { - quicklist *ql = quicklistCreate(); - quicklistPushTail(ql, F, "1111", 4); - quicklistPushTail(ql, F, "2222", 4); - quicklistPushTail(ql, F, "3333", 4); - quicklistPushTail(ql, F, "4444", 4); - ql_verify(ql, 1, 4, 4, 4); - quicklistEntry entry; - quicklistIndex(ql, 0, &entry); - if (entry.longval != 1111) - ERR("Not 1111, %lld", entry.longval); - quicklistIndex(ql, 1, &entry); - if (entry.longval != 2222) - ERR("Not 2222, %lld", entry.longval); - quicklistIndex(ql, 2, &entry); - if (entry.longval != 3333) - ERR("Not 3333, %lld", entry.longval); - quicklistIndex(ql, 3, &entry); - if (entry.longval != 4444) - ERR("Not 4444, %lld", entry.longval); - if (quicklistIndex(ql, 4, &entry)) - ERR("Index past elements: %lld", entry.longval); - quicklistIndex(ql, -1, &entry); - if (entry.longval != 4444) - ERR("Not 4444 (reverse), %lld", entry.longval); - quicklistIndex(ql, -2, &entry); - if (entry.longval != 3333) - ERR("Not 3333 (reverse), %lld", entry.longval); - quicklistIndex(ql, -3, &entry); - if (entry.longval != 2222) - ERR("Not 2222 (reverse), %lld", entry.longval); - quicklistIndex(ql, -4, &entry); - if (entry.longval != 1111) - ERR("Not 1111 (reverse), %lld", entry.longval); - if (quicklistIndex(ql, -5, &entry)) - ERR("Index past elements (reverse), %lld", entry.longval); - quicklistRelease(ql); - } - - TEST("numbers larger list read") { - quicklist *ql = quicklistCreate(); - char num[32]; - long long nums[5000]; - for (int i = 0; i < 5000; i++) { - nums[i] = -5157318210846258176 + i; - int sz = ll2string(num, sizeof(num), nums[i]); - quicklistPushTail(ql, F, num, sz); - } - quicklistPushTail(ql, F, "xxxxxxxxxxxxxxxxxxxx", 20); - quicklistEntry entry; - for (int i = 0; i < 5000; i++) { - quicklistIndex(ql, i, &entry); - if (entry.longval != nums[i]) - ERR("[%d] Not longval %lld but rather %lld", i, nums[i], - entry.longval); - entry.longval = 0xdeadbeef; - } - quicklistIndex(ql, 5000, &entry); - if (strncmp((char *)entry.value, "xxxxxxxxxxxxxxxxxxxx", 20)) - ERR("String val not match: %s", entry.value); - ql_verify(ql, 157, 5001, 32, 9); - quicklistRelease(ql); - } - - TEST("numbers larger list read B") { - quicklist *ql = quicklistCreate(); - quicklistPushTail(ql, F, "99", 2); - quicklistPushTail(ql, F, "98", 2); - quicklistPushTail(ql, F, "xxxxxxxxxxxxxxxxxxxx", 20); - quicklistPushTail(ql, F, "96", 2); - quicklistPushTail(ql, F, "95", 2); - quicklistReplaceAtIndex(ql, 1, "foo", 3); - quicklistReplaceAtIndex(ql, -1, "bar", 3); - quicklistRelease(ql); - OK; - } - - for (int f = optimize_start; f < 16; f++) { - TEST_DESC("lrem test at fill %d", f) { - quicklist *ql = quicklistCreate(); - char *words[] = { "abc", "foo", "bar", "foobar", "foobared", - "zap", "bar", "test", "foo" }; - char *result[] = { "abc", "foo", "foobar", "foobared", - "zap", "test", "foo" }; - char *resultB[] = { "abc", "foo", "foobar", - "foobared", "zap", "test" }; - for (int i = 0; i < 9; i++) - quicklistPushTail(ql, f, words[i], strlen(words[i])); - - /* lrem 0 bar */ - quicklistIter *iter = quicklistGetIterator(ql, AL_START_HEAD); - quicklistEntry entry; - int i = 0; - while (quicklistNext(iter, &entry)) { - if (quicklistCompare(entry.zi, (unsigned char *)"bar", 3)) { - quicklistDelEntry(iter, &entry); - } - i++; - } - quicklistReleaseIterator(iter); - - /* check result of lrem 0 bar */ - iter = quicklistGetIterator(ql, AL_START_HEAD); - i = 0; - int ok = 1; - while (quicklistNext(iter, &entry)) { - /* Result must be: abc, foo, foobar, foobared, zap, test, foo */ - if (strncmp((char *)entry.value, result[i], entry.sz)) { - ERR("No match at position %d, got %.*s instead of %s", i, - entry.sz, entry.value, result[i]); - ok = 0; - } - i++; - } - quicklistReleaseIterator(iter); - - quicklistPushTail(ql, f, "foo", 3); - - /* lrem -2 foo */ - iter = quicklistGetIterator(ql, AL_START_TAIL); - i = 0; - int del = 2; - while (quicklistNext(iter, &entry)) { - if (quicklistCompare(entry.zi, (unsigned char *)"foo", 3)) { - quicklistDelEntry(iter, &entry); - del--; - } - if (!del) - break; - i++; - } - quicklistReleaseIterator(iter); - - /* check result of lrem -2 foo */ - /* (we're ignoring the '2' part and still deleting all foo because - * we only have two foo) */ - iter = quicklistGetIterator(ql, AL_START_TAIL); - i = 0; - size_t resB = sizeof(resultB) / sizeof(*resultB); - while (quicklistNext(iter, &entry)) { - /* Result must be: abc, foo, foobar, foobared, zap, test, foo */ - if (strncmp((char *)entry.value, resultB[resB - 1 - i], - entry.sz)) { - ERR("No match at position %d, got %.*s instead of %s", i, - entry.sz, entry.value, resultB[resB - 1 - i]); - ok = 0; - } - i++; - } - - quicklistReleaseIterator(iter); - /* final result of all tests */ - if (ok) - OK; - quicklistRelease(ql); - } - } - - for (int f = optimize_start; f < 16; f++) { - TEST_DESC("iterate reverse + delete at fill %d", f) { - quicklist *ql = quicklistCreate(); - quicklistPushTail(ql, f, "abc", 3); - quicklistPushTail(ql, f, "def", 3); - quicklistPushTail(ql, f, "hij", 3); - quicklistPushTail(ql, f, "jkl", 3); - quicklistPushTail(ql, f, "oop", 3); - - quicklistEntry entry; + quicklistPushHead(ql, genstr("hello", i), 32); quicklistIter *iter = quicklistGetIterator(ql, AL_START_TAIL); + quicklistEntry entry; int i = 0; while (quicklistNext(iter, &entry)) { - if (quicklistCompare(entry.zi, (unsigned char *)"hij", 3)) { - quicklistDelEntry(iter, &entry); - } - i++; - } - quicklistReleaseIterator(iter); - - if (i != 5) - ERR("Didn't iterate 5 times, iterated %d times.", i); - - /* Check results after deletion of "hij" */ - iter = quicklistGetIterator(ql, AL_START_HEAD); - i = 0; - char *vals[] = { "abc", "def", "jkl", "oop" }; - while (quicklistNext(iter, &entry)) { - if (!quicklistCompare(entry.zi, (unsigned char *)vals[i], 3)) { - ERR("Value at %d didn't match %s\n", i, vals[i]); - } + char *h = genstr("hello", i); + if (strcmp((char *)entry.value, h)) + ERR("value [%s] didn't match [%s] at position %d", + entry.value, h, i); i++; } + if (i != 500) + ERR("Didn't iterate over exactly 500 elements (%d)", i); + ql_verify(ql, 16, 500, 20, 32); quicklistReleaseIterator(iter); quicklistRelease(ql); } - } - for (int f = optimize_start; f < 800; f++) { - TEST_DESC("iterator at index test at fill %d", f) { - quicklist *ql = quicklistCreate(); - char num[32]; - long long nums[5000]; - for (int i = 0; i < 760; i++) { - nums[i] = -5157318210846258176 + i; - int sz = ll2string(num, sizeof(num), nums[i]); - quicklistPushTail(ql, f, num, sz); + TEST("insert before with 0 elements") { + quicklist *ql = quicklistNew(-2, options[_i]); + quicklistEntry entry; + quicklistIndex(ql, 0, &entry); + quicklistInsertBefore(ql, &entry, "abc", 4); + ql_verify(ql, 1, 1, 1, 1); + quicklistRelease(ql); + } + + TEST("insert after with 0 elements") { + quicklist *ql = quicklistNew(-2, options[_i]); + quicklistEntry entry; + quicklistIndex(ql, 0, &entry); + quicklistInsertAfter(ql, &entry, "abc", 4); + ql_verify(ql, 1, 1, 1, 1); + quicklistRelease(ql); + } + + TEST("insert after 1 element") { + quicklist *ql = quicklistNew(-2, options[_i]); + quicklistPushHead(ql, "hello", 6); + quicklistEntry entry; + quicklistIndex(ql, 0, &entry); + quicklistInsertAfter(ql, &entry, "abc", 4); + ql_verify(ql, 1, 2, 2, 2); + quicklistRelease(ql); + } + + TEST("insert before 1 element") { + quicklist *ql = quicklistNew(-2, options[_i]); + quicklistPushHead(ql, "hello", 6); + quicklistEntry entry; + quicklistIndex(ql, 0, &entry); + quicklistInsertAfter(ql, &entry, "abc", 4); + ql_verify(ql, 1, 2, 2, 2); + quicklistRelease(ql); + } + + for (int f = optimize_start; f < 12; f++) { + TEST_DESC("insert once in elements while iterating at fill %d at " + "compress %d\n", + f, options[_i]) { + quicklist *ql = quicklistNew(f, options[_i]); + quicklistPushTail(ql, "abc", 3); + quicklistSetFill(ql, 1); + quicklistPushTail(ql, "def", 3); /* force to unique node */ + quicklistSetFill(ql, f); + quicklistPushTail(ql, "bob", 3); /* force to reset for +3 */ + quicklistPushTail(ql, "foo", 3); + quicklistPushTail(ql, "zoo", 3); + + itrprintr(ql, 0); + /* insert "bar" before "bob" while iterating over list. */ + quicklistIter *iter = quicklistGetIterator(ql, AL_START_HEAD); + quicklistEntry entry; + while (quicklistNext(iter, &entry)) { + if (!strncmp((char *)entry.value, "bob", 3)) { + /* Insert as fill = 1 so it spills into new node. */ + quicklistInsertBefore(ql, &entry, "bar", 3); + break; /* didn't we fix insert-while-iterating? */ + } + } + itrprintr(ql, 0); + + /* verify results */ + quicklistIndex(ql, 0, &entry); + if (strncmp((char *)entry.value, "abc", 3)) + ERR("Value 0 didn't match, instead got: %.*s", entry.sz, + entry.value); + quicklistIndex(ql, 1, &entry); + if (strncmp((char *)entry.value, "def", 3)) + ERR("Value 1 didn't match, instead got: %.*s", entry.sz, + entry.value); + quicklistIndex(ql, 2, &entry); + if (strncmp((char *)entry.value, "bar", 3)) + ERR("Value 2 didn't match, instead got: %.*s", entry.sz, + entry.value); + quicklistIndex(ql, 3, &entry); + if (strncmp((char *)entry.value, "bob", 3)) + ERR("Value 3 didn't match, instead got: %.*s", entry.sz, + entry.value); + quicklistIndex(ql, 4, &entry); + if (strncmp((char *)entry.value, "foo", 3)) + ERR("Value 4 didn't match, instead got: %.*s", entry.sz, + entry.value); + quicklistIndex(ql, 5, &entry); + if (strncmp((char *)entry.value, "zoo", 3)) + ERR("Value 5 didn't match, instead got: %.*s", entry.sz, + entry.value); + quicklistReleaseIterator(iter); + quicklistRelease(ql); + } + } + + for (int f = optimize_start; f < 1024; f++) { + TEST_DESC( + "insert [before] 250 new in middle of 500 elements at fill" + " %d at compress %d", + f, options[_i]) { + quicklist *ql = quicklistNew(f, options[_i]); + for (int i = 0; i < 500; i++) + quicklistPushTail(ql, genstr("hello", i), 32); + for (int i = 0; i < 250; i++) { + quicklistEntry entry; + quicklistIndex(ql, 250, &entry); + quicklistInsertBefore(ql, &entry, genstr("abc", i), 32); + } + if (f == 32) + ql_verify(ql, 25, 750, 32, 20); + quicklistRelease(ql); + } + } + + for (int f = optimize_start; f < 1024; f++) { + TEST_DESC("insert [after] 250 new in middle of 500 elements at " + "fill %d at compress %d", + f, options[_i]) { + quicklist *ql = quicklistNew(f, options[_i]); + for (int i = 0; i < 500; i++) + quicklistPushHead(ql, genstr("hello", i), 32); + for (int i = 0; i < 250; i++) { + quicklistEntry entry; + quicklistIndex(ql, 250, &entry); + quicklistInsertAfter(ql, &entry, genstr("abc", i), 32); + } + + if (ql->count != 750) + ERR("List size not 750, but rather %ld", ql->count); + + if (f == 32) + ql_verify(ql, 26, 750, 20, 32); + quicklistRelease(ql); + } + } + + TEST("duplicate empty list") { + quicklist *ql = quicklistNew(-2, options[_i]); + ql_verify(ql, 0, 0, 0, 0); + quicklist *copy = quicklistDup(ql); + ql_verify(copy, 0, 0, 0, 0); + quicklistRelease(ql); + quicklistRelease(copy); + } + + TEST("duplicate list of 1 element") { + quicklist *ql = quicklistNew(-2, options[_i]); + quicklistPushHead(ql, genstr("hello", 3), 32); + ql_verify(ql, 1, 1, 1, 1); + quicklist *copy = quicklistDup(ql); + ql_verify(copy, 1, 1, 1, 1); + quicklistRelease(ql); + quicklistRelease(copy); + } + + TEST("duplicate list of 500") { + quicklist *ql = quicklistNew(-2, options[_i]); + quicklistSetFill(ql, 32); + for (int i = 0; i < 500; i++) + quicklistPushHead(ql, genstr("hello", i), 32); + ql_verify(ql, 16, 500, 20, 32); + + quicklist *copy = quicklistDup(ql); + ql_verify(copy, 16, 500, 20, 32); + quicklistRelease(ql); + quicklistRelease(copy); + } + + for (int f = optimize_start; f < 512; f++) { + TEST_DESC("index 1,200 from 500 list at fill %d at compress %d", f, + options[_i]) { + quicklist *ql = quicklistNew(f, options[_i]); + for (int i = 0; i < 500; i++) + quicklistPushTail(ql, genstr("hello", i + 1), 32); + quicklistEntry entry; + quicklistIndex(ql, 1, &entry); + if (!strcmp((char *)entry.value, "hello2")) + OK; + else + ERR("Value: %s", entry.value); + quicklistIndex(ql, 200, &entry); + if (!strcmp((char *)entry.value, "hello201")) + OK; + else + ERR("Value: %s", entry.value); + quicklistRelease(ql); } + TEST_DESC("index -1,-2 from 500 list at fill %d at compress %d", f, + options[_i]) { + quicklist *ql = quicklistNew(f, options[_i]); + for (int i = 0; i < 500; i++) + quicklistPushTail(ql, genstr("hello", i + 1), 32); + quicklistEntry entry; + quicklistIndex(ql, -1, &entry); + if (!strcmp((char *)entry.value, "hello500")) + OK; + else + ERR("Value: %s", entry.value); + quicklistIndex(ql, -2, &entry); + if (!strcmp((char *)entry.value, "hello499")) + OK; + else + ERR("Value: %s", entry.value); + quicklistRelease(ql); + } + + TEST_DESC("index -100 from 500 list at fill %d at compress %d", f, + options[_i]) { + quicklist *ql = quicklistNew(f, options[_i]); + for (int i = 0; i < 500; i++) + quicklistPushTail(ql, genstr("hello", i + 1), 32); + quicklistEntry entry; + quicklistIndex(ql, -100, &entry); + if (!strcmp((char *)entry.value, "hello401")) + OK; + else + ERR("Value: %s", entry.value); + quicklistRelease(ql); + } + + TEST_DESC("index too big +1 from 50 list at fill %d at compress %d", + f, options[_i]) { + quicklist *ql = quicklistNew(f, options[_i]); + for (int i = 0; i < 50; i++) + quicklistPushTail(ql, genstr("hello", i + 1), 32); + quicklistEntry entry; + if (quicklistIndex(ql, 50, &entry)) + ERR("Index found at 50 with 50 list: %.*s", entry.sz, + entry.value); + else + OK; + quicklistRelease(ql); + } + } + + TEST("delete range empty list") { + quicklist *ql = quicklistNew(-2, options[_i]); + quicklistDelRange(ql, 5, 20); + ql_verify(ql, 0, 0, 0, 0); + quicklistRelease(ql); + } + + TEST("delete range of entire node in list of one node") { + quicklist *ql = quicklistNew(-2, options[_i]); + for (int i = 0; i < 32; i++) + quicklistPushHead(ql, genstr("hello", i), 32); + ql_verify(ql, 1, 32, 32, 32); + quicklistDelRange(ql, 0, 32); + ql_verify(ql, 0, 0, 0, 0); + quicklistRelease(ql); + } + + TEST("delete range of entire node with overflow counts") { + quicklist *ql = quicklistNew(-2, options[_i]); + for (int i = 0; i < 32; i++) + quicklistPushHead(ql, genstr("hello", i), 32); + ql_verify(ql, 1, 32, 32, 32); + quicklistDelRange(ql, 0, 128); + ql_verify(ql, 0, 0, 0, 0); + quicklistRelease(ql); + } + + TEST("delete middle 100 of 500 list") { + quicklist *ql = quicklistNew(-2, options[_i]); + quicklistSetFill(ql, 32); + for (int i = 0; i < 500; i++) + quicklistPushTail(ql, genstr("hello", i + 1), 32); + ql_verify(ql, 16, 500, 32, 20); + quicklistDelRange(ql, 200, 100); + ql_verify(ql, 14, 400, 32, 20); + quicklistRelease(ql); + } + + TEST("delete negative 1 from 500 list") { + quicklist *ql = quicklistNew(-2, options[_i]); + quicklistSetFill(ql, 32); + for (int i = 0; i < 500; i++) + quicklistPushTail(ql, genstr("hello", i + 1), 32); + ql_verify(ql, 16, 500, 32, 20); + quicklistDelRange(ql, -1, 1); + ql_verify(ql, 16, 499, 32, 19); + quicklistRelease(ql); + } + + TEST("delete negative 1 from 500 list with overflow counts") { + quicklist *ql = quicklistNew(-2, options[_i]); + quicklistSetFill(ql, 32); + for (int i = 0; i < 500; i++) + quicklistPushTail(ql, genstr("hello", i + 1), 32); + ql_verify(ql, 16, 500, 32, 20); + quicklistDelRange(ql, -1, 128); + ql_verify(ql, 16, 499, 32, 19); + quicklistRelease(ql); + } + + TEST("delete negative 100 from 500 list") { + quicklist *ql = quicklistNew(-2, options[_i]); + quicklistSetFill(ql, 32); + for (int i = 0; i < 500; i++) + quicklistPushTail(ql, genstr("hello", i + 1), 32); + quicklistDelRange(ql, -100, 100); + ql_verify(ql, 13, 400, 32, 16); + quicklistRelease(ql); + } + + TEST("delete -10 count 5 from 50 list") { + quicklist *ql = quicklistNew(-2, options[_i]); + quicklistSetFill(ql, 32); + for (int i = 0; i < 50; i++) + quicklistPushTail(ql, genstr("hello", i + 1), 32); + ql_verify(ql, 2, 50, 32, 18); + quicklistDelRange(ql, -10, 5); + ql_verify(ql, 2, 45, 32, 13); + quicklistRelease(ql); + } + + TEST("numbers only list read") { + quicklist *ql = quicklistNew(-2, options[_i]); + quicklistPushTail(ql, "1111", 4); + quicklistPushTail(ql, "2222", 4); + quicklistPushTail(ql, "3333", 4); + quicklistPushTail(ql, "4444", 4); + ql_verify(ql, 1, 4, 4, 4); quicklistEntry entry; - quicklistIter *iter = - quicklistGetIteratorAtIdx(ql, AL_START_HEAD, 437); - int i = 437; - while (quicklistNext(iter, &entry)) { + quicklistIndex(ql, 0, &entry); + if (entry.longval != 1111) + ERR("Not 1111, %lld", entry.longval); + quicklistIndex(ql, 1, &entry); + if (entry.longval != 2222) + ERR("Not 2222, %lld", entry.longval); + quicklistIndex(ql, 2, &entry); + if (entry.longval != 3333) + ERR("Not 3333, %lld", entry.longval); + quicklistIndex(ql, 3, &entry); + if (entry.longval != 4444) + ERR("Not 4444, %lld", entry.longval); + if (quicklistIndex(ql, 4, &entry)) + ERR("Index past elements: %lld", entry.longval); + quicklistIndex(ql, -1, &entry); + if (entry.longval != 4444) + ERR("Not 4444 (reverse), %lld", entry.longval); + quicklistIndex(ql, -2, &entry); + if (entry.longval != 3333) + ERR("Not 3333 (reverse), %lld", entry.longval); + quicklistIndex(ql, -3, &entry); + if (entry.longval != 2222) + ERR("Not 2222 (reverse), %lld", entry.longval); + quicklistIndex(ql, -4, &entry); + if (entry.longval != 1111) + ERR("Not 1111 (reverse), %lld", entry.longval); + if (quicklistIndex(ql, -5, &entry)) + ERR("Index past elements (reverse), %lld", entry.longval); + quicklistRelease(ql); + } + + TEST("numbers larger list read") { + quicklist *ql = quicklistNew(-2, options[_i]); + quicklistSetFill(ql, 32); + char num[32]; + long long nums[5000]; + for (int i = 0; i < 5000; i++) { + nums[i] = -5157318210846258176 + i; + int sz = ll2string(num, sizeof(num), nums[i]); + quicklistPushTail(ql, num, sz); + } + quicklistPushTail(ql, "xxxxxxxxxxxxxxxxxxxx", 20); + quicklistEntry entry; + for (int i = 0; i < 5000; i++) { + quicklistIndex(ql, i, &entry); if (entry.longval != nums[i]) - ERR("Expected %lld, but got %lld", entry.longval, nums[i]); - i++; + ERR("[%d] Not longval %lld but rather %lld", i, nums[i], + entry.longval); + entry.longval = 0xdeadbeef; } - quicklistReleaseIterator(iter); + quicklistIndex(ql, 5000, &entry); + if (strncmp((char *)entry.value, "xxxxxxxxxxxxxxxxxxxx", 20)) + ERR("String val not match: %s", entry.value); + ql_verify(ql, 157, 5001, 32, 9); quicklistRelease(ql); } + + TEST("numbers larger list read B") { + quicklist *ql = quicklistNew(-2, options[_i]); + quicklistPushTail(ql, "99", 2); + quicklistPushTail(ql, "98", 2); + quicklistPushTail(ql, "xxxxxxxxxxxxxxxxxxxx", 20); + quicklistPushTail(ql, "96", 2); + quicklistPushTail(ql, "95", 2); + quicklistReplaceAtIndex(ql, 1, "foo", 3); + quicklistReplaceAtIndex(ql, -1, "bar", 3); + quicklistRelease(ql); + OK; + } + + for (int f = optimize_start; f < 16; f++) { + TEST_DESC("lrem test at fill %d at compress %d", f, options[_i]) { + quicklist *ql = quicklistNew(f, options[_i]); + char *words[] = { "abc", "foo", "bar", "foobar", "foobared", + "zap", "bar", "test", "foo" }; + char *result[] = { "abc", "foo", "foobar", "foobared", + "zap", "test", "foo" }; + char *resultB[] = { "abc", "foo", "foobar", + "foobared", "zap", "test" }; + for (int i = 0; i < 9; i++) + quicklistPushTail(ql, words[i], strlen(words[i])); + + /* lrem 0 bar */ + quicklistIter *iter = quicklistGetIterator(ql, AL_START_HEAD); + quicklistEntry entry; + int i = 0; + while (quicklistNext(iter, &entry)) { + if (quicklistCompare(entry.zi, (unsigned char *)"bar", 3)) { + quicklistDelEntry(iter, &entry); + } + i++; + } + quicklistReleaseIterator(iter); + + /* check result of lrem 0 bar */ + iter = quicklistGetIterator(ql, AL_START_HEAD); + i = 0; + int ok = 1; + while (quicklistNext(iter, &entry)) { + /* Result must be: abc, foo, foobar, foobared, zap, test, + * foo */ + if (strncmp((char *)entry.value, result[i], entry.sz)) { + ERR("No match at position %d, got %.*s instead of %s", + i, entry.sz, entry.value, result[i]); + ok = 0; + } + i++; + } + quicklistReleaseIterator(iter); + + quicklistPushTail(ql, "foo", 3); + + /* lrem -2 foo */ + iter = quicklistGetIterator(ql, AL_START_TAIL); + i = 0; + int del = 2; + while (quicklistNext(iter, &entry)) { + if (quicklistCompare(entry.zi, (unsigned char *)"foo", 3)) { + quicklistDelEntry(iter, &entry); + del--; + } + if (!del) + break; + i++; + } + quicklistReleaseIterator(iter); + + /* check result of lrem -2 foo */ + /* (we're ignoring the '2' part and still deleting all foo + * because + * we only have two foo) */ + iter = quicklistGetIterator(ql, AL_START_TAIL); + i = 0; + size_t resB = sizeof(resultB) / sizeof(*resultB); + while (quicklistNext(iter, &entry)) { + /* Result must be: abc, foo, foobar, foobared, zap, test, + * foo */ + if (strncmp((char *)entry.value, resultB[resB - 1 - i], + entry.sz)) { + ERR("No match at position %d, got %.*s instead of %s", + i, entry.sz, entry.value, resultB[resB - 1 - i]); + ok = 0; + } + i++; + } + + quicklistReleaseIterator(iter); + /* final result of all tests */ + if (ok) + OK; + quicklistRelease(ql); + } + } + + for (int f = optimize_start; f < 16; f++) { + TEST_DESC("iterate reverse + delete at fill %d at compress %d", f, + options[_i]) { + quicklist *ql = quicklistNew(f, options[_i]); + quicklistPushTail(ql, "abc", 3); + quicklistPushTail(ql, "def", 3); + quicklistPushTail(ql, "hij", 3); + quicklistPushTail(ql, "jkl", 3); + quicklistPushTail(ql, "oop", 3); + + quicklistEntry entry; + quicklistIter *iter = quicklistGetIterator(ql, AL_START_TAIL); + int i = 0; + while (quicklistNext(iter, &entry)) { + if (quicklistCompare(entry.zi, (unsigned char *)"hij", 3)) { + quicklistDelEntry(iter, &entry); + } + i++; + } + quicklistReleaseIterator(iter); + + if (i != 5) + ERR("Didn't iterate 5 times, iterated %d times.", i); + + /* Check results after deletion of "hij" */ + iter = quicklistGetIterator(ql, AL_START_HEAD); + i = 0; + char *vals[] = { "abc", "def", "jkl", "oop" }; + while (quicklistNext(iter, &entry)) { + if (!quicklistCompare(entry.zi, (unsigned char *)vals[i], + 3)) { + ERR("Value at %d didn't match %s\n", i, vals[i]); + } + i++; + } + quicklistReleaseIterator(iter); + quicklistRelease(ql); + } + } + + for (int f = optimize_start; f < 800; f++) { + TEST_DESC("iterator at index test at fill %d at compress %d", f, + options[_i]) { + quicklist *ql = quicklistNew(f, options[_i]); + char num[32]; + long long nums[5000]; + for (int i = 0; i < 760; i++) { + nums[i] = -5157318210846258176 + i; + int sz = ll2string(num, sizeof(num), nums[i]); + quicklistPushTail(ql, num, sz); + } + + quicklistEntry entry; + quicklistIter *iter = + quicklistGetIteratorAtIdx(ql, AL_START_HEAD, 437); + int i = 437; + while (quicklistNext(iter, &entry)) { + if (entry.longval != nums[i]) + ERR("Expected %lld, but got %lld", entry.longval, + nums[i]); + i++; + } + quicklistReleaseIterator(iter); + quicklistRelease(ql); + } + } + + for (int f = optimize_start; f < 40; f++) { + TEST_DESC("ltrim test A at fill %d at compress %d", f, + options[_i]) { + quicklist *ql = quicklistNew(f, options[_i]); + char num[32]; + long long nums[5000]; + for (int i = 0; i < 32; i++) { + nums[i] = -5157318210846258176 + i; + int sz = ll2string(num, sizeof(num), nums[i]); + quicklistPushTail(ql, num, sz); + } + if (f == 32) + ql_verify(ql, 1, 32, 32, 32); + /* ltrim 25 53 (keep [25,32] inclusive = 7 remaining) */ + quicklistDelRange(ql, 0, 25); + quicklistDelRange(ql, 0, 0); + quicklistEntry entry; + for (int i = 0; i < 7; i++) { + quicklistIndex(ql, i, &entry); + if (entry.longval != nums[25 + i]) + ERR("Deleted invalid range! Expected %lld but got " + "%lld", + entry.longval, nums[25 + i]); + } + if (f == 32) + ql_verify(ql, 1, 7, 7, 7); + quicklistRelease(ql); + } + } + + for (int f = optimize_start; f < 40; f++) { + TEST_DESC("ltrim test B at fill %d at compress %d", f, + options[_i]) { + /* Force-disable compression because our 33 sequential + * integers don't compress and the check always fails. */ + quicklist *ql = quicklistNew(f, QUICKLIST_NOCOMPRESS); + char num[32]; + long long nums[5000]; + for (int i = 0; i < 33; i++) { + nums[i] = i; + int sz = ll2string(num, sizeof(num), nums[i]); + quicklistPushTail(ql, num, sz); + } + if (f == 32) + ql_verify(ql, 2, 33, 32, 1); + /* ltrim 5 16 (keep [5,16] inclusive = 12 remaining) */ + quicklistDelRange(ql, 0, 5); + quicklistDelRange(ql, -16, 16); + if (f == 32) + ql_verify(ql, 1, 12, 12, 12); + quicklistEntry entry; + quicklistIndex(ql, 0, &entry); + if (entry.longval != 5) + ERR("A: longval not 5, but %lld", entry.longval); + else + OK; + quicklistIndex(ql, -1, &entry); + if (entry.longval != 16) + ERR("B! got instead: %lld", entry.longval); + else + OK; + quicklistPushTail(ql, "bobobob", 7); + quicklistIndex(ql, -1, &entry); + if (strncmp((char *)entry.value, "bobobob", 7)) + ERR("Tail doesn't match bobobob, it's %.*s instead", + entry.sz, entry.value); + for (int i = 0; i < 12; i++) { + quicklistIndex(ql, i, &entry); + if (entry.longval != nums[5 + i]) + ERR("Deleted invalid range! Expected %lld but got " + "%lld", + entry.longval, nums[5 + i]); + } + quicklistRelease(ql); + } + } + + for (int f = optimize_start; f < 40; f++) { + TEST_DESC("ltrim test C at fill %d at compress %d", f, + options[_i]) { + quicklist *ql = quicklistNew(f, options[_i]); + char num[32]; + long long nums[5000]; + for (int i = 0; i < 33; i++) { + nums[i] = -5157318210846258176 + i; + int sz = ll2string(num, sizeof(num), nums[i]); + quicklistPushTail(ql, num, sz); + } + if (f == 32) + ql_verify(ql, 2, 33, 32, 1); + /* ltrim 3 3 (keep [3,3] inclusive = 1 remaining) */ + quicklistDelRange(ql, 0, 3); + quicklistDelRange(ql, -29, + 4000); /* make sure not loop forever */ + if (f == 32) + ql_verify(ql, 1, 1, 1, 1); + quicklistEntry entry; + quicklistIndex(ql, 0, &entry); + if (entry.longval != -5157318210846258173) + ERROR; + else + OK; + quicklistRelease(ql); + } + } + + for (int f = optimize_start; f < 40; f++) { + TEST_DESC("ltrim test D at fill %d at compress %d", f, + options[_i]) { + quicklist *ql = quicklistNew(f, options[_i]); + char num[32]; + long long nums[5000]; + for (int i = 0; i < 33; i++) { + nums[i] = -5157318210846258176 + i; + int sz = ll2string(num, sizeof(num), nums[i]); + quicklistPushTail(ql, num, sz); + } + if (f == 32) + ql_verify(ql, 2, 33, 32, 1); + quicklistDelRange(ql, -12, 3); + if (ql->count != 30) + ERR("Didn't delete exactly three elements! Count is: %lu", + ql->count); + quicklistRelease(ql); + } + } + + for (int f = optimize_start; f < 72; f++) { + TEST_DESC("create quicklist from ziplist at fill %d at compress %d", + f, options[_i]) { + unsigned char *zl = ziplistNew(); + long long nums[64]; + char num[64]; + for (int i = 0; i < 33; i++) { + nums[i] = -5157318210846258176 + i; + int sz = ll2string(num, sizeof(num), nums[i]); + zl = + ziplistPush(zl, (unsigned char *)num, sz, ZIPLIST_TAIL); + } + for (int i = 0; i < 33; i++) { + zl = ziplistPush(zl, (unsigned char *)genstr("hello", i), + 32, ZIPLIST_TAIL); + } + quicklist *ql = quicklistCreateFromZiplist(f, options[_i], zl); + if (f == 1) + ql_verify(ql, 66, 66, 1, 1); + else if (f == 32) + ql_verify(ql, 3, 66, 32, 2); + else if (f == 66) + ql_verify(ql, 1, 66, 66, 66); + quicklistRelease(ql); + } + } + + long long stop = mstime(); + runtime[_i] = stop - start; } - for (int f = optimize_start; f < 40; f++) { - TEST_DESC("ltrim test A at fill %d", f) { - quicklist *ql = quicklistCreate(); - char num[32]; - long long nums[5000]; - for (int i = 0; i < 32; i++) { - nums[i] = -5157318210846258176 + i; - int sz = ll2string(num, sizeof(num), nums[i]); - quicklistPushTail(ql, f, num, sz); - } - if (f == 32) - ql_verify(ql, 1, 32, 32, 32); - /* ltrim 25 53 (keep [25,32] inclusive = 7 remaining) */ - quicklistDelRange(ql, 0, 25); - quicklistDelRange(ql, 0, 0); - quicklistEntry entry; - for (int i = 0; i < 7; i++) { - quicklistIndex(ql, i, &entry); - if (entry.longval != nums[25 + i]) - ERR("Deleted invalid range! Expected %lld but got %lld", - entry.longval, nums[25 + i]); - } - if (f == 32) - ql_verify(ql, 1, 7, 7, 7); - quicklistRelease(ql); - } - } + /* Run a longer test of compression depth outside of primary test loop. */ + int list_sizes[] = { 250, 251, 500, 999, 1000 }; + long long start = mstime(); + for (int list = 0; list < (int)(sizeof(list_sizes) / sizeof(*list_sizes)); + list++) { + for (int f = optimize_start; f < 128; f++) { + for (int depth = 1; depth < 40; depth++) { + /* skip over many redundant test cases */ + TEST_DESC("verify specific compression of interior nodes with " + "%d list " + "at fill %d at compress %d", + list_sizes[list], f, depth) { + quicklist *ql = quicklistNew(f, depth); + for (int i = 0; i < list_sizes[list]; i++) { + quicklistPushTail(ql, genstr("hello TAIL", i + 1), 64); + quicklistPushHead(ql, genstr("hello HEAD", i + 1), 64); + } - for (int f = optimize_start; f < 40; f++) { - TEST_DESC("ltrim test B at fill %d", f) { - quicklist *ql = quicklistCreate(); - char num[32]; - long long nums[5000]; - for (int i = 0; i < 33; i++) { - nums[i] = i; - int sz = ll2string(num, sizeof(num), nums[i]); - quicklistPushTail(ql, f, num, sz); - } - if (f == 32) - ql_verify(ql, 2, 33, 32, 1); - /* ltrim 5 16 (keep [5,16] inclusive = 12 remaining) */ - quicklistDelRange(ql, 0, 5); - quicklistDelRange(ql, -16, 16); - if (f == 32) - ql_verify(ql, 1, 12, 12, 12); - quicklistEntry entry; - quicklistIndex(ql, 0, &entry); - if (entry.longval != 5) - ERR("A: longval not 5, but %lld", entry.longval); - else - OK; - quicklistIndex(ql, -1, &entry); - if (entry.longval != 16) - ERR("B! got instead: %lld", entry.longval); - else - OK; - quicklistPushTail(ql, f, "bobobob", 7); - quicklistIndex(ql, -1, &entry); - if (strncmp((char *)entry.value, "bobobob", 7)) - ERR("Tail doesn't match bobobob, it's %.*s instead", entry.sz, - entry.value); - for (int i = 0; i < 12; i++) { - quicklistIndex(ql, i, &entry); - if (entry.longval != nums[5 + i]) - ERR("Deleted invalid range! Expected %lld but got %lld", - entry.longval, nums[5 + i]); - } - quicklistRelease(ql); - } - } + quicklistNode *node = ql->head; + unsigned int low_raw = ql->compress; + unsigned int high_raw = ql->len - ql->compress; - for (int f = optimize_start; f < 40; f++) { - TEST_DESC("ltrim test C at fill %d", f) { - quicklist *ql = quicklistCreate(); - char num[32]; - long long nums[5000]; - for (int i = 0; i < 33; i++) { - nums[i] = -5157318210846258176 + i; - int sz = ll2string(num, sizeof(num), nums[i]); - quicklistPushTail(ql, f, num, sz); + for (unsigned int at = 0; at < ql->len; + at++, node = node->next) { + if (at < low_raw || at >= high_raw) { + if (node->encoding != QUICKLIST_NODE_ENCODING_RAW) { + ERR("Incorrect compression: node %d is " + "compressed at depth %d ((%u, %u); total " + "nodes: %u; size: %u)", + at, depth, low_raw, high_raw, ql->len, + node->sz); + } + } else { + if (node->encoding != QUICKLIST_NODE_ENCODING_LZF) { + ERR("Incorrect non-compression: node %d is NOT " + "compressed at depth %d ((%u, %u); total " + "nodes: %u; size: %u; attempted: %d)", + at, depth, low_raw, high_raw, ql->len, + node->sz, node->attempted_compress); + } + } + } + quicklistRelease(ql); + } } - if (f == 32) - ql_verify(ql, 2, 33, 32, 1); - /* ltrim 3 3 (keep [3,3] inclusive = 1 remaining) */ - quicklistDelRange(ql, 0, 3); - quicklistDelRange(ql, -29, 4000); /* make sure not loop forever */ - if (f == 32) - ql_verify(ql, 1, 1, 1, 1); - quicklistEntry entry; - quicklistIndex(ql, 0, &entry); - if (entry.longval != -5157318210846258173) - ERROR; - else - OK; - quicklistRelease(ql); } } + long long stop = mstime(); - for (int f = optimize_start; f < 40; f++) { - TEST_DESC("ltrim test D at fill %d", f) { - quicklist *ql = quicklistCreate(); - char num[32]; - long long nums[5000]; - for (int i = 0; i < 33; i++) { - nums[i] = -5157318210846258176 + i; - int sz = ll2string(num, sizeof(num), nums[i]); - quicklistPushTail(ql, f, num, sz); - } - if (f == 32) - ql_verify(ql, 2, 33, 32, 1); - quicklistDelRange(ql, -12, 3); - if (ql->count != 30) - ERR("Didn't delete exactly three elements! Count is: %lu", - ql->count); - quicklistRelease(ql); - } - } - - for (int f = optimize_start; f < 72; f++) { - TEST_DESC("create quicklist from ziplist at fill %d", f) { - unsigned char *zl = ziplistNew(); - long long nums[32]; - char num[32]; - for (int i = 0; i < 33; i++) { - nums[i] = -5157318210846258176 + i; - int sz = ll2string(num, sizeof(num), nums[i]); - zl = ziplistPush(zl, (unsigned char *)num, sz, ZIPLIST_TAIL); - } - for (int i = 0; i < 33; i++) { - zl = ziplistPush(zl, (unsigned char *)genstr("hello", i), 32, - ZIPLIST_TAIL); - } - quicklist *ql = quicklistCreateFromZiplist(f, zl); - if (f == 1) - ql_verify(ql, 66, 66, 1, 1); - else if (f == 32) - ql_verify(ql, 3, 66, 32, 2); - else if (f == 66) - ql_verify(ql, 1, 66, 66, 66); - quicklistRelease(ql); - } - } + printf("\n"); + for (size_t i = 0; i < option_count; i++) + printf("Test Loop %02d: %0.2f seconds.\n", options[i], + (float)runtime[i] / 1000); + printf("Compressions: %0.2f seconds.\n", (float)(stop - start) / 1000); + printf("\n"); if (!err) printf("ALL TESTS PASSED!\n"); diff --git a/src/quicklist.h b/src/quicklist.h index 6f21f789d..5c9530ccd 100644 --- a/src/quicklist.h +++ b/src/quicklist.h @@ -33,19 +33,50 @@ /* Node, quicklist, and Iterator are the only data structures used currently. */ +/* quicklistNode is a 32 byte struct describing a ziplist for a quicklist. + * We use bit fields keep the quicklistNode at 32 bytes. + * count: 16 bits, max 65536 (max zl bytes is 65k, so max count actually < 32k). + * encoding: 2 bits, RAW=1, LZF=2. + * container: 2 bits, NONE=1, ZIPLIST=2. + * recompress: 1 bit, bool, true if node is temporarry decompressed for usage. + * attempted_compress: 1 bit, boolean, used for verifying during testing. + * extra: 12 bits, free for future use; pads out the remainder of 32 bits */ typedef struct quicklistNode { struct quicklistNode *prev; struct quicklistNode *next; unsigned char *zl; - unsigned int count; /* cached count of items in ziplist */ - unsigned int sz; /* ziplist size in bytes */ + unsigned int sz; /* ziplist size in bytes */ + unsigned int count : 16; /* count of items in ziplist */ + unsigned int encoding : 2; /* RAW==1 or LZF==2 */ + unsigned int container : 2; /* NONE==1 or ZIPLIST==2 */ + unsigned int recompress : 1; /* was this node previous compressed? */ + unsigned int attempted_compress : 1; /* node can't compress; too small */ + unsigned int extra : 10; /* more bits to steal for future usage */ } quicklistNode; +/* quicklistLZF is a 4+N byte struct holding 'sz' followed by 'compressed'. + * 'sz' is byte length of 'compressed' field. + * 'compressed' is LZF data with total (compressed) length 'sz' + * NOTE: uncompressed length is stored in quicklistNode->sz. + * When quicklistNode->zl is compressed, node->zl points to a quicklistLZF */ +typedef struct quicklistLZF { + unsigned int sz; /* LZF size in bytes*/ + char compressed[]; +} quicklistLZF; + +/* quicklist is a 32 byte struct (on 64-bit systems) describing a quicklist. + * 'count' is the number of total entries. + * 'len' is the number of quicklist nodes. + * 'compress' is: -1 if compression disabled, otherwise it's the number + * of quicklistNodes to leave uncompressed at ends of quicklist. + * 'fill' is the user-requested (or default) fill factor. */ typedef struct quicklist { quicklistNode *head; quicklistNode *tail; - unsigned long len; /* number of quicklistNodes */ - unsigned long count; /* total count of all entries in all ziplists */ + unsigned long count; /* total count of all entries in all ziplists */ + unsigned int len; /* number of quicklistNodes */ + int fill : 16; /* fill factor for individual nodes */ + unsigned int compress : 16; /* depth of end nodes not to compress;0=off */ } quicklist; typedef struct quicklistIter { @@ -69,23 +100,40 @@ typedef struct quicklistEntry { #define QUICKLIST_HEAD 0 #define QUICKLIST_TAIL -1 +/* quicklist node encodings */ +#define QUICKLIST_NODE_ENCODING_RAW 1 +#define QUICKLIST_NODE_ENCODING_LZF 2 + +/* quicklist compression disable */ +#define QUICKLIST_NOCOMPRESS 0 + +/* quicklist container formats */ +#define QUICKLIST_NODE_CONTAINER_NONE 1 +#define QUICKLIST_NODE_CONTAINER_ZIPLIST 2 + +#define quicklistNodeIsCompressed(node) \ + ((node)->encoding == QUICKLIST_NODE_ENCODING_LZF) + /* Prototypes */ quicklist *quicklistCreate(void); +quicklist *quicklistNew(int fill, int compress); +void quicklistSetCompressDepth(quicklist *quicklist, int depth); +void quicklistSetFill(quicklist *quicklist, int fill); +void quicklistSetOptions(quicklist *quicklist, int fill, int depth); void quicklistRelease(quicklist *quicklist); -quicklist *quicklistPushHead(quicklist *quicklist, const int fill, void *value, - const size_t sz); -quicklist *quicklistPushTail(quicklist *quicklist, const int fill, void *value, - const size_t sz); -void quicklistPush(quicklist *quicklist, const int fill, void *value, - const size_t sz, int where); +int quicklistPushHead(quicklist *quicklist, void *value, const size_t sz); +int quicklistPushTail(quicklist *quicklist, void *value, const size_t sz); +void quicklistPush(quicklist *quicklist, void *value, const size_t sz, + int where); void quicklistAppendZiplist(quicklist *quicklist, unsigned char *zl); quicklist *quicklistAppendValuesFromZiplist(quicklist *quicklist, - const int fill, unsigned char *zl); -quicklist *quicklistCreateFromZiplist(int fill, unsigned char *zl); -void quicklistInsertAfter(quicklist *quicklist, const int fill, - quicklistEntry *node, void *value, const size_t sz); -void quicklistInsertBefore(quicklist *quicklist, const int fill, - quicklistEntry *node, void *value, const size_t sz); + unsigned char *zl); +quicklist *quicklistCreateFromZiplist(int fill, int compress, + unsigned char *zl); +void quicklistInsertAfter(quicklist *quicklist, quicklistEntry *node, + void *value, const size_t sz); +void quicklistInsertBefore(quicklist *quicklist, quicklistEntry *node, + void *value, const size_t sz); void quicklistDelEntry(quicklistIter *iter, quicklistEntry *entry); int quicklistReplaceAtIndex(quicklist *quicklist, long index, void *data, int sz); @@ -100,7 +148,7 @@ int quicklistIndex(const quicklist *quicklist, const long long index, quicklistEntry *entry); void quicklistRewind(quicklist *quicklist, quicklistIter *li); void quicklistRewindTail(quicklist *quicklist, quicklistIter *li); -void quicklistRotate(quicklist *quicklist, const int fill); +void quicklistRotate(quicklist *quicklist); int quicklistPopCustom(quicklist *quicklist, int where, unsigned char **data, unsigned int *sz, long long *sval, void *(*saver)(unsigned char *data, unsigned int sz)); @@ -108,6 +156,7 @@ int quicklistPop(quicklist *quicklist, int where, unsigned char **data, unsigned int *sz, long long *slong); unsigned int quicklistCount(quicklist *ql); int quicklistCompare(unsigned char *p1, unsigned char *p2, int p2_len); +size_t quicklistGetLzf(const quicklistNode *node, void **data); #ifdef REDIS_TEST int quicklistTest(int argc, char *argv[]); diff --git a/src/rdb.c b/src/rdb.c index fe9964635..209a0da11 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -209,10 +209,33 @@ int rdbTryIntegerEncoding(char *s, size_t len, unsigned char *enc) { return rdbEncodeInteger(value,enc); } -int rdbSaveLzfStringObject(rio *rdb, unsigned char *s, size_t len) { - size_t comprlen, outlen; +int rdbSaveLzfBlob(rio *rdb, void *data, size_t compress_len, + size_t original_len) { unsigned char byte; int n, nwritten = 0; + + /* Data compressed! Let's save it on disk */ + byte = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_LZF; + if ((n = rdbWriteRaw(rdb,&byte,1)) == -1) goto writeerr; + nwritten += n; + + if ((n = rdbSaveLen(rdb,compress_len)) == -1) goto writeerr; + nwritten += n; + + if ((n = rdbSaveLen(rdb,original_len)) == -1) goto writeerr; + nwritten += n; + + if ((n = rdbWriteRaw(rdb,data,compress_len)) == -1) goto writeerr; + nwritten += n; + + return nwritten; + +writeerr: + return -1; +} + +int rdbSaveLzfStringObject(rio *rdb, unsigned char *s, size_t len) { + size_t comprlen, outlen; void *out; /* We require at least four bytes compression for this to be worth it */ @@ -224,26 +247,9 @@ int rdbSaveLzfStringObject(rio *rdb, unsigned char *s, size_t len) { zfree(out); return 0; } - /* Data compressed! Let's save it on disk */ - byte = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_LZF; - if ((n = rdbWriteRaw(rdb,&byte,1)) == -1) goto writeerr; - nwritten += n; - - if ((n = rdbSaveLen(rdb,comprlen)) == -1) goto writeerr; - nwritten += n; - - if ((n = rdbSaveLen(rdb,len)) == -1) goto writeerr; - nwritten += n; - - if ((n = rdbWriteRaw(rdb,out,comprlen)) == -1) goto writeerr; - nwritten += n; - + size_t nwritten = rdbSaveLzfBlob(rdb, out, comprlen, len); zfree(out); return nwritten; - -writeerr: - zfree(out); - return -1; } robj *rdbLoadLzfStringObject(rio *rdb) { @@ -491,8 +497,15 @@ int rdbSaveObject(rio *rdb, robj *o) { nwritten += n; do { - if ((n = rdbSaveRawString(rdb,node->zl,node->sz)) == -1) return -1; - nwritten += n; + if (quicklistNodeIsCompressed(node)) { + void *data; + size_t compress_len = quicklistGetLzf(node, &data); + if ((n = rdbSaveLzfBlob(rdb,data,compress_len,node->sz)) == -1) return -1; + nwritten += n; + } else { + if ((n = rdbSaveRawString(rdb,node->zl,node->sz)) == -1) return -1; + nwritten += n; + } } while ((node = node->next)); } else { redisPanic("Unknown list encoding"); @@ -822,14 +835,15 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { if ((len = rdbLoadLen(rdb,NULL)) == REDIS_RDB_LENERR) return NULL; o = createQuicklistObject(); + quicklistSetFill(o->ptr, server.list_max_ziplist_entries); + quicklistSetCompress(o->ptr, 0 /*FIXME*/); /* Load every single element of the list */ while(len--) { if ((ele = rdbLoadEncodedStringObject(rdb)) == NULL) return NULL; dec = getDecodedObject(ele); size_t len = sdslen(dec->ptr); - size_t zlen = server.list_max_ziplist_entries; - o->ptr = quicklistPushTail(o->ptr, zlen, dec->ptr, len); + o->ptr = quicklistPushTail(o->ptr, dec->ptr, len); decrRefCount(dec); decrRefCount(ele); } diff --git a/src/t_list.c b/src/t_list.c index b40e0089a..61fdf2ad8 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -43,10 +43,7 @@ void listTypePush(robj *subject, robj *value, int where) { int pos = (where == REDIS_HEAD) ? QUICKLIST_HEAD : QUICKLIST_TAIL; value = getDecodedObject(value); size_t len = sdslen(value->ptr); - size_t zlen = server.list_max_ziplist_entries; - /* If this value is greater than our allowed values, create it in - * an isolated ziplist */ - quicklistPush(subject->ptr, zlen, value->ptr, len, pos); + quicklistPush(subject->ptr, value->ptr, len, pos); decrRefCount(value); } else { redisPanic("Unknown list encoding"); @@ -146,12 +143,11 @@ void listTypeInsert(listTypeEntry *entry, robj *value, int where) { value = getDecodedObject(value); sds str = value->ptr; size_t len = sdslen(str); - size_t zlen = server.list_max_ziplist_entries; if (where == REDIS_TAIL) { - quicklistInsertAfter((quicklist *)entry->entry.quicklist, zlen, + quicklistInsertAfter((quicklist *)entry->entry.quicklist, &entry->entry, str, len); } else if (where == REDIS_HEAD) { - quicklistInsertBefore((quicklist *)entry->entry.quicklist, zlen, + quicklistInsertBefore((quicklist *)entry->entry.quicklist, &entry->entry, str, len); } decrRefCount(value); @@ -188,7 +184,7 @@ void listTypeConvert(robj *subject, int enc) { size_t zlen = server.list_max_ziplist_entries; subject->encoding = REDIS_ENCODING_QUICKLIST; - subject->ptr = quicklistCreateFromZiplist(zlen, subject->ptr); + subject->ptr = quicklistCreateFromZiplist(zlen, 0 /*FIXME*/, subject->ptr); } else { redisPanic("Unsupported list conversion"); } @@ -211,6 +207,8 @@ void pushGenericCommand(redisClient *c, int where) { c->argv[j] = tryObjectEncoding(c->argv[j]); if (!lobj) { lobj = createQuicklistObject(); + quicklistSetFill(lobj->ptr, server.list_max_ziplist_entries); + quicklistSetCompress(lobj->ptr, 0 /*FIXME*/); dbAdd(c->db,c->argv[1],lobj); } listTypePush(lobj,c->argv[j],where); @@ -539,6 +537,8 @@ void rpoplpushHandlePush(redisClient *c, robj *dstkey, robj *dstobj, robj *value /* Create the list if the key does not exist */ if (!dstobj) { dstobj = createQuicklistObject(); + quicklistSetFill(dstobj->ptr, server.list_max_ziplist_entries); + quicklistSetCompress(dstobj->ptr, 0 /*FIXME*/); dbAdd(c->db,dstkey,dstobj); } signalModifiedKey(c->db,dstkey); From 5f506b6d2b4b5a3a06d8a5ee44f0ecc8b33a457f Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Mon, 29 Dec 2014 23:37:43 -0500 Subject: [PATCH 24/29] Cleanup quicklist style Small fixes due to a new version of clang-format (it's less crazy than the older version). --- src/quicklist.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/quicklist.c b/src/quicklist.c index 92ce50649..4d79f2d1a 100644 --- a/src/quicklist.c +++ b/src/quicklist.c @@ -40,7 +40,7 @@ #endif /* Optimization levels for size-based filling */ -static const size_t optimization_level[] = { 4096, 8192, 16384, 32768, 65536 }; +static const size_t optimization_level[] = {4096, 8192, 16384, 32768, 65536}; /* Maximum size in bytes of any multi-element ziplist. * Larger values will live in their own isolated ziplists. */ @@ -527,7 +527,7 @@ quicklist *quicklistAppendValuesFromZiplist(quicklist *quicklist, unsigned char *value; unsigned int sz; long long longval; - char longstr[32] = { 0 }; + char longstr[32] = {0}; unsigned char *p = ziplistIndex(zl, 0); while (ziplistGet(p, &value, &sz, &longval)) { @@ -1267,7 +1267,7 @@ void quicklistRotate(quicklist *quicklist) { unsigned char *value; long long longval; unsigned int sz; - char longstr[32] = { 0 }; + char longstr[32] = {0}; ziplistGet(p, &value, &sz, &longval); /* If value found is NULL, then ziplistGet populated longval instead */ @@ -1582,7 +1582,7 @@ static int _ql_verify(quicklist *ql, uint32_t len, uint32_t count, /* Generate new string concatenating integer i against string 'prefix' */ static char *genstr(char *prefix, int i) { - static char result[64] = { 0 }; + static char result[64] = {0}; snprintf(result, sizeof(result), "%s%d", prefix, i); return result; } @@ -1598,7 +1598,7 @@ int quicklistTest(int argc, char *argv[]) { printf("Starting optimization offset at: %d\n", optimize_start); - int options[] = { 0, 1, 2, 3, 4, 5, 6, 10 }; + int options[] = {0, 1, 2, 3, 4, 5, 6, 10}; size_t option_count = sizeof(options) / sizeof(*options); long long runtime[option_count]; @@ -2247,12 +2247,12 @@ int quicklistTest(int argc, char *argv[]) { for (int f = optimize_start; f < 16; f++) { TEST_DESC("lrem test at fill %d at compress %d", f, options[_i]) { quicklist *ql = quicklistNew(f, options[_i]); - char *words[] = { "abc", "foo", "bar", "foobar", "foobared", - "zap", "bar", "test", "foo" }; - char *result[] = { "abc", "foo", "foobar", "foobared", - "zap", "test", "foo" }; - char *resultB[] = { "abc", "foo", "foobar", - "foobared", "zap", "test" }; + char *words[] = {"abc", "foo", "bar", "foobar", "foobared", + "zap", "bar", "test", "foo"}; + char *result[] = {"abc", "foo", "foobar", "foobared", + "zap", "test", "foo"}; + char *resultB[] = {"abc", "foo", "foobar", + "foobared", "zap", "test"}; for (int i = 0; i < 9; i++) quicklistPushTail(ql, words[i], strlen(words[i])); @@ -2355,7 +2355,7 @@ int quicklistTest(int argc, char *argv[]) { /* Check results after deletion of "hij" */ iter = quicklistGetIterator(ql, AL_START_HEAD); i = 0; - char *vals[] = { "abc", "def", "jkl", "oop" }; + char *vals[] = {"abc", "def", "jkl", "oop"}; while (quicklistNext(iter, &entry)) { if (!quicklistCompare(entry.zi, (unsigned char *)vals[i], 3)) { @@ -2554,7 +2554,7 @@ int quicklistTest(int argc, char *argv[]) { } /* Run a longer test of compression depth outside of primary test loop. */ - int list_sizes[] = { 250, 251, 500, 999, 1000 }; + int list_sizes[] = {250, 251, 500, 999, 1000}; long long start = mstime(); for (int list = 0; list < (int)(sizeof(list_sizes) / sizeof(*list_sizes)); list++) { From bbbbfb14422ee84e4b79330f299ddacf9be23d88 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Tue, 16 Dec 2014 00:11:21 -0500 Subject: [PATCH 25/29] Add branch prediction hints to quicklist Actually makes a noticeable difference. Branch hints were selected based on profiler hotspots. --- src/quicklist.c | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/quicklist.c b/src/quicklist.c index 4d79f2d1a..8e11de988 100644 --- a/src/quicklist.c +++ b/src/quicklist.c @@ -77,6 +77,14 @@ static const size_t optimization_level[] = {4096, 8192, 16384, 32768, 65536}; (e)->sz = 0; \ } while (0) +#if __GNUC__ >= 3 +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) +#else +#define likely(x) (x) +#define unlikely(x) (x) +#endif + /* Create a new quicklist. * Free with quicklistRelease(). */ quicklist *quicklistCreate(void) { @@ -405,7 +413,7 @@ static int _quicklistNodeSizeMeetsOptimizationRequirement(const size_t sz, static int _quicklistNodeAllowInsert(const quicklistNode *node, const int fill, const size_t sz) { - if (!node) + if (unlikely(!node)) return 0; int ziplist_overhead; @@ -418,14 +426,14 @@ static int _quicklistNodeAllowInsert(const quicklistNode *node, const int fill, /* size of forward offset */ if (sz < 64) ziplist_overhead += 1; - else if (sz < 16384) + else if (likely(sz < 16384)) ziplist_overhead += 2; else ziplist_overhead += 5; /* new_sz overestimates if 'sz' encodes to an integer type */ unsigned int new_sz = node->sz + sz + ziplist_overhead; - if (_quicklistNodeSizeMeetsOptimizationRequirement(new_sz, fill)) + if (likely(_quicklistNodeSizeMeetsOptimizationRequirement(new_sz, fill))) return 1; else if (!sizeMeetsSafetyLimit(new_sz)) return 0; @@ -443,7 +451,7 @@ static int _quicklistNodeAllowMerge(const quicklistNode *a, /* approximate merged ziplist size (- 11 to remove one ziplist * header/trailer) */ unsigned int merge_sz = a->sz + b->sz - 11; - if (_quicklistNodeSizeMeetsOptimizationRequirement(merge_sz, fill)) + if (likely(_quicklistNodeSizeMeetsOptimizationRequirement(merge_sz, fill))) return 1; else if (!sizeMeetsSafetyLimit(merge_sz)) return 0; @@ -464,7 +472,8 @@ static int _quicklistNodeAllowMerge(const quicklistNode *a, * Returns 1 if new head created. */ int quicklistPushHead(quicklist *quicklist, void *value, size_t sz) { quicklistNode *orig_head = quicklist->head; - if (_quicklistNodeAllowInsert(quicklist->head, quicklist->fill, sz)) { + if (likely( + _quicklistNodeAllowInsert(quicklist->head, quicklist->fill, sz))) { quicklist->head->zl = ziplistPush(quicklist->head->zl, value, sz, ZIPLIST_HEAD); quicklistNodeUpdateSz(quicklist->head); @@ -486,7 +495,8 @@ int quicklistPushHead(quicklist *quicklist, void *value, size_t sz) { * Returns 1 if new tail created. */ int quicklistPushTail(quicklist *quicklist, void *value, size_t sz) { quicklistNode *orig_tail = quicklist->tail; - if (_quicklistNodeAllowInsert(quicklist->tail, quicklist->fill, sz)) { + if (likely( + _quicklistNodeAllowInsert(quicklist->tail, quicklist->fill, sz))) { quicklist->tail->zl = ziplistPush(quicklist->tail->zl, value, sz, ZIPLIST_TAIL); quicklistNodeUpdateSz(quicklist->tail); @@ -649,8 +659,8 @@ void quicklistDelEntry(quicklistIter *iter, quicklistEntry *entry) { int quicklistReplaceAtIndex(quicklist *quicklist, long index, void *data, int sz) { quicklistEntry entry; - if (quicklistIndex(quicklist, index, &entry)) { - // quicklistDecompressNode(entry.node); + if (likely(quicklistIndex(quicklist, index, &entry))) { + /* quicklistIndex provides an uncompressed node */ entry.node->zl = ziplistDelete(entry.node->zl, &entry.zi); entry.node->zl = ziplistInsert(entry.node->zl, entry.zi, data, sz); quicklistCompress(quicklist, entry.node); @@ -1223,7 +1233,7 @@ int quicklistIndex(const quicklist *quicklist, const long long idx, if (index >= quicklist->count) return 0; - while (n) { + while (likely(n)) { if ((accum + n->count) > index) { break; } else { @@ -1253,7 +1263,8 @@ int quicklistIndex(const quicklist *quicklist, const long long idx, quicklistDecompressNodeForUse(entry->node); entry->zi = ziplistIndex(entry->node->zl, entry->offset); ziplistGet(entry->zi, &entry->value, &entry->sz, &entry->longval); - // quicklistCompress(quicklist, entry->node); + /* The caller will use our result, so we don't re-compress here. + * The caller can recompress or delete the node as needed. */ return 1; } From 02bb515a094c081fcbc3e33c60a5dbff440eb447 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Tue, 16 Dec 2014 00:49:14 -0500 Subject: [PATCH 26/29] Config: Add quicklist, remove old list options This removes: - list-max-ziplist-entries - list-max-ziplist-value This adds: - list-max-ziplist-size - list-compress-depth Also updates config file with new sections and updates tests to use quicklist settings instead of old list settings. --- redis.conf | 35 ++++++++++++++++++++++++++++++----- src/config.c | 28 ++++++++++++++++------------ src/rdb.c | 8 +++++--- src/redis.c | 4 ++-- src/redis.h | 12 ++++++++---- src/t_list.c | 14 +++++++------- tests/unit/sort.tcl | 3 +-- tests/unit/type/list-2.tcl | 3 +-- tests/unit/type/list-3.tcl | 3 +-- tests/unit/type/list.tcl | 3 +-- 10 files changed, 72 insertions(+), 41 deletions(-) diff --git a/redis.conf b/redis.conf index 7bb94fbe9..781e726bc 100644 --- a/redis.conf +++ b/redis.conf @@ -818,11 +818,36 @@ notify-keyspace-events "" hash-max-ziplist-entries 512 hash-max-ziplist-value 64 -# Similarly to hashes, small lists are also encoded in a special way in order -# to save a lot of space. The special representation is only used when -# you are under the following limits: -list-max-ziplist-entries 512 -list-max-ziplist-value 64 +# Lists are also encoded in a special way to save a lot of space. +# The number of entries allowed per internal list node can be specified +# as a fixed maximum size or a maximum number of elements. +# For a fixed maximum size, use -5 through -1, meaning: +# -5: max size: 64 Kb <-- not recommended for normal workloads +# -4: max size: 32 Kb <-- not recommended +# -3: max size: 16 Kb <-- probably not recommended +# -2: max size: 8 Kb <-- good +# -1: max size: 4 Kb <-- good +# Positive numbers mean store up to _exactly_ that number of elements +# per list node. +# The highest performing option is usually -2 (8 Kb size) or -1 (4 Kb size), +# but if your use case is unique, adjust the settings as necessary. +list-max-ziplist-size -2 + +# Lists may also be compressed. +# Compress depth is the number of quicklist ziplist nodes from *each* side of +# the list to *exclude* from compression. The head and tail of the list +# are always uncompressed for fast push/pop operations. Settings are: +# 0: disable all list compression +# 1: depth 1 means "don't start compressing until after 1 node into the list, +# going from either the head or tail" +# So: [head]->node->node->...->node->[tail] +# [head], [tail] will always be uncompressed; inner nodes will compress. +# 2: [head]->[next]->node->node->...->node->[prev]->[tail] +# 2 here means: don't compress head or head->next or tail->prev or tail, +# but compress all nodes between them. +# 3: [head]->[next]->[next]->node->node->...->node->[prev]->[prev]->[tail] +# etc. +list-compress-depth 0 # Sets have a special encoding in just one case: when a set is composed # of just strings that happen to be integers in radix 10 in the range diff --git a/src/config.c b/src/config.c index 2140b769f..b0245954c 100644 --- a/src/config.c +++ b/src/config.c @@ -397,9 +397,13 @@ void loadServerConfigFromString(char *config) { } else if (!strcasecmp(argv[0],"hash-max-ziplist-value") && argc == 2) { server.hash_max_ziplist_value = memtoll(argv[1], NULL); } else if (!strcasecmp(argv[0],"list-max-ziplist-entries") && argc == 2){ - server.list_max_ziplist_entries = memtoll(argv[1], NULL); + /* DEAD OPTION */ } else if (!strcasecmp(argv[0],"list-max-ziplist-value") && argc == 2) { - server.list_max_ziplist_value = memtoll(argv[1], NULL); + /* DEAD OPTION */ + } else if (!strcasecmp(argv[0],"list-max-ziplist-size") && argc == 2) { + server.list_max_ziplist_size = atoi(argv[1]); + } else if (!strcasecmp(argv[0],"list-compress-depth") && argc == 2) { + server.list_compress_depth = atoi(argv[1]); } else if (!strcasecmp(argv[0],"set-max-intset-entries") && argc == 2) { server.set_max_intset_entries = memtoll(argv[1], NULL); } else if (!strcasecmp(argv[0],"zset-max-ziplist-entries") && argc == 2) { @@ -795,12 +799,12 @@ void configSetCommand(redisClient *c) { } else if (!strcasecmp(c->argv[2]->ptr,"hash-max-ziplist-value")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; server.hash_max_ziplist_value = ll; - } else if (!strcasecmp(c->argv[2]->ptr,"list-max-ziplist-entries")) { + } else if (!strcasecmp(c->argv[2]->ptr,"list-max-ziplist-size")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; - server.list_max_ziplist_entries = ll; - } else if (!strcasecmp(c->argv[2]->ptr,"list-max-ziplist-value")) { + server.list_max_ziplist_size = ll; + } else if (!strcasecmp(c->argv[2]->ptr,"list-compress-depth")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; - server.list_max_ziplist_value = ll; + server.list_compress_depth = ll; } else if (!strcasecmp(c->argv[2]->ptr,"set-max-intset-entries")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; server.set_max_intset_entries = ll; @@ -1047,10 +1051,10 @@ void configGetCommand(redisClient *c) { server.hash_max_ziplist_entries); config_get_numerical_field("hash-max-ziplist-value", server.hash_max_ziplist_value); - config_get_numerical_field("list-max-ziplist-entries", - server.list_max_ziplist_entries); - config_get_numerical_field("list-max-ziplist-value", - server.list_max_ziplist_value); + config_get_numerical_field("list-max-ziplist-size", + server.list_max_ziplist_size); + config_get_numerical_field("list-compress-depth", + server.list_compress_depth); config_get_numerical_field("set-max-intset-entries", server.set_max_intset_entries); config_get_numerical_field("zset-max-ziplist-entries", @@ -1857,8 +1861,8 @@ int rewriteConfig(char *path) { rewriteConfigNotifykeyspaceeventsOption(state); rewriteConfigNumericalOption(state,"hash-max-ziplist-entries",server.hash_max_ziplist_entries,REDIS_HASH_MAX_ZIPLIST_ENTRIES); rewriteConfigNumericalOption(state,"hash-max-ziplist-value",server.hash_max_ziplist_value,REDIS_HASH_MAX_ZIPLIST_VALUE); - rewriteConfigNumericalOption(state,"list-max-ziplist-entries",server.list_max_ziplist_entries,REDIS_LIST_MAX_ZIPLIST_ENTRIES); - rewriteConfigNumericalOption(state,"list-max-ziplist-value",server.list_max_ziplist_value,REDIS_LIST_MAX_ZIPLIST_VALUE); + rewriteConfigNumericalOption(state,"list-max-ziplist-size",server.list_max_ziplist_size,REDIS_LIST_MAX_ZIPLIST_SIZE); + rewriteConfigNumericalOption(state,"list-compress-depth",server.list_compress_depth,REDIS_LIST_COMPRESS_DEPTH); rewriteConfigNumericalOption(state,"set-max-intset-entries",server.set_max_intset_entries,REDIS_SET_MAX_INTSET_ENTRIES); rewriteConfigNumericalOption(state,"zset-max-ziplist-entries",server.zset_max_ziplist_entries,REDIS_ZSET_MAX_ZIPLIST_ENTRIES); rewriteConfigNumericalOption(state,"zset-max-ziplist-value",server.zset_max_ziplist_value,REDIS_ZSET_MAX_ZIPLIST_VALUE); diff --git a/src/rdb.c b/src/rdb.c index 209a0da11..53b47e4f0 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -835,15 +835,15 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { if ((len = rdbLoadLen(rdb,NULL)) == REDIS_RDB_LENERR) return NULL; o = createQuicklistObject(); - quicklistSetFill(o->ptr, server.list_max_ziplist_entries); - quicklistSetCompress(o->ptr, 0 /*FIXME*/); + quicklistSetOptions(o->ptr, server.list_max_ziplist_size, + server.list_compress_depth); /* Load every single element of the list */ while(len--) { if ((ele = rdbLoadEncodedStringObject(rdb)) == NULL) return NULL; dec = getDecodedObject(ele); size_t len = sdslen(dec->ptr); - o->ptr = quicklistPushTail(o->ptr, dec->ptr, len); + quicklistPushTail(o->ptr, dec->ptr, len); decrRefCount(dec); decrRefCount(ele); } @@ -985,6 +985,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { } else if (rdbtype == REDIS_RDB_TYPE_LIST_QUICKLIST) { if ((len = rdbLoadLen(rdb,NULL)) == REDIS_RDB_LENERR) return NULL; o = createQuicklistObject(); + quicklistSetOptions(o->ptr, server.list_max_ziplist_size, + server.list_compress_depth); while (len--) { if ((ele = rdbLoadStringObject(rdb)) == NULL) return NULL; diff --git a/src/redis.c b/src/redis.c index ee88eddb5..a4d9e562c 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1449,8 +1449,8 @@ void initServerConfig(void) { server.maxmemory_samples = REDIS_DEFAULT_MAXMEMORY_SAMPLES; server.hash_max_ziplist_entries = REDIS_HASH_MAX_ZIPLIST_ENTRIES; server.hash_max_ziplist_value = REDIS_HASH_MAX_ZIPLIST_VALUE; - server.list_max_ziplist_entries = REDIS_LIST_MAX_ZIPLIST_ENTRIES; - server.list_max_ziplist_value = REDIS_LIST_MAX_ZIPLIST_VALUE; + server.list_max_ziplist_size = REDIS_LIST_MAX_ZIPLIST_SIZE; + server.list_compress_depth = REDIS_LIST_COMPRESS_DEPTH; server.set_max_intset_entries = REDIS_SET_MAX_INTSET_ENTRIES; server.zset_max_ziplist_entries = REDIS_ZSET_MAX_ZIPLIST_ENTRIES; server.zset_max_ziplist_value = REDIS_ZSET_MAX_ZIPLIST_VALUE; diff --git a/src/redis.h b/src/redis.h index 38c072478..ab02275fb 100644 --- a/src/redis.h +++ b/src/redis.h @@ -331,12 +331,14 @@ typedef long long mstime_t; /* millisecond time type. */ /* Zip structure related defaults */ #define REDIS_HASH_MAX_ZIPLIST_ENTRIES 512 #define REDIS_HASH_MAX_ZIPLIST_VALUE 64 -#define REDIS_LIST_MAX_ZIPLIST_ENTRIES 512 -#define REDIS_LIST_MAX_ZIPLIST_VALUE 64 #define REDIS_SET_MAX_INTSET_ENTRIES 512 #define REDIS_ZSET_MAX_ZIPLIST_ENTRIES 128 #define REDIS_ZSET_MAX_ZIPLIST_VALUE 64 +/* List defaults */ +#define REDIS_LIST_MAX_ZIPLIST_SIZE -2 +#define REDIS_LIST_COMPRESS_DEPTH 0 + /* HyperLogLog defines */ #define REDIS_DEFAULT_HLL_SPARSE_MAX_BYTES 3000 @@ -871,12 +873,14 @@ struct redisServer { /* Zip structure config, see redis.conf for more information */ size_t hash_max_ziplist_entries; size_t hash_max_ziplist_value; - size_t list_max_ziplist_entries; - size_t list_max_ziplist_value; size_t set_max_intset_entries; size_t zset_max_ziplist_entries; size_t zset_max_ziplist_value; size_t hll_sparse_max_bytes; + /* List parameters */ + int list_max_ziplist_size; + int list_compress_depth; + /* time cache */ time_t unixtime; /* Unix time sampled every cron cycle. */ long long mstime; /* Like 'unixtime' but with milliseconds resolution. */ /* Pubsub */ diff --git a/src/t_list.c b/src/t_list.c index 61fdf2ad8..232cb5c52 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -181,10 +181,10 @@ void listTypeConvert(robj *subject, int enc) { redisAssertWithInfo(NULL,subject,subject->encoding==REDIS_ENCODING_ZIPLIST); if (enc == REDIS_ENCODING_QUICKLIST) { - size_t zlen = server.list_max_ziplist_entries; - + size_t zlen = server.list_max_ziplist_size; + int depth = server.list_compress_depth; + subject->ptr = quicklistCreateFromZiplist(zlen, depth, subject->ptr); subject->encoding = REDIS_ENCODING_QUICKLIST; - subject->ptr = quicklistCreateFromZiplist(zlen, 0 /*FIXME*/, subject->ptr); } else { redisPanic("Unsupported list conversion"); } @@ -207,8 +207,8 @@ void pushGenericCommand(redisClient *c, int where) { c->argv[j] = tryObjectEncoding(c->argv[j]); if (!lobj) { lobj = createQuicklistObject(); - quicklistSetFill(lobj->ptr, server.list_max_ziplist_entries); - quicklistSetCompress(lobj->ptr, 0 /*FIXME*/); + quicklistSetOptions(lobj->ptr, server.list_max_ziplist_size, + server.list_compress_depth); dbAdd(c->db,c->argv[1],lobj); } listTypePush(lobj,c->argv[j],where); @@ -537,8 +537,8 @@ void rpoplpushHandlePush(redisClient *c, robj *dstkey, robj *dstobj, robj *value /* Create the list if the key does not exist */ if (!dstobj) { dstobj = createQuicklistObject(); - quicklistSetFill(dstobj->ptr, server.list_max_ziplist_entries); - quicklistSetCompress(dstobj->ptr, 0 /*FIXME*/); + quicklistSetOptions(dstobj->ptr, server.list_max_ziplist_size, + server.list_compress_depth); dbAdd(c->db,dstkey,dstobj); } signalModifiedKey(c->db,dstkey); diff --git a/tests/unit/sort.tcl b/tests/unit/sort.tcl index 5ae48d450..083c4540d 100644 --- a/tests/unit/sort.tcl +++ b/tests/unit/sort.tcl @@ -1,8 +1,7 @@ start_server { tags {"sort"} overrides { - "list-max-ziplist-value" 16 - "list-max-ziplist-entries" 32 + "list-max-ziplist-size" 32 "set-max-intset-entries" 32 } } { diff --git a/tests/unit/type/list-2.tcl b/tests/unit/type/list-2.tcl index d25873912..4c7d6d91c 100644 --- a/tests/unit/type/list-2.tcl +++ b/tests/unit/type/list-2.tcl @@ -1,8 +1,7 @@ start_server { tags {"list"} overrides { - "list-max-ziplist-value" 16 - "list-max-ziplist-entries" 4 + "list-max-ziplist-size" 4 } } { source "tests/unit/type/list-common.tcl" diff --git a/tests/unit/type/list-3.tcl b/tests/unit/type/list-3.tcl index 81ef9b3e5..ece6ea2d5 100644 --- a/tests/unit/type/list-3.tcl +++ b/tests/unit/type/list-3.tcl @@ -1,8 +1,7 @@ start_server { tags {list ziplist} overrides { - "list-max-ziplist-value" 200000 - "list-max-ziplist-entries" 16 + "list-max-ziplist-size" 16 } } { test {Explicit regression for a list bug} { diff --git a/tests/unit/type/list.tcl b/tests/unit/type/list.tcl index 0c0c07761..e4d568cf1 100644 --- a/tests/unit/type/list.tcl +++ b/tests/unit/type/list.tcl @@ -1,8 +1,7 @@ start_server { tags {"list"} overrides { - "list-max-ziplist-value" 16 - "list-max-ziplist-entries" 5 + "list-max-ziplist-size" 5 } } { source "tests/unit/type/list-common.tcl" From 9e11d07909086999d54181466eafecab44a0dbe3 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Fri, 19 Dec 2014 10:41:52 -0500 Subject: [PATCH 27/29] Add more quicklist info to DEBUG OBJECT Adds: ql_compressed (boolean, 1 if compression enabled for list, 0 otherwise) Adds: ql_uncompressed_size (actual uncompressed size of all quicklistNodes) Adds: ql_ziplist_max (quicklist max ziplist fill factor) Compression ratio of the list is then ql_uncompressed_size / serializedlength We report ql_uncompressed_size for all quicklists because serializedlength is a _compressed_ representation anyway. Sample output from a large list: 127.0.0.1:6379> llen abc (integer) 38370061 127.0.0.1:6379> debug object abc Value at:0x7ff97b51d140 refcount:1 encoding:quicklist serializedlength:19878335 lru:9718164 lru_seconds_idle:5 ql_nodes:21945 ql_avg_node:1748.46 ql_ziplist_max:-2 ql_compressed:0 ql_uncompressed_size:1643187761 (1.36s) The 1.36s result time is because rdbSavedObjectLen() is serializing the object, not because of any new stats reporting. If we run DEBUG OBJECT on a compressed list, DEBUG OBJECT takes almost *zero* time because rdbSavedObjectLen() reuses already-compressed ziplists: 127.0.0.1:6379> debug object abc Value at:0x7fe5c5800040 refcount:1 encoding:quicklist serializedlength:19878335 lru:9718109 lru_seconds_idle:5 ql_nodes:21945 ql_avg_node:1748.46 ql_ziplist_max:-2 ql_compressed:1 ql_uncompressed_size:1643187761 --- src/debug.c | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/debug.c b/src/debug.c index d11e3f037..7783196a0 100644 --- a/src/debug.c +++ b/src/debug.c @@ -306,11 +306,32 @@ void debugCommand(redisClient *c) { char *nextra = extra; int remaining = sizeof(extra); quicklist *ql = val->ptr; - double avg = (double)ql->count/ql->len; + /* Add number of quicklist nodes */ int used = snprintf(nextra, remaining, " ql_nodes:%u", ql->len); nextra += used; remaining -= used; - snprintf(nextra, remaining, " ql_avg_node:%.2f", avg); + /* Add average quicklist fill factor */ + double avg = (double)ql->count/ql->len; + used = snprintf(nextra, remaining, " ql_avg_node:%.2f", avg); + nextra += used; + remaining -= used; + /* Add quicklist fill level / max ziplist size */ + used = snprintf(nextra, remaining, " ql_ziplist_max:%d", ql->fill); + nextra += used; + remaining -= used; + /* Add isCompressed? */ + int compressed = ql->compress != 0; + used = snprintf(nextra, remaining, " ql_compressed:%d", compressed); + nextra += used; + remaining -= used; + /* Add total uncompressed size */ + unsigned long sz = 0; + for (quicklistNode *node = ql->head; node; node = node->next) { + sz += node->sz; + } + used = snprintf(nextra, remaining, " ql_uncompressed_size:%lu", sz); + nextra += used; + remaining -= used; } addReplyStatusFormat(c, From 25e12d10bebfa38ce864956c0498b432c17eccdd Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Fri, 19 Dec 2014 21:26:04 -0500 Subject: [PATCH 28/29] Set optional 'static' for Quicklist+Redis This also defines REDIS_STATIC='' for building everything inside src/ and everything inside deps/lua/. --- deps/Makefile | 2 +- src/Makefile | 2 +- src/quicklist.c | 71 ++++++++++++++++++++++++++++--------------------- 3 files changed, 42 insertions(+), 33 deletions(-) diff --git a/deps/Makefile b/deps/Makefile index 1f623ea7b..71f6d3a2c 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -58,7 +58,7 @@ ifeq ($(uname_S),SunOS) LUA_CFLAGS= -D__C99FEATURES__=1 endif -LUA_CFLAGS+= -O2 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL $(CFLAGS) +LUA_CFLAGS+= -O2 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC='' $(CFLAGS) LUA_LDFLAGS+= $(LDFLAGS) # lua's Makefile defines AR="ar rcu", which is unusual, and makes it more # challenging to cross-compile lua (and redis). These defines make it easier diff --git a/src/Makefile b/src/Makefile index 16172b25f..3abef4e48 100644 --- a/src/Makefile +++ b/src/Makefile @@ -18,7 +18,7 @@ OPTIMIZATION?=-O2 DEPENDENCY_TARGETS=hiredis linenoise lua # Default settings -STD=-std=c99 -pedantic +STD=-std=c99 -pedantic -DREDIS_STATIC='' WARN=-Wall -W OPT=$(OPTIMIZATION) diff --git a/src/quicklist.c b/src/quicklist.c index 8e11de988..6682b2087 100644 --- a/src/quicklist.c +++ b/src/quicklist.c @@ -39,6 +39,10 @@ #include /* for printf (debug printing), snprintf (genstr) */ #endif +#ifndef REDIS_STATIC +#define REDIS_STATIC static +#endif + /* Optimization levels for size-based filling */ static const size_t optimization_level[] = {4096, 8192, 16384, 32768, 65536}; @@ -131,7 +135,7 @@ quicklist *quicklistNew(int fill, int compress) { return quicklist; } -static quicklistNode *quicklistCreateNode(void) { +REDIS_STATIC quicklistNode *quicklistCreateNode(void) { quicklistNode *node; node = zmalloc(sizeof(*node)); node->zl = NULL; @@ -171,7 +175,7 @@ void quicklistRelease(quicklist *quicklist) { /* Compress the ziplist in 'node' and update encoding details. * Returns 1 if ziplist compressed successfully. * Returns 0 if compression failed or if ziplist too small to compress. */ -static int __quicklistCompressNode(quicklistNode *node) { +REDIS_STATIC int __quicklistCompressNode(quicklistNode *node) { #ifdef REDIS_TEST node->attempted_compress = 1; #endif @@ -208,7 +212,7 @@ static int __quicklistCompressNode(quicklistNode *node) { /* Uncompress the ziplist in 'node' and update encoding details. * Returns 1 on successful decode, 0 on failure to decode. */ -static int __quicklistDecompressNode(quicklistNode *node) { +REDIS_STATIC int __quicklistDecompressNode(quicklistNode *node) { #ifdef REDIS_TEST node->attempted_compress = 0; #endif @@ -258,8 +262,8 @@ size_t quicklistGetLzf(const quicklistNode *node, void **data) { * The only way to guarantee interior nodes get compressed is to iterate * to our "interior" compress depth then compress the next node we find. * If compress depth is larger than the entire list, we return immediately. */ -static void __quicklistCompress(const quicklist *quicklist, - quicklistNode *node) { +REDIS_STATIC void __quicklistCompress(const quicklist *quicklist, + quicklistNode *node) { /* If length is less than our compress depth (from both sides), * we can't compress anything. */ if (!quicklistAllowsCompression(quicklist) || @@ -345,8 +349,9 @@ static void __quicklistCompress(const quicklist *quicklist, * Insert 'new_node' before 'old_node' if 'after' is 0. * Note: 'new_node' is *always* uncompressed, so if we assign it to * head or tail, we do not need to uncompress it. */ -static void __quicklistInsertNode(quicklist *quicklist, quicklistNode *old_node, - quicklistNode *new_node, int after) { +REDIS_STATIC void __quicklistInsertNode(quicklist *quicklist, + quicklistNode *old_node, + quicklistNode *new_node, int after) { if (after) { new_node->prev = old_node; if (old_node) { @@ -380,20 +385,21 @@ static void __quicklistInsertNode(quicklist *quicklist, quicklistNode *old_node, } /* Wrappers for node inserting around existing node. */ -static void _quicklistInsertNodeBefore(quicklist *quicklist, - quicklistNode *old_node, - quicklistNode *new_node) { +REDIS_STATIC void _quicklistInsertNodeBefore(quicklist *quicklist, + quicklistNode *old_node, + quicklistNode *new_node) { __quicklistInsertNode(quicklist, old_node, new_node, 0); } -static void _quicklistInsertNodeAfter(quicklist *quicklist, - quicklistNode *old_node, - quicklistNode *new_node) { +REDIS_STATIC void _quicklistInsertNodeAfter(quicklist *quicklist, + quicklistNode *old_node, + quicklistNode *new_node) { __quicklistInsertNode(quicklist, old_node, new_node, 1); } -static int _quicklistNodeSizeMeetsOptimizationRequirement(const size_t sz, - const int fill) { +REDIS_STATIC int +_quicklistNodeSizeMeetsOptimizationRequirement(const size_t sz, + const int fill) { if (fill >= 0) return 0; @@ -411,8 +417,8 @@ static int _quicklistNodeSizeMeetsOptimizationRequirement(const size_t sz, #define sizeMeetsSafetyLimit(sz) ((sz) <= SIZE_SAFETY_LIMIT) -static int _quicklistNodeAllowInsert(const quicklistNode *node, const int fill, - const size_t sz) { +REDIS_STATIC int _quicklistNodeAllowInsert(const quicklistNode *node, + const int fill, const size_t sz) { if (unlikely(!node)) return 0; @@ -443,8 +449,9 @@ static int _quicklistNodeAllowInsert(const quicklistNode *node, const int fill, return 0; } -static int _quicklistNodeAllowMerge(const quicklistNode *a, - const quicklistNode *b, const int fill) { +REDIS_STATIC int _quicklistNodeAllowMerge(const quicklistNode *a, + const quicklistNode *b, + const int fill) { if (!a || !b) return 0; @@ -569,7 +576,8 @@ quicklist *quicklistCreateFromZiplist(int fill, int compress, } \ } while (0) -static void __quicklistDelNode(quicklist *quicklist, quicklistNode *node) { +REDIS_STATIC void __quicklistDelNode(quicklist *quicklist, + quicklistNode *node) { if (node->next) node->next->prev = node->prev; if (node->prev) @@ -602,8 +610,8 @@ static void __quicklistDelNode(quicklist *quicklist, quicklistNode *node) { * * Returns 1 if the entire node was deleted, 0 if node still exists. * Also updates in/out param 'p' with the next offset in the ziplist. */ -static int quicklistDelIndex(quicklist *quicklist, quicklistNode *node, - unsigned char **p) { +REDIS_STATIC int quicklistDelIndex(quicklist *quicklist, quicklistNode *node, + unsigned char **p) { int gone = 0; node->zl = ziplistDelete(node->zl, p); @@ -683,9 +691,9 @@ int quicklistReplaceAtIndex(quicklist *quicklist, long index, void *data, * * Returns the input node picked to merge against or NULL if * merging was not possible. */ -static quicklistNode *_quicklistZiplistMerge(quicklist *quicklist, - quicklistNode *a, - quicklistNode *b) { +REDIS_STATIC quicklistNode *_quicklistZiplistMerge(quicklist *quicklist, + quicklistNode *a, + quicklistNode *b) { D("Requested merge (a,b) (%u, %u)", a->count, b->count); quicklistDecompressNode(a); @@ -721,7 +729,8 @@ static quicklistNode *_quicklistZiplistMerge(quicklist *quicklist, * - (center->prev, center) * - (center, center->next) */ -static void _quicklistMergeNodes(quicklist *quicklist, quicklistNode *center) { +REDIS_STATIC void _quicklistMergeNodes(quicklist *quicklist, + quicklistNode *center) { int fill = quicklist->fill; quicklistNode *prev, *prev_prev, *next, *next_next, *target; prev = prev_prev = next = next_next = target = NULL; @@ -784,8 +793,8 @@ static void _quicklistMergeNodes(quicklist *quicklist, quicklistNode *center) { * The input node keeps all elements not taken by the returned node. * * Returns newly created node or NULL if split not possible. */ -static quicklistNode *_quicklistSplitNode(quicklistNode *node, int offset, - int after) { +REDIS_STATIC quicklistNode *_quicklistSplitNode(quicklistNode *node, int offset, + int after) { size_t zl_sz = node->sz; quicklistNode *new_node = quicklistCreateNode(); @@ -819,8 +828,8 @@ static quicklistNode *_quicklistSplitNode(quicklistNode *node, int offset, * * If after==1, the new value is inserted after 'entry', otherwise * the new value is inserted before 'entry'. */ -static void _quicklistInsert(quicklist *quicklist, quicklistEntry *entry, - void *value, const size_t sz, int after) { +REDIS_STATIC void _quicklistInsert(quicklist *quicklist, quicklistEntry *entry, + void *value, const size_t sz, int after) { int full = 0, at_tail = 0, at_head = 0, full_next = 0, full_prev = 0; int fill = quicklist->fill; quicklistNode *node = entry->node; @@ -1359,7 +1368,7 @@ int quicklistPopCustom(quicklist *quicklist, int where, unsigned char **data, } /* Return a malloc'd copy of data passed in */ -static void *_quicklistSaver(unsigned char *data, unsigned int sz) { +REDIS_STATIC void *_quicklistSaver(unsigned char *data, unsigned int sz) { unsigned char *vstr; if (data) { vstr = zmalloc(sz); From 5870e22423e069452e9f858b80ac40bfc455bfe6 Mon Sep 17 00:00:00 2001 From: Matt Stancliff Date: Tue, 23 Dec 2014 10:10:42 -0500 Subject: [PATCH 29/29] Upgrade LZF to 3.6 (2011) from 3.5 (2009) This is lzf_c and lzf_d from http://dist.schmorp.de/liblzf/liblzf-3.6.tar.gz --- src/lzfP.h | 56 ++++++++++++++++++++++++++++++++++++++-------------- src/lzf_c.c | 42 ++++++++++++++++++--------------------- src/lzf_d.c | 57 ++++++++++++++++++++++++++++++++++++++++++----------- 3 files changed, 106 insertions(+), 49 deletions(-) diff --git a/src/lzfP.h b/src/lzfP.h index c9eae3f6a..c6d2e096c 100644 --- a/src/lzfP.h +++ b/src/lzfP.h @@ -49,7 +49,7 @@ * the difference between 15 and 14 is very small * for small blocks (and 14 is usually a bit faster). * For a low-memory/faster configuration, use HLOG == 13; - * For best compression, use 15 or 16 (or more, up to 23). + * For best compression, use 15 or 16 (or more, up to 22). */ #ifndef HLOG # define HLOG 16 @@ -94,7 +94,7 @@ /* * Avoid assigning values to errno variable? for some embedding purposes * (linux kernel for example), this is necessary. NOTE: this breaks - * the documentation in lzf.h. + * the documentation in lzf.h. Avoiding errno has no speed impact. */ #ifndef AVOID_ERRNO # define AVOID_ERRNO 0 @@ -121,16 +121,52 @@ # define CHECK_INPUT 1 #endif +/* + * Whether to store pointers or offsets inside the hash table. On + * 64 bit architetcures, pointers take up twice as much space, + * and might also be slower. Default is to autodetect. + */ +/*#define LZF_USER_OFFSETS autodetect */ + /*****************************************************************************/ /* nothing should be changed below */ +#ifdef __cplusplus +# include +# include +using namespace std; +#else +# include +# include +#endif + +#ifndef LZF_USE_OFFSETS +# if defined (WIN32) +# define LZF_USE_OFFSETS defined(_M_X64) +# else +# if __cplusplus > 199711L +# include +# else +# include +# endif +# define LZF_USE_OFFSETS (UINTPTR_MAX > 0xffffffffU) +# endif +#endif + typedef unsigned char u8; -typedef const u8 *LZF_STATE[1 << (HLOG)]; +#if LZF_USE_OFFSETS +# define LZF_HSLOT_BIAS ((const u8 *)in_data) + typedef unsigned int LZF_HSLOT; +#else +# define LZF_HSLOT_BIAS 0 + typedef const u8 *LZF_HSLOT; +#endif + +typedef LZF_HSLOT LZF_STATE[1 << (HLOG)]; #if !STRICT_ALIGN /* for unaligned accesses we need a 16 bit datatype. */ -# include # if USHRT_MAX == 65535 typedef unsigned short u16; # elif UINT_MAX == 65535 @@ -142,17 +178,7 @@ typedef const u8 *LZF_STATE[1 << (HLOG)]; #endif #if ULTRA_FAST -# if defined(VERY_FAST) -# undef VERY_FAST -# endif -#endif - -#if INIT_HTAB -# ifdef __cplusplus -# include -# else -# include -# endif +# undef VERY_FAST #endif #endif diff --git a/src/lzf_c.c b/src/lzf_c.c index 9e031ad0b..e9c69a0b8 100644 --- a/src/lzf_c.c +++ b/src/lzf_c.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2008 Marc Alexander Lehmann + * Copyright (c) 2000-2010 Marc Alexander Lehmann * * Redistribution and use in source and binary forms, with or without modifica- * tion, are permitted provided that the following conditions are met: @@ -40,8 +40,8 @@ /* * don't play with this unless you benchmark! - * decompression is not dependent on the hash function - * the hashing function might seem strange, just believe me + * the data format is not dependent on the hash function. + * the hash function might seem strange, just believe me, * it works ;) */ #ifndef FRST @@ -89,9 +89,9 @@ /* * compressed format * - * 000LLLLL ; literal - * LLLooooo oooooooo ; backref L - * 111ooooo LLLLLLLL oooooooo ; backref L+7 + * 000LLLLL ; literal, L+1=1..33 octets + * LLLooooo oooooooo ; backref L+1=1..7 octets, o+1=1..4096 offset + * 111ooooo LLLLLLLL oooooooo ; backref L+8 octets, o+1=1..4096 offset * */ @@ -106,7 +106,6 @@ lzf_compress (const void *const in_data, unsigned int in_len, #if !LZF_STATE_ARG LZF_STATE htab; #endif - const u8 **hslot; const u8 *ip = (const u8 *)in_data; u8 *op = (u8 *)out_data; const u8 *in_end = ip + in_len; @@ -133,10 +132,6 @@ lzf_compress (const void *const in_data, unsigned int in_len, #if INIT_HTAB memset (htab, 0, sizeof (htab)); -# if 0 - for (hslot = htab; hslot < htab + HSIZE; hslot++) - *hslot++ = ip; -# endif #endif lit = 0; op++; /* start run */ @@ -144,24 +139,23 @@ lzf_compress (const void *const in_data, unsigned int in_len, hval = FRST (ip); while (ip < in_end - 2) { + LZF_HSLOT *hslot; + hval = NEXT (hval, ip); hslot = htab + IDX (hval); - ref = *hslot; *hslot = ip; + ref = *hslot + LZF_HSLOT_BIAS; *hslot = ip - LZF_HSLOT_BIAS; if (1 #if INIT_HTAB && ref < ip /* the next test will actually take care of this, but this is faster */ #endif && (off = ip - ref - 1) < MAX_OFF - && ip + 4 < in_end && ref > (u8 *)in_data -#if STRICT_ALIGN - && ref[0] == ip[0] - && ref[1] == ip[1] && ref[2] == ip[2] +#if STRICT_ALIGN + && ((ref[1] << 8) | ref[0]) == ((ip[1] << 8) | ip[0]) #else && *(u16 *)ref == *(u16 *)ip - && ref[2] == ip[2] #endif ) { @@ -170,12 +164,13 @@ lzf_compress (const void *const in_data, unsigned int in_len, unsigned int maxlen = in_end - ip - len; maxlen = maxlen > MAX_REF ? MAX_REF : maxlen; + if (expect_false (op + 3 + 1 >= out_end)) /* first a faster conservative test */ + if (op - !lit + 3 + 1 >= out_end) /* second the exact but rare test */ + return 0; + op [- lit - 1] = lit - 1; /* stop run */ op -= !lit; /* undo run if length is zero */ - if (expect_false (op + 3 + 1 >= out_end)) - return 0; - for (;;) { if (expect_true (maxlen > 16)) @@ -222,6 +217,7 @@ lzf_compress (const void *const in_data, unsigned int in_len, } *op++ = off; + lit = 0; op++; /* start run */ ip += len + 1; @@ -237,12 +233,12 @@ lzf_compress (const void *const in_data, unsigned int in_len, hval = FRST (ip); hval = NEXT (hval, ip); - htab[IDX (hval)] = ip; + htab[IDX (hval)] = ip - LZF_HSLOT_BIAS; ip++; # if VERY_FAST && !ULTRA_FAST hval = NEXT (hval, ip); - htab[IDX (hval)] = ip; + htab[IDX (hval)] = ip - LZF_HSLOT_BIAS; ip++; # endif #else @@ -251,7 +247,7 @@ lzf_compress (const void *const in_data, unsigned int in_len, do { hval = NEXT (hval, ip); - htab[IDX (hval)] = ip; + htab[IDX (hval)] = ip - LZF_HSLOT_BIAS; ip++; } while (len--); diff --git a/src/lzf_d.c b/src/lzf_d.c index 6c723f5e0..c32be8e87 100644 --- a/src/lzf_d.c +++ b/src/lzf_d.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2007 Marc Alexander Lehmann + * Copyright (c) 2000-2010 Marc Alexander Lehmann * * Redistribution and use in source and binary forms, with or without modifica- * tion, are permitted provided that the following conditions are met: @@ -43,14 +43,14 @@ # define SET_ERRNO(n) errno = (n) #endif -/* +#if USE_REP_MOVSB /* small win on amd, big loss on intel */ #if (__i386 || __amd64) && __GNUC__ >= 3 # define lzf_movsb(dst, src, len) \ asm ("rep movsb" \ : "=D" (dst), "=S" (src), "=c" (len) \ : "0" (dst), "1" (src), "2" (len)); #endif -*/ +#endif unsigned int lzf_decompress (const void *const in_data, unsigned int in_len, @@ -86,9 +86,17 @@ lzf_decompress (const void *const in_data, unsigned int in_len, #ifdef lzf_movsb lzf_movsb (op, ip, ctrl); #else - do - *op++ = *ip++; - while (--ctrl); + switch (ctrl) + { + case 32: *op++ = *ip++; case 31: *op++ = *ip++; case 30: *op++ = *ip++; case 29: *op++ = *ip++; + case 28: *op++ = *ip++; case 27: *op++ = *ip++; case 26: *op++ = *ip++; case 25: *op++ = *ip++; + case 24: *op++ = *ip++; case 23: *op++ = *ip++; case 22: *op++ = *ip++; case 21: *op++ = *ip++; + case 20: *op++ = *ip++; case 19: *op++ = *ip++; case 18: *op++ = *ip++; case 17: *op++ = *ip++; + case 16: *op++ = *ip++; case 15: *op++ = *ip++; case 14: *op++ = *ip++; case 13: *op++ = *ip++; + case 12: *op++ = *ip++; case 11: *op++ = *ip++; case 10: *op++ = *ip++; case 9: *op++ = *ip++; + case 8: *op++ = *ip++; case 7: *op++ = *ip++; case 6: *op++ = *ip++; case 5: *op++ = *ip++; + case 4: *op++ = *ip++; case 3: *op++ = *ip++; case 2: *op++ = *ip++; case 1: *op++ = *ip++; + } #endif } else /* back reference */ @@ -134,12 +142,39 @@ lzf_decompress (const void *const in_data, unsigned int in_len, len += 2; lzf_movsb (op, ref, len); #else - *op++ = *ref++; - *op++ = *ref++; + switch (len) + { + default: + len += 2; - do - *op++ = *ref++; - while (--len); + if (op >= ref + len) + { + /* disjunct areas */ + memcpy (op, ref, len); + op += len; + } + else + { + /* overlapping, use octte by octte copying */ + do + *op++ = *ref++; + while (--len); + } + + break; + + case 9: *op++ = *ref++; + case 8: *op++ = *ref++; + case 7: *op++ = *ref++; + case 6: *op++ = *ref++; + case 5: *op++ = *ref++; + case 4: *op++ = *ref++; + case 3: *op++ = *ref++; + case 2: *op++ = *ref++; + case 1: *op++ = *ref++; + case 0: *op++ = *ref++; /* two octets more */ + *op++ = *ref++; + } #endif } }