gvsafronov/futriix
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #2143 from mattsta/quicklist

Browse Source 
Quicklist (linked list + ziplist)
This commit is contained in:
Salvatore Sanfilippo 2015-01-08 09:51:55 +01:00
commit 05ba119fbb
47 changed files with 3794 additions and 692 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
deps/Makefile vendored
View File

@ -58,7 +58,7 @@ ifeq ($(uname_S),SunOS)
LUA_CFLAGS= -D__C99FEATURES__=1
endif
LUA_CFLAGS+= -O2 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL $(CFLAGS)
LUA_CFLAGS+= -O2 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC='' $(CFLAGS)
LUA_LDFLAGS+= $(LDFLAGS)
# lua's Makefile defines AR="ar rcu", which is unusual, and makes it more
# challenging to cross-compile lua (and redis). These defines make it easier

35
redis.conf
View File

@ -818,11 +818,36 @@ notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
# Similarly to hashes, small lists are also encoded in a special way in order
# to save a lot of space. The special representation is only used when
# you are under the following limits:
list-max-ziplist-entries 512
list-max-ziplist-value 64
# Lists are also encoded in a special way to save a lot of space.
# The number of entries allowed per internal list node can be specified
# as a fixed maximum size or a maximum number of elements.
# For a fixed maximum size, use -5 through -1, meaning:
# -5: max size: 64 Kb <-- not recommended for normal workloads
# -4: max size: 32 Kb <-- not recommended
# -3: max size: 16 Kb <-- probably not recommended
# -2: max size: 8 Kb <-- good
# -1: max size: 4 Kb <-- good
# Positive numbers mean store up to _exactly_ that number of elements
# per list node.
# The highest performing option is usually -2 (8 Kb size) or -1 (4 Kb size),
# but if your use case is unique, adjust the settings as necessary.
list-max-ziplist-size -2
# Lists may also be compressed.
# Compress depth is the number of quicklist ziplist nodes from *each* side of
# the list to *exclude* from compression. The head and tail of the list
# are always uncompressed for fast push/pop operations. Settings are:
# 0: disable all list compression
# 1: depth 1 means "don't start compressing until after 1 node into the list,
# going from either the head or tail"
# So: [head]->node->node->...->node->[tail]
# [head], [tail] will always be uncompressed; inner nodes will compress.
# 2: [head]->[next]->node->node->...->node->[prev]->[tail]
# 2 here means: don't compress head or head->next or tail->prev or tail,
# but compress all nodes between them.
# 3: [head]->[next]->[next]->node->node->...->node->[prev]->[prev]->[tail]
# etc.
list-compress-depth 0
# Sets have a special encoding in just one case: when a set is composed
# of just strings that happen to be integers in radix 10 in the range

8
src/Makefile
View File

@ -18,7 +18,7 @@ OPTIMIZATION?=-O2
DEPENDENCY_TARGETS=hiredis linenoise lua
# Default settings
STD=-std=c99 -pedantic
STD=-std=c99 -pedantic -DREDIS_STATIC=''
WARN=-Wall -W
OPT=$(OPTIMIZATION)
@ -46,6 +46,10 @@ ifeq ($(USE_JEMALLOC),yes)
MALLOC=jemalloc
endif
ifeq ($(USE_JEMALLOC),no)
MALLOC=libc
endif
# Override default settings if possible
-include .make-settings
@ -113,7 +117,7 @@ endif
REDIS_SERVER_NAME=redis-server
REDIS_SENTINEL_NAME=redis-sentinel
REDIS_SERVER_OBJ=adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o
REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o
REDIS_CLI_NAME=redis-cli
REDIS_CLI_OBJ=anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o
REDIS_BENCHMARK_NAME=redis-benchmark

43
src/aof.c
View File

@ -770,52 +770,29 @@ int rioWriteBulkObject(rio *r, robj *obj) {
int rewriteListObject(rio *r, robj *key, robj *o) {
long long count = 0, items = listTypeLength(o);
if (o->encoding == REDIS_ENCODING_ZIPLIST) {
unsigned char *zl = o->ptr;
unsigned char *p = ziplistIndex(zl,0);
unsigned char *vstr;
unsigned int vlen;
long long vlong;
if (o->encoding == REDIS_ENCODING_QUICKLIST) {
quicklist *list = o->ptr;
quicklistIter *li = quicklistGetIterator(list, AL_START_HEAD);
quicklistEntry entry;
while(ziplistGet(p,&vstr,&vlen,&vlong)) {
while (quicklistNext(li,&entry)) {
if (count == 0) {
int cmd_items = (items > REDIS_AOF_REWRITE_ITEMS_PER_CMD) ?
REDIS_AOF_REWRITE_ITEMS_PER_CMD : items;
if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0;
if (rioWriteBulkString(r,"RPUSH",5) == 0) return 0;
if (rioWriteBulkObject(r,key) == 0) return 0;
}
if (vstr) {
if (rioWriteBulkString(r,(char*)vstr,vlen) == 0) return 0;
if (entry.value) {
if (rioWriteBulkString(r,(char*)entry.value,entry.sz) == 0) return 0;
} else {
if (rioWriteBulkLongLong(r,vlong) == 0) return 0;
if (rioWriteBulkLongLong(r,entry.longval) == 0) return 0;
}
p = ziplistNext(zl,p);
if (++count == REDIS_AOF_REWRITE_ITEMS_PER_CMD) count = 0;
items--;
}
} else if (o->encoding == REDIS_ENCODING_LINKEDLIST) {
list *list = o->ptr;
listNode *ln;
listIter li;
listRewind(list,&li);
while((ln = listNext(&li))) {
robj *eleobj = listNodeValue(ln);
if (count == 0) {
int cmd_items = (items > REDIS_AOF_REWRITE_ITEMS_PER_CMD) ?
REDIS_AOF_REWRITE_ITEMS_PER_CMD : items;
if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0;
if (rioWriteBulkString(r,"RPUSH",5) == 0) return 0;
if (rioWriteBulkObject(r,key) == 0) return 0;
}
if (rioWriteBulkObject(r,eleobj) == 0) return 0;
if (++count == REDIS_AOF_REWRITE_ITEMS_PER_CMD) count = 0;
items--;
}
quicklistReleaseIterator(li);
} else {
redisPanic("Unknown list encoding");
}

5
src/cluster.c
View File

@ -3886,10 +3886,7 @@ void clusterCommand(redisClient *c) {
server.cluster->stats_bus_messages_sent,
server.cluster->stats_bus_messages_received
);
addReplySds(c,sdscatprintf(sdsempty(),"$%lu\r\n",
(unsigned long)sdslen(info)));
addReplySds(c,info);
addReply(c,shared.crlf);
addReplyBulkSds(c, info);
} else if (!strcasecmp(c->argv[1]->ptr,"saveconfig") && c->argc == 2) {
int retval = clusterSaveConfig(1);

28
src/config.c
View File

@ -397,9 +397,13 @@ void loadServerConfigFromString(char *config) {
} else if (!strcasecmp(argv[0],"hash-max-ziplist-value") && argc == 2) {
server.hash_max_ziplist_value = memtoll(argv[1], NULL);
} else if (!strcasecmp(argv[0],"list-max-ziplist-entries") && argc == 2){
server.list_max_ziplist_entries = memtoll(argv[1], NULL);
/* DEAD OPTION */
} else if (!strcasecmp(argv[0],"list-max-ziplist-value") && argc == 2) {
server.list_max_ziplist_value = memtoll(argv[1], NULL);
/* DEAD OPTION */
} else if (!strcasecmp(argv[0],"list-max-ziplist-size") && argc == 2) {
server.list_max_ziplist_size = atoi(argv[1]);
} else if (!strcasecmp(argv[0],"list-compress-depth") && argc == 2) {
server.list_compress_depth = atoi(argv[1]);
} else if (!strcasecmp(argv[0],"set-max-intset-entries") && argc == 2) {
server.set_max_intset_entries = memtoll(argv[1], NULL);
} else if (!strcasecmp(argv[0],"zset-max-ziplist-entries") && argc == 2) {
@ -795,12 +799,12 @@ void configSetCommand(redisClient *c) {
} else if (!strcasecmp(c->argv[2]->ptr,"hash-max-ziplist-value")) {
if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt;
server.hash_max_ziplist_value = ll;
} else if (!strcasecmp(c->argv[2]->ptr,"list-max-ziplist-entries")) {
} else if (!strcasecmp(c->argv[2]->ptr,"list-max-ziplist-size")) {
if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt;
server.list_max_ziplist_entries = ll;
} else if (!strcasecmp(c->argv[2]->ptr,"list-max-ziplist-value")) {
server.list_max_ziplist_size = ll;
} else if (!strcasecmp(c->argv[2]->ptr,"list-compress-depth")) {
if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt;
server.list_max_ziplist_value = ll;
server.list_compress_depth = ll;
} else if (!strcasecmp(c->argv[2]->ptr,"set-max-intset-entries")) {
if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt;
server.set_max_intset_entries = ll;
@ -1047,10 +1051,10 @@ void configGetCommand(redisClient *c) {
server.hash_max_ziplist_entries);
config_get_numerical_field("hash-max-ziplist-value",
server.hash_max_ziplist_value);
config_get_numerical_field("list-max-ziplist-entries",
server.list_max_ziplist_entries);
config_get_numerical_field("list-max-ziplist-value",
server.list_max_ziplist_value);
config_get_numerical_field("list-max-ziplist-size",
server.list_max_ziplist_size);
config_get_numerical_field("list-compress-depth",
server.list_compress_depth);
config_get_numerical_field("set-max-intset-entries",
server.set_max_intset_entries);
config_get_numerical_field("zset-max-ziplist-entries",
@ -1857,8 +1861,8 @@ int rewriteConfig(char *path) {
rewriteConfigNotifykeyspaceeventsOption(state);
rewriteConfigNumericalOption(state,"hash-max-ziplist-entries",server.hash_max_ziplist_entries,REDIS_HASH_MAX_ZIPLIST_ENTRIES);
rewriteConfigNumericalOption(state,"hash-max-ziplist-value",server.hash_max_ziplist_value,REDIS_HASH_MAX_ZIPLIST_VALUE);
rewriteConfigNumericalOption(state,"list-max-ziplist-entries",server.list_max_ziplist_entries,REDIS_LIST_MAX_ZIPLIST_ENTRIES);
rewriteConfigNumericalOption(state,"list-max-ziplist-value",server.list_max_ziplist_value,REDIS_LIST_MAX_ZIPLIST_VALUE);
rewriteConfigNumericalOption(state,"list-max-ziplist-size",server.list_max_ziplist_size,REDIS_LIST_MAX_ZIPLIST_SIZE);
rewriteConfigNumericalOption(state,"list-compress-depth",server.list_compress_depth,REDIS_LIST_COMPRESS_DEPTH);
rewriteConfigNumericalOption(state,"set-max-intset-entries",server.set_max_intset_entries,REDIS_SET_MAX_INTSET_ENTRIES);
rewriteConfigNumericalOption(state,"zset-max-ziplist-entries",server.zset_max_ziplist_entries,REDIS_ZSET_MAX_ZIPLIST_ENTRIES);
rewriteConfigNumericalOption(state,"zset-max-ziplist-value",server.zset_max_ziplist_value,REDIS_ZSET_MAX_ZIPLIST_VALUE);

8
src/crc64.c
View File

@ -181,9 +181,13 @@ uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) {
}
/* Test main */
#ifdef TEST_MAIN
#ifdef REDIS_TEST
#include <stdio.h>
int main(void) {
#define UNUSED(x) (void)(x)
int crc64Test(int argc, char *argv[]) {
UNUSED(argc);
UNUSED(argv);
printf("e9c6d914c4b8d9ca == %016llx\n",
(unsigned long long) crc64(0,(unsigned char*)"123456789",9));
return 0;

4
src/crc64.h
View File

@ -5,4 +5,8 @@
uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l);
#ifdef REDIS_TEST
int crc64Test(int argc, char *argv[]);
#endif
#endif

55
src/debug.c
View File

@ -252,6 +252,12 @@ void computeDatasetDigest(unsigned char *final) {
}
}
void inputCatSds(void *result, const char *str) {
/* result is actually a (sds *), so re-cast it here */
sds *info = (sds *)result;
*info = sdscat(*info, str);
}
void debugCommand(redisClient *c) {
if (!strcasecmp(c->argv[1]->ptr,"segfault")) {
*((char*)-1) = 'x';
@ -295,13 +301,46 @@ void debugCommand(redisClient *c) {
val = dictGetVal(de);
strenc = strEncoding(val->encoding);
char extra[128] = {0};
if (val->encoding == REDIS_ENCODING_QUICKLIST) {
char *nextra = extra;
int remaining = sizeof(extra);
quicklist *ql = val->ptr;
/* Add number of quicklist nodes */
int used = snprintf(nextra, remaining, " ql_nodes:%u", ql->len);
nextra += used;
remaining -= used;
/* Add average quicklist fill factor */
double avg = (double)ql->count/ql->len;
used = snprintf(nextra, remaining, " ql_avg_node:%.2f", avg);
nextra += used;
remaining -= used;
/* Add quicklist fill level / max ziplist size */
used = snprintf(nextra, remaining, " ql_ziplist_max:%d", ql->fill);
nextra += used;
remaining -= used;
/* Add isCompressed? */
int compressed = ql->compress != 0;
used = snprintf(nextra, remaining, " ql_compressed:%d", compressed);
nextra += used;
remaining -= used;
/* Add total uncompressed size */
unsigned long sz = 0;
for (quicklistNode *node = ql->head; node; node = node->next) {
sz += node->sz;
}
used = snprintf(nextra, remaining, " ql_uncompressed_size:%lu", sz);
nextra += used;
remaining -= used;
}
addReplyStatusFormat(c,
"Value at:%p refcount:%d "
"encoding:%s serializedlength:%lld "
"lru:%d lru_seconds_idle:%llu",
"lru:%d lru_seconds_idle:%llu%s",
(void*)val, val->refcount,
strenc, (long long) rdbSavedObjectLen(val),
val->lru, estimateObjectIdleTime(val)/1000);
val->lru, estimateObjectIdleTime(val)/1000, extra);
} else if (!strcasecmp(c->argv[1]->ptr,"sdslen") && c->argc == 3) {
dictEntry *de;
robj *val;
@ -379,6 +418,18 @@ void debugCommand(redisClient *c) {
errstr = sdsmapchars(errstr,"\n\r"," ",2); /* no newlines in errors. */
errstr = sdscatlen(errstr,"\r\n",2);
addReplySds(c,errstr);
} else if (!strcasecmp(c->argv[1]->ptr,"jemalloc") && c->argc == 3) {
#if defined(USE_JEMALLOC)
if (!strcasecmp(c->argv[2]->ptr, "info")) {
sds info = sdsempty();
je_malloc_stats_print(inputCatSds, &info, NULL);
addReplyBulkSds(c, info);
} else {
addReplyErrorFormat(c, "Valid jemalloc debug fields: info");
}
#else
addReplyErrorFormat(c, "jemalloc support not available");
#endif
} else {
addReplyErrorFormat(c, "Unknown DEBUG subcommand or wrong number of arguments for '%s'",
(char*)c->argv[1]->ptr);

8
src/endianconv.c
View File

@ -101,12 +101,16 @@ uint64_t intrev64(uint64_t v) {
return v;
}
#ifdef TESTMAIN
#ifdef REDIS_TEST
#include <stdio.h>
int main(void) {
#define UNUSED(x) (void)(x)
int endianconvTest(int argc, char *argv[]) {
char buf[32];
UNUSED(argc);
UNUSED(argv);
sprintf(buf,"ciaoroma");
memrev16(buf);
printf("%s\n", buf);

4
src/endianconv.h
View File

@ -71,4 +71,8 @@ uint64_t intrev64(uint64_t v);
#define ntohu64(v) intrev64(v)
#endif
#ifdef REDIS_TEST
int endianconvTest(int argc, char *argv[]);
#endif
#endif

42
src/intset.c
View File

@ -365,44 +365,46 @@ size_t intsetBlobLen(intset *is) {
return sizeof(intset)+intrev32ifbe(is->length)*intrev32ifbe(is->encoding);
}
#ifdef INTSET_TEST_MAIN
#ifdef REDIS_TEST
#include <sys/time.h>
#include <time.h>
void intsetRepr(intset *is) {
int i;
for (i = 0; i < intrev32ifbe(is->length); i++) {
#if 0
static void intsetRepr(intset *is) {
for (uint32_t i = 0; i < intrev32ifbe(is->length); i++) {
printf("%lld\n", (uint64_t)_intsetGet(is,i));
}
printf("\n");
}
void error(char *err) {
static void error(char *err) {
printf("%s\n", err);
exit(1);
}
#endif
void ok(void) {
static void ok(void) {
printf("OK\n");
}
long long usec(void) {
static long long usec(void) {
struct timeval tv;
gettimeofday(&tv,NULL);
return (((long long)tv.tv_sec)*1000000)+tv.tv_usec;
}
#define assert(_e) ((_e)?(void)0:(_assert(#_e,__FILE__,__LINE__),exit(1)))
void _assert(char *estr, char *file, int line) {
static void _assert(char *estr, char *file, int line) {
printf("\n\n=== ASSERTION FAILED ===\n");
printf("==> %s:%d '%s' is not true\n",file,line,estr);
}
intset *createSet(int bits, int size) {
static intset *createSet(int bits, int size) {
uint64_t mask = (1<<bits)-1;
uint64_t i, value;
uint64_t value;
intset *is = intsetNew();
for (i = 0; i < size; i++) {
for (int i = 0; i < size; i++) {
if (bits > 32) {
value = (rand()*rand()) & mask;
} else {
@ -413,10 +415,8 @@ intset *createSet(int bits, int size) {
return is;
}
void checkConsistency(intset *is) {
int i;
for (i = 0; i < (intrev32ifbe(is->length)-1); i++) {
static void checkConsistency(intset *is) {
for (uint32_t i = 0; i < (intrev32ifbe(is->length)-1); i++) {
uint32_t encoding = intrev32ifbe(is->encoding);
if (encoding == INTSET_ENC_INT16) {
@ -432,11 +432,15 @@ void checkConsistency(intset *is) {
}
}
int main(int argc, char **argv) {
#define UNUSED(x) (void)(x)
int intsetTest(int argc, char **argv) {
uint8_t success;
int i;
intset *is;
sranddev();
srand(time(NULL));
UNUSED(argc);
UNUSED(argv);
printf("Value encodings: "); {
assert(_intsetValueEncoding(-32768) == INTSET_ENC_INT16);
@ -464,7 +468,7 @@ int main(int argc, char **argv) {
}
printf("Large number of random adds: "); {
int inserts = 0;
uint32_t inserts = 0;
is = intsetNew();
for (i = 0; i < 1024; i++) {
is = intsetAdd(is,rand()%0x800,&success);
@ -566,5 +570,7 @@ int main(int argc, char **argv) {
checkConsistency(is);
ok();
}
return 0;
}
#endif

4
src/intset.h
View File

@ -48,4 +48,8 @@ uint8_t intsetGet(intset *is, uint32_t pos, int64_t *value);
uint32_t intsetLen(intset *is);
size_t intsetBlobLen(intset *is);
#ifdef REDIS_TEST
int intsetTest(int argc, char *argv[]);
#endif
#endif // __INTSET_H

56
src/lzfP.h
View File

@ -49,7 +49,7 @@
* the difference between 15 and 14 is very small
* for small blocks (and 14 is usually a bit faster).
* For a low-memory/faster configuration, use HLOG == 13;
* For best compression, use 15 or 16 (or more, up to 23).
* For best compression, use 15 or 16 (or more, up to 22).
*/
#ifndef HLOG
# define HLOG 16
@ -94,7 +94,7 @@
/*
* Avoid assigning values to errno variable? for some embedding purposes
* (linux kernel for example), this is necessary. NOTE: this breaks
* the documentation in lzf.h.
* the documentation in lzf.h. Avoiding errno has no speed impact.
*/
#ifndef AVOID_ERRNO
# define AVOID_ERRNO 0
@ -121,16 +121,52 @@
# define CHECK_INPUT 1
#endif
/*
* Whether to store pointers or offsets inside the hash table. On
* 64 bit architetcures, pointers take up twice as much space,
* and might also be slower. Default is to autodetect.
*/
/*#define LZF_USER_OFFSETS autodetect */
/*****************************************************************************/
/* nothing should be changed below */
#ifdef __cplusplus
# include <cstring>
# include <climits>
using namespace std;
#else
# include <string.h>
# include <limits.h>
#endif
#ifndef LZF_USE_OFFSETS
# if defined (WIN32)
# define LZF_USE_OFFSETS defined(_M_X64)
# else
# if __cplusplus > 199711L
# include <cstdint>
# else
# include <stdint.h>
# endif
# define LZF_USE_OFFSETS (UINTPTR_MAX > 0xffffffffU)
# endif
#endif
typedef unsigned char u8;
typedef const u8 *LZF_STATE[1 << (HLOG)];
#if LZF_USE_OFFSETS
# define LZF_HSLOT_BIAS ((const u8 *)in_data)
typedef unsigned int LZF_HSLOT;
#else
# define LZF_HSLOT_BIAS 0
typedef const u8 *LZF_HSLOT;
#endif
typedef LZF_HSLOT LZF_STATE[1 << (HLOG)];
#if !STRICT_ALIGN
/* for unaligned accesses we need a 16 bit datatype. */
# include <limits.h>
# if USHRT_MAX == 65535
typedef unsigned short u16;
# elif UINT_MAX == 65535
@ -142,17 +178,7 @@ typedef const u8 *LZF_STATE[1 << (HLOG)];
#endif
#if ULTRA_FAST
# if defined(VERY_FAST)
# undef VERY_FAST
# endif
#endif
#if INIT_HTAB
# ifdef __cplusplus
# include <cstring>
# else
# include <string.h>
# endif
# undef VERY_FAST
#endif
#endif

42
src/lzf_c.c
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2000-2008 Marc Alexander Lehmann <schmorp@schmorp.de>
* Copyright (c) 2000-2010 Marc Alexander Lehmann <schmorp@schmorp.de>
*
* Redistribution and use in source and binary forms, with or without modifica-
* tion, are permitted provided that the following conditions are met:
@ -40,8 +40,8 @@
/*
* don't play with this unless you benchmark!
* decompression is not dependent on the hash function
* the hashing function might seem strange, just believe me
* the data format is not dependent on the hash function.
* the hash function might seem strange, just believe me,
* it works ;)
*/
#ifndef FRST
@ -89,9 +89,9 @@
/*
* compressed format
*
* 000LLLLL <L+1> ; literal
* LLLooooo oooooooo ; backref L
* 111ooooo LLLLLLLL oooooooo ; backref L+7
* 000LLLLL <L+1> ; literal, L+1=1..33 octets
* LLLooooo oooooooo ; backref L+1=1..7 octets, o+1=1..4096 offset
* 111ooooo LLLLLLLL oooooooo ; backref L+8 octets, o+1=1..4096 offset
*
*/
@ -106,7 +106,6 @@ lzf_compress (const void *const in_data, unsigned int in_len,
#if !LZF_STATE_ARG
LZF_STATE htab;
#endif
const u8 **hslot;
const u8 *ip = (const u8 *)in_data;
u8 *op = (u8 *)out_data;
const u8 *in_end = ip + in_len;
@ -133,10 +132,6 @@ lzf_compress (const void *const in_data, unsigned int in_len,
#if INIT_HTAB
memset (htab, 0, sizeof (htab));
# if 0
for (hslot = htab; hslot < htab + HSIZE; hslot++)
*hslot++ = ip;
# endif
#endif
lit = 0; op++; /* start run */
@ -144,24 +139,23 @@ lzf_compress (const void *const in_data, unsigned int in_len,
hval = FRST (ip);
while (ip < in_end - 2)
{
LZF_HSLOT *hslot;
hval = NEXT (hval, ip);
hslot = htab + IDX (hval);
ref = *hslot; *hslot = ip;
ref = *hslot + LZF_HSLOT_BIAS; *hslot = ip - LZF_HSLOT_BIAS;
if (1
#if INIT_HTAB
&& ref < ip /* the next test will actually take care of this, but this is faster */
#endif
&& (off = ip - ref - 1) < MAX_OFF
&& ip + 4 < in_end
&& ref > (u8 *)in_data
#if STRICT_ALIGN
&& ref[0] == ip[0]
&& ref[1] == ip[1]
&& ref[2] == ip[2]
#if STRICT_ALIGN
&& ((ref[1] << 8) | ref[0]) == ((ip[1] << 8) | ip[0])
#else
&& *(u16 *)ref == *(u16 *)ip
&& ref[2] == ip[2]
#endif
)
{
@ -170,12 +164,13 @@ lzf_compress (const void *const in_data, unsigned int in_len,
unsigned int maxlen = in_end - ip - len;
maxlen = maxlen > MAX_REF ? MAX_REF : maxlen;
if (expect_false (op + 3 + 1 >= out_end)) /* first a faster conservative test */
if (op - !lit + 3 + 1 >= out_end) /* second the exact but rare test */
return 0;
op [- lit - 1] = lit - 1; /* stop run */
op -= !lit; /* undo run if length is zero */
if (expect_false (op + 3 + 1 >= out_end))
return 0;
for (;;)
{
if (expect_true (maxlen > 16))
@ -222,6 +217,7 @@ lzf_compress (const void *const in_data, unsigned int in_len,
}
*op++ = off;
lit = 0; op++; /* start run */
ip += len + 1;
@ -237,12 +233,12 @@ lzf_compress (const void *const in_data, unsigned int in_len,
hval = FRST (ip);
hval = NEXT (hval, ip);
htab[IDX (hval)] = ip;
htab[IDX (hval)] = ip - LZF_HSLOT_BIAS;
ip++;
# if VERY_FAST && !ULTRA_FAST
hval = NEXT (hval, ip);
htab[IDX (hval)] = ip;
htab[IDX (hval)] = ip - LZF_HSLOT_BIAS;
ip++;
# endif
#else
@ -251,7 +247,7 @@ lzf_compress (const void *const in_data, unsigned int in_len,
do
{
hval = NEXT (hval, ip);
htab[IDX (hval)] = ip;
htab[IDX (hval)] = ip - LZF_HSLOT_BIAS;
ip++;
}
while (len--);

57
src/lzf_d.c
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2000-2007 Marc Alexander Lehmann <schmorp@schmorp.de>
* Copyright (c) 2000-2010 Marc Alexander Lehmann <schmorp@schmorp.de>
*
* Redistribution and use in source and binary forms, with or without modifica-
* tion, are permitted provided that the following conditions are met:
@ -43,14 +43,14 @@
# define SET_ERRNO(n) errno = (n)
#endif
/*
#if USE_REP_MOVSB /* small win on amd, big loss on intel */
#if (__i386 || __amd64) && __GNUC__ >= 3
# define lzf_movsb(dst, src, len) \
asm ("rep movsb" \
: "=D" (dst), "=S" (src), "=c" (len) \
: "0" (dst), "1" (src), "2" (len));
#endif
*/
#endif
unsigned int
lzf_decompress (const void *const in_data, unsigned int in_len,
@ -86,9 +86,17 @@ lzf_decompress (const void *const in_data, unsigned int in_len,
#ifdef lzf_movsb
lzf_movsb (op, ip, ctrl);
#else
do
*op++ = *ip++;
while (--ctrl);
switch (ctrl)
{
case 32: *op++ = *ip++; case 31: *op++ = *ip++; case 30: *op++ = *ip++; case 29: *op++ = *ip++;
case 28: *op++ = *ip++; case 27: *op++ = *ip++; case 26: *op++ = *ip++; case 25: *op++ = *ip++;
case 24: *op++ = *ip++; case 23: *op++ = *ip++; case 22: *op++ = *ip++; case 21: *op++ = *ip++;
case 20: *op++ = *ip++; case 19: *op++ = *ip++; case 18: *op++ = *ip++; case 17: *op++ = *ip++;
case 16: *op++ = *ip++; case 15: *op++ = *ip++; case 14: *op++ = *ip++; case 13: *op++ = *ip++;
case 12: *op++ = *ip++; case 11: *op++ = *ip++; case 10: *op++ = *ip++; case 9: *op++ = *ip++;
case 8: *op++ = *ip++; case 7: *op++ = *ip++; case 6: *op++ = *ip++; case 5: *op++ = *ip++;
case 4: *op++ = *ip++; case 3: *op++ = *ip++; case 2: *op++ = *ip++; case 1: *op++ = *ip++;
}
#endif
}
else /* back reference */
@ -134,12 +142,39 @@ lzf_decompress (const void *const in_data, unsigned int in_len,
len += 2;
lzf_movsb (op, ref, len);
#else
*op++ = *ref++;
*op++ = *ref++;
switch (len)
{
default:
len += 2;
do
*op++ = *ref++;
while (--len);
if (op >= ref + len)
{
/* disjunct areas */
memcpy (op, ref, len);
op += len;
}
else
{
/* overlapping, use octte by octte copying */
do
*op++ = *ref++;
while (--len);
}
break;
case 9: *op++ = *ref++;
case 8: *op++ = *ref++;
case 7: *op++ = *ref++;
case 6: *op++ = *ref++;
case 5: *op++ = *ref++;
case 4: *op++ = *ref++;
case 3: *op++ = *ref++;
case 2: *op++ = *ref++;
case 1: *op++ = *ref++;
case 0: *op++ = *ref++; /* two octets more */
*op++ = *ref++;
}
#endif
}
}

8
src/networking.c
View File

@ -525,6 +525,14 @@ void addReplyBulkCBuffer(redisClient *c, void *p, size_t len) {
addReply(c,shared.crlf);
}
/* Add sds to reply (takes ownership of sds and frees it) */
void addReplyBulkSds(redisClient *c, sds s) {
addReplySds(c,sdscatfmt(sdsempty(),"$%u\r\n",
(unsigned long)sdslen(s)));
addReplySds(c,s);
addReply(c,shared.crlf);
}
/* Add a C nul term string as bulk reply */
void addReplyBulkCString(redisClient *c, char *s) {
if (s == NULL) {

16
src/object.c
View File

@ -180,11 +180,10 @@ robj *dupStringObject(robj *o) {
}
}
robj *createListObject(void) {
list *l = listCreate();
robj *createQuicklistObject(void) {
quicklist *l = quicklistCreate();
robj *o = createObject(REDIS_LIST,l);
listSetFreeMethod(l,decrRefCountVoid);
o->encoding = REDIS_ENCODING_LINKEDLIST;
o->encoding = REDIS_ENCODING_QUICKLIST;
return o;
}
@ -242,11 +241,8 @@ void freeStringObject(robj *o) {
void freeListObject(robj *o) {
switch (o->encoding) {
case REDIS_ENCODING_LINKEDLIST:
listRelease((list*) o->ptr);
break;
case REDIS_ENCODING_ZIPLIST:
zfree(o->ptr);
case REDIS_ENCODING_QUICKLIST:
quicklistRelease(o->ptr);
break;
default:
redisPanic("Unknown list encoding type");
@ -678,7 +674,7 @@ char *strEncoding(int encoding) {
case REDIS_ENCODING_RAW: return "raw";
case REDIS_ENCODING_INT: return "int";
case REDIS_ENCODING_HT: return "hashtable";
case REDIS_ENCODING_LINKEDLIST: return "linkedlist";
case REDIS_ENCODING_QUICKLIST: return "quicklist";
case REDIS_ENCODING_ZIPLIST: return "ziplist";
case REDIS_ENCODING_INTSET: return "intset";
case REDIS_ENCODING_SKIPLIST: return "skiplist";

2639
src/quicklist.c Normal file
View File

File diff suppressed because it is too large Load Diff

169
src/quicklist.h Normal file
View File

@ -0,0 +1,169 @@
/* quicklist.h - A generic doubly linked quicklist implementation
*
* Copyright (c) 2014, Matt Stancliff <matt@genges.com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this quicklist of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this quicklist of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __QUICKLIST_H__
#define __QUICKLIST_H__
/* Node, quicklist, and Iterator are the only data structures used currently. */
/* quicklistNode is a 32 byte struct describing a ziplist for a quicklist.
* We use bit fields keep the quicklistNode at 32 bytes.
* count: 16 bits, max 65536 (max zl bytes is 65k, so max count actually < 32k).
* encoding: 2 bits, RAW=1, LZF=2.
* container: 2 bits, NONE=1, ZIPLIST=2.
* recompress: 1 bit, bool, true if node is temporarry decompressed for usage.
* attempted_compress: 1 bit, boolean, used for verifying during testing.
* extra: 12 bits, free for future use; pads out the remainder of 32 bits */
typedef struct quicklistNode {
struct quicklistNode *prev;
struct quicklistNode *next;
unsigned char *zl;
unsigned int sz; /* ziplist size in bytes */
unsigned int count : 16; /* count of items in ziplist */
unsigned int encoding : 2; /* RAW==1 or LZF==2 */
unsigned int container : 2; /* NONE==1 or ZIPLIST==2 */
unsigned int recompress : 1; /* was this node previous compressed? */
unsigned int attempted_compress : 1; /* node can't compress; too small */
unsigned int extra : 10; /* more bits to steal for future usage */
} quicklistNode;
/* quicklistLZF is a 4+N byte struct holding 'sz' followed by 'compressed'.
* 'sz' is byte length of 'compressed' field.
* 'compressed' is LZF data with total (compressed) length 'sz'
* NOTE: uncompressed length is stored in quicklistNode->sz.
* When quicklistNode->zl is compressed, node->zl points to a quicklistLZF */
typedef struct quicklistLZF {
unsigned int sz; /* LZF size in bytes*/
char compressed[];
} quicklistLZF;
/* quicklist is a 32 byte struct (on 64-bit systems) describing a quicklist.
* 'count' is the number of total entries.
* 'len' is the number of quicklist nodes.
* 'compress' is: -1 if compression disabled, otherwise it's the number
* of quicklistNodes to leave uncompressed at ends of quicklist.
* 'fill' is the user-requested (or default) fill factor. */
typedef struct quicklist {
quicklistNode *head;
quicklistNode *tail;
unsigned long count; /* total count of all entries in all ziplists */
unsigned int len; /* number of quicklistNodes */
int fill : 16; /* fill factor for individual nodes */
unsigned int compress : 16; /* depth of end nodes not to compress;0=off */
} quicklist;
typedef struct quicklistIter {
const quicklist *quicklist;
quicklistNode *current;
unsigned char *zi;
long offset; /* offset in current ziplist */
int direction;
} quicklistIter;
typedef struct quicklistEntry {
const quicklist *quicklist;
quicklistNode *node;
unsigned char *zi;
unsigned char *value;
unsigned int sz;
long long longval;
int offset;
} quicklistEntry;
#define QUICKLIST_HEAD 0
#define QUICKLIST_TAIL -1
/* quicklist node encodings */
#define QUICKLIST_NODE_ENCODING_RAW 1
#define QUICKLIST_NODE_ENCODING_LZF 2
/* quicklist compression disable */
#define QUICKLIST_NOCOMPRESS 0
/* quicklist container formats */
#define QUICKLIST_NODE_CONTAINER_NONE 1
#define QUICKLIST_NODE_CONTAINER_ZIPLIST 2
#define quicklistNodeIsCompressed(node) \
((node)->encoding == QUICKLIST_NODE_ENCODING_LZF)
/* Prototypes */
quicklist *quicklistCreate(void);
quicklist *quicklistNew(int fill, int compress);
void quicklistSetCompressDepth(quicklist *quicklist, int depth);
void quicklistSetFill(quicklist *quicklist, int fill);
void quicklistSetOptions(quicklist *quicklist, int fill, int depth);
void quicklistRelease(quicklist *quicklist);
int quicklistPushHead(quicklist *quicklist, void *value, const size_t sz);
int quicklistPushTail(quicklist *quicklist, void *value, const size_t sz);
void quicklistPush(quicklist *quicklist, void *value, const size_t sz,
int where);
void quicklistAppendZiplist(quicklist *quicklist, unsigned char *zl);
quicklist *quicklistAppendValuesFromZiplist(quicklist *quicklist,
unsigned char *zl);
quicklist *quicklistCreateFromZiplist(int fill, int compress,
unsigned char *zl);
void quicklistInsertAfter(quicklist *quicklist, quicklistEntry *node,
void *value, const size_t sz);
void quicklistInsertBefore(quicklist *quicklist, quicklistEntry *node,
void *value, const size_t sz);
void quicklistDelEntry(quicklistIter *iter, quicklistEntry *entry);
int quicklistReplaceAtIndex(quicklist *quicklist, long index, void *data,
int sz);
int quicklistDelRange(quicklist *quicklist, const long start, const long stop);
quicklistIter *quicklistGetIterator(const quicklist *quicklist, int direction);
quicklistIter *quicklistGetIteratorAtIdx(const quicklist *quicklist,
int direction, const long long idx);
int quicklistNext(quicklistIter *iter, quicklistEntry *node);
void quicklistReleaseIterator(quicklistIter *iter);
quicklist *quicklistDup(quicklist *orig);
int quicklistIndex(const quicklist *quicklist, const long long index,
quicklistEntry *entry);
void quicklistRewind(quicklist *quicklist, quicklistIter *li);
void quicklistRewindTail(quicklist *quicklist, quicklistIter *li);
void quicklistRotate(quicklist *quicklist);
int quicklistPopCustom(quicklist *quicklist, int where, unsigned char **data,
unsigned int *sz, long long *sval,
void *(*saver)(unsigned char *data, unsigned int sz));
int quicklistPop(quicklist *quicklist, int where, unsigned char **data,
unsigned int *sz, long long *slong);
unsigned int quicklistCount(quicklist *ql);
int quicklistCompare(unsigned char *p1, unsigned char *p2, int p2_len);
size_t quicklistGetLzf(const quicklistNode *node, void **data);
#ifdef REDIS_TEST
int quicklistTest(int argc, char *argv[]);
#endif
/* Directions for iterators */
#define AL_START_HEAD 0
#define AL_START_TAIL 1
#endif /* __QUICKLIST_H__ */

145
src/rdb.c
View File

@ -209,10 +209,33 @@ int rdbTryIntegerEncoding(char *s, size_t len, unsigned char *enc) {
return rdbEncodeInteger(value,enc);
}
int rdbSaveLzfStringObject(rio *rdb, unsigned char *s, size_t len) {
size_t comprlen, outlen;
int rdbSaveLzfBlob(rio *rdb, void *data, size_t compress_len,
size_t original_len) {
unsigned char byte;
int n, nwritten = 0;
/* Data compressed! Let's save it on disk */
byte = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_LZF;
if ((n = rdbWriteRaw(rdb,&byte,1)) == -1) goto writeerr;
nwritten += n;
if ((n = rdbSaveLen(rdb,compress_len)) == -1) goto writeerr;
nwritten += n;
if ((n = rdbSaveLen(rdb,original_len)) == -1) goto writeerr;
nwritten += n;
if ((n = rdbWriteRaw(rdb,data,compress_len)) == -1) goto writeerr;
nwritten += n;
return nwritten;
writeerr:
return -1;
}
int rdbSaveLzfStringObject(rio *rdb, unsigned char *s, size_t len) {
size_t comprlen, outlen;
void *out;
/* We require at least four bytes compression for this to be worth it */
@ -224,26 +247,9 @@ int rdbSaveLzfStringObject(rio *rdb, unsigned char *s, size_t len) {
zfree(out);
return 0;
}
/* Data compressed! Let's save it on disk */
byte = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_LZF;
if ((n = rdbWriteRaw(rdb,&byte,1)) == -1) goto writeerr;
nwritten += n;
if ((n = rdbSaveLen(rdb,comprlen)) == -1) goto writeerr;
nwritten += n;
if ((n = rdbSaveLen(rdb,len)) == -1) goto writeerr;
nwritten += n;
if ((n = rdbWriteRaw(rdb,out,comprlen)) == -1) goto writeerr;
nwritten += n;
size_t nwritten = rdbSaveLzfBlob(rdb, out, comprlen, len);
zfree(out);
return nwritten;
writeerr:
zfree(out);
return -1;
}
robj *rdbLoadLzfStringObject(rio *rdb) {
@ -433,10 +439,8 @@ int rdbSaveObjectType(rio *rdb, robj *o) {
case REDIS_STRING:
return rdbSaveType(rdb,REDIS_RDB_TYPE_STRING);
case REDIS_LIST:
if (o->encoding == REDIS_ENCODING_ZIPLIST)
return rdbSaveType(rdb,REDIS_RDB_TYPE_LIST_ZIPLIST);
else if (o->encoding == REDIS_ENCODING_LINKEDLIST)
return rdbSaveType(rdb,REDIS_RDB_TYPE_LIST);
if (o->encoding == REDIS_ENCODING_QUICKLIST)
return rdbSaveType(rdb,REDIS_RDB_TYPE_LIST_QUICKLIST);
else
redisPanic("Unknown list encoding");
case REDIS_SET:
@ -477,7 +481,7 @@ int rdbLoadObjectType(rio *rdb) {
/* Save a Redis object. Returns -1 on error, number of bytes written on success. */
int rdbSaveObject(rio *rdb, robj *o) {
int n, nwritten = 0;
int n = 0, nwritten = 0;
if (o->type == REDIS_STRING) {
/* Save a string value */
@ -485,25 +489,24 @@ int rdbSaveObject(rio *rdb, robj *o) {
nwritten += n;
} else if (o->type == REDIS_LIST) {
/* Save a list value */
if (o->encoding == REDIS_ENCODING_ZIPLIST) {
size_t l = ziplistBlobLen((unsigned char*)o->ptr);
if (o->encoding == REDIS_ENCODING_QUICKLIST) {
quicklist *ql = o->ptr;
quicklistNode *node = ql->head;
if ((n = rdbSaveRawString(rdb,o->ptr,l)) == -1) return -1;
nwritten += n;
} else if (o->encoding == REDIS_ENCODING_LINKEDLIST) {
list *list = o->ptr;
listIter li;
listNode *ln;
if ((n = rdbSaveLen(rdb,listLength(list))) == -1) return -1;
if ((n = rdbSaveLen(rdb,ql->len)) == -1) return -1;
nwritten += n;
listRewind(list,&li);
while((ln = listNext(&li))) {
robj *eleobj = listNodeValue(ln);
if ((n = rdbSaveStringObject(rdb,eleobj)) == -1) return -1;
nwritten += n;
}
do {
if (quicklistNodeIsCompressed(node)) {
void *data;
size_t compress_len = quicklistGetLzf(node, &data);
if ((n = rdbSaveLzfBlob(rdb,data,compress_len,node->sz)) == -1) return -1;
nwritten += n;
} else {
if ((n = rdbSaveRawString(rdb,node->zl,node->sz)) == -1) return -1;
nwritten += n;
}
} while ((node = node->next));
} else {
redisPanic("Unknown list encoding");
}
@ -720,7 +723,7 @@ int rdbSave(char *filename) {
char tmpfile[256];
FILE *fp;
rio rdb;
int error;
int error = 0;
snprintf(tmpfile,256,"temp-%d.rdb", (int) getpid());
fp = fopen(tmpfile,"w");
@ -831,33 +834,18 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
/* Read list value */
if ((len = rdbLoadLen(rdb,NULL)) == REDIS_RDB_LENERR) return NULL;
/* Use a real list when there are too many entries */
if (len > server.list_max_ziplist_entries) {
o = createListObject();
} else {
o = createZiplistObject();
}
o = createQuicklistObject();
quicklistSetOptions(o->ptr, server.list_max_ziplist_size,
server.list_compress_depth);
/* Load every single element of the list */
while(len--) {
if ((ele = rdbLoadEncodedStringObject(rdb)) == NULL) return NULL;
/* If we are using a ziplist and the value is too big, convert
* the object to a real list. */
if (o->encoding == REDIS_ENCODING_ZIPLIST &&
sdsEncodedObject(ele) &&
sdslen(ele->ptr) > server.list_max_ziplist_value)
listTypeConvert(o,REDIS_ENCODING_LINKEDLIST);
if (o->encoding == REDIS_ENCODING_ZIPLIST) {
dec = getDecodedObject(ele);
o->ptr = ziplistPush(o->ptr,dec->ptr,sdslen(dec->ptr),REDIS_TAIL);
decrRefCount(dec);
decrRefCount(ele);
} else {
ele = tryObjectEncoding(ele);
listAddNodeTail(o->ptr,ele);
}
dec = getDecodedObject(ele);
size_t len = sdslen(dec->ptr);
quicklistPushTail(o->ptr, dec->ptr, len);
decrRefCount(dec);
decrRefCount(ele);
}
} else if (rdbtype == REDIS_RDB_TYPE_SET) {
/* Read list/set value */
@ -994,20 +982,30 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
/* All pairs should be read by now */
redisAssert(len == 0);
} else if (rdbtype == REDIS_RDB_TYPE_LIST_QUICKLIST) {
if ((len = rdbLoadLen(rdb,NULL)) == REDIS_RDB_LENERR) return NULL;
o = createQuicklistObject();
quicklistSetOptions(o->ptr, server.list_max_ziplist_size,
server.list_compress_depth);
while (len--) {
if ((ele = rdbLoadStringObject(rdb)) == NULL) return NULL;
/* 'ele' contains a sds of the ziplist, but we need to extract
* the actual ziplist for future usage. We must copy the
* sds contents to a new buffer. */
unsigned char *zl = (unsigned char *)sdsnative(ele->ptr);
zfree(ele); /* free robj container since we keep the ziplist */
quicklistAppendZiplist(o->ptr, zl);
}
} else if (rdbtype == REDIS_RDB_TYPE_HASH_ZIPMAP ||
rdbtype == REDIS_RDB_TYPE_LIST_ZIPLIST ||
rdbtype == REDIS_RDB_TYPE_SET_INTSET ||
rdbtype == REDIS_RDB_TYPE_ZSET_ZIPLIST ||
rdbtype == REDIS_RDB_TYPE_HASH_ZIPLIST)
{
robj *aux = rdbLoadStringObject(rdb);
if (aux == NULL) return NULL;
o = createObject(REDIS_STRING,NULL); /* string is just placeholder */
o->ptr = zmalloc(sdslen(aux->ptr));
memcpy(o->ptr,aux->ptr,sdslen(aux->ptr));
decrRefCount(aux);
o = rdbLoadStringObject(rdb);
if (o == NULL) return NULL;
o->ptr = sdsnative(o->ptr);
/* Fix the object encoding, and make sure to convert the encoded
* data type into the base type if accordingly to the current
@ -1048,8 +1046,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
case REDIS_RDB_TYPE_LIST_ZIPLIST:
o->type = REDIS_LIST;
o->encoding = REDIS_ENCODING_ZIPLIST;
if (ziplistLen(o->ptr) > server.list_max_ziplist_entries)
listTypeConvert(o,REDIS_ENCODING_LINKEDLIST);
listTypeConvert(o,REDIS_ENCODING_QUICKLIST);
break;
case REDIS_RDB_TYPE_SET_INTSET:
o->type = REDIS_SET;

7
src/rdb.h
View File

@ -38,7 +38,7 @@
/* The current RDB version. When the format changes in a way that is no longer
* backward compatible this number gets incremented. */
#define REDIS_RDB_VERSION 6
#define REDIS_RDB_VERSION 7
/* Defines related to the dump file format. To store 32 bits lengths for short
* keys requires a lot of space, so we check the most significant 2 bits of
@ -74,6 +74,7 @@
#define REDIS_RDB_TYPE_SET 2
#define REDIS_RDB_TYPE_ZSET 3
#define REDIS_RDB_TYPE_HASH 4
/* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */
/* Object types for encoded objects. */
#define REDIS_RDB_TYPE_HASH_ZIPMAP 9
@ -81,9 +82,11 @@
#define REDIS_RDB_TYPE_SET_INTSET 11
#define REDIS_RDB_TYPE_ZSET_ZIPLIST 12
#define REDIS_RDB_TYPE_HASH_ZIPLIST 13
#define REDIS_RDB_TYPE_LIST_QUICKLIST 14
/* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */
/* Test if a type is an object type. */
#define rdbIsObjectType(t) ((t >= 0 && t <= 4) || (t >= 9 && t <= 13))
#define rdbIsObjectType(t) ((t >= 0 && t <= 4) || (t >= 9 && t <= 14))
/* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */
#define REDIS_RDB_OPCODE_EXPIRETIME_MS 252

12
src/redis-benchmark.c
View File

@ -738,12 +738,24 @@ int main(int argc, const char **argv) {
free(cmd);
}
if (test_is_selected("rpush")) {
len = redisFormatCommand(&cmd,"RPUSH mylist %s",data);
benchmark("RPUSH",cmd,len);
free(cmd);
}
if (test_is_selected("lpop")) {
len = redisFormatCommand(&cmd,"LPOP mylist");
benchmark("LPOP",cmd,len);
free(cmd);
}
if (test_is_selected("rpop")) {
len = redisFormatCommand(&cmd,"RPOP mylist");
benchmark("RPOP",cmd,len);
free(cmd);
}
if (test_is_selected("sadd")) {
len = redisFormatCommand(&cmd,
"SADD myset element:__rand_int__");

3
src/redis-cli.c
View File

@ -629,6 +629,9 @@ static int cliSendCommand(int argc, char **argv, int repeat) {
output_raw = 0;
if (!strcasecmp(command,"info") ||
(argc == 3 && !strcasecmp(command,"debug") &&
(!strcasecmp(argv[1],"jemalloc") &&
!strcasecmp(argv[2],"info"))) ||
(argc == 2 && !strcasecmp(command,"cluster") &&
(!strcasecmp(argv[1],"nodes") ||
!strcasecmp(argv[1],"info"))) ||

36
src/redis.c
View File

@ -1449,8 +1449,8 @@ void initServerConfig(void) {
server.maxmemory_samples = REDIS_DEFAULT_MAXMEMORY_SAMPLES;
server.hash_max_ziplist_entries = REDIS_HASH_MAX_ZIPLIST_ENTRIES;
server.hash_max_ziplist_value = REDIS_HASH_MAX_ZIPLIST_VALUE;
server.list_max_ziplist_entries = REDIS_LIST_MAX_ZIPLIST_ENTRIES;
server.list_max_ziplist_value = REDIS_LIST_MAX_ZIPLIST_VALUE;
server.list_max_ziplist_size = REDIS_LIST_MAX_ZIPLIST_SIZE;
server.list_compress_depth = REDIS_LIST_COMPRESS_DEPTH;
server.set_max_intset_entries = REDIS_SET_MAX_INTSET_ENTRIES;
server.zset_max_ziplist_entries = REDIS_ZSET_MAX_ZIPLIST_ENTRIES;
server.zset_max_ziplist_value = REDIS_ZSET_MAX_ZIPLIST_VALUE;
@ -3058,11 +3058,7 @@ void infoCommand(redisClient *c) {
addReply(c,shared.syntaxerr);
return;
}
sds info = genRedisInfoString(section);
addReplySds(c,sdscatprintf(sdsempty(),"$%lu\r\n",
(unsigned long)sdslen(info)));
addReplySds(c,info);
addReply(c,shared.crlf);
addReplyBulkSds(c, genRedisInfoString(section));
}
void monitorCommand(redisClient *c) {
@ -3659,6 +3655,32 @@ int redisIsSupervised(void) {
int main(int argc, char **argv) {
struct timeval tv;
#ifdef REDIS_TEST
if (argc == 3 && !strcasecmp(argv[1], "test")) {
if (!strcasecmp(argv[2], "ziplist")) {
return ziplistTest(argc, argv);
} else if (!strcasecmp(argv[2], "quicklist")) {
quicklistTest(argc, argv);
} else if (!strcasecmp(argv[2], "intset")) {
return intsetTest(argc, argv);
} else if (!strcasecmp(argv[2], "zipmap")) {
return zipmapTest(argc, argv);
} else if (!strcasecmp(argv[2], "sha1test")) {
return sha1Test(argc, argv);
} else if (!strcasecmp(argv[2], "util")) {
return utilTest(argc, argv);
} else if (!strcasecmp(argv[2], "sds")) {
return sdsTest(argc, argv);
} else if (!strcasecmp(argv[2], "endianconv")) {
return endianconvTest(argc, argv);
} else if (!strcasecmp(argv[2], "crc64")) {
return crc64Test(argc, argv);
}
return -1; /* test not found */
}
#endif
/* We need to initialize our libraries, and the server configuration. */
#ifdef INIT_SETPROCTITLE_REPLACEMENT
spt_init(argc, argv);

31
src/redis.h
View File

@ -65,6 +65,13 @@ typedef long long mstime_t; /* millisecond time type. */
#include "util.h" /* Misc functions useful in many places */
#include "latency.h" /* Latency monitor API */
#include "sparkline.h" /* ASII graphs API */
#include "quicklist.h"
/* Following includes allow test functions to be called from Redis main() */
#include "zipmap.h"
#include "sha1.h"
#include "endianconv.h"
#include "crc64.h"
/* Error codes */
#define REDIS_OK 0
@ -198,6 +205,7 @@ typedef long long mstime_t; /* millisecond time type. */
#define REDIS_ENCODING_INTSET 6 /* Encoded as intset */
#define REDIS_ENCODING_SKIPLIST 7 /* Encoded as skiplist */
#define REDIS_ENCODING_EMBSTR 8 /* Embedded sds string encoding */
#define REDIS_ENCODING_QUICKLIST 9 /* Encoded as linked list of ziplists */
/* Defines related to the dump file format. To store 32 bits lengths for short
* keys requires a lot of space, so we check the most significant 2 bits of
@ -323,12 +331,14 @@ typedef long long mstime_t; /* millisecond time type. */
/* Zip structure related defaults */
#define REDIS_HASH_MAX_ZIPLIST_ENTRIES 512
#define REDIS_HASH_MAX_ZIPLIST_VALUE 64
#define REDIS_LIST_MAX_ZIPLIST_ENTRIES 512
#define REDIS_LIST_MAX_ZIPLIST_VALUE 64
#define REDIS_SET_MAX_INTSET_ENTRIES 512
#define REDIS_ZSET_MAX_ZIPLIST_ENTRIES 128
#define REDIS_ZSET_MAX_ZIPLIST_VALUE 64
/* List defaults */
#define REDIS_LIST_MAX_ZIPLIST_SIZE -2
#define REDIS_LIST_COMPRESS_DEPTH 0
/* HyperLogLog defines */
#define REDIS_DEFAULT_HLL_SPARSE_MAX_BYTES 3000
@ -863,12 +873,14 @@ struct redisServer {
/* Zip structure config, see redis.conf for more information */
size_t hash_max_ziplist_entries;
size_t hash_max_ziplist_value;
size_t list_max_ziplist_entries;
size_t list_max_ziplist_value;
size_t set_max_intset_entries;
size_t zset_max_ziplist_entries;
size_t zset_max_ziplist_value;
size_t hll_sparse_max_bytes;
/* List parameters */
int list_max_ziplist_size;
int list_compress_depth;
/* time cache */
time_t unixtime; /* Unix time sampled every cron cycle. */
long long mstime; /* Like 'unixtime' but with milliseconds resolution. */
/* Pubsub */
@ -958,15 +970,13 @@ typedef struct {
robj *subject;
unsigned char encoding;
unsigned char direction; /* Iteration direction */
unsigned char *zi;
listNode *ln;
quicklistIter *iter;
} listTypeIterator;
/* Structure for an entry while iterating over a list. */
typedef struct {
listTypeIterator *li;
unsigned char *zi; /* Entry in ziplist */
listNode *ln; /* Entry in linked list */
quicklistEntry entry; /* Entry in quicklist */
} listTypeEntry;
/* Structure to hold set iteration abstraction. */
@ -1043,6 +1053,7 @@ void addReplyBulkCBuffer(redisClient *c, void *p, size_t len);
void addReplyBulkLongLong(redisClient *c, long long ll);
void addReply(redisClient *c, robj *obj);
void addReplySds(redisClient *c, sds s);
void addReplyBulkSds(redisClient *c, sds s);
void addReplyError(redisClient *c, char *err);
void addReplyStatus(redisClient *c, char *status);
void addReplyDouble(redisClient *c, double d);
@ -1092,7 +1103,7 @@ int listTypeNext(listTypeIterator *li, listTypeEntry *entry);
robj *listTypeGet(listTypeEntry *entry);
void listTypeInsert(listTypeEntry *entry, robj *value, int where);
int listTypeEqual(listTypeEntry *entry, robj *o);
void listTypeDelete(listTypeEntry *entry);
void listTypeDelete(listTypeIterator *iter, listTypeEntry *entry);
void listTypeConvert(robj *subject, int enc);
void unblockClientWaitingData(redisClient *c);
void handleClientsBlockedOnLists(void);
@ -1130,7 +1141,7 @@ robj *getDecodedObject(robj *o);
size_t stringObjectLen(robj *o);
robj *createStringObjectFromLongLong(long long value);
robj *createStringObjectFromLongDouble(long double value, int humanfriendly);
robj *createListObject(void);
robj *createQuicklistObject(void);
robj *createZiplistObject(void);
robj *createSetObject(void);
robj *createIntsetObject(void);

26
src/sds.c
View File

@ -88,6 +88,17 @@ void sdsfree(sds s) {
zfree(s-sizeof(struct sdshdr));
}
/* Remove sds header so we can use buffer as malloc'd byte array.
* Returns the contents of 's' usable as a full malloc'd C string. */
char *sdsnative(sds s) {
if (!s) return NULL;
size_t len = sdslen(s);
char *base = s-sizeof(struct sdshdr);
memmove(base, s, len);
return zrealloc(base, len);
}
/* Set the sds string length to the length as obtained with strlen(), so
* considering as content only up to the first null term character.
*
@ -962,12 +973,15 @@ sds sdsjoin(char **argv, int argc, char *sep) {
return join;
}
#ifdef SDS_TEST_MAIN
#if defined(REDIS_TEST) || defined(SDS_TEST_MAIN)
#include <stdio.h>
#include "testhelp.h"
#include "limits.h"
int main(void) {
#define UNUSED(x) (void)(x)
int sdsTest(int argc, char *argv[]) {
UNUSED(argc);
UNUSED(argv);
{
struct sdshdr *sh;
sds x = sdsnew("foo"), y;
@ -1092,7 +1106,7 @@ int main(void) {
memcmp(y,"\"\\a\\n\\x00foo\\r\"",15) == 0)
{
int oldfree;
unsigned int oldfree;
sdsfree(x);
x = sdsnew("0");
@ -1113,3 +1127,9 @@ int main(void) {
return 0;
}
#endif
#ifdef SDS_TEST_MAIN
int main(void) {
return sdsTest();
}
#endif

5
src/sds.h
View File

@ -60,6 +60,7 @@ sds sdsempty(void);
size_t sdslen(const sds s);
sds sdsdup(const sds s);
void sdsfree(sds s);
char *sdsnative(sds s);
size_t sdsavail(const sds s);
sds sdsgrowzero(sds s, size_t len);
sds sdscatlen(sds s, const void *t, size_t len);
@ -98,4 +99,8 @@ void sdsIncrLen(sds s, int incr);
sds sdsRemoveFreeSpace(sds s);
size_t sdsAllocSize(sds s);
#ifdef REDIS_TEST
int sdsTest(int argc, char *argv[]);
#endif
#endif

7
src/sentinel.c
View File

@ -577,7 +577,7 @@ void sentinelEvent(int level, char *type, sentinelRedisInstance *ri,
if (level == REDIS_WARNING && ri != NULL) {
sentinelRedisInstance *master = (ri->flags & SRI_MASTER) ?
ri : ri->master;
if (master->notification_script) {
if (master && master->notification_script) {
sentinelScheduleScriptExecution(master->notification_script,
type,msg,NULL);
}
@ -2908,10 +2908,7 @@ void sentinelInfoCommand(redisClient *c) {
dictReleaseIterator(di);
}
addReplySds(c,sdscatprintf(sdsempty(),"$%lu\r\n",
(unsigned long)sdslen(info)));
addReplySds(c,info);
addReply(c,shared.crlf);
addReplyBulkSds(c, info);
}
/* Implements Sentinel verison of the ROLE command. The output is

11
src/sha1.c
View File

@ -199,16 +199,19 @@ void SHA1Final(unsigned char digest[20], SHA1_CTX* context)
}
/* ================ end of sha1.c ================ */
#if 0
#ifdef REDIS_TEST
#define BUFSIZE 4096
int
main(int argc, char **argv)
#define UNUSED(x) (void)(x)
int sha1Test(int argc, char **argv)
{
SHA1_CTX ctx;
unsigned char hash[20], buf[BUFSIZE];
int i;
UNUSED(argc);
UNUSED(argv);
for(i=0;i<BUFSIZE;i++)
buf[i] = i;
@ -223,6 +226,4 @@ main(int argc, char **argv)
printf("\n");
return 0;
}
#endif

7
src/sha1.h
View File

@ -1,3 +1,5 @@
#ifndef SHA1_H
#define SHA1_H
/* ================ sha1.h ================ */
/*
SHA-1 in C
@ -15,3 +17,8 @@ void SHA1Transform(u_int32_t state[5], const unsigned char buffer[64]);
void SHA1Init(SHA1_CTX* context);
void SHA1Update(SHA1_CTX* context, const unsigned char* data, u_int32_t len);
void SHA1Final(unsigned char digest[20], SHA1_CTX* context);
#ifdef REDIS_TEST
int sha1Test(int argc, char **argv);
#endif
#endif

5
src/sort.c
View File

@ -220,7 +220,7 @@ void sortCommand(redisClient *c) {
if (sortval)
incrRefCount(sortval);
else
sortval = createListObject();
sortval = createQuicklistObject();
/* The SORT command has an SQL-alike syntax, parse it */
while(j < c->argc) {
@ -420,6 +420,7 @@ void sortCommand(redisClient *c) {
} else {
redisPanic("Unknown type");
}
printf("j: %d; vectorlen: %d\n", j, vectorlen);
redisAssertWithInfo(c,sortval,j == vectorlen);
/* Now it's time to load the right scores in the sorting vector */
@ -509,7 +510,7 @@ void sortCommand(redisClient *c) {
}
}
} else {
robj *sobj = createZiplistObject();
robj *sobj = createQuicklistObject();
/* STORE option specified, set the sorting result as a List object */
for (j = start; j <= end; j++) {

358
src/t_list.c
View File

@ -33,75 +33,37 @@
* List API
*----------------------------------------------------------------------------*/
/* Check the argument length to see if it requires us to convert the ziplist
* to a real list. Only check raw-encoded objects because integer encoded
* objects are never too long. */
void listTypeTryConversion(robj *subject, robj *value) {
if (subject->encoding != REDIS_ENCODING_ZIPLIST) return;
if (sdsEncodedObject(value) &&
sdslen(value->ptr) > server.list_max_ziplist_value)
listTypeConvert(subject,REDIS_ENCODING_LINKEDLIST);
}
/* The function pushes an element to the specified list object 'subject',
* at head or tail position as specified by 'where'.
*
* There is no need for the caller to increment the refcount of 'value' as
* the function takes care of it if needed. */
void listTypePush(robj *subject, robj *value, int where) {
/* Check if we need to convert the ziplist */
listTypeTryConversion(subject,value);
if (subject->encoding == REDIS_ENCODING_ZIPLIST &&
ziplistLen(subject->ptr) >= server.list_max_ziplist_entries)
listTypeConvert(subject,REDIS_ENCODING_LINKEDLIST);
if (subject->encoding == REDIS_ENCODING_ZIPLIST) {
int pos = (where == REDIS_HEAD) ? ZIPLIST_HEAD : ZIPLIST_TAIL;
if (subject->encoding == REDIS_ENCODING_QUICKLIST) {
int pos = (where == REDIS_HEAD) ? QUICKLIST_HEAD : QUICKLIST_TAIL;
value = getDecodedObject(value);
subject->ptr = ziplistPush(subject->ptr,value->ptr,sdslen(value->ptr),pos);
size_t len = sdslen(value->ptr);
quicklistPush(subject->ptr, value->ptr, len, pos);
decrRefCount(value);
} else if (subject->encoding == REDIS_ENCODING_LINKEDLIST) {
if (where == REDIS_HEAD) {
listAddNodeHead(subject->ptr,value);
} else {
listAddNodeTail(subject->ptr,value);
}
incrRefCount(value);
} else {
redisPanic("Unknown list encoding");
}
}
void *listPopSaver(unsigned char *data, unsigned int sz) {
return createStringObject((char*)data,sz);
}
robj *listTypePop(robj *subject, int where) {
long long vlong;
robj *value = NULL;
if (subject->encoding == REDIS_ENCODING_ZIPLIST) {
unsigned char *p;
unsigned char *vstr;
unsigned int vlen;
long long vlong;
int pos = (where == REDIS_HEAD) ? 0 : -1;
p = ziplistIndex(subject->ptr,pos);
if (ziplistGet(p,&vstr,&vlen,&vlong)) {
if (vstr) {
value = createStringObject((char*)vstr,vlen);
} else {
int ql_where = where == REDIS_HEAD ? QUICKLIST_HEAD : QUICKLIST_TAIL;
if (subject->encoding == REDIS_ENCODING_QUICKLIST) {
if (quicklistPopCustom(subject->ptr, ql_where, (unsigned char **)&value,
NULL, &vlong, listPopSaver)) {
if (!value)
value = createStringObjectFromLongLong(vlong);
}
/* We only need to delete an element when it exists */
subject->ptr = ziplistDelete(subject->ptr,&p);
}
} else if (subject->encoding == REDIS_ENCODING_LINKEDLIST) {
list *list = subject->ptr;
listNode *ln;
if (where == REDIS_HEAD) {
ln = listFirst(list);
} else {
ln = listLast(list);
}
if (ln != NULL) {
value = listNodeValue(ln);
incrRefCount(value);
listDelNode(list,ln);
}
} else {
redisPanic("Unknown list encoding");
@ -110,25 +72,28 @@ robj *listTypePop(robj *subject, int where) {
}
unsigned long listTypeLength(robj *subject) {
if (subject->encoding == REDIS_ENCODING_ZIPLIST) {
return ziplistLen(subject->ptr);
} else if (subject->encoding == REDIS_ENCODING_LINKEDLIST) {
return listLength((list*)subject->ptr);
if (subject->encoding == REDIS_ENCODING_QUICKLIST) {
return quicklistCount(subject->ptr);
} else {
redisPanic("Unknown list encoding");
}
}
/* Initialize an iterator at the specified index. */
listTypeIterator *listTypeInitIterator(robj *subject, long index, unsigned char direction) {
listTypeIterator *listTypeInitIterator(robj *subject, long index,
unsigned char direction) {
listTypeIterator *li = zmalloc(sizeof(listTypeIterator));
li->subject = subject;
li->encoding = subject->encoding;
li->direction = direction;
if (li->encoding == REDIS_ENCODING_ZIPLIST) {
li->zi = ziplistIndex(subject->ptr,index);
} else if (li->encoding == REDIS_ENCODING_LINKEDLIST) {
li->ln = listIndex(subject->ptr,index);
li->iter = NULL;
/* REDIS_HEAD means start at TAIL and move *towards* head.
* REDIS_TAIL means start at HEAD and move *towards tail. */
int iter_direction =
direction == REDIS_HEAD ? AL_START_TAIL : AL_START_HEAD;
if (li->encoding == REDIS_ENCODING_QUICKLIST) {
li->iter = quicklistGetIteratorAtIdx(li->subject->ptr,
iter_direction, index);
} else {
redisPanic("Unknown list encoding");
}
@ -137,6 +102,7 @@ listTypeIterator *listTypeInitIterator(robj *subject, long index, unsigned char
/* Clean up the iterator. */
void listTypeReleaseIterator(listTypeIterator *li) {
zfree(li->iter);
zfree(li);
}
@ -148,24 +114,8 @@ int listTypeNext(listTypeIterator *li, listTypeEntry *entry) {
redisAssert(li->subject->encoding == li->encoding);
entry->li = li;
if (li->encoding == REDIS_ENCODING_ZIPLIST) {
entry->zi = li->zi;
if (entry->zi != NULL) {
if (li->direction == REDIS_TAIL)
li->zi = ziplistNext(li->subject->ptr,li->zi);
else
li->zi = ziplistPrev(li->subject->ptr,li->zi);
return 1;
}
} else if (li->encoding == REDIS_ENCODING_LINKEDLIST) {
entry->ln = li->ln;
if (entry->ln != NULL) {
if (li->direction == REDIS_TAIL)
li->ln = li->ln->next;
else
li->ln = li->ln->prev;
return 1;
}
if (li->encoding == REDIS_ENCODING_QUICKLIST) {
return quicklistNext(li->iter, &entry->entry);
} else {
redisPanic("Unknown list encoding");
}
@ -174,24 +124,14 @@ int listTypeNext(listTypeIterator *li, listTypeEntry *entry) {
/* Return entry or NULL at the current position of the iterator. */
robj *listTypeGet(listTypeEntry *entry) {
listTypeIterator *li = entry->li;
robj *value = NULL;
if (li->encoding == REDIS_ENCODING_ZIPLIST) {
unsigned char *vstr;
unsigned int vlen;
long long vlong;
redisAssert(entry->zi != NULL);
if (ziplistGet(entry->zi,&vstr,&vlen,&vlong)) {
if (vstr) {
value = createStringObject((char*)vstr,vlen);
} else {
value = createStringObjectFromLongLong(vlong);
}
if (entry->li->encoding == REDIS_ENCODING_QUICKLIST) {
if (entry->entry.value) {
value = createStringObject((char *)entry->entry.value,
entry->entry.sz);
} else {
value = createStringObjectFromLongLong(entry->entry.longval);
}
} else if (li->encoding == REDIS_ENCODING_LINKEDLIST) {
redisAssert(entry->ln != NULL);
value = listNodeValue(entry->ln);
incrRefCount(value);
} else {
redisPanic("Unknown list encoding");
}
@ -199,30 +139,18 @@ robj *listTypeGet(listTypeEntry *entry) {
}
void listTypeInsert(listTypeEntry *entry, robj *value, int where) {
robj *subject = entry->li->subject;
if (entry->li->encoding == REDIS_ENCODING_ZIPLIST) {
if (entry->li->encoding == REDIS_ENCODING_QUICKLIST) {
value = getDecodedObject(value);
sds str = value->ptr;
size_t len = sdslen(str);
if (where == REDIS_TAIL) {
unsigned char *next = ziplistNext(subject->ptr,entry->zi);
/* When we insert after the current element, but the current element
* is the tail of the list, we need to do a push. */
if (next == NULL) {
subject->ptr = ziplistPush(subject->ptr,value->ptr,sdslen(value->ptr),REDIS_TAIL);
} else {
subject->ptr = ziplistInsert(subject->ptr,next,value->ptr,sdslen(value->ptr));
}
} else {
subject->ptr = ziplistInsert(subject->ptr,entry->zi,value->ptr,sdslen(value->ptr));
quicklistInsertAfter((quicklist *)entry->entry.quicklist,
&entry->entry, str, len);
} else if (where == REDIS_HEAD) {
quicklistInsertBefore((quicklist *)entry->entry.quicklist,
&entry->entry, str, len);
}
decrRefCount(value);
} else if (entry->li->encoding == REDIS_ENCODING_LINKEDLIST) {
if (where == REDIS_TAIL) {
listInsertNode(subject->ptr,entry->ln,value,AL_START_TAIL);
} else {
listInsertNode(subject->ptr,entry->ln,value,AL_START_HEAD);
}
incrRefCount(value);
} else {
redisPanic("Unknown list encoding");
}
@ -230,59 +158,33 @@ void listTypeInsert(listTypeEntry *entry, robj *value, int where) {
/* Compare the given object with the entry at the current position. */
int listTypeEqual(listTypeEntry *entry, robj *o) {
listTypeIterator *li = entry->li;
if (li->encoding == REDIS_ENCODING_ZIPLIST) {
if (entry->li->encoding == REDIS_ENCODING_QUICKLIST) {
redisAssertWithInfo(NULL,o,sdsEncodedObject(o));
return ziplistCompare(entry->zi,o->ptr,sdslen(o->ptr));
} else if (li->encoding == REDIS_ENCODING_LINKEDLIST) {
return equalStringObjects(o,listNodeValue(entry->ln));
return quicklistCompare(entry->entry.zi,o->ptr,sdslen(o->ptr));
} else {
redisPanic("Unknown list encoding");
}
}
/* Delete the element pointed to. */
void listTypeDelete(listTypeEntry *entry) {
listTypeIterator *li = entry->li;
if (li->encoding == REDIS_ENCODING_ZIPLIST) {
unsigned char *p = entry->zi;
li->subject->ptr = ziplistDelete(li->subject->ptr,&p);
/* Update position of the iterator depending on the direction */
if (li->direction == REDIS_TAIL)
li->zi = p;
else
li->zi = ziplistPrev(li->subject->ptr,p);
} else if (entry->li->encoding == REDIS_ENCODING_LINKEDLIST) {
listNode *next;
if (li->direction == REDIS_TAIL)
next = entry->ln->next;
else
next = entry->ln->prev;
listDelNode(li->subject->ptr,entry->ln);
li->ln = next;
void listTypeDelete(listTypeIterator *iter, listTypeEntry *entry) {
if (entry->li->encoding == REDIS_ENCODING_QUICKLIST) {
quicklistDelEntry(iter->iter, &entry->entry);
} else {
redisPanic("Unknown list encoding");
}
}
/* Create a quicklist from a single ziplist */
void listTypeConvert(robj *subject, int enc) {
listTypeIterator *li;
listTypeEntry entry;
redisAssertWithInfo(NULL,subject,subject->type == REDIS_LIST);
redisAssertWithInfo(NULL,subject,subject->type==REDIS_LIST);
redisAssertWithInfo(NULL,subject,subject->encoding==REDIS_ENCODING_ZIPLIST);
if (enc == REDIS_ENCODING_LINKEDLIST) {
list *l = listCreate();
listSetFreeMethod(l,decrRefCountVoid);
/* listTypeGet returns a robj with incremented refcount */
li = listTypeInitIterator(subject,0,REDIS_TAIL);
while (listTypeNext(li,&entry)) listAddNodeTail(l,listTypeGet(&entry));
listTypeReleaseIterator(li);
subject->encoding = REDIS_ENCODING_LINKEDLIST;
zfree(subject->ptr);
subject->ptr = l;
if (enc == REDIS_ENCODING_QUICKLIST) {
size_t zlen = server.list_max_ziplist_size;
int depth = server.list_compress_depth;
subject->ptr = quicklistCreateFromZiplist(zlen, depth, subject->ptr);
subject->encoding = REDIS_ENCODING_QUICKLIST;
} else {
redisPanic("Unsupported list conversion");
}
@ -304,7 +206,9 @@ void pushGenericCommand(redisClient *c, int where) {
for (j = 2; j < c->argc; j++) {
c->argv[j] = tryObjectEncoding(c->argv[j]);
if (!lobj) {
lobj = createZiplistObject();
lobj = createQuicklistObject();
quicklistSetOptions(lobj->ptr, server.list_max_ziplist_size,
server.list_compress_depth);
dbAdd(c->db,c->argv[1],lobj);
}
listTypePush(lobj,c->argv[j],where);
@ -338,13 +242,6 @@ void pushxGenericCommand(redisClient *c, robj *refval, robj *val, int where) {
checkType(c,subject,REDIS_LIST)) return;
if (refval != NULL) {
/* We're not sure if this value can be inserted yet, but we cannot
* convert the list inside the iterator. We don't want to loop over
* the list twice (once to see if the value can be inserted and once
* to do the actual insert), so we assume this value can be inserted
* and convert the ziplist to a regular list if necessary. */
listTypeTryConversion(subject,val);
/* Seek refval from head to tail */
iter = listTypeInitIterator(subject,0,REDIS_TAIL);
while (listTypeNext(iter,&entry)) {
@ -357,10 +254,6 @@ void pushxGenericCommand(redisClient *c, robj *refval, robj *val, int where) {
listTypeReleaseIterator(iter);
if (inserted) {
/* Check if the length exceeds the ziplist length threshold. */
if (subject->encoding == REDIS_ENCODING_ZIPLIST &&
ziplistLen(subject->ptr) > server.list_max_ziplist_entries)
listTypeConvert(subject,REDIS_ENCODING_LINKEDLIST);
signalModifiedKey(c->db,c->argv[1]);
notifyKeyspaceEvent(REDIS_NOTIFY_LIST,"linsert",
c->argv[1],c->db->id);
@ -418,31 +311,19 @@ void lindexCommand(redisClient *c) {
if ((getLongFromObjectOrReply(c, c->argv[2], &index, NULL) != REDIS_OK))
return;
if (o->encoding == REDIS_ENCODING_ZIPLIST) {
unsigned char *p;
unsigned char *vstr;
unsigned int vlen;
long long vlong;
p = ziplistIndex(o->ptr,index);
if (ziplistGet(p,&vstr,&vlen,&vlong)) {
if (vstr) {
value = createStringObject((char*)vstr,vlen);
if (o->encoding == REDIS_ENCODING_QUICKLIST) {
quicklistEntry entry;
if (quicklistIndex(o->ptr, index, &entry)) {
if (entry.value) {
value = createStringObject((char*)entry.value,entry.sz);
} else {
value = createStringObjectFromLongLong(vlong);
value = createStringObjectFromLongLong(entry.longval);
}
addReplyBulk(c,value);
decrRefCount(value);
} else {
addReply(c,shared.nullbulk);
}
} else if (o->encoding == REDIS_ENCODING_LINKEDLIST) {
listNode *ln = listIndex(o->ptr,index);
if (ln != NULL) {
value = listNodeValue(ln);
addReplyBulk(c,value);
} else {
addReply(c,shared.nullbulk);
}
} else {
redisPanic("Unknown list encoding");
}
@ -452,35 +333,18 @@ void lsetCommand(redisClient *c) {
robj *o = lookupKeyWriteOrReply(c,c->argv[1],shared.nokeyerr);
if (o == NULL || checkType(c,o,REDIS_LIST)) return;
long index;
robj *value = (c->argv[3] = tryObjectEncoding(c->argv[3]));
robj *value = c->argv[3];
if ((getLongFromObjectOrReply(c, c->argv[2], &index, NULL) != REDIS_OK))
return;
listTypeTryConversion(o,value);
if (o->encoding == REDIS_ENCODING_ZIPLIST) {
unsigned char *p, *zl = o->ptr;
p = ziplistIndex(zl,index);
if (p == NULL) {
if (o->encoding == REDIS_ENCODING_QUICKLIST) {
quicklist *ql = o->ptr;
int replaced = quicklistReplaceAtIndex(ql, index,
value->ptr, sdslen(value->ptr));
if (!replaced) {
addReply(c,shared.outofrangeerr);
} else {
o->ptr = ziplistDelete(o->ptr,&p);
value = getDecodedObject(value);
o->ptr = ziplistInsert(o->ptr,p,value->ptr,sdslen(value->ptr));
decrRefCount(value);
addReply(c,shared.ok);
signalModifiedKey(c->db,c->argv[1]);
notifyKeyspaceEvent(REDIS_NOTIFY_LIST,"lset",c->argv[1],c->db->id);
server.dirty++;
}
} else if (o->encoding == REDIS_ENCODING_LINKEDLIST) {
listNode *ln = listIndex(o->ptr,index);
if (ln == NULL) {
addReply(c,shared.outofrangeerr);
} else {
decrRefCount((robj*)listNodeValue(ln));
listNodeValue(ln) = value;
incrRefCount(value);
addReply(c,shared.ok);
signalModifiedKey(c->db,c->argv[1]);
notifyKeyspaceEvent(REDIS_NOTIFY_LIST,"lset",c->argv[1],c->db->id);
@ -549,43 +413,28 @@ void lrangeCommand(redisClient *c) {
/* Return the result in form of a multi-bulk reply */
addReplyMultiBulkLen(c,rangelen);
if (o->encoding == REDIS_ENCODING_ZIPLIST) {
unsigned char *p = ziplistIndex(o->ptr,start);
unsigned char *vstr;
unsigned int vlen;
long long vlong;
if (o->encoding == REDIS_ENCODING_QUICKLIST) {
listTypeIterator *iter = listTypeInitIterator(o, start, REDIS_TAIL);
while(rangelen--) {
ziplistGet(p,&vstr,&vlen,&vlong);
if (vstr) {
addReplyBulkCBuffer(c,vstr,vlen);
listTypeEntry entry;
listTypeNext(iter, &entry);
quicklistEntry *qe = &entry.entry;
if (qe->value) {
addReplyBulkCBuffer(c,qe->value,qe->sz);
} else {
addReplyBulkLongLong(c,vlong);
addReplyBulkLongLong(c,qe->longval);
}
p = ziplistNext(o->ptr,p);
}
} else if (o->encoding == REDIS_ENCODING_LINKEDLIST) {
listNode *ln;
/* If we are nearest to the end of the list, reach the element
* starting from tail and going backward, as it is faster. */
if (start > llen/2) start -= llen;
ln = listIndex(o->ptr,start);
while(rangelen--) {
addReplyBulk(c,ln->value);
ln = ln->next;
}
listTypeReleaseIterator(iter);
} else {
redisPanic("List encoding is not LINKEDLIST nor ZIPLIST!");
redisPanic("List encoding is not QUICKLIST!");
}
}
void ltrimCommand(redisClient *c) {
robj *o;
long start, end, llen, j, ltrim, rtrim;
list *list;
listNode *ln;
long start, end, llen, ltrim, rtrim;
if ((getLongFromObjectOrReply(c, c->argv[2], &start, NULL) != REDIS_OK) ||
(getLongFromObjectOrReply(c, c->argv[3], &end, NULL) != REDIS_OK)) return;
@ -612,19 +461,9 @@ void ltrimCommand(redisClient *c) {
}
/* Remove list elements to perform the trim */
if (o->encoding == REDIS_ENCODING_ZIPLIST) {
o->ptr = ziplistDeleteRange(o->ptr,0,ltrim);
o->ptr = ziplistDeleteRange(o->ptr,-rtrim,rtrim);
} else if (o->encoding == REDIS_ENCODING_LINKEDLIST) {
list = o->ptr;
for (j = 0; j < ltrim; j++) {
ln = listFirst(list);
listDelNode(list,ln);
}
for (j = 0; j < rtrim; j++) {
ln = listLast(list);
listDelNode(list,ln);
}
if (o->encoding == REDIS_ENCODING_QUICKLIST) {
quicklistDelRange(o->ptr,0,ltrim);
quicklistDelRange(o->ptr,-rtrim,rtrim);
} else {
redisPanic("Unknown list encoding");
}
@ -641,10 +480,9 @@ void ltrimCommand(redisClient *c) {
void lremCommand(redisClient *c) {
robj *subject, *obj;
obj = c->argv[3] = tryObjectEncoding(c->argv[3]);
obj = c->argv[3];
long toremove;
long removed = 0;
listTypeEntry entry;
if ((getLongFromObjectOrReply(c, c->argv[2], &toremove, NULL) != REDIS_OK))
return;
@ -652,10 +490,6 @@ void lremCommand(redisClient *c) {
subject = lookupKeyWriteOrReply(c,c->argv[1],shared.czero);
if (subject == NULL || checkType(c,subject,REDIS_LIST)) return;
/* Make sure obj is raw when we're dealing with a ziplist */
if (subject->encoding == REDIS_ENCODING_ZIPLIST)
obj = getDecodedObject(obj);
listTypeIterator *li;
if (toremove < 0) {
toremove = -toremove;
@ -664,9 +498,10 @@ void lremCommand(redisClient *c) {
li = listTypeInitIterator(subject,0,REDIS_TAIL);
}
listTypeEntry entry;
while (listTypeNext(li,&entry)) {
if (listTypeEqual(&entry,obj)) {
listTypeDelete(&entry);
listTypeDelete(li, &entry);
server.dirty++;
removed++;
if (toremove && removed == toremove) break;
@ -674,11 +509,10 @@ void lremCommand(redisClient *c) {
}
listTypeReleaseIterator(li);
/* Clean up raw encoded object */
if (subject->encoding == REDIS_ENCODING_ZIPLIST)
decrRefCount(obj);
if (listTypeLength(subject) == 0) {
dbDelete(c->db,c->argv[1]);
}
if (listTypeLength(subject) == 0) dbDelete(c->db,c->argv[1]);
addReplyLongLong(c,removed);
if (removed) signalModifiedKey(c->db,c->argv[1]);
}
@ -702,7 +536,9 @@ void lremCommand(redisClient *c) {
void rpoplpushHandlePush(redisClient *c, robj *dstkey, robj *dstobj, robj *value) {
/* Create the list if the key does not exist */
if (!dstobj) {
dstobj = createZiplistObject();
dstobj = createQuicklistObject();
quicklistSetOptions(dstobj->ptr, server.list_max_ziplist_size,
server.list_compress_depth);
dbAdd(c->db,dstkey,dstobj);
}
signalModifiedKey(c->db,dstkey);
@ -1010,7 +846,9 @@ void handleClientsBlockedOnLists(void) {
}
}
if (listTypeLength(o) == 0) dbDelete(rl->db,rl->key);
if (listTypeLength(o) == 0) {
dbDelete(rl->db,rl->key);
}
/* We don't call signalModifiedKey() as it was already called
* when an element was pushed on the list. */
}

2
src/t_zset.c
View File

@ -1382,7 +1382,7 @@ void zremrangeGenericCommand(redisClient *c, int rangetype) {
robj *key = c->argv[1];
robj *zobj;
int keyremoved = 0;
unsigned long deleted;
unsigned long deleted = 0;
zrangespec range;
zlexrangespec lexrange;
long start, end, llen;

54
src/util.c
View File

@ -529,10 +529,10 @@ int pathIsBaseName(char *path) {
return strchr(path,'/') == NULL && strchr(path,'\\') == NULL;
}
#ifdef UTIL_TEST_MAIN
#ifdef REDIS_TEST
#include <assert.h>
void test_string2ll(void) {
static void test_string2ll(void) {
char buf[32];
long long v;
@ -587,7 +587,7 @@ void test_string2ll(void) {
assert(string2ll(buf,strlen(buf),&v) == 0);
}
void test_string2l(void) {
static void test_string2l(void) {
char buf[32];
long v;
@ -636,9 +636,55 @@ void test_string2l(void) {
#endif
}
int main(int argc, char **argv) {
static void test_ll2string(void) {
char buf[32];
long long v;
int sz;
v = 0;
sz = ll2string(buf, sizeof buf, v);
assert(sz == 1);
assert(!strcmp(buf, "0"));
v = -1;
sz = ll2string(buf, sizeof buf, v);
assert(sz == 2);
assert(!strcmp(buf, "-1"));
v = 99;
sz = ll2string(buf, sizeof buf, v);
assert(sz == 2);
assert(!strcmp(buf, "99"));
v = -99;
sz = ll2string(buf, sizeof buf, v);
assert(sz == 3);
assert(!strcmp(buf, "-99"));
v = -2147483648;
sz = ll2string(buf, sizeof buf, v);
assert(sz == 11);
assert(!strcmp(buf, "-2147483648"));
v = LLONG_MIN;
sz = ll2string(buf, sizeof buf, v);
assert(sz == 20);
assert(!strcmp(buf, "-9223372036854775808"));
v = LLONG_MAX;
sz = ll2string(buf, sizeof buf, v);
assert(sz == 19);
assert(!strcmp(buf, "9223372036854775807"));
}
#define UNUSED(x) (void)(x)
int utilTest(int argc, char **argv) {
UNUSED(argc);
UNUSED(argv);
test_string2ll();
test_string2l();
test_ll2string();
return 0;
}
#endif

4
src/util.h
View File

@ -42,4 +42,8 @@ int d2string(char *buf, size_t len, double value);
sds getAbsolutePath(char *filename);
int pathIsBaseName(char *path);
#ifdef REDIS_TEST
int utilTest(int argc, char **argv);
#endif
#endif

310
src/ziplist.c
View File

@ -143,6 +143,7 @@
#define ZIPLIST_TAIL_OFFSET(zl) (*((uint32_t*)((zl)+sizeof(uint32_t))))
#define ZIPLIST_LENGTH(zl) (*((uint16_t*)((zl)+sizeof(uint32_t)*2)))
#define ZIPLIST_HEADER_SIZE (sizeof(uint32_t)*2+sizeof(uint16_t))
#define ZIPLIST_END_SIZE (sizeof(uint8_t))
#define ZIPLIST_ENTRY_HEAD(zl) ((zl)+ZIPLIST_HEADER_SIZE)
#define ZIPLIST_ENTRY_TAIL(zl) ((zl)+intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl)))
#define ZIPLIST_ENTRY_END(zl) ((zl)+intrev32ifbe(ZIPLIST_BYTES(zl))-1)
@ -162,6 +163,13 @@ typedef struct zlentry {
unsigned char *p;
} zlentry;
#define ZIPLIST_ENTRY_ZERO(zle) { \
(zle)->prevrawlensize = (zle)->prevrawlen = 0; \
(zle)->lensize = (zle)->len = (zle)->headersize = 0; \
(zle)->encoding = 0; \
(zle)->p = NULL; \
}
/* Extract the encoding from the byte pointed by 'ptr' and set it into
* 'encoding'. */
#define ZIP_ENTRY_ENCODING(ptr, encoding) do { \
@ -169,6 +177,8 @@ typedef struct zlentry {
if ((encoding) < ZIP_STR_MASK) (encoding) &= ZIP_STR_MASK; \
} while(0)
void ziplistRepr(unsigned char *zl);
/* Return bytes needed to store integer encoded by 'encoding' */
static unsigned int zipIntSize(unsigned char encoding) {
switch(encoding) {
@ -404,14 +414,12 @@ static int64_t zipLoadInteger(unsigned char *p, unsigned char encoding) {
}
/* Return a struct with all information about an entry. */
static zlentry zipEntry(unsigned char *p) {
zlentry e;
static void zipEntry(unsigned char *p, zlentry *e) {
ZIP_DECODE_PREVLEN(p, e.prevrawlensize, e.prevrawlen);
ZIP_DECODE_LENGTH(p + e.prevrawlensize, e.encoding, e.lensize, e.len);
e.headersize = e.prevrawlensize + e.lensize;
e.p = p;
return e;
ZIP_DECODE_PREVLEN(p, e->prevrawlensize, e->prevrawlen);
ZIP_DECODE_LENGTH(p + e->prevrawlensize, e->encoding, e->lensize, e->len);
e->headersize = e->prevrawlensize + e->lensize;
e->p = p;
}
/* Create a new empty ziplist. */
@ -460,13 +468,13 @@ static unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p
zlentry cur, next;
while (p[0] != ZIP_END) {
cur = zipEntry(p);
zipEntry(p, &cur);
rawlen = cur.headersize + cur.len;
rawlensize = zipPrevEncodeLength(NULL,rawlen);
/* Abort if there is no next entry. */
if (p[rawlen] == ZIP_END) break;
next = zipEntry(p+rawlen);
zipEntry(p+rawlen, &next);
/* Abort when "prevlen" has not changed. */
if (next.prevrawlen == rawlen) break;
@ -521,7 +529,7 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsig
int nextdiff = 0;
zlentry first, tail;
first = zipEntry(p);
zipEntry(p, &first);
for (i = 0; p[0] != ZIP_END && i < num; i++) {
p += zipRawEntryLength(p);
deleted++;
@ -545,7 +553,7 @@ static unsigned char *__ziplistDelete(unsigned char *zl, unsigned char *p, unsig
/* When the tail contains more than one entry, we need to take
* "nextdiff" in account as well. Otherwise, a change in the
* size of prevlen doesn't have an effect on the *tail* offset. */
tail = zipEntry(p);
zipEntry(p, &tail);
if (p[tail.headersize+tail.len] != ZIP_END) {
ZIPLIST_TAIL_OFFSET(zl) =
intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+nextdiff);
@ -635,7 +643,7 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig
/* When the tail contains more than one entry, we need to take
* "nextdiff" in account as well. Otherwise, a change in the
* size of prevlen doesn't have an effect on the *tail* offset. */
tail = zipEntry(p+reqlen);
zipEntry(p+reqlen, &tail);
if (p[reqlen+tail.headersize+tail.len] != ZIP_END) {
ZIPLIST_TAIL_OFFSET(zl) =
intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+nextdiff);
@ -665,6 +673,121 @@ static unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsig
return zl;
}
/* Merge ziplists 'first' and 'second' by appending 'second' to 'first'.
*
* NOTE: The larger ziplist is reallocated to contain the new merged ziplist.
* Either 'first' or 'second' can be used for the result. The parameter not
* used will be free'd and set to NULL.
*
* After calling this function, the input parameters are no longer valid since
* they are changed and free'd in-place.
*
* The result ziplist is the contents of 'first' followed by 'second'.
*
* On failure: returns NULL if the merge is impossible.
* On success: returns the merged ziplist (which is expanded version of either
* 'first' or 'second', also frees the other unused input ziplist, and sets the
* input ziplist argument equal to newly reallocated ziplist return value. */
unsigned char *ziplistMerge(unsigned char **first, unsigned char **second) {
/* If any params are null, we can't merge, so NULL. */
if (first == NULL || *first == NULL || second == NULL || *second == NULL)
return NULL;
/* Can't merge same list into itself. */
if (*first == *second)
return NULL;
size_t first_bytes = intrev32ifbe(ZIPLIST_BYTES(*first));
size_t first_len = intrev16ifbe(ZIPLIST_LENGTH(*first));
size_t second_bytes = intrev32ifbe(ZIPLIST_BYTES(*second));
size_t second_len = intrev16ifbe(ZIPLIST_LENGTH(*second));
int append;
unsigned char *source, *target;
size_t target_bytes, source_bytes;
/* Pick the largest ziplist so we can resize easily in-place.
* We must also track if we are now appending or prepending to
* the target ziplist. */
if (first_len >= second_len) {
/* retain first, append second to first. */
target = *first;
target_bytes = first_bytes;
source = *second;
source_bytes = second_bytes;
append = 1;
} else {
/* else, retain second, prepend first to second. */
target = *second;
target_bytes = second_bytes;
source = *first;
source_bytes = first_bytes;
append = 0;
}
/* Calculate final bytes (subtract one pair of metadata) */
size_t zlbytes = first_bytes + second_bytes -
ZIPLIST_HEADER_SIZE - ZIPLIST_END_SIZE;
size_t zllength = first_len + second_len;
/* Combined zl length should be limited within UINT16_MAX */
zllength = zllength < UINT16_MAX ? zllength : UINT16_MAX;
/* Save offset positions before we start ripping memory apart. */
size_t first_offset = intrev32ifbe(ZIPLIST_TAIL_OFFSET(*first));
size_t second_offset = intrev32ifbe(ZIPLIST_TAIL_OFFSET(*second));
/* Extend target to new zlbytes then append or prepend source. */
target = zrealloc(target, zlbytes);
if (append) {
/* append == appending to target */
/* Copy source after target (copying over original [END]):
* [TARGET - END, SOURCE - HEADER] */
memcpy(target + target_bytes - ZIPLIST_END_SIZE,
source + ZIPLIST_HEADER_SIZE,
source_bytes - ZIPLIST_HEADER_SIZE);
} else {
/* !append == prepending to target */
/* Move target *contents* exactly size of (source - [END]),
* then copy source into vacataed space (source - [END]):
* [SOURCE - END, TARGET - HEADER] */
memmove(target + source_bytes - ZIPLIST_END_SIZE,
target + ZIPLIST_HEADER_SIZE,
target_bytes - ZIPLIST_HEADER_SIZE);
memcpy(target, source, source_bytes - ZIPLIST_END_SIZE);
}
/* Update header metadata. */
ZIPLIST_BYTES(target) = intrev32ifbe(zlbytes);
ZIPLIST_LENGTH(target) = intrev16ifbe(zllength);
/* New tail offset is:
* + N bytes of first ziplist
* - 1 byte for [END] of first ziplist
* + M bytes for the offset of the original tail of the second ziplist
* - J bytes for HEADER because second_offset keeps no header. */
ZIPLIST_TAIL_OFFSET(target) = intrev32ifbe(
(first_bytes - ZIPLIST_END_SIZE) +
(second_offset - ZIPLIST_HEADER_SIZE));
/* __ziplistCascadeUpdate just fixes the prev length values until it finds a
* correct prev length value (then it assumes the rest of the list is okay).
* We tell CascadeUpdate to start at the first ziplist's tail element to fix
* the merge seam. */
target = __ziplistCascadeUpdate(target, target+first_offset);
/* Now free and NULL out what we didn't realloc */
if (append) {
zfree(*second);
*second = NULL;
*first = target;
} else {
zfree(*first);
*first = NULL;
*second = target;
}
return target;
}
unsigned char *ziplistPush(unsigned char *zl, unsigned char *s, unsigned int slen, int where) {
unsigned char *p;
p = (where == ZIPLIST_HEAD) ? ZIPLIST_ENTRY_HEAD(zl) : ZIPLIST_ENTRY_END(zl);
@ -748,7 +871,7 @@ unsigned int ziplistGet(unsigned char *p, unsigned char **sstr, unsigned int *sl
if (p == NULL || p[0] == ZIP_END) return 0;
if (sstr) *sstr = NULL;
entry = zipEntry(p);
zipEntry(p, &entry);
if (ZIP_IS_STR(entry.encoding)) {
if (sstr) {
*slen = entry.len;
@ -783,7 +906,7 @@ unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p) {
}
/* Delete a range of entries from the ziplist. */
unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigned int num) {
unsigned char *ziplistDeleteRange(unsigned char *zl, int index, unsigned int num) {
unsigned char *p = ziplistIndex(zl,index);
return (p == NULL) ? zl : __ziplistDelete(zl,p,num);
}
@ -796,7 +919,7 @@ unsigned int ziplistCompare(unsigned char *p, unsigned char *sstr, unsigned int
long long zval, sval;
if (p[0] == ZIP_END) return 0;
entry = zipEntry(p);
zipEntry(p, &entry);
if (ZIP_IS_STR(entry.encoding)) {
/* Raw compare */
if (entry.len == slen) {
@ -913,7 +1036,7 @@ void ziplistRepr(unsigned char *zl) {
intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl)));
p = ZIPLIST_ENTRY_HEAD(zl);
while(*p != ZIP_END) {
entry = zipEntry(p);
zipEntry(p, &entry);
printf(
"{"
"addr 0x%08lx, "
@ -952,14 +1075,14 @@ void ziplistRepr(unsigned char *zl) {
printf("{end}\n\n");
}
#ifdef ZIPLIST_TEST_MAIN
#ifdef REDIS_TEST
#include <sys/time.h>
#include "adlist.h"
#include "sds.h"
#define debug(f, ...) { if (DEBUG) printf(f, __VA_ARGS__); }
unsigned char *createList() {
static unsigned char *createList() {
unsigned char *zl = ziplistNew();
zl = ziplistPush(zl, (unsigned char*)"foo", 3, ZIPLIST_TAIL);
zl = ziplistPush(zl, (unsigned char*)"quux", 4, ZIPLIST_TAIL);
@ -968,7 +1091,7 @@ unsigned char *createList() {
return zl;
}
unsigned char *createIntList() {
static unsigned char *createIntList() {
unsigned char *zl = ziplistNew();
char buf[32];
@ -987,13 +1110,13 @@ unsigned char *createIntList() {
return zl;
}
long long usec(void) {
static long long usec(void) {
struct timeval tv;
gettimeofday(&tv,NULL);
return (((long long)tv.tv_sec)*1000000)+tv.tv_usec;
}
void stress(int pos, int num, int maxsize, int dnum) {
static void stress(int pos, int num, int maxsize, int dnum) {
int i,j,k;
unsigned char *zl;
char posstr[2][5] = { "HEAD", "TAIL" };
@ -1016,7 +1139,7 @@ void stress(int pos, int num, int maxsize, int dnum) {
}
}
void pop(unsigned char *zl, int where) {
static unsigned char *pop(unsigned char *zl, int where) {
unsigned char *p, *vstr;
unsigned int vlen;
long long vlong;
@ -1028,20 +1151,22 @@ void pop(unsigned char *zl, int where) {
else
printf("Pop tail: ");
if (vstr)
if (vstr) {
if (vlen && fwrite(vstr,vlen,1,stdout) == 0) perror("fwrite");
else
}
else {
printf("%lld", vlong);
}
printf("\n");
ziplistDeleteRange(zl,-1,1);
return ziplistDelete(zl,&p);
} else {
printf("ERROR: Could not pop\n");
exit(1);
}
}
int randstring(char *target, unsigned int min, unsigned int max) {
static int randstring(char *target, unsigned int min, unsigned int max) {
int p = 0;
int len = min+rand()%(max-min+1);
int minval, maxval;
@ -1067,23 +1192,24 @@ int randstring(char *target, unsigned int min, unsigned int max) {
return len;
}
void verify(unsigned char *zl, zlentry *e) {
int i;
static void verify(unsigned char *zl, zlentry *e) {
int len = ziplistLen(zl);
zlentry _e;
for (i = 0; i < len; i++) {
ZIPLIST_ENTRY_ZERO(&_e);
for (int i = 0; i < len; i++) {
memset(&e[i], 0, sizeof(zlentry));
e[i] = zipEntry(ziplistIndex(zl, i));
zipEntry(ziplistIndex(zl, i), &e[i]);
memset(&_e, 0, sizeof(zlentry));
_e = zipEntry(ziplistIndex(zl, -len+i));
zipEntry(ziplistIndex(zl, -len+i), &_e);
assert(memcmp(&e[i], &_e, sizeof(zlentry)) == 0);
}
}
int main(int argc, char **argv) {
int ziplistTest(int argc, char **argv) {
unsigned char *zl, *p;
unsigned char *entry;
unsigned int elen;
@ -1096,21 +1222,25 @@ int main(int argc, char **argv) {
zl = createIntList();
ziplistRepr(zl);
zfree(zl);
zl = createList();
ziplistRepr(zl);
pop(zl,ZIPLIST_TAIL);
zl = pop(zl,ZIPLIST_TAIL);
ziplistRepr(zl);
pop(zl,ZIPLIST_HEAD);
zl = pop(zl,ZIPLIST_HEAD);
ziplistRepr(zl);
pop(zl,ZIPLIST_TAIL);
zl = pop(zl,ZIPLIST_TAIL);
ziplistRepr(zl);
pop(zl,ZIPLIST_TAIL);
zl = pop(zl,ZIPLIST_TAIL);
ziplistRepr(zl);
zfree(zl);
printf("Get element at index 3:\n");
{
zl = createList();
@ -1126,6 +1256,7 @@ int main(int argc, char **argv) {
printf("%lld\n", value);
}
printf("\n");
zfree(zl);
}
printf("Get element at index 4 (out of range):\n");
@ -1139,6 +1270,7 @@ int main(int argc, char **argv) {
return 1;
}
printf("\n");
zfree(zl);
}
printf("Get element at index -1 (last element):\n");
@ -1156,6 +1288,7 @@ int main(int argc, char **argv) {
printf("%lld\n", value);
}
printf("\n");
zfree(zl);
}
printf("Get element at index -4 (first element):\n");
@ -1173,6 +1306,7 @@ int main(int argc, char **argv) {
printf("%lld\n", value);
}
printf("\n");
zfree(zl);
}
printf("Get element at index -5 (reverse out of range):\n");
@ -1186,6 +1320,7 @@ int main(int argc, char **argv) {
return 1;
}
printf("\n");
zfree(zl);
}
printf("Iterate list from 0 to end:\n");
@ -1203,6 +1338,7 @@ int main(int argc, char **argv) {
printf("\n");
}
printf("\n");
zfree(zl);
}
printf("Iterate list from 1 to end:\n");
@ -1220,6 +1356,7 @@ int main(int argc, char **argv) {
printf("\n");
}
printf("\n");
zfree(zl);
}
printf("Iterate list from 2 to end:\n");
@ -1237,6 +1374,7 @@ int main(int argc, char **argv) {
printf("\n");
}
printf("\n");
zfree(zl);
}
printf("Iterate starting out of range:\n");
@ -1249,6 +1387,7 @@ int main(int argc, char **argv) {
printf("ERROR\n");
}
printf("\n");
zfree(zl);
}
printf("Iterate from back to front:\n");
@ -1266,6 +1405,7 @@ int main(int argc, char **argv) {
printf("\n");
}
printf("\n");
zfree(zl);
}
printf("Iterate from back to front, deleting all items:\n");
@ -1284,6 +1424,7 @@ int main(int argc, char **argv) {
printf("\n");
}
printf("\n");
zfree(zl);
}
printf("Delete inclusive range 0,0:\n");
@ -1291,6 +1432,7 @@ int main(int argc, char **argv) {
zl = createList();
zl = ziplistDeleteRange(zl, 0, 1);
ziplistRepr(zl);
zfree(zl);
}
printf("Delete inclusive range 0,1:\n");
@ -1298,6 +1440,7 @@ int main(int argc, char **argv) {
zl = createList();
zl = ziplistDeleteRange(zl, 0, 2);
ziplistRepr(zl);
zfree(zl);
}
printf("Delete inclusive range 1,2:\n");
@ -1305,6 +1448,7 @@ int main(int argc, char **argv) {
zl = createList();
zl = ziplistDeleteRange(zl, 1, 2);
ziplistRepr(zl);
zfree(zl);
}
printf("Delete with start index out of range:\n");
@ -1312,6 +1456,7 @@ int main(int argc, char **argv) {
zl = createList();
zl = ziplistDeleteRange(zl, 5, 1);
ziplistRepr(zl);
zfree(zl);
}
printf("Delete with num overflow:\n");
@ -1319,6 +1464,7 @@ int main(int argc, char **argv) {
zl = createList();
zl = ziplistDeleteRange(zl, 1, 5);
ziplistRepr(zl);
zfree(zl);
}
printf("Delete foo while iterating:\n");
@ -1343,11 +1489,12 @@ int main(int argc, char **argv) {
}
printf("\n");
ziplistRepr(zl);
zfree(zl);
}
printf("Regression test for >255 byte strings:\n");
{
char v1[257],v2[257];
char v1[257] = {0}, v2[257] = {0};
memset(v1,'x',256);
memset(v2,'y',256);
zl = ziplistNew();
@ -1362,13 +1509,15 @@ int main(int argc, char **argv) {
assert(ziplistGet(p,&entry,&elen,&value));
assert(strncmp(v2,(char*)entry,elen) == 0);
printf("SUCCESS\n\n");
zfree(zl);
}
printf("Regression test deleting next to last entries:\n");
{
char v[3][257];
zlentry e[3];
int i;
char v[3][257] = {{0}};
zlentry e[3] = {{.prevrawlensize = 0, .prevrawlen = 0, .lensize = 0,
.len = 0, .headersize = 0, .encoding = 0, .p = NULL}};
size_t i;
for (i = 0; i < (sizeof(v)/sizeof(v[0])); i++) {
memset(v[i], 'a' + i, sizeof(v[0]));
@ -1399,6 +1548,7 @@ int main(int argc, char **argv) {
assert(e[1].prevrawlensize == 5);
printf("SUCCESS\n\n");
zfree(zl);
}
printf("Create long list and check indices:\n");
@ -1420,6 +1570,7 @@ int main(int argc, char **argv) {
assert(999-i == value);
}
printf("SUCCESS\n\n");
zfree(zl);
}
printf("Compare strings with ziplist entries:\n");
@ -1445,6 +1596,82 @@ int main(int argc, char **argv) {
return 1;
}
printf("SUCCESS\n\n");
zfree(zl);
}
printf("Merge test:\n");
{
/* create list gives us: [hello, foo, quux, 1024] */
zl = createList();
unsigned char *zl2 = createList();
unsigned char *zl3 = ziplistNew();
unsigned char *zl4 = ziplistNew();
if (ziplistMerge(&zl4, &zl4)) {
printf("ERROR: Allowed merging of one ziplist into itself.\n");
return 1;
}
/* Merge two empty ziplists, get empty result back. */
zl4 = ziplistMerge(&zl3, &zl4);
ziplistRepr(zl4);
if (ziplistLen(zl4)) {
printf("ERROR: Merging two empty ziplists created entries.\n");
return 1;
}
zfree(zl4);
zl2 = ziplistMerge(&zl, &zl2);
/* merge gives us: [hello, foo, quux, 1024, hello, foo, quux, 1024] */
ziplistRepr(zl2);
if (ziplistLen(zl2) != 8) {
printf("ERROR: Merged length not 8, but: %u\n", ziplistLen(zl2));
return 1;
}
p = ziplistIndex(zl2,0);
if (!ziplistCompare(p,(unsigned char*)"hello",5)) {
printf("ERROR: not \"hello\"\n");
return 1;
}
if (ziplistCompare(p,(unsigned char*)"hella",5)) {
printf("ERROR: \"hella\"\n");
return 1;
}
p = ziplistIndex(zl2,3);
if (!ziplistCompare(p,(unsigned char*)"1024",4)) {
printf("ERROR: not \"1024\"\n");
return 1;
}
if (ziplistCompare(p,(unsigned char*)"1025",4)) {
printf("ERROR: \"1025\"\n");
return 1;
}
p = ziplistIndex(zl2,4);
if (!ziplistCompare(p,(unsigned char*)"hello",5)) {
printf("ERROR: not \"hello\"\n");
return 1;
}
if (ziplistCompare(p,(unsigned char*)"hella",5)) {
printf("ERROR: \"hella\"\n");
return 1;
}
p = ziplistIndex(zl2,7);
if (!ziplistCompare(p,(unsigned char*)"1024",4)) {
printf("ERROR: not \"1024\"\n");
return 1;
}
if (ziplistCompare(p,(unsigned char*)"1025",4)) {
printf("ERROR: \"1025\"\n");
return 1;
}
printf("SUCCESS\n\n");
zfree(zl);
}
printf("Stress with random payloads of different encoding:\n");
@ -1464,7 +1691,7 @@ int main(int argc, char **argv) {
for (i = 0; i < 20000; i++) {
zl = ziplistNew();
ref = listCreate();
listSetFreeMethod(ref,sdsfree);
listSetFreeMethod(ref,(void (*)(void*))sdsfree);
len = rand() % 256;
/* Create lists */
@ -1532,5 +1759,4 @@ int main(int argc, char **argv) {
return 0;
}
#endif

7
src/ziplist.h
View File

@ -32,6 +32,7 @@
#define ZIPLIST_TAIL 1
unsigned char *ziplistNew(void);
unsigned char *ziplistMerge(unsigned char **first, unsigned char **second);
unsigned char *ziplistPush(unsigned char *zl, unsigned char *s, unsigned int slen, int where);
unsigned char *ziplistIndex(unsigned char *zl, int index);
unsigned char *ziplistNext(unsigned char *zl, unsigned char *p);
@ -39,8 +40,12 @@ unsigned char *ziplistPrev(unsigned char *zl, unsigned char *p);
unsigned int ziplistGet(unsigned char *p, unsigned char **sval, unsigned int *slen, long long *lval);
unsigned char *ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen);
unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p);
unsigned char *ziplistDeleteRange(unsigned char *zl, unsigned int index, unsigned int num);
unsigned char *ziplistDeleteRange(unsigned char *zl, int index, unsigned int num);
unsigned int ziplistCompare(unsigned char *p, unsigned char *s, unsigned int slen);
unsigned char *ziplistFind(unsigned char *p, unsigned char *vstr, unsigned int vlen, unsigned int skip);
unsigned int ziplistLen(unsigned char *zl);
size_t ziplistBlobLen(unsigned char *zl);
#ifdef REDIS_TEST
int ziplistTest(int argc, char *argv[]);
#endif

10
src/zipmap.c
View File

@ -370,8 +370,8 @@ size_t zipmapBlobLen(unsigned char *zm) {
return totlen;
}
#ifdef ZIPMAP_TEST_MAIN
void zipmapRepr(unsigned char *p) {
#ifdef REDIS_TEST
static void zipmapRepr(unsigned char *p) {
unsigned int l;
printf("{status %u}",*p++);
@ -404,9 +404,13 @@ void zipmapRepr(unsigned char *p) {
printf("\n");
}
int main(void) {
#define UNUSED(x) (void)(x)
int zipmapTest(int argc, char *argv[]) {
unsigned char *zm;
UNUSED(argc);
UNUSED(argv);
zm = zipmapNew();
zm = zipmapSet(zm,(unsigned char*) "name",4, (unsigned char*) "foo",3,NULL);

4
src/zipmap.h
View File

@ -46,4 +46,8 @@ unsigned int zipmapLen(unsigned char *zm);
size_t zipmapBlobLen(unsigned char *zm);
void zipmapRepr(unsigned char *p);
#ifdef REDIS_TEST
int zipmapTest(int argc, char *argv[]);
#endif
#endif

7
tests/support/test.tcl
View File

@ -19,9 +19,12 @@ proc assert_match {pattern value} {
}
}
proc assert_equal {expected value} {
proc assert_equal {expected value {detail ""}} {
if {$expected ne $value} {
error "assertion:Expected '$value' to be equal to '$expected'"
if {$detail ne ""} {
set detail " (detail: $detail)"
}
error "assertion:Expected '$value' to be equal to '$expected'$detail"
}
}

4
tests/unit/aofrw.tcl
View File

@ -77,10 +77,10 @@ start_server {tags {"aofrw"}} {
}
foreach d {string int} {
foreach e {ziplist linkedlist} {
foreach e {quicklist} {
test "AOF rewrite of list with $e encoding, $d data" {
r flushall
if {$e eq {ziplist}} {set len 10} else {set len 1000}
set len 1000
for {set j 0} {$j < $len} {incr j} {
if {$d eq {string}} {
set data [randstring 0 16 alpha]

4
tests/unit/dump.tcl
View File

@ -157,7 +157,7 @@ start_server {tags {"dump"}} {
test {MIGRATE can correctly transfer large values} {
set first [srv 0 client]
r del key
for {set j 0} {$j < 5000} {incr j} {
for {set j 0} {$j < 40000} {incr j} {
r rpush key 1 2 3 4 5 6 7 8 9 10
r rpush key "item 1" "item 2" "item 3" "item 4" "item 5" \
"item 6" "item 7" "item 8" "item 9" "item 10"
@ -175,7 +175,7 @@ start_server {tags {"dump"}} {
assert {[$first exists key] == 0}
assert {[$second exists key] == 1}
assert {[$second ttl key] == -1}
assert {[$second llen key] == 5000*20}
assert {[$second llen key] == 40000*20}
}
}

13
tests/unit/sort.tcl
View File

@ -1,8 +1,7 @@
start_server {
tags {"sort"}
overrides {
"list-max-ziplist-value" 16
"list-max-ziplist-entries" 32
"list-max-ziplist-size" 32
"set-max-intset-entries" 32
}
} {
@ -36,9 +35,9 @@ start_server {
}
foreach {num cmd enc title} {
16 lpush ziplist "Ziplist"
1000 lpush linkedlist "Linked list"
10000 lpush linkedlist "Big Linked list"
16 lpush quicklist "Old Ziplist"
1000 lpush quicklist "Old Linked list"
10000 lpush quicklist "Old Big Linked list"
16 sadd intset "Intset"
1000 sadd hashtable "Hash table"
10000 sadd hashtable "Big Hash table"
@ -85,14 +84,14 @@ start_server {
r sort tosort BY weight_* store sort-res
assert_equal $result [r lrange sort-res 0 -1]
assert_equal 16 [r llen sort-res]
assert_encoding ziplist sort-res
assert_encoding quicklist sort-res
}
test "SORT BY hash field STORE" {
r sort tosort BY wobj_*->weight store sort-res
assert_equal $result [r lrange sort-res 0 -1]
assert_equal 16 [r llen sort-res]
assert_encoding ziplist sort-res
assert_encoding quicklist sort-res
}
test "SORT extracts STORE correctly" {

9
tests/unit/type/list-2.tcl
View File

@ -1,8 +1,7 @@
start_server {
tags {"list"}
overrides {
"list-max-ziplist-value" 16
"list-max-ziplist-entries" 256
"list-max-ziplist-size" 4
}
} {
source "tests/unit/type/list-common.tcl"
@ -28,14 +27,18 @@ start_server {
for {set i 0} {$i < 1000} {incr i} {
set min [expr {int(rand()*$startlen)}]
set max [expr {$min+int(rand()*$startlen)}]
set before_len [llength $mylist]
set before_len_r [r llen mylist]
set mylist [lrange $mylist $min $max]
r ltrim mylist $min $max
assert_equal $mylist [r lrange mylist 0 -1]
assert_equal $mylist [r lrange mylist 0 -1] "failed trim"
set starting [r llen mylist]
for {set j [r llen mylist]} {$j < $startlen} {incr j} {
set str [randomInt 9223372036854775807]
r rpush mylist $str
lappend mylist $str
assert_equal $mylist [r lrange mylist 0 -1] "failed append match"
}
}
}

3
tests/unit/type/list-3.tcl
View File

@ -1,8 +1,7 @@
start_server {
tags {list ziplist}
overrides {
"list-max-ziplist-value" 200000
"list-max-ziplist-entries" 256
"list-max-ziplist-size" 16
}
} {
test {Explicit regression for a list bug} {

169
tests/unit/type/list.tcl
View File

@ -1,25 +1,24 @@
start_server {
tags {"list"}
overrides {
"list-max-ziplist-value" 16
"list-max-ziplist-entries" 256
"list-max-ziplist-size" 5
}
} {
source "tests/unit/type/list-common.tcl"
test {LPUSH, RPUSH, LLENGTH, LINDEX, LPOP - ziplist} {
# first lpush then rpush
assert_equal 1 [r lpush myziplist1 a]
assert_equal 2 [r rpush myziplist1 b]
assert_equal 3 [r rpush myziplist1 c]
assert_equal 1 [r lpush myziplist1 aa]
assert_equal 2 [r rpush myziplist1 bb]
assert_equal 3 [r rpush myziplist1 cc]
assert_equal 3 [r llen myziplist1]
assert_equal a [r lindex myziplist1 0]
assert_equal b [r lindex myziplist1 1]
assert_equal c [r lindex myziplist1 2]
assert_equal aa [r lindex myziplist1 0]
assert_equal bb [r lindex myziplist1 1]
assert_equal cc [r lindex myziplist1 2]
assert_equal {} [r lindex myziplist2 3]
assert_equal c [r rpop myziplist1]
assert_equal a [r lpop myziplist1]
assert_encoding ziplist myziplist1
assert_equal cc [r rpop myziplist1]
assert_equal aa [r lpop myziplist1]
assert_encoding quicklist myziplist1
# first rpush then lpush
assert_equal 1 [r rpush myziplist2 a]
@ -32,13 +31,13 @@ start_server {
assert_equal {} [r lindex myziplist2 3]
assert_equal a [r rpop myziplist2]
assert_equal c [r lpop myziplist2]
assert_encoding ziplist myziplist2
assert_encoding quicklist myziplist2
}
test {LPUSH, RPUSH, LLENGTH, LINDEX, LPOP - regular list} {
# first lpush then rpush
assert_equal 1 [r lpush mylist1 $largevalue(linkedlist)]
assert_encoding linkedlist mylist1
assert_encoding quicklist mylist1
assert_equal 2 [r rpush mylist1 b]
assert_equal 3 [r rpush mylist1 c]
assert_equal 3 [r llen mylist1]
@ -51,7 +50,7 @@ start_server {
# first rpush then lpush
assert_equal 1 [r rpush mylist2 $largevalue(linkedlist)]
assert_encoding linkedlist mylist2
assert_encoding quicklist mylist2
assert_equal 2 [r lpush mylist2 b]
assert_equal 3 [r lpush mylist2 c]
assert_equal 3 [r llen mylist2]
@ -74,34 +73,22 @@ start_server {
assert_equal {d c b a 0 1 2 3} [r lrange mylist 0 -1]
}
test {DEL a list - ziplist} {
assert_equal 1 [r del myziplist2]
assert_equal 0 [r exists myziplist2]
assert_equal 0 [r llen myziplist2]
}
test {DEL a list - regular list} {
test {DEL a list} {
assert_equal 1 [r del mylist2]
assert_equal 0 [r exists mylist2]
assert_equal 0 [r llen mylist2]
}
proc create_ziplist {key entries} {
proc create_list {key entries} {
r del $key
foreach entry $entries { r rpush $key $entry }
assert_encoding ziplist $key
}
proc create_linkedlist {key entries} {
r del $key
foreach entry $entries { r rpush $key $entry }
assert_encoding linkedlist $key
assert_encoding quicklist $key
}
foreach {type large} [array get largevalue] {
test "BLPOP, BRPOP: single existing list - $type" {
set rd [redis_deferring_client]
create_$type blist "a b $large c d"
create_list blist "a b $large c d"
$rd blpop blist 1
assert_equal {blist a} [$rd read]
@ -116,8 +103,8 @@ start_server {
test "BLPOP, BRPOP: multiple existing lists - $type" {
set rd [redis_deferring_client]
create_$type blist1 "a $large c"
create_$type blist2 "d $large f"
create_list blist1 "a $large c"
create_list blist2 "d $large f"
$rd blpop blist1 blist2 1
assert_equal {blist1 a} [$rd read]
@ -137,7 +124,7 @@ start_server {
test "BLPOP, BRPOP: second list has an entry - $type" {
set rd [redis_deferring_client]
r del blist1
create_$type blist2 "d $large f"
create_list blist2 "d $large f"
$rd blpop blist1 blist2 1
assert_equal {blist2 d} [$rd read]
@ -151,7 +138,7 @@ start_server {
r del target
set rd [redis_deferring_client]
create_$type blist "a b $large c d"
create_list blist "a b $large c d"
$rd brpoplpush blist target 1
assert_equal d [$rd read]
@ -517,28 +504,28 @@ start_server {
foreach {type large} [array get largevalue] {
test "LPUSHX, RPUSHX - $type" {
create_$type xlist "$large c"
create_list xlist "$large c"
assert_equal 3 [r rpushx xlist d]
assert_equal 4 [r lpushx xlist a]
assert_equal "a $large c d" [r lrange xlist 0 -1]
}
test "LINSERT - $type" {
create_$type xlist "a $large c d"
assert_equal 5 [r linsert xlist before c zz]
assert_equal "a $large zz c d" [r lrange xlist 0 10]
assert_equal 6 [r linsert xlist after c yy]
assert_equal "a $large zz c yy d" [r lrange xlist 0 10]
assert_equal 7 [r linsert xlist after d dd]
assert_equal -1 [r linsert xlist after bad ddd]
assert_equal "a $large zz c yy d dd" [r lrange xlist 0 10]
assert_equal 8 [r linsert xlist before a aa]
assert_equal -1 [r linsert xlist before bad aaa]
assert_equal "aa a $large zz c yy d dd" [r lrange xlist 0 10]
create_list xlist "a $large c d"
assert_equal 5 [r linsert xlist before c zz] "before c"
assert_equal "a $large zz c d" [r lrange xlist 0 10] "lrangeA"
assert_equal 6 [r linsert xlist after c yy] "after c"
assert_equal "a $large zz c yy d" [r lrange xlist 0 10] "lrangeB"
assert_equal 7 [r linsert xlist after d dd] "after d"
assert_equal -1 [r linsert xlist after bad ddd] "after bad"
assert_equal "a $large zz c yy d dd" [r lrange xlist 0 10] "lrangeC"
assert_equal 8 [r linsert xlist before a aa] "before a"
assert_equal -1 [r linsert xlist before bad aaa] "before bad"
assert_equal "aa a $large zz c yy d dd" [r lrange xlist 0 10] "lrangeD"
# check inserting integer encoded value
assert_equal 9 [r linsert xlist before aa 42]
assert_equal 42 [r lrange xlist 0 0]
assert_equal 9 [r linsert xlist before aa 42] "before aa"
assert_equal 42 [r lrange xlist 0 0] "lrangeE"
}
}
@ -547,55 +534,7 @@ start_server {
set e
} {*ERR*syntax*error*}
test {LPUSHX, RPUSHX convert from ziplist to list} {
set large $largevalue(linkedlist)
# convert when a large value is pushed
create_ziplist xlist a
assert_equal 2 [r rpushx xlist $large]
assert_encoding linkedlist xlist
create_ziplist xlist a
assert_equal 2 [r lpushx xlist $large]
assert_encoding linkedlist xlist
# convert when the length threshold is exceeded
create_ziplist xlist [lrepeat 256 a]
assert_equal 257 [r rpushx xlist b]
assert_encoding linkedlist xlist
create_ziplist xlist [lrepeat 256 a]
assert_equal 257 [r lpushx xlist b]
assert_encoding linkedlist xlist
}
test {LINSERT convert from ziplist to list} {
set large $largevalue(linkedlist)
# convert when a large value is inserted
create_ziplist xlist a
assert_equal 2 [r linsert xlist before a $large]
assert_encoding linkedlist xlist
create_ziplist xlist a
assert_equal 2 [r linsert xlist after a $large]
assert_encoding linkedlist xlist
# convert when the length threshold is exceeded
create_ziplist xlist [lrepeat 256 a]
assert_equal 257 [r linsert xlist before a a]
assert_encoding linkedlist xlist
create_ziplist xlist [lrepeat 256 a]
assert_equal 257 [r linsert xlist after a a]
assert_encoding linkedlist xlist
# don't convert when the value could not be inserted
create_ziplist xlist [lrepeat 256 a]
assert_equal -1 [r linsert xlist before foo a]
assert_encoding ziplist xlist
create_ziplist xlist [lrepeat 256 a]
assert_equal -1 [r linsert xlist after foo a]
assert_encoding ziplist xlist
}
foreach {type num} {ziplist 250 linkedlist 500} {
foreach {type num} {quicklist 250 quicklist 500} {
proc check_numbered_list_consistency {key} {
set len [r llen $key]
for {set i 0} {$i < $len} {incr i} {
@ -664,16 +603,16 @@ start_server {
foreach {type large} [array get largevalue] {
test "RPOPLPUSH base case - $type" {
r del mylist1 mylist2
create_$type mylist1 "a $large c d"
create_list mylist1 "a $large c d"
assert_equal d [r rpoplpush mylist1 mylist2]
assert_equal c [r rpoplpush mylist1 mylist2]
assert_equal "a $large" [r lrange mylist1 0 -1]
assert_equal "c d" [r lrange mylist2 0 -1]
assert_encoding ziplist mylist2
assert_encoding quicklist mylist2
}
test "RPOPLPUSH with the same list as src and dst - $type" {
create_$type mylist "a $large c"
create_list mylist "a $large c"
assert_equal "a $large c" [r lrange mylist 0 -1]
assert_equal c [r rpoplpush mylist mylist]
assert_equal "c a $large" [r lrange mylist 0 -1]
@ -681,8 +620,8 @@ start_server {
foreach {othertype otherlarge} [array get largevalue] {
test "RPOPLPUSH with $type source and existing target $othertype" {
create_$type srclist "a b c $large"
create_$othertype dstlist "$otherlarge"
create_list srclist "a b c $large"
create_list dstlist "$otherlarge"
assert_equal $large [r rpoplpush srclist dstlist]
assert_equal c [r rpoplpush srclist dstlist]
assert_equal "a b" [r lrange srclist 0 -1]
@ -691,7 +630,7 @@ start_server {
# When we rpoplpush'ed a large value, dstlist should be
# converted to the same encoding as srclist.
if {$type eq "linkedlist"} {
assert_encoding linkedlist dstlist
assert_encoding quicklist dstlist
}
}
}
@ -713,7 +652,7 @@ start_server {
}
test {RPOPLPUSH against non list dst key} {
create_ziplist srclist {a b c d}
create_list srclist {a b c d}
r set dstlist x
assert_error WRONGTYPE* {r rpoplpush srclist dstlist}
assert_type string dstlist
@ -727,7 +666,7 @@ start_server {
foreach {type large} [array get largevalue] {
test "Basic LPOP/RPOP - $type" {
create_$type mylist "$large 1 2"
create_list mylist "$large 1 2"
assert_equal $large [r lpop mylist]
assert_equal 2 [r rpop mylist]
assert_equal 1 [r lpop mylist]
@ -745,7 +684,7 @@ start_server {
assert_error WRONGTYPE* {r rpop notalist}
}
foreach {type num} {ziplist 250 linkedlist 500} {
foreach {type num} {quicklist 250 quicklist 500} {
test "Mass RPOP/LPOP - $type" {
r del mylist
set sum1 0
@ -765,24 +704,24 @@ start_server {
foreach {type large} [array get largevalue] {
test "LRANGE basics - $type" {
create_$type mylist "$large 1 2 3 4 5 6 7 8 9"
create_list mylist "$large 1 2 3 4 5 6 7 8 9"
assert_equal {1 2 3 4 5 6 7 8} [r lrange mylist 1 -2]
assert_equal {7 8 9} [r lrange mylist -3 -1]
assert_equal {4} [r lrange mylist 4 4]
}
test "LRANGE inverted indexes - $type" {
create_$type mylist "$large 1 2 3 4 5 6 7 8 9"
create_list mylist "$large 1 2 3 4 5 6 7 8 9"
assert_equal {} [r lrange mylist 6 2]
}
test "LRANGE out of range indexes including the full list - $type" {
create_$type mylist "$large 1 2 3"
create_list mylist "$large 1 2 3"
assert_equal "$large 1 2 3" [r lrange mylist -1000 1000]
}
test "LRANGE out of range negative end index - $type" {
create_$type mylist "$large 1 2 3"
create_list mylist "$large 1 2 3"
assert_equal $large [r lrange mylist 0 -4]
assert_equal {} [r lrange mylist 0 -5]
}
@ -796,7 +735,7 @@ start_server {
proc trim_list {type min max} {
upvar 1 large large
r del mylist
create_$type mylist "1 2 3 4 $large"
create_list mylist "1 2 3 4 $large"
r ltrim mylist $min $max
r lrange mylist 0 -1
}
@ -825,7 +764,7 @@ start_server {
foreach {type large} [array get largevalue] {
test "LSET - $type" {
create_$type mylist "99 98 $large 96 95"
create_list mylist "99 98 $large 96 95"
r lset mylist 1 foo
r lset mylist -1 bar
assert_equal "99 foo $large 96 bar" [r lrange mylist 0 -1]
@ -847,7 +786,7 @@ start_server {
foreach {type e} [array get largevalue] {
test "LREM remove all the occurrences - $type" {
create_$type mylist "$e foo bar foobar foobared zap bar test foo"
create_list mylist "$e foo bar foobar foobared zap bar test foo"
assert_equal 2 [r lrem mylist 0 bar]
assert_equal "$e foo foobar foobared zap test foo" [r lrange mylist 0 -1]
}
@ -863,7 +802,7 @@ start_server {
}
test "LREM starting from tail with negative count - $type" {
create_$type mylist "$e foo bar foobar foobared zap bar test foo foo"
create_list mylist "$e foo bar foobar foobared zap bar test foo foo"
assert_equal 1 [r lrem mylist -1 bar]
assert_equal "$e foo bar foobar foobared zap test foo foo" [r lrange mylist 0 -1]
}
@ -874,7 +813,7 @@ start_server {
}
test "LREM deleting objects that may be int encoded - $type" {
create_$type myotherlist "$e 1 2 3"
create_list myotherlist "$e 1 2 3"
assert_equal 1 [r lrem myotherlist 1 2]
assert_equal 3 [r llen myotherlist]
}