diff --git a/src/cluster.c b/src/cluster.c index 6116614ea..ef73b1093 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -4981,6 +4981,9 @@ int verifyDumpPayload(unsigned char *p, size_t len) { rdbver = (footer[1] << 8) | footer[0]; if (rdbver > RDB_VERSION) return C_ERR; + if (server.skip_checksum_validation) + return C_OK; + /* Verify CRC64 */ crc = crc64(0,p,len-8); memrev64ifbe(&crc); diff --git a/src/db.c b/src/db.c index 45865b3e8..12e1c2bbe 100644 --- a/src/db.c +++ b/src/db.c @@ -960,6 +960,7 @@ void scanGenericCommand(client *c, robj *o, unsigned long cursor) { * value, or skip it if it was not filtered: we only match keys. */ if (o && (o->type == OBJ_ZSET || o->type == OBJ_HASH)) { node = nextnode; + serverAssert(node); /* assertion for valgrind (avoid NPD) */ nextnode = listNextNode(node); if (filter) { kobj = listNodeValue(node); diff --git a/src/debug.c b/src/debug.c index d32a8ddce..7cc3c15a3 100644 --- a/src/debug.c +++ b/src/debug.c @@ -403,6 +403,7 @@ void debugCommand(client *c) { "SDSLEN -- Show low level SDS string info representing key and value.", "SEGFAULT -- Crash the server with sigsegv.", "SET-ACTIVE-EXPIRE <0|1> -- Setting it to 0 disables expiring keys in background when they are not accessed (otherwise the Redis behavior). Setting it to 1 reenables back the default.", +"SET-SKIP-CHECKSUM-VALIDATION <0|1> -- Enables or disables checksum checks for rdb or RESTORE payload.", "AOF-FLUSH-SLEEP -- Server will sleep before flushing the AOF, this is used for testing", "SLEEP -- Stop the server for . Decimals allowed.", "STRUCTSIZE -- Return the size of different Redis core C structures.", @@ -722,6 +723,11 @@ NULL { server.active_expire_enabled = atoi(c->argv[2]->ptr); addReply(c,shared.ok); + } else if (!strcasecmp(c->argv[1]->ptr,"set-skip-checksum-validation") && + c->argc == 3) + { + server.skip_checksum_validation = atoi(c->argv[2]->ptr); + addReply(c,shared.ok); } else if (!strcasecmp(c->argv[1]->ptr,"aof-flush-sleep") && c->argc == 3) { @@ -880,6 +886,14 @@ void serverLogObjectDebugInfo(const robj *o) { serverLog(LL_WARNING,"Object type: %d", o->type); serverLog(LL_WARNING,"Object encoding: %d", o->encoding); serverLog(LL_WARNING,"Object refcount: %d", o->refcount); +#if UNSAFE_CRASH_REPORT + /* This code is now disabled. o->ptr may be unreliable to print. in some + * cases a ziplist could have already been freed by realloc, but not yet + * updated to o->ptr. in other cases the call to ziplistLen may need to + * iterate on all the items in the list (and possibly crash again). + * For some cases it may be ok to crash here again, but these could cause + * invalid memory access which will bother valgrind and also possibly cause + * random memory portion to be "leaked" into the logfile. */ if (o->type == OBJ_STRING && sdsEncodedObject(o)) { serverLog(LL_WARNING,"Object raw string len: %zu", sdslen(o->ptr)); if (sdslen(o->ptr) < 4096) { @@ -900,6 +914,7 @@ void serverLogObjectDebugInfo(const robj *o) { } else if (o->type == OBJ_STREAM) { serverLog(LL_WARNING,"Stream size: %d", (int) streamLength(o)); } +#endif } void _serverAssertPrintObject(const robj *o) { diff --git a/src/intset.c b/src/intset.c index 0079199cf..964a41187 100644 --- a/src/intset.c +++ b/src/intset.c @@ -34,6 +34,7 @@ #include "intset.h" #include "zmalloc.h" #include "endianconv.h" +#include "redisassert.h" /* Note that these encodings are ordered, so: * INTSET_ENC_INT16 < INTSET_ENC_INT32 < INTSET_ENC_INT64. */ @@ -258,7 +259,9 @@ uint8_t intsetFind(intset *is, int64_t value) { /* Return random member */ int64_t intsetRandom(intset *is) { - return _intsetGet(is,rand()%intrev32ifbe(is->length)); + uint32_t len = intrev32ifbe(is->length); + assert(len); /* avoid division by zero on corrupt intset payload. */ + return _intsetGet(is,rand()%len); } /* Get the value at the given position. When this position is diff --git a/src/listpack.c b/src/listpack.c index 580954075..b403a1200 100644 --- a/src/listpack.c +++ b/src/listpack.c @@ -536,6 +536,7 @@ unsigned char *lpGet(unsigned char *p, int64_t *count, unsigned char *intbuf) { int64_t val; uint64_t uval, negstart, negmax; + assert(p); /* assertion for valgrind (avoid NPD) */ if (LP_ENCODING_IS_7BIT_UINT(p[0])) { negstart = UINT64_MAX; /* 7 bit ints are always positive. */ negmax = 0; diff --git a/src/lzf_d.c b/src/lzf_d.c index 5f55d471d..dbf1ae772 100644 --- a/src/lzf_d.c +++ b/src/lzf_d.c @@ -65,9 +65,10 @@ lzf_decompress (const void *const in_data, unsigned int in_len, u8 const *const in_end = ip + in_len; u8 *const out_end = op + out_len; - do + while (ip < in_end) { - unsigned int ctrl = *ip++; + unsigned int ctrl; + ctrl = *ip++; if (ctrl < (1 << 5)) /* literal run */ { @@ -182,7 +183,6 @@ lzf_decompress (const void *const in_data, unsigned int in_len, #endif } } - while (ip < in_end); return op - (u8 *)out_data; } diff --git a/src/networking.c b/src/networking.c index 626b117a1..82b6428ac 100644 --- a/src/networking.c +++ b/src/networking.c @@ -674,6 +674,7 @@ void addReplyLongLong(client *c, long long ll) { } void addReplyAggregateLen(client *c, long length, int prefix) { + serverAssert(length >= 0); if (prefix == '*' && length < OBJ_SHARED_BULKHDR_LEN) addReply(c,shared.mbulkhdr[length]); else diff --git a/src/quicklist.c b/src/quicklist.c index 0916d1f1b..8d68aa9ae 100644 --- a/src/quicklist.c +++ b/src/quicklist.c @@ -34,6 +34,7 @@ #include "ziplist.h" #include "util.h" /* for ll2string */ #include "lzf.h" +#include "redisassert.h" #if defined(REDIS_TEST) || defined(REDIS_TEST_VERBOSE) #include /* for printf (debug printing), snprintf (genstr) */ @@ -1289,7 +1290,8 @@ int quicklistIndex(const quicklist *quicklist, const long long idx, quicklistDecompressNodeForUse(entry->node); entry->zi = ziplistIndex(entry->node->zl, entry->offset); - ziplistGet(entry->zi, &entry->value, &entry->sz, &entry->longval); + if (!ziplistGet(entry->zi, &entry->value, &entry->sz, &entry->longval)) + assert(0); /* This can happen on corrupt ziplist with fake entry count. */ /* The caller will use our result, so we don't re-compress here. * The caller can recompress or delete the node as needed. */ return 1; diff --git a/src/rdb.c b/src/rdb.c index af527f94c..e88cabb5d 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -399,8 +399,9 @@ void *rdbLoadLzfStringObject(rio *rdb, int flags, size_t *lenptr) { /* Load the compressed representation and uncompress it to target. */ if (rioRead(rdb,c,clen) == 0) goto err; - if (lzf_decompress(c,clen,val,len) == 0) { + if (lzf_decompress(c,clen,val,len) != len) { rdbExitReportCorruptRDB("Invalid LZF compressed string"); + goto err; } zfree(c); @@ -504,6 +505,8 @@ void *rdbGenericLoadStringObject(rio *rdb, int flags, size_t *lenptr) { unsigned long long len; len = rdbLoadLen(rdb,&isencoded); + if (len == RDB_LENERR) return NULL; + if (isencoded) { switch(len) { case RDB_ENC_INT8: @@ -518,7 +521,6 @@ void *rdbGenericLoadStringObject(rio *rdb, int flags, size_t *lenptr) { } } - if (len == RDB_LENERR) return NULL; if (plain || sds) { void *buf = plain ? zmalloc(len) : sdsnewlen(SDS_NOINIT,len); if (lenptr) *lenptr = len; @@ -604,7 +606,7 @@ int rdbLoadDoubleValue(rio *rdb, double *val) { default: if (rioRead(rdb,buf,len) == 0) return -1; buf[len] = '\0'; - sscanf(buf, "%lg", val); + if (sscanf(buf, "%lg", val)!=1) return -1; return 0; } } @@ -1572,7 +1574,12 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { /* This will also be called when the set was just converted * to a regular hash table encoded set. */ if (o->encoding == OBJ_ENCODING_HT) { - dictAdd((dict*)o->ptr,sdsele,NULL); + if (dictAdd((dict*)o->ptr,sdsele,NULL) != DICT_OK) { + rdbExitReportCorruptRDB("Duplicate set members detected"); + decrRefCount(o); + sdsfree(sdsele); + return NULL; + } } else { sdsfree(sdsele); } @@ -1693,7 +1700,11 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { /* Add pair to hash table */ ret = dictAdd((dict*)o->ptr, field, value); if (ret == DICT_ERR) { - rdbExitReportCorruptRDB("Duplicate keys detected"); + rdbExitReportCorruptRDB("Duplicate hash fields detected"); + sdsfree(value); + sdsfree(field); + decrRefCount(o); + return NULL; } } @@ -1843,6 +1854,9 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { if (sdslen(nodekey) != sizeof(streamID)) { rdbExitReportCorruptRDB("Stream node key entry is not the " "size of a stream ID"); + sdsfree(nodekey); + decrRefCount(o); + return NULL; } /* Load the listpack. */ @@ -1870,14 +1884,22 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { * deletion we should remove the radix tree key if the * resulting listpack is empty. */ rdbExitReportCorruptRDB("Empty listpack inside stream"); + sdsfree(nodekey); + decrRefCount(o); + zfree(lp); + return NULL; } /* Insert the key in the radix tree. */ int retval = raxInsert(s->rax, (unsigned char*)nodekey,sizeof(streamID),lp,NULL); sdsfree(nodekey); - if (!retval) + if (!retval) { rdbExitReportCorruptRDB("Listpack re-added with existing key"); + decrRefCount(o); + zfree(lp); + return NULL; + } } /* Load total number of items inside the stream. */ s->length = rdbLoadLen(rdb,NULL); @@ -1922,9 +1944,13 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { } streamCG *cgroup = streamCreateCG(s,cgname,sdslen(cgname),&cg_id); - if (cgroup == NULL) + if (cgroup == NULL) { rdbExitReportCorruptRDB("Duplicated consumer group name %s", cgname); + decrRefCount(o); + sdsfree(cgname); + return NULL; + } sdsfree(cgname); /* Load the global PEL for this consumer group, however we'll @@ -1954,9 +1980,13 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { streamFreeNACK(nack); return NULL; } - if (!raxInsert(cgroup->pel,rawid,sizeof(rawid),nack,NULL)) + if (!raxInsert(cgroup->pel,rawid,sizeof(rawid),nack,NULL)) { rdbExitReportCorruptRDB("Duplicated global PEL entry " "loading stream consumer group"); + decrRefCount(o); + streamFreeNACK(nack); + return NULL; + } } /* Now that we loaded our global PEL, we need to load the @@ -2003,18 +2033,24 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { return NULL; } streamNACK *nack = raxFind(cgroup->pel,rawid,sizeof(rawid)); - if (nack == raxNotFound) + if (nack == raxNotFound) { rdbExitReportCorruptRDB("Consumer entry not found in " "group global PEL"); + decrRefCount(o); + return NULL; + } /* Set the NACK consumer, that was left to NULL when * loading the global PEL. Then set the same shared * NACK structure also in the consumer-specific PEL. */ nack->consumer = consumer; - if (!raxInsert(consumer->pel,rawid,sizeof(rawid),nack,NULL)) + if (!raxInsert(consumer->pel,rawid,sizeof(rawid),nack,NULL)) { rdbExitReportCorruptRDB("Duplicated consumer PEL entry " " loading a stream consumer " "group"); + decrRefCount(o); + return NULL; + } } } } @@ -2034,8 +2070,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { if (mt == NULL) { moduleTypeNameByID(name,moduleid); - serverLog(LL_WARNING,"The RDB file contains module data I can't load: no matching module '%s'", name); - exit(1); + rdbExitReportCorruptRDB("The RDB file contains module data I can't load: no matching module '%s'", name); + return NULL; } RedisModuleIO io; robj keyobj; @@ -2054,20 +2090,26 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { if (io.ver == 2) { uint64_t eof = rdbLoadLen(rdb,NULL); if (eof == RDB_LENERR) { - o = createModuleObject(mt,ptr); /* creating just in order to easily destroy */ - decrRefCount(o); + if (ptr) { + o = createModuleObject(mt,ptr); /* creating just in order to easily destroy */ + decrRefCount(o); + } return NULL; } if (eof != RDB_MODULE_OPCODE_EOF) { - serverLog(LL_WARNING,"The RDB file contains module data for the module '%s' that is not terminated by the proper module value EOF marker", name); - exit(1); + rdbExitReportCorruptRDB("The RDB file contains module data for the module '%s' that is not terminated by the proper module value EOF marker", name); + if (ptr) { + o = createModuleObject(mt,ptr); /* creating just in order to easily destroy */ + decrRefCount(o); + } + return NULL; } } if (ptr == NULL) { moduleTypeNameByID(name,moduleid); - serverLog(LL_WARNING,"The RDB file contains module data for the module type '%s', that the responsible module is not able to load. Check for modules log above for additional clues.", name); - exit(1); + rdbExitReportCorruptRDB("The RDB file contains module data for the module type '%s', that the responsible module is not able to load. Check for modules log above for additional clues.", name); + return NULL; } o = createModuleObject(mt,ptr); } else { @@ -2441,7 +2483,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { uint64_t cksum, expected = rdb->cksum; if (rioRead(rdb,&cksum,8) == 0) goto eoferr; - if (server.rdb_checksum) { + if (server.rdb_checksum && !server.skip_checksum_validation) { memrev64ifbe(&cksum); if (cksum == 0) { serverLog(LL_WARNING,"RDB file was saved with checksum disabled: no check performed."); diff --git a/src/server.c b/src/server.c index 1b6f71aef..0c3bbff03 100644 --- a/src/server.c +++ b/src/server.c @@ -2506,6 +2506,7 @@ void initServerConfig(void) { server.tlsfd_count = 0; server.sofd = -1; server.active_expire_enabled = 1; + server.skip_checksum_validation = 0; server.client_max_querybuf_len = PROTO_MAX_QUERYBUF_LEN; server.saveparams = NULL; server.loading = 0; diff --git a/src/server.h b/src/server.h index 8a35d48df..47f054b3f 100644 --- a/src/server.h +++ b/src/server.h @@ -1245,6 +1245,7 @@ struct redisServer { int active_expire_effort; /* From 1 (default) to 10, active effort. */ int active_defrag_enabled; int sanitize_dump_payload; /* Enables deep sanitization for ziplist and listpack in RDB and RESTORE. */ + int skip_checksum_validation; /* Disables checksum validateion for RDB and RESTORE payload. */ int jemalloc_bg_thread; /* Enable jemalloc background thread */ size_t active_defrag_ignore_bytes; /* minimum amount of fragmentation waste to start active defrag */ int active_defrag_threshold_lower; /* minimum percentage of fragmentation to start active defrag */ diff --git a/src/t_list.c b/src/t_list.c index acabe7be2..42b4f92df 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -728,6 +728,7 @@ void lmoveGenericCommand(client *c, int wherefrom, int whereto) { if (checkType(c,dobj,OBJ_LIST)) return; value = listTypePop(sobj,wherefrom); + serverAssert(value); /* assertion for valgrind (avoid NPD) */ /* We saved touched key, and protect it, since lmoveHandlePush * may change the client command argument vector (it does not * currently). */ diff --git a/src/t_stream.c b/src/t_stream.c index 70cc8bfa1..320c5a2fe 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -787,6 +787,7 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields) { *numfields = lpGetInteger(si->lp_ele); si->lp_ele = lpNext(si->lp,si->lp_ele); } + serverAssert(*numfields>=0); /* If current >= start, and the entry is not marked as * deleted, emit it. */ @@ -2879,6 +2880,7 @@ void xinfoReplyWithStreamInfo(client *c, stream *s) { addReplyStreamID(c,&id); /* Consumer name. */ + serverAssert(nack->consumer); /* assertion for valgrind (avoid NPD) */ addReplyBulkCBuffer(c,nack->consumer->name, sdslen(nack->consumer->name)); diff --git a/src/valgrind.sup b/src/valgrind.sup index b05843d8c..5d6367e3f 100644 --- a/src/valgrind.sup +++ b/src/valgrind.sup @@ -15,3 +15,12 @@ Memcheck:Value8 fun:lzf_compress } + +{ + + Memcheck:FishyValue + malloc(size) + fun:malloc + fun:ztrymalloc_usable + fun:ztrymalloc +} diff --git a/tests/instances.tcl b/tests/instances.tcl index d3b1b50cd..156c92706 100644 --- a/tests/instances.tcl +++ b/tests/instances.tcl @@ -155,7 +155,7 @@ proc log_crashes {} { set logs [glob */err.txt] foreach log $logs { - set res [find_valgrind_errors $log] + set res [find_valgrind_errors $log true] if {$res != ""} { puts $res incr ::failed diff --git a/tests/integration/corrupt-dump-fuzzer.tcl b/tests/integration/corrupt-dump-fuzzer.tcl new file mode 100644 index 000000000..d41ed2941 --- /dev/null +++ b/tests/integration/corrupt-dump-fuzzer.tcl @@ -0,0 +1,193 @@ +# tests of corrupt ziplist payload with valid CRC + +tags {"dump" "corruption"} { + +proc generate_collections {suffix elements} { + set rd [redis_deferring_client] + for {set j 0} {$j < $elements} {incr j} { + # add both string values and integers + if {$j % 2 == 0} {set val $j} else {set val "_$j"} + $rd hset hash$suffix $j $val + $rd lpush list$suffix $val + $rd zadd zset$suffix $j $val + $rd sadd set$suffix $val + $rd xadd stream$suffix * item 1 value $val + } + for {set j 0} {$j < $elements * 5} {incr j} { + $rd read ; # Discard replies + } + $rd close +} + +# generate keys with various types and encodings +proc generate_types {} { + r config set list-max-ziplist-size 5 + r config set hash-max-ziplist-entries 5 + r config set zset-max-ziplist-entries 5 + r config set stream-node-max-entries 5 + + # create small (ziplist / listpack encoded) objects with 3 items + generate_collections "" 3 + + # add some metadata to the stream + r xgroup create stream mygroup 0 + set records [r xreadgroup GROUP mygroup Alice COUNT 2 STREAMS stream >] + r xack stream mygroup [lindex [lindex [lindex [lindex $records 0] 1] 0] 0] + + # create other non-collection types + r incr int + r set string str + + # create bigger objects with 10 items (more than a single ziplist / listpack) + generate_collections big 10 + + # make sure our big stream also has a listpack record that has different + # field names than the master recored + r xadd streambig * item 1 value 1 + r xadd streambig * item 1 unique value +} + +proc corrupt_payload {payload} { + set len [string length $payload] + set count 1 ;# usually corrupt only one byte + if {rand() > 0.9} { set count 2 } + while { $count > 0 } { + set idx [expr {int(rand() * $len)}] + set ch [binary format c [expr {int(rand()*255)}]] + set payload [string replace $payload $idx $idx $ch] + incr count -1 + } + return $payload +} + +# fuzzy tester for corrupt RESTORE payloads +# valgrind will make sure there were no leaks in the rdb loader error handling code +foreach sanitize_dump {no yes} { + if {$::accurate} { + set min_duration [expr {60 * 10}] ;# run at least 10 minutes + set min_cycles 1000 ;# run at least 1k cycles (max 16 minutes) + } else { + set min_duration 10 ; # run at least 10 seconds + set min_cycles 10 ; # run at least 10 cycles + } + + test "Fuzzer corrupt restore payloads - sanitize_dump: $sanitize_dump" { + if {$min_duration * 2 > $::timeout} { + fail "insufficient timeout" + } + # start a server, fill with data and save an RDB file once (avoid re-save) + start_server [list overrides [list "save" "" use-exit-on-panic yes crash-memcheck-enabled no loglevel verbose] ] { + set stdout [srv 0 stdout] + r config set sanitize-dump-payload $sanitize_dump + r debug set-skip-checksum-validation 1 + set start_time [clock seconds] + generate_types + r save + set cycle 0 + set stat_terminated_in_restore 0 + set stat_terminated_in_traffic 0 + set stat_terminated_by_signal 0 + set stat_successful_restore 0 + set stat_rejected_restore 0 + set stat_traffic_commands_sent 0 + # repeatedly DUMP a random key, corrupt it and try RESTORE into a new key + while true { + set k [r randomkey] + set dump [r dump $k] + set dump [corrupt_payload $dump] + set printable_dump [string2printable $dump] + set restore_failed false + set report_and_restart false + set sent {} + # RESTORE can fail, but hopefully not terminate + if { [catch { r restore "_$k" 0 $dump REPLACE } err] } { + set restore_failed true + # skip if return failed with an error response. + if {[string match "ERR*" $err]} { + incr stat_rejected_restore + } else { + set report_and_restart true + incr stat_terminated_in_restore + write_log_line 0 "corrupt payload: $printable_dump" + if {$sanitize_dump == 1} { + puts "Server crashed in RESTORE with payload: $printable_dump" + } + } + } else { + r ping ;# an attempt to check if the server didn't terminate (this will throw an error that will terminate the tests) + } + + set print_commands false + if {!$restore_failed} { + # if RESTORE didn't fail or terminate, run some random traffic on the new key + incr stat_successful_restore + if { [ catch { + set sent [generate_fuzzy_traffic_on_key "_$k" 1] ;# traffic for 1 second + incr stat_traffic_commands_sent [llength $sent] + r del "_$k" ;# in case the server terminated, here's where we'll detect it. + } err ] } { + # if the server terminated update stats and restart it + set report_and_restart true + incr stat_terminated_in_traffic + set by_signal [count_log_message 0 "crashed by signal"] + incr stat_terminated_by_signal $by_signal + + if {$by_signal != 0 || $sanitize_dump == 1 } { + puts "Server crashed (by signal: $by_signal), with payload: $printable_dump" + set print_commands true + } + } + } + + # check valgrind report for invalid reads after each RESTORE + # payload so that we have a report that is easier to reproduce + set valgrind_errors [find_valgrind_errors [srv 0 stderr] false] + if {$valgrind_errors != ""} { + puts "valgrind found an issue for payload: $printable_dump" + set report_and_restart true + set print_commands true + } + + if {$report_and_restart} { + if {$print_commands} { + puts "violating commands:" + foreach cmd $sent { + foreach arg $cmd { + puts -nonewline "[string2printable $arg] " + } + puts "" + } + } + + # restart the server and re-apply debug configuration + write_log_line 0 "corrupt payload: $printable_dump" + restart_server 0 true true + r config set sanitize-dump-payload $sanitize_dump + r debug set-skip-checksum-validation 1 + } + + incr cycle + if { ([clock seconds]-$start_time) >= $min_duration && $cycle >= $min_cycles} { + break + } + } + if {$::verbose} { + puts "Done $cycle cycles in [expr {[clock seconds]-$start_time}] seconds." + puts "RESTORE: successful: $stat_successful_restore, rejected: $stat_rejected_restore" + puts "Total commands sent in traffic: $stat_traffic_commands_sent, crashes during traffic: $stat_terminated_in_traffic ($stat_terminated_by_signal by signal)." + } + } + # if we run sanitization we never expect the server to crash at runtime + if { $sanitize_dump == 1} { + assert_equal $stat_terminated_in_restore 0 + assert_equal $stat_terminated_in_traffic 0 + } + # make sure all terminations where due to assertion and not a SIGSEGV + assert_equal $stat_terminated_by_signal 0 + } +} + + + +} ;# tags + diff --git a/tests/integration/corrupt-dump.tcl b/tests/integration/corrupt-dump.tcl index 9864036d2..b8828fd94 100644 --- a/tests/integration/corrupt-dump.tcl +++ b/tests/integration/corrupt-dump.tcl @@ -1,4 +1,10 @@ # tests of corrupt ziplist payload with valid CRC +# * setting crash-memcheck-enabled to no to avoid issues with valgrind +# * setting use-exit-on-panic to yes so that valgrind can search for leaks +# * settng debug set-skip-checksum-validation to 1 on some tests for which we +# didn't bother to fake a valid checksum +# * some tests set sanitize-dump-payload to no and some to yet, depending on +# what we want to test tags {"dump" "corruption"} { @@ -18,27 +24,27 @@ test {corrupt payload: #7445 - with sanitize} { test {corrupt payload: #7445 - without sanitize - 1} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { r config set sanitize-dump-payload no - r config set crash-memcheck-enabled no ;# avoid valgrind issues r restore key 0 $corrupt_payload_7445 catch {r lindex key 2} - verify_log_message 0 "*ASSERTION FAILED*" 0 + assert_equal [count_log_message 0 "crashed by signal"] 0 + assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 } } test {corrupt payload: #7445 - without sanitize - 2} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { r config set sanitize-dump-payload no - r config set crash-memcheck-enabled no ;# avoid valgrind issues r restore key 0 $corrupt_payload_7445 catch {r lset key 2 "BEEF"} - verify_log_message 0 "*ASSERTION FAILED*" 0 + assert_equal [count_log_message 0 "crashed by signal"] 0 + assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 } } test {corrupt payload: hash with valid zip list header, invalid entry len} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no r restore key 0 "\x0D\x1B\x1B\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x02\x61\x00\x04\x02\x62\x00\x04\x14\x63\x00\x04\x02\x64\x00\xFF\x09\x00\xD9\x10\x54\x92\x15\xF5\x5F\x52" - r config set crash-memcheck-enabled no ;# avoid valgrind issues r config set hash-max-ziplist-entries 1 catch {r hset key b b} verify_log_message 0 "*zipEntrySafe*" 0 @@ -47,6 +53,7 @@ test {corrupt payload: hash with valid zip list header, invalid entry len} { test {corrupt payload: invalid zlbytes header} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no catch { r restore key 0 "\x0D\x1B\x25\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x02\x61\x00\x04\x02\x62\x00\x04\x02\x63\x00\x04\x02\x64\x00\xFF\x09\x00\xB7\xF7\x6E\x9F\x43\x43\x14\xC6" } err @@ -56,8 +63,8 @@ test {corrupt payload: invalid zlbytes header} { test {corrupt payload: valid zipped hash header, dup records} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no r restore key 0 "\x0D\x1B\x1B\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x02\x61\x00\x04\x02\x62\x00\x04\x02\x61\x00\x04\x02\x64\x00\xFF\x09\x00\xA1\x98\x36\x78\xCC\x8E\x93\x2E" - r config set crash-memcheck-enabled no ;# avoid valgrind issues r config set hash-max-ziplist-entries 1 # cause an assertion when converting to hash table catch {r hset key b b} @@ -67,10 +74,11 @@ test {corrupt payload: valid zipped hash header, dup records} { test {corrupt payload: quicklist big ziplist prev len} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no r restore key 0 "\x0E\x01\x13\x13\x00\x00\x00\x0E\x00\x00\x00\x02\x00\x00\x02\x61\x00\x0E\x02\x62\x00\xFF\x09\x00\x49\x97\x30\xB2\x0D\xA1\xED\xAA" - r config set crash-memcheck-enabled no ;# avoid valgrind issues catch {r lindex key -2} - verify_log_message 0 "*ASSERTION FAILED*" 0 + assert_equal [count_log_message 0 "crashed by signal"] 0 + assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 } } @@ -87,19 +95,24 @@ test {corrupt payload: quicklist small ziplist prev len} { test {corrupt payload: quicklist ziplist wrong count} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no r restore key 0 "\x0E\x01\x13\x13\x00\x00\x00\x0E\x00\x00\x00\x03\x00\x00\x02\x61\x00\x04\x02\x62\x00\xFF\x09\x00\x4D\xE2\x0A\x2F\x08\x25\xDF\x91" - r lpush key a - # check that the server didn't crash - r ping + # we'll be able to push, but iterating on the list will assert + r lpush key header + r rpush key footer + catch { [r lrange key -1 -1] } + assert_equal [count_log_message 0 "crashed by signal"] 0 + assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 } } test {corrupt payload: #3080 - quicklist} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no catch { r RESTORE key 0 "\x0E\x01\x80\x00\x00\x00\x10\x41\x41\x41\x41\x41\x41\x41\x41\x02\x00\x00\x80\x41\x41\x41\x41\x07\x00\x03\xC7\x1D\xEF\x54\x68\xCC\xF3" - r DUMP key - } + r DUMP key ;# DUMP was used in the original issue, but now even with shallow sanitization restore safely fails, so this is dead code + } err assert_match "*Bad data format*" $err verify_log_message 0 "*Ziplist integrity check failed*" 0 } @@ -107,9 +120,11 @@ test {corrupt payload: #3080 - quicklist} { test {corrupt payload: #3080 - ziplist} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + # shallow sanitization is enough for restore to safely reject the payload with wrong size + r config set sanitize-dump-payload no catch { r RESTORE key 0 "\x0A\x80\x00\x00\x00\x10\x41\x41\x41\x41\x41\x41\x41\x41\x02\x00\x00\x80\x41\x41\x41\x41\x07\x00\x39\x5B\x49\xE0\xC1\xC6\xDD\x76" - } + } err assert_match "*Bad data format*" $err verify_log_message 0 "*Ziplist integrity check failed*" 0 } @@ -118,7 +133,7 @@ test {corrupt payload: #3080 - ziplist} { test {corrupt payload: load corrupted rdb with no CRC - #3505} { set server_path [tmpdir "server.rdb-corruption-test"] exec cp tests/assets/corrupt_ziplist.rdb $server_path - set srv [start_server [list overrides [list "dir" $server_path "dbfilename" "corrupt_ziplist.rdb" loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no]]] + set srv [start_server [list overrides [list "dir" $server_path "dbfilename" "corrupt_ziplist.rdb" loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no sanitize-dump-payload no]]] # wait for termination wait_for_condition 100 50 { @@ -135,6 +150,7 @@ test {corrupt payload: load corrupted rdb with no CRC - #3505} { test {corrupt payload: listpack invalid size header} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no catch { r restore key 0 "\x0F\x01\x10\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x02\x40\x55\x5F\x00\x00\x00\x0F\x00\x01\x01\x00\x01\x02\x01\x88\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x00\x01\x00\x01\x00\x01\x00\x01\x02\x02\x88\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x61\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x88\x62\x00\x00\x00\x00\x00\x00\x00\x09\x08\x01\xFF\x0A\x01\x00\x00\x09\x00\x45\x91\x0A\x87\x2F\xA5\xF9\x2E" } err @@ -145,21 +161,25 @@ test {corrupt payload: listpack invalid size header} { test {corrupt payload: listpack too long entry len} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no r restore key 0 "\x0F\x01\x10\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x02\x40\x55\x55\x00\x00\x00\x0F\x00\x01\x01\x00\x01\x02\x01\x88\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x00\x01\x00\x01\x00\x01\x00\x01\x02\x02\x89\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x61\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x88\x62\x00\x00\x00\x00\x00\x00\x00\x09\x08\x01\xFF\x0A\x01\x00\x00\x09\x00\x40\x63\xC9\x37\x03\xA2\xE5\x68" catch { r xinfo stream key full } err - verify_log_message 0 "*ASSERTION FAILED*" 0 + assert_equal [count_log_message 0 "crashed by signal"] 0 + assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 } } test {corrupt payload: listpack very long entry len} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no r restore key 0 "\x0F\x01\x10\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x02\x40\x55\x55\x00\x00\x00\x0F\x00\x01\x01\x00\x01\x02\x01\x88\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x00\x01\x00\x01\x00\x01\x00\x01\x02\x02\x88\x31\x00\x00\x00\x00\x00\x00\x00\x09\x88\x61\x00\x00\x00\x00\x00\x00\x00\x09\x88\x32\x00\x00\x00\x00\x00\x00\x00\x09\x9C\x62\x00\x00\x00\x00\x00\x00\x00\x09\x08\x01\xFF\x0A\x01\x00\x00\x09\x00\x63\x6F\x42\x8E\x7C\xB5\xA2\x9D" catch { r xinfo stream key full } err - verify_log_message 0 "*ASSERTION FAILED*" 0 + assert_equal [count_log_message 0 "crashed by signal"] 0 + assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 } } @@ -174,5 +194,247 @@ test {corrupt payload: listpack too long entry prev len} { } } +test {corrupt payload: fuzzer findings - NPD in streamIteratorGetID} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no + r debug set-skip-checksum-validation 1 + catch { + r RESTORE key 0 "\x0F\x01\x10\x00\x00\x01\x73\xBD\x68\x48\x71\x00\x00\x00\x00\x00\x00\x00\x00\x40\x42\x42\x00\x00\x00\x18\x00\x03\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x00\x01\x02\x01\x00\x01\x00\x01\x01\x01\x00\x01\x05\x01\x02\x01\x00\x01\x01\x01\x01\x01\x82\x5F\x31\x03\x05\x01\x02\x01\x00\x01\x02\x01\x01\x01\x02\x01\x48\x01\xFF\x03\x81\x00\x00\x01\x73\xBD\x68\x48\x71\x02\x01\x07\x6D\x79\x67\x72\x6F\x75\x70\x81\x00\x00\x01\x73\xBD\x68\x48\x71\x00\x01\x00\x00\x01\x73\xBD\x68\x48\x71\x00\x00\x00\x00\x00\x00\x00\x00\x72\x48\x68\xBD\x73\x01\x00\x00\x01\x01\x05\x41\x6C\x69\x63\x65\x72\x48\x68\xBD\x73\x01\x00\x00\x01\x00\x00\x01\x73\xBD\x68\x48\x71\x00\x00\x00\x00\x00\x00\x00\x00\x09\x00\x80\xCD\xB0\xD5\x1A\xCE\xFF\x10" + r XREVRANGE key 725 233 + } + assert_equal [count_log_message 0 "crashed by signal"] 0 + assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 + } +} + +test {corrupt payload: fuzzer findings - listpack NPD on invalid stream} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no + r debug set-skip-checksum-validation 1 + catch { + r RESTORE _stream 0 "\x0F\x01\x10\x00\x00\x01\x73\xDC\xB6\x6B\xF1\x00\x00\x00\x00\x00\x00\x00\x00\x40\x42\x42\x00\x00\x00\x18\x00\x03\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x00\x01\x02\x01\x00\x01\x00\x01\x01\x01\x00\x01\x05\x01\x02\x01\x1F\x01\x00\x01\x01\x01\x6D\x5F\x31\x03\x05\x01\x02\x01\x29\x01\x00\x01\x01\x01\x02\x01\x05\x01\xFF\x03\x81\x00\x00\x01\x73\xDC\xB6\x6C\x1A\x00\x01\x07\x6D\x79\x67\x72\x6F\x75\x70\x81\x00\x00\x01\x73\xDC\xB6\x6B\xF1\x00\x01\x00\x00\x01\x73\xDC\xB6\x6B\xF1\x00\x00\x00\x00\x00\x00\x00\x00\x4B\x6C\xB6\xDC\x73\x01\x00\x00\x01\x01\x05\x41\x6C\x69\x63\x65\x3D\x6C\xB6\xDC\x73\x01\x00\x00\x01\x00\x00\x01\x73\xDC\xB6\x6B\xF1\x00\x00\x00\x00\x00\x00\x00\x00\x09\x00\xC7\x7D\x1C\xD7\x04\xFF\xE6\x9D" + r XREAD STREAMS _stream 519389898758 + } + assert_equal [count_log_message 0 "crashed by signal"] 0 + assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 + } +} + +test {corrupt payload: fuzzer findings - NPD in quicklistIndex} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no + r debug set-skip-checksum-validation 1 + catch { + r RESTORE key 0 "\x0E\x01\x13\x13\x00\x00\x00\x10\x00\x00\x00\x03\x12\x00\xF3\x02\x02\x5F\x31\x04\xF1\xFF\x09\x00\xC9\x4B\x31\xFE\x61\xC0\x96\xFE" + r LSET key 290 290 + } + assert_equal [count_log_message 0 "crashed by signal"] 0 + assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 + } +} + +test {corrupt payload: fuzzer findings - invalid read in ziplistFind} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no + r debug set-skip-checksum-validation 1 + catch { + r RESTORE key 0 "\x0D\x19\x19\x00\x00\x00\x16\x00\x00\x00\x06\x00\x00\xF1\x02\xF1\x02\xF2\x02\x02\x5F\x31\x04\x99\x02\xF3\xFF\x09\x00\xC5\xB8\x10\xC0\x8A\xF9\x16\xDF" + r HEXISTS key -688319650333 + } + assert_equal [count_log_message 0 "crashed by signal"] 0 + assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 + } +} + + +test {corrupt payload: fuzzer findings - invalid ziplist encoding} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload yes + r debug set-skip-checksum-validation 1 + catch { + r RESTORE _listbig 0 "\x0E\x02\x1B\x1B\x00\x00\x00\x16\x00\x00\x00\x05\x00\x00\x02\x5F\x39\x04\xF9\x02\x86\x5F\x37\x04\xF7\x02\x02\x5F\x35\xFF\x19\x19\x00\x00\x00\x16\x00\x00\x00\x05\x00\x00\xF5\x02\x02\x5F\x33\x04\xF3\x02\x02\x5F\x31\x04\xF1\xFF\x09\x00\x0C\xFC\x99\x2C\x23\x45\x15\x60" + } err + assert_match "*Bad data format*" $err + verify_log_message 0 "*Ziplist integrity check failed*" 0 + } +} + +test {corrupt payload: fuzzer findings - hash crash} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload yes + r debug set-skip-checksum-validation 1 + r RESTORE _hash 0 "\x0D\x19\x19\x00\x00\x00\x16\x00\x00\x00\x06\x00\x00\xF1\x02\xF1\x02\xF2\x02\x02\x5F\x31\x04\xF3\x02\xF3\xFF\x09\x00\x38\xB8\x10\xC0\x8A\xF9\x16\xDF" + r HSET _hash 394891450 1635910264 + r HMGET _hash 887312884855 + } +} + +test {corrupt payload: fuzzer findings - uneven entry count in hash} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no + r debug set-skip-checksum-validation 1 + r RESTORE _hashbig 0 "\x0D\x3D\x3D\x00\x00\x00\x38\x00\x00\x00\x14\x00\x00\xF2\x02\x02\x5F\x31\x04\x1C\x02\xF7\x02\xF1\x02\xF1\x02\xF5\x02\xF5\x02\xF4\x02\x02\x5F\x33\x04\xF6\x02\x02\x5F\x35\x04\xF8\x02\x02\x5F\x37\x04\xF9\x02\xF9\x02\xF3\x02\xF3\x02\xFA\x02\x02\x5F\x39\xFF\x09\x00\x73\xB7\x68\xC8\x97\x24\x8E\x88" + catch { r HSCAN _hashbig -250 } + assert_equal [count_log_message 0 "crashed by signal"] 0 + assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 + } +} + +test {corrupt payload: fuzzer findings - invalid read in lzf_decompress} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no + r debug set-skip-checksum-validation 1 + catch { r RESTORE _setbig 0 "\x02\x03\x02\x5F\x31\xC0\x02\xC3\x00\x09\x00\xE6\xDC\x76\x44\xFF\xEB\x3D\xFE" } err + assert_match "*Bad data format*" $err + } +} + +test {corrupt payload: fuzzer findings - leak in rdbloading due to dup entry in set} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no + r debug set-skip-checksum-validation 1 + catch { r RESTORE _setbig 0 "\x02\x0A\x02\x5F\x39\xC0\x06\x02\x5F\x31\xC0\x00\xC0\x04\x02\x5F\x35\xC0\x02\xC0\x08\x02\x5F\x31\x02\x5F\x33\x09\x00\x7A\x5A\xFB\x90\x3A\xE9\x3C\xBE" } err + assert_match "*Bad data format*" $err + } +} + +test {corrupt payload: fuzzer findings - empty intset div by zero} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no + r debug set-skip-checksum-validation 1 + r RESTORE _setbig 0 "\x02\xC0\xC0\x06\x02\x5F\x39\xC0\x02\x02\x5F\x33\xC0\x00\x02\x5F\x31\xC0\x04\xC0\x08\x02\x5F\x37\x02\x5F\x35\x09\x00\xC5\xD4\x6D\xBA\xAD\x14\xB7\xE7" + catch {r SRANDMEMBER _setbig } + } +} + +test {corrupt payload: fuzzer findings - valgrind ziplist - crash report prints freed memory} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no + r debug set-skip-checksum-validation 1 + r RESTORE _zsetbig 0 "\x0C\x3D\x3D\x00\x00\x00\x3A\x00\x00\x00\x14\x00\x00\xF1\x02\xF1\x02\x02\x5F\x31\x04\xF2\x02\xF3\x02\xF3\x02\x02\x5F\x33\x04\xF4\x02\xEE\x02\xF5\x02\x02\x5F\x35\x04\xF6\x02\xF7\x02\xF7\x02\x02\x5F\x37\x04\xF8\x02\xF9\x02\xF9\x02\x02\x5F\x39\x04\xFA\xFF\x09\x00\xAE\xF9\x77\x2A\x47\x24\x33\xF6" + catch { r ZREMRANGEBYSCORE _zsetbig -1050966020 724 } + assert_equal [count_log_message 0 "crashed by signal"] 0 + assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 + } +} + +test {corrupt payload: fuzzer findings - valgrind ziplist prevlen reaches outside the ziplist} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no + r debug set-skip-checksum-validation 1 + r RESTORE _listbig 0 "\x0E\x02\x1B\x1B\x00\x00\x00\x16\x00\x00\x00\x05\x00\x00\x02\x5F\x39\x04\xF9\x02\x02\x5F\x37\x04\xF7\x02\x02\x5F\x35\xFF\x19\x19\x00\x00\x00\x16\x00\x00\x00\x05\x00\x00\xF5\x02\x02\x5F\x33\x04\xF3\x95\x02\x5F\x31\x04\xF1\xFF\x09\x00\x0C\xFC\x99\x2C\x23\x45\x15\x60" + catch { r RPOP _listbig } + catch { r RPOP _listbig } + catch { r RPUSH _listbig 949682325 } + assert_equal [count_log_message 0 "crashed by signal"] 0 + assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 + } +} + +test {corrupt payload: fuzzer findings - valgrind - bad rdbLoadDoubleValue} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no + r debug set-skip-checksum-validation 1 + catch { r RESTORE _list 0 "\x03\x01\x11\x11\x00\x00\x00\x0A\x00\x00\x00\x01\x00\x00\xD0\x07\x1A\xE9\x02\xFF\x09\x00\x1A\x06\x07\x32\x41\x28\x3A\x46" } err + assert_match "*Bad data format*" $err + } +} + +test {corrupt payload: fuzzer findings - valgrind ziplist prev too big} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no + r debug set-skip-checksum-validation 1 + r RESTORE _list 0 "\x0E\x01\x13\x13\x00\x00\x00\x10\x00\x00\x00\x03\x00\x00\xF3\x02\x02\x5F\x31\xC1\xF1\xFF\x09\x00\xC9\x4B\x31\xFE\x61\xC0\x96\xFE" + catch { r RPUSHX _list -45 } + catch { r LREM _list -748 -840} + assert_equal [count_log_message 0 "crashed by signal"] 0 + assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 + } +} + +test {corrupt payload: fuzzer findings - lzf decompression fails, avoid valgrind invalid read} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no + r debug set-skip-checksum-validation 1 + catch {r RESTORE _stream 0 "\x0F\x02\x10\x00\x00\x01\x73\xDD\xAA\x2A\xB9\x00\x00\x00\x00\x00\x00\x00\x00\xC3\x40\x4B\x40\x5C\x18\x5C\x00\x00\x00\x24\x00\x05\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x00\x20\x01\x00\x01\x20\x03\x00\x05\x20\x1C\x40\x07\x05\x01\x01\x82\x5F\x31\x03\x80\x0D\x40\x00\x00\x02\x60\x19\x40\x27\x40\x19\x00\x33\x60\x19\x40\x29\x02\x01\x01\x04\x20\x19\x00\xFF\x10\x00\x00\x01\x73\xDD\xAA\x2A\xBC\x00\x00\x00\x00\x00\x00\x00\x00\xC3\x40\x4D\x40\x5E\x18\x5E\x00\x00\x00\x24\x00\x05\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x00\x20\x01\x06\x01\x01\x82\x5F\x35\x03\x05\x20\x1E\x17\x0B\x03\x01\x01\x06\x01\x40\x0B\x00\x01\x60\x0D\x02\x82\x5F\x37\x60\x19\x80\x00\x00\x08\x60\x19\x80\x27\x02\x82\x5F\x39\x20\x19\x00\xFF\x0A\x81\x00\x00\x01\x73\xDD\xAA\x2A\xBE\x00\x00\x09\x00\x21\x85\x77\x43\x71\x7B\x17\x88"} err + assert_match "*Bad data format*" $err + } +} + +test {corrupt payload: fuzzer findings - stream bad lp_count} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload yes + r debug set-skip-checksum-validation 1 + catch { r RESTORE _stream 0 "\x0F\x01\x10\x00\x00\x01\x73\xDE\xDF\x7D\x9B\x00\x00\x00\x00\x00\x00\x00\x00\x40\x42\x42\x00\x00\x00\x18\x00\x03\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x00\x01\x02\x01\x00\x01\x00\x01\x01\x01\x00\x01\x56\x01\x02\x01\x22\x01\x00\x01\x01\x01\x82\x5F\x31\x03\x05\x01\x02\x01\x2C\x01\x00\x01\x01\x01\x02\x01\x05\x01\xFF\x03\x81\x00\x00\x01\x73\xDE\xDF\x7D\xC7\x00\x01\x07\x6D\x79\x67\x72\x6F\x75\x70\x81\x00\x00\x01\x73\xDE\xDF\x7D\x9B\x00\x01\x00\x00\x01\x73\xDE\xDF\x7D\x9B\x00\x00\x00\x00\x00\x00\x00\x00\xF9\x7D\xDF\xDE\x73\x01\x00\x00\x01\x01\x05\x41\x6C\x69\x63\x65\xEB\x7D\xDF\xDE\x73\x01\x00\x00\x01\x00\x00\x01\x73\xDE\xDF\x7D\x9B\x00\x00\x00\x00\x00\x00\x00\x00\x09\x00\xB2\xA8\xA7\x5F\x1B\x61\x72\xD5"} err + assert_match "*Bad data format*" $err + r ping + } +} + +test {corrupt payload: fuzzer findings - stream bad lp_count - unsanitized} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no + r debug set-skip-checksum-validation 1 + r RESTORE _stream 0 "\x0F\x01\x10\x00\x00\x01\x73\xDE\xDF\x7D\x9B\x00\x00\x00\x00\x00\x00\x00\x00\x40\x42\x42\x00\x00\x00\x18\x00\x03\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x00\x01\x02\x01\x00\x01\x00\x01\x01\x01\x00\x01\x56\x01\x02\x01\x22\x01\x00\x01\x01\x01\x82\x5F\x31\x03\x05\x01\x02\x01\x2C\x01\x00\x01\x01\x01\x02\x01\x05\x01\xFF\x03\x81\x00\x00\x01\x73\xDE\xDF\x7D\xC7\x00\x01\x07\x6D\x79\x67\x72\x6F\x75\x70\x81\x00\x00\x01\x73\xDE\xDF\x7D\x9B\x00\x01\x00\x00\x01\x73\xDE\xDF\x7D\x9B\x00\x00\x00\x00\x00\x00\x00\x00\xF9\x7D\xDF\xDE\x73\x01\x00\x00\x01\x01\x05\x41\x6C\x69\x63\x65\xEB\x7D\xDF\xDE\x73\x01\x00\x00\x01\x00\x00\x01\x73\xDE\xDF\x7D\x9B\x00\x00\x00\x00\x00\x00\x00\x00\x09\x00\xB2\xA8\xA7\x5F\x1B\x61\x72\xD5" + catch { r XREVRANGE _stream 638932639 738} + assert_equal [count_log_message 0 "crashed by signal"] 0 + assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 + } +} + +test {corrupt payload: fuzzer findings - stream integrity check issue} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload yes + r debug set-skip-checksum-validation 1 + catch { r RESTORE _stream 0 "\x0F\x02\x10\x00\x00\x01\x75\x2D\xA2\x90\x67\x00\x00\x00\x00\x00\x00\x00\x00\xC3\x40\x4F\x40\x5C\x18\x5C\x00\x00\x00\x24\x00\x05\x01\x00\x01\x4A\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x00\x20\x01\x00\x01\x20\x03\x00\x05\x20\x1C\x40\x09\x05\x01\x01\x82\x5F\x31\x03\x80\x0D\x00\x02\x20\x0D\x00\x02\xA0\x19\x00\x03\x20\x0B\x02\x82\x5F\x33\xA0\x19\x00\x04\x20\x0D\x00\x04\x20\x19\x00\xFF\x10\x00\x00\x01\x75\x2D\xA2\x90\x67\x00\x00\x00\x00\x00\x00\x00\x05\xC3\x40\x56\x40\x60\x18\x60\x00\x00\x00\x24\x00\x05\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x00\x20\x01\x06\x01\x01\x82\x5F\x35\x03\x05\x20\x1E\x40\x0B\x03\x01\x01\x06\x01\x80\x0B\x00\x02\x20\x0B\x02\x82\x5F\x37\x60\x19\x03\x01\x01\xDF\xFB\x20\x05\x00\x08\x60\x1A\x20\x0C\x00\xFC\x20\x05\x02\x82\x5F\x39\x20\x1B\x00\xFF\x0A\x81\x00\x00\x01\x75\x2D\xA2\x90\x68\x01\x00\x09\x00\x1D\x6F\xC0\x69\x8A\xDE\xF7\x92" } err + assert_match "*Bad data format*" $err + } +} + +test {corrupt payload: fuzzer findings - infinite loop} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no + r debug set-skip-checksum-validation 1 + r RESTORE _stream 0 "\x0F\x01\x10\x00\x00\x01\x75\x3A\xA6\xD0\x93\x00\x00\x00\x00\x00\x00\x00\x00\x40\x42\x42\x00\x00\x00\x18\x00\x03\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x00\x01\x02\x01\x00\x01\x00\x01\x01\x01\x00\x01\x05\x01\x02\x01\x00\x01\x01\x01\x01\x01\x82\x5F\x31\x03\xFD\x01\x02\x01\x00\x01\x02\x01\x01\x01\x02\x01\x05\x01\xFF\x03\x81\x00\x00\x01\x75\x3A\xA6\xD0\x93\x02\x01\x07\x6D\x79\x67\x72\x6F\x75\x70\x81\x00\x00\x01\x75\x3A\xA6\xD0\x93\x00\x01\x00\x00\x01\x75\x3A\xA6\xD0\x93\x00\x00\x00\x00\x00\x00\x00\x00\x94\xD0\xA6\x3A\x75\x01\x00\x00\x01\x01\x05\x41\x6C\x69\x63\x65\x94\xD0\xA6\x3A\x75\x01\x00\x00\x01\x00\x00\x01\x75\x3A\xA6\xD0\x93\x00\x00\x00\x00\x00\x00\x00\x00\x09\x00\xC4\x09\xAD\x69\x7E\xEE\xA6\x2F" + catch { r XREVRANGE _stream 288270516 971031845 } + assert_equal [count_log_message 0 "crashed by signal"] 0 + assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 + } +} + +test {corrupt payload: fuzzer findings - invalid tail offset after removal} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no + r debug set-skip-checksum-validation 1 + r RESTORE _zset 0 "\x0C\x19\x19\x00\x00\x00\x02\x00\x00\x00\x06\x00\x00\xF1\x02\xF1\x02\x02\x5F\x31\x04\xF2\x02\xF3\x02\xF3\xFF\x09\x00\x4D\x72\x7B\x97\xCD\x9A\x70\xC1" + catch {r ZPOPMIN _zset} + catch {r ZPOPMAX _zset} + assert_equal [count_log_message 0 "crashed by signal"] 0 + assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 + } +} + +test {corrupt payload: fuzzer findings - negative reply length} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no + r debug set-skip-checksum-validation 1 + r RESTORE _stream 0 "\x0F\x01\x10\x00\x00\x01\x75\xCF\xA1\x16\xA7\x00\x00\x00\x00\x00\x00\x00\x00\x40\x42\x42\x00\x00\x00\x18\x00\x03\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x00\x01\x02\x01\x00\x01\x00\x01\x01\x01\x00\x01\x05\x01\x02\x01\x00\x01\x01\x01\x01\x01\x14\x5F\x31\x03\x05\x01\x02\x01\x00\x01\x02\x01\x01\x01\x02\x01\x05\x01\xFF\x03\x81\x00\x00\x01\x75\xCF\xA1\x16\xA7\x02\x01\x07\x6D\x79\x67\x72\x6F\x75\x70\x81\x00\x00\x01\x75\xCF\xA1\x16\xA7\x01\x01\x00\x00\x01\x75\xCF\xA1\x16\xA7\x00\x00\x00\x00\x00\x00\x00\x01\xA7\x16\xA1\xCF\x75\x01\x00\x00\x01\x01\x05\x41\x6C\x69\x63\x65\xA7\x16\xA1\xCF\x75\x01\x00\x00\x01\x00\x00\x01\x75\xCF\xA1\x16\xA7\x00\x00\x00\x00\x00\x00\x00\x01\x09\x00\x1B\x42\x52\xB8\xDD\x5C\xE5\x4E" + catch {r XADD _stream * -956 -2601503852} + catch {r XINFO STREAM _stream FULL} + assert_equal [count_log_message 0 "crashed by signal"] 0 + assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 + } +} + +test {corrupt payload: fuzzer findings - valgrind negative malloc} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload yes + r debug set-skip-checksum-validation 1 + catch {r RESTORE _key 0 "\x0E\x01\x81\xD6\xD6\x00\x00\x00\x0A\x00\x00\x00\x01\x00\x00\x40\xC8\x6F\x2F\x36\xE2\xDF\xE3\x2E\x26\x64\x8B\x87\xD1\x7A\xBD\xFF\xEF\xEF\x63\x65\xF6\xF8\x8C\x4E\xEC\x96\x89\x56\x88\xF8\x3D\x96\x5A\x32\xBD\xD1\x36\xD8\x02\xE6\x66\x37\xCB\x34\x34\xC4\x52\xA7\x2A\xD5\x6F\x2F\x7E\xEE\xA2\x94\xD9\xEB\xA9\x09\x38\x3B\xE1\xA9\x60\xB6\x4E\x09\x44\x1F\x70\x24\xAA\x47\xA8\x6E\x30\xE1\x13\x49\x4E\xA1\x92\xC4\x6C\xF0\x35\x83\xD9\x4F\xD9\x9C\x0A\x0D\x7A\xE7\xB1\x61\xF5\xC1\x2D\xDC\xC3\x0E\x87\xA6\x80\x15\x18\xBA\x7F\x72\xDD\x14\x75\x46\x44\x0B\xCA\x9C\x8F\x1C\x3C\xD7\xDA\x06\x62\x18\x7E\x15\x17\x24\xAB\x45\x21\x27\xC2\xBC\xBB\x86\x6E\xD8\xBD\x8E\x50\xE0\xE0\x88\xA4\x9B\x9D\x15\x2A\x98\xFF\x5E\x78\x6C\x81\xFC\xA8\xC9\xC8\xE6\x61\xC8\xD1\x4A\x7F\x81\xD6\xA6\x1A\xAD\x4C\xC1\xA2\x1C\x90\x68\x15\x2A\x8A\x36\xC0\x58\xC3\xCC\xA6\x54\x19\x12\x0F\xEB\x46\xFF\x6E\xE3\xA7\x92\xF8\xFF\x09\x00\xD0\x71\xF7\x9F\xF7\x6A\xD6\x2E"} err + assert_match "*Bad data format*" $err + r ping + } +} + } ;# tags diff --git a/tests/integration/psync2.tcl b/tests/integration/psync2.tcl index 1b996ffd4..8f8db5a92 100644 --- a/tests/integration/psync2.tcl +++ b/tests/integration/psync2.tcl @@ -280,7 +280,7 @@ start_server {} { set sync_partial_err [status $R($master_id) sync_partial_err] catch { $R($slave_id) config rewrite - restart_server [expr {0-$slave_id}] true + restart_server [expr {0-$slave_id}] true false set R($slave_id) [srv [expr {0-$slave_id}] client] } # note: just waiting for connected_slaves==4 has a race condition since @@ -329,7 +329,7 @@ start_server {} { catch { $R($slave_id) config rewrite - restart_server [expr {0-$slave_id}] true + restart_server [expr {0-$slave_id}] true false set R($slave_id) [srv [expr {0-$slave_id}] client] } diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl index d29f9d88e..aadfe281f 100644 --- a/tests/integration/rdb.tcl +++ b/tests/integration/rdb.tcl @@ -164,7 +164,7 @@ test {client freed during loading} { # 100mb of rdb, 100k keys will load in more than 1 second r debug populate 100000 key 1000 - restart_server 0 false + restart_server 0 false false # make sure it's still loading assert_equal [s loading] 1 diff --git a/tests/support/server.tcl b/tests/support/server.tcl index ae93ad007..e5b167a35 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -13,7 +13,7 @@ proc start_server_error {config_file error} { } proc check_valgrind_errors stderr { - set res [find_valgrind_errors $stderr] + set res [find_valgrind_errors $stderr true] if {$res != ""} { send_data_packet $::test_server_fd err "Valgrind error: $res\n" } @@ -437,7 +437,7 @@ proc start_server {options {code undefined}} { while 1 { # check that the server actually started and is ready for connections - if {[exec grep -i "Ready to accept" | wc -l < $stdout] > 0} { + if {[count_message_lines $stdout "Ready to accept"] > 0} { break } after 10 @@ -511,13 +511,19 @@ proc start_server {options {code undefined}} { } } -proc restart_server {level wait_ready} { +proc restart_server {level wait_ready rotate_logs} { set srv [lindex $::servers end+$level] kill_server $srv + set pid [dict get $srv "pid"] set stdout [dict get $srv "stdout"] set stderr [dict get $srv "stderr"] - set config_file [dict get $srv "config_file"] + if {$rotate_logs} { + set ts [clock format [clock seconds] -format %y%m%d%H%M%S] + file rename $stdout $stdout.$ts.$pid + file rename $stderr $stderr.$ts.$pid + } + set prev_ready_count [count_message_lines $stdout "Ready to accept"] # if we're inside a test, write the test name to the server log file if {[info exists ::cur_test]} { @@ -526,7 +532,7 @@ proc restart_server {level wait_ready} { close $fd } - set prev_ready_count [exec grep -i "Ready to accept" | wc -l < $stdout] + set config_file [dict get $srv "config_file"] set pid [spawn_server $config_file $stdout $stderr] @@ -541,7 +547,7 @@ proc restart_server {level wait_ready} { if {$wait_ready} { while 1 { # check that the server actually started and is ready for connections - if {[exec grep -i "Ready to accept" | wc -l < $stdout] > $prev_ready_count + 1} { + if {[count_message_lines $stdout "Ready to accept"] > $prev_ready_count} { break } after 10 diff --git a/tests/support/util.tcl b/tests/support/util.tcl index aadec7cd1..040a7f7d9 100644 --- a/tests/support/util.tcl +++ b/tests/support/util.tcl @@ -454,22 +454,31 @@ proc colorstr {color str} { } } -proc find_valgrind_errors {stderr} { +proc find_valgrind_errors {stderr on_termination} { set fd [open $stderr] set buf [read $fd] close $fd # Look for stack trace (" at 0x") and other errors (Invalid, Mismatched, etc). # Look for "Warnings", but not the "set address range perms". These don't indicate any real concern. - # Look for the absense of a leak free summary (happens when redis isn't terminated properly). + # corrupt-dump unit, not sure why but it seems they don't indicate any real concern. if {[regexp -- { at 0x} $buf] || [regexp -- {^(?=.*Warning)(?:(?!set address range perms).)*$} $buf] || [regexp -- {Invalid} $buf] || [regexp -- {Mismatched} $buf] || [regexp -- {uninitialized} $buf] || [regexp -- {has a fishy} $buf] || - [regexp -- {overlap} $buf] || - (![regexp -- {definitely lost: 0 bytes} $buf] && + [regexp -- {overlap} $buf]} { + return $buf + } + + # If the process didn't terminate yet, we can't look for the summary report + if {!$on_termination} { + return "" + } + + # Look for the absense of a leak free summary (happens when redis isn't terminated properly). + if {(![regexp -- {definitely lost: 0 bytes} $buf] && ![regexp -- {no leaks are possible} $buf])} { return $buf } @@ -547,3 +556,116 @@ proc cmdrstat {cmd r} { set _ $value } } + +proc generate_fuzzy_traffic_on_key {key duration} { + # Commands per type, blocking commands removed + # TODO: extract these from help.h or elsewhere, and improve to include other types + set string_commands {APPEND BITCOUNT BITFIELD BITOP BITPOS DECR DECRBY GET GETBIT GETRANGE GETSET INCR INCRBY INCRBYFLOAT MGET MSET MSETNX PSETEX SET SETBIT SETEX SETNX SETRANGE STRALGO STRLEN} + set hash_commands {HDEL HEXISTS HGET HGETALL HINCRBY HINCRBYFLOAT HKEYS HLEN HMGET HMSET HSCAN HSET HSETNX HSTRLEN HVALS} + set zset_commands {ZADD ZCARD ZCOUNT ZINCRBY ZINTERSTORE ZLEXCOUNT ZPOPMAX ZPOPMIN ZRANGE ZRANGEBYLEX ZRANGEBYSCORE ZRANK ZREM ZREMRANGEBYLEX ZREMRANGEBYRANK ZREMRANGEBYSCORE ZREVRANGE ZREVRANGEBYLEX ZREVRANGEBYSCORE ZREVRANK ZSCAN ZSCORE ZUNIONSTORE} + set list_commands {LINDEX LINSERT LLEN LPOP LPOS LPUSH LPUSHX LRANGE LREM LSET LTRIM RPOP RPOPLPUSH RPUSH RPUSHX} + set set_commands {SADD SCARD SDIFF SDIFFSTORE SINTER SINTERSTORE SISMEMBER SMEMBERS SMOVE SPOP SRANDMEMBER SREM SSCAN SUNION SUNIONSTORE} + set stream_commands {XACK XADD XCLAIM XDEL XGROUP XINFO XLEN XPENDING XRANGE XREAD XREADGROUP XREVRANGE XTRIM} + set commands [dict create string $string_commands hash $hash_commands zset $zset_commands list $list_commands set $set_commands stream $stream_commands] + + set type [r type $key] + set cmds [dict get $commands $type] + set start_time [clock seconds] + set sent {} + set succeeded 0 + while {([clock seconds]-$start_time) < $duration} { + # find a random command for our key type + set cmd_idx [expr {int(rand()*[llength $cmds])}] + set cmd [lindex $cmds $cmd_idx] + # get the command details from redis + if { [ catch { + set cmd_info [lindex [r command info $cmd] 0] + } err ] } { + # if we failed, it means redis crashed after the previous command + return $sent + } + # try to build a valid command argument + set arity [lindex $cmd_info 1] + set arity [expr $arity < 0 ? - $arity: $arity] + set firstkey [lindex $cmd_info 3] + set i 1 + if {$cmd == "XINFO"} { + lappend cmd "STREAM" + lappend cmd $key + lappend cmd "FULL" + incr i 3 + } + if {$cmd == "XREAD"} { + lappend cmd "STREAMS" + lappend cmd $key + randpath { + lappend cmd \$ + } { + lappend cmd [randomValue] + } + incr i 3 + } + if {$cmd == "XADD"} { + lappend cmd $key + randpath { + lappend cmd "*" + } { + lappend cmd [randomValue] + } + lappend cmd [randomValue] + lappend cmd [randomValue] + incr i 4 + } + for {} {$i < $arity} {incr i} { + if {$i == $firstkey} { + lappend cmd $key + } else { + lappend cmd [randomValue] + } + } + # execute the command, we expect commands to fail on syntax errors + lappend sent $cmd + if { ! [ catch { + r {*}$cmd + } err ] } { + incr succeeded + } + } + + # print stats so that we know if we managed to generate commands that actually made senes + #if {$::verbose} { + # set count [llength $sent] + # puts "Fuzzy traffic sent: $count, succeeded: $succeeded" + #} + + # return the list of commands we sent + return $sent +} + +# write line to server log file +proc write_log_line {srv_idx msg} { + set logfile [srv $srv_idx stdout] + set fd [open $logfile "a+"] + puts $fd "### $msg" + close $fd +} + +proc string2printable s { + set res {} + set has_special_chars false + foreach i [split $s {}] { + scan $i %c int + # non printable characters, including space and excluding: " \ $ { } + if {$int < 32 || $int > 122 || $int == 34 || $int == 36 || $int == 92} { + set has_special_chars true + } + # TCL8.5 has issues mixing \x notation and normal chars in the same + # source code string, so we'll convert the entire string. + append res \\x[format %02X $int] + } + if {!$has_special_chars} { + return $s + } + set res "\"$res\"" + return $res +} diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 998e82bd9..a1fae8d75 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -45,6 +45,7 @@ set ::all_tests { integration/aof integration/rdb integration/corrupt-dump + integration/corrupt-dump-fuzzer integration/convert-zipmap-hash-on-load integration/logging integration/psync2 diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl index 2d50b417b..86da17657 100644 --- a/tests/unit/introspection.tcl +++ b/tests/unit/introspection.tcl @@ -160,7 +160,7 @@ start_server {tags {"introspection"}} { # Rewrite entire configuration, restart and confirm the # server is able to parse it and start. assert_equal [r debug config-rewrite-force-all] "OK" - restart_server 0 0 + restart_server 0 false false assert_equal [r ping] "PONG" # Verify no changes were introduced