Relaxed RDB version check (#1604)
New config `rdb-version-check` with values: * `strict`: Reject future RDB versions. * `relaxed`: Try parsing future RDB versions and fail only when an unknown RDB opcode or type is encountered. This can make it possible for Valkey 8.1 to try read a dump from for example Valkey 9.0 or later on a best-effort basis. The conditions for when this is expected to work can be defined when the future Valkey versions are released. Loading is expected to fail in the following cases: * If the data set contains any new key types or other data elements not supported by the current version. * If the RDB contains new representations or encodings of existing key types or other data elements. This change also prepares for the next RDB version bump. A range of RDB versions (12-79) is reserved, since it's expected to be used by foreign software RDB versions, so Valkey will not accept versions in this range even with the `relaxed` version check. The DUMP/RESTORE format has no magic string; only the RDB version number. This change also prepares for the magic string to change from REDIS to VALKEY next time we bump the RDB version. Related to #1108. --------- Signed-off-by: Viktor Söderqvist <viktor.soderqvist@est.tech> Co-authored-by: Madelyn Olson <madelyneolson@gmail.com>
This commit is contained in:
Viktor Söderqvist
GitHub
parent
7699a3a94a
commit
e9b8970e72
@ -162,7 +162,10 @@ int verifyDumpPayload(unsigned char *p, size_t len, uint16_t *rdbver_ptr) {
|
||||
if (rdbver_ptr) {
|
||||
*rdbver_ptr = rdbver;
|
||||
}
|
||||
if (rdbver > RDB_VERSION) return C_ERR;
|
||||
if ((rdbver >= RDB_FOREIGN_VERSION_MIN && rdbver <= RDB_FOREIGN_VERSION_MAX) ||
|
||||
(rdbver > RDB_VERSION && server.rdb_version_check == RDB_VERSION_CHECK_STRICT)) {
|
||||
return C_ERR;
|
||||
}
|
||||
|
||||
if (server.skip_checksum_validation) return C_OK;
|
||||
|
||||
|
||||
@ -160,6 +160,10 @@ configEnum log_timestamp_format_enum[] = {{"legacy", LOG_TIMESTAMP_LEGACY},
|
||||
{"milliseconds", LOG_TIMESTAMP_MILLISECONDS},
|
||||
{NULL, 0}};
|
||||
|
||||
configEnum rdb_version_check_enum[] = {{"strict", RDB_VERSION_CHECK_STRICT},
|
||||
{"relaxed", RDB_VERSION_CHECK_RELAXED},
|
||||
{NULL, 0}};
|
||||
|
||||
/* Output buffer limits presets. */
|
||||
clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = {
|
||||
{0, 0, 0}, /* normal */
|
||||
@ -3244,6 +3248,7 @@ standardConfig static_configs[] = {
|
||||
createEnumConfig("shutdown-on-sigterm", NULL, MODIFIABLE_CONFIG | MULTI_ARG_CONFIG, shutdown_on_sig_enum, server.shutdown_on_sigterm, 0, isValidShutdownOnSigFlags, NULL),
|
||||
createEnumConfig("log-format", NULL, MODIFIABLE_CONFIG, log_format_enum, server.log_format, LOG_FORMAT_LEGACY, NULL, NULL),
|
||||
createEnumConfig("log-timestamp-format", NULL, MODIFIABLE_CONFIG, log_timestamp_format_enum, server.log_timestamp_format, LOG_TIMESTAMP_LEGACY, NULL, NULL),
|
||||
createEnumConfig("rdb-version-check", NULL, MODIFIABLE_CONFIG, rdb_version_check_enum, server.rdb_version_check, RDB_VERSION_CHECK_STRICT, NULL, NULL),
|
||||
|
||||
/* Integer configs */
|
||||
createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.dbnum, 16, INTEGER_CONFIG, NULL, NULL),
|
||||
|
||||
15
src/rdb.c
15
src/rdb.c
@ -42,6 +42,7 @@
|
||||
#include <math.h>
|
||||
#include <fcntl.h>
|
||||
#include <stdatomic.h>
|
||||
#include <stdbool.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/resource.h>
|
||||
@ -1418,6 +1419,7 @@ int rdbSaveRio(int req, rio *rdb, int *error, int rdbflags, rdbSaveInfo *rsi) {
|
||||
int j;
|
||||
|
||||
if (server.rdb_checksum) rdb->update_cksum = rioGenericUpdateChecksum;
|
||||
/* TODO: Change this to "VALKEY%03d" next time we bump the RDB version. */
|
||||
snprintf(magic, sizeof(magic), "REDIS%04d", RDB_VERSION);
|
||||
if (rdbWriteRaw(rdb, magic, 9) == -1) goto werr;
|
||||
if (rdbSaveInfoAuxFields(rdb, rdbflags, rsi) == -1) goto werr;
|
||||
@ -3023,17 +3025,24 @@ int rdbLoadRioWithLoadingCtx(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadin
|
||||
char buf[1024];
|
||||
int error;
|
||||
long long empty_keys_skipped = 0;
|
||||
bool is_valkey_magic;
|
||||
|
||||
rdb->update_cksum = rdbLoadProgressCallback;
|
||||
rdb->max_processing_chunk = server.loading_process_events_interval_bytes;
|
||||
if (rioRead(rdb, buf, 9) == 0) goto eoferr;
|
||||
buf[9] = '\0';
|
||||
if (memcmp(buf, "REDIS", 5) != 0) {
|
||||
if (memcmp(buf, "REDIS0", 6) == 0) {
|
||||
is_valkey_magic = false;
|
||||
} else if (memcmp(buf, "VALKEY", 6) == 0) {
|
||||
is_valkey_magic = true;
|
||||
} else {
|
||||
serverLog(LL_WARNING, "Wrong signature trying to load DB from file");
|
||||
return C_ERR;
|
||||
}
|
||||
rdbver = atoi(buf + 5);
|
||||
if (rdbver < 1 || rdbver > RDB_VERSION) {
|
||||
rdbver = atoi(buf + 6);
|
||||
if (rdbver < 1 ||
|
||||
(rdbver >= RDB_FOREIGN_VERSION_MIN && !is_valkey_magic) ||
|
||||
(rdbver > RDB_VERSION && server.rdb_version_check == RDB_VERSION_CHECK_STRICT)) {
|
||||
serverLog(LL_WARNING, "Can't handle RDB format version %d", rdbver);
|
||||
return C_ERR;
|
||||
}
|
||||
|
||||
20
src/rdb.h
20
src/rdb.h
@ -37,9 +37,27 @@
|
||||
#include "server.h"
|
||||
|
||||
/* The current RDB version. When the format changes in a way that is no longer
|
||||
* backward compatible this number gets incremented. */
|
||||
* backward compatible this number gets incremented.
|
||||
*
|
||||
* RDB 11 is the last open-source Redis RDB version, used by Valkey 7.x and 8.x.
|
||||
*
|
||||
* RDB 12+ are non-open-source Redis formats.
|
||||
*
|
||||
* Next time we bump the Valkey RDB version, use much higher version to avoid
|
||||
* collisions with non-OSS Redis RDB versions. For example, we could use RDB
|
||||
* version 90 for Valkey 9.0.
|
||||
*
|
||||
* In an RDB file/stream, we also check the magic string REDIS or VALKEY but in
|
||||
* the DUMP/RESTORE format, there is only the RDB version number and no magic
|
||||
* string. */
|
||||
#define RDB_VERSION 11
|
||||
|
||||
/* Reserved range for foreign (unsupported, non-OSS) RDB format. */
|
||||
#define RDB_FOREIGN_VERSION_MIN 12
|
||||
#define RDB_FOREIGN_VERSION_MAX 79
|
||||
static_assert(RDB_VERSION < RDB_FOREIGN_VERSION_MIN || RDB_VERSION > RDB_FOREIGN_VERSION_MAX,
|
||||
"RDB version in foreign version range");
|
||||
|
||||
/* Defines related to the dump file format. To store 32 bits lengths for short
|
||||
* keys requires a lot of space, so we check the most significant 2 bits of
|
||||
* the first byte to interpreter the length:
|
||||
|
||||
@ -600,6 +600,9 @@ typedef enum { LOG_TIMESTAMP_LEGACY = 0,
|
||||
LOG_TIMESTAMP_ISO8601,
|
||||
LOG_TIMESTAMP_MILLISECONDS } log_timestamp_type;
|
||||
|
||||
typedef enum { RDB_VERSION_CHECK_STRICT = 0,
|
||||
RDB_VERSION_CHECK_RELAXED } rdb_version_check_type;
|
||||
|
||||
/* common sets of actions to pause/unpause */
|
||||
#define PAUSE_ACTIONS_CLIENT_WRITE_SET \
|
||||
(PAUSE_ACTION_CLIENT_WRITE | PAUSE_ACTION_EXPIRE | PAUSE_ACTION_EVICT | PAUSE_ACTION_REPLICA)
|
||||
@ -1768,6 +1771,7 @@ struct valkeyServer {
|
||||
int active_defrag_enabled;
|
||||
int sanitize_dump_payload; /* Enables deep sanitization for ziplist and listpack in RDB and RESTORE. */
|
||||
int skip_checksum_validation; /* Disable checksum validation for RDB and RESTORE payload. */
|
||||
int rdb_version_check; /* Try to load RDB produced by a future version. */
|
||||
int jemalloc_bg_thread; /* Enable jemalloc background thread */
|
||||
int active_defrag_configuration_changed; /* Config changed; need to recompute active_defrag_cpu_percent. */
|
||||
size_t active_defrag_ignore_bytes; /* minimum amount of fragmentation waste to start active defrag */
|
||||
|
||||
BIN
tests/assets/encodings-rdb987.rdb
Normal file
BIN
tests/assets/encodings-rdb987.rdb
Normal file
Binary file not shown.
@ -1,9 +1,21 @@
|
||||
tags {"rdb external:skip"} {
|
||||
|
||||
# Helper function to start a server and kill it, just to check the error
|
||||
# logged.
|
||||
set defaults {}
|
||||
proc start_server_and_kill_it {overrides code} {
|
||||
upvar defaults defaults srv srv server_path server_path
|
||||
set config [concat $defaults $overrides]
|
||||
set srv [start_server [list overrides $config keep_persistence true]]
|
||||
uplevel 1 $code
|
||||
kill_server $srv
|
||||
}
|
||||
|
||||
set server_path [tmpdir "server.rdb-encoding-test"]
|
||||
|
||||
# Copy RDB with different encodings in server path
|
||||
exec cp tests/assets/encodings.rdb $server_path
|
||||
exec cp tests/assets/encodings-rdb987.rdb $server_path
|
||||
exec cp tests/assets/list-quicklist.rdb $server_path
|
||||
|
||||
start_server [list overrides [list "dir" $server_path "dbfilename" "list-quicklist.rdb" save ""]] {
|
||||
@ -15,11 +27,7 @@ start_server [list overrides [list "dir" $server_path "dbfilename" "list-quickli
|
||||
} {7}
|
||||
}
|
||||
|
||||
start_server [list overrides [list "dir" $server_path "dbfilename" "encodings.rdb"]] {
|
||||
test "RDB encoding loading test" {
|
||||
r select 0
|
||||
csvdump r
|
||||
} {"0","compressible","string","aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
set csv_dump {"0","compressible","string","aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
"0","hash","hash","a","1","aa","10","aaa","100","b","2","bb","20","bbb","200","c","3","cc","30","ccc","300","ddd","400","eee","5000000000",
|
||||
"0","hash_zipped","hash","a","1","b","2","c","3",
|
||||
"0","list","list","1","2","3","a","b","c","100000","6000000000","1","2","3","a","b","c","100000","6000000000","1","2","3","a","b","c","100000","6000000000",
|
||||
@ -33,6 +41,32 @@ start_server [list overrides [list "dir" $server_path "dbfilename" "encodings.rd
|
||||
"0","zset","zset","a","1","b","2","c","3","aa","10","bb","20","cc","30","aaa","100","bbb","200","ccc","300","aaaa","1000","cccc","123456789","bbbb","5000000000",
|
||||
"0","zset_zipped","zset","a","1","b","2","c","3",
|
||||
}
|
||||
|
||||
start_server [list overrides [list "dir" $server_path "dbfilename" "encodings.rdb"]] {
|
||||
test "RDB encoding loading test" {
|
||||
r select 0
|
||||
csvdump r
|
||||
} $csv_dump
|
||||
}
|
||||
|
||||
start_server_and_kill_it [list "dir" $server_path "dbfilename" "encodings-rdb987.rdb"] {
|
||||
test "RDB future version loading, strict version check" {
|
||||
wait_for_condition 50 100 {
|
||||
[string match {*Fatal error loading*} \
|
||||
[exec tail -1 < [dict get $srv stdout]]]
|
||||
} else {
|
||||
fail "Server started even if RDB version check failed"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
start_server [list overrides [list "dir" $server_path \
|
||||
"dbfilename" "encodings-rdb987.rdb" \
|
||||
"rdb-version-check" "relaxed"]] {
|
||||
test "RDB future version loading, relaxed version check" {
|
||||
r select 0
|
||||
csvdump r
|
||||
} $csv_dump
|
||||
}
|
||||
|
||||
set server_path [tmpdir "server.rdb-startup-test"]
|
||||
@ -80,17 +114,6 @@ start_server [list overrides [list "dir" $server_path] keep_persistence true] {
|
||||
r del stream
|
||||
}
|
||||
|
||||
# Helper function to start a server and kill it, just to check the error
|
||||
# logged.
|
||||
set defaults {}
|
||||
proc start_server_and_kill_it {overrides code} {
|
||||
upvar defaults defaults srv srv server_path server_path
|
||||
set config [concat $defaults $overrides]
|
||||
set srv [start_server [list overrides $config keep_persistence true]]
|
||||
uplevel 1 $code
|
||||
kill_server $srv
|
||||
}
|
||||
|
||||
# Make the RDB file unreadable
|
||||
file attributes [file join $server_path dump.rdb] -permissions 0222
|
||||
|
||||
|
||||
@ -124,6 +124,24 @@ start_server {tags {"dump"}} {
|
||||
close_replication_stream $repl
|
||||
} {} {needs:repl}
|
||||
|
||||
test {RESTORE key with future RDB version, strict version check} {
|
||||
# str len RDB 222 CRC64 checksum
|
||||
# | | | |
|
||||
set bar_dump "\x00\x03bar\xde\x00\x0fYUza\xd3\xec\xe0"
|
||||
assert_error {ERR DUMP payload version or checksum are wrong} {r restore foo 0 $bar_dump replace}
|
||||
}
|
||||
|
||||
test {RESTORE key with future RDB version, relaxed version check} {
|
||||
# str len RDB 222 CRC64 checksum
|
||||
# | | | |
|
||||
set bar_dump "\x00\x03bar\xde\x00\x0fYUza\xd3\xec\xe0"
|
||||
r config set rdb-version-check relaxed
|
||||
catch {r restore foo 0 $bar_dump replace} e
|
||||
r config set rdb-version-check strict
|
||||
assert_equal {bar} [r get foo]
|
||||
set e
|
||||
} {OK}
|
||||
|
||||
test {DUMP of non existing key returns nil} {
|
||||
r dump nonexisting_key
|
||||
} {}
|
||||
|
||||
@ -549,6 +549,15 @@ rdbcompression yes
|
||||
# tell the loading code to skip the check.
|
||||
rdbchecksum yes
|
||||
|
||||
# Valkey can try to load an RDB dump produced by a future version of Valkey.
|
||||
# This can only work on a best-effort basis, because future RDB versions may
|
||||
# contain information that's not known to the current version. If no new features
|
||||
# are used, it may be possible to import the data produced by a later version,
|
||||
# but loading is aborted if unknown information is encountered. Possible values
|
||||
# are 'strict' and 'relaxed'. This also applies to replication and the RESTORE
|
||||
# command.
|
||||
rdb-version-check strict
|
||||
|
||||
# Enables or disables full sanitization checks for ziplist and listpack etc when
|
||||
# loading an RDB or RESTORE payload. This reduces the chances of a assertion or
|
||||
# crash later on while processing commands.
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user