Adapt redis-check-aof tool for Multi Part Aof (#10061)
Modifications of this PR: 1. Support the verification of `Multi Part AOF`, while still maintaining support for the old-style `AOF/RDB-preamble`. `redis-check-aof` will automatically choose which mode to use according to the incoming file format. `Usage: redis-check-aof [--fix|--truncate-to-timestamp $timestamp] <AOF/manifest>` 2. Refactor part of the code to make it easier to understand 3. Currently only supports truncate (`--fix` or `--truncate-to-timestamp`) the last AOF file (may be `BASE` or `INCR`) The reasons for 3 above: - for `--fix`: Only the last AOF may be truncated, this is guaranteed by redis - for `--truncate-to-timestamp`: Normally, we only have `BASE` + `INCR` files at most, and `BASE` cannot be truncated(It only contains a timestamp annotation at the beginning of the file), so only `INCR` can be truncated. If we have a `BASE+INCR1+INCR2` file (meaning we have an interrupted AOFRW), Only `INCR2` files can be truncated at this time. If we still insist on truncate `INCR1`, we need to manually delete `INCR2` and update the manifest file, then re-run `redis-check-aof` - If we want to support truncate any file, we need to add very complicated code to support the atomic modification of multiple file deletion and update manifest, I think this is unnecessary
This commit is contained in:
chenyang8094
GitHub
parent
f7f68c654a
commit
a50aa29bde
55
src/aof.c
55
src/aof.c
@ -47,6 +47,8 @@ off_t getBaseAndIncrAppendOnlyFilesSize(aofManifest *am);
|
||||
int getBaseAndIncrAppendOnlyFilesNum(aofManifest *am);
|
||||
int aofFileExist(char *filename);
|
||||
int rewriteAppendOnlyFile(char *filename);
|
||||
aofManifest *aofLoadManifestFromFile(sds am_filepath);
|
||||
void aofManifestFreeAndUpdate(aofManifest *am);
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
* AOF Manifest file implementation.
|
||||
@ -226,13 +228,8 @@ sds getAofManifestAsString(aofManifest *am) {
|
||||
* in order to support seamless upgrades from previous versions which did not
|
||||
* use them.
|
||||
*/
|
||||
#define MANIFEST_MAX_LINE 1024
|
||||
void aofLoadManifestFromDisk(void) {
|
||||
const char *err = NULL;
|
||||
long long maxseq = 0;
|
||||
|
||||
server.aof_manifest = aofManifestCreate();
|
||||
|
||||
if (!dirExists(server.aof_dirname)) {
|
||||
serverLog(LL_NOTICE, "The AOF directory %s doesn't exist", server.aof_dirname);
|
||||
return;
|
||||
@ -247,16 +244,26 @@ void aofLoadManifestFromDisk(void) {
|
||||
return;
|
||||
}
|
||||
|
||||
aofManifest *am = aofLoadManifestFromFile(am_filepath);
|
||||
if (am) aofManifestFreeAndUpdate(am);
|
||||
sdsfree(am_name);
|
||||
sdsfree(am_filepath);
|
||||
}
|
||||
|
||||
/* Generic manifest loading function, used in `aofLoadManifestFromDisk` and redis-check-aof tool. */
|
||||
#define MANIFEST_MAX_LINE 1024
|
||||
aofManifest *aofLoadManifestFromFile(sds am_filepath) {
|
||||
const char *err = NULL;
|
||||
long long maxseq = 0;
|
||||
|
||||
aofManifest *am = aofManifestCreate();
|
||||
FILE *fp = fopen(am_filepath, "r");
|
||||
if (fp == NULL) {
|
||||
serverLog(LL_WARNING, "Fatal error: can't open the AOF manifest "
|
||||
"file %s for reading: %s", am_name, strerror(errno));
|
||||
"file %s for reading: %s", am_filepath, strerror(errno));
|
||||
exit(1);
|
||||
}
|
||||
|
||||
sdsfree(am_name);
|
||||
sdsfree(am_filepath);
|
||||
|
||||
char buf[MANIFEST_MAX_LINE+1];
|
||||
sds *argv = NULL;
|
||||
int argc;
|
||||
@ -292,14 +299,14 @@ void aofLoadManifestFromDisk(void) {
|
||||
|
||||
line = sdstrim(sdsnew(buf), " \t\r\n");
|
||||
if (!sdslen(line)) {
|
||||
err = "The AOF manifest file is invalid format";
|
||||
err = "Invalid AOF manifest file format";
|
||||
goto loaderr;
|
||||
}
|
||||
|
||||
argv = sdssplitargs(line, &argc);
|
||||
/* 'argc < 6' was done for forward compatibility. */
|
||||
if (argv == NULL || argc < 6 || (argc % 2)) {
|
||||
err = "The AOF manifest file is invalid format";
|
||||
err = "Invalid AOF manifest file format";
|
||||
goto loaderr;
|
||||
}
|
||||
|
||||
@ -321,7 +328,7 @@ void aofLoadManifestFromDisk(void) {
|
||||
|
||||
/* We have to make sure we load all the information. */
|
||||
if (!ai->file_name || !ai->file_seq || !ai->file_type) {
|
||||
err = "The AOF manifest file is invalid format";
|
||||
err = "Invalid AOF manifest file format";
|
||||
goto loaderr;
|
||||
}
|
||||
|
||||
@ -329,21 +336,21 @@ void aofLoadManifestFromDisk(void) {
|
||||
argv = NULL;
|
||||
|
||||
if (ai->file_type == AOF_FILE_TYPE_BASE) {
|
||||
if (server.aof_manifest->base_aof_info) {
|
||||
if (am->base_aof_info) {
|
||||
err = "Found duplicate base file information";
|
||||
goto loaderr;
|
||||
}
|
||||
server.aof_manifest->base_aof_info = ai;
|
||||
server.aof_manifest->curr_base_file_seq = ai->file_seq;
|
||||
am->base_aof_info = ai;
|
||||
am->curr_base_file_seq = ai->file_seq;
|
||||
} else if (ai->file_type == AOF_FILE_TYPE_HIST) {
|
||||
listAddNodeTail(server.aof_manifest->history_aof_list, ai);
|
||||
listAddNodeTail(am->history_aof_list, ai);
|
||||
} else if (ai->file_type == AOF_FILE_TYPE_INCR) {
|
||||
if (ai->file_seq <= maxseq) {
|
||||
err = "Found a non-monotonic sequence number";
|
||||
goto loaderr;
|
||||
}
|
||||
listAddNodeTail(server.aof_manifest->incr_aof_list, ai);
|
||||
server.aof_manifest->curr_incr_file_seq = ai->file_seq;
|
||||
listAddNodeTail(am->incr_aof_list, ai);
|
||||
am->curr_incr_file_seq = ai->file_seq;
|
||||
maxseq = ai->file_seq;
|
||||
} else {
|
||||
err = "Unknown AOF file type";
|
||||
@ -356,7 +363,7 @@ void aofLoadManifestFromDisk(void) {
|
||||
}
|
||||
|
||||
fclose(fp);
|
||||
return;
|
||||
return am;
|
||||
|
||||
loaderr:
|
||||
/* Sanitizer suppression: may report a false positive if we goto loaderr
|
||||
@ -1526,15 +1533,15 @@ uxeof: /* Unexpected AOF end of file. */
|
||||
}
|
||||
}
|
||||
}
|
||||
serverLog(LL_WARNING,"Unexpected end of file reading the append only file %s. You can: \
|
||||
1) Make a backup of your AOF file, then use ./redis-check-aof --fix <filename>. \
|
||||
2) Alternatively you can set the 'aof-load-truncated' configuration option to yes and restart the server.", filename);
|
||||
serverLog(LL_WARNING, "Unexpected end of file reading the append only file %s. You can: "
|
||||
"1) Make a backup of your AOF file, then use ./redis-check-aof --fix <filename.manifest>. "
|
||||
"2) Alternatively you can set the 'aof-load-truncated' configuration option to yes and restart the server.", filename);
|
||||
ret = AOF_FAILED;
|
||||
goto cleanup;
|
||||
|
||||
fmterr: /* Format error. */
|
||||
serverLog(LL_WARNING,"Bad file format reading the append only file %s: \
|
||||
make a backup of your AOF file, then use ./redis-check-aof --fix <filename>", filename);
|
||||
serverLog(LL_WARNING, "Bad file format reading the append only file %s: "
|
||||
"make a backup of your AOF file, then use ./redis-check-aof --fix <filename.manifest>", filename);
|
||||
ret = AOF_FAILED;
|
||||
/* fall through to cleanup. */
|
||||
|
||||
|
||||
@ -30,6 +30,24 @@
|
||||
|
||||
#include "server.h"
|
||||
#include <sys/stat.h>
|
||||
#include <sys/types.h>
|
||||
#include <regex.h>
|
||||
#include <libgen.h>
|
||||
|
||||
#define AOF_CHECK_OK 0
|
||||
#define AOF_CHECK_EMPTY 1
|
||||
#define AOF_CHECK_TRUNCATED 2
|
||||
#define AOF_CHECK_TIMESTAMP_TRUNCATED 3
|
||||
|
||||
typedef enum {
|
||||
AOF_RESP,
|
||||
AOF_RDB_PREAMBLE,
|
||||
AOF_MULTI_PART,
|
||||
} input_file_type;
|
||||
|
||||
aofManifest *aofManifestCreate(void);
|
||||
void aofManifestFree(aofManifest *am);
|
||||
aofManifest *aofLoadManifestFromFile(sds am_filepath);
|
||||
|
||||
#define ERROR(...) { \
|
||||
char __buf[1024]; \
|
||||
@ -51,47 +69,6 @@ int consumeNewline(char *buf) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
int readAnnotations(FILE *fp) {
|
||||
char buf[AOF_ANNOTATION_LINE_MAX_LEN];
|
||||
while (1) {
|
||||
epos = ftello(fp);
|
||||
if (fgets(buf, sizeof(buf), fp) == NULL) {
|
||||
return 0;
|
||||
}
|
||||
if (buf[0] == '#') {
|
||||
if (to_timestamp && strncmp(buf, "#TS:", 4) == 0) {
|
||||
time_t ts = strtol(buf+4, NULL, 10);
|
||||
if (ts <= to_timestamp) continue;
|
||||
if (epos == 0) {
|
||||
printf("AOF has nothing before timestamp %ld, "
|
||||
"aborting...\n", to_timestamp);
|
||||
fclose(fp);
|
||||
exit(1);
|
||||
}
|
||||
/* Truncate remaining AOF if exceeding 'to_timestamp' */
|
||||
if (ftruncate(fileno(fp), epos) == -1) {
|
||||
printf("Failed to truncate AOF to timestamp %ld\n",
|
||||
to_timestamp);
|
||||
exit(1);
|
||||
} else {
|
||||
printf("Successfully truncated AOF to timestamp %ld\n",
|
||||
to_timestamp);
|
||||
fclose(fp);
|
||||
exit(0);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
} else {
|
||||
if (fseek(fp, -(ftello(fp)-epos), SEEK_CUR) == -1) {
|
||||
ERROR("Fseek error: %s", strerror(errno));
|
||||
return 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
int readLong(FILE *fp, char prefix, long *target) {
|
||||
char buf[128], *eptr;
|
||||
epos = ftello(fp);
|
||||
@ -133,9 +110,13 @@ int readString(FILE *fp, char** target) {
|
||||
len += 2;
|
||||
*target = (char*)zmalloc(len);
|
||||
if (!readBytes(fp,*target,len)) {
|
||||
zfree(*target);
|
||||
*target = NULL;
|
||||
return 0;
|
||||
}
|
||||
if (!consumeNewline(*target+len-2)) {
|
||||
zfree(*target);
|
||||
*target = NULL;
|
||||
return 0;
|
||||
}
|
||||
(*target)[len-2] = '\0';
|
||||
@ -146,38 +127,154 @@ int readArgc(FILE *fp, long *target) {
|
||||
return readLong(fp,'*',target);
|
||||
}
|
||||
|
||||
off_t process(FILE *fp) {
|
||||
/* Used to decode a RESP record in the AOF file to obtain the original
|
||||
* redis command, and also check whether the command is MULTI/EXEC. If the
|
||||
* command is MULTI, the parameter out_multi will be incremented by one, and
|
||||
* if the command is EXEC, the parameter out_multi will be decremented
|
||||
* by one. The parameter out_multi will be used by the upper caller to determine
|
||||
* whether the AOF file contains unclosed transactions.
|
||||
**/
|
||||
int processRESP(FILE *fp, char *filename, int *out_multi) {
|
||||
long argc;
|
||||
off_t pos = 0;
|
||||
int i, multi = 0;
|
||||
char *str;
|
||||
|
||||
if (!readArgc(fp, &argc)) return 0;
|
||||
|
||||
for (int i = 0; i < argc; i++) {
|
||||
if (!readString(fp, &str)) return 0;
|
||||
if (i == 0) {
|
||||
if (strcasecmp(str, "multi") == 0) {
|
||||
if ((*out_multi)++) {
|
||||
ERROR("Unexpected MULTI in AOF %s", filename);
|
||||
zfree(str);
|
||||
return 0;
|
||||
}
|
||||
} else if (strcasecmp(str, "exec") == 0) {
|
||||
if (--(*out_multi)) {
|
||||
ERROR("Unexpected EXEC in AOF %s", filename);
|
||||
zfree(str);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
zfree(str);
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Used to parse an annotation in the AOF file, the annotation starts with '#'
|
||||
* in AOF. Currently AOF only contains timestamp annotations, but this function
|
||||
* can easily be extended to handle other annotations.
|
||||
*
|
||||
* The processing rule of time annotation is that once the timestamp is found to
|
||||
* be greater than 'to_timestamp', the AOF after the annotation is truncated.
|
||||
* Note that in Multi Part AOF, this truncation is only allowed when the last_file
|
||||
* parameter is 1.
|
||||
**/
|
||||
int processAnnotations(FILE *fp, char *filename, int last_file) {
|
||||
char buf[AOF_ANNOTATION_LINE_MAX_LEN];
|
||||
|
||||
epos = ftello(fp);
|
||||
if (fgets(buf, sizeof(buf), fp) == NULL) {
|
||||
printf("Failed to read annotations from AOF %s, aborting...\n", filename);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
if (to_timestamp && strncmp(buf, "#TS:", 4) == 0) {
|
||||
char *endptr;
|
||||
errno = 0;
|
||||
time_t ts = strtol(buf+4, &endptr, 10);
|
||||
if (errno != 0 || *endptr != '\r') {
|
||||
printf("Invalid timestamp annotation\n");
|
||||
exit(1);
|
||||
}
|
||||
if (ts <= to_timestamp) return 1;
|
||||
if (epos == 0) {
|
||||
printf("AOF %s has nothing before timestamp %ld, "
|
||||
"aborting...\n", filename, to_timestamp);
|
||||
exit(1);
|
||||
}
|
||||
if (!last_file) {
|
||||
printf("Failed to truncate AOF %s to timestamp %ld to offset %ld because it is not the last file.\n",
|
||||
filename, to_timestamp, (long int)epos);
|
||||
printf("If you insist, please delete all files after this file according to the manifest "
|
||||
"file and delete the corresponding records in manifest file manually. Then re-run redis-check-aof.\n");
|
||||
exit(1);
|
||||
}
|
||||
/* Truncate remaining AOF if exceeding 'to_timestamp' */
|
||||
if (ftruncate(fileno(fp), epos) == -1) {
|
||||
printf("Failed to truncate AOF %s to timestamp %ld\n",
|
||||
filename, to_timestamp);
|
||||
exit(1);
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Used to check the validity of a single AOF file. The AOF file can be:
|
||||
* 1. Old-style AOF
|
||||
* 2. Old-style RDB-preamble AOF
|
||||
* 3. BASE or INCR in Multi Part AOF
|
||||
* */
|
||||
int checkSingleAof(char *aof_filename, char *aof_filepath, int last_file, int fix, int preamble) {
|
||||
off_t pos = 0, diff;
|
||||
int multi = 0;
|
||||
char buf[2];
|
||||
|
||||
FILE *fp = fopen(aof_filepath, "r+");
|
||||
if (fp == NULL) {
|
||||
printf("Cannot open file: %s, aborting...\n", aof_filename);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
struct redis_stat sb;
|
||||
if (redis_fstat(fileno(fp),&sb) == -1) {
|
||||
printf("Cannot stat file: %s, aborting...\n", aof_filename);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
off_t size = sb.st_size;
|
||||
if (size == 0) {
|
||||
return AOF_CHECK_EMPTY;
|
||||
}
|
||||
|
||||
if (preamble) {
|
||||
char *argv[2] = {NULL, aof_filename};
|
||||
if (redis_check_rdb_main(2, argv, fp) == C_ERR) {
|
||||
printf("RDB preamble of AOF file is not sane, aborting.\n");
|
||||
exit(1);
|
||||
} else {
|
||||
printf("RDB preamble is OK, proceeding with AOF tail...\n");
|
||||
}
|
||||
}
|
||||
|
||||
while(1) {
|
||||
if (!multi) pos = ftello(fp);
|
||||
if (!readAnnotations(fp)) break;
|
||||
if (!readArgc(fp, &argc)) break;
|
||||
|
||||
for (i = 0; i < argc; i++) {
|
||||
if (!readString(fp,&str)) break;
|
||||
if (i == 0) {
|
||||
if (strcasecmp(str, "multi") == 0) {
|
||||
if (multi++) {
|
||||
ERROR("Unexpected MULTI");
|
||||
break;
|
||||
}
|
||||
} else if (strcasecmp(str, "exec") == 0) {
|
||||
if (--multi) {
|
||||
ERROR("Unexpected EXEC");
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (fgets(buf, sizeof(buf), fp) == NULL) {
|
||||
if (feof(fp)) {
|
||||
break;
|
||||
}
|
||||
zfree(str);
|
||||
printf("Failed to read from AOF %s, aborting...\n", aof_filename);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
/* Stop if the loop did not finish */
|
||||
if (i < argc) {
|
||||
if (str) zfree(str);
|
||||
if (fseek(fp, -1, SEEK_CUR) == -1) {
|
||||
printf("Failed to fseek in AOF %s: %s", aof_filename, strerror(errno));
|
||||
exit(1);
|
||||
}
|
||||
|
||||
if (buf[0] == '#') {
|
||||
if (!processAnnotations(fp, aof_filepath, last_file)) {
|
||||
fclose(fp);
|
||||
return AOF_CHECK_TIMESTAMP_TRUNCATED;
|
||||
}
|
||||
} else if (buf[0] == '*'){
|
||||
if (!processRESP(fp, aof_filepath, &multi)) break;
|
||||
} else {
|
||||
printf("AOF %s format error\n", aof_filename);
|
||||
break;
|
||||
}
|
||||
}
|
||||
@ -185,31 +282,270 @@ off_t process(FILE *fp) {
|
||||
if (feof(fp) && multi && strlen(error) == 0) {
|
||||
ERROR("Reached EOF before reading EXEC for MULTI");
|
||||
}
|
||||
|
||||
if (strlen(error) > 0) {
|
||||
printf("%s\n", error);
|
||||
}
|
||||
return pos;
|
||||
|
||||
diff = size-pos;
|
||||
|
||||
/* In truncate-to-timestamp mode, just exit if there is nothing to truncate. */
|
||||
if (diff == 0 && to_timestamp) {
|
||||
printf("Truncate nothing in AOF %s to timestamp %ld\n", aof_filename, to_timestamp);
|
||||
fclose(fp);
|
||||
return AOF_CHECK_OK;
|
||||
}
|
||||
|
||||
printf("AOF analyzed: filename=%s, size=%lld, ok_up_to=%lld, ok_up_to_line=%lld, diff=%lld\n",
|
||||
aof_filename, (long long) size, (long long) pos, line, (long long) diff);
|
||||
if (diff > 0) {
|
||||
if (fix) {
|
||||
if (!last_file) {
|
||||
printf("Failed to truncate AOF %s because it is not the last file\n", aof_filename);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
char buf[2];
|
||||
printf("This will shrink the AOF %s from %lld bytes, with %lld bytes, to %lld bytes\n",
|
||||
aof_filename, (long long)size, (long long)diff, (long long)pos);
|
||||
printf("Continue? [y/N]: ");
|
||||
if (fgets(buf, sizeof(buf), stdin) == NULL || strncasecmp(buf, "y", 1) != 0) {
|
||||
printf("Aborting...\n");
|
||||
exit(1);
|
||||
}
|
||||
if (ftruncate(fileno(fp), pos) == -1) {
|
||||
printf("Failed to truncate AOF %s\n", aof_filename);
|
||||
exit(1);
|
||||
} else {
|
||||
fclose(fp);
|
||||
return AOF_CHECK_TRUNCATED;
|
||||
}
|
||||
} else {
|
||||
printf("AOF %s is not valid. Use the --fix option to try fixing it.\n", aof_filename);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
fclose(fp);
|
||||
return AOF_CHECK_OK;
|
||||
}
|
||||
|
||||
/* Used to determine whether the file is a RDB file. These two possibilities:
|
||||
* 1. The file is an old style RDB-preamble AOF
|
||||
* 2. The file is a BASE AOF in Multi Part AOF
|
||||
* */
|
||||
int fileIsRDB(char *filepath) {
|
||||
FILE *fp = fopen(filepath, "r");
|
||||
if (fp == NULL) {
|
||||
printf("Cannot open file: %s\n", filepath);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
struct redis_stat sb;
|
||||
if (redis_fstat(fileno(fp), &sb) == -1) {
|
||||
printf("Cannot stat file: %s\n", filepath);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
off_t size = sb.st_size;
|
||||
if (size == 0) {
|
||||
fclose(fp);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (size >= 8) { /* There must be at least room for the RDB header. */
|
||||
char sig[5];
|
||||
int rdb_file = fread(sig, sizeof(sig), 1, fp) == 1 &&
|
||||
memcmp(sig, "REDIS", sizeof(sig)) == 0;
|
||||
if (rdb_file) {
|
||||
fclose(fp);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
fclose(fp);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Used to determine whether the file is a manifest file. */
|
||||
#define MANIFEST_MAX_LINE 1024
|
||||
int fileIsManifest(char *filepath) {
|
||||
int is_manifest = 0;
|
||||
FILE *fp = fopen(filepath, "r");
|
||||
if (fp == NULL) {
|
||||
printf("Cannot open file: %s\n", filepath);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
struct redis_stat sb;
|
||||
if (redis_fstat(fileno(fp), &sb) == -1) {
|
||||
printf("Cannot stat file: %s\n", filepath);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
off_t size = sb.st_size;
|
||||
if (size == 0) {
|
||||
fclose(fp);
|
||||
return 0;
|
||||
}
|
||||
|
||||
char buf[MANIFEST_MAX_LINE+1];
|
||||
while (1) {
|
||||
if (fgets(buf, MANIFEST_MAX_LINE+1, fp) == NULL) {
|
||||
if (feof(fp)) {
|
||||
break;
|
||||
} else {
|
||||
printf("Cannot read file: %s\n", filepath);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
/* Skip comments lines */
|
||||
if (buf[0] == '#') {
|
||||
continue;
|
||||
} else if (!memcmp(buf, "file", strlen("file"))) {
|
||||
is_manifest = 1;
|
||||
}
|
||||
}
|
||||
|
||||
fclose(fp);
|
||||
return is_manifest;
|
||||
}
|
||||
|
||||
/* Get the format of the file to be checked. It can be:
|
||||
* AOF_RESP: Old-style AOF
|
||||
* AOF_RDB_PREAMBLE: Old-style RDB-preamble AOF
|
||||
* AOF_MULTI_PART: manifest in Multi Part AOF
|
||||
*
|
||||
* redis-check-aof tool will automatically perform different
|
||||
* verification logic according to different file formats.
|
||||
* */
|
||||
input_file_type getInputFileType(char *filepath) {
|
||||
if (fileIsManifest(filepath)) {
|
||||
return AOF_MULTI_PART;
|
||||
} else if (fileIsRDB(filepath)) {
|
||||
return AOF_RDB_PREAMBLE;
|
||||
} else {
|
||||
return AOF_RESP;
|
||||
}
|
||||
}
|
||||
|
||||
/* Check if Multi Part AOF is valid. It will check the BASE file and INCR files
|
||||
* at once according to the manifest instructions (this is somewhat similar to
|
||||
* redis' AOF loading).
|
||||
*
|
||||
* When the verification is successful, we can guarantee:
|
||||
* 1. The manifest file format is valid
|
||||
* 2. Both BASE AOF and INCR AOFs format are valid
|
||||
* 3. No BASE or INCR AOFs files are missing
|
||||
*
|
||||
* Note that in Multi Part AOF, we only allow truncation for the last AOF file.
|
||||
* */
|
||||
void checkMultiPartAof(char *dirpath, char *manifest_filepath, int fix) {
|
||||
int total_num = 0, aof_num = 0, last_file;
|
||||
int ret;
|
||||
|
||||
printf("Start checking Multi Part AOF\n");
|
||||
aofManifest *am = aofLoadManifestFromFile(manifest_filepath);
|
||||
|
||||
if (am->base_aof_info) total_num++;
|
||||
if (am->incr_aof_list) total_num += listLength(am->incr_aof_list);
|
||||
|
||||
if (am->base_aof_info) {
|
||||
sds aof_filename = am->base_aof_info->file_name;
|
||||
sds aof_filepath = makePath(dirpath, aof_filename);
|
||||
last_file = ++aof_num == total_num;
|
||||
int aof_preable = fileIsRDB(aof_filepath);
|
||||
|
||||
printf("Start to check BASE AOF (%s format).\n", aof_preable ? "RDB":"RESP");
|
||||
ret = checkSingleAof(aof_filename, aof_filepath, last_file, fix, aof_preable);
|
||||
if (ret == AOF_CHECK_OK) {
|
||||
printf("BASE AOF %s is valid\n", aof_filename);
|
||||
} else if (ret == AOF_CHECK_EMPTY) {
|
||||
printf("BASE AOF %s is empty\n", aof_filename);
|
||||
} else if (ret == AOF_CHECK_TIMESTAMP_TRUNCATED) {
|
||||
printf("Successfully truncated AOF %s to timestamp %ld\n",
|
||||
aof_filename, to_timestamp);
|
||||
} else if (ret == AOF_CHECK_TRUNCATED) {
|
||||
printf("Successfully truncated AOF %s\n", aof_filename);
|
||||
}
|
||||
sdsfree(aof_filepath);
|
||||
}
|
||||
|
||||
if (listLength(am->incr_aof_list)) {
|
||||
listNode *ln;
|
||||
listIter li;
|
||||
|
||||
printf("Start to check INCR files.\n");
|
||||
listRewind(am->incr_aof_list, &li);
|
||||
while ((ln = listNext(&li)) != NULL) {
|
||||
aofInfo *ai = (aofInfo*)ln->value;
|
||||
sds aof_filename = (char*)ai->file_name;
|
||||
sds aof_filepath = makePath(dirpath, aof_filename);
|
||||
last_file = ++aof_num == total_num;
|
||||
ret = checkSingleAof(aof_filename, aof_filepath, last_file, fix, 0);
|
||||
if (ret == AOF_CHECK_OK) {
|
||||
printf("INCR AOF %s is valid\n", aof_filename);
|
||||
} else if (ret == AOF_CHECK_EMPTY) {
|
||||
printf("INCR AOF %s is empty\n", aof_filename);
|
||||
} else if (ret == AOF_CHECK_TIMESTAMP_TRUNCATED) {
|
||||
printf("Successfully truncated AOF %s to timestamp %ld\n",
|
||||
aof_filename, to_timestamp);
|
||||
} else if (ret == AOF_CHECK_TRUNCATED) {
|
||||
printf("Successfully truncated AOF %s\n", aof_filename);
|
||||
}
|
||||
sdsfree(aof_filepath);
|
||||
}
|
||||
}
|
||||
|
||||
aofManifestFree(am);
|
||||
printf("All AOF files and manifest are valid\n");
|
||||
}
|
||||
|
||||
/* Check if old style AOF is valid. Internally, it will identify whether
|
||||
* the AOF is in RDB-preamble format, and will eventually call `checkSingleAof`
|
||||
* to do the check. */
|
||||
void checkOldStyleAof(char *filepath, int fix, int preamble) {
|
||||
printf("Start checking Old-Style AOF\n");
|
||||
int ret = checkSingleAof(filepath, filepath, 1, fix, preamble);
|
||||
if (ret == AOF_CHECK_OK) {
|
||||
printf("AOF %s is valid\n", filepath);
|
||||
} else if (ret == AOF_CHECK_EMPTY) {
|
||||
printf("AOF %s is empty\n", filepath);
|
||||
} else if (ret == AOF_CHECK_TIMESTAMP_TRUNCATED) {
|
||||
printf("Successfully truncated AOF %s to timestamp %ld\n",
|
||||
filepath, to_timestamp);
|
||||
} else if (ret == AOF_CHECK_TRUNCATED) {
|
||||
printf("Successfully truncated AOF %s\n", filepath);
|
||||
}
|
||||
}
|
||||
|
||||
int redis_check_aof_main(int argc, char **argv) {
|
||||
char *filename;
|
||||
char *filepath;
|
||||
char temp_filepath[PATH_MAX + 1];
|
||||
char *dirpath;
|
||||
int fix = 0;
|
||||
|
||||
if (argc < 2) {
|
||||
goto invalid_args;
|
||||
} else if (argc == 2) {
|
||||
filename = argv[1];
|
||||
filepath = argv[1];
|
||||
} else if (argc == 3) {
|
||||
if (!strcmp(argv[1],"--fix")) {
|
||||
filename = argv[2];
|
||||
if (!strcmp(argv[1], "--fix")) {
|
||||
filepath = argv[2];
|
||||
fix = 1;
|
||||
} else {
|
||||
goto invalid_args;
|
||||
}
|
||||
} else if (argc == 4) {
|
||||
if (!strcmp(argv[1], "--truncate-to-timestamp")) {
|
||||
to_timestamp = strtol(argv[2],NULL,10);
|
||||
filename = argv[3];
|
||||
char *endptr;
|
||||
errno = 0;
|
||||
to_timestamp = strtol(argv[2], &endptr, 10);
|
||||
if (errno != 0 || *endptr != '\0') {
|
||||
printf("Invalid timestamp, aborting...\n");
|
||||
exit(1);
|
||||
}
|
||||
filepath = argv[3];
|
||||
} else {
|
||||
goto invalid_args;
|
||||
}
|
||||
@ -217,85 +553,28 @@ int redis_check_aof_main(int argc, char **argv) {
|
||||
goto invalid_args;
|
||||
}
|
||||
|
||||
FILE *fp = fopen(filename,"r+");
|
||||
if (fp == NULL) {
|
||||
printf("Cannot open file: %s\n", filename);
|
||||
exit(1);
|
||||
/* In the glibc implementation dirname may modify their argument. */
|
||||
memcpy(temp_filepath, filepath, strlen(filepath));
|
||||
dirpath = dirname(temp_filepath);
|
||||
|
||||
/* Select the corresponding verification method according to the input file type. */
|
||||
input_file_type type = getInputFileType(filepath);
|
||||
switch (type) {
|
||||
case AOF_MULTI_PART:
|
||||
checkMultiPartAof(dirpath, filepath, fix);
|
||||
break;
|
||||
case AOF_RESP:
|
||||
checkOldStyleAof(filepath, fix, 0);
|
||||
break;
|
||||
case AOF_RDB_PREAMBLE:
|
||||
checkOldStyleAof(filepath, fix, 1);
|
||||
break;
|
||||
}
|
||||
|
||||
struct redis_stat sb;
|
||||
if (redis_fstat(fileno(fp),&sb) == -1) {
|
||||
printf("Cannot stat file: %s\n", filename);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
off_t size = sb.st_size;
|
||||
if (size == 0) {
|
||||
printf("Empty file: %s\n", filename);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
/* This AOF file may have an RDB preamble. Check this to start, and if this
|
||||
* is the case, start processing the RDB part. */
|
||||
if (size >= 8) { /* There must be at least room for the RDB header. */
|
||||
char sig[5];
|
||||
int has_preamble = fread(sig,sizeof(sig),1,fp) == 1 &&
|
||||
memcmp(sig,"REDIS",sizeof(sig)) == 0;
|
||||
rewind(fp);
|
||||
if (has_preamble) {
|
||||
printf("The AOF appears to start with an RDB preamble.\n"
|
||||
"Checking the RDB preamble to start:\n");
|
||||
if (redis_check_rdb_main(argc,argv,fp) == C_ERR) {
|
||||
printf("RDB preamble of AOF file is not sane, aborting.\n");
|
||||
exit(1);
|
||||
} else {
|
||||
printf("RDB preamble is OK, proceeding with AOF tail...\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
off_t pos = process(fp);
|
||||
off_t diff = size-pos;
|
||||
|
||||
/* In truncate-to-timestamp mode, just exit if there is nothing to truncate. */
|
||||
if (diff == 0 && to_timestamp) {
|
||||
printf("Truncate nothing in AOF to timestamp %ld\n", to_timestamp);
|
||||
fclose(fp);
|
||||
exit(0);
|
||||
}
|
||||
|
||||
printf("AOF analyzed: size=%lld, ok_up_to=%lld, ok_up_to_line=%lld, diff=%lld\n",
|
||||
(long long) size, (long long) pos, line, (long long) diff);
|
||||
if (diff > 0) {
|
||||
if (fix) {
|
||||
char buf[2];
|
||||
printf("This will shrink the AOF from %lld bytes, with %lld bytes, to %lld bytes\n",(long long)size,(long long)diff,(long long)pos);
|
||||
printf("Continue? [y/N]: ");
|
||||
if (fgets(buf,sizeof(buf),stdin) == NULL ||
|
||||
strncasecmp(buf,"y",1) != 0) {
|
||||
printf("Aborting...\n");
|
||||
exit(1);
|
||||
}
|
||||
if (ftruncate(fileno(fp), pos) == -1) {
|
||||
printf("Failed to truncate AOF\n");
|
||||
exit(1);
|
||||
} else {
|
||||
printf("Successfully truncated AOF\n");
|
||||
}
|
||||
} else {
|
||||
printf("AOF is not valid. "
|
||||
"Use the --fix option to try fixing it.\n");
|
||||
exit(1);
|
||||
}
|
||||
} else {
|
||||
printf("AOF is valid\n");
|
||||
}
|
||||
|
||||
fclose(fp);
|
||||
exit(0);
|
||||
|
||||
invalid_args:
|
||||
printf("Usage: %s [--fix|--truncate-to-timestamp $timestamp] <file.aof>\n",
|
||||
argv[0]);
|
||||
printf("Usage: %s [--fix|--truncate-to-timestamp $timestamp] <file.manifest|file.aof>\n",
|
||||
argv[0]);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
@ -100,7 +100,7 @@ tags {"external:skip"} {
|
||||
fail "AOF loading didn't fail"
|
||||
}
|
||||
|
||||
assert_equal 1 [count_message_lines $server_path/stdout "The AOF manifest file is invalid format"]
|
||||
assert_equal 1 [count_message_lines $server_path/stdout "Invalid AOF manifest file format"]
|
||||
}
|
||||
|
||||
clean_aof_persistence $aof_dirpath
|
||||
@ -186,7 +186,7 @@ tags {"external:skip"} {
|
||||
fail "AOF loading didn't fail"
|
||||
}
|
||||
|
||||
assert_equal 2 [count_message_lines $server_path/stdout "The AOF manifest file is invalid format"]
|
||||
assert_equal 2 [count_message_lines $server_path/stdout "Invalid AOF manifest file format"]
|
||||
}
|
||||
|
||||
clean_aof_persistence $aof_dirpath
|
||||
@ -213,7 +213,7 @@ tags {"external:skip"} {
|
||||
fail "AOF loading didn't fail"
|
||||
}
|
||||
|
||||
assert_equal 3 [count_message_lines $server_path/stdout "The AOF manifest file is invalid format"]
|
||||
assert_equal 3 [count_message_lines $server_path/stdout "Invalid AOF manifest file format"]
|
||||
}
|
||||
|
||||
clean_aof_persistence $aof_dirpath
|
||||
@ -267,7 +267,7 @@ tags {"external:skip"} {
|
||||
fail "AOF loading didn't fail"
|
||||
}
|
||||
|
||||
assert_equal 4 [count_message_lines $server_path/stdout "The AOF manifest file is invalid format"]
|
||||
assert_equal 4 [count_message_lines $server_path/stdout "Invalid AOF manifest file format"]
|
||||
}
|
||||
|
||||
clean_aof_persistence $aof_dirpath
|
||||
|
||||
@ -4,6 +4,7 @@ set server_path [tmpdir server.aof]
|
||||
set aof_dirname "appendonlydir"
|
||||
set aof_basename "appendonly.aof"
|
||||
set aof_dirpath "$server_path/$aof_dirname"
|
||||
set aof_base_file "$server_path/$aof_dirname/${aof_basename}.1$::base_aof_sufix$::aof_format_suffix"
|
||||
set aof_file "$server_path/$aof_dirname/${aof_basename}.1$::incr_aof_sufix$::aof_format_suffix"
|
||||
set aof_manifest_file "$server_path/$aof_dirname/$aof_basename$::manifest_suffix"
|
||||
|
||||
@ -142,7 +143,7 @@ tags {"aof external:skip"} {
|
||||
## Test that redis-check-aof indeed sees this AOF is not valid
|
||||
test "Short read: Utility should confirm the AOF is not valid" {
|
||||
catch {
|
||||
exec src/redis-check-aof $aof_file
|
||||
exec src/redis-check-aof $aof_manifest_file
|
||||
} result
|
||||
assert_match "*not valid*" $result
|
||||
}
|
||||
@ -154,13 +155,13 @@ tags {"aof external:skip"} {
|
||||
}
|
||||
|
||||
catch {
|
||||
exec src/redis-check-aof $aof_file
|
||||
exec src/redis-check-aof $aof_manifest_file
|
||||
} result
|
||||
assert_match "*ok_up_to_line=8*" $result
|
||||
}
|
||||
|
||||
test "Short read: Utility should be able to fix the AOF" {
|
||||
set result [exec src/redis-check-aof --fix $aof_file << "y\n"]
|
||||
set result [exec src/redis-check-aof --fix $aof_manifest_file << "y\n"]
|
||||
assert_match "*Successfully truncated AOF*" $result
|
||||
}
|
||||
|
||||
@ -444,7 +445,7 @@ tags {"aof external:skip"} {
|
||||
|
||||
test {Truncate AOF to specific timestamp} {
|
||||
# truncate to timestamp 1628217473
|
||||
exec src/redis-check-aof --truncate-to-timestamp 1628217473 $aof_file
|
||||
exec src/redis-check-aof --truncate-to-timestamp 1628217473 $aof_manifest_file
|
||||
start_server_aof [list dir $server_path] {
|
||||
set c [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
|
||||
wait_done_loading $c
|
||||
@ -454,7 +455,7 @@ tags {"aof external:skip"} {
|
||||
}
|
||||
|
||||
# truncate to timestamp 1628217471
|
||||
exec src/redis-check-aof --truncate-to-timestamp 1628217471 $aof_file
|
||||
exec src/redis-check-aof --truncate-to-timestamp 1628217471 $aof_manifest_file
|
||||
start_server_aof [list dir $server_path] {
|
||||
set c [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
|
||||
wait_done_loading $c
|
||||
@ -464,7 +465,7 @@ tags {"aof external:skip"} {
|
||||
}
|
||||
|
||||
# truncate to timestamp 1628217470
|
||||
exec src/redis-check-aof --truncate-to-timestamp 1628217470 $aof_file
|
||||
exec src/redis-check-aof --truncate-to-timestamp 1628217470 $aof_manifest_file
|
||||
start_server_aof [list dir $server_path] {
|
||||
set c [redis [dict get $srv host] [dict get $srv port] 0 $::tls]
|
||||
wait_done_loading $c
|
||||
@ -473,7 +474,7 @@ tags {"aof external:skip"} {
|
||||
}
|
||||
|
||||
# truncate to timestamp 1628217469
|
||||
catch {exec src/redis-check-aof --truncate-to-timestamp 1628217469 $aof_file} e
|
||||
catch {exec src/redis-check-aof --truncate-to-timestamp 1628217469 $aof_manifest_file} e
|
||||
assert_match {*aborting*} $e
|
||||
}
|
||||
|
||||
@ -515,4 +516,120 @@ tags {"aof external:skip"} {
|
||||
assert_equal [r get foo] 102
|
||||
}
|
||||
}
|
||||
|
||||
test {Test redis-check-aof for old style resp AOF} {
|
||||
create_aof $aof_dirpath $aof_file {
|
||||
append_to_aof [formatCommand set foo hello]
|
||||
append_to_aof [formatCommand set bar world]
|
||||
}
|
||||
|
||||
catch {
|
||||
exec src/redis-check-aof $aof_file
|
||||
} result
|
||||
assert_match "*Start checking Old-Style AOF*is valid*" $result
|
||||
}
|
||||
|
||||
test {Test redis-check-aof for old style rdb-preamble AOF} {
|
||||
catch {
|
||||
exec src/redis-check-aof tests/assets/rdb-preamble.aof
|
||||
} result
|
||||
assert_match "*Start checking Old-Style AOF*RDB preamble is OK, proceeding with AOF tail*is valid*" $result
|
||||
}
|
||||
|
||||
test {Test redis-check-aof for Multi Part AOF with resp AOF base} {
|
||||
create_aof $aof_dirpath $aof_base_file {
|
||||
append_to_aof [formatCommand set foo hello]
|
||||
append_to_aof [formatCommand set bar world]
|
||||
}
|
||||
|
||||
create_aof $aof_dirpath $aof_file {
|
||||
append_to_aof [formatCommand set foo hello]
|
||||
append_to_aof [formatCommand set bar world]
|
||||
}
|
||||
|
||||
create_aof_manifest $aof_dirpath $aof_manifest_file {
|
||||
append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n"
|
||||
append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
|
||||
}
|
||||
|
||||
catch {
|
||||
exec src/redis-check-aof $aof_manifest_file
|
||||
} result
|
||||
assert_match "*Start checking Multi Part AOF*Start to check BASE AOF (RESP format)*BASE AOF*is valid*Start to check INCR files*INCR AOF*is valid*All AOF files and manifest are valid*" $result
|
||||
}
|
||||
|
||||
test {Test redis-check-aof for Multi Part AOF with rdb-preamble AOF base} {
|
||||
exec cp tests/assets/rdb-preamble.aof $aof_base_file
|
||||
|
||||
create_aof $aof_dirpath $aof_file {
|
||||
append_to_aof [formatCommand set foo hello]
|
||||
append_to_aof [formatCommand set bar world]
|
||||
}
|
||||
|
||||
create_aof_manifest $aof_dirpath $aof_manifest_file {
|
||||
append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n"
|
||||
append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
|
||||
}
|
||||
|
||||
catch {
|
||||
exec src/redis-check-aof $aof_manifest_file
|
||||
} result
|
||||
assert_match "*Start checking Multi Part AOF*Start to check BASE AOF (RDB format)*DB preamble is OK, proceeding with AOF tail*BASE AOF*is valid*Start to check INCR files*INCR AOF*is valid*All AOF files and manifest are valid*" $result
|
||||
}
|
||||
|
||||
test {Test redis-check-aof only truncates the last file for Multi Part AOF in fix mode} {
|
||||
create_aof $aof_dirpath $aof_base_file {
|
||||
append_to_aof [formatCommand set foo hello]
|
||||
append_to_aof [formatCommand multi]
|
||||
append_to_aof [formatCommand set bar world]
|
||||
}
|
||||
|
||||
create_aof $aof_dirpath $aof_file {
|
||||
append_to_aof [formatCommand set foo hello]
|
||||
append_to_aof [formatCommand set bar world]
|
||||
}
|
||||
|
||||
create_aof_manifest $aof_dirpath $aof_manifest_file {
|
||||
append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n"
|
||||
append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
|
||||
}
|
||||
|
||||
catch {
|
||||
exec src/redis-check-aof $aof_manifest_file
|
||||
} result
|
||||
assert_match "*not valid*" $result
|
||||
|
||||
catch {
|
||||
exec src/redis-check-aof --fix $aof_manifest_file
|
||||
} result
|
||||
assert_match "*Failed to truncate AOF*because it is not the last file*" $result
|
||||
}
|
||||
|
||||
test {Test redis-check-aof only truncates the last file for Multi Part AOF in truncate-to-timestamp mode} {
|
||||
create_aof $aof_dirpath $aof_base_file {
|
||||
append_to_aof "#TS:1628217470\r\n"
|
||||
append_to_aof [formatCommand set foo1 bar1]
|
||||
append_to_aof "#TS:1628217471\r\n"
|
||||
append_to_aof [formatCommand set foo2 bar2]
|
||||
append_to_aof "#TS:1628217472\r\n"
|
||||
append_to_aof "#TS:1628217473\r\n"
|
||||
append_to_aof [formatCommand set foo3 bar3]
|
||||
append_to_aof "#TS:1628217474\r\n"
|
||||
}
|
||||
|
||||
create_aof $aof_dirpath $aof_file {
|
||||
append_to_aof [formatCommand set foo hello]
|
||||
append_to_aof [formatCommand set bar world]
|
||||
}
|
||||
|
||||
create_aof_manifest $aof_dirpath $aof_manifest_file {
|
||||
append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n"
|
||||
append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n"
|
||||
}
|
||||
|
||||
catch {
|
||||
exec src/redis-check-aof --truncate-to-timestamp 1628217473 $aof_manifest_file
|
||||
} result
|
||||
assert_match "*Failed to truncate AOF*to timestamp*because it is not the last file*" $result
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user