diff --git a/src/aof.c b/src/aof.c index 7fca84ef8..d7ba6b318 100644 --- a/src/aof.c +++ b/src/aof.c @@ -47,6 +47,8 @@ off_t getBaseAndIncrAppendOnlyFilesSize(aofManifest *am); int getBaseAndIncrAppendOnlyFilesNum(aofManifest *am); int aofFileExist(char *filename); int rewriteAppendOnlyFile(char *filename); +aofManifest *aofLoadManifestFromFile(sds am_filepath); +void aofManifestFreeAndUpdate(aofManifest *am); /* ---------------------------------------------------------------------------- * AOF Manifest file implementation. @@ -226,13 +228,8 @@ sds getAofManifestAsString(aofManifest *am) { * in order to support seamless upgrades from previous versions which did not * use them. */ -#define MANIFEST_MAX_LINE 1024 void aofLoadManifestFromDisk(void) { - const char *err = NULL; - long long maxseq = 0; - server.aof_manifest = aofManifestCreate(); - if (!dirExists(server.aof_dirname)) { serverLog(LL_NOTICE, "The AOF directory %s doesn't exist", server.aof_dirname); return; @@ -247,16 +244,26 @@ void aofLoadManifestFromDisk(void) { return; } + aofManifest *am = aofLoadManifestFromFile(am_filepath); + if (am) aofManifestFreeAndUpdate(am); + sdsfree(am_name); + sdsfree(am_filepath); +} + +/* Generic manifest loading function, used in `aofLoadManifestFromDisk` and redis-check-aof tool. */ +#define MANIFEST_MAX_LINE 1024 +aofManifest *aofLoadManifestFromFile(sds am_filepath) { + const char *err = NULL; + long long maxseq = 0; + + aofManifest *am = aofManifestCreate(); FILE *fp = fopen(am_filepath, "r"); if (fp == NULL) { serverLog(LL_WARNING, "Fatal error: can't open the AOF manifest " - "file %s for reading: %s", am_name, strerror(errno)); + "file %s for reading: %s", am_filepath, strerror(errno)); exit(1); } - sdsfree(am_name); - sdsfree(am_filepath); - char buf[MANIFEST_MAX_LINE+1]; sds *argv = NULL; int argc; @@ -292,14 +299,14 @@ void aofLoadManifestFromDisk(void) { line = sdstrim(sdsnew(buf), " \t\r\n"); if (!sdslen(line)) { - err = "The AOF manifest file is invalid format"; + err = "Invalid AOF manifest file format"; goto loaderr; } argv = sdssplitargs(line, &argc); /* 'argc < 6' was done for forward compatibility. */ if (argv == NULL || argc < 6 || (argc % 2)) { - err = "The AOF manifest file is invalid format"; + err = "Invalid AOF manifest file format"; goto loaderr; } @@ -321,7 +328,7 @@ void aofLoadManifestFromDisk(void) { /* We have to make sure we load all the information. */ if (!ai->file_name || !ai->file_seq || !ai->file_type) { - err = "The AOF manifest file is invalid format"; + err = "Invalid AOF manifest file format"; goto loaderr; } @@ -329,21 +336,21 @@ void aofLoadManifestFromDisk(void) { argv = NULL; if (ai->file_type == AOF_FILE_TYPE_BASE) { - if (server.aof_manifest->base_aof_info) { + if (am->base_aof_info) { err = "Found duplicate base file information"; goto loaderr; } - server.aof_manifest->base_aof_info = ai; - server.aof_manifest->curr_base_file_seq = ai->file_seq; + am->base_aof_info = ai; + am->curr_base_file_seq = ai->file_seq; } else if (ai->file_type == AOF_FILE_TYPE_HIST) { - listAddNodeTail(server.aof_manifest->history_aof_list, ai); + listAddNodeTail(am->history_aof_list, ai); } else if (ai->file_type == AOF_FILE_TYPE_INCR) { if (ai->file_seq <= maxseq) { err = "Found a non-monotonic sequence number"; goto loaderr; } - listAddNodeTail(server.aof_manifest->incr_aof_list, ai); - server.aof_manifest->curr_incr_file_seq = ai->file_seq; + listAddNodeTail(am->incr_aof_list, ai); + am->curr_incr_file_seq = ai->file_seq; maxseq = ai->file_seq; } else { err = "Unknown AOF file type"; @@ -356,7 +363,7 @@ void aofLoadManifestFromDisk(void) { } fclose(fp); - return; + return am; loaderr: /* Sanitizer suppression: may report a false positive if we goto loaderr @@ -1526,15 +1533,15 @@ uxeof: /* Unexpected AOF end of file. */ } } } - serverLog(LL_WARNING,"Unexpected end of file reading the append only file %s. You can: \ - 1) Make a backup of your AOF file, then use ./redis-check-aof --fix . \ - 2) Alternatively you can set the 'aof-load-truncated' configuration option to yes and restart the server.", filename); + serverLog(LL_WARNING, "Unexpected end of file reading the append only file %s. You can: " + "1) Make a backup of your AOF file, then use ./redis-check-aof --fix . " + "2) Alternatively you can set the 'aof-load-truncated' configuration option to yes and restart the server.", filename); ret = AOF_FAILED; goto cleanup; fmterr: /* Format error. */ - serverLog(LL_WARNING,"Bad file format reading the append only file %s: \ - make a backup of your AOF file, then use ./redis-check-aof --fix ", filename); + serverLog(LL_WARNING, "Bad file format reading the append only file %s: " + "make a backup of your AOF file, then use ./redis-check-aof --fix ", filename); ret = AOF_FAILED; /* fall through to cleanup. */ diff --git a/src/redis-check-aof.c b/src/redis-check-aof.c index 01f42ec1b..d2ff21068 100644 --- a/src/redis-check-aof.c +++ b/src/redis-check-aof.c @@ -30,6 +30,24 @@ #include "server.h" #include +#include +#include +#include + +#define AOF_CHECK_OK 0 +#define AOF_CHECK_EMPTY 1 +#define AOF_CHECK_TRUNCATED 2 +#define AOF_CHECK_TIMESTAMP_TRUNCATED 3 + +typedef enum { + AOF_RESP, + AOF_RDB_PREAMBLE, + AOF_MULTI_PART, +} input_file_type; + +aofManifest *aofManifestCreate(void); +void aofManifestFree(aofManifest *am); +aofManifest *aofLoadManifestFromFile(sds am_filepath); #define ERROR(...) { \ char __buf[1024]; \ @@ -51,47 +69,6 @@ int consumeNewline(char *buf) { return 1; } -int readAnnotations(FILE *fp) { - char buf[AOF_ANNOTATION_LINE_MAX_LEN]; - while (1) { - epos = ftello(fp); - if (fgets(buf, sizeof(buf), fp) == NULL) { - return 0; - } - if (buf[0] == '#') { - if (to_timestamp && strncmp(buf, "#TS:", 4) == 0) { - time_t ts = strtol(buf+4, NULL, 10); - if (ts <= to_timestamp) continue; - if (epos == 0) { - printf("AOF has nothing before timestamp %ld, " - "aborting...\n", to_timestamp); - fclose(fp); - exit(1); - } - /* Truncate remaining AOF if exceeding 'to_timestamp' */ - if (ftruncate(fileno(fp), epos) == -1) { - printf("Failed to truncate AOF to timestamp %ld\n", - to_timestamp); - exit(1); - } else { - printf("Successfully truncated AOF to timestamp %ld\n", - to_timestamp); - fclose(fp); - exit(0); - } - } - continue; - } else { - if (fseek(fp, -(ftello(fp)-epos), SEEK_CUR) == -1) { - ERROR("Fseek error: %s", strerror(errno)); - return 0; - } - return 1; - } - } - return 1; -} - int readLong(FILE *fp, char prefix, long *target) { char buf[128], *eptr; epos = ftello(fp); @@ -133,9 +110,13 @@ int readString(FILE *fp, char** target) { len += 2; *target = (char*)zmalloc(len); if (!readBytes(fp,*target,len)) { + zfree(*target); + *target = NULL; return 0; } if (!consumeNewline(*target+len-2)) { + zfree(*target); + *target = NULL; return 0; } (*target)[len-2] = '\0'; @@ -146,38 +127,154 @@ int readArgc(FILE *fp, long *target) { return readLong(fp,'*',target); } -off_t process(FILE *fp) { +/* Used to decode a RESP record in the AOF file to obtain the original + * redis command, and also check whether the command is MULTI/EXEC. If the + * command is MULTI, the parameter out_multi will be incremented by one, and + * if the command is EXEC, the parameter out_multi will be decremented + * by one. The parameter out_multi will be used by the upper caller to determine + * whether the AOF file contains unclosed transactions. + **/ +int processRESP(FILE *fp, char *filename, int *out_multi) { long argc; - off_t pos = 0; - int i, multi = 0; char *str; + if (!readArgc(fp, &argc)) return 0; + + for (int i = 0; i < argc; i++) { + if (!readString(fp, &str)) return 0; + if (i == 0) { + if (strcasecmp(str, "multi") == 0) { + if ((*out_multi)++) { + ERROR("Unexpected MULTI in AOF %s", filename); + zfree(str); + return 0; + } + } else if (strcasecmp(str, "exec") == 0) { + if (--(*out_multi)) { + ERROR("Unexpected EXEC in AOF %s", filename); + zfree(str); + return 0; + } + } + } + zfree(str); + } + + return 1; +} + +/* Used to parse an annotation in the AOF file, the annotation starts with '#' + * in AOF. Currently AOF only contains timestamp annotations, but this function + * can easily be extended to handle other annotations. + * + * The processing rule of time annotation is that once the timestamp is found to + * be greater than 'to_timestamp', the AOF after the annotation is truncated. + * Note that in Multi Part AOF, this truncation is only allowed when the last_file + * parameter is 1. + **/ +int processAnnotations(FILE *fp, char *filename, int last_file) { + char buf[AOF_ANNOTATION_LINE_MAX_LEN]; + + epos = ftello(fp); + if (fgets(buf, sizeof(buf), fp) == NULL) { + printf("Failed to read annotations from AOF %s, aborting...\n", filename); + exit(1); + } + + if (to_timestamp && strncmp(buf, "#TS:", 4) == 0) { + char *endptr; + errno = 0; + time_t ts = strtol(buf+4, &endptr, 10); + if (errno != 0 || *endptr != '\r') { + printf("Invalid timestamp annotation\n"); + exit(1); + } + if (ts <= to_timestamp) return 1; + if (epos == 0) { + printf("AOF %s has nothing before timestamp %ld, " + "aborting...\n", filename, to_timestamp); + exit(1); + } + if (!last_file) { + printf("Failed to truncate AOF %s to timestamp %ld to offset %ld because it is not the last file.\n", + filename, to_timestamp, (long int)epos); + printf("If you insist, please delete all files after this file according to the manifest " + "file and delete the corresponding records in manifest file manually. Then re-run redis-check-aof.\n"); + exit(1); + } + /* Truncate remaining AOF if exceeding 'to_timestamp' */ + if (ftruncate(fileno(fp), epos) == -1) { + printf("Failed to truncate AOF %s to timestamp %ld\n", + filename, to_timestamp); + exit(1); + } else { + return 0; + } + } + return 1; +} + +/* Used to check the validity of a single AOF file. The AOF file can be: + * 1. Old-style AOF + * 2. Old-style RDB-preamble AOF + * 3. BASE or INCR in Multi Part AOF + * */ +int checkSingleAof(char *aof_filename, char *aof_filepath, int last_file, int fix, int preamble) { + off_t pos = 0, diff; + int multi = 0; + char buf[2]; + + FILE *fp = fopen(aof_filepath, "r+"); + if (fp == NULL) { + printf("Cannot open file: %s, aborting...\n", aof_filename); + exit(1); + } + + struct redis_stat sb; + if (redis_fstat(fileno(fp),&sb) == -1) { + printf("Cannot stat file: %s, aborting...\n", aof_filename); + exit(1); + } + + off_t size = sb.st_size; + if (size == 0) { + return AOF_CHECK_EMPTY; + } + + if (preamble) { + char *argv[2] = {NULL, aof_filename}; + if (redis_check_rdb_main(2, argv, fp) == C_ERR) { + printf("RDB preamble of AOF file is not sane, aborting.\n"); + exit(1); + } else { + printf("RDB preamble is OK, proceeding with AOF tail...\n"); + } + } + while(1) { if (!multi) pos = ftello(fp); - if (!readAnnotations(fp)) break; - if (!readArgc(fp, &argc)) break; - - for (i = 0; i < argc; i++) { - if (!readString(fp,&str)) break; - if (i == 0) { - if (strcasecmp(str, "multi") == 0) { - if (multi++) { - ERROR("Unexpected MULTI"); - break; - } - } else if (strcasecmp(str, "exec") == 0) { - if (--multi) { - ERROR("Unexpected EXEC"); - break; - } - } + if (fgets(buf, sizeof(buf), fp) == NULL) { + if (feof(fp)) { + break; } - zfree(str); + printf("Failed to read from AOF %s, aborting...\n", aof_filename); + exit(1); } - /* Stop if the loop did not finish */ - if (i < argc) { - if (str) zfree(str); + if (fseek(fp, -1, SEEK_CUR) == -1) { + printf("Failed to fseek in AOF %s: %s", aof_filename, strerror(errno)); + exit(1); + } + + if (buf[0] == '#') { + if (!processAnnotations(fp, aof_filepath, last_file)) { + fclose(fp); + return AOF_CHECK_TIMESTAMP_TRUNCATED; + } + } else if (buf[0] == '*'){ + if (!processRESP(fp, aof_filepath, &multi)) break; + } else { + printf("AOF %s format error\n", aof_filename); break; } } @@ -185,31 +282,270 @@ off_t process(FILE *fp) { if (feof(fp) && multi && strlen(error) == 0) { ERROR("Reached EOF before reading EXEC for MULTI"); } + if (strlen(error) > 0) { printf("%s\n", error); } - return pos; + + diff = size-pos; + + /* In truncate-to-timestamp mode, just exit if there is nothing to truncate. */ + if (diff == 0 && to_timestamp) { + printf("Truncate nothing in AOF %s to timestamp %ld\n", aof_filename, to_timestamp); + fclose(fp); + return AOF_CHECK_OK; + } + + printf("AOF analyzed: filename=%s, size=%lld, ok_up_to=%lld, ok_up_to_line=%lld, diff=%lld\n", + aof_filename, (long long) size, (long long) pos, line, (long long) diff); + if (diff > 0) { + if (fix) { + if (!last_file) { + printf("Failed to truncate AOF %s because it is not the last file\n", aof_filename); + exit(1); + } + + char buf[2]; + printf("This will shrink the AOF %s from %lld bytes, with %lld bytes, to %lld bytes\n", + aof_filename, (long long)size, (long long)diff, (long long)pos); + printf("Continue? [y/N]: "); + if (fgets(buf, sizeof(buf), stdin) == NULL || strncasecmp(buf, "y", 1) != 0) { + printf("Aborting...\n"); + exit(1); + } + if (ftruncate(fileno(fp), pos) == -1) { + printf("Failed to truncate AOF %s\n", aof_filename); + exit(1); + } else { + fclose(fp); + return AOF_CHECK_TRUNCATED; + } + } else { + printf("AOF %s is not valid. Use the --fix option to try fixing it.\n", aof_filename); + exit(1); + } + } + fclose(fp); + return AOF_CHECK_OK; +} + +/* Used to determine whether the file is a RDB file. These two possibilities: + * 1. The file is an old style RDB-preamble AOF + * 2. The file is a BASE AOF in Multi Part AOF + * */ +int fileIsRDB(char *filepath) { + FILE *fp = fopen(filepath, "r"); + if (fp == NULL) { + printf("Cannot open file: %s\n", filepath); + exit(1); + } + + struct redis_stat sb; + if (redis_fstat(fileno(fp), &sb) == -1) { + printf("Cannot stat file: %s\n", filepath); + exit(1); + } + + off_t size = sb.st_size; + if (size == 0) { + fclose(fp); + return 0; + } + + if (size >= 8) { /* There must be at least room for the RDB header. */ + char sig[5]; + int rdb_file = fread(sig, sizeof(sig), 1, fp) == 1 && + memcmp(sig, "REDIS", sizeof(sig)) == 0; + if (rdb_file) { + fclose(fp); + return 1; + } + } + + fclose(fp); + return 0; +} + +/* Used to determine whether the file is a manifest file. */ +#define MANIFEST_MAX_LINE 1024 +int fileIsManifest(char *filepath) { + int is_manifest = 0; + FILE *fp = fopen(filepath, "r"); + if (fp == NULL) { + printf("Cannot open file: %s\n", filepath); + exit(1); + } + + struct redis_stat sb; + if (redis_fstat(fileno(fp), &sb) == -1) { + printf("Cannot stat file: %s\n", filepath); + exit(1); + } + + off_t size = sb.st_size; + if (size == 0) { + fclose(fp); + return 0; + } + + char buf[MANIFEST_MAX_LINE+1]; + while (1) { + if (fgets(buf, MANIFEST_MAX_LINE+1, fp) == NULL) { + if (feof(fp)) { + break; + } else { + printf("Cannot read file: %s\n", filepath); + exit(1); + } + } + + /* Skip comments lines */ + if (buf[0] == '#') { + continue; + } else if (!memcmp(buf, "file", strlen("file"))) { + is_manifest = 1; + } + } + + fclose(fp); + return is_manifest; +} + +/* Get the format of the file to be checked. It can be: + * AOF_RESP: Old-style AOF + * AOF_RDB_PREAMBLE: Old-style RDB-preamble AOF + * AOF_MULTI_PART: manifest in Multi Part AOF + * + * redis-check-aof tool will automatically perform different + * verification logic according to different file formats. + * */ +input_file_type getInputFileType(char *filepath) { + if (fileIsManifest(filepath)) { + return AOF_MULTI_PART; + } else if (fileIsRDB(filepath)) { + return AOF_RDB_PREAMBLE; + } else { + return AOF_RESP; + } +} + +/* Check if Multi Part AOF is valid. It will check the BASE file and INCR files + * at once according to the manifest instructions (this is somewhat similar to + * redis' AOF loading). + * + * When the verification is successful, we can guarantee: + * 1. The manifest file format is valid + * 2. Both BASE AOF and INCR AOFs format are valid + * 3. No BASE or INCR AOFs files are missing + * + * Note that in Multi Part AOF, we only allow truncation for the last AOF file. + * */ +void checkMultiPartAof(char *dirpath, char *manifest_filepath, int fix) { + int total_num = 0, aof_num = 0, last_file; + int ret; + + printf("Start checking Multi Part AOF\n"); + aofManifest *am = aofLoadManifestFromFile(manifest_filepath); + + if (am->base_aof_info) total_num++; + if (am->incr_aof_list) total_num += listLength(am->incr_aof_list); + + if (am->base_aof_info) { + sds aof_filename = am->base_aof_info->file_name; + sds aof_filepath = makePath(dirpath, aof_filename); + last_file = ++aof_num == total_num; + int aof_preable = fileIsRDB(aof_filepath); + + printf("Start to check BASE AOF (%s format).\n", aof_preable ? "RDB":"RESP"); + ret = checkSingleAof(aof_filename, aof_filepath, last_file, fix, aof_preable); + if (ret == AOF_CHECK_OK) { + printf("BASE AOF %s is valid\n", aof_filename); + } else if (ret == AOF_CHECK_EMPTY) { + printf("BASE AOF %s is empty\n", aof_filename); + } else if (ret == AOF_CHECK_TIMESTAMP_TRUNCATED) { + printf("Successfully truncated AOF %s to timestamp %ld\n", + aof_filename, to_timestamp); + } else if (ret == AOF_CHECK_TRUNCATED) { + printf("Successfully truncated AOF %s\n", aof_filename); + } + sdsfree(aof_filepath); + } + + if (listLength(am->incr_aof_list)) { + listNode *ln; + listIter li; + + printf("Start to check INCR files.\n"); + listRewind(am->incr_aof_list, &li); + while ((ln = listNext(&li)) != NULL) { + aofInfo *ai = (aofInfo*)ln->value; + sds aof_filename = (char*)ai->file_name; + sds aof_filepath = makePath(dirpath, aof_filename); + last_file = ++aof_num == total_num; + ret = checkSingleAof(aof_filename, aof_filepath, last_file, fix, 0); + if (ret == AOF_CHECK_OK) { + printf("INCR AOF %s is valid\n", aof_filename); + } else if (ret == AOF_CHECK_EMPTY) { + printf("INCR AOF %s is empty\n", aof_filename); + } else if (ret == AOF_CHECK_TIMESTAMP_TRUNCATED) { + printf("Successfully truncated AOF %s to timestamp %ld\n", + aof_filename, to_timestamp); + } else if (ret == AOF_CHECK_TRUNCATED) { + printf("Successfully truncated AOF %s\n", aof_filename); + } + sdsfree(aof_filepath); + } + } + + aofManifestFree(am); + printf("All AOF files and manifest are valid\n"); +} + +/* Check if old style AOF is valid. Internally, it will identify whether + * the AOF is in RDB-preamble format, and will eventually call `checkSingleAof` + * to do the check. */ +void checkOldStyleAof(char *filepath, int fix, int preamble) { + printf("Start checking Old-Style AOF\n"); + int ret = checkSingleAof(filepath, filepath, 1, fix, preamble); + if (ret == AOF_CHECK_OK) { + printf("AOF %s is valid\n", filepath); + } else if (ret == AOF_CHECK_EMPTY) { + printf("AOF %s is empty\n", filepath); + } else if (ret == AOF_CHECK_TIMESTAMP_TRUNCATED) { + printf("Successfully truncated AOF %s to timestamp %ld\n", + filepath, to_timestamp); + } else if (ret == AOF_CHECK_TRUNCATED) { + printf("Successfully truncated AOF %s\n", filepath); + } } int redis_check_aof_main(int argc, char **argv) { - char *filename; + char *filepath; + char temp_filepath[PATH_MAX + 1]; + char *dirpath; int fix = 0; if (argc < 2) { goto invalid_args; } else if (argc == 2) { - filename = argv[1]; + filepath = argv[1]; } else if (argc == 3) { - if (!strcmp(argv[1],"--fix")) { - filename = argv[2]; + if (!strcmp(argv[1], "--fix")) { + filepath = argv[2]; fix = 1; } else { goto invalid_args; } } else if (argc == 4) { if (!strcmp(argv[1], "--truncate-to-timestamp")) { - to_timestamp = strtol(argv[2],NULL,10); - filename = argv[3]; + char *endptr; + errno = 0; + to_timestamp = strtol(argv[2], &endptr, 10); + if (errno != 0 || *endptr != '\0') { + printf("Invalid timestamp, aborting...\n"); + exit(1); + } + filepath = argv[3]; } else { goto invalid_args; } @@ -217,85 +553,28 @@ int redis_check_aof_main(int argc, char **argv) { goto invalid_args; } - FILE *fp = fopen(filename,"r+"); - if (fp == NULL) { - printf("Cannot open file: %s\n", filename); - exit(1); + /* In the glibc implementation dirname may modify their argument. */ + memcpy(temp_filepath, filepath, strlen(filepath)); + dirpath = dirname(temp_filepath); + + /* Select the corresponding verification method according to the input file type. */ + input_file_type type = getInputFileType(filepath); + switch (type) { + case AOF_MULTI_PART: + checkMultiPartAof(dirpath, filepath, fix); + break; + case AOF_RESP: + checkOldStyleAof(filepath, fix, 0); + break; + case AOF_RDB_PREAMBLE: + checkOldStyleAof(filepath, fix, 1); + break; } - struct redis_stat sb; - if (redis_fstat(fileno(fp),&sb) == -1) { - printf("Cannot stat file: %s\n", filename); - exit(1); - } - - off_t size = sb.st_size; - if (size == 0) { - printf("Empty file: %s\n", filename); - exit(1); - } - - /* This AOF file may have an RDB preamble. Check this to start, and if this - * is the case, start processing the RDB part. */ - if (size >= 8) { /* There must be at least room for the RDB header. */ - char sig[5]; - int has_preamble = fread(sig,sizeof(sig),1,fp) == 1 && - memcmp(sig,"REDIS",sizeof(sig)) == 0; - rewind(fp); - if (has_preamble) { - printf("The AOF appears to start with an RDB preamble.\n" - "Checking the RDB preamble to start:\n"); - if (redis_check_rdb_main(argc,argv,fp) == C_ERR) { - printf("RDB preamble of AOF file is not sane, aborting.\n"); - exit(1); - } else { - printf("RDB preamble is OK, proceeding with AOF tail...\n"); - } - } - } - - off_t pos = process(fp); - off_t diff = size-pos; - - /* In truncate-to-timestamp mode, just exit if there is nothing to truncate. */ - if (diff == 0 && to_timestamp) { - printf("Truncate nothing in AOF to timestamp %ld\n", to_timestamp); - fclose(fp); - exit(0); - } - - printf("AOF analyzed: size=%lld, ok_up_to=%lld, ok_up_to_line=%lld, diff=%lld\n", - (long long) size, (long long) pos, line, (long long) diff); - if (diff > 0) { - if (fix) { - char buf[2]; - printf("This will shrink the AOF from %lld bytes, with %lld bytes, to %lld bytes\n",(long long)size,(long long)diff,(long long)pos); - printf("Continue? [y/N]: "); - if (fgets(buf,sizeof(buf),stdin) == NULL || - strncasecmp(buf,"y",1) != 0) { - printf("Aborting...\n"); - exit(1); - } - if (ftruncate(fileno(fp), pos) == -1) { - printf("Failed to truncate AOF\n"); - exit(1); - } else { - printf("Successfully truncated AOF\n"); - } - } else { - printf("AOF is not valid. " - "Use the --fix option to try fixing it.\n"); - exit(1); - } - } else { - printf("AOF is valid\n"); - } - - fclose(fp); exit(0); invalid_args: - printf("Usage: %s [--fix|--truncate-to-timestamp $timestamp] \n", - argv[0]); + printf("Usage: %s [--fix|--truncate-to-timestamp $timestamp] \n", + argv[0]); exit(1); } diff --git a/tests/integration/aof-multi-part.tcl b/tests/integration/aof-multi-part.tcl index 52877404c..480d336fb 100644 --- a/tests/integration/aof-multi-part.tcl +++ b/tests/integration/aof-multi-part.tcl @@ -100,7 +100,7 @@ tags {"external:skip"} { fail "AOF loading didn't fail" } - assert_equal 1 [count_message_lines $server_path/stdout "The AOF manifest file is invalid format"] + assert_equal 1 [count_message_lines $server_path/stdout "Invalid AOF manifest file format"] } clean_aof_persistence $aof_dirpath @@ -186,7 +186,7 @@ tags {"external:skip"} { fail "AOF loading didn't fail" } - assert_equal 2 [count_message_lines $server_path/stdout "The AOF manifest file is invalid format"] + assert_equal 2 [count_message_lines $server_path/stdout "Invalid AOF manifest file format"] } clean_aof_persistence $aof_dirpath @@ -213,7 +213,7 @@ tags {"external:skip"} { fail "AOF loading didn't fail" } - assert_equal 3 [count_message_lines $server_path/stdout "The AOF manifest file is invalid format"] + assert_equal 3 [count_message_lines $server_path/stdout "Invalid AOF manifest file format"] } clean_aof_persistence $aof_dirpath @@ -267,7 +267,7 @@ tags {"external:skip"} { fail "AOF loading didn't fail" } - assert_equal 4 [count_message_lines $server_path/stdout "The AOF manifest file is invalid format"] + assert_equal 4 [count_message_lines $server_path/stdout "Invalid AOF manifest file format"] } clean_aof_persistence $aof_dirpath diff --git a/tests/integration/aof.tcl b/tests/integration/aof.tcl index cab1d3745..3d8c08c51 100644 --- a/tests/integration/aof.tcl +++ b/tests/integration/aof.tcl @@ -4,6 +4,7 @@ set server_path [tmpdir server.aof] set aof_dirname "appendonlydir" set aof_basename "appendonly.aof" set aof_dirpath "$server_path/$aof_dirname" +set aof_base_file "$server_path/$aof_dirname/${aof_basename}.1$::base_aof_sufix$::aof_format_suffix" set aof_file "$server_path/$aof_dirname/${aof_basename}.1$::incr_aof_sufix$::aof_format_suffix" set aof_manifest_file "$server_path/$aof_dirname/$aof_basename$::manifest_suffix" @@ -142,7 +143,7 @@ tags {"aof external:skip"} { ## Test that redis-check-aof indeed sees this AOF is not valid test "Short read: Utility should confirm the AOF is not valid" { catch { - exec src/redis-check-aof $aof_file + exec src/redis-check-aof $aof_manifest_file } result assert_match "*not valid*" $result } @@ -154,13 +155,13 @@ tags {"aof external:skip"} { } catch { - exec src/redis-check-aof $aof_file + exec src/redis-check-aof $aof_manifest_file } result assert_match "*ok_up_to_line=8*" $result } test "Short read: Utility should be able to fix the AOF" { - set result [exec src/redis-check-aof --fix $aof_file << "y\n"] + set result [exec src/redis-check-aof --fix $aof_manifest_file << "y\n"] assert_match "*Successfully truncated AOF*" $result } @@ -444,7 +445,7 @@ tags {"aof external:skip"} { test {Truncate AOF to specific timestamp} { # truncate to timestamp 1628217473 - exec src/redis-check-aof --truncate-to-timestamp 1628217473 $aof_file + exec src/redis-check-aof --truncate-to-timestamp 1628217473 $aof_manifest_file start_server_aof [list dir $server_path] { set c [redis [dict get $srv host] [dict get $srv port] 0 $::tls] wait_done_loading $c @@ -454,7 +455,7 @@ tags {"aof external:skip"} { } # truncate to timestamp 1628217471 - exec src/redis-check-aof --truncate-to-timestamp 1628217471 $aof_file + exec src/redis-check-aof --truncate-to-timestamp 1628217471 $aof_manifest_file start_server_aof [list dir $server_path] { set c [redis [dict get $srv host] [dict get $srv port] 0 $::tls] wait_done_loading $c @@ -464,7 +465,7 @@ tags {"aof external:skip"} { } # truncate to timestamp 1628217470 - exec src/redis-check-aof --truncate-to-timestamp 1628217470 $aof_file + exec src/redis-check-aof --truncate-to-timestamp 1628217470 $aof_manifest_file start_server_aof [list dir $server_path] { set c [redis [dict get $srv host] [dict get $srv port] 0 $::tls] wait_done_loading $c @@ -473,7 +474,7 @@ tags {"aof external:skip"} { } # truncate to timestamp 1628217469 - catch {exec src/redis-check-aof --truncate-to-timestamp 1628217469 $aof_file} e + catch {exec src/redis-check-aof --truncate-to-timestamp 1628217469 $aof_manifest_file} e assert_match {*aborting*} $e } @@ -515,4 +516,120 @@ tags {"aof external:skip"} { assert_equal [r get foo] 102 } } + + test {Test redis-check-aof for old style resp AOF} { + create_aof $aof_dirpath $aof_file { + append_to_aof [formatCommand set foo hello] + append_to_aof [formatCommand set bar world] + } + + catch { + exec src/redis-check-aof $aof_file + } result + assert_match "*Start checking Old-Style AOF*is valid*" $result + } + + test {Test redis-check-aof for old style rdb-preamble AOF} { + catch { + exec src/redis-check-aof tests/assets/rdb-preamble.aof + } result + assert_match "*Start checking Old-Style AOF*RDB preamble is OK, proceeding with AOF tail*is valid*" $result + } + + test {Test redis-check-aof for Multi Part AOF with resp AOF base} { + create_aof $aof_dirpath $aof_base_file { + append_to_aof [formatCommand set foo hello] + append_to_aof [formatCommand set bar world] + } + + create_aof $aof_dirpath $aof_file { + append_to_aof [formatCommand set foo hello] + append_to_aof [formatCommand set bar world] + } + + create_aof_manifest $aof_dirpath $aof_manifest_file { + append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n" + append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n" + } + + catch { + exec src/redis-check-aof $aof_manifest_file + } result + assert_match "*Start checking Multi Part AOF*Start to check BASE AOF (RESP format)*BASE AOF*is valid*Start to check INCR files*INCR AOF*is valid*All AOF files and manifest are valid*" $result + } + + test {Test redis-check-aof for Multi Part AOF with rdb-preamble AOF base} { + exec cp tests/assets/rdb-preamble.aof $aof_base_file + + create_aof $aof_dirpath $aof_file { + append_to_aof [formatCommand set foo hello] + append_to_aof [formatCommand set bar world] + } + + create_aof_manifest $aof_dirpath $aof_manifest_file { + append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n" + append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n" + } + + catch { + exec src/redis-check-aof $aof_manifest_file + } result + assert_match "*Start checking Multi Part AOF*Start to check BASE AOF (RDB format)*DB preamble is OK, proceeding with AOF tail*BASE AOF*is valid*Start to check INCR files*INCR AOF*is valid*All AOF files and manifest are valid*" $result + } + + test {Test redis-check-aof only truncates the last file for Multi Part AOF in fix mode} { + create_aof $aof_dirpath $aof_base_file { + append_to_aof [formatCommand set foo hello] + append_to_aof [formatCommand multi] + append_to_aof [formatCommand set bar world] + } + + create_aof $aof_dirpath $aof_file { + append_to_aof [formatCommand set foo hello] + append_to_aof [formatCommand set bar world] + } + + create_aof_manifest $aof_dirpath $aof_manifest_file { + append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n" + append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n" + } + + catch { + exec src/redis-check-aof $aof_manifest_file + } result + assert_match "*not valid*" $result + + catch { + exec src/redis-check-aof --fix $aof_manifest_file + } result + assert_match "*Failed to truncate AOF*because it is not the last file*" $result + } + + test {Test redis-check-aof only truncates the last file for Multi Part AOF in truncate-to-timestamp mode} { + create_aof $aof_dirpath $aof_base_file { + append_to_aof "#TS:1628217470\r\n" + append_to_aof [formatCommand set foo1 bar1] + append_to_aof "#TS:1628217471\r\n" + append_to_aof [formatCommand set foo2 bar2] + append_to_aof "#TS:1628217472\r\n" + append_to_aof "#TS:1628217473\r\n" + append_to_aof [formatCommand set foo3 bar3] + append_to_aof "#TS:1628217474\r\n" + } + + create_aof $aof_dirpath $aof_file { + append_to_aof [formatCommand set foo hello] + append_to_aof [formatCommand set bar world] + } + + create_aof_manifest $aof_dirpath $aof_manifest_file { + append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n" + append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n" + } + + catch { + exec src/redis-check-aof --truncate-to-timestamp 1628217473 $aof_manifest_file + } result + assert_match "*Failed to truncate AOF*to timestamp*because it is not the last file*" $result + } }