From 5ce78b135d7fd76589973607b707dd7dbcab0792 Mon Sep 17 00:00:00 2001 From: Andrey Date: Sun, 11 Oct 2015 15:01:15 +0300 Subject: [PATCH 1/2] Introduce support of comments. --- include/rapidjson/reader.h | 48 ++++++++++++++++++------- test/unittest/readertest.cpp | 70 ++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 12 deletions(-) diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h index 9a17301..2603992 100644 --- a/include/rapidjson/reader.h +++ b/include/rapidjson/reader.h @@ -140,6 +140,7 @@ enum ParseFlag { kParseIterativeFlag = 4, //!< Iterative(constant complexity in terms of function call stack size) parsing. kParseStopWhenDoneFlag = 8, //!< After parsing a complete JSON root from stream, stop further processing the rest of stream. When this flag is used, parser will not generate kParseErrorDocumentRootNotSingular error. kParseFullPrecisionFlag = 16, //!< Parse number in full precision (but slower). + kParseCommentsFlag = 32, //!< Allow one-line (//) and multi-line (/**/) comments. kParseDefaultFlags = RAPIDJSON_PARSE_DEFAULT_FLAGS //!< Default parse flags. Can be customized by defining RAPIDJSON_PARSE_DEFAULT_FLAGS }; @@ -398,7 +399,7 @@ public: ClearStackOnExit scope(*this); - SkipWhitespace(is); + SkipWhitespaceAndComments(is); if (is.Peek() == '\0') { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentEmpty, is.Tell()); @@ -409,7 +410,7 @@ public: RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); if (!(parseFlags & kParseStopWhenDoneFlag)) { - SkipWhitespace(is); + SkipWhitespaceAndComments(is); if (is.Peek() != '\0') { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentRootNotSingular, is.Tell()); @@ -462,6 +463,29 @@ private: ClearStackOnExit& operator=(const ClearStackOnExit&); }; + template + void SkipWhitespaceAndComments(InputStream& is) { + SkipWhitespace(is); + + if (parseFlags & kParseCommentsFlag) { + while (is.Peek() == '/') { + is.Take(); + + if (is.Peek() == '*') { + is.Take(); + while (is.Take() != '*' || is.Take() != '/') { } + } else if (is.Peek() == '/') { + is.Take(); + while (is.Take() != '\n') { } + } else { + RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); + } + + SkipWhitespace(is); + } + } + } + // Parse object: { string : value, ... } template void ParseObject(InputStream& is, Handler& handler) { @@ -471,7 +495,7 @@ private: if (!handler.StartObject()) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - SkipWhitespace(is); + SkipWhitespaceAndComments(is); if (is.Peek() == '}') { is.Take(); @@ -487,22 +511,22 @@ private: ParseString(is, handler, true); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - SkipWhitespace(is); + SkipWhitespaceAndComments(is); if (is.Take() != ':') RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); - SkipWhitespace(is); + SkipWhitespaceAndComments(is); ParseValue(is, handler); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - SkipWhitespace(is); + SkipWhitespaceAndComments(is); ++memberCount; switch (is.Take()) { - case ',': SkipWhitespace(is); break; + case ',': SkipWhitespaceAndComments(is); break; case '}': if (!handler.EndObject(memberCount)) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); @@ -521,7 +545,7 @@ private: if (!handler.StartArray()) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); - SkipWhitespace(is); + SkipWhitespaceAndComments(is); if (is.Peek() == ']') { is.Take(); @@ -535,10 +559,10 @@ private: RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; ++elementCount; - SkipWhitespace(is); + SkipWhitespaceAndComments(is); switch (is.Take()) { - case ',': SkipWhitespace(is); break; + case ',': SkipWhitespaceAndComments(is); break; case ']': if (!handler.EndArray(elementCount)) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); @@ -1404,7 +1428,7 @@ private: ClearStackOnExit scope(*this); IterativeParsingState state = IterativeParsingStartState; - SkipWhitespace(is); + SkipWhitespaceAndComments(is); while (is.Peek() != '\0') { Token t = Tokenize(is.Peek()); IterativeParsingState n = Predict(state, t); @@ -1421,7 +1445,7 @@ private: if ((parseFlags & kParseStopWhenDoneFlag) && state == IterativeParsingFinishState) break; - SkipWhitespace(is); + SkipWhitespaceAndComments(is); } // Handle the end of file. diff --git a/test/unittest/readertest.cpp b/test/unittest/readertest.cpp index 9106063..010a1a9 100644 --- a/test/unittest/readertest.cpp +++ b/test/unittest/readertest.cpp @@ -1349,6 +1349,76 @@ TEST(Reader, ParseTerminationByHandler) { TEST_TERMINATION(12, "{\"a\":[1]"); // non-empty array } +TEST(Reader, ParseComments) { + const char* json = + "// Here is a one-line comment.\n" + "{// And here's another one\n" + " /*And here's an in-line one.*/\"hello\" : \"world\"," + " \"t\" :/* And one with '*' symbol*/true ," + "/* A multiline comment\n" + " goes here*/" + " \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3]" + "}/*And the last one to be sure */"; + + StringStream s(json); + ParseObjectHandler h; + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + EXPECT_EQ(20u, h.step_); +} + +TEST(Reader, ParseEmptyInlineComment) { + const char* json = "{/**/\"hello\" : \"world\", \"t\" : true, \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }"; + + StringStream s(json); + ParseObjectHandler h; + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + EXPECT_EQ(20u, h.step_); +} + +TEST(Reader, ParseEmptyOnelineComment) { + const char* json = "{//\n\"hello\" : \"world\", \"t\" : true, \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }"; + + StringStream s(json); + ParseObjectHandler h; + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + EXPECT_EQ(20u, h.step_); +} + +TEST(Reader, InlineCommentsAreDisabledByDefault) { + { + const char* json = "{/* Inline comment. */\"hello\" : \"world\", \"t\" : true, \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }"; + + StringStream s(json); + ParseObjectHandler h; + Reader reader; + EXPECT_FALSE(reader.Parse(s, h)); + } + + { + const char* json = + "{\"hello\" : /* Multiline comment starts here\n" + " continues here\n" + " and ends here */\"world\", \"t\" :true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }"; + + StringStream s(json); + ParseObjectHandler h; + Reader reader; + EXPECT_FALSE(reader.Parse(s, h)); + } +} + +TEST(Reader, OnelineCommentsAreDisabledByDefault) { + const char* json = "{// One-line comment\n\"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }"; + + StringStream s(json); + ParseObjectHandler h; + Reader reader; + EXPECT_FALSE(reader.Parse(s, h)); +} + #ifdef __GNUC__ RAPIDJSON_DIAG_POP #endif From f7960ac0e8740f81980e8cd11284cac54f9f44be Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 13 Oct 2015 23:57:54 +0300 Subject: [PATCH 2/2] Comments parsing fixes. * Comments parsing function correctly handles EOF. * Since SkipWhitespaceAndComments can generate errors, its calls should be followed by RAPIDJSON_PARSE_ERROR_EARLY_RETURN macro. * Some tests to make the bug never appear again. --- include/rapidjson/reader.h | 35 +++++++++++++++++++++++++++++++---- test/unittest/readertest.cpp | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h index 2603992..fba9f19 100644 --- a/include/rapidjson/reader.h +++ b/include/rapidjson/reader.h @@ -400,6 +400,7 @@ public: ClearStackOnExit scope(*this); SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); if (is.Peek() == '\0') { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentEmpty, is.Tell()); @@ -411,6 +412,7 @@ public: if (!(parseFlags & kParseStopWhenDoneFlag)) { SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); if (is.Peek() != '\0') { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentRootNotSingular, is.Tell()); @@ -473,10 +475,21 @@ private: if (is.Peek() == '*') { is.Take(); - while (is.Take() != '*' || is.Take() != '/') { } + while (true) { + if (is.Peek() == '\0') + RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); + + if (is.Take() == '*') { + if (is.Peek() == '\0') + RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); + + if (is.Take() == '/') + break; + } + } } else if (is.Peek() == '/') { is.Take(); - while (is.Take() != '\n') { } + while (is.Peek() != '\0' && is.Take() != '\n') { } } else { RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); } @@ -496,6 +509,7 @@ private: RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; if (is.Peek() == '}') { is.Take(); @@ -512,21 +526,27 @@ private: RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; if (is.Take() != ':') RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; ParseValue(is, handler); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; ++memberCount; switch (is.Take()) { - case ',': SkipWhitespaceAndComments(is); break; + case ',': + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + break; case '}': if (!handler.EndObject(memberCount)) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); @@ -546,6 +566,7 @@ private: RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; if (is.Peek() == ']') { is.Take(); @@ -560,9 +581,13 @@ private: ++elementCount; SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; switch (is.Take()) { - case ',': SkipWhitespaceAndComments(is); break; + case ',': + SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + break; case ']': if (!handler.EndArray(elementCount)) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); @@ -1429,6 +1454,7 @@ private: IterativeParsingState state = IterativeParsingStartState; SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); while (is.Peek() != '\0') { Token t = Tokenize(is.Peek()); IterativeParsingState n = Predict(state, t); @@ -1446,6 +1472,7 @@ private: break; SkipWhitespaceAndComments(is); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); } // Handle the end of file. diff --git a/test/unittest/readertest.cpp b/test/unittest/readertest.cpp index 010a1a9..71d7113 100644 --- a/test/unittest/readertest.cpp +++ b/test/unittest/readertest.cpp @@ -1387,6 +1387,19 @@ TEST(Reader, ParseEmptyOnelineComment) { EXPECT_EQ(20u, h.step_); } +TEST(Reader, ParseMultipleCommentsInARow) { + const char* json = + "{/* first comment *//* second */\n" + "/* third */ /*fourth*/// last one\n" + "\"hello\" : \"world\", \"t\" : true, \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }"; + + StringStream s(json); + ParseObjectHandler h; + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + EXPECT_EQ(20u, h.step_); +} + TEST(Reader, InlineCommentsAreDisabledByDefault) { { const char* json = "{/* Inline comment. */\"hello\" : \"world\", \"t\" : true, \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }"; @@ -1419,6 +1432,26 @@ TEST(Reader, OnelineCommentsAreDisabledByDefault) { EXPECT_FALSE(reader.Parse(s, h)); } +TEST(Reader, EofAfterOneLineComment) { + const char* json = "{\"hello\" : \"world\" // EOF is here -->\0 \n}"; + + StringStream s(json); + ParseObjectHandler h; + Reader reader; + EXPECT_FALSE(reader.Parse(s, h)); + EXPECT_EQ(kParseErrorObjectMissCommaOrCurlyBracket, reader.GetParseErrorCode()); +} + +TEST(Reader, IncompleteMultilineComment) { + const char* json = "{\"hello\" : \"world\" /* EOF is here -->\0 */}"; + + StringStream s(json); + ParseObjectHandler h; + Reader reader; + EXPECT_FALSE(reader.Parse(s, h)); + EXPECT_EQ(kParseErrorUnspecificSyntaxError, reader.GetParseErrorCode()); +} + #ifdef __GNUC__ RAPIDJSON_DIAG_POP #endif