gvsafronov/rapidjson
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added trailing comma support to iterative parser

Browse Source 
This also fixes cases where the iterative parser should have produced
kParseErrorValueInvalid rather than kParseErrorUnspecifiedSyntaxError
when expecting a value (after a colon in an object, after a comma in an
array, and at the start of an array.)
This commit is contained in:
Nicholas Fraser 2016-03-20 12:52:48 -04:00
parent 7c0e9d941d
commit 68217548f3
2 changed files with 79 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
include/rapidjson/reader.h
View File

@ -640,9 +640,9 @@ private:
if (parseFlags & kParseTrailingCommasFlag) { if (parseFlags & kParseTrailingCommasFlag) {
if (is.Peek() == '}') { if (is.Peek() == '}') {
is.Take();
if (RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount))) if (RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount)))
RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell());
is.Take();
return; return;
} }
} }
@ -689,9 +689,9 @@ private:
if (parseFlags & kParseTrailingCommasFlag) { if (parseFlags & kParseTrailingCommasFlag) {
if (is.Peek() == ']') { if (is.Peek() == ']') {
is.Take();
if (RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount))) if (RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount)))
RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell());
is.Take();
return; return;
} }
} }
@ -1541,7 +1541,7 @@ private:
IterativeParsingErrorState, // Left bracket IterativeParsingErrorState, // Left bracket
IterativeParsingErrorState, // Right bracket IterativeParsingErrorState, // Right bracket
IterativeParsingErrorState, // Left curly bracket IterativeParsingErrorState, // Left curly bracket
IterativeParsingErrorState, // Right curly bracket IterativeParsingObjectFinishState, // Right curly bracket
IterativeParsingErrorState, // Comma IterativeParsingErrorState, // Comma
IterativeParsingErrorState, // Colon IterativeParsingErrorState, // Colon
IterativeParsingMemberKeyState, // String IterativeParsingMemberKeyState, // String
@ -1587,7 +1587,7 @@ private:
// ElementDelimiter // ElementDelimiter
{ {
IterativeParsingArrayInitialState, // Left bracket(push Element state) IterativeParsingArrayInitialState, // Left bracket(push Element state)
IterativeParsingErrorState, // Right bracket IterativeParsingArrayFinishState, // Right bracket
IterativeParsingObjectInitialState, // Left curly bracket(push Element state) IterativeParsingObjectInitialState, // Left curly bracket(push Element state)
IterativeParsingErrorState, // Right curly bracket IterativeParsingErrorState, // Right curly bracket
IterativeParsingErrorState, // Comma IterativeParsingErrorState, // Comma
@ -1689,6 +1689,11 @@ private:
case IterativeParsingObjectFinishState: case IterativeParsingObjectFinishState:
{ {
// Transit from delimiter is only allowed when trailing commas are enabled
if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingMemberDelimiterState) {
RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorObjectMissName, is.Tell());
return IterativeParsingErrorState;
}
// Get member count. // Get member count.
SizeType c = *stack_.template Pop<SizeType>(1); SizeType c = *stack_.template Pop<SizeType>(1);
// If the object is not empty, count the last member. // If the object is not empty, count the last member.
@ -1714,6 +1719,11 @@ private:
case IterativeParsingArrayFinishState: case IterativeParsingArrayFinishState:
{ {
// Transit from delimiter is only allowed when trailing commas are enabled
if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingElementDelimiterState) {
RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorValueInvalid, is.Tell());
return IterativeParsingErrorState;
}
// Get element count. // Get element count.
SizeType c = *stack_.template Pop<SizeType>(1); SizeType c = *stack_.template Pop<SizeType>(1);
// If the array is not empty, count the last element. // If the array is not empty, count the last element.
@ -1773,6 +1783,9 @@ private:
case IterativeParsingMemberDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); return; case IterativeParsingMemberDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); return;
case IterativeParsingMemberKeyState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); return; case IterativeParsingMemberKeyState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); return;
case IterativeParsingMemberValueState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); return; case IterativeParsingMemberValueState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); return;
case IterativeParsingKeyValueDelimiterState:
case IterativeParsingArrayInitialState:
case IterativeParsingElementDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); return;
case IterativeParsingElementState: RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); return; case IterativeParsingElementState: RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); return;
default: RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); return; default: RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); return;
} }

78
test/unittest/readertest.cpp
View File

@ -1127,6 +1127,16 @@ TEST(Reader, IterativeParsing_ErrorHandling) {
TESTERRORHANDLING("{\"a\": 1", kParseErrorObjectMissCommaOrCurlyBracket, 7u); TESTERRORHANDLING("{\"a\": 1", kParseErrorObjectMissCommaOrCurlyBracket, 7u);
TESTERRORHANDLING("[1 2 3]", kParseErrorArrayMissCommaOrSquareBracket, 3u); TESTERRORHANDLING("[1 2 3]", kParseErrorArrayMissCommaOrSquareBracket, 3u);
TESTERRORHANDLING("{\"a: 1", kParseErrorStringMissQuotationMark, 6u); TESTERRORHANDLING("{\"a: 1", kParseErrorStringMissQuotationMark, 6u);
TESTERRORHANDLING("{\"a\":}", kParseErrorValueInvalid, 5u);
TESTERRORHANDLING("{\"a\":]", kParseErrorValueInvalid, 5u);
TESTERRORHANDLING("[1,2,}", kParseErrorValueInvalid, 5u);
TESTERRORHANDLING("[}]", kParseErrorValueInvalid, 1u);
TESTERRORHANDLING("[,]", kParseErrorValueInvalid, 1u);
TESTERRORHANDLING("[1,,]", kParseErrorValueInvalid, 3u);
// Trailing commas are not allowed without kParseTrailingCommasFlag
TESTERRORHANDLING("{\"a\": 1,}", kParseErrorObjectMissName, 8u);
TESTERRORHANDLING("[1,2,3,]", kParseErrorValueInvalid, 7u);
// Any JSON value can be a valid root element in RFC7159. // Any JSON value can be a valid root element in RFC7159.
TESTERRORHANDLING("\"ab", kParseErrorStringMissQuotationMark, 3u); TESTERRORHANDLING("\"ab", kParseErrorStringMissQuotationMark, 3u);
@ -1560,12 +1570,13 @@ TEST(Reader, NumbersAsStrings) {
} }
} }
TEST(Reader, TrailingCommas) { template <unsigned extraFlags>
void TestTrailingCommas() {
{ {
StringStream s("[1,2,3,]"); StringStream s("[1,2,3,]");
ParseArrayHandler<3> h; ParseArrayHandler<3> h;
Reader reader; Reader reader;
EXPECT_TRUE(reader.Parse<kParseTrailingCommasFlag>(s, h)); EXPECT_TRUE(reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h));
EXPECT_EQ(5u, h.step_); EXPECT_EQ(5u, h.step_);
} }
{ {
@ -1574,7 +1585,7 @@ TEST(Reader, TrailingCommas) {
StringStream s(json); StringStream s(json);
ParseObjectHandler h; ParseObjectHandler h;
Reader reader; Reader reader;
EXPECT_TRUE(reader.Parse<kParseTrailingCommasFlag>(s, h)); EXPECT_TRUE(reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h));
EXPECT_EQ(20u, h.step_); EXPECT_EQ(20u, h.step_);
} }
{ {
@ -1584,7 +1595,7 @@ TEST(Reader, TrailingCommas) {
StringStream s(json); StringStream s(json);
ParseObjectHandler h; ParseObjectHandler h;
Reader reader; Reader reader;
EXPECT_TRUE(reader.Parse<kParseTrailingCommasFlag>(s, h)); EXPECT_TRUE(reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h));
EXPECT_EQ(20u, h.step_); EXPECT_EQ(20u, h.step_);
} }
{ {
@ -1594,18 +1605,27 @@ TEST(Reader, TrailingCommas) {
StringStream s(json); StringStream s(json);
ParseObjectHandler h; ParseObjectHandler h;
Reader reader; Reader reader;
EXPECT_TRUE(reader.Parse<kParseTrailingCommasFlag|kParseCommentsFlag>(s, h)); EXPECT_TRUE(reader.Parse<extraFlags|kParseTrailingCommasFlag|kParseCommentsFlag>(s, h));
EXPECT_EQ(20u, h.step_); EXPECT_EQ(20u, h.step_);
} }
} }
TEST(Reader, MultipleTrailingCommaErrors) { TEST(Reader, TrailingCommas) {
TestTrailingCommas<kParseNoFlags>();
}
TEST(Reader, TrailingCommasIterative) {
TestTrailingCommas<kParseIterativeFlag>();
}
template <unsigned extraFlags>
void TestMultipleTrailingCommaErrors() {
// only a single trailing comma is allowed. // only a single trailing comma is allowed.
{ {
StringStream s("[1,2,3,,]"); StringStream s("[1,2,3,,]");
ParseArrayHandler<3> h; ParseArrayHandler<3> h;
Reader reader; Reader reader;
ParseResult r = reader.Parse<kParseTrailingCommasFlag>(s, h); ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
EXPECT_TRUE(reader.HasParseError()); EXPECT_TRUE(reader.HasParseError());
EXPECT_EQ(kParseErrorValueInvalid, r.Code()); EXPECT_EQ(kParseErrorValueInvalid, r.Code());
EXPECT_EQ(7u, r.Offset()); EXPECT_EQ(7u, r.Offset());
@ -1616,21 +1636,30 @@ TEST(Reader, MultipleTrailingCommaErrors) {
StringStream s(json); StringStream s(json);
ParseObjectHandler h; ParseObjectHandler h;
Reader reader; Reader reader;
ParseResult r = reader.Parse<kParseTrailingCommasFlag>(s, h); ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
EXPECT_TRUE(reader.HasParseError()); EXPECT_TRUE(reader.HasParseError());
EXPECT_EQ(kParseErrorObjectMissName, r.Code()); EXPECT_EQ(kParseErrorObjectMissName, r.Code());
EXPECT_EQ(95u, r.Offset()); EXPECT_EQ(95u, r.Offset());
} }
} }
TEST(Reader, EmptyExceptForCommaErrors) { TEST(Reader, MultipleTrailingCommaErrors) {
TestMultipleTrailingCommaErrors<kParseNoFlags>();
}
TEST(Reader, MultipleTrailingCommaErrorsIterative) {
TestMultipleTrailingCommaErrors<kParseIterativeFlag>();
}
template <unsigned extraFlags>
void TestEmptyExceptForCommaErrors() {
// not allowed even with trailing commas enabled; the // not allowed even with trailing commas enabled; the
// trailing comma must follow a value. // trailing comma must follow a value.
{ {
StringStream s("[,]"); StringStream s("[,]");
ParseArrayHandler<3> h; ParseArrayHandler<3> h;
Reader reader; Reader reader;
ParseResult r = reader.Parse<kParseTrailingCommasFlag>(s, h); ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
EXPECT_TRUE(reader.HasParseError()); EXPECT_TRUE(reader.HasParseError());
EXPECT_EQ(kParseErrorValueInvalid, r.Code()); EXPECT_EQ(kParseErrorValueInvalid, r.Code());
EXPECT_EQ(1u, r.Offset()); EXPECT_EQ(1u, r.Offset());
@ -1639,34 +1668,51 @@ TEST(Reader, EmptyExceptForCommaErrors) {
StringStream s("{,}"); StringStream s("{,}");
ParseObjectHandler h; ParseObjectHandler h;
Reader reader; Reader reader;
ParseResult r = reader.Parse<kParseTrailingCommasFlag>(s, h); ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
EXPECT_TRUE(reader.HasParseError()); EXPECT_TRUE(reader.HasParseError());
EXPECT_EQ(kParseErrorObjectMissName, r.Code()); EXPECT_EQ(kParseErrorObjectMissName, r.Code());
EXPECT_EQ(1u, r.Offset()); EXPECT_EQ(1u, r.Offset());
} }
} }
TEST(Reader, TrailingCommaHandlerTermination) { TEST(Reader, EmptyExceptForCommaErrors) {
TestEmptyExceptForCommaErrors<kParseNoFlags>();
}
TEST(Reader, EmptyExceptForCommaErrorsIterative) {
TestEmptyExceptForCommaErrors<kParseIterativeFlag>();
}
template <unsigned extraFlags>
void TestTrailingCommaHandlerTermination() {
{ {
HandlerTerminateAtEndArray h; HandlerTerminateAtEndArray h;
Reader reader; Reader reader;
StringStream s("[1,2,3,]"); StringStream s("[1,2,3,]");
ParseResult r = reader.Parse<kParseTrailingCommasFlag>(s, h); ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
EXPECT_TRUE(reader.HasParseError()); EXPECT_TRUE(reader.HasParseError());
EXPECT_EQ(kParseErrorTermination, r.Code()); EXPECT_EQ(kParseErrorTermination, r.Code());
EXPECT_EQ(8u, r.Offset()); EXPECT_EQ(7u, r.Offset());
} }
{ {
HandlerTerminateAtEndObject h; HandlerTerminateAtEndObject h;
Reader reader; Reader reader;
StringStream s("{\"t\": true, \"f\": false,}"); StringStream s("{\"t\": true, \"f\": false,}");
ParseResult r = reader.Parse<kParseTrailingCommasFlag>(s, h); ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
EXPECT_TRUE(reader.HasParseError()); EXPECT_TRUE(reader.HasParseError());
EXPECT_EQ(kParseErrorTermination, r.Code()); EXPECT_EQ(kParseErrorTermination, r.Code());
EXPECT_EQ(24u, r.Offset()); EXPECT_EQ(23u, r.Offset());
} }
} }
TEST(Reader, TrailingCommaHandlerTermination) {
TestTrailingCommaHandlerTermination<kParseNoFlags>();
}
TEST(Reader, TrailingCommaHandlerTerminationIterative) {
TestTrailingCommaHandlerTermination<kParseIterativeFlag>();
}
#ifdef __GNUC__ #ifdef __GNUC__
RAPIDJSON_DIAG_POP RAPIDJSON_DIAG_POP
#endif #endif