gvsafronov/rapidjson
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added trailing comma support to iterative parser

Browse Source 
This also fixes cases where the iterative parser should have produced
kParseErrorValueInvalid rather than kParseErrorUnspecifiedSyntaxError
when expecting a value (after a colon in an object, after a comma in an
array, and at the start of an array.)
This commit is contained in:
Nicholas Fraser 2016-03-20 12:52:48 -04:00
parent 7c0e9d941d
commit 68217548f3
2 changed files with 79 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
include/rapidjson/reader.h
View File

@ -640,9 +640,9 @@ private:
if (parseFlags & kParseTrailingCommasFlag) {
if (is.Peek() == '}') {
is.Take();
if (RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount)))
RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell());
is.Take();
return;
}
}
@ -689,9 +689,9 @@ private:
if (parseFlags & kParseTrailingCommasFlag) {
if (is.Peek() == ']') {
is.Take();
if (RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount)))
RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell());
is.Take();
return;
}
}
@ -1541,7 +1541,7 @@ private:
IterativeParsingErrorState, // Left bracket
IterativeParsingErrorState, // Right bracket
IterativeParsingErrorState, // Left curly bracket
IterativeParsingErrorState, // Right curly bracket
IterativeParsingObjectFinishState, // Right curly bracket
IterativeParsingErrorState, // Comma
IterativeParsingErrorState, // Colon
IterativeParsingMemberKeyState, // String
@ -1587,7 +1587,7 @@ private:
// ElementDelimiter
{
IterativeParsingArrayInitialState, // Left bracket(push Element state)
IterativeParsingErrorState, // Right bracket
IterativeParsingArrayFinishState, // Right bracket
IterativeParsingObjectInitialState, // Left curly bracket(push Element state)
IterativeParsingErrorState, // Right curly bracket
IterativeParsingErrorState, // Comma
@ -1689,6 +1689,11 @@ private:
case IterativeParsingObjectFinishState:
{
// Transit from delimiter is only allowed when trailing commas are enabled
if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingMemberDelimiterState) {
RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorObjectMissName, is.Tell());
return IterativeParsingErrorState;
}
// Get member count.
SizeType c = *stack_.template Pop<SizeType>(1);
// If the object is not empty, count the last member.
@ -1714,6 +1719,11 @@ private:
case IterativeParsingArrayFinishState:
{
// Transit from delimiter is only allowed when trailing commas are enabled
if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingElementDelimiterState) {
RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorValueInvalid, is.Tell());
return IterativeParsingErrorState;
}
// Get element count.
SizeType c = *stack_.template Pop<SizeType>(1);
// If the array is not empty, count the last element.
@ -1773,6 +1783,9 @@ private:
case IterativeParsingMemberDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); return;
case IterativeParsingMemberKeyState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); return;
case IterativeParsingMemberValueState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); return;
case IterativeParsingKeyValueDelimiterState:
case IterativeParsingArrayInitialState:
case IterativeParsingElementDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); return;
case IterativeParsingElementState: RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); return;
default: RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); return;
}

78
test/unittest/readertest.cpp
View File

@ -1127,6 +1127,16 @@ TEST(Reader, IterativeParsing_ErrorHandling) {
TESTERRORHANDLING("{\"a\": 1", kParseErrorObjectMissCommaOrCurlyBracket, 7u);
TESTERRORHANDLING("[1 2 3]", kParseErrorArrayMissCommaOrSquareBracket, 3u);
TESTERRORHANDLING("{\"a: 1", kParseErrorStringMissQuotationMark, 6u);
TESTERRORHANDLING("{\"a\":}", kParseErrorValueInvalid, 5u);
TESTERRORHANDLING("{\"a\":]", kParseErrorValueInvalid, 5u);
TESTERRORHANDLING("[1,2,}", kParseErrorValueInvalid, 5u);
TESTERRORHANDLING("[}]", kParseErrorValueInvalid, 1u);
TESTERRORHANDLING("[,]", kParseErrorValueInvalid, 1u);
TESTERRORHANDLING("[1,,]", kParseErrorValueInvalid, 3u);
// Trailing commas are not allowed without kParseTrailingCommasFlag
TESTERRORHANDLING("{\"a\": 1,}", kParseErrorObjectMissName, 8u);
TESTERRORHANDLING("[1,2,3,]", kParseErrorValueInvalid, 7u);
// Any JSON value can be a valid root element in RFC7159.
TESTERRORHANDLING("\"ab", kParseErrorStringMissQuotationMark, 3u);
@ -1560,12 +1570,13 @@ TEST(Reader, NumbersAsStrings) {
}
}
TEST(Reader, TrailingCommas) {
template <unsigned extraFlags>
void TestTrailingCommas() {
{
StringStream s("[1,2,3,]");
ParseArrayHandler<3> h;
Reader reader;
EXPECT_TRUE(reader.Parse<kParseTrailingCommasFlag>(s, h));
EXPECT_TRUE(reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h));
EXPECT_EQ(5u, h.step_);
}
{
@ -1574,7 +1585,7 @@ TEST(Reader, TrailingCommas) {
StringStream s(json);
ParseObjectHandler h;
Reader reader;
EXPECT_TRUE(reader.Parse<kParseTrailingCommasFlag>(s, h));
EXPECT_TRUE(reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h));
EXPECT_EQ(20u, h.step_);
}
{
@ -1584,7 +1595,7 @@ TEST(Reader, TrailingCommas) {
StringStream s(json);
ParseObjectHandler h;
Reader reader;
EXPECT_TRUE(reader.Parse<kParseTrailingCommasFlag>(s, h));
EXPECT_TRUE(reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h));
EXPECT_EQ(20u, h.step_);
}
{
@ -1594,18 +1605,27 @@ TEST(Reader, TrailingCommas) {
StringStream s(json);
ParseObjectHandler h;
Reader reader;
EXPECT_TRUE(reader.Parse<kParseTrailingCommasFlag|kParseCommentsFlag>(s, h));
EXPECT_TRUE(reader.Parse<extraFlags|kParseTrailingCommasFlag|kParseCommentsFlag>(s, h));
EXPECT_EQ(20u, h.step_);
}
}
TEST(Reader, MultipleTrailingCommaErrors) {
TEST(Reader, TrailingCommas) {
TestTrailingCommas<kParseNoFlags>();
}
TEST(Reader, TrailingCommasIterative) {
TestTrailingCommas<kParseIterativeFlag>();
}
template <unsigned extraFlags>
void TestMultipleTrailingCommaErrors() {
// only a single trailing comma is allowed.
{
StringStream s("[1,2,3,,]");
ParseArrayHandler<3> h;
Reader reader;
ParseResult r = reader.Parse<kParseTrailingCommasFlag>(s, h);
ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
EXPECT_TRUE(reader.HasParseError());
EXPECT_EQ(kParseErrorValueInvalid, r.Code());
EXPECT_EQ(7u, r.Offset());
@ -1616,21 +1636,30 @@ TEST(Reader, MultipleTrailingCommaErrors) {
StringStream s(json);
ParseObjectHandler h;
Reader reader;
ParseResult r = reader.Parse<kParseTrailingCommasFlag>(s, h);
ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
EXPECT_TRUE(reader.HasParseError());
EXPECT_EQ(kParseErrorObjectMissName, r.Code());
EXPECT_EQ(95u, r.Offset());
}
}
TEST(Reader, EmptyExceptForCommaErrors) {
TEST(Reader, MultipleTrailingCommaErrors) {
TestMultipleTrailingCommaErrors<kParseNoFlags>();
}
TEST(Reader, MultipleTrailingCommaErrorsIterative) {
TestMultipleTrailingCommaErrors<kParseIterativeFlag>();
}
template <unsigned extraFlags>
void TestEmptyExceptForCommaErrors() {
// not allowed even with trailing commas enabled; the
// trailing comma must follow a value.
{
StringStream s("[,]");
ParseArrayHandler<3> h;
Reader reader;
ParseResult r = reader.Parse<kParseTrailingCommasFlag>(s, h);
ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
EXPECT_TRUE(reader.HasParseError());
EXPECT_EQ(kParseErrorValueInvalid, r.Code());
EXPECT_EQ(1u, r.Offset());
@ -1639,34 +1668,51 @@ TEST(Reader, EmptyExceptForCommaErrors) {
StringStream s("{,}");
ParseObjectHandler h;
Reader reader;
ParseResult r = reader.Parse<kParseTrailingCommasFlag>(s, h);
ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
EXPECT_TRUE(reader.HasParseError());
EXPECT_EQ(kParseErrorObjectMissName, r.Code());
EXPECT_EQ(1u, r.Offset());
}
}
TEST(Reader, TrailingCommaHandlerTermination) {
TEST(Reader, EmptyExceptForCommaErrors) {
TestEmptyExceptForCommaErrors<kParseNoFlags>();
}
TEST(Reader, EmptyExceptForCommaErrorsIterative) {
TestEmptyExceptForCommaErrors<kParseIterativeFlag>();
}
template <unsigned extraFlags>
void TestTrailingCommaHandlerTermination() {
{
HandlerTerminateAtEndArray h;
Reader reader;
StringStream s("[1,2,3,]");
ParseResult r = reader.Parse<kParseTrailingCommasFlag>(s, h);
ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
EXPECT_TRUE(reader.HasParseError());
EXPECT_EQ(kParseErrorTermination, r.Code());
EXPECT_EQ(8u, r.Offset());
EXPECT_EQ(7u, r.Offset());
}
{
HandlerTerminateAtEndObject h;
Reader reader;
StringStream s("{\"t\": true, \"f\": false,}");
ParseResult r = reader.Parse<kParseTrailingCommasFlag>(s, h);
ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
EXPECT_TRUE(reader.HasParseError());
EXPECT_EQ(kParseErrorTermination, r.Code());
EXPECT_EQ(24u, r.Offset());
EXPECT_EQ(23u, r.Offset());
}
}
TEST(Reader, TrailingCommaHandlerTermination) {
TestTrailingCommaHandlerTermination<kParseNoFlags>();
}
TEST(Reader, TrailingCommaHandlerTerminationIterative) {
TestTrailingCommaHandlerTermination<kParseIterativeFlag>();
}
#ifdef __GNUC__
RAPIDJSON_DIAG_POP
#endif