gvsafronov/rapidjson
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add kParseStopWhenDoneFlag, its implementation and related unit tests

Browse Source
This commit is contained in:
Milo Yip 2014-07-27 17:10:32 +08:00
parent c4ce48cde9
commit c81386413b
2 changed files with 115 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

185
include/rapidjson/reader.h
View File

@ -65,7 +65,8 @@ enum ParseFlag {
kParseDefaultFlags = 0, //!< Default parse flags. Non-destructive parsing. Text strings are decoded into allocated buffer. kParseDefaultFlags = 0, //!< Default parse flags. Non-destructive parsing. Text strings are decoded into allocated buffer.
kParseInsituFlag = 1, //!< In-situ(destructive) parsing. kParseInsituFlag = 1, //!< In-situ(destructive) parsing.
kParseValidateEncodingFlag = 2, //!< Validate encoding of JSON strings. kParseValidateEncodingFlag = 2, //!< Validate encoding of JSON strings.
kParseIterativeFlag = 4 //!< Iterative(constant complexity in terms of function call stack size) parsing. kParseIterativeFlag = 4, //!< Iterative(constant complexity in terms of function call stack size) parsing.
kParseStopWhenDoneFlag = 8 //!< After parsing a complete JSON root from stream, stop further processing the rest of stream. When this flag is used, parser will not generate kParseErrorDocumentRootNotSingular error.
}; };
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
@ -273,11 +274,10 @@ public:
typedef typename SourceEncoding::Ch Ch; //!< SourceEncoding character type typedef typename SourceEncoding::Ch Ch; //!< SourceEncoding character type
//! Constructor. //! Constructor.
/*! \param limit Parsing stack size limit(in bytes). Pass 0 means no limit. /*! \param allocator Optional allocator for allocating stack memory. (Only use for non-destructive parsing)
\param allocator Optional allocator for allocating stack memory. (Only use for non-destructive parsing)
\param stackCapacity stack capacity in bytes for storing a single decoded string. (Only use for non-destructive parsing) \param stackCapacity stack capacity in bytes for storing a single decoded string. (Only use for non-destructive parsing)
*/ */
GenericReader(size_t limit = 0, Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity) : stack_(allocator, stackCapacity), kStackSizeLimit_(limit), parseResult_() {} GenericReader(Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity) : stack_(allocator, stackCapacity), parseResult_() {}
//! Parse JSON text. //! Parse JSON text.
/*! \tparam parseFlags Combination of \ref ParseFlag. /*! \tparam parseFlags Combination of \ref ParseFlag.
@ -310,11 +310,13 @@ public:
} }
RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_);
SkipWhitespace(is); if (!(parseFlags & kParseStopWhenDoneFlag)) {
SkipWhitespace(is);
if (is.Peek() != '\0') { if (is.Peek() != '\0') {
RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentRootNotSingular, is.Tell()); RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentRootNotSingular, is.Tell());
RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_);
}
} }
} }
@ -513,7 +515,7 @@ private:
typedef typename TargetEncoding::Ch Ch; typedef typename TargetEncoding::Ch Ch;
StackStream(internal::Stack<Allocator>& stack) : stack_(stack), length_(0) {} StackStream(internal::Stack<Allocator>& stack) : stack_(stack), length_(0) {}
void Put(Ch c) { RAPIDJSON_FORCEINLINE void Put(Ch c) {
*stack_.template Push<Ch>() = c; *stack_.template Push<Ch>() = c;
++length_; ++length_;
} }
@ -572,11 +574,6 @@ private:
is.Take(); is.Take();
Ch e = is.Take(); Ch e = is.Take();
if ((sizeof(Ch) == 1 || unsigned(e) < 256) && escape[(unsigned char)e]) { if ((sizeof(Ch) == 1 || unsigned(e) < 256) && escape[(unsigned char)e]) {
if (!(parseFlags & kParseInsituFlag)) {
if (!CheckStackSpaceQuota(sizeof(Ch))) {
RAPIDJSON_PARSE_ERROR(kParseErrorStackSizeLimitExceeded, is.Tell() - 1);
}
}
os.Put(escape[(unsigned char)e]); os.Put(escape[(unsigned char)e]);
} }
else if (e == 'u') { // Unicode else if (e == 'u') { // Unicode
@ -597,11 +594,6 @@ private:
} }
else if (c == '"') { // Closing double quote else if (c == '"') { // Closing double quote
is.Take(); is.Take();
if (!(parseFlags & kParseInsituFlag)) {
if (!CheckStackSpaceQuota(sizeof(Ch))) {
RAPIDJSON_PARSE_ERROR(kParseErrorStackSizeLimitExceeded, is.Tell() - 1);
}
}
os.Put('\0'); // null-terminate the string os.Put('\0'); // null-terminate the string
return; return;
} }
@ -829,44 +821,52 @@ private:
}; };
// Tokens // Tokens
enum IterativeParsingToken { enum Token {
IterativeParsingLeftBracketToken = 0, LeftBracketToken = 0,
IterativeParsingRightBracketToken, RightBracketToken,
IterativeParsingLeftCurlyBracketToken, LeftCurlyBracketToken,
IterativeParsingRightCurlyBracketToken, RightCurlyBracketToken,
IterativeParsingCommaToken, CommaToken,
IterativeParsingColonToken, ColonToken,
IterativeParsingStringToken, StringToken,
IterativeParsingFalseToken, FalseToken,
IterativeParsingTrueToken, TrueToken,
IterativeParsingNullToken, NullToken,
IterativeParsingNumberToken, NumberToken,
cIterativeParsingTokenCount kTokenCount
}; };
IterativeParsingToken Tokenize(Ch c) { RAPIDJSON_FORCEINLINE Token Tokenize(Ch c) {
switch (c) { #define N NumberToken
case '[': return IterativeParsingLeftBracketToken; #define N16 N,N,N,N,N,N,N,N,N,N,N,N,N,N,N,N
case ']': return IterativeParsingRightBracketToken; // Maps from ASCII to Token
case '{': return IterativeParsingLeftCurlyBracketToken; static const unsigned char tokenMap[256] = {
case '}': return IterativeParsingRightCurlyBracketToken; N16, // 00~0F
case ',': return IterativeParsingCommaToken; N16, // 10~1F
case ':': return IterativeParsingColonToken; N, N, StringToken, N, N, N, N, N, N, N, N, N, CommaToken, N, N, N, // 20~2F
case '"': return IterativeParsingStringToken; N, N, N, N, N, N, N, N, N, N, ColonToken, N, N, N, N, N, // 30~3F
case 'f': return IterativeParsingFalseToken; N16, // 40~4F
case 't': return IterativeParsingTrueToken; N, N, N, N, N, N, N, N, N, N, N, LeftBracketToken, N, RightBracketToken, N, N, // 50~5F
case 'n': return IterativeParsingNullToken; N, N, N, N, N, N, FalseToken, N, N, N, N, N, N, N, NullToken, N, // 60~6F
default: return IterativeParsingNumberToken; N, N, N, N, TrueToken, N, N, N, N, N, N, LeftCurlyBracketToken, N, RightCurlyBracketToken, N, N, // 70~7F
} N16, N16, N16, N16, N16, N16, N16, N16 // 80~FF
};
#undef N
#undef N16
if (sizeof(Ch) == 1 || unsigned(c) < 256)
return (Token)tokenMap[(unsigned char)c];
else
return NumberToken;
} }
IterativeParsingState Predict(IterativeParsingState state, IterativeParsingToken token) { RAPIDJSON_FORCEINLINE IterativeParsingState Predict(IterativeParsingState state, Token token) {
// current state x one lookahead token -> new state // current state x one lookahead token -> new state
static const char G[cIterativeParsingStateCount][cIterativeParsingTokenCount] = { static const char G[cIterativeParsingStateCount][kTokenCount] = {
// Start // Start
{ {
IterativeParsingArrayInitialState, // Left bracket IterativeParsingArrayInitialState, // Left bracket
@ -1025,11 +1025,7 @@ private:
// Make an advance in the token stream and state based on the candidate destination state which was returned by Transit(). // Make an advance in the token stream and state based on the candidate destination state which was returned by Transit().
// May return a new state on state pop. // May return a new state on state pop.
template <unsigned parseFlags, typename InputStream, typename Handler> template <unsigned parseFlags, typename InputStream, typename Handler>
IterativeParsingState Transit(IterativeParsingState src, IterativeParsingToken token, IterativeParsingState dst, InputStream& is, Handler& handler) { RAPIDJSON_FORCEINLINE IterativeParsingState Transit(IterativeParsingState src, Token token, IterativeParsingState dst, InputStream& is, Handler& handler) {
int c = 0;
IterativeParsingState n;
bool hr;
switch (dst) { switch (dst) {
case IterativeParsingStartState: case IterativeParsingStartState:
RAPIDJSON_ASSERT(false); RAPIDJSON_ASSERT(false);
@ -1043,27 +1039,20 @@ private:
case IterativeParsingObjectInitialState: case IterativeParsingObjectInitialState:
case IterativeParsingArrayInitialState: case IterativeParsingArrayInitialState:
{
// Push the state(Element or MemeberValue) if we are nested in another array or value of member. // Push the state(Element or MemeberValue) if we are nested in another array or value of member.
// In this way we can get the correct state on ObjectFinish or ArrayFinish by frame pop. // In this way we can get the correct state on ObjectFinish or ArrayFinish by frame pop.
n = src; IterativeParsingState n = src;
if (src == IterativeParsingArrayInitialState || src == IterativeParsingElementDelimiterState) if (src == IterativeParsingArrayInitialState || src == IterativeParsingElementDelimiterState)
n = IterativeParsingElementState; n = IterativeParsingElementState;
else if (src == IterativeParsingKeyValueDelimiterState) else if (src == IterativeParsingKeyValueDelimiterState)
n = IterativeParsingMemberValueState; n = IterativeParsingMemberValueState;
// Check stack space limit.
if (!CheckStackSpaceQuota(sizeof(IterativeParsingState) + sizeof(int))) {
RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorStackSizeLimitExceeded, is.Tell());
return IterativeParsingErrorState;
}
// Push current state. // Push current state.
*stack_.template Push<IterativeParsingState>(1) = n; *stack_.template Push<SizeType>(1) = n;
// Initialize and push the member/element count. // Initialize and push the member/element count.
*stack_.template Push<int>(1) = 0; *stack_.template Push<SizeType>(1) = 0;
// Call handler // Call handler
if (dst == IterativeParsingObjectInitialState) bool hr = (dst == IterativeParsingObjectInitialState) ? handler.StartObject() : handler.StartArray();
hr = handler.StartObject();
else
hr = handler.StartArray();
// On handler short circuits the parsing. // On handler short circuits the parsing.
if (!hr) { if (!hr) {
RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell());
@ -1073,6 +1062,7 @@ private:
is.Take(); is.Take();
return dst; return dst;
} }
}
case IterativeParsingMemberKeyState: case IterativeParsingMemberKeyState:
ParseString<parseFlags>(is, handler); ParseString<parseFlags>(is, handler);
@ -1082,7 +1072,7 @@ private:
return dst; return dst;
case IterativeParsingKeyValueDelimiterState: case IterativeParsingKeyValueDelimiterState:
if (token == IterativeParsingColonToken) { if (token == ColonToken) {
is.Take(); is.Take();
return dst; return dst;
} }
@ -1109,22 +1099,23 @@ private:
case IterativeParsingElementDelimiterState: case IterativeParsingElementDelimiterState:
is.Take(); is.Take();
// Update member/element count. // Update member/element count.
*stack_.template Top<int>() = *stack_.template Top<int>() + 1; *stack_.template Top<SizeType>() = *stack_.template Top<SizeType>() + 1;
return dst; return dst;
case IterativeParsingObjectFinishState: case IterativeParsingObjectFinishState:
{
// Get member count. // Get member count.
c = *stack_.template Pop<int>(1); SizeType c = *stack_.template Pop<SizeType>(1);
// If the object is not empty, count the last member. // If the object is not empty, count the last member.
if (src == IterativeParsingMemberValueState) if (src == IterativeParsingMemberValueState)
++c; ++c;
// Restore the state. // Restore the state.
n = *stack_.template Pop<IterativeParsingState>(1); IterativeParsingState n = static_cast<IterativeParsingState>(*stack_.template Pop<SizeType>(1));
// Transit to Finish state if this is the topmost scope. // Transit to Finish state if this is the topmost scope.
if (n == IterativeParsingStartState) if (n == IterativeParsingStartState)
n = IterativeParsingFinishState; n = IterativeParsingFinishState;
// Call handler // Call handler
hr = handler.EndObject(c); bool hr = handler.EndObject(c);
// On handler short circuits the parsing. // On handler short circuits the parsing.
if (!hr) { if (!hr) {
RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell());
@ -1134,20 +1125,22 @@ private:
is.Take(); is.Take();
return n; return n;
} }
}
case IterativeParsingArrayFinishState: case IterativeParsingArrayFinishState:
{
// Get element count. // Get element count.
c = *stack_.template Pop<int>(1); SizeType c = *stack_.template Pop<SizeType>(1);
// If the array is not empty, count the last element. // If the array is not empty, count the last element.
if (src == IterativeParsingElementState) if (src == IterativeParsingElementState)
++c; ++c;
// Restore the state. // Restore the state.
n = *stack_.template Pop<IterativeParsingState>(1); IterativeParsingState n = static_cast<IterativeParsingState>(*stack_.template Pop<SizeType>(1));
// Transit to Finish state if this is the topmost scope. // Transit to Finish state if this is the topmost scope.
if (n == IterativeParsingStartState) if (n == IterativeParsingStartState)
n = IterativeParsingFinishState; n = IterativeParsingFinishState;
// Call handler // Call handler
hr = handler.EndArray(c); bool hr = handler.EndArray(c);
// On handler short circuits the parsing. // On handler short circuits the parsing.
if (!hr) { if (!hr) {
RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell());
@ -1157,6 +1150,7 @@ private:
is.Take(); is.Take();
return n; return n;
} }
}
default: default:
RAPIDJSON_ASSERT(false); RAPIDJSON_ASSERT(false);
@ -1171,29 +1165,16 @@ private:
return; return;
} }
if (src == IterativeParsingStartState && is.Peek() == '\0') switch (src) {
RAPIDJSON_PARSE_ERROR(kParseErrorDocumentEmpty, is.Tell()); case IterativeParsingStartState: RAPIDJSON_PARSE_ERROR(is.Peek() == '\0' ? kParseErrorDocumentEmpty : kParseErrorDocumentRootNotObjectOrArray, is.Tell());
case IterativeParsingFinishState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentRootNotSingular, is.Tell());
else if (src == IterativeParsingStartState) case IterativeParsingObjectInitialState:
RAPIDJSON_PARSE_ERROR(kParseErrorDocumentRootNotObjectOrArray, is.Tell()); case IterativeParsingMemberDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell());
case IterativeParsingMemberKeyState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell());
else if (src == IterativeParsingFinishState) case IterativeParsingMemberValueState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell());
RAPIDJSON_PARSE_ERROR(kParseErrorDocumentRootNotSingular, is.Tell()); case IterativeParsingElementState: RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell());
default: RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell());
else if (src == IterativeParsingObjectInitialState || src == IterativeParsingMemberDelimiterState) }
RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell());
else if (src == IterativeParsingMemberKeyState)
RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell());
else if (src == IterativeParsingMemberValueState)
RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell());
else if (src == IterativeParsingElementState)
RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell());
else
RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell());
} }
template <unsigned parseFlags, typename InputStream, typename Handler> template <unsigned parseFlags, typename InputStream, typename Handler>
@ -1204,7 +1185,7 @@ private:
SkipWhitespace(is); SkipWhitespace(is);
while (is.Peek() != '\0') { while (is.Peek() != '\0') {
IterativeParsingToken t = Tokenize(is.Peek()); Token t = Tokenize(is.Peek());
IterativeParsingState n = Predict(state, t); IterativeParsingState n = Predict(state, t);
IterativeParsingState d = Transit<parseFlags>(state, t, n, is, handler); IterativeParsingState d = Transit<parseFlags>(state, t, n, is, handler);
@ -1214,6 +1195,11 @@ private:
} }
state = d; state = d;
// Do not further consume streams if a root JSON has been parsed.
if ((parseFlags & kParseStopWhenDoneFlag) && state == IterativeParsingFinishState)
break;
SkipWhitespace(is); SkipWhitespace(is);
} }
@ -1224,13 +1210,8 @@ private:
return parseResult_; return parseResult_;
} }
bool CheckStackSpaceQuota(size_t size) const {
return kStackSizeLimit_ == 0 || (stack_.GetSize() + size <= kStackSizeLimit_);
}
static const size_t kDefaultStackCapacity = 256; //!< Default stack capacity in bytes for storing a single decoded string. static const size_t kDefaultStackCapacity = 256; //!< Default stack capacity in bytes for storing a single decoded string.
internal::Stack<Allocator> stack_; //!< A stack for storing decoded string temporarily during non-destructive parsing. internal::Stack<Allocator> stack_; //!< A stack for storing decoded string temporarily during non-destructive parsing.
const size_t kStackSizeLimit_; //!< Stack size limit(in bytes). A value of 0 means no limit.
ParseResult parseResult_; ParseResult parseResult_;
}; // class GenericReader }; // class GenericReader

44
test/unittest/readertest.cpp
View File

@ -545,6 +545,38 @@ TEST(Reader, Parse_EmptyObject) {
EXPECT_EQ(2u, h.step_); EXPECT_EQ(2u, h.step_);
} }
struct ParseMultipleRootHandler : BaseReaderHandler<> {
ParseMultipleRootHandler() : step_(0) {}
bool Default() { ADD_FAILURE(); return false; }
bool StartObject() { EXPECT_EQ(0u, step_); step_++; return true; }
bool EndObject(SizeType) { EXPECT_EQ(1u, step_); step_++; return true; }
bool StartArray() { EXPECT_EQ(2u, step_); step_++; return true; }
bool EndArray(SizeType) { EXPECT_EQ(3u, step_); step_++; return true; }
unsigned step_;
};
template <unsigned parseFlags>
void TestMultipleRoot() {
StringStream s("{}[]a");
ParseMultipleRootHandler h;
Reader reader;
EXPECT_TRUE(reader.Parse<parseFlags>(s, h));
EXPECT_EQ(2u, h.step_);
EXPECT_TRUE(reader.Parse<parseFlags>(s, h));
EXPECT_EQ(4u, h.step_);
EXPECT_EQ('a', s.Peek());
}
TEST(Reader, Parse_MultipleRoot) {
TestMultipleRoot<kParseStopWhenDoneFlag>();
}
TEST(Reader, ParseIterative_MultipleRoot) {
TestMultipleRoot<kParseIterativeFlag | kParseStopWhenDoneFlag>();
}
#define TEST_ERROR(errorCode, str) \ #define TEST_ERROR(errorCode, str) \
{ \ { \
char buffer[1001]; \ char buffer[1001]; \
@ -932,18 +964,6 @@ TEST(Reader, IterativeParsing_ShortCircuit) {
} }
} }
TEST(Reader, IterativeParsing_LimitStackSize) {
BaseReaderHandler<> handler;
Reader reader(20);
StringStream is("[[[]]]");
ParseResult r = reader.Parse<kParseIterativeFlag>(is, handler);
EXPECT_TRUE(reader.HasParseError());
EXPECT_EQ(kParseErrorStackSizeLimitExceeded, r.Code());
EXPECT_EQ(2u, r.Offset());
}
#ifdef __GNUC__ #ifdef __GNUC__
RAPIDJSON_DIAG_POP RAPIDJSON_DIAG_POP
#endif #endif