diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h index 69d8cd8..084cb1e 100644 --- a/include/rapidjson/reader.h +++ b/include/rapidjson/reader.h @@ -65,7 +65,8 @@ enum ParseFlag { kParseDefaultFlags = 0, //!< Default parse flags. Non-destructive parsing. Text strings are decoded into allocated buffer. kParseInsituFlag = 1, //!< In-situ(destructive) parsing. kParseValidateEncodingFlag = 2, //!< Validate encoding of JSON strings. - kParseIterativeFlag = 4 //!< Iterative(constant complexity in terms of function call stack size) parsing. + kParseIterativeFlag = 4, //!< Iterative(constant complexity in terms of function call stack size) parsing. + kParseStopWhenDoneFlag = 8 //!< After parsing a complete JSON root from stream, stop further processing the rest of stream. When this flag is used, parser will not generate kParseErrorDocumentRootNotSingular error. }; /////////////////////////////////////////////////////////////////////////////// @@ -273,11 +274,10 @@ public: typedef typename SourceEncoding::Ch Ch; //!< SourceEncoding character type //! Constructor. - /*! \param limit Parsing stack size limit(in bytes). Pass 0 means no limit. - \param allocator Optional allocator for allocating stack memory. (Only use for non-destructive parsing) + /*! \param allocator Optional allocator for allocating stack memory. (Only use for non-destructive parsing) \param stackCapacity stack capacity in bytes for storing a single decoded string. (Only use for non-destructive parsing) */ - GenericReader(size_t limit = 0, Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity) : stack_(allocator, stackCapacity), kStackSizeLimit_(limit), parseResult_() {} + GenericReader(Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity) : stack_(allocator, stackCapacity), parseResult_() {} //! Parse JSON text. /*! \tparam parseFlags Combination of \ref ParseFlag. @@ -310,11 +310,13 @@ public: } RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); - SkipWhitespace(is); + if (!(parseFlags & kParseStopWhenDoneFlag)) { + SkipWhitespace(is); - if (is.Peek() != '\0') { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentRootNotSingular, is.Tell()); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + if (is.Peek() != '\0') { + RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentRootNotSingular, is.Tell()); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_); + } } } @@ -513,7 +515,7 @@ private: typedef typename TargetEncoding::Ch Ch; StackStream(internal::Stack& stack) : stack_(stack), length_(0) {} - void Put(Ch c) { + RAPIDJSON_FORCEINLINE void Put(Ch c) { *stack_.template Push() = c; ++length_; } @@ -572,11 +574,6 @@ private: is.Take(); Ch e = is.Take(); if ((sizeof(Ch) == 1 || unsigned(e) < 256) && escape[(unsigned char)e]) { - if (!(parseFlags & kParseInsituFlag)) { - if (!CheckStackSpaceQuota(sizeof(Ch))) { - RAPIDJSON_PARSE_ERROR(kParseErrorStackSizeLimitExceeded, is.Tell() - 1); - } - } os.Put(escape[(unsigned char)e]); } else if (e == 'u') { // Unicode @@ -597,11 +594,6 @@ private: } else if (c == '"') { // Closing double quote is.Take(); - if (!(parseFlags & kParseInsituFlag)) { - if (!CheckStackSpaceQuota(sizeof(Ch))) { - RAPIDJSON_PARSE_ERROR(kParseErrorStackSizeLimitExceeded, is.Tell() - 1); - } - } os.Put('\0'); // null-terminate the string return; } @@ -829,44 +821,52 @@ private: }; // Tokens - enum IterativeParsingToken { - IterativeParsingLeftBracketToken = 0, - IterativeParsingRightBracketToken, + enum Token { + LeftBracketToken = 0, + RightBracketToken, - IterativeParsingLeftCurlyBracketToken, - IterativeParsingRightCurlyBracketToken, + LeftCurlyBracketToken, + RightCurlyBracketToken, - IterativeParsingCommaToken, - IterativeParsingColonToken, + CommaToken, + ColonToken, - IterativeParsingStringToken, - IterativeParsingFalseToken, - IterativeParsingTrueToken, - IterativeParsingNullToken, - IterativeParsingNumberToken, + StringToken, + FalseToken, + TrueToken, + NullToken, + NumberToken, - cIterativeParsingTokenCount + kTokenCount }; - IterativeParsingToken Tokenize(Ch c) { - switch (c) { - case '[': return IterativeParsingLeftBracketToken; - case ']': return IterativeParsingRightBracketToken; - case '{': return IterativeParsingLeftCurlyBracketToken; - case '}': return IterativeParsingRightCurlyBracketToken; - case ',': return IterativeParsingCommaToken; - case ':': return IterativeParsingColonToken; - case '"': return IterativeParsingStringToken; - case 'f': return IterativeParsingFalseToken; - case 't': return IterativeParsingTrueToken; - case 'n': return IterativeParsingNullToken; - default: return IterativeParsingNumberToken; - } + RAPIDJSON_FORCEINLINE Token Tokenize(Ch c) { +#define N NumberToken +#define N16 N,N,N,N,N,N,N,N,N,N,N,N,N,N,N,N + // Maps from ASCII to Token + static const unsigned char tokenMap[256] = { + N16, // 00~0F + N16, // 10~1F + N, N, StringToken, N, N, N, N, N, N, N, N, N, CommaToken, N, N, N, // 20~2F + N, N, N, N, N, N, N, N, N, N, ColonToken, N, N, N, N, N, // 30~3F + N16, // 40~4F + N, N, N, N, N, N, N, N, N, N, N, LeftBracketToken, N, RightBracketToken, N, N, // 50~5F + N, N, N, N, N, N, FalseToken, N, N, N, N, N, N, N, NullToken, N, // 60~6F + N, N, N, N, TrueToken, N, N, N, N, N, N, LeftCurlyBracketToken, N, RightCurlyBracketToken, N, N, // 70~7F + N16, N16, N16, N16, N16, N16, N16, N16 // 80~FF + }; +#undef N +#undef N16 + + if (sizeof(Ch) == 1 || unsigned(c) < 256) + return (Token)tokenMap[(unsigned char)c]; + else + return NumberToken; } - IterativeParsingState Predict(IterativeParsingState state, IterativeParsingToken token) { + RAPIDJSON_FORCEINLINE IterativeParsingState Predict(IterativeParsingState state, Token token) { // current state x one lookahead token -> new state - static const char G[cIterativeParsingStateCount][cIterativeParsingTokenCount] = { + static const char G[cIterativeParsingStateCount][kTokenCount] = { // Start { IterativeParsingArrayInitialState, // Left bracket @@ -1025,11 +1025,7 @@ private: // Make an advance in the token stream and state based on the candidate destination state which was returned by Transit(). // May return a new state on state pop. template - IterativeParsingState Transit(IterativeParsingState src, IterativeParsingToken token, IterativeParsingState dst, InputStream& is, Handler& handler) { - int c = 0; - IterativeParsingState n; - bool hr; - + RAPIDJSON_FORCEINLINE IterativeParsingState Transit(IterativeParsingState src, Token token, IterativeParsingState dst, InputStream& is, Handler& handler) { switch (dst) { case IterativeParsingStartState: RAPIDJSON_ASSERT(false); @@ -1043,27 +1039,20 @@ private: case IterativeParsingObjectInitialState: case IterativeParsingArrayInitialState: + { // Push the state(Element or MemeberValue) if we are nested in another array or value of member. // In this way we can get the correct state on ObjectFinish or ArrayFinish by frame pop. - n = src; + IterativeParsingState n = src; if (src == IterativeParsingArrayInitialState || src == IterativeParsingElementDelimiterState) n = IterativeParsingElementState; else if (src == IterativeParsingKeyValueDelimiterState) n = IterativeParsingMemberValueState; - // Check stack space limit. - if (!CheckStackSpaceQuota(sizeof(IterativeParsingState) + sizeof(int))) { - RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorStackSizeLimitExceeded, is.Tell()); - return IterativeParsingErrorState; - } // Push current state. - *stack_.template Push(1) = n; + *stack_.template Push(1) = n; // Initialize and push the member/element count. - *stack_.template Push(1) = 0; + *stack_.template Push(1) = 0; // Call handler - if (dst == IterativeParsingObjectInitialState) - hr = handler.StartObject(); - else - hr = handler.StartArray(); + bool hr = (dst == IterativeParsingObjectInitialState) ? handler.StartObject() : handler.StartArray(); // On handler short circuits the parsing. if (!hr) { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); @@ -1073,6 +1062,7 @@ private: is.Take(); return dst; } + } case IterativeParsingMemberKeyState: ParseString(is, handler); @@ -1082,7 +1072,7 @@ private: return dst; case IterativeParsingKeyValueDelimiterState: - if (token == IterativeParsingColonToken) { + if (token == ColonToken) { is.Take(); return dst; } @@ -1109,22 +1099,23 @@ private: case IterativeParsingElementDelimiterState: is.Take(); // Update member/element count. - *stack_.template Top() = *stack_.template Top() + 1; + *stack_.template Top() = *stack_.template Top() + 1; return dst; case IterativeParsingObjectFinishState: + { // Get member count. - c = *stack_.template Pop(1); + SizeType c = *stack_.template Pop(1); // If the object is not empty, count the last member. if (src == IterativeParsingMemberValueState) ++c; // Restore the state. - n = *stack_.template Pop(1); + IterativeParsingState n = static_cast(*stack_.template Pop(1)); // Transit to Finish state if this is the topmost scope. if (n == IterativeParsingStartState) n = IterativeParsingFinishState; // Call handler - hr = handler.EndObject(c); + bool hr = handler.EndObject(c); // On handler short circuits the parsing. if (!hr) { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); @@ -1134,20 +1125,22 @@ private: is.Take(); return n; } + } case IterativeParsingArrayFinishState: + { // Get element count. - c = *stack_.template Pop(1); + SizeType c = *stack_.template Pop(1); // If the array is not empty, count the last element. if (src == IterativeParsingElementState) ++c; // Restore the state. - n = *stack_.template Pop(1); + IterativeParsingState n = static_cast(*stack_.template Pop(1)); // Transit to Finish state if this is the topmost scope. if (n == IterativeParsingStartState) n = IterativeParsingFinishState; // Call handler - hr = handler.EndArray(c); + bool hr = handler.EndArray(c); // On handler short circuits the parsing. if (!hr) { RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); @@ -1157,6 +1150,7 @@ private: is.Take(); return n; } + } default: RAPIDJSON_ASSERT(false); @@ -1171,29 +1165,16 @@ private: return; } - if (src == IterativeParsingStartState && is.Peek() == '\0') - RAPIDJSON_PARSE_ERROR(kParseErrorDocumentEmpty, is.Tell()); - - else if (src == IterativeParsingStartState) - RAPIDJSON_PARSE_ERROR(kParseErrorDocumentRootNotObjectOrArray, is.Tell()); - - else if (src == IterativeParsingFinishState) - RAPIDJSON_PARSE_ERROR(kParseErrorDocumentRootNotSingular, is.Tell()); - - else if (src == IterativeParsingObjectInitialState || src == IterativeParsingMemberDelimiterState) - RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); - - else if (src == IterativeParsingMemberKeyState) - RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); - - else if (src == IterativeParsingMemberValueState) - RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); - - else if (src == IterativeParsingElementState) - RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); - - else - RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); + switch (src) { + case IterativeParsingStartState: RAPIDJSON_PARSE_ERROR(is.Peek() == '\0' ? kParseErrorDocumentEmpty : kParseErrorDocumentRootNotObjectOrArray, is.Tell()); + case IterativeParsingFinishState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentRootNotSingular, is.Tell()); + case IterativeParsingObjectInitialState: + case IterativeParsingMemberDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); + case IterativeParsingMemberKeyState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); + case IterativeParsingMemberValueState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); + case IterativeParsingElementState: RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); + default: RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); + } } template @@ -1204,7 +1185,7 @@ private: SkipWhitespace(is); while (is.Peek() != '\0') { - IterativeParsingToken t = Tokenize(is.Peek()); + Token t = Tokenize(is.Peek()); IterativeParsingState n = Predict(state, t); IterativeParsingState d = Transit(state, t, n, is, handler); @@ -1214,6 +1195,11 @@ private: } state = d; + + // Do not further consume streams if a root JSON has been parsed. + if ((parseFlags & kParseStopWhenDoneFlag) && state == IterativeParsingFinishState) + break; + SkipWhitespace(is); } @@ -1224,13 +1210,8 @@ private: return parseResult_; } - bool CheckStackSpaceQuota(size_t size) const { - return kStackSizeLimit_ == 0 || (stack_.GetSize() + size <= kStackSizeLimit_); - } - static const size_t kDefaultStackCapacity = 256; //!< Default stack capacity in bytes for storing a single decoded string. internal::Stack stack_; //!< A stack for storing decoded string temporarily during non-destructive parsing. - const size_t kStackSizeLimit_; //!< Stack size limit(in bytes). A value of 0 means no limit. ParseResult parseResult_; }; // class GenericReader diff --git a/test/unittest/readertest.cpp b/test/unittest/readertest.cpp index 1919e94..78079e5 100644 --- a/test/unittest/readertest.cpp +++ b/test/unittest/readertest.cpp @@ -545,6 +545,38 @@ TEST(Reader, Parse_EmptyObject) { EXPECT_EQ(2u, h.step_); } +struct ParseMultipleRootHandler : BaseReaderHandler<> { + ParseMultipleRootHandler() : step_(0) {} + + bool Default() { ADD_FAILURE(); return false; } + bool StartObject() { EXPECT_EQ(0u, step_); step_++; return true; } + bool EndObject(SizeType) { EXPECT_EQ(1u, step_); step_++; return true; } + bool StartArray() { EXPECT_EQ(2u, step_); step_++; return true; } + bool EndArray(SizeType) { EXPECT_EQ(3u, step_); step_++; return true; } + + unsigned step_; +}; + +template +void TestMultipleRoot() { + StringStream s("{}[]a"); + ParseMultipleRootHandler h; + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + EXPECT_EQ(2u, h.step_); + EXPECT_TRUE(reader.Parse(s, h)); + EXPECT_EQ(4u, h.step_); + EXPECT_EQ('a', s.Peek()); +} + +TEST(Reader, Parse_MultipleRoot) { + TestMultipleRoot(); +} + +TEST(Reader, ParseIterative_MultipleRoot) { + TestMultipleRoot(); +} + #define TEST_ERROR(errorCode, str) \ { \ char buffer[1001]; \ @@ -932,18 +964,6 @@ TEST(Reader, IterativeParsing_ShortCircuit) { } } -TEST(Reader, IterativeParsing_LimitStackSize) { - BaseReaderHandler<> handler; - Reader reader(20); - StringStream is("[[[]]]"); - - ParseResult r = reader.Parse(is, handler); - - EXPECT_TRUE(reader.HasParseError()); - EXPECT_EQ(kParseErrorStackSizeLimitExceeded, r.Code()); - EXPECT_EQ(2u, r.Offset()); -} - #ifdef __GNUC__ RAPIDJSON_DIAG_POP #endif