Try to resolve issue #35: implement iterative parsing.
This commit is contained in:
thebusytypist
parent
c12286a072
commit
3006fa7d8c
@ -46,7 +46,8 @@ namespace rapidjson {
|
||||
enum ParseFlag {
|
||||
kParseDefaultFlags = 0, //!< Default parse flags. Non-destructive parsing. Text strings are decoded into allocated buffer.
|
||||
kParseInsituFlag = 1, //!< In-situ(destructive) parsing.
|
||||
kParseValidateEncodingFlag = 2 //!< Validate encoding of JSON strings.
|
||||
kParseValidateEncodingFlag = 2, //!< Validate encoding of JSON strings.
|
||||
kParseNonRecursiveFlag = 4 //!< Non-recursive(constant complexity in terms of function call stack size) parsing.
|
||||
};
|
||||
|
||||
//! Error code of parsing.
|
||||
@ -69,7 +70,7 @@ enum ParseErrorCode {
|
||||
kParseErrorStringUnicodeSurrogateInvalid, //!< The surrogate pair in string is invalid.
|
||||
kParseErrorStringEscapeInvalid, //!< Invalid escape character in string.
|
||||
kParseErrorStringMissQuotationMark, //!< Missing a closing quotation mark in string.
|
||||
kParseErrorStringInvalidEncoding, //!< Invalid encoidng in string.
|
||||
kParseErrorStringInvalidEncoding, //!< Invalid encoding in string.
|
||||
|
||||
kParseErrorNumberTooBig, //!< Number too big to be stored in double.
|
||||
kParseErrorNumberMissFraction, //!< Miss fraction part in number.
|
||||
@ -134,7 +135,7 @@ namespace internal {
|
||||
template<typename Stream, int = StreamTraits<Stream>::copyOptimization>
|
||||
class StreamLocalCopy;
|
||||
|
||||
//! Do copy optimziation.
|
||||
//! Do copy optimization.
|
||||
template<typename Stream>
|
||||
class StreamLocalCopy<Stream, 1> {
|
||||
public:
|
||||
@ -297,6 +298,9 @@ public:
|
||||
parseErrorCode_ = kParseErrorNone;
|
||||
errorOffset_ = 0;
|
||||
|
||||
if (parseFlags & kParseNonRecursiveFlag)
|
||||
return NonRecursiveParse<parseFlags>(is, handler);
|
||||
|
||||
SkipWhitespace(is);
|
||||
|
||||
if (is.Peek() == '\0')
|
||||
@ -748,6 +752,220 @@ private:
|
||||
}
|
||||
}
|
||||
|
||||
// Non-recursive parsing
|
||||
enum NonRecursiveParsingState {
|
||||
NonRecursiveParsingStartState,
|
||||
NonRecursiveParsingFinishState,
|
||||
NonRecursiveParsingErrorState,
|
||||
// Object states
|
||||
NonRecursiveParsingObjectInitialState,
|
||||
NonRecursiveParsingObjectContentState,
|
||||
// Array states
|
||||
NonRecursiveParsingArrayInitialState,
|
||||
NonRecursiveParsingArrayContentState
|
||||
};
|
||||
|
||||
template <typename InputStream, typename Handler>
|
||||
NonRecursiveParsingState TransitToCompoundValueTypeState(NonRecursiveParsingState state, InputStream& is, Handler& handler) {
|
||||
// For compound value type(object and array), we should push the current state and start a new stack frame for this type.
|
||||
NonRecursiveParsingState r = NonRecursiveParsingErrorState;
|
||||
|
||||
switch (is.Take()) {
|
||||
case '{':
|
||||
handler.StartObject();
|
||||
r = NonRecursiveParsingObjectInitialState;
|
||||
// Push current state.
|
||||
*stack_.template Push<NonRecursiveParsingState>(1) = state;
|
||||
// Initialize and push member count.
|
||||
*stack_.template Push<int>(1) = 0;
|
||||
break;
|
||||
case '[':
|
||||
handler.StartArray();
|
||||
r = NonRecursiveParsingArrayInitialState;
|
||||
// Push current state.
|
||||
*stack_.template Push<NonRecursiveParsingState>(1) = state;
|
||||
// Initialize and push element count.
|
||||
*stack_.template Push<int>(1) = 0;
|
||||
break;
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
// Inner transition of object or array states(ObjectInitial->ObjectContent, ArrayInitial->ArrayContent).
|
||||
template <unsigned parseFlags, typename InputStream, typename Handler>
|
||||
NonRecursiveParsingState TransitByValue(NonRecursiveParsingState state, InputStream& is, Handler& handler) {
|
||||
RAPIDJSON_ASSERT(
|
||||
state == NonRecursiveParsingObjectInitialState ||
|
||||
state == NonRecursiveParsingArrayInitialState ||
|
||||
state == NonRecursiveParsingObjectContentState ||
|
||||
state == NonRecursiveParsingArrayContentState);
|
||||
|
||||
NonRecursiveParsingState t;
|
||||
if (state == NonRecursiveParsingObjectInitialState)
|
||||
t = NonRecursiveParsingObjectContentState;
|
||||
else if (state == NonRecursiveParsingArrayInitialState)
|
||||
t = NonRecursiveParsingArrayContentState;
|
||||
else
|
||||
t = state;
|
||||
|
||||
NonRecursiveParsingState r = NonRecursiveParsingErrorState;
|
||||
|
||||
switch (is.Peek()) {
|
||||
// For plain value state is not changed.
|
||||
case 'n': ParseNull <parseFlags>(is, handler); r = t; break;
|
||||
case 't': ParseTrue <parseFlags>(is, handler); r = t; break;
|
||||
case 'f': ParseFalse <parseFlags>(is, handler); r = t; break;
|
||||
case '"': ParseString<parseFlags>(is, handler); r = t; break;
|
||||
// Transit when value is object or array.
|
||||
case '{':
|
||||
case '[':
|
||||
r = TransitToCompoundValueTypeState(state, is, handler); break;
|
||||
default: ParseNumber<parseFlags>(is, handler); r = t; break;
|
||||
}
|
||||
|
||||
if (HasParseError())
|
||||
r = NonRecursiveParsingErrorState;
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
// Transit from object related states(ObjectInitial, ObjectContent).
|
||||
template <unsigned parseFlags, typename InputStream, typename Handler>
|
||||
NonRecursiveParsingState TransitFromObjectStates(NonRecursiveParsingState state, InputStream& is, Handler& handler) {
|
||||
NonRecursiveParsingState r = NonRecursiveParsingErrorState;
|
||||
|
||||
switch (is.Peek()) {
|
||||
case '}': {
|
||||
is.Take();
|
||||
// Get member count(include an extra one for non-empty object).
|
||||
int memberCount = *stack_.template Pop<int>(1);
|
||||
if (state == NonRecursiveParsingObjectContentState)
|
||||
++memberCount;
|
||||
// Restore the parent stack frame.
|
||||
r = *stack_.template Pop<NonRecursiveParsingState>(1);
|
||||
// Transit to ContentState since a member/an element was just parsed.
|
||||
if (r == NonRecursiveParsingArrayInitialState)
|
||||
r = NonRecursiveParsingArrayContentState;
|
||||
else if (r == NonRecursiveParsingObjectInitialState)
|
||||
r = NonRecursiveParsingObjectContentState;
|
||||
// If we return to the topmost frame mark it finished.
|
||||
if (r == NonRecursiveParsingStartState)
|
||||
r = NonRecursiveParsingFinishState;
|
||||
handler.EndObject(memberCount);
|
||||
break;
|
||||
}
|
||||
case ',':
|
||||
is.Take();
|
||||
r = NonRecursiveParsingObjectContentState;
|
||||
// Update member count.
|
||||
*stack_.template Top<int>() = *stack_.template Top<int>() + 1;
|
||||
break;
|
||||
case '"':
|
||||
// Should be a key-value pair.
|
||||
ParseString<parseFlags>(is, handler);
|
||||
if (HasParseError()) {
|
||||
r = NonRecursiveParsingErrorState;
|
||||
RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorObjectMissName, is.Tell());
|
||||
break;
|
||||
}
|
||||
|
||||
SkipWhitespace(is);
|
||||
|
||||
if (is.Take() != ':') {
|
||||
r = NonRecursiveParsingErrorState;
|
||||
RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorObjectMissColon, is.Tell());
|
||||
break;
|
||||
}
|
||||
|
||||
SkipWhitespace(is);
|
||||
|
||||
r = TransitByValue<parseFlags>(state, is, handler);
|
||||
|
||||
break;
|
||||
default:
|
||||
r = NonRecursiveParsingErrorState;
|
||||
RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell());
|
||||
break;
|
||||
}
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
// Transit from array related states(ArrayInitial, ArrayContent).
|
||||
template <unsigned parseFlags, typename InputStream, typename Handler>
|
||||
NonRecursiveParsingState TransitFromArrayStates(NonRecursiveParsingState state, InputStream& is, Handler& handler) {
|
||||
NonRecursiveParsingState r = NonRecursiveParsingErrorState;
|
||||
|
||||
switch (is.Peek()) {
|
||||
case ']': {
|
||||
is.Take();
|
||||
// Get element count(include an extra one for non-empty array).
|
||||
int elementCount = *stack_.template Pop<int>(1);
|
||||
if (state == NonRecursiveParsingArrayContentState)
|
||||
++elementCount;
|
||||
// Restore the parent stack frame.
|
||||
r = *stack_.template Pop<NonRecursiveParsingState>(1);
|
||||
// Transit to ContentState since a member/an element was just parsed.
|
||||
if (r == NonRecursiveParsingArrayInitialState)
|
||||
r = NonRecursiveParsingArrayContentState;
|
||||
else if (r == NonRecursiveParsingObjectInitialState)
|
||||
r = NonRecursiveParsingObjectContentState;
|
||||
// If we return to the topmost frame mark it finished.
|
||||
if (r == NonRecursiveParsingStartState)
|
||||
r = NonRecursiveParsingFinishState;
|
||||
handler.EndArray(elementCount);
|
||||
break;
|
||||
}
|
||||
case ',':
|
||||
is.Take();
|
||||
r = NonRecursiveParsingArrayContentState;
|
||||
// Update element count.
|
||||
*stack_.template Top<int>() = *stack_.template Top<int>() + 1;
|
||||
break;
|
||||
default:
|
||||
// Should be a single value.
|
||||
r = TransitByValue<parseFlags>(state, is, handler);
|
||||
break;
|
||||
}
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
template <unsigned parseFlags, typename InputStream, typename Handler>
|
||||
NonRecursiveParsingState Transit(NonRecursiveParsingState state, InputStream& is, Handler& handler) {
|
||||
NonRecursiveParsingState r = NonRecursiveParsingErrorState;
|
||||
|
||||
switch (state) {
|
||||
case NonRecursiveParsingStartState:
|
||||
r = TransitToCompoundValueTypeState(state, is, handler);
|
||||
break;
|
||||
case NonRecursiveParsingObjectInitialState:
|
||||
case NonRecursiveParsingObjectContentState:
|
||||
r = TransitFromObjectStates<parseFlags>(state, is, handler);
|
||||
break;
|
||||
case NonRecursiveParsingArrayInitialState:
|
||||
case NonRecursiveParsingArrayContentState:
|
||||
r = TransitFromArrayStates<parseFlags>(state, is, handler);
|
||||
break;
|
||||
}
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
template <unsigned parseFlags, typename InputStream, typename Handler>
|
||||
bool NonRecursiveParse(InputStream& is, Handler& handler) {
|
||||
NonRecursiveParsingState state = NonRecursiveParsingStartState;
|
||||
|
||||
SkipWhitespace(is);
|
||||
while (is.Peek() != '\0' && state != NonRecursiveParsingErrorState) {
|
||||
state = Transit<parseFlags>(state, is, handler);
|
||||
SkipWhitespace(is);
|
||||
}
|
||||
|
||||
stack_.Clear();
|
||||
return state == NonRecursiveParsingFinishState && !HasParseError();
|
||||
}
|
||||
|
||||
static const size_t kDefaultStackCapacity = 256; //!< Default stack capacity in bytes for storing a single decoded string.
|
||||
internal::Stack<Allocator> stack_; //!< A stack for storing decoded string temporarily during non-destructive parsing.
|
||||
ParseErrorCode parseErrorCode_;
|
||||
|
||||
@ -650,7 +650,7 @@ struct StreamTraits<CustomStringStream<Encoding> > {
|
||||
enum { copyOptimization = 1 };
|
||||
};
|
||||
|
||||
} // namespace rapdijson
|
||||
} // namespace rapidjson
|
||||
#endif
|
||||
|
||||
TEST(Reader, CustomStringStream) {
|
||||
@ -706,6 +706,200 @@ TEST(Reader, Parse_IStreamWrapper_StringStream) {
|
||||
EXPECT_FALSE(reader.HasParseError());
|
||||
}
|
||||
|
||||
TEST(Reader, NonRecursiveParsing) {
|
||||
StringStream json("[1,true,false,null,\"string\",{\"array\":[1]}]");
|
||||
Reader reader;
|
||||
BaseReaderHandler<> handler;
|
||||
|
||||
Reader::NonRecursiveParsingState r;
|
||||
|
||||
// [
|
||||
r = reader.Transit<kParseNonRecursiveFlag>(
|
||||
Reader::NonRecursiveParsingStartState,
|
||||
json,
|
||||
handler);
|
||||
EXPECT_FALSE(reader.HasParseError());
|
||||
EXPECT_EQ(Reader::NonRecursiveParsingArrayInitialState, r);
|
||||
|
||||
// 1
|
||||
r = reader.Transit<kParseNonRecursiveFlag>(
|
||||
r,
|
||||
json,
|
||||
handler);
|
||||
EXPECT_FALSE(reader.HasParseError());
|
||||
EXPECT_EQ(Reader::NonRecursiveParsingArrayContentState, r);
|
||||
EXPECT_EQ(0, *reader.stack_.template Top<int>()); // element count
|
||||
|
||||
// ,
|
||||
r = reader.Transit<kParseNonRecursiveFlag>(
|
||||
r,
|
||||
json,
|
||||
handler);
|
||||
EXPECT_FALSE(reader.HasParseError());
|
||||
EXPECT_EQ(Reader::NonRecursiveParsingArrayContentState, r);
|
||||
EXPECT_EQ(1, *reader.stack_.template Top<int>()); // element count
|
||||
|
||||
// true
|
||||
r = reader.Transit<kParseNonRecursiveFlag>(
|
||||
r,
|
||||
json,
|
||||
handler);
|
||||
EXPECT_FALSE(reader.HasParseError());
|
||||
EXPECT_EQ(Reader::NonRecursiveParsingArrayContentState, r);
|
||||
EXPECT_EQ(1, *reader.stack_.template Top<int>()); // element count
|
||||
|
||||
// ,
|
||||
r = reader.Transit<kParseNonRecursiveFlag>(
|
||||
r,
|
||||
json,
|
||||
handler);
|
||||
EXPECT_FALSE(reader.HasParseError());
|
||||
EXPECT_EQ(Reader::NonRecursiveParsingArrayContentState, r);
|
||||
EXPECT_EQ(2, *reader.stack_.template Top<int>()); // element count
|
||||
|
||||
// false
|
||||
r = reader.Transit<kParseNonRecursiveFlag>(
|
||||
r,
|
||||
json,
|
||||
handler);
|
||||
EXPECT_FALSE(reader.HasParseError());
|
||||
EXPECT_EQ(Reader::NonRecursiveParsingArrayContentState, r);
|
||||
EXPECT_EQ(2, *reader.stack_.template Top<int>()); // element count
|
||||
|
||||
// ,
|
||||
r = reader.Transit<kParseNonRecursiveFlag>(
|
||||
r,
|
||||
json,
|
||||
handler);
|
||||
EXPECT_FALSE(reader.HasParseError());
|
||||
EXPECT_EQ(Reader::NonRecursiveParsingArrayContentState, r);
|
||||
EXPECT_EQ(3, *reader.stack_.template Top<int>()); // element count
|
||||
|
||||
// null
|
||||
r = reader.Transit<kParseNonRecursiveFlag>(
|
||||
r,
|
||||
json,
|
||||
handler);
|
||||
EXPECT_FALSE(reader.HasParseError());
|
||||
EXPECT_EQ(Reader::NonRecursiveParsingArrayContentState, r);
|
||||
EXPECT_EQ(3, *reader.stack_.template Top<int>()); // element count
|
||||
|
||||
// ,
|
||||
r = reader.Transit<kParseNonRecursiveFlag>(
|
||||
r,
|
||||
json,
|
||||
handler);
|
||||
EXPECT_FALSE(reader.HasParseError());
|
||||
EXPECT_EQ(Reader::NonRecursiveParsingArrayContentState, r);
|
||||
EXPECT_EQ(4, *reader.stack_.template Top<int>()); // element count
|
||||
|
||||
// "string"
|
||||
r = reader.Transit<kParseNonRecursiveFlag>(
|
||||
r,
|
||||
json,
|
||||
handler);
|
||||
EXPECT_FALSE(reader.HasParseError());
|
||||
EXPECT_EQ(Reader::NonRecursiveParsingArrayContentState, r);
|
||||
EXPECT_EQ(4, *reader.stack_.template Top<int>()); // element count
|
||||
|
||||
// ,
|
||||
r = reader.Transit<kParseNonRecursiveFlag>(
|
||||
r,
|
||||
json,
|
||||
handler);
|
||||
EXPECT_FALSE(reader.HasParseError());
|
||||
EXPECT_EQ(Reader::NonRecursiveParsingArrayContentState, r);
|
||||
EXPECT_EQ(5, *reader.stack_.template Top<int>()); // element count
|
||||
|
||||
// {
|
||||
r = reader.Transit<kParseNonRecursiveFlag>(
|
||||
r,
|
||||
json,
|
||||
handler);
|
||||
EXPECT_FALSE(reader.HasParseError());
|
||||
EXPECT_EQ(Reader::NonRecursiveParsingObjectInitialState, r);
|
||||
EXPECT_EQ(0, *reader.stack_.template Top<int>()); // member count
|
||||
|
||||
// "array":[
|
||||
r = reader.Transit<kParseNonRecursiveFlag>(
|
||||
r,
|
||||
json,
|
||||
handler);
|
||||
EXPECT_FALSE(reader.HasParseError());
|
||||
EXPECT_EQ(Reader::NonRecursiveParsingArrayInitialState, r);
|
||||
EXPECT_EQ(0, *reader.stack_.template Top<int>()); // element count
|
||||
|
||||
// 1
|
||||
r = reader.Transit<kParseNonRecursiveFlag>(
|
||||
r,
|
||||
json,
|
||||
handler);
|
||||
EXPECT_FALSE(reader.HasParseError());
|
||||
EXPECT_EQ(Reader::NonRecursiveParsingArrayContentState, r);
|
||||
EXPECT_EQ(0, *reader.stack_.template Top<int>()); // element count
|
||||
|
||||
// ]
|
||||
r = reader.Transit<kParseNonRecursiveFlag>(
|
||||
r,
|
||||
json,
|
||||
handler);
|
||||
EXPECT_FALSE(reader.HasParseError());
|
||||
EXPECT_EQ(Reader::NonRecursiveParsingObjectContentState, r);
|
||||
EXPECT_EQ(0, *reader.stack_.template Top<int>()); // member count
|
||||
|
||||
// }
|
||||
r = reader.Transit<kParseNonRecursiveFlag>(
|
||||
r,
|
||||
json,
|
||||
handler);
|
||||
EXPECT_FALSE(reader.HasParseError());
|
||||
EXPECT_EQ(Reader::NonRecursiveParsingArrayContentState, r);
|
||||
EXPECT_EQ(5, *reader.stack_.template Top<int>()); // element count
|
||||
|
||||
// ]
|
||||
r = reader.Transit<kParseNonRecursiveFlag>(
|
||||
r,
|
||||
json,
|
||||
handler);
|
||||
EXPECT_FALSE(reader.HasParseError());
|
||||
EXPECT_EQ(Reader::NonRecursiveParsingFinishState, r);
|
||||
}
|
||||
|
||||
struct CountHandler : BaseReaderHandler<> {
|
||||
void EndObject(SizeType memberCount) {
|
||||
MemberCount = memberCount;
|
||||
}
|
||||
|
||||
void EndArray(SizeType elementCount) {
|
||||
ElementCount = elementCount;
|
||||
}
|
||||
|
||||
SizeType MemberCount;
|
||||
SizeType ElementCount;
|
||||
};
|
||||
|
||||
TEST(Reader, NonRecursiveParsing_MemberCounting) {
|
||||
StringStream json("{\"array\": []}");
|
||||
Reader reader;
|
||||
CountHandler handler;
|
||||
|
||||
reader.NonRecursiveParse<kParseNonRecursiveFlag>(json, handler);
|
||||
|
||||
EXPECT_FALSE(reader.HasParseError());
|
||||
EXPECT_EQ(1, handler.MemberCount);
|
||||
}
|
||||
|
||||
TEST(Reader, NonRecursiveParsing_ElementCounting) {
|
||||
StringStream json("[{}]");
|
||||
Reader reader;
|
||||
CountHandler handler;
|
||||
|
||||
reader.NonRecursiveParse<kParseNonRecursiveFlag>(json, handler);
|
||||
|
||||
EXPECT_FALSE(reader.HasParseError());
|
||||
EXPECT_EQ(1, handler.ElementCount);
|
||||
}
|
||||
|
||||
#ifdef __GNUC__
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user