gvsafronov/rapidjson
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use lookup table for Tokenize()

Browse Source
This commit is contained in:
Milo Yip 2014-07-26 21:23:37 +08:00
parent 01126defd2
commit 89865cb919
1 changed files with 40 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
include/rapidjson/reader.h
View File

@ -829,44 +829,52 @@ private:
}; };
// Tokens // Tokens
enum IterativeParsingToken { enum Token {
IterativeParsingLeftBracketToken = 0, LeftBracketToken = 0,
IterativeParsingRightBracketToken, RightBracketToken,
IterativeParsingLeftCurlyBracketToken, LeftCurlyBracketToken,
IterativeParsingRightCurlyBracketToken, RightCurlyBracketToken,
IterativeParsingCommaToken, CommaToken,
IterativeParsingColonToken, ColonToken,
IterativeParsingStringToken, StringToken,
IterativeParsingFalseToken, FalseToken,
IterativeParsingTrueToken, TrueToken,
IterativeParsingNullToken, NullToken,
IterativeParsingNumberToken, NumberToken,
cIterativeParsingTokenCount kTokenCount
}; };
IterativeParsingToken Tokenize(Ch c) { RAPIDJSON_FORCEINLINE Token Tokenize(Ch c) {
switch (c) { #define N NumberToken
case '[': return IterativeParsingLeftBracketToken; #define N16 N,N,N,N,N,N,N,N,N,N,N,N,N,N,N,N
case ']': return IterativeParsingRightBracketToken; // Maps from ASCII to Token
case '{': return IterativeParsingLeftCurlyBracketToken; static const unsigned char tokenMap[256] = {
case '}': return IterativeParsingRightCurlyBracketToken; N16, // 00~0F
case ',': return IterativeParsingCommaToken; N16, // 10~1F
case ':': return IterativeParsingColonToken; N, N, StringToken, N, N, N, N, N, N, N, N, N, CommaToken, N, N, N, // 20~2F
case '"': return IterativeParsingStringToken; N, N, N, N, N, N, N, N, N, N, ColonToken, N, N, N, N, N, // 30~3F
case 'f': return IterativeParsingFalseToken; N16, // 40~4F
case 't': return IterativeParsingTrueToken; N, N, N, N, N, N, N, N, N, N, N, LeftBracketToken, N, RightBracketToken, N, N, // 50~5F
case 'n': return IterativeParsingNullToken; N, N, N, N, N, N, FalseToken, N, N, N, N, N, N, N, NullToken, N, // 60~6F
default: return IterativeParsingNumberToken; N, N, N, N, TrueToken, N, N, N, N, N, N, LeftCurlyBracketToken, N, RightCurlyBracketToken, N, N, // 70~7F
} N16, N16, N16, N16, N16, N16, N16, N16 // 80~FF
};
#undef N
#undef N16
if (sizeof(Ch) == 1 || unsigned(c) < 256)
return (Token)tokenMap[(unsigned char)c];
else
return NumberToken;
} }
IterativeParsingState Predict(IterativeParsingState state, IterativeParsingToken token) { IterativeParsingState Predict(IterativeParsingState state, Token token) {
// current state x one lookahead token -> new state // current state x one lookahead token -> new state
static const char G[cIterativeParsingStateCount][cIterativeParsingTokenCount] = { static const char G[cIterativeParsingStateCount][kTokenCount] = {
// Start // Start
{ {
IterativeParsingArrayInitialState, // Left bracket IterativeParsingArrayInitialState, // Left bracket
@ -1025,7 +1033,7 @@ private:
// Make an advance in the token stream and state based on the candidate destination state which was returned by Transit(). // Make an advance in the token stream and state based on the candidate destination state which was returned by Transit().
// May return a new state on state pop. // May return a new state on state pop.
template <unsigned parseFlags, typename InputStream, typename Handler> template <unsigned parseFlags, typename InputStream, typename Handler>
IterativeParsingState Transit(IterativeParsingState src, IterativeParsingToken token, IterativeParsingState dst, InputStream& is, Handler& handler) { IterativeParsingState Transit(IterativeParsingState src, Token token, IterativeParsingState dst, InputStream& is, Handler& handler) {
//int c = 0; //int c = 0;
//IterativeParsingState n; //IterativeParsingState n;
//bool hr; //bool hr;
@ -1081,7 +1089,7 @@ private:
return dst; return dst;
case IterativeParsingKeyValueDelimiterState: case IterativeParsingKeyValueDelimiterState:
if (token == IterativeParsingColonToken) { if (token == ColonToken) {
is.Take(); is.Take();
return dst; return dst;
} }
@ -1207,7 +1215,7 @@ private:
SkipWhitespace(is); SkipWhitespace(is);
while (is.Peek() != '\0') { while (is.Peek() != '\0') {
IterativeParsingToken t = Tokenize(is.Peek()); Token t = Tokenize(is.Peek());
IterativeParsingState n = Predict(state, t); IterativeParsingState n = Predict(state, t);
IterativeParsingState d = Transit<parseFlags>(state, t, n, is, handler); IterativeParsingState d = Transit<parseFlags>(state, t, n, is, handler);