diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h index b1474db..9ffc68f 100644 --- a/include/rapidjson/reader.h +++ b/include/rapidjson/reader.h @@ -679,7 +679,14 @@ private: *stack_.template Push() = c; ++length_; } + + RAPIDJSON_FORCEINLINE void* Push(size_t count) { + length_ += count; + return stack_.template Push(count); + } + size_t Length() const { return length_; } + Ch* Pop() { return stack_.template Pop(length_); } @@ -740,6 +747,10 @@ private: is.Take(); // Skip '\"' for (;;) { + // Scan and copy string before "\\\"" or < 0x20. This is an optional optimzation. + if (!(parseFlags & kParseValidateEncodingFlag)) + ScanCopyUnescapedString(is, os); + Ch c = is.Peek(); if (RAPIDJSON_UNLIKELY(c == '\\')) { // Escape is.Take(); @@ -769,10 +780,12 @@ private: os.Put('\0'); // null-terminate the string return; } - else if (RAPIDJSON_UNLIKELY(c == '\0')) - RAPIDJSON_PARSE_ERROR(kParseErrorStringMissQuotationMark, is.Tell() - 1); - else if (RAPIDJSON_UNLIKELY(static_cast(c) < 0x20)) // RFC 4627: unescaped = %x20-21 / %x23-5B / %x5D-10FFFF - RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, is.Tell() - 1); + else if (RAPIDJSON_UNLIKELY(static_cast(c) < 0x20)) { // RFC 4627: unescaped = %x20-21 / %x23-5B / %x5D-10FFFF + if (c == '\0') + RAPIDJSON_PARSE_ERROR(kParseErrorStringMissQuotationMark, is.Tell() - 1); + else + RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, is.Tell() - 1); + } else { if (RAPIDJSON_UNLIKELY((parseFlags & kParseValidateEncodingFlag ? !Transcoder::Validate(is, os) : @@ -782,6 +795,59 @@ private: } } + template + static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InputStream&, OutputStream&) { + // Do nothing for generic version + } + +#if 0 //defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) + static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(StringStream& is, StackStream& os) { + const char* p = is.src_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(*p < 0x20)) { + is.src_ = p; + return; + } + else + os.Put(*p++); + + // The rest of string using SIMD + static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; + static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; + static const char space[16] = { 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20 }; + const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); + const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); + const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); + + for (;; p += 16) { + const __m128i s = _mm_load_si128(reinterpret_cast(p)); + __m128i x = _mm_cmpeq_epi8(s, dq); + x = _mm_or_si128(x, _mm_cmpeq_epi8(s, bs)); + x = _mm_or_si128(x, _mm_cmplt_epi8(s, sp)); + unsigned short r = static_cast(_mm_movemask_epi8(x)); + if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped + size_t length; + #ifdef _MSC_VER // Find the index of first escaped + unsigned long offset; + _BitScanForward(&offset, r); + length = offset; + #else + length = static_cast(__builtin_ffs(r) - 1); + #endif + memcpy(os.Push(length), p, length); + p += length; + break; + } + _mm_storeu_si128(reinterpret_cast<__m128i *>(os.Push(16)), s); + } + + is.src_ = p; + } +#endif + template class NumberStream; @@ -887,7 +953,7 @@ private: while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { if (RAPIDJSON_UNLIKELY(i64 >= RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC))) // 2^63 = 9223372036854775808 if (RAPIDJSON_LIKELY(i64 != RAPIDJSON_UINT64_C2(0x0CCCCCCC, 0xCCCCCCCC) || s.Peek() > '8')) { - d = i64; + d = static_cast(i64); useDouble = true; break; } @@ -898,7 +964,7 @@ private: while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { if (RAPIDJSON_UNLIKELY(i64 >= RAPIDJSON_UINT64_C2(0x19999999, 0x99999999))) // 2^64 - 1 = 18446744073709551615 if (RAPIDJSON_LIKELY(i64 != RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || s.Peek() > '5')) { - d = i64; + d = static_cast(i64); useDouble = true; break; } @@ -969,7 +1035,7 @@ private: int exp = 0; if (s.Peek() == 'e' || s.Peek() == 'E') { if (!useDouble) { - d = use64bit ? i64 : i; + d = static_cast(use64bit ? i64 : i); useDouble = true; } s.Take(); diff --git a/test/perftest/rapidjsontest.cpp b/test/perftest/rapidjsontest.cpp index b9ac395..4693755 100644 --- a/test/perftest/rapidjsontest.cpp +++ b/test/perftest/rapidjsontest.cpp @@ -95,6 +95,26 @@ TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler)) { } } +#define TEST_TYPED(index, Name)\ +TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler_##Name)) {\ + for (size_t i = 0; i < kTrialCount * 10; i++) {\ + StringStream s(types_[index]);\ + BaseReaderHandler<> h;\ + Reader reader;\ + EXPECT_TRUE(reader.Parse(s, h));\ + }\ +} + +TEST_TYPED(0, Booleans) +TEST_TYPED(1, Floats) +TEST_TYPED(2, Guids) +TEST_TYPED(3, Integers) +TEST_TYPED(4, Mixed) +TEST_TYPED(5, Nulls) +TEST_TYPED(6, Paragraphs) + +#undef TEST_TYPED + TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler_FullPrecision)) { for (size_t i = 0; i < kTrialCount; i++) { StringStream s(json_); @@ -293,7 +313,7 @@ TEST_F(RapidJson, Writer_StringBuffer_##Name) {\ const char* str = s.GetString();\ (void)str;\ }\ -}\ +} TEST_TYPED(0, Booleans) TEST_TYPED(1, Floats) @@ -303,6 +323,8 @@ TEST_TYPED(4, Mixed) TEST_TYPED(5, Nulls) TEST_TYPED(6, Paragraphs) +#undef TEST_TYPED + TEST_F(RapidJson, PrettyWriter_StringBuffer) { for (size_t i = 0; i < kTrialCount; i++) { StringBuffer s(0, 2048 * 1024); diff --git a/test/unittest/readertest.cpp b/test/unittest/readertest.cpp index f2a0be1..6ef1351 100644 --- a/test/unittest/readertest.cpp +++ b/test/unittest/readertest.cpp @@ -14,6 +14,14 @@ #include "unittest.h" +// __SSE2__ and __SSE4_2__ are recognized by gcc, clang, and the Intel compiler. +// We use -march=native with gmake to enable -msse2 and -msse4.2, if supported. +#if defined(__SSE4_2__) +# define RAPIDJSON_SSE42 +#elif defined(__SSE2__) +# define RAPIDJSON_SSE2 +#endif + #include "rapidjson/reader.h" #include "rapidjson/internal/dtoa.h" #include "rapidjson/internal/itoa.h"