From ba2b751b4df386f6685bfb9a672560aeba78d956 Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Sun, 28 Feb 2016 17:33:49 +0100 Subject: [PATCH 01/26] Added kParseNumbersAsStringsFlag --- include/rapidjson/reader.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h index 226c9d1..c06a4fc 100644 --- a/include/rapidjson/reader.h +++ b/include/rapidjson/reader.h @@ -148,6 +148,7 @@ enum ParseFlag { kParseStopWhenDoneFlag = 8, //!< After parsing a complete JSON root from stream, stop further processing the rest of stream. When this flag is used, parser will not generate kParseErrorDocumentRootNotSingular error. kParseFullPrecisionFlag = 16, //!< Parse number in full precision (but slower). kParseCommentsFlag = 32, //!< Allow one-line (//) and multi-line (/**/) comments. + kParseNumbersAsStringsFlag = 64, //!< Parse all numbers (ints/doubles) as strings. kParseDefaultFlags = RAPIDJSON_PARSE_DEFAULT_FLAGS //!< Default parse flags. Can be customized by defining RAPIDJSON_PARSE_DEFAULT_FLAGS }; From 4abcfd1e284c44b1461188599ba5e568baa7b9e0 Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Sun, 28 Feb 2016 17:58:01 +0100 Subject: [PATCH 02/26] Added Number() to rapidjson::Handler --- include/rapidjson/reader.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h index c06a4fc..6ee1f43 100644 --- a/include/rapidjson/reader.h +++ b/include/rapidjson/reader.h @@ -170,6 +170,8 @@ concept Handler { bool Int64(int64_t i); bool Uint64(uint64_t i); bool Double(double d); + /// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length) + bool Number(const Ch* str, SizeType length, bool copy); bool String(const Ch* str, SizeType length, bool copy); bool StartObject(); bool Key(const Ch* str, SizeType length, bool copy); @@ -200,6 +202,8 @@ struct BaseReaderHandler { bool Int64(int64_t) { return static_cast(*this).Default(); } bool Uint64(uint64_t) { return static_cast(*this).Default(); } bool Double(double) { return static_cast(*this).Default(); } + /// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length) + bool Number(const Ch*, SizeType, bool) { return static_cast(*this).Default(); } bool String(const Ch*, SizeType, bool) { return static_cast(*this).Default(); } bool StartObject() { return static_cast(*this).Default(); } bool Key(const Ch* str, SizeType len, bool copy) { return static_cast(*this).String(str, len, copy); } From d2d5f6f91984a533107a52cdad40e34c9b86906d Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Sun, 28 Feb 2016 17:58:34 +0100 Subject: [PATCH 03/26] ParseNumber() handles kParseNumbersAsStringsFlag --- include/rapidjson/reader.h | 57 +++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h index 6ee1f43..5f8b1a9 100644 --- a/include/rapidjson/reader.h +++ b/include/rapidjson/reader.h @@ -1097,6 +1097,8 @@ private: NumberStream s(*this, copy.s); size_t startOffset = s.Tell(); + typename InputStream::Ch *head = is.PutBegin(); + // Parse minus bool minus = Consume(s, '-'); @@ -1268,31 +1270,42 @@ private: // Finish parsing, call event according to the type of number. bool cont = true; - size_t length = s.Length(); - const char* decimal = s.Pop(); // Pop stack no matter if it will be used or not. - if (useDouble) { - int p = exp + expFrac; - if (parseFlags & kParseFullPrecisionFlag) - d = internal::StrtodFullPrecision(d, p, decimal, length, decimalPosition, exp); - else - d = internal::StrtodNormalPrecision(d, p); + if (parseFlags & kParseNumbersAsStringsFlag) + { + s.Pop(); // Pop stack no matter if it will be used or not. + const size_t length = s.Tell() - startOffset; - cont = handler.Double(minus ? -d : d); + cont = handler.Number(head, length, (parseFlags & kParseInsituFlag) ? false : true); } - else { - if (use64bit) { - if (minus) - cont = handler.Int64(static_cast(~i64 + 1)); - else - cont = handler.Uint64(i64); - } - else { - if (minus) - cont = handler.Int(static_cast(~i + 1)); - else - cont = handler.Uint(i); - } + else + { + size_t length = s.Length(); + const char* decimal = s.Pop(); // Pop stack no matter if it will be used or not. + + if (useDouble) { + int p = exp + expFrac; + if (parseFlags & kParseFullPrecisionFlag) + d = internal::StrtodFullPrecision(d, p, decimal, length, decimalPosition, exp); + else + d = internal::StrtodNormalPrecision(d, p); + + cont = handler.Double(minus ? -d : d); + } + else { + if (use64bit) { + if (minus) + cont = handler.Int64(static_cast(~i64 + 1)); + else + cont = handler.Uint64(i64); + } + else { + if (minus) + cont = handler.Int(static_cast(~i + 1)); + else + cont = handler.Uint(i); + } + } } if (RAPIDJSON_UNLIKELY(!cont)) RAPIDJSON_PARSE_ERROR(kParseErrorTermination, startOffset); From eeb13bdb4caf2fff4c4bf054be8ba47e6aaf395b Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Sun, 28 Feb 2016 18:12:57 +0100 Subject: [PATCH 04/26] Added Writer::Number() --- include/rapidjson/writer.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/include/rapidjson/writer.h b/include/rapidjson/writer.h index 4bda076..7df5696 100644 --- a/include/rapidjson/writer.h +++ b/include/rapidjson/writer.h @@ -171,6 +171,12 @@ public: */ bool Double(double d) { Prefix(kNumberType); return WriteDouble(d); } + bool Number(const Ch* str, SizeType length, bool copy = false) { + (void)copy; + Prefix(kNumberType); + return WriteString(str, length); + } + bool String(const Ch* str, SizeType length, bool copy = false) { (void)copy; Prefix(kStringType); From 334461b421364baf46732d3915b613e568a22b51 Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Sun, 28 Feb 2016 18:28:19 +0100 Subject: [PATCH 05/26] Added Hasher::Number() --- include/rapidjson/schema.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/rapidjson/schema.h b/include/rapidjson/schema.h index 9b033af..7a362d5 100644 --- a/include/rapidjson/schema.h +++ b/include/rapidjson/schema.h @@ -183,6 +183,11 @@ public: return WriteNumber(n); } + bool Number(const Ch* str, SizeType len, bool) { + WriteBuffer(kNumberType, str, len * sizeof(Ch)); + return true; + } + bool String(const Ch* str, SizeType len, bool) { WriteBuffer(kStringType, str, len * sizeof(Ch)); return true; From 4f94ec9b0bc63f3de9d1f59b16a9cdf8e7532832 Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Sun, 28 Feb 2016 18:38:06 +0100 Subject: [PATCH 06/26] Added GenericDocument::Number() --- include/rapidjson/document.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/include/rapidjson/document.h b/include/rapidjson/document.h index be09be4..7b46873 100644 --- a/include/rapidjson/document.h +++ b/include/rapidjson/document.h @@ -2325,6 +2325,14 @@ public: bool Uint64(uint64_t i) { new (stack_.template Push()) ValueType(i); return true; } bool Double(double d) { new (stack_.template Push()) ValueType(d); return true; } + bool Number(const Ch* str, SizeType length, bool copy) { + if (copy) + new (stack_.template Push()) ValueType(str, length, GetAllocator()); + else + new (stack_.template Push()) ValueType(str, length); + return true; + } + bool String(const Ch* str, SizeType length, bool copy) { if (copy) new (stack_.template Push()) ValueType(str, length, GetAllocator()); From 2bbfe0d8a8df25f2a61a19544360e0a5482e673d Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Sun, 28 Feb 2016 18:50:04 +0100 Subject: [PATCH 07/26] Number() -> RawNumber() to avoid name clashes with the union Number --- include/rapidjson/document.h | 2 +- include/rapidjson/reader.h | 6 +++--- include/rapidjson/schema.h | 2 +- include/rapidjson/writer.h | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/rapidjson/document.h b/include/rapidjson/document.h index 7b46873..18c2527 100644 --- a/include/rapidjson/document.h +++ b/include/rapidjson/document.h @@ -2325,7 +2325,7 @@ public: bool Uint64(uint64_t i) { new (stack_.template Push()) ValueType(i); return true; } bool Double(double d) { new (stack_.template Push()) ValueType(d); return true; } - bool Number(const Ch* str, SizeType length, bool copy) { + bool RawNumber(const Ch* str, SizeType length, bool copy) { if (copy) new (stack_.template Push()) ValueType(str, length, GetAllocator()); else diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h index 5f8b1a9..7bec7cb 100644 --- a/include/rapidjson/reader.h +++ b/include/rapidjson/reader.h @@ -171,7 +171,7 @@ concept Handler { bool Uint64(uint64_t i); bool Double(double d); /// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length) - bool Number(const Ch* str, SizeType length, bool copy); + bool RawNumber(const Ch* str, SizeType length, bool copy); bool String(const Ch* str, SizeType length, bool copy); bool StartObject(); bool Key(const Ch* str, SizeType length, bool copy); @@ -203,7 +203,7 @@ struct BaseReaderHandler { bool Uint64(uint64_t) { return static_cast(*this).Default(); } bool Double(double) { return static_cast(*this).Default(); } /// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length) - bool Number(const Ch*, SizeType, bool) { return static_cast(*this).Default(); } + bool RawNumber(const Ch*, SizeType, bool) { return static_cast(*this).Default(); } bool String(const Ch*, SizeType, bool) { return static_cast(*this).Default(); } bool StartObject() { return static_cast(*this).Default(); } bool Key(const Ch* str, SizeType len, bool copy) { return static_cast(*this).String(str, len, copy); } @@ -1276,7 +1276,7 @@ private: s.Pop(); // Pop stack no matter if it will be used or not. const size_t length = s.Tell() - startOffset; - cont = handler.Number(head, length, (parseFlags & kParseInsituFlag) ? false : true); + cont = handler.RawNumber(head, length, (parseFlags & kParseInsituFlag) ? false : true); } else { diff --git a/include/rapidjson/schema.h b/include/rapidjson/schema.h index 7a362d5..752767e 100644 --- a/include/rapidjson/schema.h +++ b/include/rapidjson/schema.h @@ -183,7 +183,7 @@ public: return WriteNumber(n); } - bool Number(const Ch* str, SizeType len, bool) { + bool RawNumber(const Ch* str, SizeType len, bool) { WriteBuffer(kNumberType, str, len * sizeof(Ch)); return true; } diff --git a/include/rapidjson/writer.h b/include/rapidjson/writer.h index 7df5696..3957422 100644 --- a/include/rapidjson/writer.h +++ b/include/rapidjson/writer.h @@ -171,7 +171,7 @@ public: */ bool Double(double d) { Prefix(kNumberType); return WriteDouble(d); } - bool Number(const Ch* str, SizeType length, bool copy = false) { + bool RawNumber(const Ch* str, SizeType length, bool copy = false) { (void)copy; Prefix(kNumberType); return WriteString(str, length); From fb5c464221aaf676c61656cff7264b454d7ead68 Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Sun, 28 Feb 2016 19:35:21 +0100 Subject: [PATCH 08/26] Added PrettyWriter::RawNumber() --- include/rapidjson/prettywriter.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/include/rapidjson/prettywriter.h b/include/rapidjson/prettywriter.h index 4f8eba9..490db55 100644 --- a/include/rapidjson/prettywriter.h +++ b/include/rapidjson/prettywriter.h @@ -74,6 +74,12 @@ public: bool Uint64(uint64_t u64) { PrettyPrefix(kNumberType); return Base::WriteUint64(u64); } bool Double(double d) { PrettyPrefix(kNumberType); return Base::WriteDouble(d); } + bool RawNumber(const Ch* str, SizeType length, bool copy = false) { + (void)copy; + PrettyPrefix(kNumberType); + return Base::WriteString(str, length); + } + bool String(const Ch* str, SizeType length, bool copy = false) { (void)copy; PrettyPrefix(kStringType); From f76174422d0761a1640e6751b2b28faf6e7e0fce Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Sun, 28 Feb 2016 19:20:31 +0100 Subject: [PATCH 09/26] Fixed Ch type --- include/rapidjson/reader.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h index 7bec7cb..7019c41 100644 --- a/include/rapidjson/reader.h +++ b/include/rapidjson/reader.h @@ -1271,7 +1271,7 @@ private: // Finish parsing, call event according to the type of number. bool cont = true; - if (parseFlags & kParseNumbersAsStringsFlag) + if ((parseFlags & kParseNumbersAsStringsFlag) && (parseFlags & kParseInsituFlag)) { s.Pop(); // Pop stack no matter if it will be used or not. const size_t length = s.Tell() - startOffset; From 6143f97b4589742ceed239978806864115e5442f Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Mon, 29 Feb 2016 10:52:36 +0100 Subject: [PATCH 10/26] BaseReaderHandler::RawNumber() calls String() by default --- include/rapidjson/reader.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h index 7019c41..eedac84 100644 --- a/include/rapidjson/reader.h +++ b/include/rapidjson/reader.h @@ -203,7 +203,7 @@ struct BaseReaderHandler { bool Uint64(uint64_t) { return static_cast(*this).Default(); } bool Double(double) { return static_cast(*this).Default(); } /// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length) - bool RawNumber(const Ch*, SizeType, bool) { return static_cast(*this).Default(); } + bool RawNumber(const Ch* str, SizeType len, bool copy) { return static_cast(*this).String(str, len, copy); } bool String(const Ch*, SizeType, bool) { return static_cast(*this).Default(); } bool StartObject() { return static_cast(*this).Default(); } bool Key(const Ch* str, SizeType len, bool copy) { return static_cast(*this).String(str, len, copy); } From a214bdff4d5336c9871b2c53456d4bca9f20feee Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Mon, 29 Feb 2016 10:54:55 +0100 Subject: [PATCH 11/26] Removed unnecessary kParseInsituFlag check in ParseNumber() --- include/rapidjson/reader.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h index eedac84..d787085 100644 --- a/include/rapidjson/reader.h +++ b/include/rapidjson/reader.h @@ -1271,7 +1271,7 @@ private: // Finish parsing, call event according to the type of number. bool cont = true; - if ((parseFlags & kParseNumbersAsStringsFlag) && (parseFlags & kParseInsituFlag)) + if (parseFlags & kParseNumbersAsStringsFlag) { s.Pop(); // Pop stack no matter if it will be used or not. const size_t length = s.Tell() - startOffset; From 7d4891e243147ab523bb89489165a283b6872a5d Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Mon, 29 Feb 2016 16:11:31 +0100 Subject: [PATCH 12/26] Added NumberStream::Push() --- include/rapidjson/reader.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h index d787085..721d8cd 100644 --- a/include/rapidjson/reader.h +++ b/include/rapidjson/reader.h @@ -1058,6 +1058,8 @@ private: RAPIDJSON_FORCEINLINE Ch Peek() const { return is.Peek(); } RAPIDJSON_FORCEINLINE Ch TakePush() { return is.Take(); } RAPIDJSON_FORCEINLINE Ch Take() { return is.Take(); } + RAPIDJSON_FORCEINLINE void Push( char c ) {} + size_t Tell() { return is.Tell(); } size_t Length() { return 0; } const char* Pop() { return 0; } @@ -1080,6 +1082,10 @@ private: return Base::is.Take(); } + RAPIDJSON_FORCEINLINE void Push(char c) { + stackStream.Put(c); + } + size_t Length() { return stackStream.Length(); } const char* Pop() { From e23bc0d1a5ab65518d5df1924eb3f0dc3ffcd4e8 Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Mon, 29 Feb 2016 16:12:30 +0100 Subject: [PATCH 13/26] Fixed numbers-as-strings for in-situ streams --- include/rapidjson/reader.h | 40 +++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h index 721d8cd..8990441 100644 --- a/include/rapidjson/reader.h +++ b/include/rapidjson/reader.h @@ -1100,10 +1100,12 @@ private: template void ParseNumber(InputStream& is, Handler& handler) { internal::StreamLocalCopy copy(is); - NumberStream s(*this, copy.s); - size_t startOffset = s.Tell(); + NumberStream s(*this, copy.s); - typename InputStream::Ch *head = is.PutBegin(); + size_t startOffset = s.Tell(); // Parse minus bool minus = Consume(s, '-'); @@ -1189,6 +1191,9 @@ private: int expFrac = 0; size_t decimalPosition; if (Consume(s, '.')) { + if (((parseFlags & kParseNumbersAsStringsFlag) != 0) && ((parseFlags & kParseInsituFlag) == 0)) { + s.Push('.'); + } decimalPosition = s.Length(); if (RAPIDJSON_UNLIKELY(!(s.Peek() >= '0' && s.Peek() <= '9'))) @@ -1236,7 +1241,11 @@ private: // Parse exp = e [ minus / plus ] 1*DIGIT int exp = 0; if (Consume(s, 'e') || Consume(s, 'E')) { - if (!useDouble) { + if ( ((parseFlags & kParseNumbersAsStringsFlag) != 0) && ((parseFlags & kParseInsituFlag) == 0) ) { + s.Push( 'e' ); + } + + if (!useDouble) { d = static_cast(use64bit ? i64 : i); useDouble = true; } @@ -1277,15 +1286,24 @@ private: // Finish parsing, call event according to the type of number. bool cont = true; - if (parseFlags & kParseNumbersAsStringsFlag) - { - s.Pop(); // Pop stack no matter if it will be used or not. - const size_t length = s.Tell() - startOffset; + if (parseFlags & kParseNumbersAsStringsFlag) { + + if (parseFlags & kParseInsituFlag) { + s.Pop(); // Pop stack no matter if it will be used or not. + typename InputStream::Ch *head = is.PutBegin(); + const size_t length = s.Tell() - startOffset; + RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); + const typename TargetEncoding::Ch* const str = reinterpret_cast(head); + cont = handler.RawNumber(str, SizeType(length), false); + } + else { + const char* str = s.Pop(); + SizeType length = static_cast(s.Length()) - 1; + cont = handler.RawNumber(str, SizeType(length), true); + } - cont = handler.RawNumber(head, length, (parseFlags & kParseInsituFlag) ? false : true); } - else - { + else { size_t length = s.Length(); const char* decimal = s.Pop(); // Pop stack no matter if it will be used or not. From f83b2c09e86eb9880786d01a2399bc6ef1dbee3e Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Mon, 29 Feb 2016 16:31:54 +0100 Subject: [PATCH 14/26] =?UTF-8?q?Fixed=20warning=20unused=20parameter=20?= =?UTF-8?q?=E2=80=98c=E2=80=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- include/rapidjson/reader.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h index 8990441..c1ef26f 100644 --- a/include/rapidjson/reader.h +++ b/include/rapidjson/reader.h @@ -1058,7 +1058,7 @@ private: RAPIDJSON_FORCEINLINE Ch Peek() const { return is.Peek(); } RAPIDJSON_FORCEINLINE Ch TakePush() { return is.Take(); } RAPIDJSON_FORCEINLINE Ch Take() { return is.Take(); } - RAPIDJSON_FORCEINLINE void Push( char c ) {} + RAPIDJSON_FORCEINLINE void Push(char) {} size_t Tell() { return is.Tell(); } size_t Length() { return 0; } From 9b13eacdf16f398866cf8da91bbfe28287e8ba5a Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Mon, 29 Feb 2016 17:40:43 +0100 Subject: [PATCH 15/26] Add 0-character during in-situ numbers-as-strings parsing --- include/rapidjson/reader.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h index c1ef26f..352dbd8 100644 --- a/include/rapidjson/reader.h +++ b/include/rapidjson/reader.h @@ -1290,9 +1290,10 @@ private: if (parseFlags & kParseInsituFlag) { s.Pop(); // Pop stack no matter if it will be used or not. - typename InputStream::Ch *head = is.PutBegin(); + typename InputStream::Ch* head = is.PutBegin(); const size_t length = s.Tell() - startOffset; RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); + *(head + length) = '\0'; const typename TargetEncoding::Ch* const str = reinterpret_cast(head); cont = handler.RawNumber(str, SizeType(length), false); } From 43a2c5694ea58f92878710a17d680fe4d78b02a9 Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Mon, 29 Feb 2016 17:42:48 +0100 Subject: [PATCH 16/26] Play nice with different encodings --- include/rapidjson/reader.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h index 352dbd8..a4cb0af 100644 --- a/include/rapidjson/reader.h +++ b/include/rapidjson/reader.h @@ -1298,8 +1298,14 @@ private: cont = handler.RawNumber(str, SizeType(length), false); } else { - const char* str = s.Pop(); - SizeType length = static_cast(s.Length()) - 1; + StackStream stackStream(stack_); + SizeType numCharsToCopy = s.Length(); + while (numCharsToCopy--) { + Transcoder::Transcode(is, stackStream); + } + stackStream.Put('\0'); + const typename TargetEncoding::Ch* str = stackStream.Pop(); + const SizeType length = static_cast(stackStream.Length()) - 1; cont = handler.RawNumber(str, SizeType(length), true); } From 3dba370486091ef1e9b8b5c70ed037933168b73f Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Mon, 29 Feb 2016 17:51:32 +0100 Subject: [PATCH 17/26] Added IterativeParsingReaderHandler::RawNumber() --- include/rapidjson/reader.h | 6 +++--- test/unittest/readertest.cpp | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h index a4cb0af..e6c7471 100644 --- a/include/rapidjson/reader.h +++ b/include/rapidjson/reader.h @@ -1299,11 +1299,11 @@ private: } else { StackStream stackStream(stack_); - SizeType numCharsToCopy = s.Length(); + SizeType numCharsToCopy = s.Length(); while (numCharsToCopy--) { - Transcoder::Transcode(is, stackStream); + Transcoder::Transcode(is, stackStream); } - stackStream.Put('\0'); + stackStream.Put('\0'); const typename TargetEncoding::Ch* str = stackStream.Pop(); const SizeType length = static_cast(stackStream.Length()) - 1; cont = handler.RawNumber(str, SizeType(length), true); diff --git a/test/unittest/readertest.cpp b/test/unittest/readertest.cpp index 221fd2f..8690da6 100644 --- a/test/unittest/readertest.cpp +++ b/test/unittest/readertest.cpp @@ -1170,6 +1170,8 @@ struct IterativeParsingReaderHandler { bool Double(double) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_DOUBLE; return true; } + bool RawNumber(const Ch*, SizeType, bool) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_STRING; return true; } + bool String(const Ch*, SizeType, bool) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_STRING; return true; } bool StartObject() { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_STARTOBJECT; return true; } From 1ffb3359152f3205224248b57242679e9bf93397 Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Mon, 29 Feb 2016 18:05:05 +0100 Subject: [PATCH 18/26] Added RawNumber() to fix unit tests --- test/unittest/readertest.cpp | 30 ++++++++++++++++-------------- test/unittest/valuetest.cpp | 24 +++++++++++++----------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/test/unittest/readertest.cpp b/test/unittest/readertest.cpp index 8690da6..b99936b 100644 --- a/test/unittest/readertest.cpp +++ b/test/unittest/readertest.cpp @@ -1351,12 +1351,13 @@ struct TerminateHandler { bool Int64(int64_t) { return e != 4; } bool Uint64(uint64_t) { return e != 5; } bool Double(double) { return e != 6; } - bool String(const char*, SizeType, bool) { return e != 7; } - bool StartObject() { return e != 8; } - bool Key(const char*, SizeType, bool) { return e != 9; } - bool EndObject(SizeType) { return e != 10; } - bool StartArray() { return e != 11; } - bool EndArray(SizeType) { return e != 12; } + bool RawNumber(const char*, SizeType, bool) { return e != 7; } + bool String(const char*, SizeType, bool) { return e != 8; } + bool StartObject() { return e != 9; } + bool Key(const char*, SizeType, bool) { return e != 10; } + bool EndObject(SizeType) { return e != 11; } + bool StartArray() { return e != 12; } + bool EndArray(SizeType) { return e != 13; } }; #define TEST_TERMINATION(e, json)\ @@ -1377,14 +1378,15 @@ TEST(Reader, ParseTerminationByHandler) { TEST_TERMINATION(4, "[-1234567890123456789"); TEST_TERMINATION(5, "[1234567890123456789"); TEST_TERMINATION(6, "[0.5]"); - TEST_TERMINATION(7, "[\"a\""); - TEST_TERMINATION(8, "[{"); - TEST_TERMINATION(9, "[{\"a\""); - TEST_TERMINATION(10, "[{}"); - TEST_TERMINATION(10, "[{\"a\":1}"); // non-empty object - TEST_TERMINATION(11, "{\"a\":["); - TEST_TERMINATION(12, "{\"a\":[]"); - TEST_TERMINATION(12, "{\"a\":[1]"); // non-empty array + // RawNumber() is never called + TEST_TERMINATION(8, "[\"a\""); + TEST_TERMINATION(9, "[{"); + TEST_TERMINATION(10, "[{\"a\""); + TEST_TERMINATION(11, "[{}"); + TEST_TERMINATION(11, "[{\"a\":1}"); // non-empty object + TEST_TERMINATION(12, "{\"a\":["); + TEST_TERMINATION(13, "{\"a\":[]"); + TEST_TERMINATION(13, "{\"a\":[1]"); // non-empty array } TEST(Reader, ParseComments) { diff --git a/test/unittest/valuetest.cpp b/test/unittest/valuetest.cpp index af20aaf..4e0f47b 100644 --- a/test/unittest/valuetest.cpp +++ b/test/unittest/valuetest.cpp @@ -1644,12 +1644,13 @@ struct TerminateHandler { bool Int64(int64_t) { return e != 4; } bool Uint64(uint64_t) { return e != 5; } bool Double(double) { return e != 6; } - bool String(const char*, SizeType, bool) { return e != 7; } - bool StartObject() { return e != 8; } - bool Key(const char*, SizeType, bool) { return e != 9; } - bool EndObject(SizeType) { return e != 10; } - bool StartArray() { return e != 11; } - bool EndArray(SizeType) { return e != 12; } + bool RawNumber(const char*, SizeType, bool) { return e != 7; } + bool String(const char*, SizeType, bool) { return e != 8; } + bool StartObject() { return e != 9; } + bool Key(const char*, SizeType, bool) { return e != 10; } + bool EndObject(SizeType) { return e != 11; } + bool StartArray() { return e != 12; } + bool EndArray(SizeType) { return e != 13; } }; #define TEST_TERMINATION(e, json)\ @@ -1670,12 +1671,13 @@ TEST(Value, AcceptTerminationByHandler) { TEST_TERMINATION(4, "[-1234567890123456789]"); TEST_TERMINATION(5, "[9223372036854775808]"); TEST_TERMINATION(6, "[0.5]"); - TEST_TERMINATION(7, "[\"a\"]"); - TEST_TERMINATION(8, "[{}]"); - TEST_TERMINATION(9, "[{\"a\":1}]"); - TEST_TERMINATION(10, "[{}]"); - TEST_TERMINATION(11, "{\"a\":[]}"); + // RawNumber() is never called + TEST_TERMINATION(8, "[\"a\"]"); + TEST_TERMINATION(9, "[{}]"); + TEST_TERMINATION(10, "[{\"a\":1}]"); + TEST_TERMINATION(11, "[{}]"); TEST_TERMINATION(12, "{\"a\":[]}"); + TEST_TERMINATION(13, "{\"a\":[]}"); } struct ValueIntComparer { From 22d22145d2f8a3aa940e67fefdec4f6024f789fb Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Mon, 29 Feb 2016 18:05:24 +0100 Subject: [PATCH 19/26] Added GenericSchemaValidator::RawNumber() --- include/rapidjson/schema.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/rapidjson/schema.h b/include/rapidjson/schema.h index 752767e..26da8a6 100644 --- a/include/rapidjson/schema.h +++ b/include/rapidjson/schema.h @@ -1684,6 +1684,8 @@ RAPIDJSON_MULTILINEMACRO_END bool Int64(int64_t i) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int64, (CurrentContext(), i), (i)); } bool Uint64(uint64_t u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint64, (CurrentContext(), u), (u)); } bool Double(double d) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Double, (CurrentContext(), d), (d)); } + bool RawNumber(const Ch* str, SizeType length, bool copy) + { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } bool String(const Ch* str, SizeType length, bool copy) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } From 9748595960210037081e65568ff5518d1c669fbc Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Mon, 29 Feb 2016 18:25:14 +0100 Subject: [PATCH 20/26] Added MyHandler::RawNumber() --- example/simplereader/simplereader.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/example/simplereader/simplereader.cpp b/example/simplereader/simplereader.cpp index edbdb63..5aae8a1 100644 --- a/example/simplereader/simplereader.cpp +++ b/example/simplereader/simplereader.cpp @@ -12,6 +12,10 @@ struct MyHandler { bool Int64(int64_t i) { cout << "Int64(" << i << ")" << endl; return true; } bool Uint64(uint64_t u) { cout << "Uint64(" << u << ")" << endl; return true; } bool Double(double d) { cout << "Double(" << d << ")" << endl; return true; } + bool RawNumber(const char* str, SizeType length, bool copy) { + cout << "Number(" << str << ", " << length << ", " << boolalpha << copy << ")" << endl; + return true; + } bool String(const char* str, SizeType length, bool copy) { cout << "String(" << str << ", " << length << ", " << boolalpha << copy << ")" << endl; return true; From a9c79db5808867f6654c12ac2e6563856e2f2953 Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Mon, 29 Feb 2016 18:48:15 +0100 Subject: [PATCH 21/26] Added CapitalizeFilter::RawNumber() --- example/capitalize/capitalize.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/example/capitalize/capitalize.cpp b/example/capitalize/capitalize.cpp index adc32b5..7da37e9 100644 --- a/example/capitalize/capitalize.cpp +++ b/example/capitalize/capitalize.cpp @@ -24,7 +24,8 @@ struct CapitalizeFilter { bool Int64(int64_t i) { return out_.Int64(i); } bool Uint64(uint64_t u) { return out_.Uint64(u); } bool Double(double d) { return out_.Double(d); } - bool String(const char* str, SizeType length, bool) { + bool RawNumber(const char* str, SizeType length, bool copy) { return out_.RawNumber(str, length, copy); } + bool String(const char* str, SizeType length, bool) { buffer_.clear(); for (SizeType i = 0; i < length; i++) buffer_.push_back(static_cast(std::toupper(str[i]))); From 5642a81f248cc7e96c8c6898335186a695c41191 Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Mon, 29 Feb 2016 20:21:28 +0100 Subject: [PATCH 22/26] Added JsonxWriter::RawNumber() --- example/jsonx/jsonx.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/example/jsonx/jsonx.cpp b/example/jsonx/jsonx.cpp index c253ac0..1346b57 100644 --- a/example/jsonx/jsonx.cpp +++ b/example/jsonx/jsonx.cpp @@ -57,6 +57,13 @@ public: return WriteNumberElement(buffer, sprintf(buffer, "%.17g", d)); } + bool RawNumber(const char* str, SizeType length, bool) { + return + WriteStartElement("number") && + WriteEscapedText(str, length) && + WriteEndElement("number"); + } + bool String(const char* str, SizeType length, bool) { return WriteStartElement("string") && From bea3790a74798a47ffa3cdc3828ad1af2d8017b1 Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Wed, 2 Mar 2016 02:06:33 +0100 Subject: [PATCH 23/26] Don't insert terminating zero --- include/rapidjson/reader.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h index e6c7471..056da94 100644 --- a/include/rapidjson/reader.h +++ b/include/rapidjson/reader.h @@ -1293,7 +1293,7 @@ private: typename InputStream::Ch* head = is.PutBegin(); const size_t length = s.Tell() - startOffset; RAPIDJSON_ASSERT(length <= 0xFFFFFFFF); - *(head + length) = '\0'; +// *(head + length) = '\0'; const typename TargetEncoding::Ch* const str = reinterpret_cast(head); cont = handler.RawNumber(str, SizeType(length), false); } From 57eae5595ee31f1d3661d6f2475fd7d74acaca0c Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Tue, 1 Mar 2016 22:37:54 +0100 Subject: [PATCH 24/26] Added new unit test for kParseNumbersAsStringsFlag --- test/unittest/readertest.cpp | 40 ++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/test/unittest/readertest.cpp b/test/unittest/readertest.cpp index b99936b..34a1ed2 100644 --- a/test/unittest/readertest.cpp +++ b/test/unittest/readertest.cpp @@ -1512,6 +1512,46 @@ TEST(Reader, UnrecognizedComment) { EXPECT_EQ(kParseErrorUnspecificSyntaxError, reader.GetParseErrorCode()); } +struct NumbersAsStringsHandler { + bool Null() { return true; } + bool Bool(bool) { return true; } + bool Int(int) { return true; } + bool Uint(unsigned) { return true; } + bool Int64(int64_t) { return true; } + bool Uint64(uint64_t) { return true; } + bool Double(double) { return true; } + // 'str' is not null-terminated + bool RawNumber(const char* str, SizeType length, bool) { + EXPECT_TRUE(str != nullptr); + EXPECT_TRUE(strncmp(str, "3.1416", length) == 0); + return true; + } + bool String(const char*, SizeType, bool) { return true; } + bool StartObject() { return true; } + bool Key(const char*, SizeType, bool) { return true; } + bool EndObject(SizeType) { return true; } + bool StartArray() { return true; } + bool EndArray(SizeType) { return true; } +}; + +TEST(Reader, NumbersAsStrings) { + { + const char* json = "{ \"pi\": 3.1416 } "; + StringStream s(json); + NumbersAsStringsHandler h; + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + } + { + char* json = StrDup("{ \"pi\": 3.1416 } "); + InsituStringStream s(json); + NumbersAsStringsHandler h; + Reader reader; + EXPECT_TRUE(reader.Parse(s, h)); + free(json); + } +} + #ifdef __GNUC__ RAPIDJSON_DIAG_POP #endif From ae785ffb5240e8cfbc87e131d999828e59845cc6 Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Wed, 2 Mar 2016 02:21:38 +0100 Subject: [PATCH 25/26] Don't use nullptr --- test/unittest/readertest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unittest/readertest.cpp b/test/unittest/readertest.cpp index 34a1ed2..31ed81d 100644 --- a/test/unittest/readertest.cpp +++ b/test/unittest/readertest.cpp @@ -1522,7 +1522,7 @@ struct NumbersAsStringsHandler { bool Double(double) { return true; } // 'str' is not null-terminated bool RawNumber(const char* str, SizeType length, bool) { - EXPECT_TRUE(str != nullptr); + EXPECT_TRUE(str != 0); EXPECT_TRUE(strncmp(str, "3.1416", length) == 0); return true; } From b5966c329097327e757c6b5714430124396c82a5 Mon Sep 17 00:00:00 2001 From: Sergey Kosarevsky Date: Wed, 2 Mar 2016 03:07:53 +0100 Subject: [PATCH 26/26] Added missing static_cast --- include/rapidjson/reader.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h index 056da94..87d8afd 100644 --- a/include/rapidjson/reader.h +++ b/include/rapidjson/reader.h @@ -1299,7 +1299,7 @@ private: } else { StackStream stackStream(stack_); - SizeType numCharsToCopy = s.Length(); + SizeType numCharsToCopy = static_cast(s.Length()); while (numCharsToCopy--) { Transcoder::Transcode(is, stackStream); }