From bb0621108802d7839059cede4dee5daca7205e28 Mon Sep 17 00:00:00 2001 From: jack_perisich Date: Tue, 27 Jul 2021 19:50:51 -0400 Subject: [PATCH 1/9] Fix small errors in dtoa output for certain doubles --- include/rapidjson/internal/dtoa.h | 10 +++++++--- test/unittest/dtoatest.cpp | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/include/rapidjson/internal/dtoa.h b/include/rapidjson/internal/dtoa.h index 621402f..9f6ae3b 100644 --- a/include/rapidjson/internal/dtoa.h +++ b/include/rapidjson/internal/dtoa.h @@ -58,7 +58,11 @@ inline int CountDecimalDigit32(uint32_t n) { } inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buffer, int* len, int* K) { - static const uint32_t kPow10[] = { 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000 }; + static const uint64_t kPow10[] = { 1U, 10U, 100U, 1000U, 10000U, 100000U, 1000000U, 10000000U, 100000000U, + 1000000000U, 10000000000U, 100000000000U, 1000000000000U, + 10000000000000U, 100000000000000U, 1000000000000000U, + 10000000000000000U, 100000000000000000U, 1000000000000000000U, + 10000000000000000000U }; const DiyFp one(uint64_t(1) << -Mp.e, Mp.e); const DiyFp wp_w = Mp - W; uint32_t p1 = static_cast(Mp.f >> -one.e); @@ -86,7 +90,7 @@ inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buff uint64_t tmp = (static_cast(p1) << -one.e) + p2; if (tmp <= delta) { *K += kappa; - GrisuRound(buffer, *len, delta, tmp, static_cast(kPow10[kappa]) << -one.e, wp_w.f); + GrisuRound(buffer, *len, delta, tmp, kPow10[kappa] << -one.e, wp_w.f); return; } } @@ -103,7 +107,7 @@ inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buff if (p2 < delta) { *K += kappa; int index = -kappa; - GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * (index < 9 ? kPow10[index] : 0)); + GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * (index < 20 ? kPow10[index] : 0)); return; } } diff --git a/test/unittest/dtoatest.cpp b/test/unittest/dtoatest.cpp index 66576bd..3ec8982 100644 --- a/test/unittest/dtoatest.cpp +++ b/test/unittest/dtoatest.cpp @@ -38,6 +38,7 @@ TEST(dtoa, normal) { TEST_DTOA(0.123456789012, "0.123456789012"); TEST_DTOA(1234567.8, "1234567.8"); TEST_DTOA(-79.39773355813419, "-79.39773355813419"); + TEST_DTOA(-36.973846435546875, "-36.973846435546875"); TEST_DTOA(0.000001, "0.000001"); TEST_DTOA(0.0000001, "1e-7"); TEST_DTOA(1e30, "1e30"); From b952a592a480e551517468bbf4794061653b840c Mon Sep 17 00:00:00 2001 From: Kyrega Date: Tue, 10 Aug 2021 09:04:30 +0200 Subject: [PATCH 2/9] Fix RawNumber for longer char types --- include/rapidjson/internal/biginteger.h | 15 ++- include/rapidjson/internal/strtod.h | 15 ++- include/rapidjson/reader.h | 38 ++++--- test/unittest/readertest.cpp | 142 ++++++++++++++++++++++++ 4 files changed, 180 insertions(+), 30 deletions(-) diff --git a/include/rapidjson/internal/biginteger.h b/include/rapidjson/internal/biginteger.h index 1245578..7b14563 100644 --- a/include/rapidjson/internal/biginteger.h +++ b/include/rapidjson/internal/biginteger.h @@ -37,7 +37,8 @@ public: digits_[0] = u; } - BigInteger(const char* decimals, size_t length) : count_(1) { + template + BigInteger(const Ch* decimals, size_t length) : count_(1) { RAPIDJSON_ASSERT(length > 0); digits_[0] = 0; size_t i = 0; @@ -221,7 +222,8 @@ public: bool IsZero() const { return count_ == 1 && digits_[0] == 0; } private: - void AppendDecimal64(const char* begin, const char* end) { + template + void AppendDecimal64(const Ch* begin, const Ch* end) { uint64_t u = ParseUint64(begin, end); if (IsZero()) *this = u; @@ -236,11 +238,12 @@ private: digits_[count_++] = digit; } - static uint64_t ParseUint64(const char* begin, const char* end) { + template + static uint64_t ParseUint64(const Ch* begin, const Ch* end) { uint64_t r = 0; - for (const char* p = begin; p != end; ++p) { - RAPIDJSON_ASSERT(*p >= '0' && *p <= '9'); - r = r * 10u + static_cast(*p - '0'); + for (const Ch* p = begin; p != end; ++p) { + RAPIDJSON_ASSERT(*p >= Ch('0') && *p <= Ch('9')); + r = r * 10u + static_cast(*p - Ch('0')); } return r; } diff --git a/include/rapidjson/internal/strtod.h b/include/rapidjson/internal/strtod.h index d61a67a..55f0e38 100644 --- a/include/rapidjson/internal/strtod.h +++ b/include/rapidjson/internal/strtod.h @@ -128,17 +128,18 @@ inline bool StrtodFast(double d, int p, double* result) { } // Compute an approximation and see if it is within 1/2 ULP -inline bool StrtodDiyFp(const char* decimals, int dLen, int dExp, double* result) { +template +inline bool StrtodDiyFp(const Ch* decimals, int dLen, int dExp, double* result) { uint64_t significand = 0; int i = 0; // 2^64 - 1 = 18446744073709551615, 1844674407370955161 = 0x1999999999999999 for (; i < dLen; i++) { if (significand > RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || - (significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > '5')) + (significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > Ch('5'))) break; - significand = significand * 10u + static_cast(decimals[i] - '0'); + significand = significand * 10u + static_cast(decimals[i] - Ch('0')); } - if (i < dLen && decimals[i] >= '5') // Rounding + if (i < dLen && decimals[i] >= Ch('5')) // Rounding significand++; int remaining = dLen - i; @@ -205,7 +206,8 @@ inline bool StrtodDiyFp(const char* decimals, int dLen, int dExp, double* result return halfWay - static_cast(error) >= precisionBits || precisionBits >= halfWay + static_cast(error); } -inline double StrtodBigInteger(double approx, const char* decimals, int dLen, int dExp) { +template +inline double StrtodBigInteger(double approx, const Ch* decimals, int dLen, int dExp) { RAPIDJSON_ASSERT(dLen >= 0); const BigInteger dInt(decimals, static_cast(dLen)); Double a(approx); @@ -223,7 +225,8 @@ inline double StrtodBigInteger(double approx, const char* decimals, int dLen, in return a.NextPositiveDouble(); } -inline double StrtodFullPrecision(double d, int p, const char* decimals, size_t length, size_t decimalPosition, int exp) { +template +inline double StrtodFullPrecision(double d, int p, const Ch* decimals, size_t length, size_t decimalPosition, int exp) { RAPIDJSON_ASSERT(d >= 0.0); RAPIDJSON_ASSERT(length >= 1); diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h index 09ace4e..8f7e533 100644 --- a/include/rapidjson/reader.h +++ b/include/rapidjson/reader.h @@ -1404,11 +1404,11 @@ private: } #endif // RAPIDJSON_NEON - template + template class NumberStream; - template - class NumberStream { + template + class NumberStream { public: typedef typename InputStream::Ch Ch; @@ -1421,7 +1421,7 @@ private: size_t Tell() { return is.Tell(); } size_t Length() { return 0; } - const char* Pop() { return 0; } + const StackCharacter* Pop() { return 0; } protected: NumberStream& operator=(const NumberStream&); @@ -1429,35 +1429,35 @@ private: InputStream& is; }; - template - class NumberStream : public NumberStream { - typedef NumberStream Base; + template + class NumberStream : public NumberStream { + typedef NumberStream Base; public: NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is), stackStream(reader.stack_) {} RAPIDJSON_FORCEINLINE Ch TakePush() { - stackStream.Put(static_cast(Base::is.Peek())); + stackStream.Put(static_cast(Base::is.Peek())); return Base::is.Take(); } - RAPIDJSON_FORCEINLINE void Push(char c) { + RAPIDJSON_FORCEINLINE void Push(StackCharacter c) { stackStream.Put(c); } size_t Length() { return stackStream.Length(); } - const char* Pop() { + const StackCharacter* Pop() { stackStream.Put('\0'); return stackStream.Pop(); } private: - StackStream stackStream; + StackStream stackStream; }; - template - class NumberStream : public NumberStream { - typedef NumberStream Base; + template + class NumberStream : public NumberStream { + typedef NumberStream Base; public: NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is) {} @@ -1466,8 +1466,10 @@ private: template void ParseNumber(InputStream& is, Handler& handler) { + typedef std::conditional<(parseFlags & kParseNumbersAsStringsFlag) != 0, typename TargetEncoding::Ch, char>::type NumberCharacter; + internal::StreamLocalCopy copy(is); - NumberStream(s.Length()); - StringStream srcStream(s.Pop()); + GenericStringStream> srcStream(s.Pop()); StackStream dstStream(stack_); while (numCharsToCopy--) { - Transcoder, TargetEncoding>::Transcode(srcStream, dstStream); + Transcoder, TargetEncoding>::Transcode(srcStream, dstStream); } dstStream.Put('\0'); const typename TargetEncoding::Ch* str = dstStream.Pop(); @@ -1705,7 +1707,7 @@ private: } else { size_t length = s.Length(); - const char* decimal = s.Pop(); // Pop stack no matter if it will be used or not. + const NumberCharacter* decimal = s.Pop(); // Pop stack no matter if it will be used or not. if (useDouble) { int p = exp + expFrac; diff --git a/test/unittest/readertest.cpp b/test/unittest/readertest.cpp index d3fcdef..995d6db 100644 --- a/test/unittest/readertest.cpp +++ b/test/unittest/readertest.cpp @@ -1392,6 +1392,36 @@ private: std::istream& is_; }; +class WIStreamWrapper { +public: + typedef wchar_t Ch; + + WIStreamWrapper(std::wistream& is) : is_(is) {} + + Ch Peek() const { + int c = is_.peek(); + return c == std::char_traits::eof() ? Ch('\0') : static_cast(c); + } + + Ch Take() { + int c = is_.get(); + return c == std::char_traits::eof() ? Ch('\0') : static_cast(c); + } + + size_t Tell() const { return static_cast(is_.tellg()); } + + Ch* PutBegin() { assert(false); return 0; } + void Put(Ch) { assert(false); } + void Flush() { assert(false); } + size_t PutEnd(Ch*) { assert(false); return 0; } + +private: + WIStreamWrapper(const IStreamWrapper&); + WIStreamWrapper& operator=(const IStreamWrapper&); + + std::wistream& is_; +}; + TEST(Reader, Parse_IStreamWrapper_StringStream) { const char* json = "[1,2,3,4]"; @@ -1991,6 +2021,118 @@ TEST(Reader, NumbersAsStrings) { } } +struct NumbersAsStringsHandlerWChar_t { + bool Null() { return true; } + bool Bool(bool) { return true; } + bool Int(int) { return true; } + bool Uint(unsigned) { return true; } + bool Int64(int64_t) { return true; } + bool Uint64(uint64_t) { return true; } + bool Double(double) { return true; } + // 'str' is not null-terminated + bool RawNumber(const wchar_t* str, SizeType length, bool) { + EXPECT_TRUE(str != 0); + EXPECT_TRUE(expected_len_ == length); + EXPECT_TRUE(wcsncmp(str, expected_, length) == 0); + return true; + } + bool String(const wchar_t*, SizeType, bool) { return true; } + bool StartObject() { return true; } + bool Key(const wchar_t*, SizeType, bool) { return true; } + bool EndObject(SizeType) { return true; } + bool StartArray() { return true; } + bool EndArray(SizeType) { return true; } + + NumbersAsStringsHandlerWChar_t(const wchar_t* expected) + : expected_(expected) + , expected_len_(wcslen(expected)) {} + + const wchar_t* expected_; + size_t expected_len_; +}; + +TEST(Reader, NumbersAsStringsWChar_t) { + { + const wchar_t* json = L"{ \"pi\": 3.1416 } "; + GenericStringStream> s(json); + NumbersAsStringsHandlerWChar_t h(L"3.1416"); + GenericReader, UTF16<>> reader; + EXPECT_TRUE(reader.Parse(s, h)); + } + { + wchar_t* json = StrDup(L"{ \"pi\": 3.1416 } "); + GenericInsituStringStream> s(json); + NumbersAsStringsHandlerWChar_t h(L"3.1416"); + GenericReader, UTF16<>> reader; + EXPECT_TRUE(reader.Parse(s, h)); + free(json); + } + { + const wchar_t* json = L"{ \"gigabyte\": 1.0e9 } "; + GenericStringStream> s(json); + NumbersAsStringsHandlerWChar_t h(L"1.0e9"); + GenericReader, UTF16<>> reader; + EXPECT_TRUE(reader.Parse(s, h)); + } + { + wchar_t* json = StrDup(L"{ \"gigabyte\": 1.0e9 } "); + GenericInsituStringStream> s(json); + NumbersAsStringsHandlerWChar_t h(L"1.0e9"); + GenericReader, UTF16<>> reader; + EXPECT_TRUE(reader.Parse(s, h)); + free(json); + } + { + const wchar_t* json = L"{ \"pi\": 314.159e-2 } "; + GenericStringStream> s(json); + NumbersAsStringsHandlerWChar_t h(L"314.159e-2"); + GenericReader, UTF16<>> reader; + EXPECT_TRUE(reader.Parse(s, h)); + } + { + wchar_t* json = StrDup(L"{ \"gigabyte\": 314.159e-2 } "); + GenericInsituStringStream> s(json); + NumbersAsStringsHandlerWChar_t h(L"314.159e-2"); + GenericReader, UTF16<>> reader; + EXPECT_TRUE(reader.Parse(s, h)); + free(json); + } + { + const wchar_t* json = L"{ \"negative\": -1.54321 } "; + GenericStringStream> s(json); + NumbersAsStringsHandlerWChar_t h(L"-1.54321"); + GenericReader, UTF16<>> reader; + EXPECT_TRUE(reader.Parse(s, h)); + } + { + wchar_t* json = StrDup(L"{ \"negative\": -1.54321 } "); + GenericInsituStringStream> s(json); + NumbersAsStringsHandlerWChar_t h(L"-1.54321"); + GenericReader, UTF16<>> reader; + EXPECT_TRUE(reader.Parse(s, h)); + free(json); + } + { + const wchar_t* json = L"{ \"pi\": 314.159e-2 } "; + std::wstringstream ss(json); + WIStreamWrapper s(ss); + NumbersAsStringsHandlerWChar_t h(L"314.159e-2"); + GenericReader, UTF16<>> reader; + EXPECT_TRUE(reader.Parse(s, h)); + } + { + wchar_t n1e319[321]; // '1' followed by 319 '0' + n1e319[0] = L'1'; + for(int i = 1; i < 320; i++) + n1e319[i] = L'0'; + n1e319[320] = L'\0'; + GenericStringStream> s(n1e319); + NumbersAsStringsHandlerWChar_t h(n1e319); + GenericReader, UTF16<>> reader; + EXPECT_TRUE(reader.Parse(s, h)); + } +} + template void TestTrailingCommas() { { From 7fac34f7bbe451b10ab70c08cf6820387f7a1d92 Mon Sep 17 00:00:00 2001 From: Kyrega Date: Tue, 10 Aug 2021 09:33:17 +0200 Subject: [PATCH 3/9] Added typename --- include/rapidjson/reader.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h index 8f7e533..a4e5b3b 100644 --- a/include/rapidjson/reader.h +++ b/include/rapidjson/reader.h @@ -1466,7 +1466,7 @@ private: template void ParseNumber(InputStream& is, Handler& handler) { - typedef std::conditional<(parseFlags & kParseNumbersAsStringsFlag) != 0, typename TargetEncoding::Ch, char>::type NumberCharacter; + typedef typename std::conditional<(parseFlags & kParseNumbersAsStringsFlag) != 0, typename TargetEncoding::Ch, char>::type NumberCharacter; internal::StreamLocalCopy copy(is); NumberStream Date: Tue, 10 Aug 2021 09:40:03 +0200 Subject: [PATCH 4/9] Using unsigned for WIStreamWrapper --- test/unittest/readertest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unittest/readertest.cpp b/test/unittest/readertest.cpp index 995d6db..c805eef 100644 --- a/test/unittest/readertest.cpp +++ b/test/unittest/readertest.cpp @@ -1399,12 +1399,12 @@ public: WIStreamWrapper(std::wistream& is) : is_(is) {} Ch Peek() const { - int c = is_.peek(); + unsigned c = is_.peek(); return c == std::char_traits::eof() ? Ch('\0') : static_cast(c); } Ch Take() { - int c = is_.get(); + unsigned c = is_.get(); return c == std::char_traits::eof() ? Ch('\0') : static_cast(c); } From 8710d7e9893e826001c4de7c45e247d9a0084e78 Mon Sep 17 00:00:00 2001 From: Kyrega Date: Tue, 10 Aug 2021 10:12:29 +0200 Subject: [PATCH 5/9] Do not depend on c++11 conditional --- include/rapidjson/reader.h | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h index a4e5b3b..906f657 100644 --- a/include/rapidjson/reader.h +++ b/include/rapidjson/reader.h @@ -1464,9 +1464,24 @@ private: RAPIDJSON_FORCEINLINE Ch Take() { return Base::TakePush(); } }; + template + struct NumberCharacterConditional { + typedef char type; + }; + + template<> + struct NumberCharacterConditional { + typedef typename TargetEncoding::Ch type; + }; + + template<> + struct NumberCharacterConditional { + typedef char type; + }; + template void ParseNumber(InputStream& is, Handler& handler) { - typedef typename std::conditional<(parseFlags & kParseNumbersAsStringsFlag) != 0, typename TargetEncoding::Ch, char>::type NumberCharacter; + typedef NumberCharacterConditional<(parseFlags& kParseNumbersAsStringsFlag) != 0 >::type NumberCharacter; internal::StreamLocalCopy copy(is); NumberStream Date: Tue, 10 Aug 2021 10:20:18 +0200 Subject: [PATCH 6/9] Use rapidjson internal::SelectIf --- include/rapidjson/reader.h | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h index 906f657..542b7c0 100644 --- a/include/rapidjson/reader.h +++ b/include/rapidjson/reader.h @@ -1464,24 +1464,9 @@ private: RAPIDJSON_FORCEINLINE Ch Take() { return Base::TakePush(); } }; - template - struct NumberCharacterConditional { - typedef char type; - }; - - template<> - struct NumberCharacterConditional { - typedef typename TargetEncoding::Ch type; - }; - - template<> - struct NumberCharacterConditional { - typedef char type; - }; - template void ParseNumber(InputStream& is, Handler& handler) { - typedef NumberCharacterConditional<(parseFlags& kParseNumbersAsStringsFlag) != 0 >::type NumberCharacter; + typedef typename internal::SelectIf, typename TargetEncoding::Ch, char>::Type NumberCharacter; internal::StreamLocalCopy copy(is); NumberStream Date: Tue, 10 Aug 2021 10:46:21 +0200 Subject: [PATCH 7/9] No default template parameter for older compilers --- include/rapidjson/internal/biginteger.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/rapidjson/internal/biginteger.h b/include/rapidjson/internal/biginteger.h index 7b14563..009372b 100644 --- a/include/rapidjson/internal/biginteger.h +++ b/include/rapidjson/internal/biginteger.h @@ -37,7 +37,7 @@ public: digits_[0] = u; } - template + template BigInteger(const Ch* decimals, size_t length) : count_(1) { RAPIDJSON_ASSERT(length > 0); digits_[0] = 0; From a3d52c75b7d4099e86199659be7ccd2869781f70 Mon Sep 17 00:00:00 2001 From: Kyrega Date: Tue, 10 Aug 2021 10:51:45 +0200 Subject: [PATCH 8/9] No default template parameter for older compilers --- include/rapidjson/internal/biginteger.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/rapidjson/internal/biginteger.h b/include/rapidjson/internal/biginteger.h index 009372b..514a176 100644 --- a/include/rapidjson/internal/biginteger.h +++ b/include/rapidjson/internal/biginteger.h @@ -222,7 +222,7 @@ public: bool IsZero() const { return count_ == 1 && digits_[0] == 0; } private: - template + template void AppendDecimal64(const Ch* begin, const Ch* end) { uint64_t u = ParseUint64(begin, end); if (IsZero()) @@ -238,7 +238,7 @@ private: digits_[count_++] = digit; } - template + template static uint64_t ParseUint64(const Ch* begin, const Ch* end) { uint64_t r = 0; for (const Ch* p = begin; p != end; ++p) { From 22ee8b07c89fc791dd2a1e290539888bc1235fad Mon Sep 17 00:00:00 2001 From: Kyrega Date: Tue, 10 Aug 2021 11:00:40 +0200 Subject: [PATCH 9/9] Correct WIStreamWrapper --- test/unittest/readertest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unittest/readertest.cpp b/test/unittest/readertest.cpp index c805eef..057940f 100644 --- a/test/unittest/readertest.cpp +++ b/test/unittest/readertest.cpp @@ -1416,8 +1416,8 @@ public: size_t PutEnd(Ch*) { assert(false); return 0; } private: - WIStreamWrapper(const IStreamWrapper&); - WIStreamWrapper& operator=(const IStreamWrapper&); + WIStreamWrapper(const WIStreamWrapper&); + WIStreamWrapper& operator=(const WIStreamWrapper&); std::wistream& is_; };