Merge pull request #1926 from Kyrega/master

Fix RawNumber for characters with sizeof(Ch) > sizeof(char)

include/rapidjson/internal/biginteger.h

@@ -37,7 +37,8 @@ public:
         digits_[0] = u;
     }
 
-    BigInteger(const char* decimals, size_t length) : count_(1) {
+    template<typename Ch>
+    BigInteger(const Ch* decimals, size_t length) : count_(1) {
         RAPIDJSON_ASSERT(length > 0);
         digits_[0] = 0;
         size_t i = 0;
@@ -221,7 +222,8 @@ public:
     bool IsZero() const { return count_ == 1 && digits_[0] == 0; }
 
 private:
-    void AppendDecimal64(const char* begin, const char* end) {
+    template<typename Ch>
+    void AppendDecimal64(const Ch* begin, const Ch* end) {
         uint64_t u = ParseUint64(begin, end);
         if (IsZero())
             *this = u;
@@ -236,11 +238,12 @@ private:
         digits_[count_++] = digit;
     }
 
-    static uint64_t ParseUint64(const char* begin, const char* end) {
+    template<typename Ch>
+    static uint64_t ParseUint64(const Ch* begin, const Ch* end) {
         uint64_t r = 0;
-        for (const char* p = begin; p != end; ++p) {
+        for (const Ch* p = begin; p != end; ++p) {
-            RAPIDJSON_ASSERT(*p >= '0' && *p <= '9');
+            RAPIDJSON_ASSERT(*p >= Ch('0') && *p <= Ch('9'));
-            r = r * 10u + static_cast<unsigned>(*p - '0');
+            r = r * 10u + static_cast<unsigned>(*p - Ch('0'));
         }
         return r;
     }

include/rapidjson/internal/strtod.h

@@ -128,17 +128,18 @@ inline bool StrtodFast(double d, int p, double* result) {
 }
 
 // Compute an approximation and see if it is within 1/2 ULP
-inline bool StrtodDiyFp(const char* decimals, int dLen, int dExp, double* result) {
+template<typename Ch>
+inline bool StrtodDiyFp(const Ch* decimals, int dLen, int dExp, double* result) {
     uint64_t significand = 0;
     int i = 0;   // 2^64 - 1 = 18446744073709551615, 1844674407370955161 = 0x1999999999999999    
     for (; i < dLen; i++) {
         if (significand  >  RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) ||
-            (significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > '5'))
+            (significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > Ch('5')))
             break;
-        significand = significand * 10u + static_cast<unsigned>(decimals[i] - '0');
+        significand = significand * 10u + static_cast<unsigned>(decimals[i] - Ch('0'));
     }
     
-    if (i < dLen && decimals[i] >= '5') // Rounding
+    if (i < dLen && decimals[i] >= Ch('5')) // Rounding
         significand++;
 
     int remaining = dLen - i;
@@ -205,7 +206,8 @@ inline bool StrtodDiyFp(const char* decimals, int dLen, int dExp, double* result
     return halfWay - static_cast<unsigned>(error) >= precisionBits || precisionBits >= halfWay + static_cast<unsigned>(error);
 }
 
-inline double StrtodBigInteger(double approx, const char* decimals, int dLen, int dExp) {
+template<typename Ch>
+inline double StrtodBigInteger(double approx, const Ch* decimals, int dLen, int dExp) {
     RAPIDJSON_ASSERT(dLen >= 0);
     const BigInteger dInt(decimals, static_cast<unsigned>(dLen));
     Double a(approx);
@@ -223,7 +225,8 @@ inline double StrtodBigInteger(double approx, const char* decimals, int dLen, in
         return a.NextPositiveDouble();
 }
 
-inline double StrtodFullPrecision(double d, int p, const char* decimals, size_t length, size_t decimalPosition, int exp) {
+template<typename Ch>
+inline double StrtodFullPrecision(double d, int p, const Ch* decimals, size_t length, size_t decimalPosition, int exp) {
     RAPIDJSON_ASSERT(d >= 0.0);
     RAPIDJSON_ASSERT(length >= 1);
 

include/rapidjson/reader.h

@@ -1404,11 +1404,11 @@ private:
     }
 #endif // RAPIDJSON_NEON
 
-    template<typename InputStream, bool backup, bool pushOnTake>
+    template<typename InputStream, typename StackCharacter, bool backup, bool pushOnTake>
     class NumberStream;
 
-    template<typename InputStream>
+    template<typename InputStream, typename StackCharacter>
-    class NumberStream<InputStream, false, false> {
+    class NumberStream<InputStream, StackCharacter, false, false> {
     public:
         typedef typename InputStream::Ch Ch;
 
@@ -1421,7 +1421,7 @@ private:
 
         size_t Tell() { return is.Tell(); }
         size_t Length() { return 0; }
-        const char* Pop() { return 0; }
+        const StackCharacter* Pop() { return 0; }
 
     protected:
         NumberStream& operator=(const NumberStream&);
@@ -1429,35 +1429,35 @@ private:
         InputStream& is;
     };
 
-    template<typename InputStream>
+    template<typename InputStream, typename StackCharacter>
-    class NumberStream<InputStream, true, false> : public NumberStream<InputStream, false, false> {
+    class NumberStream<InputStream, StackCharacter, true, false> : public NumberStream<InputStream, StackCharacter, false, false> {
-        typedef NumberStream<InputStream, false, false> Base;
+        typedef NumberStream<InputStream, StackCharacter, false, false> Base;
     public:
         NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is), stackStream(reader.stack_) {}
 
         RAPIDJSON_FORCEINLINE Ch TakePush() {
-            stackStream.Put(static_cast<char>(Base::is.Peek()));
+            stackStream.Put(static_cast<StackCharacter>(Base::is.Peek()));
             return Base::is.Take();
         }
 
-        RAPIDJSON_FORCEINLINE void Push(char c) {
+        RAPIDJSON_FORCEINLINE void Push(StackCharacter c) {
             stackStream.Put(c);
         }
 
         size_t Length() { return stackStream.Length(); }
 
-        const char* Pop() {
+        const StackCharacter* Pop() {
             stackStream.Put('\0');
             return stackStream.Pop();
         }
 
     private:
-        StackStream<char> stackStream;
+        StackStream<StackCharacter> stackStream;
     };
 
-    template<typename InputStream>
+    template<typename InputStream, typename StackCharacter>
-    class NumberStream<InputStream, true, true> : public NumberStream<InputStream, true, false> {
+    class NumberStream<InputStream, StackCharacter, true, true> : public NumberStream<InputStream, StackCharacter, true, false> {
-        typedef NumberStream<InputStream, true, false> Base;
+        typedef NumberStream<InputStream, StackCharacter, true, false> Base;
     public:
         NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is) {}
 
@@ -1466,8 +1466,10 @@ private:
 
     template<unsigned parseFlags, typename InputStream, typename Handler>
     void ParseNumber(InputStream& is, Handler& handler) {
+        typedef typename internal::SelectIf<internal::BoolType<(parseFlags & kParseNumbersAsStringsFlag) != 0>, typename TargetEncoding::Ch, char>::Type NumberCharacter;
+
         internal::StreamLocalCopy<InputStream> copy(is);
-        NumberStream<InputStream,
+        NumberStream<InputStream, NumberCharacter,
             ((parseFlags & kParseNumbersAsStringsFlag) != 0) ?
                 ((parseFlags & kParseInsituFlag) == 0) :
                 ((parseFlags & kParseFullPrecisionFlag) != 0),
@@ -1692,10 +1694,10 @@ private:
             }
             else {
                 SizeType numCharsToCopy = static_cast<SizeType>(s.Length());
-                StringStream srcStream(s.Pop());
+                GenericStringStream<UTF8<NumberCharacter>> srcStream(s.Pop());
                 StackStream<typename TargetEncoding::Ch> dstStream(stack_);
                 while (numCharsToCopy--) {
-                    Transcoder<UTF8<>, TargetEncoding>::Transcode(srcStream, dstStream);
+                    Transcoder<UTF8<typename TargetEncoding::Ch>, TargetEncoding>::Transcode(srcStream, dstStream);
                 }
                 dstStream.Put('\0');
                 const typename TargetEncoding::Ch* str = dstStream.Pop();
@@ -1705,7 +1707,7 @@ private:
         }
         else {
            size_t length = s.Length();
-           const char* decimal = s.Pop();  // Pop stack no matter if it will be used or not.
+           const NumberCharacter* decimal = s.Pop();  // Pop stack no matter if it will be used or not.
 
            if (useDouble) {
                int p = exp + expFrac;

test/unittest/readertest.cpp

@@ -1392,6 +1392,36 @@ private:
     std::istream& is_;
 };
 
+class WIStreamWrapper {
+public:
+  typedef wchar_t Ch;
+
+  WIStreamWrapper(std::wistream& is) : is_(is) {}
+
+  Ch Peek() const {
+    unsigned c = is_.peek();
+    return c == std::char_traits<wchar_t>::eof() ? Ch('\0') : static_cast<Ch>(c);
+  }
+
+  Ch Take() {
+    unsigned c = is_.get();
+    return c == std::char_traits<wchar_t>::eof() ? Ch('\0') : static_cast<Ch>(c);
+  }
+
+  size_t Tell() const { return static_cast<size_t>(is_.tellg()); }
+
+  Ch* PutBegin() { assert(false); return 0; }
+  void Put(Ch) { assert(false); }
+  void Flush() { assert(false); }
+  size_t PutEnd(Ch*) { assert(false); return 0; }
+
+private:
+  WIStreamWrapper(const WIStreamWrapper&);
+  WIStreamWrapper& operator=(const WIStreamWrapper&);
+
+  std::wistream& is_;
+};
+
 TEST(Reader, Parse_IStreamWrapper_StringStream) {
     const char* json = "[1,2,3,4]";
 
@@ -1991,6 +2021,118 @@ TEST(Reader, NumbersAsStrings) {
     }
 }
 
+struct NumbersAsStringsHandlerWChar_t {
+  bool Null() { return true; }
+  bool Bool(bool) { return true; }
+  bool Int(int) { return true; }
+  bool Uint(unsigned) { return true; }
+  bool Int64(int64_t) { return true; }
+  bool Uint64(uint64_t) { return true; }
+  bool Double(double) { return true; }
+  // 'str' is not null-terminated
+  bool RawNumber(const wchar_t* str, SizeType length, bool) {
+    EXPECT_TRUE(str != 0);
+    EXPECT_TRUE(expected_len_ == length);
+    EXPECT_TRUE(wcsncmp(str, expected_, length) == 0);
+    return true;
+  }
+  bool String(const wchar_t*, SizeType, bool) { return true; }
+  bool StartObject() { return true; }
+  bool Key(const wchar_t*, SizeType, bool) { return true; }
+  bool EndObject(SizeType) { return true; }
+  bool StartArray() { return true; }
+  bool EndArray(SizeType) { return true; }
+
+  NumbersAsStringsHandlerWChar_t(const wchar_t* expected)
+    : expected_(expected)
+    , expected_len_(wcslen(expected)) {}
+
+  const wchar_t* expected_;
+  size_t expected_len_;
+};
+
+TEST(Reader, NumbersAsStringsWChar_t) {
+  {
+    const wchar_t* json = L"{ \"pi\": 3.1416 } ";
+    GenericStringStream<UTF16<>> s(json);
+    NumbersAsStringsHandlerWChar_t h(L"3.1416");
+    GenericReader<UTF16<>, UTF16<>> reader;
+    EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
+  }
+  {
+    wchar_t* json = StrDup(L"{ \"pi\": 3.1416 } ");
+    GenericInsituStringStream<UTF16<>> s(json);
+    NumbersAsStringsHandlerWChar_t h(L"3.1416");
+    GenericReader<UTF16<>, UTF16<>> reader;
+    EXPECT_TRUE(reader.Parse<kParseInsituFlag | kParseNumbersAsStringsFlag>(s, h));
+    free(json);
+  }
+  {
+    const wchar_t* json = L"{ \"gigabyte\": 1.0e9 } ";
+    GenericStringStream<UTF16<>> s(json);
+    NumbersAsStringsHandlerWChar_t h(L"1.0e9");
+    GenericReader<UTF16<>, UTF16<>> reader;
+    EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
+  }
+  {
+    wchar_t* json = StrDup(L"{ \"gigabyte\": 1.0e9 } ");
+    GenericInsituStringStream<UTF16<>> s(json);
+    NumbersAsStringsHandlerWChar_t h(L"1.0e9");
+    GenericReader<UTF16<>, UTF16<>> reader;
+    EXPECT_TRUE(reader.Parse<kParseInsituFlag | kParseNumbersAsStringsFlag>(s, h));
+    free(json);
+  }
+  {
+    const wchar_t* json = L"{ \"pi\": 314.159e-2 } ";
+    GenericStringStream<UTF16<>> s(json);
+    NumbersAsStringsHandlerWChar_t h(L"314.159e-2");
+    GenericReader<UTF16<>, UTF16<>> reader;
+    EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
+  }
+  {
+    wchar_t* json = StrDup(L"{ \"gigabyte\": 314.159e-2 } ");
+    GenericInsituStringStream<UTF16<>> s(json);
+    NumbersAsStringsHandlerWChar_t h(L"314.159e-2");
+    GenericReader<UTF16<>, UTF16<>> reader;
+    EXPECT_TRUE(reader.Parse<kParseInsituFlag | kParseNumbersAsStringsFlag>(s, h));
+    free(json);
+  }
+  {
+    const wchar_t* json = L"{ \"negative\": -1.54321 } ";
+    GenericStringStream<UTF16<>> s(json);
+    NumbersAsStringsHandlerWChar_t h(L"-1.54321");
+    GenericReader<UTF16<>, UTF16<>> reader;
+    EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
+  }
+  {
+    wchar_t* json = StrDup(L"{ \"negative\": -1.54321 } ");
+    GenericInsituStringStream<UTF16<>> s(json);
+    NumbersAsStringsHandlerWChar_t h(L"-1.54321");
+    GenericReader<UTF16<>, UTF16<>> reader;
+    EXPECT_TRUE(reader.Parse<kParseInsituFlag | kParseNumbersAsStringsFlag>(s, h));
+    free(json);
+  }
+  {
+    const wchar_t* json = L"{ \"pi\": 314.159e-2 } ";
+    std::wstringstream ss(json);
+    WIStreamWrapper s(ss);
+    NumbersAsStringsHandlerWChar_t h(L"314.159e-2");
+    GenericReader<UTF16<>, UTF16<>> reader;
+    EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
+  }
+  {
+    wchar_t n1e319[321];   // '1' followed by 319 '0'
+    n1e319[0] = L'1';
+    for(int i = 1; i < 320; i++)
+      n1e319[i] = L'0';
+    n1e319[320] = L'\0';
+    GenericStringStream<UTF16<>> s(n1e319);
+    NumbersAsStringsHandlerWChar_t h(n1e319);
+    GenericReader<UTF16<>, UTF16<>> reader;
+    EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
+  }
+}
+
 template <unsigned extraFlags>
 void TestTrailingCommas() {
     {