gvsafronov/rapidjson
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1926 from Kyrega/master

Browse Source 
Fix RawNumber for characters with sizeof(Ch) > sizeof(char)
This commit is contained in:
Milo Yip 2021-08-13 12:25:41 +08:00 committed by GitHub
commit 00dbcf2c6e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 180 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
include/rapidjson/internal/biginteger.h
View File

@ -37,7 +37,8 @@ public:
digits_[0] = u;
}
BigInteger(const char* decimals, size_t length) : count_(1) {
template<typename Ch>
BigInteger(const Ch* decimals, size_t length) : count_(1) {
RAPIDJSON_ASSERT(length > 0);
digits_[0] = 0;
size_t i = 0;
@ -221,7 +222,8 @@ public:
bool IsZero() const { return count_ == 1 && digits_[0] == 0; }
private:
void AppendDecimal64(const char* begin, const char* end) {
template<typename Ch>
void AppendDecimal64(const Ch* begin, const Ch* end) {
uint64_t u = ParseUint64(begin, end);
if (IsZero())
*this = u;
@ -236,11 +238,12 @@ private:
digits_[count_++] = digit;
}
static uint64_t ParseUint64(const char* begin, const char* end) {
template<typename Ch>
static uint64_t ParseUint64(const Ch* begin, const Ch* end) {
uint64_t r = 0;
for (const char* p = begin; p != end; ++p) {
RAPIDJSON_ASSERT(*p >= '0' && *p <= '9');
r = r * 10u + static_cast<unsigned>(*p - '0');
for (const Ch* p = begin; p != end; ++p) {
RAPIDJSON_ASSERT(*p >= Ch('0') && *p <= Ch('9'));
r = r * 10u + static_cast<unsigned>(*p - Ch('0'));
}
return r;
}

15
include/rapidjson/internal/strtod.h
View File

@ -128,17 +128,18 @@ inline bool StrtodFast(double d, int p, double* result) {
}
// Compute an approximation and see if it is within 1/2 ULP
inline bool StrtodDiyFp(const char* decimals, int dLen, int dExp, double* result) {
template<typename Ch>
inline bool StrtodDiyFp(const Ch* decimals, int dLen, int dExp, double* result) {
uint64_t significand = 0;
int i = 0; // 2^64 - 1 = 18446744073709551615, 1844674407370955161 = 0x1999999999999999
for (; i < dLen; i++) {
if (significand > RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) ||
(significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > '5'))
(significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > Ch('5')))
break;
significand = significand * 10u + static_cast<unsigned>(decimals[i] - '0');
significand = significand * 10u + static_cast<unsigned>(decimals[i] - Ch('0'));
}
if (i < dLen && decimals[i] >= '5') // Rounding
if (i < dLen && decimals[i] >= Ch('5')) // Rounding
significand++;
int remaining = dLen - i;
@ -205,7 +206,8 @@ inline bool StrtodDiyFp(const char* decimals, int dLen, int dExp, double* result
return halfWay - static_cast<unsigned>(error) >= precisionBits || precisionBits >= halfWay + static_cast<unsigned>(error);
}
inline double StrtodBigInteger(double approx, const char* decimals, int dLen, int dExp) {
template<typename Ch>
inline double StrtodBigInteger(double approx, const Ch* decimals, int dLen, int dExp) {
RAPIDJSON_ASSERT(dLen >= 0);
const BigInteger dInt(decimals, static_cast<unsigned>(dLen));
Double a(approx);
@ -223,7 +225,8 @@ inline double StrtodBigInteger(double approx, const char* decimals, int dLen, in
return a.NextPositiveDouble();
}
inline double StrtodFullPrecision(double d, int p, const char* decimals, size_t length, size_t decimalPosition, int exp) {
template<typename Ch>
inline double StrtodFullPrecision(double d, int p, const Ch* decimals, size_t length, size_t decimalPosition, int exp) {
RAPIDJSON_ASSERT(d >= 0.0);
RAPIDJSON_ASSERT(length >= 1);

38
include/rapidjson/reader.h
View File

@ -1404,11 +1404,11 @@ private:
}
#endif // RAPIDJSON_NEON
template<typename InputStream, bool backup, bool pushOnTake>
template<typename InputStream, typename StackCharacter, bool backup, bool pushOnTake>
class NumberStream;
template<typename InputStream>
class NumberStream<InputStream, false, false> {
template<typename InputStream, typename StackCharacter>
class NumberStream<InputStream, StackCharacter, false, false> {
public:
typedef typename InputStream::Ch Ch;
@ -1421,7 +1421,7 @@ private:
size_t Tell() { return is.Tell(); }
size_t Length() { return 0; }
const char* Pop() { return 0; }
const StackCharacter* Pop() { return 0; }
protected:
NumberStream& operator=(const NumberStream&);
@ -1429,35 +1429,35 @@ private:
InputStream& is;
};
template<typename InputStream>
class NumberStream<InputStream, true, false> : public NumberStream<InputStream, false, false> {
typedef NumberStream<InputStream, false, false> Base;
template<typename InputStream, typename StackCharacter>
class NumberStream<InputStream, StackCharacter, true, false> : public NumberStream<InputStream, StackCharacter, false, false> {
typedef NumberStream<InputStream, StackCharacter, false, false> Base;
public:
NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is), stackStream(reader.stack_) {}
RAPIDJSON_FORCEINLINE Ch TakePush() {
stackStream.Put(static_cast<char>(Base::is.Peek()));
stackStream.Put(static_cast<StackCharacter>(Base::is.Peek()));
return Base::is.Take();
}
RAPIDJSON_FORCEINLINE void Push(char c) {
RAPIDJSON_FORCEINLINE void Push(StackCharacter c) {
stackStream.Put(c);
}
size_t Length() { return stackStream.Length(); }
const char* Pop() {
const StackCharacter* Pop() {
stackStream.Put('\0');
return stackStream.Pop();
}
private:
StackStream<char> stackStream;
StackStream<StackCharacter> stackStream;
};
template<typename InputStream>
class NumberStream<InputStream, true, true> : public NumberStream<InputStream, true, false> {
typedef NumberStream<InputStream, true, false> Base;
template<typename InputStream, typename StackCharacter>
class NumberStream<InputStream, StackCharacter, true, true> : public NumberStream<InputStream, StackCharacter, true, false> {
typedef NumberStream<InputStream, StackCharacter, true, false> Base;
public:
NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is) {}
@ -1466,8 +1466,10 @@ private:
template<unsigned parseFlags, typename InputStream, typename Handler>
void ParseNumber(InputStream& is, Handler& handler) {
typedef typename internal::SelectIf<internal::BoolType<(parseFlags & kParseNumbersAsStringsFlag) != 0>, typename TargetEncoding::Ch, char>::Type NumberCharacter;
internal::StreamLocalCopy<InputStream> copy(is);
NumberStream<InputStream,
NumberStream<InputStream, NumberCharacter,
((parseFlags & kParseNumbersAsStringsFlag) != 0) ?
((parseFlags & kParseInsituFlag) == 0) :
((parseFlags & kParseFullPrecisionFlag) != 0),
@ -1692,10 +1694,10 @@ private:
}
else {
SizeType numCharsToCopy = static_cast<SizeType>(s.Length());
StringStream srcStream(s.Pop());
GenericStringStream<UTF8<NumberCharacter>> srcStream(s.Pop());
StackStream<typename TargetEncoding::Ch> dstStream(stack_);
while (numCharsToCopy--) {
Transcoder<UTF8<>, TargetEncoding>::Transcode(srcStream, dstStream);
Transcoder<UTF8<typename TargetEncoding::Ch>, TargetEncoding>::Transcode(srcStream, dstStream);
}
dstStream.Put('\0');
const typename TargetEncoding::Ch* str = dstStream.Pop();
@ -1705,7 +1707,7 @@ private:
}
else {
size_t length = s.Length();
const char* decimal = s.Pop(); // Pop stack no matter if it will be used or not.
const NumberCharacter* decimal = s.Pop(); // Pop stack no matter if it will be used or not.
if (useDouble) {
int p = exp + expFrac;

142
test/unittest/readertest.cpp
View File

@ -1392,6 +1392,36 @@ private:
std::istream& is_;
};
class WIStreamWrapper {
public:
typedef wchar_t Ch;
WIStreamWrapper(std::wistream& is) : is_(is) {}
Ch Peek() const {
unsigned c = is_.peek();
return c == std::char_traits<wchar_t>::eof() ? Ch('\0') : static_cast<Ch>(c);
}
Ch Take() {
unsigned c = is_.get();
return c == std::char_traits<wchar_t>::eof() ? Ch('\0') : static_cast<Ch>(c);
}
size_t Tell() const { return static_cast<size_t>(is_.tellg()); }
Ch* PutBegin() { assert(false); return 0; }
void Put(Ch) { assert(false); }
void Flush() { assert(false); }
size_t PutEnd(Ch*) { assert(false); return 0; }
private:
WIStreamWrapper(const WIStreamWrapper&);
WIStreamWrapper& operator=(const WIStreamWrapper&);
std::wistream& is_;
};
TEST(Reader, Parse_IStreamWrapper_StringStream) {
const char* json = "[1,2,3,4]";
@ -1991,6 +2021,118 @@ TEST(Reader, NumbersAsStrings) {
}
}
struct NumbersAsStringsHandlerWChar_t {
bool Null() { return true; }
bool Bool(bool) { return true; }
bool Int(int) { return true; }
bool Uint(unsigned) { return true; }
bool Int64(int64_t) { return true; }
bool Uint64(uint64_t) { return true; }
bool Double(double) { return true; }
// 'str' is not null-terminated
bool RawNumber(const wchar_t* str, SizeType length, bool) {
EXPECT_TRUE(str != 0);
EXPECT_TRUE(expected_len_ == length);
EXPECT_TRUE(wcsncmp(str, expected_, length) == 0);
return true;
}
bool String(const wchar_t*, SizeType, bool) { return true; }
bool StartObject() { return true; }
bool Key(const wchar_t*, SizeType, bool) { return true; }
bool EndObject(SizeType) { return true; }
bool StartArray() { return true; }
bool EndArray(SizeType) { return true; }
NumbersAsStringsHandlerWChar_t(const wchar_t* expected)
: expected_(expected)
, expected_len_(wcslen(expected)) {}
const wchar_t* expected_;
size_t expected_len_;
};
TEST(Reader, NumbersAsStringsWChar_t) {
{
const wchar_t* json = L"{ \"pi\": 3.1416 } ";
GenericStringStream<UTF16<>> s(json);
NumbersAsStringsHandlerWChar_t h(L"3.1416");
GenericReader<UTF16<>, UTF16<>> reader;
EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
}
{
wchar_t* json = StrDup(L"{ \"pi\": 3.1416 } ");
GenericInsituStringStream<UTF16<>> s(json);
NumbersAsStringsHandlerWChar_t h(L"3.1416");
GenericReader<UTF16<>, UTF16<>> reader;
EXPECT_TRUE(reader.Parse<kParseInsituFlag | kParseNumbersAsStringsFlag>(s, h));
free(json);
}
{
const wchar_t* json = L"{ \"gigabyte\": 1.0e9 } ";
GenericStringStream<UTF16<>> s(json);
NumbersAsStringsHandlerWChar_t h(L"1.0e9");
GenericReader<UTF16<>, UTF16<>> reader;
EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
}
{
wchar_t* json = StrDup(L"{ \"gigabyte\": 1.0e9 } ");
GenericInsituStringStream<UTF16<>> s(json);
NumbersAsStringsHandlerWChar_t h(L"1.0e9");
GenericReader<UTF16<>, UTF16<>> reader;
EXPECT_TRUE(reader.Parse<kParseInsituFlag | kParseNumbersAsStringsFlag>(s, h));
free(json);
}
{
const wchar_t* json = L"{ \"pi\": 314.159e-2 } ";
GenericStringStream<UTF16<>> s(json);
NumbersAsStringsHandlerWChar_t h(L"314.159e-2");
GenericReader<UTF16<>, UTF16<>> reader;
EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
}
{
wchar_t* json = StrDup(L"{ \"gigabyte\": 314.159e-2 } ");
GenericInsituStringStream<UTF16<>> s(json);
NumbersAsStringsHandlerWChar_t h(L"314.159e-2");
GenericReader<UTF16<>, UTF16<>> reader;
EXPECT_TRUE(reader.Parse<kParseInsituFlag | kParseNumbersAsStringsFlag>(s, h));
free(json);
}
{
const wchar_t* json = L"{ \"negative\": -1.54321 } ";
GenericStringStream<UTF16<>> s(json);
NumbersAsStringsHandlerWChar_t h(L"-1.54321");
GenericReader<UTF16<>, UTF16<>> reader;
EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
}
{
wchar_t* json = StrDup(L"{ \"negative\": -1.54321 } ");
GenericInsituStringStream<UTF16<>> s(json);
NumbersAsStringsHandlerWChar_t h(L"-1.54321");
GenericReader<UTF16<>, UTF16<>> reader;
EXPECT_TRUE(reader.Parse<kParseInsituFlag | kParseNumbersAsStringsFlag>(s, h));
free(json);
}
{
const wchar_t* json = L"{ \"pi\": 314.159e-2 } ";
std::wstringstream ss(json);
WIStreamWrapper s(ss);
NumbersAsStringsHandlerWChar_t h(L"314.159e-2");
GenericReader<UTF16<>, UTF16<>> reader;
EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
}
{
wchar_t n1e319[321]; // '1' followed by 319 '0'
n1e319[0] = L'1';
for(int i = 1; i < 320; i++)
n1e319[i] = L'0';
n1e319[320] = L'\0';
GenericStringStream<UTF16<>> s(n1e319);
NumbersAsStringsHandlerWChar_t h(n1e319);
GenericReader<UTF16<>, UTF16<>> reader;
EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
}
}
template <unsigned extraFlags>
void TestTrailingCommas() {
{