Fix implicit signed/unsigned conversion and a small glitch in the error computation

This commit is contained in:
abolz 2018-06-15 13:44:15 +02:00
parent 2ea43433e2
commit 16c97cd7c5

View File

@ -162,10 +162,10 @@ inline bool StrtodDiyFp(const char* decimals, int dLen, int dExp, double* result
DiyFp(RAPIDJSON_UINT64_C2(0xf4240000, 00000000), -44), // 10^6
DiyFp(RAPIDJSON_UINT64_C2(0x98968000, 00000000), -40) // 10^7
};
int adjustment = dExp - actualExp - 1;
RAPIDJSON_ASSERT(adjustment >= 0 && adjustment < 7);
v = v * kPow10[adjustment];
if (dLen + static_cast<unsigned>(adjustment)> 19u) // has more digits than decimal digits in 64-bit
int adjustment = dExp - actualExp;
RAPIDJSON_ASSERT(adjustment >= 1 && adjustment < 8);
v = v * kPow10[adjustment - 1];
if (dLen + adjustment > 19u) // has more digits than decimal digits in 64-bit
error += kUlp / 2;
}