gvsafronov/rapidjson
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Later clang compilers will warn on float -> double promotion because it can add precision. In the context of RapidJSON – especially with its float methods on GenericValue – I think this warning holds no water and should be ignored.

Browse Source 
Trim whitespace off the end of various lines.

Added an additional NumberStream specialization that will always perform a TakePush() even when just Take() is called. This supports RawNumber parsing by pushing onto our StackStream particular parts of the number that currently aren't captured because of full precision double parsing, such as the negative sign, scientific number exponents, etc.

RawNumber parsing fails with input streams that don't have copy optimization, such as the BasicIStreamWrapper stream. To work around this, instead do the Transcode copy operation by reading from a UTF8 StringStream instead of the original InputStream. Since the NumberStream downcasts all input Ch into chars, we know we're dealing with UTF8/ASCII compatible stack characters during the Transcoding.
This commit is contained in:
Jarred Nicholls 2016-03-28 14:31:36 -04:00
parent 4fdcb10c3e
commit 926d7ffcc8
3 changed files with 168 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
include/rapidjson/reader.h
View File

@ -1063,11 +1063,11 @@ private:
}
#endif
template<typename InputStream, bool backup>
template<typename InputStream, bool backup, bool pushOnTake>
class NumberStream;
template<typename InputStream>
class NumberStream<InputStream, false> {
class NumberStream<InputStream, false, false> {
public:
typedef typename InputStream::Ch Ch;
@ -1090,10 +1090,10 @@ private:
};
template<typename InputStream>
class NumberStream<InputStream, true> : public NumberStream<InputStream, false> {
typedef NumberStream<InputStream, false> Base;
class NumberStream<InputStream, true, false> : public NumberStream<InputStream, false, false> {
typedef NumberStream<InputStream, false, false> Base;
public:
NumberStream(GenericReader& reader, InputStream& is) : NumberStream<InputStream, false>(reader, is), stackStream(reader.stack_) {}
NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is), stackStream(reader.stack_) {}
~NumberStream() {}
RAPIDJSON_FORCEINLINE Ch TakePush() {
@ -1116,13 +1116,25 @@ private:
StackStream<char> stackStream;
};
template<typename InputStream>
class NumberStream<InputStream, true, true> : public NumberStream<InputStream, true, false> {
typedef NumberStream<InputStream, true, false> Base;
public:
NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is) {}
~NumberStream() {}
RAPIDJSON_FORCEINLINE Ch Take() { return Base::TakePush(); }
};
template<unsigned parseFlags, typename InputStream, typename Handler>
void ParseNumber(InputStream& is, Handler& handler) {
internal::StreamLocalCopy<InputStream> copy(is);
NumberStream<InputStream,
((parseFlags & kParseNumbersAsStringsFlag) != 0) ?
((parseFlags & kParseInsituFlag) == 0) :
((parseFlags & kParseFullPrecisionFlag) != 0)> s(*this, copy.s);
((parseFlags & kParseFullPrecisionFlag) != 0),
(parseFlags & kParseNumbersAsStringsFlag) != 0 &&
(parseFlags & kParseInsituFlag) == 0> s(*this, copy.s);
size_t startOffset = s.Tell();
@ -1210,9 +1222,6 @@ private:
int expFrac = 0;
size_t decimalPosition;
if (Consume(s, '.')) {
if (((parseFlags & kParseNumbersAsStringsFlag) != 0) && ((parseFlags & kParseInsituFlag) == 0)) {
s.Push('.');
}
decimalPosition = s.Length();
if (RAPIDJSON_UNLIKELY(!(s.Peek() >= '0' && s.Peek() <= '9')))
@ -1260,10 +1269,6 @@ private:
// Parse exp = e [ minus / plus ] 1*DIGIT
int exp = 0;
if (Consume(s, 'e') || Consume(s, 'E')) {
if ( ((parseFlags & kParseNumbersAsStringsFlag) != 0) && ((parseFlags & kParseInsituFlag) == 0) ) {
s.Push( 'e' );
}
if (!useDouble) {
d = static_cast<double>(use64bit ? i64 : i);
useDouble = true;
@ -1316,14 +1321,15 @@ private:
cont = handler.RawNumber(str, SizeType(length), false);
}
else {
StackStream<typename TargetEncoding::Ch> stackStream(stack_);
SizeType numCharsToCopy = static_cast<SizeType>(s.Length());
StringStream srcStream(s.Pop());
StackStream<typename TargetEncoding::Ch> dstStream(stack_);
while (numCharsToCopy--) {
Transcoder<SourceEncoding, TargetEncoding>::Transcode(is, stackStream);
Transcoder<UTF8<>, TargetEncoding>::Transcode(srcStream, dstStream);
}
stackStream.Put('\0');
const typename TargetEncoding::Ch* str = stackStream.Pop();
const SizeType length = static_cast<SizeType>(stackStream.Length()) - 1;
dstStream.Put('\0');
const typename TargetEncoding::Ch* str = dstStream.Pop();
const SizeType length = static_cast<SizeType>(dstStream.Length()) - 1;
cont = handler.RawNumber(str, SizeType(length), true);
}
}

5
test/unittest/CMakeLists.txt
View File

@ -38,6 +38,11 @@ if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wall -Wextra -Weffc++ -Wswitch-default -Wfloat-equal")
elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wall -Wextra -Weffc++ -Wswitch-default -Wfloat-equal -Wimplicit-fallthrough -Weverything")
# If the user is running a newer version of Clang that includes the
# -Wdouble-promotion, we will ignore that warning.
if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 3.7)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-double-promotion")
endif()
elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
add_definitions(-D_CRT_SECURE_NO_WARNINGS=1)
endif()

73
test/unittest/readertest.cpp
View File

@ -241,13 +241,13 @@ static void TestParseDouble() {
TEST_DOUBLE(fullPrecision, "0.017976931348623157e+310", 1.7976931348623157e+308); // Max double in another form
// Since
// abs((2^-1022 - 2^-1074) - 2.2250738585072012e-308) = 3.109754131239141401123495768877590405345064751974375599... ¡Á 10^-324
// abs((2^-1022) - 2.2250738585072012e-308) = 1.830902327173324040642192159804623318305533274168872044... ¡Á 10 ^ -324
// abs((2^-1022 - 2^-1074) - 2.2250738585072012e-308) = 3.109754131239141401123495768877590405345064751974375599... <EFBFBD><EFBFBD> 10^-324
// abs((2^-1022) - 2.2250738585072012e-308) = 1.830902327173324040642192159804623318305533274168872044... <EFBFBD><EFBFBD> 10 ^ -324
// So 2.2250738585072012e-308 should round to 2^-1022 = 2.2250738585072014e-308
TEST_DOUBLE(fullPrecision, "2.2250738585072012e-308", 2.2250738585072014e-308); // http://www.exploringbinary.com/java-hangs-when-converting-2-2250738585072012e-308/
// More closer to normal/subnormal boundary
// boundary = 2^-1022 - 2^-1075 = 2.225073858507201136057409796709131975934819546351645648... ¡Á 10^-308
// boundary = 2^-1022 - 2^-1075 = 2.225073858507201136057409796709131975934819546351645648... <EFBFBD><EFBFBD> 10^-308
TEST_DOUBLE(fullPrecision, "2.22507385850720113605740979670913197593481954635164564e-308", 2.2250738585072009e-308);
TEST_DOUBLE(fullPrecision, "2.22507385850720113605740979670913197593481954635164565e-308", 2.2250738585072014e-308);
@ -1541,7 +1541,8 @@ struct NumbersAsStringsHandler {
// 'str' is not null-terminated
bool RawNumber(const char* str, SizeType length, bool) {
EXPECT_TRUE(str != 0);
EXPECT_TRUE(strncmp(str, "3.1416", length) == 0);
EXPECT_TRUE(expected_len_ == length);
EXPECT_TRUE(strncmp(str, expected_, length) == 0);
return true;
}
bool String(const char*, SizeType, bool) { return true; }
@ -1550,24 +1551,84 @@ struct NumbersAsStringsHandler {
bool EndObject(SizeType) { return true; }
bool StartArray() { return true; }
bool EndArray(SizeType) { return true; }
NumbersAsStringsHandler(const char* expected)
: expected_(expected)
, expected_len_(strlen(expected)) {}
const char* expected_;
size_t expected_len_;
};
TEST(Reader, NumbersAsStrings) {
{
const char* json = "{ \"pi\": 3.1416 } ";
StringStream s(json);
NumbersAsStringsHandler h;
NumbersAsStringsHandler h("3.1416");
Reader reader;
EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
}
{
char* json = StrDup("{ \"pi\": 3.1416 } ");
InsituStringStream s(json);
NumbersAsStringsHandler h;
NumbersAsStringsHandler h("3.1416");
Reader reader;
EXPECT_TRUE(reader.Parse<kParseInsituFlag|kParseNumbersAsStringsFlag>(s, h));
free(json);
}
{
const char* json = "{ \"gigabyte\": 1.0e9 } ";
StringStream s(json);
NumbersAsStringsHandler h("1.0e9");
Reader reader;
EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
}
{
char* json = StrDup("{ \"gigabyte\": 1.0e9 } ");
InsituStringStream s(json);
NumbersAsStringsHandler h("1.0e9");
Reader reader;
EXPECT_TRUE(reader.Parse<kParseInsituFlag|kParseNumbersAsStringsFlag>(s, h));
free(json);
}
{
const char* json = "{ \"pi\": 314.159e-2 } ";
StringStream s(json);
NumbersAsStringsHandler h("314.159e-2");
Reader reader;
EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
}
{
char* json = StrDup("{ \"gigabyte\": 314.159e-2 } ");
InsituStringStream s(json);
NumbersAsStringsHandler h("314.159e-2");
Reader reader;
EXPECT_TRUE(reader.Parse<kParseInsituFlag|kParseNumbersAsStringsFlag>(s, h));
free(json);
}
{
const char* json = "{ \"negative\": -1.54321 } ";
StringStream s(json);
NumbersAsStringsHandler h("-1.54321");
Reader reader;
EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
}
{
char* json = StrDup("{ \"negative\": -1.54321 } ");
InsituStringStream s(json);
NumbersAsStringsHandler h("-1.54321");
Reader reader;
EXPECT_TRUE(reader.Parse<kParseInsituFlag|kParseNumbersAsStringsFlag>(s, h));
free(json);
}
{
const char* json = "{ \"pi\": 314.159e-2 } ";
std::stringstream ss(json);
IStreamWrapper s(ss);
NumbersAsStringsHandler h("314.159e-2");
Reader reader;
EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
}
}
template <unsigned extraFlags>