gvsafronov/rapidjson
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Later clang compilers will warn on float -> double promotion because it can add precision. In the context of RapidJSON – especially with its float methods on GenericValue – I think this warning holds no water and should be ignored.

Browse Source 
Trim whitespace off the end of various lines.

Added an additional NumberStream specialization that will always perform a TakePush() even when just Take() is called. This supports RawNumber parsing by pushing onto our StackStream particular parts of the number that currently aren't captured because of full precision double parsing, such as the negative sign, scientific number exponents, etc.

RawNumber parsing fails with input streams that don't have copy optimization, such as the BasicIStreamWrapper stream. To work around this, instead do the Transcode copy operation by reading from a UTF8 StringStream instead of the original InputStream. Since the NumberStream downcasts all input Ch into chars, we know we're dealing with UTF8/ASCII compatible stack characters during the Transcoding.
This commit is contained in:
Jarred Nicholls 2016-03-28 14:31:36 -04:00
parent 4fdcb10c3e
commit 926d7ffcc8
3 changed files with 168 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
include/rapidjson/reader.h
View File

@ -1063,11 +1063,11 @@ private:
} }
#endif #endif
template<typename InputStream, bool backup> template<typename InputStream, bool backup, bool pushOnTake>
class NumberStream; class NumberStream;
template<typename InputStream> template<typename InputStream>
class NumberStream<InputStream, false> { class NumberStream<InputStream, false, false> {
public: public:
typedef typename InputStream::Ch Ch; typedef typename InputStream::Ch Ch;
@ -1090,10 +1090,10 @@ private:
}; };
template<typename InputStream> template<typename InputStream>
class NumberStream<InputStream, true> : public NumberStream<InputStream, false> { class NumberStream<InputStream, true, false> : public NumberStream<InputStream, false, false> {
typedef NumberStream<InputStream, false> Base; typedef NumberStream<InputStream, false, false> Base;
public: public:
NumberStream(GenericReader& reader, InputStream& is) : NumberStream<InputStream, false>(reader, is), stackStream(reader.stack_) {} NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is), stackStream(reader.stack_) {}
~NumberStream() {} ~NumberStream() {}
RAPIDJSON_FORCEINLINE Ch TakePush() { RAPIDJSON_FORCEINLINE Ch TakePush() {
@ -1101,9 +1101,9 @@ private:
return Base::is.Take(); return Base::is.Take();
} }
RAPIDJSON_FORCEINLINE void Push(char c) { RAPIDJSON_FORCEINLINE void Push(char c) {
stackStream.Put(c); stackStream.Put(c);
} }
size_t Length() { return stackStream.Length(); } size_t Length() { return stackStream.Length(); }
@ -1116,13 +1116,25 @@ private:
StackStream<char> stackStream; StackStream<char> stackStream;
}; };
template<typename InputStream>
class NumberStream<InputStream, true, true> : public NumberStream<InputStream, true, false> {
typedef NumberStream<InputStream, true, false> Base;
public:
NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is) {}
~NumberStream() {}
RAPIDJSON_FORCEINLINE Ch Take() { return Base::TakePush(); }
};
template<unsigned parseFlags, typename InputStream, typename Handler> template<unsigned parseFlags, typename InputStream, typename Handler>
void ParseNumber(InputStream& is, Handler& handler) { void ParseNumber(InputStream& is, Handler& handler) {
internal::StreamLocalCopy<InputStream> copy(is); internal::StreamLocalCopy<InputStream> copy(is);
NumberStream<InputStream, NumberStream<InputStream,
((parseFlags & kParseNumbersAsStringsFlag) != 0) ? ((parseFlags & kParseNumbersAsStringsFlag) != 0) ?
((parseFlags & kParseInsituFlag) == 0) : ((parseFlags & kParseInsituFlag) == 0) :
((parseFlags & kParseFullPrecisionFlag) != 0)> s(*this, copy.s); ((parseFlags & kParseFullPrecisionFlag) != 0),
(parseFlags & kParseNumbersAsStringsFlag) != 0 &&
(parseFlags & kParseInsituFlag) == 0> s(*this, copy.s);
size_t startOffset = s.Tell(); size_t startOffset = s.Tell();
@ -1210,9 +1222,6 @@ private:
int expFrac = 0; int expFrac = 0;
size_t decimalPosition; size_t decimalPosition;
if (Consume(s, '.')) { if (Consume(s, '.')) {
if (((parseFlags & kParseNumbersAsStringsFlag) != 0) && ((parseFlags & kParseInsituFlag) == 0)) {
s.Push('.');
}
decimalPosition = s.Length(); decimalPosition = s.Length();
if (RAPIDJSON_UNLIKELY(!(s.Peek() >= '0' && s.Peek() <= '9'))) if (RAPIDJSON_UNLIKELY(!(s.Peek() >= '0' && s.Peek() <= '9')))
@ -1260,11 +1269,7 @@ private:
// Parse exp = e [ minus / plus ] 1*DIGIT // Parse exp = e [ minus / plus ] 1*DIGIT
int exp = 0; int exp = 0;
if (Consume(s, 'e') || Consume(s, 'E')) { if (Consume(s, 'e') || Consume(s, 'E')) {
if ( ((parseFlags & kParseNumbersAsStringsFlag) != 0) && ((parseFlags & kParseInsituFlag) == 0) ) { if (!useDouble) {
s.Push( 'e' );
}
if (!useDouble) {
d = static_cast<double>(use64bit ? i64 : i); d = static_cast<double>(use64bit ? i64 : i);
useDouble = true; useDouble = true;
} }
@ -1316,14 +1321,15 @@ private:
cont = handler.RawNumber(str, SizeType(length), false); cont = handler.RawNumber(str, SizeType(length), false);
} }
else { else {
StackStream<typename TargetEncoding::Ch> stackStream(stack_);
SizeType numCharsToCopy = static_cast<SizeType>(s.Length()); SizeType numCharsToCopy = static_cast<SizeType>(s.Length());
StringStream srcStream(s.Pop());
StackStream<typename TargetEncoding::Ch> dstStream(stack_);
while (numCharsToCopy--) { while (numCharsToCopy--) {
Transcoder<SourceEncoding, TargetEncoding>::Transcode(is, stackStream); Transcoder<UTF8<>, TargetEncoding>::Transcode(srcStream, dstStream);
} }
stackStream.Put('\0'); dstStream.Put('\0');
const typename TargetEncoding::Ch* str = stackStream.Pop(); const typename TargetEncoding::Ch* str = dstStream.Pop();
const SizeType length = static_cast<SizeType>(stackStream.Length()) - 1; const SizeType length = static_cast<SizeType>(dstStream.Length()) - 1;
cont = handler.RawNumber(str, SizeType(length), true); cont = handler.RawNumber(str, SizeType(length), true);
} }
} }

5
test/unittest/CMakeLists.txt
View File

@ -38,6 +38,11 @@ if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wall -Wextra -Weffc++ -Wswitch-default -Wfloat-equal") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wall -Wextra -Weffc++ -Wswitch-default -Wfloat-equal")
elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wall -Wextra -Weffc++ -Wswitch-default -Wfloat-equal -Wimplicit-fallthrough -Weverything") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wall -Wextra -Weffc++ -Wswitch-default -Wfloat-equal -Wimplicit-fallthrough -Weverything")
# If the user is running a newer version of Clang that includes the
# -Wdouble-promotion, we will ignore that warning.
if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 3.7)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-double-promotion")
endif()
elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
add_definitions(-D_CRT_SECURE_NO_WARNINGS=1) add_definitions(-D_CRT_SECURE_NO_WARNINGS=1)
endif() endif()

99
test/unittest/readertest.cpp
View File

@ -241,13 +241,13 @@ static void TestParseDouble() {
TEST_DOUBLE(fullPrecision, "0.017976931348623157e+310", 1.7976931348623157e+308); // Max double in another form TEST_DOUBLE(fullPrecision, "0.017976931348623157e+310", 1.7976931348623157e+308); // Max double in another form
// Since // Since
// abs((2^-1022 - 2^-1074) - 2.2250738585072012e-308) = 3.109754131239141401123495768877590405345064751974375599... ¡Á 10^-324 // abs((2^-1022 - 2^-1074) - 2.2250738585072012e-308) = 3.109754131239141401123495768877590405345064751974375599... <EFBFBD><EFBFBD> 10^-324
// abs((2^-1022) - 2.2250738585072012e-308) = 1.830902327173324040642192159804623318305533274168872044... ¡Á 10 ^ -324 // abs((2^-1022) - 2.2250738585072012e-308) = 1.830902327173324040642192159804623318305533274168872044... <EFBFBD><EFBFBD> 10 ^ -324
// So 2.2250738585072012e-308 should round to 2^-1022 = 2.2250738585072014e-308 // So 2.2250738585072012e-308 should round to 2^-1022 = 2.2250738585072014e-308
TEST_DOUBLE(fullPrecision, "2.2250738585072012e-308", 2.2250738585072014e-308); // http://www.exploringbinary.com/java-hangs-when-converting-2-2250738585072012e-308/ TEST_DOUBLE(fullPrecision, "2.2250738585072012e-308", 2.2250738585072014e-308); // http://www.exploringbinary.com/java-hangs-when-converting-2-2250738585072012e-308/
// More closer to normal/subnormal boundary // More closer to normal/subnormal boundary
// boundary = 2^-1022 - 2^-1075 = 2.225073858507201136057409796709131975934819546351645648... ¡Á 10^-308 // boundary = 2^-1022 - 2^-1075 = 2.225073858507201136057409796709131975934819546351645648... <EFBFBD><EFBFBD> 10^-308
TEST_DOUBLE(fullPrecision, "2.22507385850720113605740979670913197593481954635164564e-308", 2.2250738585072009e-308); TEST_DOUBLE(fullPrecision, "2.22507385850720113605740979670913197593481954635164564e-308", 2.2250738585072009e-308);
TEST_DOUBLE(fullPrecision, "2.22507385850720113605740979670913197593481954635164565e-308", 2.2250738585072014e-308); TEST_DOUBLE(fullPrecision, "2.22507385850720113605740979670913197593481954635164565e-308", 2.2250738585072014e-308);
@ -1541,7 +1541,8 @@ struct NumbersAsStringsHandler {
// 'str' is not null-terminated // 'str' is not null-terminated
bool RawNumber(const char* str, SizeType length, bool) { bool RawNumber(const char* str, SizeType length, bool) {
EXPECT_TRUE(str != 0); EXPECT_TRUE(str != 0);
EXPECT_TRUE(strncmp(str, "3.1416", length) == 0); EXPECT_TRUE(expected_len_ == length);
EXPECT_TRUE(strncmp(str, expected_, length) == 0);
return true; return true;
} }
bool String(const char*, SizeType, bool) { return true; } bool String(const char*, SizeType, bool) { return true; }
@ -1550,24 +1551,84 @@ struct NumbersAsStringsHandler {
bool EndObject(SizeType) { return true; } bool EndObject(SizeType) { return true; }
bool StartArray() { return true; } bool StartArray() { return true; }
bool EndArray(SizeType) { return true; } bool EndArray(SizeType) { return true; }
NumbersAsStringsHandler(const char* expected)
: expected_(expected)
, expected_len_(strlen(expected)) {}
const char* expected_;
size_t expected_len_;
}; };
TEST(Reader, NumbersAsStrings) { TEST(Reader, NumbersAsStrings) {
{ {
const char* json = "{ \"pi\": 3.1416 } "; const char* json = "{ \"pi\": 3.1416 } ";
StringStream s(json); StringStream s(json);
NumbersAsStringsHandler h; NumbersAsStringsHandler h("3.1416");
Reader reader; Reader reader;
EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h)); EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
} }
{ {
char* json = StrDup("{ \"pi\": 3.1416 } "); char* json = StrDup("{ \"pi\": 3.1416 } ");
InsituStringStream s(json); InsituStringStream s(json);
NumbersAsStringsHandler h; NumbersAsStringsHandler h("3.1416");
Reader reader; Reader reader;
EXPECT_TRUE(reader.Parse<kParseInsituFlag|kParseNumbersAsStringsFlag>(s, h)); EXPECT_TRUE(reader.Parse<kParseInsituFlag|kParseNumbersAsStringsFlag>(s, h));
free(json); free(json);
} }
{
const char* json = "{ \"gigabyte\": 1.0e9 } ";
StringStream s(json);
NumbersAsStringsHandler h("1.0e9");
Reader reader;
EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
}
{
char* json = StrDup("{ \"gigabyte\": 1.0e9 } ");
InsituStringStream s(json);
NumbersAsStringsHandler h("1.0e9");
Reader reader;
EXPECT_TRUE(reader.Parse<kParseInsituFlag|kParseNumbersAsStringsFlag>(s, h));
free(json);
}
{
const char* json = "{ \"pi\": 314.159e-2 } ";
StringStream s(json);
NumbersAsStringsHandler h("314.159e-2");
Reader reader;
EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
}
{
char* json = StrDup("{ \"gigabyte\": 314.159e-2 } ");
InsituStringStream s(json);
NumbersAsStringsHandler h("314.159e-2");
Reader reader;
EXPECT_TRUE(reader.Parse<kParseInsituFlag|kParseNumbersAsStringsFlag>(s, h));
free(json);
}
{
const char* json = "{ \"negative\": -1.54321 } ";
StringStream s(json);
NumbersAsStringsHandler h("-1.54321");
Reader reader;
EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
}
{
char* json = StrDup("{ \"negative\": -1.54321 } ");
InsituStringStream s(json);
NumbersAsStringsHandler h("-1.54321");
Reader reader;
EXPECT_TRUE(reader.Parse<kParseInsituFlag|kParseNumbersAsStringsFlag>(s, h));
free(json);
}
{
const char* json = "{ \"pi\": 314.159e-2 } ";
std::stringstream ss(json);
IStreamWrapper s(ss);
NumbersAsStringsHandler h("314.159e-2");
Reader reader;
EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
}
} }
template <unsigned extraFlags> template <unsigned extraFlags>