From 1bfa188d18785062179d0e760aca27e7af00b743 Mon Sep 17 00:00:00 2001 From: Milo Yip Date: Fri, 15 Apr 2016 13:50:24 +0800 Subject: [PATCH 01/19] Improve encodings coverage --- test/unittest/encodingstest.cpp | 25 ++++++++++++++++ test/unittest/writertest.cpp | 52 ++++++++++++++++++++++++--------- 2 files changed, 63 insertions(+), 14 deletions(-) diff --git a/test/unittest/encodingstest.cpp b/test/unittest/encodingstest.cpp index b3cbb76..4104880 100644 --- a/test/unittest/encodingstest.cpp +++ b/test/unittest/encodingstest.cpp @@ -423,3 +423,28 @@ TEST(EncodingsTest, UTF32) { } } } + +TEST(EncodingsTest, ASCII) { + StringBuffer os, os2; + for (unsigned codepoint = 0; codepoint < 128; codepoint++) { + os.Clear(); + ASCII<>::Encode(os, codepoint); + const ASCII<>::Ch* encodedStr = os.GetString(); + { + StringStream is(encodedStr); + unsigned decodedCodepoint; + bool result = ASCII<>::Decode(is, &decodedCodepoint); + if (!result || codepoint != decodedCodepoint) + std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl; + } + + // Validate + { + StringStream is(encodedStr); + os2.Clear(); + bool result = ASCII<>::Validate(is, os2); + EXPECT_TRUE(result); + EXPECT_EQ(0, StrCmp(encodedStr, os2.GetString())); + } + } +} diff --git a/test/unittest/writertest.cpp b/test/unittest/writertest.cpp index 238aa79..4e08d7e 100644 --- a/test/unittest/writertest.cpp +++ b/test/unittest/writertest.cpp @@ -18,6 +18,7 @@ #include "rapidjson/reader.h" #include "rapidjson/writer.h" #include "rapidjson/stringbuffer.h" +#include "rapidjson/memorybuffer.h" using namespace rapidjson; @@ -107,35 +108,58 @@ TEST(Writer, Double) { } +// UTF8 -> TargetEncoding -> UTF8 +template +void TestTranscode(const char* json) { + StringStream s(json); + GenericStringBuffer buffer; + Writer, UTF8<>, TargetEncoding> writer(buffer); + Reader reader; + reader.Parse(s, writer); + + StringBuffer buffer2; + Writer writer2(buffer2); + GenericReader > reader2; + GenericStringStream s2(buffer.GetString()); + reader2.Parse(s2, writer2); + + EXPECT_STREQ(json, buffer2.GetString()); +} + TEST(Writer, Transcode) { const char json[] = "{\"hello\":\"world\",\"t\":true,\"f\":false,\"n\":null,\"i\":123,\"pi\":3.1416,\"a\":[1,2,3],\"dollar\":\"\x24\",\"cents\":\"\xC2\xA2\",\"euro\":\"\xE2\x82\xAC\",\"gclef\":\"\xF0\x9D\x84\x9E\"}"; // UTF8 -> UTF16 -> UTF8 - { - StringStream s(json); - StringBuffer buffer; - Writer, UTF8<> > writer(buffer); - GenericReader, UTF16<> > reader; - reader.Parse(s, writer); - EXPECT_STREQ(json, buffer.GetString()); - } + TestTranscode >(json); - // UTF8 -> UTF8 -> ASCII -> UTF8 -> UTF8 + // UTF8 -> ASCII -> UTF8 + TestTranscode >(json); + + // UTF8 -> UTF16 -> UTF8 + TestTranscode >(json); + + // UTF8 -> UTF32 -> UTF8 + TestTranscode >(json); + + // UTF8 -> AutoUTF (UTF16BE) -> UTF8 { StringStream s(json); - StringBuffer buffer; - Writer, ASCII<> > writer(buffer); + MemoryBuffer buffer; + AutoUTFOutputStream os(buffer, kUTF16BE, true); + Writer, UTF8<>, AutoUTF > writer(os); Reader reader; reader.Parse(s, writer); StringBuffer buffer2; Writer writer2(buffer2); - GenericReader, UTF8<> > reader2; - StringStream s2(buffer.GetString()); - reader2.Parse(s2, writer2); + GenericReader, UTF8<> > reader2; + MemoryStream s2(buffer.GetBuffer(), buffer.GetSize()); + AutoUTFInputStream is(s2); + reader2.Parse(is, writer2); EXPECT_STREQ(json, buffer2.GetString()); } + } #include From a8970be54315d342c798234a14ebeb811151a48b Mon Sep 17 00:00:00 2001 From: Milo Yip Date: Fri, 15 Apr 2016 14:15:08 +0800 Subject: [PATCH 02/19] Improve UTF8::Encode() coverage via writing to AutoUTF --- test/unittest/writertest.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/unittest/writertest.cpp b/test/unittest/writertest.cpp index 4e08d7e..7db1c62 100644 --- a/test/unittest/writertest.cpp +++ b/test/unittest/writertest.cpp @@ -141,11 +141,12 @@ TEST(Writer, Transcode) { // UTF8 -> UTF32 -> UTF8 TestTranscode >(json); - // UTF8 -> AutoUTF (UTF16BE) -> UTF8 - { + // UTF8 -> AutoUTF -> UTF8 + UTFType types[] = { kUTF8, kUTF16LE , kUTF16BE, kUTF32LE , kUTF32BE }; + for (size_t i = 0; i < 5; i++) { StringStream s(json); MemoryBuffer buffer; - AutoUTFOutputStream os(buffer, kUTF16BE, true); + AutoUTFOutputStream os(buffer, types[i], true); Writer, UTF8<>, AutoUTF > writer(os); Reader reader; reader.Parse(s, writer); From 8f9ff88c29bfc294a7905751020a1b42bcaadb54 Mon Sep 17 00:00:00 2001 From: Milo Yip Date: Fri, 15 Apr 2016 17:03:23 +0800 Subject: [PATCH 03/19] Add Writer. ScanWriteUnescapedString to try to improve coverage --- test/unittest/writertest.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/unittest/writertest.cpp b/test/unittest/writertest.cpp index 7db1c62..cd0a32e 100644 --- a/test/unittest/writertest.cpp +++ b/test/unittest/writertest.cpp @@ -95,6 +95,18 @@ TEST(Writer, String) { #endif } +TEST(Writer, ScanWriteUnescapedString) { + const char json[] = "[\" \\\"\"]"; + char buffer2[sizeof(json) + 32]; + + // Use different offset to test different alignments + for (int i = 0; i < 32; i++) { + char* p = buffer2 + i; + memcpy(p, json, sizeof(json)); + TEST_ROUNDTRIP(p); + } +} + TEST(Writer, Double) { TEST_ROUNDTRIP("[1.2345,1.2345678,0.123456789012,1234567.8]"); TEST_ROUNDTRIP("0.0"); From 8fcc65bf581b6da1c3fc04a5c39222f2dbdce08f Mon Sep 17 00:00:00 2001 From: Milo Yip Date: Fri, 15 Apr 2016 19:51:50 +0800 Subject: [PATCH 04/19] Adjust ScanWriteUnescapedString test case --- test/unittest/writertest.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/unittest/writertest.cpp b/test/unittest/writertest.cpp index cd0a32e..af09f8b 100644 --- a/test/unittest/writertest.cpp +++ b/test/unittest/writertest.cpp @@ -96,7 +96,8 @@ TEST(Writer, String) { } TEST(Writer, ScanWriteUnescapedString) { - const char json[] = "[\" \\\"\"]"; + const char json[] = "[\" \\\"0123456789ABCDEF\"]"; + // ^ scanning stops here. char buffer2[sizeof(json) + 32]; // Use different offset to test different alignments From 3da4afd259667ce57b5b41db4a0e589ba861d1a9 Mon Sep 17 00:00:00 2001 From: Milo Yip Date: Sat, 16 Apr 2016 15:19:34 +0800 Subject: [PATCH 05/19] Another trial on writer coverage --- test/unittest/simdtest.cpp | 38 ++++++++++++++++++++---------------- test/unittest/writertest.cpp | 6 +++++- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/test/unittest/simdtest.cpp b/test/unittest/simdtest.cpp index 1b6fcef..84f8cb0 100644 --- a/test/unittest/simdtest.cpp +++ b/test/unittest/simdtest.cpp @@ -105,24 +105,28 @@ struct ScanCopyUnescapedStringHandler : BaseReaderHandler, ScanCopyUnesca template void TestScanCopyUnescapedString() { - for (size_t step = 0; step < 1024; step++) { - char json[1024 + 5]; - char *p = json; - *p ++= '\"'; - for (size_t i = 0; i < step; i++) - *p++= "ABCD"[i % 4]; - *p++ = '\\'; - *p++ = '\\'; - *p++ = '\"'; - *p++ = '\0'; + char buffer[1024 + 5 + 32]; - StreamType s(json); - Reader reader; - ScanCopyUnescapedStringHandler h; - reader.Parse(s, h); - EXPECT_TRUE(memcmp(h.buffer, json + 1, step) == 0); - EXPECT_EQ('\\', h.buffer[step]); // escaped - EXPECT_EQ('\0', h.buffer[step + 1]); + for (size_t offset = 0; offset < 32; offset++) { + for (size_t step = 0; step < 1024; step++) { + char* json = buffer + offset; + char *p = json; + *p++ = '\"'; + for (size_t i = 0; i < step; i++) + *p++ = "ABCD"[i % 4]; + *p++ = '\\'; + *p++ = '\\'; + *p++ = '\"'; + *p++ = '\0'; + + StreamType s(json); + Reader reader; + ScanCopyUnescapedStringHandler h; + reader.Parse(s, h); + EXPECT_TRUE(memcmp(h.buffer, json + 1, step) == 0); + EXPECT_EQ('\\', h.buffer[step]); // escaped + EXPECT_EQ('\0', h.buffer[step + 1]); + } } } diff --git a/test/unittest/writertest.cpp b/test/unittest/writertest.cpp index af09f8b..9c68c53 100644 --- a/test/unittest/writertest.cpp +++ b/test/unittest/writertest.cpp @@ -448,6 +448,10 @@ TEST(Writer, NaN) { StringBuffer buffer; Writer writer(buffer); EXPECT_FALSE(writer.Double(nan)); + + GenericStringBuffer > buffer2; + Writer > > writer2(buffer2); + EXPECT_FALSE(writer2.Double(nan)); } TEST(Writer, Inf) { @@ -456,7 +460,7 @@ TEST(Writer, Inf) { StringBuffer buffer; { Writer writer(buffer); - EXPECT_FALSE(writer.Double(inf)); + EXPECT_FALSE(writer.Double(inf)); } { Writer writer(buffer); From a6f9cb85abfb5ae0a508c7a1119497bc223c36cc Mon Sep 17 00:00:00 2001 From: Milo Yip Date: Sat, 16 Apr 2016 16:11:34 +0800 Subject: [PATCH 06/19] Third trial on writer coverage --- test/unittest/simdtest.cpp | 79 ++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 38 deletions(-) diff --git a/test/unittest/simdtest.cpp b/test/unittest/simdtest.cpp index 84f8cb0..a81b4c1 100644 --- a/test/unittest/simdtest.cpp +++ b/test/unittest/simdtest.cpp @@ -136,47 +136,50 @@ TEST(SIMD, SIMD_SUFFIX(ScanCopyUnescapedString)) { } TEST(SIMD, SIMD_SUFFIX(ScanWriteUnescapedString)) { - for (size_t step = 0; step < 1024; step++) { - char s[2048 + 1]; - char *p = s; - for (size_t i = 0; i < step; i++) - *p++= "ABCD"[i % 4]; - char escape = "\0\n\\\""[step % 4]; - *p++ = escape; - for (size_t i = 0; i < step; i++) - *p++= "ABCD"[i % 4]; + char buffer[2048 + 1 + 32]; + for (size_t offset = 0; offset < 32; offset++) { + for (size_t step = 0; step < 1024; step++) { + char* s = buffer + offset; + char* p = s; + for (size_t i = 0; i < step; i++) + *p++ = "ABCD"[i % 4]; + char escape = "\0\n\\\""[step % 4]; + *p++ = escape; + for (size_t i = 0; i < step; i++) + *p++ = "ABCD"[i % 4]; - StringBuffer sb; - Writer writer(sb); - writer.String(s, SizeType(step * 2 + 1)); - const char* q = sb.GetString(); - EXPECT_EQ('\"', *q++); - for (size_t i = 0; i < step; i++) - EXPECT_EQ("ABCD"[i % 4], *q++); - if (escape == '\0') { - EXPECT_EQ('\\', *q++); - EXPECT_EQ('u', *q++); - EXPECT_EQ('0', *q++); - EXPECT_EQ('0', *q++); - EXPECT_EQ('0', *q++); - EXPECT_EQ('0', *q++); - } - else if (escape == '\n') { - EXPECT_EQ('\\', *q++); - EXPECT_EQ('n', *q++); - } - else if (escape == '\\') { - EXPECT_EQ('\\', *q++); - EXPECT_EQ('\\', *q++); - } - else if (escape == '\"') { - EXPECT_EQ('\\', *q++); + StringBuffer sb; + Writer writer(sb); + writer.String(s, SizeType(step * 2 + 1)); + const char* q = sb.GetString(); EXPECT_EQ('\"', *q++); + for (size_t i = 0; i < step; i++) + EXPECT_EQ("ABCD"[i % 4], *q++); + if (escape == '\0') { + EXPECT_EQ('\\', *q++); + EXPECT_EQ('u', *q++); + EXPECT_EQ('0', *q++); + EXPECT_EQ('0', *q++); + EXPECT_EQ('0', *q++); + EXPECT_EQ('0', *q++); + } + else if (escape == '\n') { + EXPECT_EQ('\\', *q++); + EXPECT_EQ('n', *q++); + } + else if (escape == '\\') { + EXPECT_EQ('\\', *q++); + EXPECT_EQ('\\', *q++); + } + else if (escape == '\"') { + EXPECT_EQ('\\', *q++); + EXPECT_EQ('\"', *q++); + } + for (size_t i = 0; i < step; i++) + EXPECT_EQ("ABCD"[i % 4], *q++); + EXPECT_EQ('\"', *q++); + EXPECT_EQ('\0', *q++); } - for (size_t i = 0; i < step; i++) - EXPECT_EQ("ABCD"[i % 4], *q++); - EXPECT_EQ('\"', *q++); - EXPECT_EQ('\0', *q++); } } From bdfa0447ece96ea0bee5824ba656bdec69d1c74f Mon Sep 17 00:00:00 2001 From: Milo Yip Date: Sat, 16 Apr 2016 21:44:33 +0800 Subject: [PATCH 07/19] Add test cases for ScanCopyUnescapedString --- test/unittest/simdtest.cpp | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/test/unittest/simdtest.cpp b/test/unittest/simdtest.cpp index a81b4c1..b01b559 100644 --- a/test/unittest/simdtest.cpp +++ b/test/unittest/simdtest.cpp @@ -100,13 +100,15 @@ struct ScanCopyUnescapedStringHandler : BaseReaderHandler, ScanCopyUnesca memcpy(buffer, str, length + 1); return true; } - char buffer[1024 + 5]; + char buffer[1024 + 5 + 32]; }; template void TestScanCopyUnescapedString() { char buffer[1024 + 5 + 32]; + char backup[1024 + 5 + 32]; + // Test "ABCDABCD...\\" for (size_t offset = 0; offset < 32; offset++) { for (size_t step = 0; step < 1024; step++) { char* json = buffer + offset; @@ -118,16 +120,41 @@ void TestScanCopyUnescapedString() { *p++ = '\\'; *p++ = '\"'; *p++ = '\0'; + strcpy(backup, json); // insitu parsing will overwrite buffer, so need to backup first StreamType s(json); Reader reader; ScanCopyUnescapedStringHandler h; reader.Parse(s, h); - EXPECT_TRUE(memcmp(h.buffer, json + 1, step) == 0); + EXPECT_TRUE(memcmp(h.buffer, backup + 1, step) == 0); EXPECT_EQ('\\', h.buffer[step]); // escaped EXPECT_EQ('\0', h.buffer[step + 1]); } } + + // Test "\\ABCDABCD..." + for (size_t offset = 0; offset < 32; offset++) { + for (size_t step = 0; step < 1024; step++) { + char* json = buffer + offset; + char *p = json; + *p++ = '\"'; + *p++ = '\\'; + *p++ = '\\'; + for (size_t i = 0; i < step; i++) + *p++ = "ABCD"[i % 4]; + *p++ = '\"'; + *p++ = '\0'; + strcpy(backup, json); // insitu parsing will overwrite buffer, so need to backup first + + StreamType s(json); + Reader reader; + ScanCopyUnescapedStringHandler h; + reader.Parse(s, h); + EXPECT_TRUE(memcmp(h.buffer + 1, backup + 3, step) == 0); + EXPECT_EQ('\\', h.buffer[0]); // escaped + EXPECT_EQ('\0', h.buffer[step + 1]); + } + } } TEST(SIMD, SIMD_SUFFIX(ScanCopyUnescapedString)) { From fdd443120f753d375fa2d71260c43abada40a6da Mon Sep 17 00:00:00 2001 From: Milo Yip Date: Sat, 16 Apr 2016 22:09:23 +0800 Subject: [PATCH 08/19] Move break into same line to make coverage happy --- include/rapidjson/reader.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h index 8882a5d..243e0d2 100644 --- a/include/rapidjson/reader.h +++ b/include/rapidjson/reader.h @@ -635,8 +635,7 @@ private: RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell()); return; default: - RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); - break; + RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); break; // This useless break is only for making warning and coverage happy } if (parseFlags & kParseTrailingCommasFlag) { From c71825f80ea2eb0f40efc7494361f5fde81eb642 Mon Sep 17 00:00:00 2001 From: Milo Yip Date: Sat, 16 Apr 2016 22:14:38 +0800 Subject: [PATCH 09/19] Improve Value::IsFloat() coverage --- test/unittest/valuetest.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/unittest/valuetest.cpp b/test/unittest/valuetest.cpp index aac0a44..feec049 100644 --- a/test/unittest/valuetest.cpp +++ b/test/unittest/valuetest.cpp @@ -402,6 +402,7 @@ TEST(Value, Int) { EXPECT_TRUE(x.IsUint64()); EXPECT_FALSE(x.IsDouble()); + EXPECT_FALSE(x.IsFloat()); EXPECT_FALSE(x.IsNull()); EXPECT_FALSE(x.IsBool()); EXPECT_FALSE(x.IsFalse()); @@ -456,6 +457,7 @@ TEST(Value, Uint) { EXPECT_NEAR(1234.0, x.GetDouble(), 0.0); // Number can always be cast as double but !IsDouble(). EXPECT_FALSE(x.IsDouble()); + EXPECT_FALSE(x.IsFloat()); EXPECT_FALSE(x.IsNull()); EXPECT_FALSE(x.IsBool()); EXPECT_FALSE(x.IsFalse()); @@ -500,6 +502,7 @@ TEST(Value, Int64) { EXPECT_TRUE(x.IsUint64()); EXPECT_FALSE(x.IsDouble()); + EXPECT_FALSE(x.IsFloat()); EXPECT_FALSE(x.IsNull()); EXPECT_FALSE(x.IsBool()); EXPECT_FALSE(x.IsFalse()); @@ -561,6 +564,7 @@ TEST(Value, Uint64) { EXPECT_TRUE(x.IsUint64()); EXPECT_FALSE(x.IsDouble()); + EXPECT_FALSE(x.IsFloat()); EXPECT_FALSE(x.IsNull()); EXPECT_FALSE(x.IsBool()); EXPECT_FALSE(x.IsFalse()); From ecd8fa3437d5f1f0560cba296f55ceeddb58baff Mon Sep 17 00:00:00 2001 From: Milo Yip Date: Sat, 16 Apr 2016 23:04:40 +0800 Subject: [PATCH 10/19] Improve coverage of regex --- include/rapidjson/internal/regex.h | 12 +++++------- test/unittest/regextest.cpp | 11 +++++++++++ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/include/rapidjson/internal/regex.h b/include/rapidjson/internal/regex.h index d317daa..c206294 100644 --- a/include/rapidjson/internal/regex.h +++ b/include/rapidjson/internal/regex.h @@ -375,14 +375,14 @@ private: bool Eval(Stack& operandStack, Operator op) { switch (op) { case kConcatenation: - if (operandStack.GetSize() >= sizeof(Frag) * 2) { + RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag) * 2); + { Frag e2 = *operandStack.template Pop(1); Frag e1 = *operandStack.template Pop(1); Patch(e1.out, e2.start); *operandStack.template Push() = Frag(e1.start, e2.out, Min(e1.minIndex, e2.minIndex)); - return true; } - return false; + return true; case kAlternation: if (operandStack.GetSize() >= sizeof(Frag) * 2) { @@ -430,8 +430,7 @@ private: bool EvalQuantifier(Stack& operandStack, unsigned n, unsigned m) { RAPIDJSON_ASSERT(n <= m); - if (operandStack.GetSize() < sizeof(Frag)) - return false; + RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag)); if (n == 0) { if (m == 0) // a{0} not support @@ -647,8 +646,7 @@ private: // Return whether the added states is a match state bool AddState(Stack& l, SizeType index) const { - if (index == kRegexInvalidState) - return true; + RAPIDJSON_ASSERT(index != kRegexInvalidState); const State& s = GetState(index); if (s.out1 != kRegexInvalidState) { // Split diff --git a/test/unittest/regextest.cpp b/test/unittest/regextest.cpp index e3371d1..b497df6 100644 --- a/test/unittest/regextest.cpp +++ b/test/unittest/regextest.cpp @@ -17,6 +17,14 @@ using namespace rapidjson::internal; +TEST(Regex, Single) { + Regex re("a"); + ASSERT_TRUE(re.IsValid()); + EXPECT_TRUE(re.Match("a")); + EXPECT_FALSE(re.Match("")); + EXPECT_FALSE(re.Match("b")); +} + TEST(Regex, Concatenation) { Regex re("abc"); ASSERT_TRUE(re.IsValid()); @@ -560,6 +568,9 @@ TEST(Regex, Invalid) { TEST_INVALID("a{1,0}"); TEST_INVALID("a{-1,0}"); TEST_INVALID("a{-1,1}"); + TEST_INVALID("a{4294967296}"); // overflow of unsigned + TEST_INVALID("a{1a}"); + TEST_INVALID("["); TEST_INVALID("[]"); TEST_INVALID("[^]"); TEST_INVALID("[\\a]"); From 26e69ffde95ba4773ab06db6457b78f308716f4b Mon Sep 17 00:00:00 2001 From: Milo Yip Date: Sun, 17 Apr 2016 00:48:02 +0800 Subject: [PATCH 11/19] Fix a bug in schema minimum/maximum keywords for 64-bit integer --- include/rapidjson/schema.h | 9 +++++ test/unittest/schematest.cpp | 74 +++++++++++++++++++++++++++++++++++- 2 files changed, 82 insertions(+), 1 deletion(-) diff --git a/include/rapidjson/schema.h b/include/rapidjson/schema.h index e12e7d2..5efbf24 100644 --- a/include/rapidjson/schema.h +++ b/include/rapidjson/schema.h @@ -1108,6 +1108,9 @@ private: if (exclusiveMinimum_ ? i <= minimum_.GetInt64() : i < minimum_.GetInt64()) RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); } + else if (minimum_.IsUint64()) { + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); // i <= max(int64_t) < minimum.GetUint64() + } else if (!CheckDoubleMinimum(context, static_cast(i))) return false; } @@ -1117,6 +1120,8 @@ private: if (exclusiveMaximum_ ? i >= maximum_.GetInt64() : i > maximum_.GetInt64()) RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); } + else if (maximum_.IsUint64()) + /* do nothing */; // i <= max(int64_t) < maximum_.GetUint64() else if (!CheckDoubleMaximum(context, static_cast(i))) return false; } @@ -1142,6 +1147,8 @@ private: if (exclusiveMinimum_ ? i <= minimum_.GetUint64() : i < minimum_.GetUint64()) RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); } + else if (minimum_.IsInt64()) + /* do nothing */; // i >= 0 > minimum.Getint64() else if (!CheckDoubleMinimum(context, static_cast(i))) return false; } @@ -1151,6 +1158,8 @@ private: if (exclusiveMaximum_ ? i >= maximum_.GetUint64() : i > maximum_.GetUint64()) RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); } + else if (maximum_.IsInt64()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); // i >= 0 > maximum_ else if (!CheckDoubleMaximum(context, static_cast(i))) return false; } diff --git a/test/unittest/schematest.cpp b/test/unittest/schematest.cpp index 7182ad2..23aac0e 100644 --- a/test/unittest/schematest.cpp +++ b/test/unittest/schematest.cpp @@ -51,6 +51,10 @@ TEST(SchemaValidator, Hasher) { TEST_HASHER("false", "null", false); TEST_HASHER("1", "1", true); + TEST_HASHER("2147483648", "2147483648", true); // 2^31 can only be fit in unsigned + TEST_HASHER("-2147483649", "-2147483649", true); // -2^31 - 1 can only be fit in int64_t + TEST_HASHER("2147483648", "2147483648", true); // 2^31 can only be fit in unsigned + TEST_HASHER("4294967296", "4294967296", true); // 2^32 can only be fit in int64_t TEST_HASHER("1.5", "1.5", true); TEST_HASHER("1", "1.0", true); TEST_HASHER("1", "-1", false); @@ -316,6 +320,10 @@ TEST(SchemaValidator, String) { VALIDATE(s, "\"I'm a string\"", true); INVALIDATE(s, "42", "", "type", ""); + INVALIDATE(s, "2147483648", "", "type", ""); // 2^31 can only be fit in unsigned + INVALIDATE(s, "-2147483649", "", "type", ""); // -2^31 - 1 can only be fit in int64_t + INVALIDATE(s, "4294967296", "", "type", ""); // 2^32 can only be fit in int64_t + INVALIDATE(s, "3.1415926", "", "type", ""); } TEST(SchemaValidator, String_LengthRange) { @@ -340,6 +348,16 @@ TEST(SchemaValidator, String_Pattern) { INVALIDATE(s, "\"(888)555-1212 ext. 532\"", "", "pattern", ""); INVALIDATE(s, "\"(800)FLOWERS\"", "", "pattern", ""); } + +TEST(SchemaValidator, String_Pattern_Invalid) { + Document sd; + sd.Parse("{\"type\":\"string\",\"pattern\":\"a{0}\"}"); // TODO: report regex is invalid somehow + SchemaDocument s(sd); + + VALIDATE(s, "\"\"", true); + VALIDATE(s, "\"a\"", true); + VALIDATE(s, "\"aa\"", true); +} #endif TEST(SchemaValidator, Integer) { @@ -349,6 +367,10 @@ TEST(SchemaValidator, Integer) { VALIDATE(s, "42", true); VALIDATE(s, "-1", true); + VALIDATE(s, "2147483648", true); // 2^31 can only be fit in unsigned + VALIDATE(s, "-2147483649", true); // -2^31 - 1 can only be fit in int64_t + VALIDATE(s, "2147483648", true); // 2^31 can only be fit in unsigned + VALIDATE(s, "4294967296", true); // 2^32 can only be fit in int64_t INVALIDATE(s, "3.1415926", "", "type", ""); INVALIDATE(s, "\"42\"", "", "type", ""); } @@ -368,11 +390,34 @@ TEST(SchemaValidator, Integer_Range) { TEST(SchemaValidator, Integer_Range64Boundary) { Document sd; - sd.Parse("{\"type\":\"integer\",\"minimum\":-9223372036854775807,\"maximum\":18446744073709551614}"); + sd.Parse("{\"type\":\"integer\",\"minimum\":-9223372036854775807,\"maximum\":9223372036854775806}"); SchemaDocument s(sd); INVALIDATE(s, "-9223372036854775808", "", "minimum", ""); VALIDATE(s, "-9223372036854775807", true); + VALIDATE(s, "-2147483648", true); // int min + VALIDATE(s, "0", true); + VALIDATE(s, "2147483647", true); // int max + VALIDATE(s, "2147483648", true); // unsigned first + VALIDATE(s, "4294967296", true); // unsigned max + VALIDATE(s, "9223372036854775806", true); + INVALIDATE(s, "9223372036854775807", "", "maximum", ""); + INVALIDATE(s, "18446744073709551615", "", "maximum", ""); // uint64_t max +} + +TEST(SchemaValidator, Integer_RangeU64Boundary) { + Document sd; + sd.Parse("{\"type\":\"integer\",\"minimum\":9223372036854775808,\"maximum\":18446744073709551614}"); + SchemaDocument s(sd); + + INVALIDATE(s, "-9223372036854775808", "", "minimum", ""); + INVALIDATE(s, "9223372036854775807", "", "minimum", ""); + INVALIDATE(s, "-2147483648", "", "minimum", ""); // int min + INVALIDATE(s, "0", "", "minimum", ""); + INVALIDATE(s, "2147483647", "", "minimum", ""); // int max + INVALIDATE(s, "2147483648", "", "minimum", ""); // unsigned first + INVALIDATE(s, "4294967296", "", "minimum", ""); // unsigned max + VALIDATE(s, "9223372036854775808", true); VALIDATE(s, "18446744073709551614", true); INVALIDATE(s, "18446744073709551615", "", "maximum", ""); } @@ -418,10 +463,37 @@ TEST(SchemaValidator, Number_Range) { INVALIDATE(s, "-1", "", "minimum", ""); VALIDATE(s, "0", true); + VALIDATE(s, "0.1", true); VALIDATE(s, "10", true); VALIDATE(s, "99", true); + VALIDATE(s, "99.9", true); INVALIDATE(s, "100", "", "maximum", ""); + INVALIDATE(s, "100.0", "", "maximum", ""); + INVALIDATE(s, "101.5", "", "maximum", ""); +} + +TEST(SchemaValidator, Number_RangeDouble) { + Document sd; + sd.Parse("{\"type\":\"number\",\"minimum\":0.1,\"maximum\":100.1,\"exclusiveMaximum\":true}"); + SchemaDocument s(sd); + + INVALIDATE(s, "-9223372036854775808", "", "minimum", ""); + INVALIDATE(s, "-2147483648", "", "minimum", ""); // int min + INVALIDATE(s, "-1", "", "minimum", ""); + VALIDATE(s, "0.1", true); + VALIDATE(s, "10", true); + VALIDATE(s, "99", true); + VALIDATE(s, "100", true); INVALIDATE(s, "101", "", "maximum", ""); + INVALIDATE(s, "101.5", "", "maximum", ""); + INVALIDATE(s, "18446744073709551614", "", "maximum", ""); + INVALIDATE(s, "18446744073709551615", "", "maximum", ""); + INVALIDATE(s, "2147483647", "", "maximum", ""); // int max + INVALIDATE(s, "2147483648", "", "maximum", ""); // unsigned first + INVALIDATE(s, "4294967296", "", "maximum", ""); // unsigned max + INVALIDATE(s, "9223372036854775808", "", "maximum", ""); + INVALIDATE(s, "18446744073709551614", "", "maximum", ""); + INVALIDATE(s, "18446744073709551615", "", "maximum", ""); } TEST(SchemaValidator, Number_MultipleOf) { From e7149d665941068ccf8c565e77495521331cf390 Mon Sep 17 00:00:00 2001 From: Milo Yip Date: Sun, 17 Apr 2016 00:58:32 +0800 Subject: [PATCH 12/19] Fix memory leak for invalid regex --- include/rapidjson/schema.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/rapidjson/schema.h b/include/rapidjson/schema.h index 5efbf24..4fdb854 100644 --- a/include/rapidjson/schema.h +++ b/include/rapidjson/schema.h @@ -1006,6 +1006,7 @@ private: RegexType* r = new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString()); if (!r->IsValid()) { r->~RegexType(); + AllocatorType::Free(r); r = 0; } return r; From 954f80872d885deca3841a5669b69ce34d286540 Mon Sep 17 00:00:00 2001 From: Milo Yip Date: Sun, 17 Apr 2016 01:55:55 +0800 Subject: [PATCH 13/19] Improve schema minimum/maximum/multipleOf coverage --- test/unittest/schematest.cpp | 50 +++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/test/unittest/schematest.cpp b/test/unittest/schematest.cpp index 23aac0e..4ceacd6 100644 --- a/test/unittest/schematest.cpp +++ b/test/unittest/schematest.cpp @@ -55,6 +55,7 @@ TEST(SchemaValidator, Hasher) { TEST_HASHER("-2147483649", "-2147483649", true); // -2^31 - 1 can only be fit in int64_t TEST_HASHER("2147483648", "2147483648", true); // 2^31 can only be fit in unsigned TEST_HASHER("4294967296", "4294967296", true); // 2^32 can only be fit in int64_t + TEST_HASHER("9223372036854775808", "9223372036854775808", true); // 2^63 can only be fit in uint64_t TEST_HASHER("1.5", "1.5", true); TEST_HASHER("1", "1.0", true); TEST_HASHER("1", "-1", false); @@ -399,7 +400,7 @@ TEST(SchemaValidator, Integer_Range64Boundary) { VALIDATE(s, "0", true); VALIDATE(s, "2147483647", true); // int max VALIDATE(s, "2147483648", true); // unsigned first - VALIDATE(s, "4294967296", true); // unsigned max + VALIDATE(s, "4294967295", true); // unsigned max VALIDATE(s, "9223372036854775806", true); INVALIDATE(s, "9223372036854775807", "", "maximum", ""); INVALIDATE(s, "18446744073709551615", "", "maximum", ""); // uint64_t max @@ -416,7 +417,7 @@ TEST(SchemaValidator, Integer_RangeU64Boundary) { INVALIDATE(s, "0", "", "minimum", ""); INVALIDATE(s, "2147483647", "", "minimum", ""); // int max INVALIDATE(s, "2147483648", "", "minimum", ""); // unsigned first - INVALIDATE(s, "4294967296", "", "minimum", ""); // unsigned max + INVALIDATE(s, "4294967295", "", "minimum", ""); // unsigned max VALIDATE(s, "9223372036854775808", true); VALIDATE(s, "18446744073709551614", true); INVALIDATE(s, "18446744073709551615", "", "maximum", ""); @@ -472,6 +473,26 @@ TEST(SchemaValidator, Number_Range) { INVALIDATE(s, "101.5", "", "maximum", ""); } +TEST(SchemaValidator, Number_RangeInt) { + Document sd; + sd.Parse("{\"type\":\"number\",\"minimum\":-100,\"maximum\":-1,\"exclusiveMaximum\":true}"); + SchemaDocument s(sd); + + INVALIDATE(s, "-101", "", "minimum", ""); + INVALIDATE(s, "-100.1", "", "minimum", ""); + VALIDATE(s, "-100", true); + VALIDATE(s, "-2", true); + INVALIDATE(s, "-1", "", "maximum", ""); + INVALIDATE(s, "-0.9", "", "maximum", ""); + INVALIDATE(s, "0", "", "maximum", ""); + INVALIDATE(s, "2147483647", "", "maximum", ""); // int max + INVALIDATE(s, "2147483648", "", "maximum", ""); // unsigned first + INVALIDATE(s, "4294967295", "", "maximum", ""); // unsigned max + INVALIDATE(s, "9223372036854775808", "", "maximum", ""); + INVALIDATE(s, "18446744073709551614", "", "maximum", ""); + INVALIDATE(s, "18446744073709551615", "", "maximum", ""); +} + TEST(SchemaValidator, Number_RangeDouble) { Document sd; sd.Parse("{\"type\":\"number\",\"minimum\":0.1,\"maximum\":100.1,\"exclusiveMaximum\":true}"); @@ -490,12 +511,28 @@ TEST(SchemaValidator, Number_RangeDouble) { INVALIDATE(s, "18446744073709551615", "", "maximum", ""); INVALIDATE(s, "2147483647", "", "maximum", ""); // int max INVALIDATE(s, "2147483648", "", "maximum", ""); // unsigned first - INVALIDATE(s, "4294967296", "", "maximum", ""); // unsigned max + INVALIDATE(s, "4294967295", "", "maximum", ""); // unsigned max INVALIDATE(s, "9223372036854775808", "", "maximum", ""); INVALIDATE(s, "18446744073709551614", "", "maximum", ""); INVALIDATE(s, "18446744073709551615", "", "maximum", ""); } +TEST(SchemaValidator, Number_RangeDoubleU64Boundary) { + Document sd; + sd.Parse("{\"type\":\"number\",\"minimum\":9223372036854775808.0,\"maximum\":18446744073709550000.0}"); + SchemaDocument s(sd); + + INVALIDATE(s, "-9223372036854775808", "", "minimum", ""); + INVALIDATE(s, "-2147483648", "", "minimum", ""); // int min + INVALIDATE(s, "0", "", "minimum", ""); + INVALIDATE(s, "2147483647", "", "minimum", ""); // int max + INVALIDATE(s, "2147483648", "", "minimum", ""); // unsigned first + INVALIDATE(s, "4294967295", "", "minimum", ""); // unsigned max + VALIDATE(s, "9223372036854775808", true); + VALIDATE(s, "18446744073709540000", true); + INVALIDATE(s, "18446744073709551615", "", "maximum", ""); +} + TEST(SchemaValidator, Number_MultipleOf) { Document sd; sd.Parse("{\"type\":\"number\",\"multipleOf\":10.0}"); @@ -506,6 +543,13 @@ TEST(SchemaValidator, Number_MultipleOf) { VALIDATE(s, "-10", true); VALIDATE(s, "20", true); INVALIDATE(s, "23", "", "multipleOf", ""); + INVALIDATE(s, "-2147483648", "", "multipleOf", ""); // int min + VALIDATE(s, "-2147483640", true); + INVALIDATE(s, "2147483647", "", "multipleOf", ""); // int max + INVALIDATE(s, "2147483648", "", "multipleOf", ""); // unsigned first + VALIDATE(s, "2147483650", true); + INVALIDATE(s, "4294967295", "", "multipleOf", ""); // unsigned max + VALIDATE(s, "4294967300", true); } TEST(SchemaValidator, Number_MultipleOfOne) { From ed6fdb6d78d32c25c7e6482dbe3e2b0fe49f8bff Mon Sep 17 00:00:00 2001 From: Milo Yip Date: Sun, 17 Apr 2016 09:09:25 +0800 Subject: [PATCH 14/19] Improve coverage for SchemaValidator:::AppendToken() --- test/unittest/schematest.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/unittest/schematest.cpp b/test/unittest/schematest.cpp index 4ceacd6..ff8b5d6 100644 --- a/test/unittest/schematest.cpp +++ b/test/unittest/schematest.cpp @@ -960,6 +960,19 @@ TEST(SchemaValidator, AllOf_Nested) { INVALIDATE(s, "123", "", "allOf", ""); } +TEST(SchemaValidator, EscapedPointer) { + Document sd; + sd.Parse( + "{" + " \"type\": \"object\"," + " \"properties\": {" + " \"~/\": { \"type\": \"number\" }" + " }" + "}"); + SchemaDocument s(sd); + INVALIDATE(s, "{\"~/\":true}", "/properties/~0~1", "type", "/~0~1"); +} + template static char* ReadFile(const char* filename, Allocator& allocator) { const char *paths[] = { From cb2f340d55a9114ab34bb08b9cf0187fe8b83a81 Mon Sep 17 00:00:00 2001 From: Milo Yip Date: Sun, 17 Apr 2016 09:11:00 +0800 Subject: [PATCH 15/19] Remove ISchemaStateFactory::ReallocState() --- include/rapidjson/schema.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/include/rapidjson/schema.h b/include/rapidjson/schema.h index 4fdb854..f7a5237 100644 --- a/include/rapidjson/schema.h +++ b/include/rapidjson/schema.h @@ -159,7 +159,6 @@ public: virtual uint64_t GetHashCode(void* hasher) = 0; virtual void DestroryHasher(void* hasher) = 0; virtual void* MallocState(size_t size) = 0; - virtual void* ReallocState(void* originalPtr, size_t originalSize, size_t newSize) = 0; virtual void FreeState(void* p) = 0; }; @@ -1776,10 +1775,6 @@ RAPIDJSON_MULTILINEMACRO_END return GetStateAllocator().Malloc(size); } - virtual void* ReallocState(void* originalPtr, size_t originalSize, size_t newSize) { - return GetStateAllocator().Realloc(originalPtr, originalSize, newSize); - } - virtual void FreeState(void* p) { return StateAllocator::Free(p); } From ba0a137b9c99db1af641575ae589d2c757146c31 Mon Sep 17 00:00:00 2001 From: Milo Yip Date: Sun, 17 Apr 2016 09:17:05 +0800 Subject: [PATCH 16/19] Remove unnecessary code in GenericSchemaDocument::CreateSchemaRecursive() --- include/rapidjson/schema.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/rapidjson/schema.h b/include/rapidjson/schema.h index f7a5237..33b6a10 100644 --- a/include/rapidjson/schema.h +++ b/include/rapidjson/schema.h @@ -1437,8 +1437,6 @@ private: const SchemaType* s = GetSchema(pointer); if (!s) CreateSchema(schema, pointer, v, document); - else if (schema) - *schema = s; for (typename ValueType::ConstMemberIterator itr = v.MemberBegin(); itr != v.MemberEnd(); ++itr) CreateSchemaRecursive(0, pointer.Append(itr->name, allocator_), itr->value, document); From a28e4befed0ee03ab64843695c43358a04f0d05e Mon Sep 17 00:00:00 2001 From: Milo Yip Date: Sun, 17 Apr 2016 09:34:04 +0800 Subject: [PATCH 17/19] Improve coverage of Regex by removing default case. --- include/rapidjson/internal/regex.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/include/rapidjson/internal/regex.h b/include/rapidjson/internal/regex.h index c206294..c0a3ec5 100644 --- a/include/rapidjson/internal/regex.h +++ b/include/rapidjson/internal/regex.h @@ -413,7 +413,8 @@ private: } return false; - case kOneOrMore: + default: + RAPIDJSON_ASSERT(op == kOneOrMore); if (operandStack.GetSize() >= sizeof(Frag)) { Frag e = *operandStack.template Pop(1); SizeType s = NewState(kRegexInvalidState, e.start, 0); @@ -422,9 +423,6 @@ private: return true; } return false; - - default: - return false; } } From 01aeebf9bfd49e39556fa853bb860b77417025f7 Mon Sep 17 00:00:00 2001 From: Milo Yip Date: Sun, 17 Apr 2016 09:47:29 +0800 Subject: [PATCH 18/19] Improve reader coverage by removing a default case --- include/rapidjson/reader.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/rapidjson/reader.h b/include/rapidjson/reader.h index 243e0d2..16e2d07 100644 --- a/include/rapidjson/reader.h +++ b/include/rapidjson/reader.h @@ -1792,8 +1792,7 @@ private: case IterativeParsingKeyValueDelimiterState: case IterativeParsingArrayInitialState: case IterativeParsingElementDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); return; - case IterativeParsingElementState: RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); return; - default: RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); return; + default: RAPIDJSON_ASSERT(src == IterativeParsingElementState); RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); return; } } From d7ee08621a364693cdc610a98fb5bf556efef084 Mon Sep 17 00:00:00 2001 From: Milo Yip Date: Sun, 17 Apr 2016 10:11:40 +0800 Subject: [PATCH 19/19] Update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c70a6e3..c5d126a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,7 +51,8 @@ This project adheres to [Semantic Versioning](http://semver.org/). * Fix Document::Parse(const Ch*) for transcoding (#478) * encodings.h: fix typo in preprocessor condition (#495) * Custom Microsoft headers are necessary only for Visual Studio 2012 and lower (#559) -* +* Fix memory leak for invalid regex (26e69ffde95ba4773ab06db6457b78f308716f4b) +* Fix a bug in schema minimum/maximum keywords for 64-bit integer (e7149d665941068ccf8c565e77495521331cf390) ### Changed * Clarify problematic JSON license (#392)