From a006648398561295f28ddace74e37f00a260374b Mon Sep 17 00:00:00 2001 From: Milo Yip Date: Sat, 30 Jan 2016 01:00:01 +0800 Subject: [PATCH] Update json schema suite and add perf test --- bin/jsonschema/README.md | 91 ++++++-- bin/jsonschema/bin/jsonschema_suite | 0 bin/jsonschema/remotes/.DS_Store | Bin 0 -> 6148 bytes bin/jsonschema/tests/.DS_Store | Bin 0 -> 6148 bytes .../tests/draft3/additionalProperties.json | Bin 2194 -> 2745 bytes bin/jsonschema/tests/draft3/default.json | Bin 0 -> 1273 bytes .../tests/draft3/optional/bignum.json | Bin 1663 -> 3075 bytes .../tests/draft3/optional/format.json | Bin 6577 -> 6751 bytes bin/jsonschema/tests/draft3/pattern.json | Bin 582 -> 857 bytes bin/jsonschema/tests/draft3/ref.json | Bin 3922 -> 4385 bytes bin/jsonschema/tests/draft3/type.json | Bin 13225 -> 13217 bytes bin/jsonschema/tests/draft4/.DS_Store | Bin 0 -> 6148 bytes .../tests/draft4/additionalProperties.json | Bin 2194 -> 2745 bytes bin/jsonschema/tests/draft4/default.json | Bin 0 -> 1273 bytes bin/jsonschema/tests/draft4/maxLength.json | Bin 895 -> 896 bytes .../tests/draft4/optional/bignum.json | Bin 1663 -> 3075 bytes .../tests/draft4/optional/format.json | Bin 4434 -> 4608 bytes bin/jsonschema/tests/draft4/pattern.json | Bin 582 -> 857 bytes bin/jsonschema/tests/draft4/ref.json | Bin 3903 -> 4366 bytes bin/jsonschema/tests/draft4/type.json | Bin 9306 -> 9298 bytes bin/jsonschema/tox.ini | 8 + test/perftest/CMakeLists.txt | 3 +- test/perftest/schematest.cpp | 213 ++++++++++++++++++ test/unittest/schematest.cpp | 2 +- 24 files changed, 299 insertions(+), 18 deletions(-) mode change 100644 => 100755 bin/jsonschema/bin/jsonschema_suite create mode 100644 bin/jsonschema/remotes/.DS_Store create mode 100644 bin/jsonschema/tests/.DS_Store create mode 100644 bin/jsonschema/tests/draft3/default.json create mode 100644 bin/jsonschema/tests/draft4/.DS_Store create mode 100644 bin/jsonschema/tests/draft4/default.json create mode 100644 bin/jsonschema/tox.ini create mode 100644 test/perftest/schematest.cpp diff --git a/bin/jsonschema/README.md b/bin/jsonschema/README.md index 12c49c0..6d9da94 100644 --- a/bin/jsonschema/README.md +++ b/bin/jsonschema/README.md @@ -60,21 +60,80 @@ Who Uses the Test Suite This suite is being used by: - * [json-schema-validator (Java)](https://github.com/fge/json-schema-validator) - * [jsonschema (python)](https://github.com/Julian/jsonschema) - * [aeson-schema (haskell)](https://github.com/timjb/aeson-schema) - * [direct-schema (javascript)](https://github.com/IreneKnapp/direct-schema) - * [jsonschema (javascript)](https://github.com/tdegrunt/jsonschema) - * [JaySchema (javascript)](https://github.com/natesilva/jayschema) - * [z-schema (javascript)](https://github.com/zaggino/z-schema) - * [jassi (javascript)](https://github.com/iclanzan/jassi) - * [json-schema-valid (javascript)](https://github.com/ericgj/json-schema-valid) - * [jesse (Erlang)](https://github.com/klarna/jesse) - * [json-schema (PHP)](https://github.com/justinrainbow/json-schema) - * [gojsonschema (Go)](https://github.com/sigu-399/gojsonschema) - * [json_schema (Dart)](https://github.com/patefacio/json_schema) - * [tv4 (JavaScript)](https://github.com/geraintluff/tv4) - * [Jsonary (JavaScript)](https://github.com/jsonary-js/jsonary) +### Coffeescript ### + +* [jsck](https://github.com/pandastrike/jsck) + +### Dart ### + +* [json_schema](https://github.com/patefacio/json_schema) + +### Erlang ### + +* [jesse](https://github.com/klarna/jesse) + +### Go ### + +* [gojsonschema](https://github.com/sigu-399/gojsonschema) +* [validate-json](https://github.com/cesanta/validate-json) + +### Haskell ### + +* [aeson-schema](https://github.com/timjb/aeson-schema) +* [hjsonschema](https://github.com/seagreen/hjsonschema) + +### Java ### + +* [json-schema-validator](https://github.com/fge/json-schema-validator) + +### JavaScript ### + +* [json-schema-benchmark](https://github.com/Muscula/json-schema-benchmark) +* [direct-schema](https://github.com/IreneKnapp/direct-schema) +* [is-my-json-valid](https://github.com/mafintosh/is-my-json-valid) +* [jassi](https://github.com/iclanzan/jassi) +* [JaySchema](https://github.com/natesilva/jayschema) +* [json-schema-valid](https://github.com/ericgj/json-schema-valid) +* [Jsonary](https://github.com/jsonary-js/jsonary) +* [jsonschema](https://github.com/tdegrunt/jsonschema) +* [request-validator](https://github.com/bugventure/request-validator) +* [skeemas](https://github.com/Prestaul/skeemas) +* [tv4](https://github.com/geraintluff/tv4) +* [z-schema](https://github.com/zaggino/z-schema) +* [jsen](https://github.com/bugventure/jsen) +* [ajv](https://github.com/epoberezkin/ajv) + +### Node.js ### + +The JSON Schema Test Suite is also available as an +[npm](https://www.npmjs.com/package/json-schema-test-suite) package. +Node-specific support is maintained on the [node branch](https://github.com/json-schema/JSON-Schema-Test-Suite/tree/node). +See [NODE-README.md](https://github.com/json-schema/JSON-Schema-Test-Suite/blob/node/NODE-README.md) +for more information. + +### .NET ### + +* [Newtonsoft.Json.Schema](https://github.com/JamesNK/Newtonsoft.Json.Schema) + +### PHP ### + +* [json-schema](https://github.com/justinrainbow/json-schema) + +### Python ### + +* [jsonschema](https://github.com/Julian/jsonschema) + +### Ruby ### + +* [json-schema](https://github.com/hoxworth/json-schema) + +### Rust ### + +* [valico](https://github.com/rustless/valico) + +### Swift ### + +* [JSONSchema](https://github.com/kylef/JSONSchema.swift) If you use it as well, please fork and send a pull request adding yourself to the list :). @@ -85,5 +144,5 @@ Contributing If you see something missing or incorrect, a pull request is most welcome! There are some sanity checks in place for testing the test suite. You can run -them with `bin/jsonschema_suite check`. They will be run automatically by +them with `bin/jsonschema_suite check` or `tox`. They will be run automatically by [Travis CI](https://travis-ci.org/) as well. diff --git a/bin/jsonschema/bin/jsonschema_suite b/bin/jsonschema/bin/jsonschema_suite old mode 100644 new mode 100755 diff --git a/bin/jsonschema/remotes/.DS_Store b/bin/jsonschema/remotes/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..1d098a4103d67f2b3b336934f3d7f42b462861a1 GIT binary patch literal 6148 zcmeHKOHRWu5S@X7Ds|H(?0kjZAgaO%dI3btXOYNNp?e>&D;7NuZ$2PWB6c8zW+eOB z^Ks%A#p59&UhngYXh}qKG(ncZgot|5bmq<%K-M+xX_ue7{;rgMVxhmNl6SwP2P)K4 zrjz#{8T!Z7rYpnNcX53hIFz={`93>*C>7{`CI$;>GS&XK|+FoU?3O>27-Z~ zU;sH=WWF$rJ{SlFf`JbPv%iZPLMpAHP(0stF?-7xoF z0$3~ntcg<~A}|dqFsPa>h6Ww+l6f_83JkhvHXoWdYj!B=x8wZc>7q4|BNdLa}rlnfC~I81+?j&yFH$iwRQ10tF;CG0=JwmxEbb7!QkZ>=;as-E60zX b6nVww*sqCGpwkg|I*>mDrVEV<{I&w$e{~fm literal 0 HcmV?d00001 diff --git a/bin/jsonschema/tests/draft3/additionalProperties.json b/bin/jsonschema/tests/draft3/additionalProperties.json index eb334c985cec4c1493c0e5bf3995b73dc46f24d9..40831f9e9aa135fbd9b698b3a212cea5276e72c7 100644 GIT binary patch delta 73 zcmV-P0Ji^<5xEtxNe7W0BNJm`ZXjiNX>)WSVtF8GbaQ2FW|3?pkt`IGz64{F{Q(Y> f(giz{BnGgPQ3xE9djb=ae+Uqh?gJjPNe4**xz`yN delta 12 TcmdlfI!SPYC;R3CPESSv9fAYs diff --git a/bin/jsonschema/tests/draft3/default.json b/bin/jsonschema/tests/draft3/default.json new file mode 100644 index 0000000000000000000000000000000000000000..17629779fbeabea738a101fba2b305990e096466 GIT binary patch literal 1273 zcmds#!EVDK42JK03d`qRrtQ>c*c(JW(BLFUDMa9Gs;a*G0)#-aY`sjHG!9X~#(e($ zR|lBZEqwhSN^-ST11qEP2&nYC=#>P%4g`&{AgK{!&-j$DxVoc@n6h)3P-?JdAj^u} zmsp55#_R?{&_5Ufm|C*Mdp?}jgd1^=P@NW{6uqxb^6p;1GdcEr)hc)0iLbX6Cd%^B zk%qa?D<}tUpjtJGqrripefE-(;TaP>by3@|WK6Y9J*qK{s_@eY1>6W{x&};Tj q8e7Q8X2i3QxG0g5vIG4|-W3AlpZGEAgRLu=p9XH)dw8SS|oB;ROQ# diff --git a/bin/jsonschema/tests/draft3/optional/format.json b/bin/jsonschema/tests/draft3/optional/format.json index fc86b03a45281a52d05df3c5da94d20780340668..3ca7319dda0472fece466eddcf9d76a8f4a1efee 100644 GIT binary patch delta 41 xcmdmJeBWe)3g^VN?4kul`6c? delta 16 Xcmca_ve9^h3g_ekcCO8LIGY6kIIjj2 diff --git a/bin/jsonschema/tests/draft3/pattern.json b/bin/jsonschema/tests/draft3/pattern.json index befc4b560f7c332a101f28790c3f7fbc2b093b2e..25e7299731491bf95ab79d1cda9b332778c2ae1e 100644 GIT binary patch delta 74 zcmX@ca+7U?GZUlEL`HS~%wmPS{1S!4yyT4hqSTa$MoLVH+7lyOg>w^2k~30^6%rMS eOOuLAiZb)kC+?78tEfm!tgM`PU2n26lQ00Iav9(N delta 11 Scmcb~c8q0%GZQmcEEfP63Ih`W diff --git a/bin/jsonschema/tests/draft3/ref.json b/bin/jsonschema/tests/draft3/ref.json index c9840192a4777cbee3a342fdd41eea60dab2193a..903ecb6bce13eac7dac93233425d0e78e87bb203 100644 GIT binary patch delta 176 zcmca4w@_(AHYc+}X5Qp{9)-zmoF#C!C|4Pry^PCo@&z^n*5aJR;*808Jc_*4N?^8< zm4cy>@#KfhB9jexIVSI8b7C#Y%t=XvssXbx)vy<&7A2?Vm4H<7S1Unz80v&3+w%)d Rj^|IEyATvdT_Y=PCiy2e|Af^9c$}*5X%~yoZl#^9LS7765O| B4srkh diff --git a/bin/jsonschema/tests/draft3/type.json b/bin/jsonschema/tests/draft3/type.json index 8f108899744d753fa88eac5fcab1af18d78f4077..337da1206dae10a50e1d1459db0fbc1910958fc8 100644 GIT binary patch delta 47 zcmV+~0MP%bXQ5}XH5HQ&6eE*6_W%3VUuAagp+S52($4T FxEP`S5oQ1Y delta 53 zcmV-50LuTNXQ^kfH5HRp7AKSM86A_K8bXs#8V!@40|%2_79f*=8XuGJ3{bOL76Js5 LbQdVIfEpng4xthV diff --git a/bin/jsonschema/tests/draft4/.DS_Store b/bin/jsonschema/tests/draft4/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..ef142295ea00d1d19fc3d30cc4e82dd207530825 GIT binary patch literal 6148 zcmeHK%}T^D5T4N$3SRc8x4go>L0IYo$i9Huf(orExO?vd?#YAC<2OGpmKHn+A~FM+ zFPY3tnh%;}h={j`c0;r#q6$rrrL!PnUYt5}=L;ZfjzTYVPhI=kbPI|8qDj8JqCx}h z=^1$X{)bX@53|YcakFbmKiF=rZcj9aqIv5BBrVO0ha4q-$4St!$ zB7YhZqhKHy_-738s@~OGY|8J}+4khFO=x#$BH}kn2ZH|O5rBc5BUd_U^GW*f%Z{U= TWD&cD1LGl}goFwPeu04xBO^7X literal 0 HcmV?d00001 diff --git a/bin/jsonschema/tests/draft4/additionalProperties.json b/bin/jsonschema/tests/draft4/additionalProperties.json index eb334c985cec4c1493c0e5bf3995b73dc46f24d9..40831f9e9aa135fbd9b698b3a212cea5276e72c7 100644 GIT binary patch delta 73 zcmV-P0Ji^<5xEtxNe7W0BNJm`ZXjiNX>)WSVtF8GbaQ2FW|3?pkt`IGz64{F{Q(Y> f(giz{BnGgPQ3xE9djb=ae+Uqh?gJjPNe4**xz`yN delta 12 TcmdlfI!SPYC;R3CPESSv9fAYs diff --git a/bin/jsonschema/tests/draft4/default.json b/bin/jsonschema/tests/draft4/default.json new file mode 100644 index 0000000000000000000000000000000000000000..17629779fbeabea738a101fba2b305990e096466 GIT binary patch literal 1273 zcmds#!EVDK42JK03d`qRrtQ>c*c(JW(BLFUDMa9Gs;a*G0)#-aY`sjHG!9X~#(e($ zR|lBZEqwhSN^-ST11qEP2&nYC=#>P%4g`&{AgK{!&-j$DxVoc@n6h)3P-?JdAj^u} zmsp55#_R?{&_5Ufm|C*Mdp?}jgd1^=P@NW{6uqxb^6p;1GdcEr)hc)0iLbX6Cd%^B zk%qa?D<}tUpjtJGqrripefE-(;TaP>by3@|WK6Y9J*qK{s_@eY1>6W{x&};Tj q8e7Q8X2i3QxG0g5vIG4|-W3AlpZGEAgRLu=p9XnN|V-Lx2XZ diff --git a/bin/jsonschema/tests/draft4/optional/bignum.json b/bin/jsonschema/tests/draft4/optional/bignum.json index cd479949cb725c18933dfdb4e210a8a6c9d56b50..ccc7c17fe8d504002d8f6100b6cfd3e15d8aca46 100644 GIT binary patch delta 174 zcmey*(=4%}jcKAn2}fRPdSXdtS?a{e28_BJ53?}BMJ69()MM10tjQd(*@el1W%3+m zFRskI%-qskB`XEp$?utSdCO9ZDivTROio}?2C8pnnJS`?p9dFK$SciFN-Zj$e2!Uc dvK}ixvKf+WDwDHVC7CkwCbzO;H%^i*4FF&xImZA1 delta 15 WcmZpc_|LPUjcF6>H)dw8SS|oB;ROQ# diff --git a/bin/jsonschema/tests/draft4/optional/format.json b/bin/jsonschema/tests/draft4/optional/format.json index 53c5d2519056c62773afe23b2cdccf1c9a8d3ede..aacfd119843cb003b06c74d7e283a0aeeca9902a 100644 GIT binary patch delta 37 tcmcbl)S$BAHuJ>w?1BYF`6c! delta 16 YcmZorxumq=HuL0vY+Rcsv*hvu06l01i~s-t diff --git a/bin/jsonschema/tests/draft4/pattern.json b/bin/jsonschema/tests/draft4/pattern.json index befc4b560f7c332a101f28790c3f7fbc2b093b2e..25e7299731491bf95ab79d1cda9b332778c2ae1e 100644 GIT binary patch delta 74 zcmX@ca+7U?GZUlEL`HS~%wmPS{1S!4yyT4hqSTa$MoLVH+7lyOg>w^2k~30^6%rMS eOOuLAiZb)kC+?78tEfm!tgM`PU2n26lQ00Iav9(N delta 11 Scmcb~c8q0%GZQmcEEfP63Ih`W diff --git a/bin/jsonschema/tests/draft4/ref.json b/bin/jsonschema/tests/draft4/ref.json index b38ff0313f1fc03c240af138709a7d5e0133b037..7e805522492e7bfa5d6738e6427bef9cf1aa2f46 100644 GIT binary patch delta 176 zcmdll*Qc~0o0C}~GjDP}kHX|O&Js9Vl&cKRUdH7(`2w2(YjI9uamM649!1`2B`{mb zO2N>`c=AJLk;w+U9FzC4IkA>x=Ai;`3GN_g1pqUXHeLV# delta 48 zcmeBE+Ap^un{%=dqwM4foJErvS!E}ybCrPU16=l#e+vjq*5X%~yoZl#^9LR|765l` B4W5#`CYJhGGTE4WP77ZlojUUVYM +#include +#include + +#define ARRAY_SIZE(a) sizeof(a) / sizeof(a[0]) + +using namespace rapidjson; + +template +static char* ReadFile(const char* filename, Allocator& allocator) { + const char *paths[] = { + "", + "bin/", + "../bin/", + "../../bin/", + "../../../bin/" + }; + char buffer[1024]; + FILE *fp = 0; + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) { + sprintf(buffer, "%s%s", paths[i], filename); + fp = fopen(buffer, "rb"); + if (fp) + break; + } + + if (!fp) + return 0; + + fseek(fp, 0, SEEK_END); + size_t length = static_cast(ftell(fp)); + fseek(fp, 0, SEEK_SET); + char* json = reinterpret_cast(allocator.Malloc(length + 1)); + size_t readLength = fread(json, 1, length, fp); + json[readLength] = '\0'; + fclose(fp); + return json; +} + +class Schema : public PerfTest { +public: + Schema() {} + + virtual void SetUp() { + PerfTest::SetUp(); + + const char* filenames[] = { + "additionalItems.json", + "additionalProperties.json", + "allOf.json", + "anyOf.json", + "default.json", + "definitions.json", + "dependencies.json", + "enum.json", + "items.json", + "maximum.json", + "maxItems.json", + "maxLength.json", + "maxProperties.json", + "minimum.json", + "minItems.json", + "minLength.json", + "minProperties.json", + "multipleOf.json", + "not.json", + "oneOf.json", + "pattern.json", + "patternProperties.json", + "properties.json", + "ref.json", + "refRemote.json", + "required.json", + "type.json", + "uniqueItems.json" + }; + + char jsonBuffer[65536]; + MemoryPoolAllocator<> jsonAllocator(jsonBuffer, sizeof(jsonBuffer)); + + for (size_t i = 0; i < ARRAY_SIZE(filenames); i++) { + char filename[FILENAME_MAX]; + sprintf(filename, "jsonschema/tests/draft4/%s", filenames[i]); + char* json = ReadFile(filename, jsonAllocator); + if (!json) { + printf("json test suite file %s not found", filename); + return; + } + + Document d; + d.Parse(json); + if (d.HasParseError()) { + printf("json test suite file %s has parse error", filename); + return; + } + + for (Value::ConstValueIterator schemaItr = d.Begin(); schemaItr != d.End(); ++schemaItr) { + if (IsExcludeTestSuite((*schemaItr)["description"].GetString())) + continue; + + TestSuite* ts = new TestSuite; + ts->schema = new SchemaDocument((*schemaItr)["schema"]); + + const Value& tests = (*schemaItr)["tests"]; + for (Value::ConstValueIterator testItr = tests.Begin(); testItr != tests.End(); ++testItr) { + if (IsExcludeTest((*testItr)["description"].GetString())) + continue; + + Document* d2 = new Document; + d2->CopyFrom((*testItr)["data"], d2->GetAllocator()); + ts->tests.push_back(d2); + } + testSuites.push_back(ts); + } + } + } + + virtual void TearDown() { + PerfTest::TearDown(); + for (TestSuiteList::const_iterator itr = testSuites.begin(); itr != testSuites.end(); ++itr) + delete *itr; + testSuites.clear(); + } + +private: + // Using the same exclusion in https://github.com/json-schema/JSON-Schema-Test-Suite + static bool IsExcludeTestSuite(const std::string& description) { + const char* excludeTestSuites[] = { + //lost failing these tests + "remote ref", + "remote ref, containing refs itself", + "fragment within remote ref", + "ref within remote ref", + "change resolution scope", + // these below were added to get jsck in the benchmarks) + "uniqueItems validation", + "valid definition", + "invalid definition" + }; + + for (size_t i = 0; i < ARRAY_SIZE(excludeTestSuites); i++) + if (excludeTestSuites[i] == description) + return true; + return false; + } + + // Using the same exclusion in https://github.com/json-schema/JSON-Schema-Test-Suite + static bool IsExcludeTest(const std::string& description) { + const char* excludeTests[] = { + //lots of validators fail these + "invalid definition, invalid definition schema", + "maxLength validation, two supplementary Unicode code points is long enough", + "minLength validation, one supplementary Unicode code point is not long enough", + //this is to get tv4 in the benchmarks + "heterogeneous enum validation, something else is invalid" + }; + + for (size_t i = 0; i < ARRAY_SIZE(excludeTests); i++) + if (excludeTests[i] == description) + return true; + return false; + } + + Schema(const Schema&); + Schema& operator=(const Schema&); + +protected: + typedef std::vector DocumentList; + + struct TestSuite { + TestSuite() : schema() {} + ~TestSuite() { + delete schema; + for (DocumentList::iterator itr = tests.begin(); itr != tests.end(); ++itr) + delete *itr; + } + SchemaDocument* schema; + DocumentList tests; + }; + + typedef std::vector TestSuiteList; + TestSuiteList testSuites; +}; + +TEST_F(Schema, TestSuite) { + char validatorBuffer[65536]; + MemoryPoolAllocator<> validatorAllocator(validatorBuffer, sizeof(validatorBuffer)); + + int testCount = 0; + clock_t start = clock(); + for (int i = 0; i < 10000; i++) { + for (TestSuiteList::const_iterator itr = testSuites.begin(); itr != testSuites.end(); ++itr) { + const TestSuite& ts = **itr; + GenericSchemaValidator >, MemoryPoolAllocator<> > validator(*ts.schema, &validatorAllocator); + for (DocumentList::const_iterator testItr = ts.tests.begin(); testItr != ts.tests.end(); ++testItr) { + validator.Reset(); + (*testItr)->Accept(validator); + testCount++; + } + validatorAllocator.Clear(); + } + } + clock_t end = clock(); + double duration = double(end - start) / CLOCKS_PER_SEC; + printf("%d tests in %f s -> %f tests per sec\n", testCount, duration, testCount / duration); +} + +#endif diff --git a/test/unittest/schematest.cpp b/test/unittest/schematest.cpp index 95b5bb0..b87ca9c 100644 --- a/test/unittest/schematest.cpp +++ b/test/unittest/schematest.cpp @@ -992,11 +992,11 @@ private: TEST(SchemaValidator, TestSuite) { const char* filenames[] = { - "properties.json", "additionalItems.json", "additionalProperties.json", "allOf.json", "anyOf.json", + "default.json", "definitions.json", "dependencies.json", "enum.json",