Update json schema suite and add perf test

bin/jsonschema/README.md

@@ -60,21 +60,80 @@ Who Uses the Test Suite
 
 This suite is being used by:
 
-  * [json-schema-validator (Java)](https://github.com/fge/json-schema-validator)
+### Coffeescript ###
-  * [jsonschema (python)](https://github.com/Julian/jsonschema)
+
-  * [aeson-schema (haskell)](https://github.com/timjb/aeson-schema)
+* [jsck](https://github.com/pandastrike/jsck)
-  * [direct-schema (javascript)](https://github.com/IreneKnapp/direct-schema)
+
-  * [jsonschema (javascript)](https://github.com/tdegrunt/jsonschema)
+### Dart ###
-  * [JaySchema (javascript)](https://github.com/natesilva/jayschema)
+
-  * [z-schema (javascript)](https://github.com/zaggino/z-schema)
+* [json_schema](https://github.com/patefacio/json_schema) 
-  * [jassi (javascript)](https://github.com/iclanzan/jassi)
+
-  * [json-schema-valid (javascript)](https://github.com/ericgj/json-schema-valid)
+### Erlang ###
-  * [jesse (Erlang)](https://github.com/klarna/jesse)
+
-  * [json-schema (PHP)](https://github.com/justinrainbow/json-schema)
+* [jesse](https://github.com/klarna/jesse)
-  * [gojsonschema (Go)](https://github.com/sigu-399/gojsonschema) 
+
-  * [json_schema (Dart)](https://github.com/patefacio/json_schema) 
+### Go ###
-  * [tv4 (JavaScript)](https://github.com/geraintluff/tv4)
+
-  * [Jsonary (JavaScript)](https://github.com/jsonary-js/jsonary)
+* [gojsonschema](https://github.com/sigu-399/gojsonschema) 
+* [validate-json](https://github.com/cesanta/validate-json)
+
+### Haskell ###
+
+* [aeson-schema](https://github.com/timjb/aeson-schema)
+* [hjsonschema](https://github.com/seagreen/hjsonschema)
+
+### Java ###
+
+* [json-schema-validator](https://github.com/fge/json-schema-validator)
+
+### JavaScript ###
+
+* [json-schema-benchmark](https://github.com/Muscula/json-schema-benchmark)
+* [direct-schema](https://github.com/IreneKnapp/direct-schema)
+* [is-my-json-valid](https://github.com/mafintosh/is-my-json-valid)
+* [jassi](https://github.com/iclanzan/jassi)
+* [JaySchema](https://github.com/natesilva/jayschema)
+* [json-schema-valid](https://github.com/ericgj/json-schema-valid)
+* [Jsonary](https://github.com/jsonary-js/jsonary)
+* [jsonschema](https://github.com/tdegrunt/jsonschema)
+* [request-validator](https://github.com/bugventure/request-validator)
+* [skeemas](https://github.com/Prestaul/skeemas)
+* [tv4](https://github.com/geraintluff/tv4)
+* [z-schema](https://github.com/zaggino/z-schema)
+* [jsen](https://github.com/bugventure/jsen)
+* [ajv](https://github.com/epoberezkin/ajv)
+
+### Node.js ###
+
+The JSON Schema Test Suite is also available as an
+[npm](https://www.npmjs.com/package/json-schema-test-suite) package.
+Node-specific support is maintained on the [node branch](https://github.com/json-schema/JSON-Schema-Test-Suite/tree/node).
+See [NODE-README.md](https://github.com/json-schema/JSON-Schema-Test-Suite/blob/node/NODE-README.md)
+for more information.
+
+### .NET ###
+
+* [Newtonsoft.Json.Schema](https://github.com/JamesNK/Newtonsoft.Json.Schema)
+
+### PHP ###
+
+* [json-schema](https://github.com/justinrainbow/json-schema)
+
+### Python ###
+
+* [jsonschema](https://github.com/Julian/jsonschema)
+
+### Ruby ###
+
+* [json-schema](https://github.com/hoxworth/json-schema)
+
+### Rust ###
+
+* [valico](https://github.com/rustless/valico)
+
+### Swift ###
+
+* [JSONSchema](https://github.com/kylef/JSONSchema.swift)
 
 If you use it as well, please fork and send a pull request adding yourself to
 the list :).
@@ -85,5 +144,5 @@ Contributing
 If you see something missing or incorrect, a pull request is most welcome!
 
 There are some sanity checks in place for testing the test suite. You can run
-them with `bin/jsonschema_suite check`. They will be run automatically by
+them with `bin/jsonschema_suite check` or `tox`. They will be run automatically by
 [Travis CI](https://travis-ci.org/) as well.

bin/jsonschema/tox.ini

@@ -0,0 +1,8 @@
+[tox]
+minversion = 1.6
+envlist = py27
+skipsdist = True
+
+[testenv]
+deps = jsonschema
+commands = {envpython} bin/jsonschema_suite check

test/perftest/CMakeLists.txt

@@ -2,7 +2,8 @@ set(PERFTEST_SOURCES
     misctest.cpp
     perftest.cpp
     platformtest.cpp
-    rapidjsontest.cpp)
+    rapidjsontest.cpp
+    schematest.cpp)
 
 add_executable(perftest ${PERFTEST_SOURCES})
 target_link_libraries(perftest ${TEST_LIBRARIES})

test/perftest/schematest.cpp

@@ -0,0 +1,213 @@
+#include "perftest.h"
+
+#if TEST_RAPIDJSON
+
+#include "rapidjson/schema.h"
+#include <ctime>
+#include <string>
+#include <vector>
+
+#define ARRAY_SIZE(a) sizeof(a) / sizeof(a[0])
+
+using namespace rapidjson;
+
+template <typename Allocator>
+static char* ReadFile(const char* filename, Allocator& allocator) {
+    const char *paths[] = {
+        "",
+        "bin/",
+        "../bin/",
+        "../../bin/",
+        "../../../bin/"
+    };
+    char buffer[1024];
+    FILE *fp = 0;
+    for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) {
+        sprintf(buffer, "%s%s", paths[i], filename);
+        fp = fopen(buffer, "rb");
+        if (fp)
+            break;
+    }
+
+    if (!fp)
+        return 0;
+
+    fseek(fp, 0, SEEK_END);
+    size_t length = static_cast<size_t>(ftell(fp));
+    fseek(fp, 0, SEEK_SET);
+    char* json = reinterpret_cast<char*>(allocator.Malloc(length + 1));
+    size_t readLength = fread(json, 1, length, fp);
+    json[readLength] = '\0';
+    fclose(fp);
+    return json;
+}
+
+class Schema : public PerfTest {
+public:
+    Schema() {}
+
+    virtual void SetUp() {
+        PerfTest::SetUp();
+
+        const char* filenames[] = {
+            "additionalItems.json",
+            "additionalProperties.json",
+            "allOf.json",
+            "anyOf.json",
+            "default.json",
+            "definitions.json",
+            "dependencies.json",
+            "enum.json",
+            "items.json",
+            "maximum.json",
+            "maxItems.json",
+            "maxLength.json",
+            "maxProperties.json",
+            "minimum.json",
+            "minItems.json",
+            "minLength.json",
+            "minProperties.json",
+            "multipleOf.json",
+            "not.json",
+            "oneOf.json",
+            "pattern.json",
+            "patternProperties.json",
+            "properties.json",
+            "ref.json",
+            "refRemote.json",
+            "required.json",
+            "type.json",
+            "uniqueItems.json"
+        };
+
+        char jsonBuffer[65536];
+        MemoryPoolAllocator<> jsonAllocator(jsonBuffer, sizeof(jsonBuffer));
+
+        for (size_t i = 0; i < ARRAY_SIZE(filenames); i++) {
+            char filename[FILENAME_MAX];
+            sprintf(filename, "jsonschema/tests/draft4/%s", filenames[i]);
+            char* json = ReadFile(filename, jsonAllocator);
+            if (!json) {
+                printf("json test suite file %s not found", filename);
+                return;
+            }
+
+            Document d;
+            d.Parse(json);
+            if (d.HasParseError()) {
+                printf("json test suite file %s has parse error", filename);
+                return;
+            }
+
+            for (Value::ConstValueIterator schemaItr = d.Begin(); schemaItr != d.End(); ++schemaItr) {
+                if (IsExcludeTestSuite((*schemaItr)["description"].GetString()))
+                    continue;
+
+                TestSuite* ts = new TestSuite;
+                ts->schema = new SchemaDocument((*schemaItr)["schema"]);
+
+                const Value& tests = (*schemaItr)["tests"];
+                for (Value::ConstValueIterator testItr = tests.Begin(); testItr != tests.End(); ++testItr) {
+                    if (IsExcludeTest((*testItr)["description"].GetString()))
+                        continue;
+
+                    Document* d2 = new Document;
+                    d2->CopyFrom((*testItr)["data"], d2->GetAllocator());
+                    ts->tests.push_back(d2);
+                }
+                testSuites.push_back(ts);
+            }
+        }
+    }
+
+    virtual void TearDown() {
+        PerfTest::TearDown();
+        for (TestSuiteList::const_iterator itr = testSuites.begin(); itr != testSuites.end(); ++itr)
+            delete *itr;
+        testSuites.clear();
+    }
+
+private:
+    // Using the same exclusion in https://github.com/json-schema/JSON-Schema-Test-Suite
+    static bool IsExcludeTestSuite(const std::string& description) {
+        const char* excludeTestSuites[] = {
+            //lost failing these tests
+            "remote ref",
+            "remote ref, containing refs itself",
+            "fragment within remote ref",
+            "ref within remote ref",
+            "change resolution scope",
+            // these below were added to get jsck in the benchmarks)
+            "uniqueItems validation",
+            "valid definition",
+            "invalid definition"
+        };
+
+        for (size_t i = 0; i < ARRAY_SIZE(excludeTestSuites); i++)
+            if (excludeTestSuites[i] == description)
+                return true;
+        return false;
+    }
+
+    // Using the same exclusion in https://github.com/json-schema/JSON-Schema-Test-Suite
+    static bool IsExcludeTest(const std::string& description) {
+        const char* excludeTests[] = {
+            //lots of validators fail these
+            "invalid definition, invalid definition schema",
+            "maxLength validation, two supplementary Unicode code points is long enough",
+            "minLength validation, one supplementary Unicode code point is not long enough",
+            //this is to get tv4 in the benchmarks
+            "heterogeneous enum validation, something else is invalid"
+        };
+
+        for (size_t i = 0; i < ARRAY_SIZE(excludeTests); i++)
+            if (excludeTests[i] == description)
+                return true;
+        return false;
+    }
+
+    Schema(const Schema&);
+    Schema& operator=(const Schema&);
+
+protected:
+    typedef std::vector<Document*> DocumentList;
+
+    struct TestSuite {
+        TestSuite() : schema() {}
+        ~TestSuite() {
+            delete schema;
+            for (DocumentList::iterator itr = tests.begin(); itr != tests.end(); ++itr)
+                delete *itr;
+        }
+        SchemaDocument* schema;
+        DocumentList tests;
+    };
+
+    typedef std::vector<TestSuite* > TestSuiteList;
+    TestSuiteList testSuites;
+};
+
+TEST_F(Schema, TestSuite) {
+    char validatorBuffer[65536];
+    MemoryPoolAllocator<> validatorAllocator(validatorBuffer, sizeof(validatorBuffer));
+
+    int testCount = 0;
+    clock_t start = clock();
+    for (int i = 0; i < 10000; i++) {
+        for (TestSuiteList::const_iterator itr = testSuites.begin(); itr != testSuites.end(); ++itr) {
+            const TestSuite& ts = **itr;
+            GenericSchemaValidator<SchemaDocument, BaseReaderHandler<UTF8<> >, MemoryPoolAllocator<> >  validator(*ts.schema, &validatorAllocator);
+            for (DocumentList::const_iterator testItr = ts.tests.begin(); testItr != ts.tests.end(); ++testItr) {
+                validator.Reset();
+                (*testItr)->Accept(validator);
+                testCount++;
+            }
+            validatorAllocator.Clear();
+        }
+    }
+    clock_t end = clock();
+    double duration = double(end - start) / CLOCKS_PER_SEC;
+    printf("%d tests in %f s -> %f tests per sec\n", testCount, duration, testCount / duration);
+}
+
+#endif

test/unittest/schematest.cpp

@@ -992,11 +992,11 @@ private:
 
 TEST(SchemaValidator, TestSuite) {
     const char* filenames[] = {
-        "properties.json",
         "additionalItems.json",
         "additionalProperties.json",
         "allOf.json",
         "anyOf.json",
+        "default.json",
         "definitions.json",
         "dependencies.json",
         "enum.json",