From 3a65e2dd7f37684081ad8eb62fb64fbc418c4a47 Mon Sep 17 00:00:00 2001 From: Pave Pimenov Date: Sat, 12 Sep 2020 19:53:06 +0300 Subject: [PATCH 01/27] fix https://github.com/Tencent/rapidjson/issues/1778 (part 1) --- include/rapidjson/schema.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/rapidjson/schema.h b/include/rapidjson/schema.h index fc39d06..4e5f217 100644 --- a/include/rapidjson/schema.h +++ b/include/rapidjson/schema.h @@ -475,12 +475,12 @@ public: AssignIfExist(allOf_, *schemaDocument, p, value, GetAllOfString(), document); AssignIfExist(anyOf_, *schemaDocument, p, value, GetAnyOfString(), document); AssignIfExist(oneOf_, *schemaDocument, p, value, GetOneOfString(), document); - } - if (const ValueType* v = GetMember(value, GetNotString())) { + if (const ValueType* v = GetMember(value, GetNotString())) { schemaDocument->CreateSchema(¬_, p.Append(GetNotString(), allocator_), *v, document); notValidatorIndex_ = validatorCount_; validatorCount_++; + } } // Object @@ -915,7 +915,7 @@ public: } if (additionalPropertiesSchema_) { - if (additionalPropertiesSchema_ && context.patternPropertiesSchemaCount > 0) { + if (context.patternPropertiesSchemaCount > 0) { context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = additionalPropertiesSchema_; context.valueSchema = typeless_; context.valuePatternValidatorType = Context::kPatternValidatorWithAdditionalProperty; From 24ebd51287f7f3f6aca11a3c7cc8eea8110c10ce Mon Sep 17 00:00:00 2001 From: Laurent Stacul Date: Mon, 22 Feb 2021 16:11:42 +0000 Subject: [PATCH 02/27] Fix recursive operator== call in C++20 (#1846) --- include/rapidjson/document.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/rapidjson/document.h b/include/rapidjson/document.h index 028235e..0910122 100644 --- a/include/rapidjson/document.h +++ b/include/rapidjson/document.h @@ -1083,6 +1083,7 @@ public: */ template RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator!=(const T& rhs) const { return !(*this == rhs); } +#ifndef __cpp_lib_three_way_comparison //! Equal-to operator with arbitrary types (symmetric version) /*! \return (rhs == lhs) */ @@ -1093,6 +1094,7 @@ public: */ template friend RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator!=(const T& lhs, const GenericValue& rhs) { return !(rhs == lhs); } //@} +#endif //!@name Type //@{ From 1c2c8e085a8b2561dff17bedb689d2eb0609b689 Mon Sep 17 00:00:00 2001 From: miloyip Date: Tue, 2 Mar 2021 11:15:31 +0800 Subject: [PATCH 03/27] doc: fix incorrect template parameters in EncodedOutputStream example Fix #1851 --- doc/stream.md | 2 +- doc/stream.zh-cn.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/stream.md b/doc/stream.md index 04c865c..0573549 100644 --- a/doc/stream.md +++ b/doc/stream.md @@ -231,7 +231,7 @@ FileWriteStream bos(fp, writeBuffer, sizeof(writeBuffer)); typedef EncodedOutputStream, FileWriteStream> OutputStream; OutputStream eos(bos, true); // Write BOM -Writer, UTF8<>> writer(eos); +Writer, UTF32LE<>> writer(eos); d.Accept(writer); // This generates UTF32-LE file from UTF-8 in memory fclose(fp); diff --git a/doc/stream.zh-cn.md b/doc/stream.zh-cn.md index 7f2f356..6e379bb 100644 --- a/doc/stream.zh-cn.md +++ b/doc/stream.zh-cn.md @@ -231,7 +231,7 @@ FileWriteStream bos(fp, writeBuffer, sizeof(writeBuffer)); typedef EncodedOutputStream, FileWriteStream> OutputStream; OutputStream eos(bos, true); // 写入 BOM -Writer, UTF8<>> writer(eos); +Writer, UTF32LE<>> writer(eos); d.Accept(writer); // 这里从内存的 UTF-8 生成 UTF32-LE 文件 fclose(fp); From cdb2d4757dce200a5026fb6277202e56f8f25a63 Mon Sep 17 00:00:00 2001 From: ylavic Date: Fri, 12 Mar 2021 15:14:30 +0100 Subject: [PATCH 04/27] Provide RAPIDJSON_HAS_CXX11 and use it for RAPIDJSON_HAS_CXX11_RVALUE_REFS and RAPIDJSON_HAS_CXX11_NOEXCEPT. --- include/rapidjson/rapidjson.h | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/include/rapidjson/rapidjson.h b/include/rapidjson/rapidjson.h index 78aa89a..fb6dd51 100644 --- a/include/rapidjson/rapidjson.h +++ b/include/rapidjson/rapidjson.h @@ -541,8 +541,14 @@ RAPIDJSON_NAMESPACE_END /////////////////////////////////////////////////////////////////////////////// // C++11 features +#ifndef RAPIDJSON_HAS_CXX11 +#define RAPIDJSON_HAS_CXX11 (__cplusplus >= 201103L) +#endif + #ifndef RAPIDJSON_HAS_CXX11_RVALUE_REFS -#if defined(__clang__) +#if RAPIDJSON_HAS_CXX11 +#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 +#elif defined(__clang__) #if __has_feature(cxx_rvalue_references) && \ (defined(_MSC_VER) || defined(_LIBCPP_VERSION) || defined(__GLIBCXX__) && __GLIBCXX__ >= 20080306) #define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 @@ -560,7 +566,9 @@ RAPIDJSON_NAMESPACE_END #endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS #ifndef RAPIDJSON_HAS_CXX11_NOEXCEPT -#if defined(__clang__) +#if RAPIDJSON_HAS_CXX11 +#define RAPIDJSON_HAS_CXX11_NOEXCEPT 1 +#elif defined(__clang__) #define RAPIDJSON_HAS_CXX11_NOEXCEPT __has_feature(cxx_noexcept) #elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,6,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ (defined(_MSC_VER) && _MSC_VER >= 1900) || \ @@ -570,11 +578,13 @@ RAPIDJSON_NAMESPACE_END #define RAPIDJSON_HAS_CXX11_NOEXCEPT 0 #endif #endif +#ifndef RAPIDJSON_NOEXCEPT #if RAPIDJSON_HAS_CXX11_NOEXCEPT #define RAPIDJSON_NOEXCEPT noexcept #else -#define RAPIDJSON_NOEXCEPT /* noexcept */ +#define RAPIDJSON_NOEXCEPT throw() #endif // RAPIDJSON_HAS_CXX11_NOEXCEPT +#endif // no automatic detection, yet #ifndef RAPIDJSON_HAS_CXX11_TYPETRAITS From d51dd2d0e9a1002bd0182b95a7bd246ecd0e3fbb Mon Sep 17 00:00:00 2001 From: ylavic Date: Fri, 12 Mar 2021 15:31:10 +0100 Subject: [PATCH 05/27] RAPIDJSON_NOEXCEPT_ASSERT should assert regardless of RAPIDJSON_HAS_CXX11_NOEXCEPT. --- include/rapidjson/rapidjson.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/include/rapidjson/rapidjson.h b/include/rapidjson/rapidjson.h index fb6dd51..8034c49 100644 --- a/include/rapidjson/rapidjson.h +++ b/include/rapidjson/rapidjson.h @@ -638,12 +638,8 @@ RAPIDJSON_NAMESPACE_END #ifndef RAPIDJSON_NOEXCEPT_ASSERT #ifdef RAPIDJSON_ASSERT_THROWS -#if RAPIDJSON_HAS_CXX11_NOEXCEPT -#define RAPIDJSON_NOEXCEPT_ASSERT(x) -#else #include #define RAPIDJSON_NOEXCEPT_ASSERT(x) assert(x) -#endif // RAPIDJSON_HAS_CXX11_NOEXCEPT #else #define RAPIDJSON_NOEXCEPT_ASSERT(x) RAPIDJSON_ASSERT(x) #endif // RAPIDJSON_ASSERT_THROWS From c033292aeaf014accfb5b4054d8fdac36358266c Mon Sep 17 00:00:00 2001 From: ylavic Date: Sat, 27 Apr 2019 03:41:19 +0200 Subject: [PATCH 06/27] Safer GenericValue& operator=(GenericValue& rhs). When rhs is a sub-Value of *this, destroying *this also destroys/frees rhs, thus the following RawAssign(rhs) crashes. Address this by saving/moving rhs to a temporary first, which clears rhs and avoids its destruction with *this. The crash can be reproduced in test Value.MergeDuplicateKey by using the CrtAllocator instead of the default Document's MemoryPoolAllocator. --- include/rapidjson/document.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/include/rapidjson/document.h b/include/rapidjson/document.h index 0910122..61d031b 100644 --- a/include/rapidjson/document.h +++ b/include/rapidjson/document.h @@ -916,8 +916,13 @@ public: */ GenericValue& operator=(GenericValue& rhs) RAPIDJSON_NOEXCEPT { if (RAPIDJSON_LIKELY(this != &rhs)) { + // Can't destroy "this" before assigning "rhs", otherwise "rhs" + // could be used after free if it's an sub-Value of "this", + // hence the temporary danse. + GenericValue temp; + temp.RawAssign(rhs); this->~GenericValue(); - RawAssign(rhs); + RawAssign(temp); } return *this; } From 50cb424c348fe82814196e9ccc04e7efb69d82ca Mon Sep 17 00:00:00 2001 From: ylavic Date: Mon, 15 Mar 2021 23:56:55 +0100 Subject: [PATCH 07/27] Test assignment from inner Value. --- test/unittest/valuetest.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/unittest/valuetest.cpp b/test/unittest/valuetest.cpp index 00f0652..f34d4b0 100644 --- a/test/unittest/valuetest.cpp +++ b/test/unittest/valuetest.cpp @@ -1119,6 +1119,16 @@ TEST(Value, Array) { z.SetArray(); EXPECT_TRUE(z.IsArray()); EXPECT_TRUE(z.Empty()); + + // PR #1503: assign from inner Value + { + CrtAllocator a; // Free() is not a noop + GenericValue, CrtAllocator> nullValue; + GenericValue, CrtAllocator> arrayValue(kArrayType); + arrayValue.PushBack(nullValue, a); + arrayValue = arrayValue[0]; // shouldn't crash (use after free) + EXPECT_TRUE(arrayValue.IsNull()); + } } TEST(Value, ArrayHelper) { From 49e4dd619fadabeee4685ac23a2154908a650b92 Mon Sep 17 00:00:00 2001 From: ylavic Date: Fri, 12 Mar 2021 15:16:02 +0100 Subject: [PATCH 08/27] Provide StdAllocator, STL compatible, for use with STL types. --- include/rapidjson/allocators.h | 421 +++++++++++++++++++++++++++++---- 1 file changed, 369 insertions(+), 52 deletions(-) diff --git a/include/rapidjson/allocators.h b/include/rapidjson/allocators.h index 44ec529..03b2dcc 100644 --- a/include/rapidjson/allocators.h +++ b/include/rapidjson/allocators.h @@ -17,6 +17,8 @@ #include "rapidjson.h" +#include + RAPIDJSON_NAMESPACE_BEGIN /////////////////////////////////////////////////////////////////////////////// @@ -89,7 +91,14 @@ public: } return RAPIDJSON_REALLOC(originalPtr, newSize); } - static void Free(void *ptr) { RAPIDJSON_FREE(ptr); } + static void Free(void *ptr) RAPIDJSON_NOEXCEPT { RAPIDJSON_FREE(ptr); } + + bool operator==(const CrtAllocator&) const RAPIDJSON_NOEXCEPT { + return true; + } + bool operator!=(const CrtAllocator&) const RAPIDJSON_NOEXCEPT { + return false; + } }; /////////////////////////////////////////////////////////////////////////////// @@ -113,6 +122,36 @@ public: */ template class MemoryPoolAllocator { + //! Chunk header for perpending to each chunk. + /*! Chunks are stored as a singly linked list. + */ + struct ChunkHeader { + size_t capacity; //!< Capacity of the chunk in bytes (excluding the header itself). + size_t size; //!< Current size of allocated memory in bytes. + ChunkHeader *next; //!< Next chunk in the linked list. + }; + + struct SharedData { + ChunkHeader *chunkHead; //!< Head of the chunk linked-list. Only the head chunk serves allocation. + BaseAllocator* ownBaseAllocator; //!< base allocator created by this object. + size_t refcount; + bool ownBuffer; + }; + + static const size_t SIZEOF_SHARED_DATA = RAPIDJSON_ALIGN(sizeof(SharedData)); + static const size_t SIZEOF_CHUNK_HEADER = RAPIDJSON_ALIGN(sizeof(ChunkHeader)); + + static inline ChunkHeader *GetChunkHead(SharedData *shared) + { + return reinterpret_cast(reinterpret_cast(shared) + SIZEOF_SHARED_DATA); + } + static inline uint8_t *GetChunkBuffer(SharedData *shared) + { + return reinterpret_cast(shared->chunkHead) + SIZEOF_CHUNK_HEADER; + } + + static const size_t kDefaultChunkCapacity = RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY; //!< Default chunk capacity. + public: static const bool kNeedFree = false; //!< Tell users that no need to call Free() with this allocator. (concept Allocator) @@ -120,9 +159,26 @@ public: /*! \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. \param baseAllocator The allocator for allocating memory chunks. */ + explicit MemoryPoolAllocator(size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : - chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(0), baseAllocator_(baseAllocator), ownBaseAllocator_(0) + chunk_capacity_(chunkSize), + baseAllocator_(baseAllocator ? baseAllocator : RAPIDJSON_NEW(BaseAllocator)()), + shared_(static_cast(baseAllocator_ ? baseAllocator_->Malloc(SIZEOF_SHARED_DATA + SIZEOF_CHUNK_HEADER) : 0)) { + RAPIDJSON_ASSERT(baseAllocator_ != 0); + RAPIDJSON_ASSERT(shared_ != 0); + if (baseAllocator) { + shared_->ownBaseAllocator = 0; + } + else { + shared_->ownBaseAllocator = baseAllocator_; + } + shared_->chunkHead = GetChunkHead(shared_); + shared_->chunkHead->capacity = 0; + shared_->chunkHead->size = 0; + shared_->chunkHead->next = 0; + shared_->ownBuffer = true; + shared_->refcount = 1; } //! Constructor with user-supplied buffer. @@ -136,41 +192,77 @@ public: \param baseAllocator The allocator for allocating memory chunks. */ MemoryPoolAllocator(void *buffer, size_t size, size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : - chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(buffer), baseAllocator_(baseAllocator), ownBaseAllocator_(0) + chunk_capacity_(chunkSize), + baseAllocator_(baseAllocator), + shared_(static_cast(AlignBuffer(buffer, size))) { - RAPIDJSON_ASSERT(buffer != 0); - RAPIDJSON_ASSERT(size > sizeof(ChunkHeader)); - chunkHead_ = reinterpret_cast(buffer); - chunkHead_->capacity = size - sizeof(ChunkHeader); - chunkHead_->size = 0; - chunkHead_->next = 0; + RAPIDJSON_ASSERT(size >= SIZEOF_SHARED_DATA + SIZEOF_CHUNK_HEADER); + shared_->chunkHead = GetChunkHead(shared_); + shared_->chunkHead->capacity = size - SIZEOF_SHARED_DATA - SIZEOF_CHUNK_HEADER; + shared_->chunkHead->size = 0; + shared_->chunkHead->next = 0; + shared_->ownBaseAllocator = 0; + shared_->ownBuffer = false; + shared_->refcount = 1; + } + + MemoryPoolAllocator(const MemoryPoolAllocator& rhs) RAPIDJSON_NOEXCEPT : + chunk_capacity_(rhs.chunk_capacity_), + baseAllocator_(rhs.baseAllocator_), + shared_(rhs.shared_) + { + RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); + ++shared_->refcount; + } + MemoryPoolAllocator& operator=(const MemoryPoolAllocator& rhs) RAPIDJSON_NOEXCEPT + { + RAPIDJSON_NOEXCEPT_ASSERT(rhs.shared_->refcount > 0); + ++rhs.shared_->refcount; + + this->~MemoryPoolAllocator(); + baseAllocator_ = rhs.baseAllocator_; + chunk_capacity_ = rhs.chunk_capacity_; + shared_ = rhs.shared_; + return *this; } //! Destructor. /*! This deallocates all memory chunks, excluding the user-supplied buffer. */ - ~MemoryPoolAllocator() { + ~MemoryPoolAllocator() RAPIDJSON_NOEXCEPT { + if (shared_->refcount > 1) { + --shared_->refcount; + return; + } Clear(); - RAPIDJSON_DELETE(ownBaseAllocator_); + BaseAllocator *a = shared_->ownBaseAllocator; + if (shared_->ownBuffer) { + baseAllocator_->Free(shared_); + } + RAPIDJSON_DELETE(a); } - //! Deallocates all memory chunks, excluding the user-supplied buffer. - void Clear() { - while (chunkHead_ && chunkHead_ != userBuffer_) { - ChunkHeader* next = chunkHead_->next; - baseAllocator_->Free(chunkHead_); - chunkHead_ = next; + //! Deallocates all memory chunks, excluding the first/user one. + void Clear() RAPIDJSON_NOEXCEPT { + RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); + for (;;) { + ChunkHeader* c = shared_->chunkHead; + if (!c->next) { + break; + } + shared_->chunkHead = c->next; + baseAllocator_->Free(c); } - if (chunkHead_ && chunkHead_ == userBuffer_) - chunkHead_->size = 0; // Clear user buffer + shared_->chunkHead->size = 0; } //! Computes the total capacity of allocated memory chunks. /*! \return total capacity in bytes. */ - size_t Capacity() const { + size_t Capacity() const RAPIDJSON_NOEXCEPT { + RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); size_t capacity = 0; - for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) + for (ChunkHeader* c = shared_->chunkHead; c != 0; c = c->next) capacity += c->capacity; return capacity; } @@ -178,25 +270,35 @@ public: //! Computes the memory blocks allocated. /*! \return total used bytes. */ - size_t Size() const { + size_t Size() const RAPIDJSON_NOEXCEPT { + RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); size_t size = 0; - for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) + for (ChunkHeader* c = shared_->chunkHead; c != 0; c = c->next) size += c->size; return size; } + //! Whether the allocator is shared. + /*! \return true or false. + */ + bool Shared() const RAPIDJSON_NOEXCEPT { + RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); + return shared_->refcount > 1; + } + //! Allocates a memory block. (concept Allocator) void* Malloc(size_t size) { + RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); if (!size) return NULL; size = RAPIDJSON_ALIGN(size); - if (chunkHead_ == 0 || chunkHead_->size + size > chunkHead_->capacity) + if (RAPIDJSON_UNLIKELY(shared_->chunkHead->size + size > shared_->chunkHead->capacity)) if (!AddChunk(chunk_capacity_ > size ? chunk_capacity_ : size)) return NULL; - void *buffer = reinterpret_cast(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size; - chunkHead_->size += size; + void *buffer = GetChunkBuffer(shared_) + shared_->chunkHead->size; + shared_->chunkHead->size += size; return buffer; } @@ -205,6 +307,7 @@ public: if (originalPtr == 0) return Malloc(newSize); + RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); if (newSize == 0) return NULL; @@ -216,10 +319,10 @@ public: return originalPtr; // Simply expand it if it is the last allocation and there is sufficient space - if (originalPtr == reinterpret_cast(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size - originalSize) { + if (originalPtr == GetChunkBuffer(shared_) + shared_->chunkHead->size - originalSize) { size_t increment = static_cast(newSize - originalSize); - if (chunkHead_->size + increment <= chunkHead_->capacity) { - chunkHead_->size += increment; + if (shared_->chunkHead->size + increment <= shared_->chunkHead->capacity) { + shared_->chunkHead->size += increment; return originalPtr; } } @@ -235,50 +338,264 @@ public: } //! Frees a memory block (concept Allocator) - static void Free(void *ptr) { (void)ptr; } // Do nothing + static void Free(void *ptr) RAPIDJSON_NOEXCEPT { (void)ptr; } // Do nothing + + //! Compare (equality) with another MemoryPoolAllocator + bool operator==(const MemoryPoolAllocator& rhs) const RAPIDJSON_NOEXCEPT { + RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); + RAPIDJSON_NOEXCEPT_ASSERT(rhs.shared_->refcount > 0); + return shared_ == rhs.shared_; + } + //! Compare (inequality) with another MemoryPoolAllocator + bool operator!=(const MemoryPoolAllocator& rhs) const RAPIDJSON_NOEXCEPT { + return !operator==(rhs); + } private: - //! Copy constructor is not permitted. - MemoryPoolAllocator(const MemoryPoolAllocator& rhs) /* = delete */; - //! Copy assignment operator is not permitted. - MemoryPoolAllocator& operator=(const MemoryPoolAllocator& rhs) /* = delete */; - //! Creates a new chunk. /*! \param capacity Capacity of the chunk in bytes. \return true if success. */ bool AddChunk(size_t capacity) { if (!baseAllocator_) - ownBaseAllocator_ = baseAllocator_ = RAPIDJSON_NEW(BaseAllocator)(); - if (ChunkHeader* chunk = reinterpret_cast(baseAllocator_->Malloc(RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + capacity))) { + shared_->ownBaseAllocator = baseAllocator_ = RAPIDJSON_NEW(BaseAllocator)(); + if (ChunkHeader* chunk = static_cast(baseAllocator_->Malloc(SIZEOF_CHUNK_HEADER + capacity))) { chunk->capacity = capacity; chunk->size = 0; - chunk->next = chunkHead_; - chunkHead_ = chunk; + chunk->next = shared_->chunkHead; + shared_->chunkHead = chunk; return true; } else return false; } - static const int kDefaultChunkCapacity = RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY; //!< Default chunk capacity. + static inline void* AlignBuffer(void* buf, size_t &size) + { + RAPIDJSON_NOEXCEPT_ASSERT(buf != 0); + const uintptr_t mask = sizeof(void*) - 1; + const uintptr_t ubuf = reinterpret_cast(buf); + if (RAPIDJSON_UNLIKELY(ubuf & mask)) { + const uintptr_t abuf = (ubuf + mask) & ~mask; + RAPIDJSON_ASSERT(size >= abuf - ubuf); + buf = reinterpret_cast(abuf); + size -= abuf - ubuf; + } + return buf; + } - //! Chunk header for perpending to each chunk. - /*! Chunks are stored as a singly linked list. - */ - struct ChunkHeader { - size_t capacity; //!< Capacity of the chunk in bytes (excluding the header itself). - size_t size; //!< Current size of allocated memory in bytes. - ChunkHeader *next; //!< Next chunk in the linked list. + size_t chunk_capacity_; //!< The minimum capacity of chunk when they are allocated. + BaseAllocator* baseAllocator_; //!< base allocator for allocating memory chunks. + SharedData *shared_; //!< The shared data of the allocator +}; + + +template +inline T* Realloc(A& a, T* old_p, size_t old_n, size_t new_n) +{ + RAPIDJSON_NOEXCEPT_ASSERT(old_n <= SIZE_MAX / sizeof(T) && new_n <= SIZE_MAX / sizeof(T)); + return static_cast(a.Realloc(old_p, old_n * sizeof(T), new_n * sizeof(T))); +} + +template +inline T *Malloc(A& a, size_t n = 1) +{ + return Realloc(a, NULL, 0, n); +} + +template +inline void Free(A& a, T *p, size_t n = 1) +{ + static_cast(Realloc(a, p, n, 0)); +} + + +#ifdef __GNUC__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) // std::allocator can safely be inherited +#endif + +template +class StdAllocator : + public std::allocator +{ + typedef std::allocator allocator_type; + +public: + typedef BaseAllocator BaseAllocatorType; + + StdAllocator() RAPIDJSON_NOEXCEPT : + allocator_type(), + baseAllocator_() + { } + + StdAllocator(const StdAllocator& rhs) RAPIDJSON_NOEXCEPT : + allocator_type(rhs), + baseAllocator_(rhs.baseAllocator_) + { } + + template + StdAllocator(const StdAllocator& rhs) RAPIDJSON_NOEXCEPT : + allocator_type(rhs), + baseAllocator_(rhs.baseAllocator_) + { } + + /* implicit */ + StdAllocator(const BaseAllocator& allocator) RAPIDJSON_NOEXCEPT : + allocator_type(), + baseAllocator_(allocator) + { } + + ~StdAllocator() RAPIDJSON_NOEXCEPT + { } + + typedef typename allocator_type::value_type value_type; + typedef typename allocator_type::pointer pointer; + typedef typename allocator_type::const_pointer const_pointer; + typedef typename allocator_type::reference reference; + typedef typename allocator_type::const_reference const_reference; + typedef typename allocator_type::size_type size_type; + typedef typename allocator_type::difference_type difference_type; + + template + struct rebind { + typedef StdAllocator other; }; - ChunkHeader *chunkHead_; //!< Head of the chunk linked-list. Only the head chunk serves allocation. - size_t chunk_capacity_; //!< The minimum capacity of chunk when they are allocated. - void *userBuffer_; //!< User supplied buffer. - BaseAllocator* baseAllocator_; //!< base allocator for allocating memory chunks. - BaseAllocator* ownBaseAllocator_; //!< base allocator created by this object. +#if RAPIDJSON_HAS_CXX11 + using allocator_type::max_size; + using allocator_type::address; + using allocator_type::construct; + using allocator_type::destroy; +#else + size_t max_size() const RAPIDJSON_NOEXCEPT + { + return allocator_type::max_size(); + } + + pointer address(reference r) const RAPIDJSON_NOEXCEPT + { + return allocator_type::address(r); + } + const_pointer address(const_reference r) const RAPIDJSON_NOEXCEPT + { + return allocator_type::address(r); + } + + void construct(pointer p, const_reference r) + { + allocator_type::construct(p, r); + } + void destroy(pointer p) + { + allocator_type::destroy(p); + } +#endif + + template + U* allocate(size_type n = 1, const void* = 0) + { + return RAPIDJSON_NAMESPACE::Malloc(baseAllocator_, n); + } + template + void deallocate(U* p, size_type n = 1) + { + RAPIDJSON_NAMESPACE::Free(baseAllocator_, p, n); + } + + pointer allocate(size_type n = 1, const void* = 0) + { + return allocate(n); + } + void deallocate(pointer p, size_type n = 1) + { + deallocate(p, n); + } + + template + bool operator==(const StdAllocator& rhs) const RAPIDJSON_NOEXCEPT + { + return baseAllocator_ == rhs.baseAllocator_; + } + template + bool operator!=(const StdAllocator& rhs) const RAPIDJSON_NOEXCEPT + { + return !operator==(rhs); + } + + //! rapidjson Allocator concept + void* Malloc(size_t size) + { + return baseAllocator_.Malloc(size); + } + void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) + { + return baseAllocator_.Realloc(originalPtr, originalSize, newSize); + } + static void Free(void *ptr) RAPIDJSON_NOEXCEPT + { + BaseAllocator::Free(ptr); + } + +private: + template + friend class StdAllocator; // access to StdAllocator.* + + BaseAllocator baseAllocator_; }; +template +class StdAllocator : + public std::allocator +{ + typedef std::allocator allocator_type; + +public: + typedef BaseAllocator BaseAllocatorType; + + StdAllocator() RAPIDJSON_NOEXCEPT : + allocator_type(), + baseAllocator_() + { } + + StdAllocator(const StdAllocator& rhs) RAPIDJSON_NOEXCEPT : + allocator_type(rhs), + baseAllocator_(rhs.baseAllocator_) + { } + + template + StdAllocator(const StdAllocator& rhs) RAPIDJSON_NOEXCEPT : + allocator_type(rhs), + baseAllocator_(rhs.baseAllocator_) + { } + + /* implicit */ + StdAllocator(const BaseAllocator& allocator) RAPIDJSON_NOEXCEPT : + allocator_type(), + baseAllocator_(allocator) + { } + + ~StdAllocator() RAPIDJSON_NOEXCEPT + { } + + typedef typename allocator_type::value_type value_type; + + template + struct rebind { + typedef StdAllocator other; + }; + +private: + template + friend class StdAllocator; // access to StdAllocator.* + + BaseAllocator baseAllocator_; +}; + +#ifdef __GNUC__ +RAPIDJSON_DIAG_POP +#endif + RAPIDJSON_NAMESPACE_END #endif // RAPIDJSON_ENCODINGS_H_ From 2e6f761458960dbf2d6aff9dc0f82db1bc243526 Mon Sep 17 00:00:00 2001 From: ylavic Date: Sat, 13 Mar 2021 02:11:17 +0100 Subject: [PATCH 09/27] Tests for StdAllocator. --- test/unittest/allocatorstest.cpp | 166 ++++++++++++++++++++++++++++++- 1 file changed, 165 insertions(+), 1 deletion(-) diff --git a/test/unittest/allocatorstest.cpp b/test/unittest/allocatorstest.cpp index c541f04..b8a2191 100644 --- a/test/unittest/allocatorstest.cpp +++ b/test/unittest/allocatorstest.cpp @@ -16,6 +16,9 @@ #include "rapidjson/allocators.h" +#include +#include + using namespace rapidjson; template @@ -47,19 +50,180 @@ void TestAllocator(Allocator& a) { EXPECT_TRUE(a.Realloc(a.Malloc(1), 1, 0) == 0); } +struct TestStdAllocatorData { + TestStdAllocatorData(int &constructions, int &destructions) : + constructions_(&constructions), + destructions_(&destructions) + { + ++*constructions_; + } + TestStdAllocatorData(const TestStdAllocatorData& rhs) : + constructions_(rhs.constructions_), + destructions_(rhs.destructions_) + { + ++*constructions_; + } + TestStdAllocatorData& operator=(const TestStdAllocatorData& rhs) + { + this->~TestStdAllocatorData(); + constructions_ = rhs.constructions_; + destructions_ = rhs.destructions_; + ++*constructions_; + return *this; + } + ~TestStdAllocatorData() + { + ++*destructions_; + } +private: + TestStdAllocatorData(); + int *constructions_, + *destructions_; +}; + +template +void TestStdAllocator(const Allocator& a) { + typedef StdAllocator VoidAllocator; + typedef typename VoidAllocator::template rebind::other BoolAllocator; + BoolAllocator ba(a), ba2(a); + EXPECT_TRUE(ba == ba2); + EXPECT_FALSE(ba!= ba2); + ba.deallocate(ba.allocate()); + EXPECT_TRUE(ba == ba2); + EXPECT_FALSE(ba != ba2); + + unsigned long long ll = 0, *llp = ≪ + const unsigned long long cll = 0, *cllp = &cll; + StdAllocator lla(a); + EXPECT_EQ(lla.address(ll), llp); + EXPECT_EQ(lla.address(cll), cllp); + EXPECT_TRUE(lla.max_size() > 0 && lla.max_size() <= SIZE_MAX / sizeof(unsigned long long)); + + int *arr; + StdAllocator ia(a); + arr = ia.allocate(10 * sizeof(int)); + EXPECT_TRUE(arr != 0); + for (int i = 0; i < 10; ++i) { + arr[i] = 0x0f0f0f0f; + } + ia.deallocate(arr, 10); + arr = (int *)ia.Malloc(10 * sizeof(int)); + EXPECT_TRUE(arr != 0); + for (int i = 0; i < 10; ++i) { + arr[i] = 0x0f0f0f0f; + } + arr = (int *)ia.Realloc(arr, 10 * sizeof(int), 20 * sizeof(int)); + EXPECT_TRUE(arr != 0); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(arr[i], 0x0f0f0f0f); + } + for (int i = 10; i < 20; i++) { + arr[i] = 0x0f0f0f0f; + } + ia.Free(arr); + + int cons = 0, dest = 0; + StdAllocator da(a); + for (int i = 1; i < 10; i++) { + TestStdAllocatorData *d = da.allocate(); + EXPECT_TRUE(d != 0); + + da.destroy(new(d) TestStdAllocatorData(cons, dest)); + EXPECT_EQ(cons, i); + EXPECT_EQ(dest, i); + + da.deallocate(d); + } + + typedef StdAllocator CharAllocator; + typedef std::basic_string, CharAllocator> String; + CharAllocator ca(a); + String s(ca); + for (int i = 0; i < 26; i++) { + s.push_back(static_cast('A' + i)); + } + EXPECT_TRUE(s == "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); +} + TEST(Allocator, CrtAllocator) { CrtAllocator a; + TestAllocator(a); + TestStdAllocator(a); + + CrtAllocator a2; + EXPECT_TRUE(a == a2); + EXPECT_FALSE(a != a2); + a2.Free(a2.Malloc(1)); + EXPECT_TRUE(a == a2); + EXPECT_FALSE(a != a2); } TEST(Allocator, MemoryPoolAllocator) { - MemoryPoolAllocator<> a; + const size_t capacity = RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY; + MemoryPoolAllocator<> a(capacity); + + a.Clear(); // noop + EXPECT_EQ(a.Size(), 0u); + EXPECT_EQ(a.Capacity(), 0u); + EXPECT_EQ(a.Shared(), false); + { + MemoryPoolAllocator<> a2(a); + EXPECT_EQ(a2.Shared(), true); + EXPECT_EQ(a.Shared(), true); + EXPECT_TRUE(a == a2); + EXPECT_FALSE(a != a2); + a2.Free(a2.Malloc(1)); + EXPECT_TRUE(a == a2); + EXPECT_FALSE(a != a2); + } + EXPECT_EQ(a.Shared(), false); + EXPECT_EQ(a.Capacity(), capacity); + EXPECT_EQ(a.Size(), 8u); // aligned + a.Clear(); + EXPECT_EQ(a.Capacity(), 0u); + EXPECT_EQ(a.Size(), 0u); + TestAllocator(a); + TestStdAllocator(a); for (size_t i = 1; i < 1000; i++) { EXPECT_TRUE(a.Malloc(i) != 0); EXPECT_LE(a.Size(), a.Capacity()); } + + CrtAllocator baseAllocator; + a = MemoryPoolAllocator<>(capacity, &baseAllocator); + EXPECT_EQ(a.Capacity(), 0u); + EXPECT_EQ(a.Size(), 0u); + a.Free(a.Malloc(1)); + EXPECT_EQ(a.Capacity(), capacity); + EXPECT_EQ(a.Size(), 8u); // aligned + + { + a.Clear(); + const size_t bufSize = 1024; + char *buffer = (char *)a.Malloc(bufSize); + MemoryPoolAllocator<> aligned_a(buffer, bufSize); + EXPECT_TRUE(aligned_a.Capacity() > 0 && aligned_a.Capacity() <= bufSize); + EXPECT_EQ(aligned_a.Size(), 0u); + aligned_a.Free(aligned_a.Malloc(1)); + EXPECT_TRUE(aligned_a.Capacity() > 0 && aligned_a.Capacity() <= bufSize); + EXPECT_EQ(aligned_a.Size(), 8u); // aligned + } + + { + a.Clear(); + const size_t bufSize = 1024; + char *buffer = (char *)a.Malloc(bufSize); + RAPIDJSON_ASSERT(bufSize % sizeof(void*) == 0); + MemoryPoolAllocator<> unaligned_a(buffer + 1, bufSize - 1); + EXPECT_TRUE(unaligned_a.Capacity() > 0 && unaligned_a.Capacity() <= bufSize - sizeof(void*)); + EXPECT_EQ(unaligned_a.Size(), 0u); + unaligned_a.Free(unaligned_a.Malloc(1)); + EXPECT_TRUE(unaligned_a.Capacity() > 0 && unaligned_a.Capacity() <= bufSize - sizeof(void*)); + EXPECT_EQ(unaligned_a.Size(), 8u); // aligned + } } TEST(Allocator, Alignment) { From 3d77d11e28aaa32bd8c8a35b2d306a586c953865 Mon Sep 17 00:00:00 2001 From: miloyip Date: Wed, 24 Mar 2021 16:51:12 +0800 Subject: [PATCH 10/27] add traverse as pointer example --- example/traverseaspointer.cpp | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 example/traverseaspointer.cpp diff --git a/example/traverseaspointer.cpp b/example/traverseaspointer.cpp new file mode 100644 index 0000000..7e0c899 --- /dev/null +++ b/example/traverseaspointer.cpp @@ -0,0 +1,39 @@ +#include "rapidjson/document.h" +#include "rapidjson/filereadstream.h" +#include "rapidjson/pointer.h" +#include "rapidjson/stringbuffer.h" +#include + +using namespace rapidjson; + +void traverse(const Value& v, const Pointer& p) { + StringBuffer sb; + p.Stringify(sb); + std::cout << sb.GetString() << std::endl; + + switch (v.GetType()) { + case kArrayType: + for (SizeType i = 0; i != v.Size(); ++i) + traverse(v[i], p.Append(i)); + break; + case kObjectType: + for (Value::ConstMemberIterator m = v.MemberBegin(); m != v.MemberEnd(); ++m) + traverse(m->value, p.Append(m->name.GetString(), m->name.GetStringLength())); + break; + default: + break; + } +} + +int main(int, char*[]) { + char readBuffer[65536]; + FileReadStream is(stdin, readBuffer, sizeof(readBuffer)); + + Document d; + d.ParseStream(is); + + Pointer root; + traverse(d, root); + + return 0; +} From 02f42604bd8d1ca7a65fb981c520d61174ab7585 Mon Sep 17 00:00:00 2001 From: ylavic Date: Sun, 28 Mar 2021 12:31:52 +0200 Subject: [PATCH 11/27] Make StdAllocator C++17-20 compatible. --- include/rapidjson/allocators.h | 67 +++++++++++++++++++++++++------- include/rapidjson/rapidjson.h | 31 +++++++++++---- test/unittest/allocatorstest.cpp | 6 +-- 3 files changed, 78 insertions(+), 26 deletions(-) diff --git a/include/rapidjson/allocators.h b/include/rapidjson/allocators.h index 03b2dcc..4265214 100644 --- a/include/rapidjson/allocators.h +++ b/include/rapidjson/allocators.h @@ -19,6 +19,11 @@ #include +#if RAPIDJSON_HAS_CXX11 +#include +#include +#endif + RAPIDJSON_NAMESPACE_BEGIN /////////////////////////////////////////////////////////////////////////////// @@ -420,6 +425,11 @@ class StdAllocator : public std::allocator { typedef std::allocator allocator_type; +#if RAPIDJSON_HAS_CXX11 + typedef std::allocator_traits traits_type; +#else + typedef allocator_type traits_type; +#endif public: typedef BaseAllocator BaseAllocatorType; @@ -449,30 +459,51 @@ public: ~StdAllocator() RAPIDJSON_NOEXCEPT { } - typedef typename allocator_type::value_type value_type; - typedef typename allocator_type::pointer pointer; - typedef typename allocator_type::const_pointer const_pointer; - typedef typename allocator_type::reference reference; - typedef typename allocator_type::const_reference const_reference; - typedef typename allocator_type::size_type size_type; - typedef typename allocator_type::difference_type difference_type; - template struct rebind { typedef StdAllocator other; }; + typedef typename traits_type::size_type size_type; + typedef typename traits_type::difference_type difference_type; + typedef typename traits_type::value_type value_type; + typedef typename traits_type::pointer pointer; + typedef typename traits_type::const_pointer const_pointer; + #if RAPIDJSON_HAS_CXX11 - using allocator_type::max_size; - using allocator_type::address; - using allocator_type::construct; - using allocator_type::destroy; -#else + + typedef typename std::add_lvalue_reference::type &reference; + typedef typename std::add_lvalue_reference::type>::type &const_reference; + + pointer address(reference r) const RAPIDJSON_NOEXCEPT + { + return std::addressof(r); + } + const_pointer address(const_reference r) const RAPIDJSON_NOEXCEPT + { + return std::addressof(r); + } + size_t max_size() const RAPIDJSON_NOEXCEPT { - return allocator_type::max_size(); + return std::numeric_limits::max() / sizeof(value_type); } + template + void construct(pointer p, Args &&...args) + { + ::new (static_cast(p)) value_type(std::forward(args)...); + } + void destroy(pointer p) + { + p->~T(); + } + +#else // !RAPIDJSON_HAS_CXX11 + + typedef typename allocator_type::reference reference; + typedef typename allocator_type::const_reference const_reference; + pointer address(reference r) const RAPIDJSON_NOEXCEPT { return allocator_type::address(r); @@ -482,6 +513,11 @@ public: return allocator_type::address(r); } + size_t max_size() const RAPIDJSON_NOEXCEPT + { + return allocator_type::max_size(); + } + void construct(pointer p, const_reference r) { allocator_type::construct(p, r); @@ -490,7 +526,8 @@ public: { allocator_type::destroy(p); } -#endif + +#endif // !RAPIDJSON_HAS_CXX11 template U* allocate(size_type n = 1, const void* = 0) diff --git a/include/rapidjson/rapidjson.h b/include/rapidjson/rapidjson.h index 8034c49..8dcb0a0 100644 --- a/include/rapidjson/rapidjson.h +++ b/include/rapidjson/rapidjson.h @@ -124,6 +124,17 @@ #define RAPIDJSON_NAMESPACE_END } #endif +/////////////////////////////////////////////////////////////////////////////// +// __cplusplus macro + +//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN + +#if defined(_MSC_VER) +#define RAPIDJSON_CPLUSPLUS _MSVC_LANG +#else +#define RAPIDJSON_CPLUSPLUS __cplusplus +#endif + /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_HAS_STDSTRING @@ -411,7 +422,7 @@ RAPIDJSON_NAMESPACE_END // Prefer C++11 static_assert, if available #ifndef RAPIDJSON_STATIC_ASSERT -#if __cplusplus >= 201103L || ( defined(_MSC_VER) && _MSC_VER >= 1800 ) +#if RAPIDJSON_CPLUSPLUS >= 201103L || ( defined(_MSC_VER) && _MSC_VER >= 1800 ) #define RAPIDJSON_STATIC_ASSERT(x) \ static_assert(x, RAPIDJSON_STRINGIFY(x)) #endif // C++11 @@ -542,7 +553,7 @@ RAPIDJSON_NAMESPACE_END // C++11 features #ifndef RAPIDJSON_HAS_CXX11 -#define RAPIDJSON_HAS_CXX11 (__cplusplus >= 201103L) +#define RAPIDJSON_HAS_CXX11 (RAPIDJSON_CPLUSPLUS >= 201103L) #endif #ifndef RAPIDJSON_HAS_CXX11_RVALUE_REFS @@ -610,12 +621,16 @@ RAPIDJSON_NAMESPACE_END /////////////////////////////////////////////////////////////////////////////// // C++17 features -#if defined(__has_cpp_attribute) -# if __has_cpp_attribute(fallthrough) -# define RAPIDJSON_DELIBERATE_FALLTHROUGH [[fallthrough]] -# else -# define RAPIDJSON_DELIBERATE_FALLTHROUGH -# endif +#ifndef RAPIDJSON_HAS_CXX17 +#define RAPIDJSON_HAS_CXX17 (RAPIDJSON_CPLUSPLUS >= 201703L) +#endif + +#if RAPIDJSON_HAS_CXX17 +# define RAPIDJSON_DELIBERATE_FALLTHROUGH [[fallthrough]] +#elif defined(__has_cpp_attribute) && __has_cpp_attribute(fallthrough) +# define RAPIDJSON_DELIBERATE_FALLTHROUGH __attribute__((fallthrough)) +#elif defined(__has_cpp_attribute) && __has_cpp_attribute(clang::fallthrough) +# define RAPIDJSON_DELIBERATE_FALLTHROUGH [[clang::fallthrough]] #else # define RAPIDJSON_DELIBERATE_FALLTHROUGH #endif diff --git a/test/unittest/allocatorstest.cpp b/test/unittest/allocatorstest.cpp index b8a2191..76e34b5 100644 --- a/test/unittest/allocatorstest.cpp +++ b/test/unittest/allocatorstest.cpp @@ -107,12 +107,12 @@ void TestStdAllocator(const Allocator& a) { arr[i] = 0x0f0f0f0f; } ia.deallocate(arr, 10); - arr = (int *)ia.Malloc(10 * sizeof(int)); + arr = Malloc(ia, 10); EXPECT_TRUE(arr != 0); for (int i = 0; i < 10; ++i) { arr[i] = 0x0f0f0f0f; } - arr = (int *)ia.Realloc(arr, 10 * sizeof(int), 20 * sizeof(int)); + arr = Realloc(ia, arr, 10, 20); EXPECT_TRUE(arr != 0); for (int i = 0; i < 10; ++i) { EXPECT_EQ(arr[i], 0x0f0f0f0f); @@ -120,7 +120,7 @@ void TestStdAllocator(const Allocator& a) { for (int i = 10; i < 20; i++) { arr[i] = 0x0f0f0f0f; } - ia.Free(arr); + Free(ia, arr, 20); int cons = 0, dest = 0; StdAllocator da(a); From 08cf9a56c0f3d648ef615c3d5f60d08941ebd3ec Mon Sep 17 00:00:00 2001 From: ylavic Date: Mon, 29 Mar 2021 00:17:24 +0200 Subject: [PATCH 12/27] Make StdAllocator C++17-20 compatible. --- include/rapidjson/allocators.h | 18 +++++++++--------- include/rapidjson/rapidjson.h | 14 ++++++++++---- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/include/rapidjson/allocators.h b/include/rapidjson/allocators.h index 4265214..2871542 100644 --- a/include/rapidjson/allocators.h +++ b/include/rapidjson/allocators.h @@ -20,7 +20,6 @@ #include #if RAPIDJSON_HAS_CXX11 -#include #include #endif @@ -466,6 +465,7 @@ public: typedef typename traits_type::size_type size_type; typedef typename traits_type::difference_type difference_type; + typedef typename traits_type::value_type value_type; typedef typename traits_type::pointer pointer; typedef typename traits_type::const_pointer const_pointer; @@ -484,19 +484,19 @@ public: return std::addressof(r); } - size_t max_size() const RAPIDJSON_NOEXCEPT + size_type max_size() const RAPIDJSON_NOEXCEPT { - return std::numeric_limits::max() / sizeof(value_type); + return traits_type::max_size(*this); } template - void construct(pointer p, Args &&...args) + void construct(pointer p, Args&&... args) { - ::new (static_cast(p)) value_type(std::forward(args)...); + traits_type::construct(*this, p, std::forward(args)...); } void destroy(pointer p) { - p->~T(); + traits_type::destroy(*this, p); } #else // !RAPIDJSON_HAS_CXX11 @@ -513,7 +513,7 @@ public: return allocator_type::address(r); } - size_t max_size() const RAPIDJSON_NOEXCEPT + size_type max_size() const RAPIDJSON_NOEXCEPT { return allocator_type::max_size(); } @@ -615,13 +615,13 @@ public: ~StdAllocator() RAPIDJSON_NOEXCEPT { } - typedef typename allocator_type::value_type value_type; - template struct rebind { typedef StdAllocator other; }; + typedef typename allocator_type::value_type value_type; + private: template friend class StdAllocator; // access to StdAllocator.* diff --git a/include/rapidjson/rapidjson.h b/include/rapidjson/rapidjson.h index 8dcb0a0..1093581 100644 --- a/include/rapidjson/rapidjson.h +++ b/include/rapidjson/rapidjson.h @@ -135,6 +135,8 @@ #define RAPIDJSON_CPLUSPLUS __cplusplus #endif +//!@endcond + /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_HAS_STDSTRING @@ -627,10 +629,14 @@ RAPIDJSON_NAMESPACE_END #if RAPIDJSON_HAS_CXX17 # define RAPIDJSON_DELIBERATE_FALLTHROUGH [[fallthrough]] -#elif defined(__has_cpp_attribute) && __has_cpp_attribute(fallthrough) -# define RAPIDJSON_DELIBERATE_FALLTHROUGH __attribute__((fallthrough)) -#elif defined(__has_cpp_attribute) && __has_cpp_attribute(clang::fallthrough) -# define RAPIDJSON_DELIBERATE_FALLTHROUGH [[clang::fallthrough]] +#elif defined(__has_cpp_attribute) +# if __has_cpp_attribute(clang::fallthrough) +# define RAPIDJSON_DELIBERATE_FALLTHROUGH [[clang::fallthrough]] +# elif __has_cpp_attribute(fallthrough) +# define RAPIDJSON_DELIBERATE_FALLTHROUGH __attribute__((fallthrough)) +# else +# define RAPIDJSON_DELIBERATE_FALLTHROUGH +# endif #else # define RAPIDJSON_DELIBERATE_FALLTHROUGH #endif From e336667b4a05b20d4c641def7130bf0c1fbd4cc2 Mon Sep 17 00:00:00 2001 From: ylavic Date: Mon, 29 Mar 2021 17:42:30 +0200 Subject: [PATCH 13/27] Handle C++17 (and C++11 with MSVC) in CI. --- .travis.yml | 43 ++++++++++++++++++++++++++++--------------- CMakeLists.txt | 19 ++++++++++++++++++- appveyor.yml | 32 +++++++++++++++++++++++++++++++- 3 files changed, 77 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8f34664..7e7a996 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,51 +28,63 @@ env: matrix: include: # gcc - - env: CONF=release ARCH=x86 CXX11=ON + - env: CONF=release ARCH=x86 CXX11=ON CXX17=OFF compiler: gcc arch: amd64 - - env: CONF=release ARCH=x86_64 CXX11=ON + - env: CONF=release ARCH=x86_64 CXX11=ON CXX17=OFF compiler: gcc arch: amd64 - - env: CONF=debug ARCH=x86 CXX11=OFF + - env: CONF=debug ARCH=x86 CXX11=OFF CXX17=OFF compiler: gcc arch: amd64 - - env: CONF=debug ARCH=x86_64 CXX11=OFF + - env: CONF=debug ARCH=x86_64 CXX11=OFF CXX17=OFF compiler: gcc arch: amd64 - - env: CONF=release ARCH=aarch64 CXX11=ON + - env: CONF=debug ARCH=x86 CXX11=OFF CXX17=ON + compiler: gcc + arch: amd64 + - env: CONF=debug ARCH=x86_64 CXX11=OFF CXX17=ON + compiler: gcc + arch: amd64 + - env: CONF=release ARCH=aarch64 CXX11=ON CXX17=OFF compiler: gcc arch: arm64 - - env: CONF=debug ARCH=aarch64 CXX11=OFF + - env: CONF=release ARCH=aarch64 CXX11=OFF CXX17=OFF + compiler: gcc + arch: arm64 + - env: CONF=release ARCH=aarch64 CXX11=OFF CXX17=ON compiler: gcc arch: arm64 # clang - - env: CONF=debug ARCH=x86 CXX11=ON CCACHE_CPP2=yes + - env: CONF=release ARCH=x86 CXX11=ON CXX17=OFF CCACHE_CPP2=yes compiler: clang arch: amd64 - - env: CONF=debug ARCH=x86_64 CXX11=ON CCACHE_CPP2=yes + - env: CONF=release ARCH=x86_64 CXX11=ON CXX17=OFF CCACHE_CPP2=yes compiler: clang arch: amd64 - - env: CONF=debug ARCH=x86 CXX11=OFF CCACHE_CPP2=yes + - env: CONF=debug ARCH=x86 CXX11=OFF CXX17=OFF CCACHE_CPP2=yes compiler: clang arch: amd64 - - env: CONF=debug ARCH=x86_64 CXX11=OFF CCACHE_CPP2=yes + - env: CONF=debug ARCH=x86_64 CXX11=OFF CXX17=OFF CCACHE_CPP2=yes compiler: clang arch: amd64 - - env: CONF=release ARCH=x86 CXX11=ON CCACHE_CPP2=yes + - env: CONF=debug ARCH=x86 CXX11=OFF CXX17=ON CCACHE_CPP2=yes compiler: clang arch: amd64 - - env: CONF=release ARCH=x86_64 CXX11=ON CCACHE_CPP2=yes + - env: CONF=debug ARCH=x86_64 CXX11=OFF CXX17=ON CCACHE_CPP2=yes compiler: clang arch: amd64 - - env: CONF=debug ARCH=aarch64 CXX11=ON CCACHE_CPP2=yes + - env: CONF=debug ARCH=aarch64 CXX11=ON CXX17=OFF CCACHE_CPP2=yes compiler: clang arch: arm64 - - env: CONF=debug ARCH=aarch64 CXX11=OFF CCACHE_CPP2=yes + - env: CONF=debug ARCH=aarch64 CXX11=OFF CXX17=OFF CCACHE_CPP2=yes + compiler: clang + arch: arm64 + - env: CONF=debug ARCH=aarch64 CXX11=OFF CXX17=ON CCACHE_CPP2=yes compiler: clang arch: arm64 # coverage report - - env: CONF=debug ARCH=x86 CXX11=ON GCOV_FLAGS='--coverage' + - env: CONF=debug ARCH=x86 CXX11=ON CXX17=OFF GCOV_FLAGS='--coverage' compiler: gcc arch: amd64 cache: @@ -135,6 +147,7 @@ script: (cd build && cmake -DRAPIDJSON_HAS_STDSTRING=ON -DRAPIDJSON_BUILD_CXX11=$CXX11 + -DRAPIDJSON_BUILD_CXX17=$CXX17 -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_BUILD_TYPE=$CONF -DCMAKE_CXX_FLAGS="$ARCH_FLAGS $GCOV_FLAGS" diff --git a/CMakeLists.txt b/CMakeLists.txt index 3b9ac51..dcca04f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,7 +35,8 @@ option(RAPIDJSON_BUILD_TESTS "Build rapidjson perftests and unittests." ON) option(RAPIDJSON_BUILD_THIRDPARTY_GTEST "Use gtest installation in `thirdparty/gtest` by default if available" OFF) -option(RAPIDJSON_BUILD_CXX11 "Build rapidjson with C++11 (gcc/clang)" ON) +option(RAPIDJSON_BUILD_CXX11 "Build rapidjson with C++11" ON) +option(RAPIDJSON_BUILD_CXX17 "Build rapidjson with C++17" OFF) if(RAPIDJSON_BUILD_CXX11) set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED TRUE) @@ -77,6 +78,8 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") endif() + elseif (RAPIDJSON_BUILD_CXX17 AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS "5.0") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") endif() if (RAPIDJSON_BUILD_ASAN) if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS "4.8.0") @@ -105,6 +108,8 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(EXTRA_CXX_FLAGS -Weffc++ -Wswitch-default -Wfloat-equal -Wconversion -Wimplicit-fallthrough) if (RAPIDJSON_BUILD_CXX11 AND CMAKE_VERSION VERSION_LESS 3.1) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + elseif (RAPIDJSON_BUILD_CXX17 AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS "4.0") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") endif() if (RAPIDJSON_BUILD_ASAN) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") @@ -119,6 +124,18 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") elseif (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") add_definitions(-D_CRT_SECURE_NO_WARNINGS=1) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc") + # CMake >= 3.10 should handle the above CMAKE_CXX_STANDARD fine, otherwise use /std:c++XX with MSVC >= 19.10 + if (RAPIDJSON_BUILD_CXX11 AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS "19.10") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /std:c++11") + elseif (RAPIDJSON_BUILD_CXX17 AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS "19.14") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /std:c++17") + endif() + # Always compile with /WX + if(CMAKE_CXX_FLAGS MATCHES "/WX-") + string(REGEX REPLACE "/WX-" "/WX" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /WX") + endif() elseif (CMAKE_CXX_COMPILER_ID MATCHES "XL") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -qarch=auto") endif() diff --git a/appveyor.yml b/appveyor.yml index 376dc19..fc48130 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -13,37 +13,67 @@ environment: - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2013 VS_VERSION: 10 2010 VS_PLATFORM: win32 + CXX11: OFF + CXX17: OFF - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2013 VS_VERSION: 10 2010 VS_PLATFORM: x64 + CXX11: OFF + CXX17: OFF - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2013 VS_VERSION: 11 2012 VS_PLATFORM: win32 + CXX11: OFF + CXX17: OFF - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2013 VS_VERSION: 11 2012 VS_PLATFORM: x64 + CXX11: OFF + CXX17: OFF - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2013 VS_VERSION: 12 2013 VS_PLATFORM: win32 + CXX11: OFF + CXX17: OFF - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2013 VS_VERSION: 12 2013 VS_PLATFORM: x64 + CXX11: OFF + CXX17: OFF - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 VS_VERSION: 14 2015 VS_PLATFORM: win32 + CXX11: OFF + CXX17: OFF - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 VS_VERSION: 14 2015 VS_PLATFORM: x64 + CXX11: OFF + CXX17: OFF - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 VS_VERSION: 15 2017 VS_PLATFORM: win32 + CXX11: OFF + CXX17: OFF - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 VS_VERSION: 15 2017 VS_PLATFORM: x64 + CXX11: OFF + CXX17: OFF + - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + VS_VERSION: 15 2017 + VS_PLATFORM: x64 + CXX11: ON + CXX17: OFF + - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + VS_VERSION: 15 2017 + VS_PLATFORM: x64 + CXX11: OFF + CXX17: ON before_build: - git submodule update --init --recursive -- cmake -H. -BBuild/VS -G "Visual Studio %VS_VERSION%" -DCMAKE_GENERATOR_PLATFORM=%VS_PLATFORM% -DCMAKE_VERBOSE_MAKEFILE=ON -DBUILD_SHARED_LIBS=true -Wno-dev +- cmake -H. -BBuild/VS -G "Visual Studio %VS_VERSION%" -DCMAKE_GENERATOR_PLATFORM=%VS_PLATFORM% -DCMAKE_VERBOSE_MAKEFILE=ON -DBUILD_SHARED_LIBS=true -DRAPIDJSON_BUILD_CXX11=%CXX11% -DRAPIDJSON_BUILD_CXX17=%CXX17% -Wno-dev build: project: Build\VS\RapidJSON.sln From 6bed9b266f248de5d973e16bc27fdc13e68ad15d Mon Sep 17 00:00:00 2001 From: ylavic Date: Tue, 30 Mar 2021 13:47:04 +0200 Subject: [PATCH 14/27] Don't define StdAllocator from C++17. --- include/rapidjson/allocators.h | 2 ++ test/unittest/allocatorstest.cpp | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/include/rapidjson/allocators.h b/include/rapidjson/allocators.h index 2871542..711ac30 100644 --- a/include/rapidjson/allocators.h +++ b/include/rapidjson/allocators.h @@ -581,6 +581,7 @@ private: BaseAllocator baseAllocator_; }; +#if !RAPIDJSON_HAS_CXX17 // std::allocator deprecated in C++17 template class StdAllocator : public std::allocator @@ -628,6 +629,7 @@ private: BaseAllocator baseAllocator_; }; +#endif #ifdef __GNUC__ RAPIDJSON_DIAG_POP diff --git a/test/unittest/allocatorstest.cpp b/test/unittest/allocatorstest.cpp index 76e34b5..ec1b2ce 100644 --- a/test/unittest/allocatorstest.cpp +++ b/test/unittest/allocatorstest.cpp @@ -83,8 +83,12 @@ private: template void TestStdAllocator(const Allocator& a) { +#if RAPIDJSON_HAS_CXX17 + typedef StdAllocator BoolAllocator; +#else typedef StdAllocator VoidAllocator; typedef typename VoidAllocator::template rebind::other BoolAllocator; +#endif BoolAllocator ba(a), ba2(a); EXPECT_TRUE(ba == ba2); EXPECT_FALSE(ba!= ba2); From a8bd931766b9fc069731b9e86df60ff377991195 Mon Sep 17 00:00:00 2001 From: ylavic Date: Tue, 30 Mar 2021 15:58:17 +0200 Subject: [PATCH 15/27] Tests for C++17 with VS 2019. --- appveyor.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index fc48130..2e591ee 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -70,6 +70,11 @@ environment: VS_PLATFORM: x64 CXX11: OFF CXX17: ON + - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 + VS_VERSION: 16 2019 + VS_PLATFORM: x64 + CXX11: OFF + CXX17: ON before_build: - git submodule update --init --recursive From 117276c4135b0a3fbc0641a5c9f4d1ad9b44d785 Mon Sep 17 00:00:00 2001 From: ylavic Date: Sat, 27 Apr 2019 03:55:23 +0200 Subject: [PATCH 16/27] Fix would-crash tests if the default allocator used were kNeedFree. The allocator cannot be destroyed before the Document, otherwise the Value destructor double frees. --- test/unittest/documenttest.cpp | 2 ++ test/unittest/valuetest.cpp | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/unittest/documenttest.cpp b/test/unittest/documenttest.cpp index 472165f..c3d1e48 100644 --- a/test/unittest/documenttest.cpp +++ b/test/unittest/documenttest.cpp @@ -325,6 +325,8 @@ TEST(Document, Swap) { EXPECT_TRUE(d1.IsNull()); // reset document, including allocator + // so clear o before so that it doesnt contain dangling elements + o.Clear(); Document().Swap(d2); EXPECT_TRUE(d2.IsNull()); EXPECT_NE(&d2.GetAllocator(), &a); diff --git a/test/unittest/valuetest.cpp b/test/unittest/valuetest.cpp index f34d4b0..0a6b325 100644 --- a/test/unittest/valuetest.cpp +++ b/test/unittest/valuetest.cpp @@ -1078,9 +1078,9 @@ static void TestArray(T& x, Allocator& allocator) { } TEST(Value, Array) { + Value::AllocatorType allocator; Value x(kArrayType); const Value& y = x; - Value::AllocatorType allocator; EXPECT_EQ(kArrayType, x.GetType()); EXPECT_TRUE(x.IsArray()); @@ -1491,9 +1491,9 @@ static void TestObject(T& x, Allocator& allocator) { } TEST(Value, Object) { + Value::AllocatorType allocator; Value x(kObjectType); const Value& y = x; // const version - Value::AllocatorType allocator; EXPECT_EQ(kObjectType, x.GetType()); EXPECT_TRUE(x.IsObject()); From 683010b02d407953d73609cd7973e2999342e97b Mon Sep 17 00:00:00 2001 From: ylavic Date: Fri, 2 Apr 2021 14:35:29 +0200 Subject: [PATCH 17/27] Add rvalue copy and assignment to MemoryPoolAllocator and StdAllocator. --- include/rapidjson/allocators.h | 42 +++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/include/rapidjson/allocators.h b/include/rapidjson/allocators.h index 711ac30..cf8e75a 100644 --- a/include/rapidjson/allocators.h +++ b/include/rapidjson/allocators.h @@ -222,7 +222,6 @@ public: { RAPIDJSON_NOEXCEPT_ASSERT(rhs.shared_->refcount > 0); ++rhs.shared_->refcount; - this->~MemoryPoolAllocator(); baseAllocator_ = rhs.baseAllocator_; chunk_capacity_ = rhs.chunk_capacity_; @@ -230,10 +229,35 @@ public: return *this; } +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + MemoryPoolAllocator(MemoryPoolAllocator&& rhs) RAPIDJSON_NOEXCEPT : + chunk_capacity_(rhs.chunk_capacity_), + baseAllocator_(rhs.baseAllocator_), + shared_(rhs.shared_) + { + RAPIDJSON_NOEXCEPT_ASSERT(rhs.shared_->refcount > 0); + rhs.shared_ = 0; + } + MemoryPoolAllocator& operator=(MemoryPoolAllocator&& rhs) RAPIDJSON_NOEXCEPT + { + RAPIDJSON_NOEXCEPT_ASSERT(rhs.shared_->refcount > 0); + this->~MemoryPoolAllocator(); + baseAllocator_ = rhs.baseAllocator_; + chunk_capacity_ = rhs.chunk_capacity_; + shared_ = rhs.shared_; + rhs.shared_ = 0; + return *this; + } +#endif + //! Destructor. /*! This deallocates all memory chunks, excluding the user-supplied buffer. */ ~MemoryPoolAllocator() RAPIDJSON_NOEXCEPT { + if (!shared_) { + // do nothing if moved + return; + } if (shared_->refcount > 1) { --shared_->refcount; return; @@ -449,6 +473,17 @@ public: baseAllocator_(rhs.baseAllocator_) { } +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + StdAllocator(StdAllocator&& rhs) RAPIDJSON_NOEXCEPT : + allocator_type(std::move(rhs)), + baseAllocator_(std::move(rhs.baseAllocator_)) + { } +#endif +#if RAPIDJSON_HAS_CXX11 + using propagate_on_container_move_assignment = std::true_type; + using propagate_on_container_swap = std::true_type; +#endif + /* implicit */ StdAllocator(const BaseAllocator& allocator) RAPIDJSON_NOEXCEPT : allocator_type(), @@ -549,6 +584,10 @@ public: deallocate(p, n); } +#if RAPIDJSON_HAS_CXX11 + using is_always_equal = std::is_empty; +#endif + template bool operator==(const StdAllocator& rhs) const RAPIDJSON_NOEXCEPT { @@ -561,6 +600,7 @@ public: } //! rapidjson Allocator concept + static const bool kNeedFree = BaseAllocator::kNeedFree; void* Malloc(size_t size) { return baseAllocator_.Malloc(size); From 5c764d9a8187f2464397ed4dffffe001b5120356 Mon Sep 17 00:00:00 2001 From: ylavic Date: Sat, 3 Apr 2021 03:21:09 +0200 Subject: [PATCH 18/27] Tests for Allocators copy by rvalue reference. --- test/unittest/allocatorstest.cpp | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/test/unittest/allocatorstest.cpp b/test/unittest/allocatorstest.cpp index ec1b2ce..2ffc325 100644 --- a/test/unittest/allocatorstest.cpp +++ b/test/unittest/allocatorstest.cpp @@ -16,8 +16,10 @@ #include "rapidjson/allocators.h" +#include #include -#include +#include +#include using namespace rapidjson; @@ -141,12 +143,34 @@ void TestStdAllocator(const Allocator& a) { typedef StdAllocator CharAllocator; typedef std::basic_string, CharAllocator> String; +#if RAPIDJSON_HAS_CXX11 + String s(CharAllocator{a}); +#else CharAllocator ca(a); String s(ca); +#endif for (int i = 0; i < 26; i++) { s.push_back(static_cast('A' + i)); } EXPECT_TRUE(s == "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); + + typedef StdAllocator, Allocator> MapAllocator; + typedef std::map, MapAllocator> Map; +#if RAPIDJSON_HAS_CXX11 + Map map(std::less(), MapAllocator{a}); +#else + MapAllocator ma(a); + Map map(std::less(), ma); +#endif + for (int i = 0; i < 10; i++) { + map.insert(std::make_pair(i, (i % 2) == 0)); + } + EXPECT_TRUE(map.size() == 10); + for (int i = 0; i < 10; i++) { + typename Map::iterator it = map.find(i); + EXPECT_TRUE(it != map.end()); + EXPECT_TRUE(it->second == ((i % 2) == 0)); + } } TEST(Allocator, CrtAllocator) { From aa0675ffd71010697ed9890b05f0bf904da3835c Mon Sep 17 00:00:00 2001 From: ylavic Date: Sat, 3 Apr 2021 16:40:09 +0200 Subject: [PATCH 19/27] Try some tests with -D_GLIBCXX_DEBUG and coverage with -O0. --- .travis.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7e7a996..4be5ef2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,10 +40,10 @@ matrix: - env: CONF=debug ARCH=x86_64 CXX11=OFF CXX17=OFF compiler: gcc arch: amd64 - - env: CONF=debug ARCH=x86 CXX11=OFF CXX17=ON + - env: CONF=debug ARCH=x86 CXX11=OFF CXX17=ON CXX_FLAGS='-D_GLIBCXX_DEBUG' compiler: gcc - arch: amd64 - - env: CONF=debug ARCH=x86_64 CXX11=OFF CXX17=ON + arch: amd64/ + - env: CONF=debug ARCH=x86_64 CXX11=OFF CXX17=ON CXX_FLAGS='-D_GLIBCXX_DEBUG' compiler: gcc arch: amd64 - env: CONF=release ARCH=aarch64 CXX11=ON CXX17=OFF @@ -84,7 +84,7 @@ matrix: compiler: clang arch: arm64 # coverage report - - env: CONF=debug ARCH=x86 CXX11=ON CXX17=OFF GCOV_FLAGS='--coverage' + - env: CONF=debug ARCH=x86 GCOV_FLAGS='--coverage' CXX_FLAGS='-O0' CXX11=OFF CXX17=OFF compiler: gcc arch: amd64 cache: @@ -93,7 +93,7 @@ matrix: after_success: - pip install --user cpp-coveralls - coveralls -r .. --gcov-options '\-lp' -e thirdparty -e example -e test -e build/CMakeFiles -e include/rapidjson/msinttypes -e include/rapidjson/internal/meta.h -e include/rapidjson/error/en.h - - env: CONF=debug ARCH=x86_64 GCOV_FLAGS='--coverage' + - env: CONF=debug ARCH=x86_64 GCOV_FLAGS='--coverage' CXX_FLAGS='-O0' CXX11=ON CXX17=OFF compiler: gcc arch: amd64 cache: @@ -102,7 +102,7 @@ matrix: after_success: - pip install --user cpp-coveralls - coveralls -r .. --gcov-options '\-lp' -e thirdparty -e example -e test -e build/CMakeFiles -e include/rapidjson/msinttypes -e include/rapidjson/internal/meta.h -e include/rapidjson/error/en.h - - env: CONF=debug ARCH=aarch64 GCOV_FLAGS='--coverage' + - env: CONF=debug ARCH=aarch64 GCOV_FLAGS='--coverage' CXX_FLAGS='-O0' CXX11=OFF CXX17=ON compiler: gcc arch: arm64 cache: @@ -150,7 +150,7 @@ script: -DRAPIDJSON_BUILD_CXX17=$CXX17 -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_BUILD_TYPE=$CONF - -DCMAKE_CXX_FLAGS="$ARCH_FLAGS $GCOV_FLAGS" + -DCMAKE_CXX_FLAGS="$ARCH_FLAGS $GCOV_FLAGS $CXX_FLAGS" -DCMAKE_EXE_LINKER_FLAGS=$GCOV_FLAGS ..) - cd build From 71f0fa7eb385c7d78c2100b676f4eca1d47b0fe0 Mon Sep 17 00:00:00 2001 From: ylavic Date: Thu, 1 Apr 2021 18:20:09 +0200 Subject: [PATCH 20/27] Set RAPIDJSON_USE_MEMBERSMAP to use a (std::multi)map for object members. When RAPIDJSON_USE_MEMBERSMAP is defined, an object Value will store its members in an (re)allocated array of Members like before, but also in an std::multimap where the key and value reference the corresponding Member by its Data and index in the array, respectively, and in a relocatable manner. The layout of the members map/array is now: {multimap*}<>{capacity}<>{Member[capacity]}<>{multimap::iterator[capacity]} where <> stands for the RAPIDJSON_ALIGN-ment of each part, if needed. This layout needs to be reallocated when the current capacity is exhausted, which requires to take care of the multimap and its iterators explicitely. The multimap is allocated separately and only its pointer is saved in this layout, so it can easily be restored in its new position. As for the old/alive iterators, they must move to their new offset according to the new capacity. With this in place, it's immediate to get the multimap::iterator from a MemberIterator and vice versa, thus the same complexity applies for the operations with MemberIterator or MapIterator. For FindMember() and RemoveMember(), the complexity drops from O(n) to the multimap/rbtree's O(log n). For EraseMember() it drops from O(n-m) to O((log n)-m), m representing the move/copy of the trailing members. For AddMember() though, the complexity grows from O(1) to O(log n) due to the insertion in the multimap too. Consequently parsing will be slower, up to ~20% measured in perftests on my laptop (since it's mainly composed of insertions). But later work on the Document (usually the goal of parsing...) will be much faster; the new DocumentFind perftest included in this commit is 8 times faster with RAPIDJSON_USE_MEMBERSMAP (still on my laptop). Overall the tests are 4% slower (mainly composed of parsing), and notably 15% slower for schemas parsing/validation (which supposedly comes from the larger JSON files parsing, still). As a side note, when RAPIDJSON_USE_MEMBERSMAP is not defined, this commit does nothing (same results for perftest with regard to previous versions). Finally, the multimap is allocated and constructed using StdAllocator, so they will use the same Allocator than for any other Value allocation, and thus will benefit from the same performance/safety/security/whatever provided by the user given Allocator. --- CMakeLists.txt | 5 + include/rapidjson/allocators.h | 14 +- include/rapidjson/document.h | 383 ++++++++++++++++++++++++++++----- include/rapidjson/rapidjson.h | 22 ++ 4 files changed, 365 insertions(+), 59 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dcca04f..dc2072a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -52,6 +52,11 @@ if(RAPIDJSON_HAS_STDSTRING) add_definitions(-DRAPIDJSON_HAS_STDSTRING) endif() +option(RAPIDJSON_USE_MEMBERSMAP "" OFF) +if(RAPIDJSON_USE_MEMBERSMAP) + add_definitions(-DRAPIDJSON_USE_MEMBERSMAP=1) +endif() + find_program(CCACHE_FOUND ccache) if(CCACHE_FOUND) set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache) diff --git a/include/rapidjson/allocators.h b/include/rapidjson/allocators.h index cf8e75a..3ec83c1 100644 --- a/include/rapidjson/allocators.h +++ b/include/rapidjson/allocators.h @@ -16,6 +16,7 @@ #define RAPIDJSON_ALLOCATORS_H_ #include "rapidjson.h" +#include "internal/meta.h" #include @@ -158,6 +159,7 @@ class MemoryPoolAllocator { public: static const bool kNeedFree = false; //!< Tell users that no need to call Free() with this allocator. (concept Allocator) + static const bool kRefCounted = true; //!< Tell users that this allocator is reference counted on copy //! Constructor with chunkSize. /*! \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. @@ -417,6 +419,16 @@ private: SharedData *shared_; //!< The shared data of the allocator }; +namespace internal { + template + struct IsRefCounted : + public FalseType + { }; + template + struct IsRefCounted::Type> : + public TrueType + { }; +} template inline T* Realloc(A& a, T* old_p, size_t old_n, size_t new_n) @@ -437,7 +449,6 @@ inline void Free(A& a, T *p, size_t n = 1) static_cast(Realloc(a, p, n, 0)); } - #ifdef __GNUC__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) // std::allocator can safely be inherited @@ -601,6 +612,7 @@ public: //! rapidjson Allocator concept static const bool kNeedFree = BaseAllocator::kNeedFree; + static const bool kRefCounted = internal::IsRefCounted::Value; void* Malloc(size_t size) { return baseAllocator_.Malloc(size); diff --git a/include/rapidjson/document.h b/include/rapidjson/document.h index 61d031b..54e2936 100644 --- a/include/rapidjson/document.h +++ b/include/rapidjson/document.h @@ -46,8 +46,8 @@ RAPIDJSON_DIAG_OFF(effc++) #include // std::random_access_iterator_tag #endif -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS -#include // std::move +#if RAPIDJSON_USE_MEMBERSMAP +#include // std::multimap #endif RAPIDJSON_NAMESPACE_BEGIN @@ -732,18 +732,8 @@ public: template GenericValue(const GenericValue& rhs, Allocator& allocator, bool copyConstStrings = false) { switch (rhs.GetType()) { - case kObjectType: { - SizeType count = rhs.data_.o.size; - Member* lm = reinterpret_cast(allocator.Malloc(count * sizeof(Member))); - const typename GenericValue::Member* rm = rhs.GetMembersPointer(); - for (SizeType i = 0; i < count; i++) { - new (&lm[i].name) GenericValue(rm[i].name, allocator, copyConstStrings); - new (&lm[i].value) GenericValue(rm[i].value, allocator, copyConstStrings); - } - data_.f.flags = kObjectFlag; - data_.o.size = data_.o.capacity = count; - SetMembersPointer(lm); - } + case kObjectType: + DoCopyMembers(rhs, allocator, copyConstStrings); break; case kArrayType: { SizeType count = rhs.data_.a.size; @@ -879,25 +869,30 @@ public: /*! Need to destruct elements of array, members of object, or copy-string. */ ~GenericValue() { - if (Allocator::kNeedFree) { // Shortcut by Allocator's trait + // With RAPIDJSON_USE_MEMBERSMAP, the maps need to be destroyed to release + // their Allocator if it's refcounted (e.g. MemoryPoolAllocator). + if (Allocator::kNeedFree || (RAPIDJSON_USE_MEMBERSMAP+0 && + internal::IsRefCounted::Value)) { switch(data_.f.flags) { case kArrayFlag: { GenericValue* e = GetElementsPointer(); for (GenericValue* v = e; v != e + data_.a.size; ++v) v->~GenericValue(); - Allocator::Free(e); + if (Allocator::kNeedFree) { // Shortcut by Allocator's trait + Allocator::Free(e); + } } break; case kObjectFlag: - for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) - m->~Member(); - Allocator::Free(GetMembersPointer()); + DoFreeMembers(); break; case kCopyStringFlag: - Allocator::Free(const_cast(GetStringPointer())); + if (Allocator::kNeedFree) { // Shortcut by Allocator's trait + Allocator::Free(const_cast(GetStringPointer())); + } break; default: @@ -1265,10 +1260,7 @@ public: */ GenericValue& MemberReserve(SizeType newCapacity, Allocator &allocator) { RAPIDJSON_ASSERT(IsObject()); - if (newCapacity > data_.o.capacity) { - SetMembersPointer(reinterpret_cast(allocator.Realloc(GetMembersPointer(), data_.o.capacity * sizeof(Member), newCapacity * sizeof(Member)))); - data_.o.capacity = newCapacity; - } + DoReserveMembers(newCapacity, allocator); return *this; } @@ -1342,11 +1334,7 @@ public: MemberIterator FindMember(const GenericValue& name) { RAPIDJSON_ASSERT(IsObject()); RAPIDJSON_ASSERT(name.IsString()); - MemberIterator member = MemberBegin(); - for ( ; member != MemberEnd(); ++member) - if (name.StringEqual(member->name)) - break; - return member; + return DoFindMember(name); } template ConstMemberIterator FindMember(const GenericValue& name) const { return const_cast(*this).FindMember(name); } @@ -1375,14 +1363,7 @@ public: GenericValue& AddMember(GenericValue& name, GenericValue& value, Allocator& allocator) { RAPIDJSON_ASSERT(IsObject()); RAPIDJSON_ASSERT(name.IsString()); - - ObjectData& o = data_.o; - if (o.size >= o.capacity) - MemberReserve(o.capacity == 0 ? kDefaultObjectCapacity : (o.capacity + (o.capacity + 1) / 2), allocator); - Member* members = GetMembersPointer(); - members[o.size].name.RawAssign(name); - members[o.size].value.RawAssign(value); - o.size++; + DoAddMember(name, value, allocator); return *this; } @@ -1516,9 +1497,7 @@ public: */ void RemoveAllMembers() { RAPIDJSON_ASSERT(IsObject()); - for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) - m->~Member(); - data_.o.size = 0; + DoClearMembers(); } //! Remove a member in object by its name. @@ -1562,14 +1541,7 @@ public: RAPIDJSON_ASSERT(data_.o.size > 0); RAPIDJSON_ASSERT(GetMembersPointer() != 0); RAPIDJSON_ASSERT(m >= MemberBegin() && m < MemberEnd()); - - MemberIterator last(GetMembersPointer() + (data_.o.size - 1)); - if (data_.o.size > 1 && m != last) - *m = *last; // Move the last one to this place - else - m->~Member(); // Only one left, just destroy - --data_.o.size; - return m; + return DoRemoveMember(m); } //! Remove a member from an object by iterator. @@ -1601,13 +1573,7 @@ public: RAPIDJSON_ASSERT(first >= MemberBegin()); RAPIDJSON_ASSERT(first <= last); RAPIDJSON_ASSERT(last <= MemberEnd()); - - MemberIterator pos = MemberBegin() + (first - MemberBegin()); - for (MemberIterator itr = pos; itr != last; ++itr) - itr->~Member(); - std::memmove(static_cast(&*pos), &*last, static_cast(MemberEnd() - last) * sizeof(Member)); - data_.o.size -= static_cast(last - first); - return pos; + return DoEraseMembers(first, last); } //! Erase a member in object by its name. @@ -1858,12 +1824,12 @@ public: //!@name String //@{ - const Ch* GetString() const { RAPIDJSON_ASSERT(IsString()); return (data_.f.flags & kInlineStrFlag) ? data_.ss.str : GetStringPointer(); } + const Ch* GetString() const { RAPIDJSON_ASSERT(IsString()); return DataString(data_); } //! Get the length of string. /*! Since rapidjson permits "\\u0000" in the json string, strlen(v.GetString()) may not equal to v.GetStringLength(). */ - SizeType GetStringLength() const { RAPIDJSON_ASSERT(IsString()); return ((data_.f.flags & kInlineStrFlag) ? (data_.ss.GetLength()) : data_.s.length); } + SizeType GetStringLength() const { RAPIDJSON_ASSERT(IsString()); return DataStringLength(data_); } //! Set this value as a string without copying source string. /*! This version has better performance with supplied length, and also support string containing null character. @@ -2112,6 +2078,13 @@ private: Flag f; }; // 16 bytes in 32-bit mode, 24 bytes in 64-bit mode, 16 bytes in 64-bit with RAPIDJSON_48BITPOINTER_OPTIMIZATION + static RAPIDJSON_FORCEINLINE const Ch* DataString(const Data& data) { + return (data.f.flags & kInlineStrFlag) ? data.ss.str : RAPIDJSON_GETPOINTER(Ch, data.s.str); + } + static RAPIDJSON_FORCEINLINE SizeType DataStringLength(const Data& data) { + return (data.f.flags & kInlineStrFlag) ? data.ss.GetLength() : data.s.length; + } + RAPIDJSON_FORCEINLINE const Ch* GetStringPointer() const { return RAPIDJSON_GETPOINTER(Ch, data_.s.str); } RAPIDJSON_FORCEINLINE const Ch* SetStringPointer(const Ch* str) { return RAPIDJSON_SETPOINTER(Ch, data_.s.str, str); } RAPIDJSON_FORCEINLINE GenericValue* GetElementsPointer() const { return RAPIDJSON_GETPOINTER(GenericValue, data_.a.elements); } @@ -2119,6 +2092,286 @@ private: RAPIDJSON_FORCEINLINE Member* GetMembersPointer() const { return RAPIDJSON_GETPOINTER(Member, data_.o.members); } RAPIDJSON_FORCEINLINE Member* SetMembersPointer(Member* members) { return RAPIDJSON_SETPOINTER(Member, data_.o.members, members); } +#if RAPIDJSON_USE_MEMBERSMAP + + struct MapTraits { + struct Less { + bool operator()(const Data& s1, const Data& s2) const { + SizeType n1 = DataStringLength(s1), n2 = DataStringLength(s2); + int cmp = std::memcmp(DataString(s1), DataString(s2), sizeof(Ch) * (n1 < n2 ? n1 : n2)); + return cmp < 0 || (cmp == 0 && n1 < n2); + } + }; + typedef std::pair Pair; + typedef std::multimap > Map; + typedef typename Map::iterator Iterator; + }; + typedef typename MapTraits::Map Map; + typedef typename MapTraits::Less MapLess; + typedef typename MapTraits::Pair MapPair; + typedef typename MapTraits::Iterator MapIterator; + + // + // Layout of the members' map/array, re(al)located according to the needed capacity: + // + // {Map*}<>{capacity}<>{Member[capacity]}<>{MapIterator[capacity]} + // + // (where <> stands for the RAPIDJSON_ALIGN-ment, if needed) + // + + static RAPIDJSON_FORCEINLINE size_t GetMapLayoutSize(SizeType capacity) { + return RAPIDJSON_ALIGN(sizeof(Map*)) + + RAPIDJSON_ALIGN(sizeof(SizeType)) + + RAPIDJSON_ALIGN(capacity * sizeof(Member)) + + capacity * sizeof(MapIterator); + } + + static RAPIDJSON_FORCEINLINE SizeType &GetMapCapacity(Map* &map) { + return *reinterpret_cast(reinterpret_cast(&map) + + RAPIDJSON_ALIGN(sizeof(Map*))); + } + + static RAPIDJSON_FORCEINLINE Member* GetMapMembers(Map* &map) { + return reinterpret_cast(reinterpret_cast(&map) + + RAPIDJSON_ALIGN(sizeof(Map*)) + + RAPIDJSON_ALIGN(sizeof(SizeType))); + } + + static RAPIDJSON_FORCEINLINE MapIterator* GetMapIterators(Map* &map) { + return reinterpret_cast(reinterpret_cast(&map) + + RAPIDJSON_ALIGN(sizeof(Map*)) + + RAPIDJSON_ALIGN(sizeof(SizeType)) + + RAPIDJSON_ALIGN(GetMapCapacity(map) * sizeof(Member))); + } + + static RAPIDJSON_FORCEINLINE Map* &GetMap(Member* members) { + RAPIDJSON_ASSERT(members != 0); + return *reinterpret_cast(reinterpret_cast(members) - + RAPIDJSON_ALIGN(sizeof(SizeType)) - + RAPIDJSON_ALIGN(sizeof(Map*))); + } + + // Some compilers' debug mechanisms want all iterators to be destroyed, for their accounting.. + RAPIDJSON_FORCEINLINE MapIterator DropMapIterator(MapIterator& rhs) { +#if RAPIDJSON_HAS_CXX11 + MapIterator ret = std::move(rhs); +#else + MapIterator ret = rhs; +#endif + rhs.~MapIterator(); + return ret; + } + + Map* &DoReallocMap(Map** oldMap, SizeType newCapacity, Allocator& allocator) { + Map **newMap = static_cast(allocator.Malloc(GetMapLayoutSize(newCapacity))); + GetMapCapacity(*newMap) = newCapacity; + if (!oldMap) { + *newMap = new (allocator.Malloc(sizeof(Map))) Map(MapLess(), allocator); + } + else { + *newMap = *oldMap; + size_t count = (*oldMap)->size(); + std::memcpy(static_cast(GetMapMembers(*newMap)), + static_cast(GetMapMembers(*oldMap)), + count * sizeof(Member)); + MapIterator *oldIt = GetMapIterators(*oldMap), + *newIt = GetMapIterators(*newMap); + while (count--) { + new (&newIt[count]) MapIterator(DropMapIterator(oldIt[count])); + } + Allocator::Free(oldMap); + } + return *newMap; + } + + RAPIDJSON_FORCEINLINE Member* DoAllocMembers(SizeType capacity, Allocator& allocator) { + return GetMapMembers(DoReallocMap(0, capacity, allocator)); + } + + void DoReserveMembers(SizeType newCapacity, Allocator& allocator) { + ObjectData& o = data_.o; + if (newCapacity > o.capacity) { + Member* oldMembers = GetMembersPointer(); + Map **oldMap = oldMembers ? &GetMap(oldMembers) : 0, + *&newMap = DoReallocMap(oldMap, newCapacity, allocator); + RAPIDJSON_SETPOINTER(Member, o.members, GetMapMembers(newMap)); + o.capacity = newCapacity; + } + } + + template + MemberIterator DoFindMember(const GenericValue& name) { + if (Member* members = GetMembersPointer()) { + Map* &map = GetMap(members); + MapIterator mit = map->find(reinterpret_cast(name.data_)); + if (mit != map->end()) { + return MemberIterator(&members[mit->second]); + } + } + return MemberEnd(); + } + + void DoClearMembers() { + if (Member* members = GetMembersPointer()) { + Map* &map = GetMap(members); + MapIterator* mit = GetMapIterators(map); + for (SizeType i = 0; i < data_.o.size; i++) { + map->erase(DropMapIterator(mit[i])); + members[i].~Member(); + } + data_.o.size = 0; + } + } + + void DoFreeMembers() { + if (Member* members = GetMembersPointer()) { + GetMap(members)->~Map(); + for (SizeType i = 0; i < data_.o.size; i++) { + members[i].~Member(); + } + if (Allocator::kNeedFree) { // Shortcut by Allocator's trait + Map** map = &GetMap(members); + Allocator::Free(*map); + Allocator::Free(map); + } + } + } + +#else // !RAPIDJSON_USE_MEMBERSMAP + + RAPIDJSON_FORCEINLINE Member* DoAllocMembers(SizeType capacity, Allocator& allocator) { + return Malloc(allocator, capacity); + } + + void DoReserveMembers(SizeType newCapacity, Allocator& allocator) { + ObjectData& o = data_.o; + if (newCapacity > o.capacity) { + Member* newMembers = Realloc(allocator, GetMembersPointer(), o.capacity, newCapacity); + RAPIDJSON_SETPOINTER(Member, o.members, newMembers); + o.capacity = newCapacity; + } + } + + template + MemberIterator DoFindMember(const GenericValue& name) { + MemberIterator member = MemberBegin(); + for ( ; member != MemberEnd(); ++member) + if (name.StringEqual(member->name)) + break; + return member; + } + + void DoClearMembers() { + for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) + m->~Member(); + data_.o.size = 0; + } + + void DoFreeMembers() { + for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) + m->~Member(); + Allocator::Free(GetMembersPointer()); + } + +#endif // !RAPIDJSON_USE_MEMBERSMAP + + void DoAddMember(GenericValue& name, GenericValue& value, Allocator& allocator) { + ObjectData& o = data_.o; + if (o.size >= o.capacity) + DoReserveMembers(o.capacity ? (o.capacity + (o.capacity + 1) / 2) : kDefaultObjectCapacity, allocator); + Member* members = GetMembersPointer(); + Member* m = members + o.size; + m->name.RawAssign(name); + m->value.RawAssign(value); +#if RAPIDJSON_USE_MEMBERSMAP + Map* &map = GetMap(members); + MapIterator* mit = GetMapIterators(map); + new (&mit[o.size]) MapIterator(map->insert(MapPair(m->name.data_, o.size))); +#endif + ++o.size; + } + + MemberIterator DoRemoveMember(MemberIterator m) { + ObjectData& o = data_.o; + Member* members = GetMembersPointer(); +#if RAPIDJSON_USE_MEMBERSMAP + Map* &map = GetMap(members); + MapIterator* mit = GetMapIterators(map); + SizeType mpos = static_cast(&*m - members); + map->erase(DropMapIterator(mit[mpos])); +#endif + MemberIterator last(members + (o.size - 1)); + if (o.size > 1 && m != last) { +#if RAPIDJSON_USE_MEMBERSMAP + new (&mit[mpos]) MapIterator(DropMapIterator(mit[&*last - members])); + mit[mpos]->second = mpos; +#endif + *m = *last; // Move the last one to this place + } + else { + m->~Member(); // Only one left, just destroy + } + --o.size; + return m; + } + + MemberIterator DoEraseMembers(ConstMemberIterator first, ConstMemberIterator last) { + ObjectData& o = data_.o; + MemberIterator beg = MemberBegin(), + pos = beg + (first - beg), + end = MemberEnd(); +#if RAPIDJSON_USE_MEMBERSMAP + Map* &map = GetMap(GetMembersPointer()); + MapIterator* mit = GetMapIterators(map); +#endif + for (MemberIterator itr = pos; itr != last; ++itr) { +#if RAPIDJSON_USE_MEMBERSMAP + map->erase(DropMapIterator(mit[itr - beg])); +#endif + itr->~Member(); + } +#if RAPIDJSON_USE_MEMBERSMAP + if (first != last) { + // Move remaining members/iterators + MemberIterator next = pos + (last - first); + for (MemberIterator itr = pos; next != end; ++itr, ++next) { + std::memcpy(static_cast(&*itr), &*next, sizeof(Member)); + SizeType mpos = static_cast(itr - beg); + new (&mit[mpos]) MapIterator(DropMapIterator(mit[next - beg])); + mit[mpos]->second = mpos; + } + } +#else + std::memmove(static_cast(&*pos), &*last, + static_cast(end - last) * sizeof(Member)); +#endif + o.size -= static_cast(last - first); + return pos; + } + + template + void DoCopyMembers(const GenericValue& rhs, Allocator& allocator, bool copyConstStrings) { + RAPIDJSON_ASSERT(rhs.GetType() == kObjectType); + + data_.f.flags = kObjectFlag; + SizeType count = rhs.data_.o.size; + Member* lm = DoAllocMembers(count, allocator); + const typename GenericValue::Member* rm = rhs.GetMembersPointer(); +#if RAPIDJSON_USE_MEMBERSMAP + Map* &map = GetMap(lm); + MapIterator* mit = GetMapIterators(map); +#endif + for (SizeType i = 0; i < count; i++) { + new (&lm[i].name) GenericValue(rm[i].name, allocator, copyConstStrings); + new (&lm[i].value) GenericValue(rm[i].value, allocator, copyConstStrings); +#if RAPIDJSON_USE_MEMBERSMAP + new (&mit[i]) MapIterator(map->insert(MapPair(lm[i].name.data_, i))); +#endif + } + data_.o.size = data_.o.capacity = count; + SetMembersPointer(lm); + } + // Initialize this value as array with initial data, without calling destructor. void SetArrayRaw(GenericValue* values, SizeType count, Allocator& allocator) { data_.f.flags = kArrayFlag; @@ -2136,9 +2389,16 @@ private: void SetObjectRaw(Member* members, SizeType count, Allocator& allocator) { data_.f.flags = kObjectFlag; if (count) { - Member* m = static_cast(allocator.Malloc(count * sizeof(Member))); + Member* m = DoAllocMembers(count, allocator); SetMembersPointer(m); std::memcpy(static_cast(m), members, count * sizeof(Member)); +#if RAPIDJSON_USE_MEMBERSMAP + Map* &map = GetMap(m); + MapIterator* mit = GetMapIterators(map); + for (SizeType i = 0; i < count; i++) { + new (&mit[i]) MapIterator(map->insert(MapPair(m[i].name.data_, i))); + } +#endif } else SetMembersPointer(0); @@ -2259,6 +2519,13 @@ public: #endif ~GenericDocument() { + // Clear the ::ValueType before ownAllocator is destroyed, ~ValueType() + // runs last and may access its elements or members which would be freed + // with an allocator like MemoryPoolAllocator (CrtAllocator does not + // free its data when destroyed, but MemoryPoolAllocator does). + if (ownAllocator_) { + ValueType::SetNull(); + } Destroy(); } diff --git a/include/rapidjson/rapidjson.h b/include/rapidjson/rapidjson.h index 1093581..a4e8953 100644 --- a/include/rapidjson/rapidjson.h +++ b/include/rapidjson/rapidjson.h @@ -162,6 +162,24 @@ #include #endif // RAPIDJSON_HAS_STDSTRING +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_USE_MEMBERSMAP + +/*! \def RAPIDJSON_USE_MEMBERSMAP + \ingroup RAPIDJSON_CONFIG + \brief Enable RapidJSON support for object members handling in a \c std::multimap + + By defining this preprocessor symbol to \c 1, \ref rapidjson::GenericValue object + members are stored in a \c std::multimap for faster lookup and deletion times, a + trade off with a slightly slower insertion time and a small object allocat(or)ed + memory overhead. + + \hideinitializer +*/ +#ifndef RAPIDJSON_USE_MEMBERSMAP +#define RAPIDJSON_USE_MEMBERSMAP 0 // not by default +#endif + /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_NO_INT64DEFINE @@ -578,6 +596,10 @@ RAPIDJSON_NAMESPACE_END #endif #endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS +#include // std::move +#endif + #ifndef RAPIDJSON_HAS_CXX11_NOEXCEPT #if RAPIDJSON_HAS_CXX11 #define RAPIDJSON_HAS_CXX11_NOEXCEPT 1 From fc08f4f61b09c737d5fb84e557a3ee831cfce806 Mon Sep 17 00:00:00 2001 From: ylavic Date: Thu, 1 Apr 2021 18:19:53 +0200 Subject: [PATCH 21/27] Tests for Members in std::multimap. --- bin/types/alotofkeys.json | Bin 0 -> 30003 bytes test/perftest/perftest.h | 9 +++++---- test/perftest/rapidjsontest.cpp | 22 ++++++++++++++++++++-- 3 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 bin/types/alotofkeys.json diff --git a/bin/types/alotofkeys.json b/bin/types/alotofkeys.json new file mode 100644 index 0000000000000000000000000000000000000000..3fc052e34fdd414108ae7d6f1ce0ef1213e175fd GIT binary patch literal 30003 zcmZ9V%g(eqmZs-<6{#Bc2(XcQhOq&2GT;;D8cIFuQLFF0pGc{t&s(Z2?fi4c9|722 zYrQA_&%gft@Bix5pzcrjZ}qO-rcJrImBzkra8uhYX&TMG42{n2uzS~S|Mfrr{lEVA zfB*OY_A-|50**J)gyZSL%Ly4-z69qGNaD(Y3Up3fY7+K;t~e?L38`MyfqOm`pp z{N?*{XZy-I_TB7?mCKUOYX~~1_lb8m|NU%x9pN+c{+)|=Ij>EtjJ&dYb(6kFpZ#&{ z@whuQK`s}5f>9}VrmN0Mf2)%U0H|A;fcT(=`Ixf`gx`^Z1ZY%BI@0Gi}k0W{0 z#d}wso8NX%`Z?!emxo)WZy%+izuQS%*HWIppWRDJZ|Gy6&;FX{xQzSsCZ|^A{UeQU zu6`)}JFC~IJ!t*+v$J@z+ZU}r_ZBnFvrl<$$0jaCXBpj;)P3*wn!a$UHu-S_ryJldxE&Fj;Py-}|l#GdwZ3!O2aYpzbX zB|q137-epY;1x7!u2*)-#jpY{k- z^y--J!FS&!^0&Qfc;h+wGAJ$oe)eT89;YbBy$`1ro$8WJc^#J6hV$D(<2#?8Ec3RU zt9qNipLLy(b7~hg9OFzB0nAi4dws^0( z3}$clH~sx=&z|m+b=&lO16RJ!HZT3B^zeLgcwRd!!ngA^i!F)9j(V7uFnK+c?>EWe;5RQ?z?HclTnBadCHlSHHR2w7$Pb z?Wb>B{Vd<24|9%rKF-Okv-;N8&-TuJQrU$i{QX|B`K6t=Je_Y6VzbS3MeVidT_aA> z#PAJ6(Nj9uA;w7e_p|*lo2U*k-CwhgNAz1TR*r6HZ!sE?i|WJGOv1gb2}h{v!t&eX zpAli~MDDVk_kK=(_TS%JmQ~HG407nNV-rJpwGBG8y;@g$bm8i4^DaJ>vzL9-Ro5Gq zj&t#7C5KzwM$xVJp_Nc!<`+Fv9Lw4yXAWtH`4e~bn_W>`+^0S7&0(It6L-*i^ES(R zKCDR|{?S}A_;E>dT=!}Zt~$2;yxh@^We`^ElW+52w(XIl5brvUJO3H))$mbsRcm8Z zVlnG?u8q&%(KRw;oA)K;;mTKNhLjd%ca!Ex%kx^QTBf@}_?1)pQQbOI!TW0ot?>Oj z9Lx26&Gn62l;MO^Iga5kEQgSQqg+xeN>pnRkdsm)7qX_m)1uvb3^3~dP_Ek z!w!3ix4nZ}k1ZXIZSQTa)<)l}Lrups5B@cR_C~5_;jiH6;L}mFt6sZ?_BoaiFWde7 z%~SrqV`lNJd05_VS7q3mx!b$&m3?`SHCsIgxsD~fy~#c%ovO7l%9fgUj$#yMv-`^N8McP^+&inn+ z+hw-pE7jXS6(>T#w4h_%hRecY~SIwPP(PLr4OIJre>O@ zufJE=s~cK&tXgQ1+O==iltMo3?e=q&^G%=B9BBxfSbFvSz0b=sj{4Zo*~Mm1m*}lK zIe^+WPVE_&z1iCw%Q$~J@A?eYniTw|{)x}rQB8;QVigQ&>Ki)5)pU)RM=|Txxa}&E z<(2<_ZKco?+_2+WS^4eXdC3QWS`D+fA`n!jW^9_bak(8v90{=&CrecoUg0A z>O16}%Fs!9+;6*#@rX`*O{c~BR{hm|e{00{cGolW z@4a)dRe!bkaceL0E_6HZahSs{;pn|ybvwI_-gTVaF5|nt`{>8H+6&6Ya#&qDj>2dE zq}$i68BH^J5rWxm_GXZA*`ki+6?@fh*mc$WDTeb6$1A!^&be9XPv<$$jp&OR+;_au z=dIVh+?T)CJMKqtZSsRt{GaD|pPlP+znNoqQ*`zbva@PC)}ep0DStn^Mxjr?>@l~U z4z0agxvC_O-duLy4>j!Z(SvHX^`*St)iX-(_vZb)ziGYBJ6()&IbOSaKHa>}FuhND z&OMfJDc3Yn{QchL9L7V4&(A9x^f_F3_muXbZ4{fSz2~u8rcwK^Z3XMJy3cYIvvq0X zA#MvzvtX~ax}U>3O}&_=Zrc3SEZ4mD{%&;D^ShQcDcauNe%tcVZoP50KX2MY>O&m* z@9D=}mcogQ^k8zc9UdnWpi#`n*5qwdz#{F~m<% zfn@Jq*LI&bb9$|~20P2#q`8e^Z0^4CXD+9yYUoFg*bGBsC*>mt={8Ke7+g1Y`YlaS zW}(!?>q=vo*ooEmGVFe0DxY!f_qH@Rkqv&?Y)tAyGVd$9Z8M*tnr`p$kD1QUiI^O8 z?*_%(XVof23b%!>$&OGz@j@`PHZS9uLPd^0Y@7`CfUuP}*d9%M`&HA@rvN?ZH4ny49 z_3QYU`Mu@bJnt2&`n%_}mZd)gt98Gofzx@C(-zvx+uY30*!IIRNpan!chOY+RgBY? z$40x!x>5rkuko-SqWeTF)|PQNHo;x28y%fi=8b?=!F_EeOw2^1L^pbPbdL+;}QoYh`s_-F-4H zj-qkYchQbd_uhV=gG_jM7TcU}c z(d6O(f3)*r6W>(TZd2Zp0dXQ2*%3FfOOZYQebR0nyggQl7F8s1$T}8 z!(sVqR$#A0p7wYA)M?o6scRYR-tJlb8VAMt`uo|f4YcZeZ{IM?fu{Qd-~!TE_FeZe zEiBY(fRT6;0o4ydgHwZDlhw8+pnPrbujgy^V{qyWegw zv3b5b(E#nmeySWGm{k{ZO;j zc5BtaAai|Q4k1`AlpE*^mnKCHD!y#PU*Y@2ytyvx^cfAACyw9!pygUMs^fP01pG(-g=Gg6Qqn39&9Ep1O9r^UHk9lPL z`Qn}Vi;Ww&lht`O3A+oI+9FMN8`H1H9kwLnzKY?#xBL2QD0_Q1z2mX(Yq)*4SoO@Yuq&Q+ z{;t#=R70*|(WN?bj^Fvm<9Fn6`cpg?Gv~ggU;eh(+UZ-5wzauUw%Q9H1$P;n zCV!{geQCFik|!T(Qs1WRRUf;a(!Gb`?r%udURYuysb+_u;d@6nW=5L(Jii8DWn!G( zWWRoHIVc>=K5N|re++M~>3trDR~&a-zA_~d&pYb<12oVBcW~ci;T%4`>iO;mQ$;%t z+*_%AT0hmv-6<=#@SXh#>ikJ{WWYdirSiUi&#UL#>;>`Q3xoNYhjX~Ub-9+d8)G{| zhNSW2*gG!pU4M7g9t7Uwx3<$^M3GGzVPrlR8Q3ps(*$>u63C3!>NCsX-0=5%-4h1x zq+ly8CcZapK)5-^pQGG9B+XHE+No_jKdju+3PXOuhB!Q(yG$b{{Vpf{K{u5(ap6aI z&htAB`4dS4_8+m@gZZ0ZH;$(SL)l6h%DBjQ>rNWC;L4r> zJ%v!BknYs%Kh{2Gyjn)1-V2u8Kx4XJtCG!u-_?8}kp-a<5b24$lH}oMai9&5; zopyV&tITMgrAPHQRn?>ppq3R(iSY@Arh= zuVr%P94OI$*JNO3pXZ*(bEd()`T3qtiLKA{;xGMwEK0i~7h&~tv`5uH2l!7xmlXH& zDXlimTlCjJ8=3HUN3S(a#eYFqAV)^Gs-`se$1yWPPT4Jj87inYub)Ph^BInCdZ4;p zG0OIj-acK`vgo@}PVcfk-8?=~+Vx)8GB0EN9QCNpWp2$VDnJ>mtoQ!yuJ}UnZEv{J z%x=|3{Jvgn3qzjX%Il}`ZXW^DCNcxe-+MKGOcFg++AxRq)9_3=Pi>yN4sP=j@NTEx z_2T+jF?}x;Hj=z^|KZl`Wzuk!Va}i1b*DFI&v&(r5 zK$sL+2KuSm3#Z~N`c1~?wP&U<1&18DSosxwce?TIdh8nFLEQGFjsJVE`mJH`omTi3 z0F~o<^hcA|98sP*w_IJi0Z5}D!=R3UN1kX zTTk|XUe}W`>sHCHw+ColkG1W}x;^c>R_E1jHuoOy5=Fm;=Zo7hLP0@yZ^pgX!lxqi zlena|#+AL=`^qq=Kj8bOox3As2C0!cu*fra?czF1bDr8ubnW^9R#xko!fVxQfn#UW zW$@;u#B*!kZjG~BT?98?4=9dX^YoXg+Jo=efr_;^EoC$l<4 zlNv|qK~Jh@L|dn{(RnVJyGxCbOsdRC{yDtVF`frBZAnnfOIrq6{bsnst}CaF z(x-&Yu0H$qzy*_{{z%%X@7nJH;*&vo&|v}VyJrY(M$_kRzcp5Uj$wHAUFZdU=B^=s zzcaYy>ZwI*JWo$=n&)La#$LGYy{ekRqV2xl-2fD&h5f*Qb0hCC=cv+}`iY9u1tYqy z1IqMOO*t4L`JR^+ZSc>#x#c&X%zI(H&kW0Y>|g(0FD>;uIZ@TYgM-klyZdD9pb=bS zeLKLpj3{5C**eza$8mYVB2I72)p~BAU1oiqXfFUwL&IqomUS-bFlzIC0!%IHd4?d$ zd#ZX)K^D+kIU&&j4MJ8B`q8fL(!gO8ZL}}zdWSp3yJZZzQiV}8{7ZKa8Ixg|&^mn6 zX_w8->-d(wrInOdONi=n3?(2A4RjLZ;3#|60TP7Fs_V>iN=CPN zoJ+L_(`%Z%FtFu^eN+NO%m9#MRf!Bf^r7q6b(%Y32jfzzI6kpHd2)k!ML0(W(0S^* zv2RSC#2H@u_~e6tL6oQP`#ep5*JLQCJ%Ch8qlFZTwRB8xSQiyp+Cg{4BbeE@_u5`y zt*7E~9v!tU^6a2Q2U(0#YP!@(Gk9|uDL$ZA>t&)e%R0!enl+B&*(7CEwD)f6)02EO z6Wr1B0Tw!u+jxHzok6LZIGwc8g5=QT*0Tn$UEB5PJ z?_AB2?02`dW9boaEeO2aaSdx}&Myvr-92ZDw_HGB9GJ>tt2)SZF}Bd@>k6L7Ea93( z195YzUnh^p9y=YPAMfj=Lt9N%_x*6rn-gN8vZzruJxFJwNKA{|xCxyNE=^)CPCpOK zRCS+&Lz_XfPUE<==eGCWh6`e%qa1UWx4i|x9MPcw9;hxaplK#RStJ(r=Oze!jinwCdvg!yu1xNr2Mjt!n7{z9Nk`_+Z~chuhIl zZ+j!@1Cz4T1GJBb1AS!-szGv7o$buhbnCEttLwjeI1Ud1s)0j6{;pv_`uV(Iq7k;h z1EE&yxn+2R+Gs$w?g11Ap5psn+W>!=IDUDT{v2mBz*aQt{Z%s#LlLXvTb}sRe#yeq zq{6iRjXzxjoB<3x{7u`SC<1G$YN+gB=pyhStu{lET*d(~4BR_{@--Pk3^CN*@a%N@ zrm^bz=jB*Ee3X%2@Vee>-nG_$$y z2)Ri^=}R&n?ycRziOy=N zFt=$@h1tMIb)&ND>H7;XlGn&xke}!$D*&JST5_C||3}Zi@YmX*ph|nKvm1gBw3Fuf zCOzKoFhr}FA;NFS1AQoM)dR1?z8%j9rtrS)O6?fN_%$@m!ZuI#-rJ%nQqK>#d6%bp zz7~iMAqv8=3<=-r_I!@@bcoG1sHE5}%??U|A_!`RZB*648O=_d_HAz3uTyW>6-dfb zkcbYMnO7Wxcz#T650?XET-77~D-`bMxuO0Rhe*S>$YZ?}XF!rIl?e8&Ei?7wQ^*7V zo*RL~(R&nnJ!DvBklePDFR}18CN1rkda%F2Ec-D)QJHG*o9k?@CtdG1e-hAHplteG z-ocJO&Cn?|_MZmK9}OqdBmREx*Utz-eAaLnYeeKNEf~l3a8xkuY%d^_KG2<}Nw3IP zt3GzPoVl-7@9Vrxh`gJQQq1HZveWcw=y?|AOtc(L9NRQjz4y)_nr*j~(We=vAnTRA zmhX?Qs@W3fM_iR_-zjDX1zw%)8x-mxMT?^4ZeTG7uKOF{NNGIAQ5AGDnwder%x(c0 z)!*w4_F7#x(*v&eMjyOv;+dM$ice01HEN!&N9-MEp= z+8icSr$0&M4Lhp$MK!|$vM$DIU3r}v_S^7|B`@nr%{KN+M*7uzcFxw;@L|aoZA|hQ%YWG{yR-w@FUNj}*Cl zt?8u=JmamXKb(>RJ&l#96hQFCn1<2+&`WuK$BD^dlWkp78jAPIq5~?*P9mW7_vnGw z!}q!Z?uLd$-w|~XV&V?C_wSmVZrZof%?7i<{NUZ-oM-qj@eCHw378nbv%$~d%ef-X z|NB|XMIkyww^>@JzXi4nF}h_L%;UK;0rukFsgSIgYe5?J_p?ty4V;SUuARBwp{N)A zX&RKvV`+?P4XqDKY(Y!S{AH_NWy6GVuQK_C^#rbj%RlcA&f5uYJ4kYE9m8J#RlO`sER_!t4V) z=*}0hSpos;PlWFvTFtkm?|s^;fA-HktV=LAXu&eB8jf!0o8*h{CP43ZzNTMab~D*W zn^SeYbGZ-1R`?}WRATU!n=(gKD1Od(FcQslcZ2i*%Wg(~S)Dm3!chZlK)=H0L!j-w zGwR7JFgm;Ed1eHw`RP>{(dC|FHHYfHAaIlI@qD5nTMm>#O8%8+CCxFP76ZT+^-z@m z(!5l4VLjuAV$zsyoJ6$PLWU@o?WO^N&KlXltFS>L4dzPTr>bQw=+?ugePBG!n6IhJ z8}<(?;h5@h{%0N_GW$3s+>Tj9cS22N6K$lp@fl@ z$f4T%&*&~Be_D>ScuRa4*!x8f`5qc}vEV47EmxD`yzEZdRG(!Qzab4N;aGj{P2Js< zV^erm%=ju!D*_Z#Bty_V&!MY&-~^@sn|A>Cc0DV$f2}DuN?5sVJ;KUSGXA2({)YY1 z_NmsDc_k9|6RePLo+rXroB4*E#qQrE)!zQae*oFjYd38Yf3nhGe z(3`cxN(+Lx!S$^~8kmt@%W1_$P^z`@a%uYq7cpALBh2;$y#|4Mo-YY?pcRrc07tr1 zo*|K|=j+(@9azHe`Jn)uOn@4dsaQeH~4b#H7vVV!v8hnhF<{z=(Rmq#?afCZ$@O# zT~qpS)m(P?9jW<xC;kN$gQ@rLmhF4H_nm)oeWl0cP11y}FpKIDStUfYeHPv8 zaQOLatLyb{>ePeQG79Eg+EpX%0&7-gbOp2o3{OZXaF?A%FANZW?{hbq@mNhW%=dhX z4ijI14f;m;6REX3{5f4}XDD#U9YG=feiqjq77-3hj5+|P>5q#yLOiuP3L)Gh9|u{ztoHUx@yZl^IY z9xd+=u-ZB*Cd+Xx?P!0Bz#HguIQd(u85Yr1d%?9^q2#Gh5?`psvVB%59aR19=S5pQ zE%;4DxpGw(9(-*-A{O?NNrdBL2l~o;+Z$o5~<(&#P5# z0dLgd`kYL>5=_DKb={LD+w>O`XM2YFv(bZi7vz(L(QrpOf%|H@_Mfm(zBCx{1-N7c zwFPBDTb*?n3Ku(HNit?Mt}ECaX#ghyg(B}F?9{IioeT;vsiD+!8S_$Pm^T;VXa1G4 zx7{-Uh0ti@sGB8PLTe#`7XyZnud2^MOQd1%IC1)Cz7VL)G-lgF!EYgW0?}wvZ(~c1QmfMo{@qy*}yC$Q+cX!TA3EOc| z1Ld$Lxd{{^S5cflUrV(x8wY+vLz=7gESga@rcV%iD;^~1wHu?1d1Mz3z#k?x$W)A_ zHjF1zW7Uh&W-@0$K7BVD`C~_2=Ed1Q_UW2aKC@oLp)QsS&L>3++*S8E49j8hW8ym;G3(IwFKCNj7)>k!_QM7S*&%%g z3~*tas&EF3NA{MWn+eT`;$+ZI+imX{mc3sr+}<$ZXV}~Uv(!T|{k_jv^rk7%1C5s- z?u=L(Yg=(pnL8&2?vfQjkid9|T75WTRrgWi4{CgFXyz#)qW?DJqEK@2w*%kd41r^% z9}p~{Z}`Tl-`ijacMyYZy&?RZ8MjVqi$hE`MbyJ+~88EAh#PmvZ$Jw6f z%!J?I+b}L0b3qy5J`3x)eAn$5g{tdi*@h$X9Db_M`DmCAU_%N_D09uxKNk@VDBw4M zIZOpd^_yjCIj0c5cD4naFwe6MjcD7spfTkV(t`*R@t#5=t+c9YieU1aepXa{uJ`br z-$^SA-aWjpA8vL_u_r}!E&pE>7&>{5G5j>@AhO(d1`+6q&f<9s#eBFD$8m5vS2KCN6 z_9!$qBz1f(8p9JTFUj--F{A(Hf7_0#ziK0RBvROzfC8Uo4H6r_Sw@(n^15qBlG{L+ zHv@KEPxt=!H%BI+aMO^N5p^iSpr@a|aV+6|L<-QMZbJnI(LRI`nR+!_HTVAFp+@TR zVY&c#DH$O5B27#p6}-^)KTg7CiWw{b?11()9QybBcj3wk82x<6SHdRB&WyUb+tsa>vTWM z$#3qVTPUG{{^YMV+wn}Vi=hW7F!2XexuYwB>nd{TTaA# zE+g;-WL?B&eb2OfjoVJLNV#A|YR}Ye#&oXR zJ_U)@C*#>=9STHV#JYU_l`(Q4c0hb;sm1QC&bnnXs0y~D#nsv&yLgbCkJEW$dU^)_ zILZLJc_YKX0!kpk-+Og`E{`n~Xq5exyn@Sh=>r#dtYahWmqMP<5njOE^P%Q|s8)Cr z@PI82m(|bz2X~yiC%4|3@hUs@ea|?JTE_3yu4CMoqUyZtk#Tko&ZhYFG$7|NY%%!e_xGUGF zMkmgt$QwoN(2%?#Lb&!3oSl-6nDce}qui%alK%b8_qL-pxo1ftjT<*6!2`hQpe8n8 zQf9_s+k+wtuNcyhwEX=n{)){m-GezB@#R+7#XHjT3^M)fAmGdHyRBI9`>fC^r2Ky(j+Yv6J;h<5cn_}>L zPgHJKTYf0=Y?f`Ndc!Z8cRG5t;tY{w*b))McPXc_Q1v-o{8wm2pn|ue6-gBlmhXnIGSrk7MHR6vD`19j5uvYE; z=TPuFE-oFIV7j9aiUhZJ`9$lD2K{oap5G3$J%Ibq5%G6j^^9Z(i4*HUx(`RrmiOIw zDs7LH*mEiDyv7ea|1Vv8tp}$0$!PZJe4XNEq?b`28~@M4kXMFdBRL%B!i2Tree~6A zwOYbykcGzJ!5X({nQ#&ee{gmTtnk0@ZbwNl8HdC@SFfg?d!@qEWR z$0j}!5@u_q#t0)@4M>3HeTo}oy%!C#C`TN|(T{1K3nTg8`<&+6VI*xa3Jd^r4mOrk zB#LRdIPeAzuSWHZOykeG9Zj{5z{Iis+wh=hMST(D4!Pdpu4`vt7@KKJ&sZ9gT2NE0 zdv(TQe(Q#Wa;_(}iJT5^(PE%TWJ^&R!QfA4q5By(>0|!kXGeM4%LcOsVst`w`1?t4 zGZP#Tp5O~-J>>`wCajxnIB=^yc%F{}`Gvx_I2C@Tc7Q|}15Lof&fqC6*6aB4<2HST zv3kA*l=U^=jS#=p{y-H4IPa;_4~}p9Xvzcb{Ps1G!^1rlj&#ZGAq-xx?(hdr zQOW8)15sseKiMxlW8Avi)0^!@-t-F+fG>Jmge491#xM;(tH2 zcfJU`S!!OxM>HjHka#_)+hlz}k0b+By*JIq1=C****g`7U5H&*z37K_3h4!av7nnk z@sjfk2=t+`Gc!+h1AKs)p;l>2?^VqUD7)}T1S!Q38bbCCwmNM;`9SCIr0pol<8c}e zFt9(Sn7{X#CJ26TMD1ZAaskOZoRgf#V(3jgXdno+SQcC}C%iZ2lfR!0&Zu7+T&6(x z_r5m*B_5$PxZ8aLLTXo%iwI^CBXbZEn!leF81~1Fu(#&Gf5_^0jMoc{nh37 z2R3z7Pz7OIud4%{!n6a_18q_)aSxZAN(S-r=!-|VzKh16;E)~MQC9O{3#Mm_Xzb+gy(6D;8;Z;Mr{ZPU=fd?jf_!lh|GnNc594s5 zziZZh+cP;M9kylY#ZTU&E=7@Dh1kB#FEIMiRDJA;SaHfwYE1T|0bcw1iwhx!hG?4c zwcbTX#~0BWU?-rv`}@7353DofUD}NLQ`K8#qWDFgep0ZrIW?ulw$q86fFRdis_0!TNf80+9HTkmz}lj}XwgJNl+w zIUGSn9|V07$wB7GX@M!HsvZrJq$|2@wdLkqdm@9x)a}GS5;H&-QC4(sKWa&3sv?P0 zfA1A1VlQ#ujvLBN0wu7f>Mj;e#*YXpb>qvI&+A@E9FG+>#z>qI4*BI}gzXN-9jy3( z3>?c~L)+)b>Vho6y=_HgbI;X2qSXY@ys5b~&De3&jDlnqnhp^_XB%FgzuLJS%STpK zuCDhSu$QEkXA4`51z`QUXl~^2N6Qe&1c3?S&R3$@diMUU*2Y3q(}TVm3SO;%?Z`}T zV%b>a0Pp=oU<1rCvBvzg8nCk+8Ko!*g5+q2t9{vof3m?hd=66{dN{es5^QC%;Ah@AFr z!VrZE0t?D>_9rzl5bQ=4BJeR!hy;_>iqw>@I_u&pXR#hI7HDyCHG4X6awW9RWDT~r z2CCmmci*`6T7;O@XTPUS5NqFY8|L*CnoPc@2prampBV*k2Nn^S1r6l^$N2Yp8>R$& z)Q<#ZM>P%zq)@vZ-EdOU9{`WG#kQF+M4`hX&gSoDTO8ntI2hzgl0a25S3O%! zHI!dK&JxidL^IAvg+;Z0C1la`3+W`N3%AxUxktEAzBZR@AVwJtB#1nkBS3-tg-Qzn z!4fl^(zq~gd%94M^YtKhCkOnY`DYHB$wwi>CU*D!&wMkosl^+S@6Z_jG@HP0jA5h+ zs8RLkt`)n86^o`ip(g%&pJ94XU@%~j%<0gXwtjLrQLx2xfO*ULIh7JhyT0hVuw+)B z^(X%E5e#Ix$4D=U_IN@x22GC{-D6AzGQjc$bEZ+aS3NKx(M@b&oRCW00IBjJc(rR# zw#x|&8M`<5OtzyS97coc{rz5Ilb?wRx449fo%k({xs-go8N+jIz-5s$8cgSmxPBg< z3P_G5IR)obXtd)=?>ldO*V;tD5e>|O;`CW0hmzS) z3wG6cHI5%vNy2PUTpboSJTB7*BaMC9hu~%sL%d&G-~$AhYM(0n8Fyp+Io~o7Xc>@z zByQPa1)8x2F-|U=L^*7Tj8AdTT~j@yLj0!%lPn2W3{BWQ@kD-zAAUc`tmzyUPly+B zOQ>SVX8Ze$00kfg_Vr^G=71fAFlTO!r;Q*^MfhXHekcff)i5eBtA9TmU&xx5Fk1U6SKanv`<4K%3Fe4iId6+Mtg$y)o(W2tT zwmqTF6w<1O@+gLTL#oeYf#-{&)O^Gn0o3#>ln+>=_+<AtT2ihkJ#~X`?1wcWd1#=pfKNYu`Mm_Sc6cz+lOIj<yiH{t03ORbe3=S-U1k|q6p{kznc#dv?Pis&vcx;B`V4x4Vi4b{G z8e8-FOA?bYVH|<%t8;5_?$gixRbU>(p20R9~!)xF9)_!DB_Yb2%?mF0}o8=*4-=aYq^$DQ;K=&Z&W% zP+zPtHv+F_?{E?k8^!Wb@g7^+RpP6#6ru%0UP!$^7P@uxK3AzYksJE6o9 z0~)*inf{Tt+1;*X!E_zEVGhEvP_f2*xlBeJYG?N*b1^EFTT1y}- zWao3ZVJxy~7|;=-R?mp;nwd(AUkK&oeuo#HnsH{O7)sSQehAN)8TlLXCweOpdRUGvB)Ndpw6J~gzN^VK zJ?w}?L=cl~lezC8Pt z3|f0I(ORwo40qVriNQ=yTkUWNF4OY3SBzJrPUok>nc3^JnJb-^jEmdFc?A} z|NY)?UXWW7=Sh?!^V7~sw#R=QWeqlCBv2e+DoA1zFzq>A&9FoO^GY0UB#Pew-!8Z@ zC+f+G{%k}atqm62_^ZS6bp6#-XFGrU-mCsYg_RC&(6T8DE_31{$h1~UKD#0|Z9O~e zU_*^R7vx+A+})0Z);32#Lo4nr2x?7q?SjUy%joe@_2AfOb9LslX$7i^Kmj^V6g9$$ zIR-F(*$=TBc@fNI2Pq4YnU7$rYNk`~&he^BCNAtiUJ26Pd@Fjmmvj=69Ty5dGDeA% zns+7C9>970+!uajED)vs+$%V8|bnB`5&7Ok@9Fi#` zbb2~byRl=RvQy)E|4IN13Kpmk5`i7k$ZBl_Vg`^5N048dzQQMmZ_0_x zc&rUCujg5nL+g#6NEoNTKM~$E$;&%%DN$;_8WOI4q~nQA@2{{UFMw|Bw2{BU9f5n-Aw*% zHK(}W`nE{CtO>(6eis@xOy;WABjM*J<0mi*T@SWHKz)z` zi--|V!rCZ?Y2t2o{seR2H2Z4)#U4W1?u(~_sGOvhj^tv(K+C?8+4mp_A@4h)vu_oz z93Xx3@AdX(06rRf!f43n=NV6JApje}iXsqcv^4PD_X15Z@52*TpFJItr(Hts(dzHK zK8xUDbC5kdZoj}6a>Pbn%W-gG%%b2``TMDor-{zd-v+6IO`lzu!xK!)>7_ z6u1)(WCgNH^Xv5?bENfk;H1c2KK{g7 z1BFt%g+PO8AI54gY%w?>5s&t>bDZO-zX!&;vqTI;62aI>sVl=iULPP=?T5uG{Cl4O zVSFE&`d{0E$&&ZsWD(J~^9Bu+(F-pIAxQx&PF2qcD^*hx(<);b#h^7ku1}EL2L^`Z zPXYvZ@!1Prm_i^7r0P`+f5;mNFnhtUOJoE|63^Wui@+pDSP9NRW)4zS^ZiJBSG^h9 z9jO0(1P$Ru47`rOP%sAxedY}9q(bQ)ffvnbR#M8R3P-~9N8T3mebKZlYujU{PKLxI zLTB~IslmH7c*h*~jzCCW9rO43qCY0olJ;p@!JSPH6vV+!4xU%YeFSo-rT-avJpL6s zeAR3<5J&b7w_I>=f`1__*arNb=frD-bkAwPJ6}koJh96i4G!JE-^=^Mc!>M`WZr1t zGD$lPdS>#iA+|_&x#Xqer808Bkx3*fyucnF{F-ZzFA0+m z!xK6>WYPJnu9uU6w_3dEU>bseX*iycc3_Tfax_hnfw{)8iOg=wjYe=DOj_a*`ht%`nlqj2f`vVkkfG82w!v zz?Hs~W0(*{K({;cz<@D;3yYmGF0H)yg}h)w^ME#CC@PEs6)hR)kkhd81^ukJHPH>&luJtE>|BG1!^O&L!%sNYY&0<}p@9H!R;IE#?Q zsu$Js%Dav(MYIE1)#4JUR+)4UUQ?Gsf7tKs%+G3DFaWOPUjMyb(AJfVy3p{x6NG7* zh*~lY2=^pKmo$;y!=oU7BB9((qzP2Nxqp9$Ydek!X{{L_-nr6t9cn5v_OvWI;`+#F zzX-%p#N1TphPv2x1SxxmJBAmuHH?PHCyD%uXcF^RLgGD>5y9Sj^i5yYeK|i^>n$ue z`U^sLc!#1x*9O`su^A-BBP1}%{$%$T=ijl|-0 z#Mx6`<3orbFYF|y?(e-K6@;jBXp4c>iPO@cS>%0HJ1$h-3wgB!_!QtpvdK%ZfK-1! zyEX7P56lxlEp`q}p@i^bK;m^~q+_E*5 +#include #ifdef RAPIDJSON_SSE2 #define SIMD_SUFFIX(name) name##_SSE2 @@ -52,7 +53,7 @@ public: // Parse as a document EXPECT_FALSE(doc_.Parse(json_).HasParseError()); - for (size_t i = 0; i < 7; i++) + for (size_t i = 0; i < 8; i++) EXPECT_FALSE(typesDoc_[i].Parse(types_[i]).HasParseError()); } @@ -68,7 +69,7 @@ private: protected: char *temp_; Document doc_; - Document typesDoc_[7]; + Document typesDoc_[8]; }; TEST_F(RapidJson, SIMD_SUFFIX(ReaderParseInsitu_DummyHandler)) { @@ -335,6 +336,23 @@ TEST_F(RapidJson, DocumentAccept) { } } +TEST_F(RapidJson, DocumentFind) { + typedef Document::ValueType ValueType; + typedef ValueType::ConstMemberIterator ConstMemberIterator; + const Document &doc = typesDoc_[7]; // alotofkeys.json + if (doc.IsObject()) { + std::vector keys; + for (ConstMemberIterator it = doc.MemberBegin(); it != doc.MemberEnd(); ++it) { + keys.push_back(&it->name); + } + for (size_t i = 0; i < kTrialCount; i++) { + for (size_t j = 0; j < keys.size(); j++) { + EXPECT_TRUE(doc.FindMember(*keys[j]) != doc.MemberEnd()); + } + } + } +} + struct NullStream { typedef char Ch; From be4a5a9087e1fafa5b0bc90750f9a3790b4af295 Mon Sep 17 00:00:00 2001 From: ylavic Date: Thu, 1 Apr 2021 23:55:15 +0200 Subject: [PATCH 22/27] Turn some Tests to RAPIDJSON_USE_MEMBERSMAP in CI. --- .travis.yml | 49 ++++++++++++++++++++++++++++--------------------- appveyor.yml | 15 ++++++++++++++- 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4be5ef2..17d8f03 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,59 +28,65 @@ env: matrix: include: # gcc - - env: CONF=release ARCH=x86 CXX11=ON CXX17=OFF + - env: CONF=release ARCH=x86 CXX11=ON CXX17=OFF MEMBERSMAP=OFF compiler: gcc arch: amd64 - - env: CONF=release ARCH=x86_64 CXX11=ON CXX17=OFF + - env: CONF=release ARCH=x86_64 CXX11=ON CXX17=OFF MEMBERSMAP=OFF compiler: gcc arch: amd64 - - env: CONF=debug ARCH=x86 CXX11=OFF CXX17=OFF + - env: CONF=release ARCH=x86_64 CXX11=ON CXX17=OFF MEMBERSMAP=ON compiler: gcc arch: amd64 - - env: CONF=debug ARCH=x86_64 CXX11=OFF CXX17=OFF + - env: CONF=debug ARCH=x86 CXX11=OFF CXX17=OFF MEMBERSMAP=OFF compiler: gcc arch: amd64 - - env: CONF=debug ARCH=x86 CXX11=OFF CXX17=ON CXX_FLAGS='-D_GLIBCXX_DEBUG' - compiler: gcc - arch: amd64/ - - env: CONF=debug ARCH=x86_64 CXX11=OFF CXX17=ON CXX_FLAGS='-D_GLIBCXX_DEBUG' + - env: CONF=debug ARCH=x86_64 CXX11=OFF CXX17=OFF MEMBERSMAP=OFF compiler: gcc arch: amd64 - - env: CONF=release ARCH=aarch64 CXX11=ON CXX17=OFF + - env: CONF=debug ARCH=x86 CXX11=OFF CXX17=ON MEMBERSMAP=ON CXX_FLAGS='-D_GLIBCXX_DEBUG' + compiler: gcc + arch: amd64 + - env: CONF=debug ARCH=x86_64 CXX11=OFF CXX17=ON MEMBERSMAP=ON CXX_FLAGS='-D_GLIBCXX_DEBUG' + compiler: gcc + arch: amd64 + - env: CONF=release ARCH=aarch64 CXX11=ON CXX17=OFF MEMBERSMAP=OFF compiler: gcc arch: arm64 - - env: CONF=release ARCH=aarch64 CXX11=OFF CXX17=OFF + - env: CONF=release ARCH=aarch64 CXX11=OFF CXX17=OFF MEMBERSMAP=OFF compiler: gcc arch: arm64 - - env: CONF=release ARCH=aarch64 CXX11=OFF CXX17=ON + - env: CONF=release ARCH=aarch64 CXX11=OFF CXX17=ON MEMBERSMAP=ON compiler: gcc arch: arm64 # clang - - env: CONF=release ARCH=x86 CXX11=ON CXX17=OFF CCACHE_CPP2=yes + - env: CONF=release ARCH=x86 CXX11=ON CXX17=OFF MEMBERSMAP=ON CCACHE_CPP2=yes compiler: clang arch: amd64 - - env: CONF=release ARCH=x86_64 CXX11=ON CXX17=OFF CCACHE_CPP2=yes + - env: CONF=release ARCH=x86_64 CXX11=ON CXX17=OFF MEMBERSMAP=ON CCACHE_CPP2=yes compiler: clang arch: amd64 - - env: CONF=debug ARCH=x86 CXX11=OFF CXX17=OFF CCACHE_CPP2=yes + - env: CONF=release ARCH=x86_64 CXX11=ON CXX17=OFF MEMBERSMAP=OFF CCACHE_CPP2=yes compiler: clang arch: amd64 - - env: CONF=debug ARCH=x86_64 CXX11=OFF CXX17=OFF CCACHE_CPP2=yes + - env: CONF=debug ARCH=x86 CXX11=OFF CXX17=OFF MEMBERSMAP=ON CCACHE_CPP2=yes compiler: clang arch: amd64 - - env: CONF=debug ARCH=x86 CXX11=OFF CXX17=ON CCACHE_CPP2=yes + - env: CONF=debug ARCH=x86_64 CXX11=OFF CXX17=OFF MEMBERSMAP=ON CCACHE_CPP2=yes compiler: clang arch: amd64 - - env: CONF=debug ARCH=x86_64 CXX11=OFF CXX17=ON CCACHE_CPP2=yes + - env: CONF=debug ARCH=x86 CXX11=OFF CXX17=ON MEMBERSMAP=OFF CCACHE_CPP2=yes compiler: clang arch: amd64 - - env: CONF=debug ARCH=aarch64 CXX11=ON CXX17=OFF CCACHE_CPP2=yes + - env: CONF=debug ARCH=x86_64 CXX11=OFF CXX17=ON MEMBERSMAP=OFF CCACHE_CPP2=yes + compiler: clang + arch: amd64 + - env: CONF=debug ARCH=aarch64 CXX11=ON CXX17=OFF MEMBERSMAP=ON CCACHE_CPP2=yes compiler: clang arch: arm64 - - env: CONF=debug ARCH=aarch64 CXX11=OFF CXX17=OFF CCACHE_CPP2=yes + - env: CONF=debug ARCH=aarch64 CXX11=OFF CXX17=OFF MEMBERSMAP=ON CCACHE_CPP2=yes compiler: clang arch: arm64 - - env: CONF=debug ARCH=aarch64 CXX11=OFF CXX17=ON CCACHE_CPP2=yes + - env: CONF=debug ARCH=aarch64 CXX11=OFF CXX17=ON MEMBERSMAP=OFF CCACHE_CPP2=yes compiler: clang arch: arm64 # coverage report @@ -93,7 +99,7 @@ matrix: after_success: - pip install --user cpp-coveralls - coveralls -r .. --gcov-options '\-lp' -e thirdparty -e example -e test -e build/CMakeFiles -e include/rapidjson/msinttypes -e include/rapidjson/internal/meta.h -e include/rapidjson/error/en.h - - env: CONF=debug ARCH=x86_64 GCOV_FLAGS='--coverage' CXX_FLAGS='-O0' CXX11=ON CXX17=OFF + - env: CONF=debug ARCH=x86_64 GCOV_FLAGS='--coverage' CXX_FLAGS='-O0' CXX11=ON CXX17=OFF MEMBERSMAP=ON compiler: gcc arch: amd64 cache: @@ -146,6 +152,7 @@ script: eval "ARCH_FLAGS=\${ARCH_FLAGS_${ARCH}}" ; (cd build && cmake -DRAPIDJSON_HAS_STDSTRING=ON + -DRAPIDJSON_USE_MEMBERSMAP=$MEMBERSMAP -DRAPIDJSON_BUILD_CXX11=$CXX11 -DRAPIDJSON_BUILD_CXX17=$CXX17 -DCMAKE_VERBOSE_MAKEFILE=ON diff --git a/appveyor.yml b/appveyor.yml index 2e591ee..4044ba6 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -15,70 +15,83 @@ environment: VS_PLATFORM: win32 CXX11: OFF CXX17: OFF + MEMBERSMAP: OFF - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2013 VS_VERSION: 10 2010 VS_PLATFORM: x64 CXX11: OFF CXX17: OFF + MEMBERSMAP: ON - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2013 VS_VERSION: 11 2012 VS_PLATFORM: win32 CXX11: OFF CXX17: OFF + MEMBERSMAP: ON - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2013 VS_VERSION: 11 2012 VS_PLATFORM: x64 CXX11: OFF CXX17: OFF + MEMBERSMAP: OFF - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2013 VS_VERSION: 12 2013 VS_PLATFORM: win32 CXX11: OFF CXX17: OFF + MEMBERSMAP: OFF - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2013 VS_VERSION: 12 2013 VS_PLATFORM: x64 CXX11: OFF CXX17: OFF + MEMBERSMAP: ON - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 VS_VERSION: 14 2015 VS_PLATFORM: win32 CXX11: OFF CXX17: OFF + MEMBERSMAP: ON - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 VS_VERSION: 14 2015 VS_PLATFORM: x64 CXX11: OFF CXX17: OFF + MEMBERSMAP: OFF - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 VS_VERSION: 15 2017 VS_PLATFORM: win32 CXX11: OFF CXX17: OFF + MEMBERSMAP: OFF - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 VS_VERSION: 15 2017 VS_PLATFORM: x64 CXX11: OFF CXX17: OFF + MEMBERSMAP: ON - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 VS_VERSION: 15 2017 VS_PLATFORM: x64 CXX11: ON CXX17: OFF + MEMBERSMAP: OFF - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 VS_VERSION: 15 2017 VS_PLATFORM: x64 CXX11: OFF CXX17: ON + MEMBERSMAP: OFF - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 VS_VERSION: 16 2019 VS_PLATFORM: x64 CXX11: OFF CXX17: ON + MEMBERSMAP: ON before_build: - git submodule update --init --recursive -- cmake -H. -BBuild/VS -G "Visual Studio %VS_VERSION%" -DCMAKE_GENERATOR_PLATFORM=%VS_PLATFORM% -DCMAKE_VERBOSE_MAKEFILE=ON -DBUILD_SHARED_LIBS=true -DRAPIDJSON_BUILD_CXX11=%CXX11% -DRAPIDJSON_BUILD_CXX17=%CXX17% -Wno-dev +- cmake -H. -BBuild/VS -G "Visual Studio %VS_VERSION%" -DCMAKE_GENERATOR_PLATFORM=%VS_PLATFORM% -DCMAKE_VERBOSE_MAKEFILE=ON -DBUILD_SHARED_LIBS=true -DRAPIDJSON_BUILD_CXX11=%CXX11% -DRAPIDJSON_BUILD_CXX17=%CXX17% -DRAPIDJSON_USE_MEMBERSMAP=%MEMBERSMAP% -Wno-dev build: project: Build\VS\RapidJSON.sln From 3168d7c343c1bb55e9c93c052431c6d4c29ca797 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Sun, 18 Apr 2021 18:06:35 +0200 Subject: [PATCH 23/27] add a test that provokes a compile time error on windows --- test/unittest/CMakeLists.txt | 1 + test/unittest/platformtest.cpp | 37 ++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 test/unittest/platformtest.cpp diff --git a/test/unittest/CMakeLists.txt b/test/unittest/CMakeLists.txt index fc8803e..0a8f2a3 100644 --- a/test/unittest/CMakeLists.txt +++ b/test/unittest/CMakeLists.txt @@ -16,6 +16,7 @@ set(UNITTEST_SOURCES jsoncheckertest.cpp namespacetest.cpp pointertest.cpp + platformtest.cpp prettywritertest.cpp ostreamwrappertest.cpp readertest.cpp diff --git a/test/unittest/platformtest.cpp b/test/unittest/platformtest.cpp new file mode 100644 index 0000000..5f9f539 --- /dev/null +++ b/test/unittest/platformtest.cpp @@ -0,0 +1,37 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2021 THL A29 Limited, a Tencent company, and Milo Yip. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#include "unittest.h" + +// see https://github.com/Tencent/rapidjson/issues/1448 +// including windows.h on purpose to provoke a compile time problem as GetObject is a +// macro that gets defined when windows.h is included +#ifdef _WIN32 +#include +#endif + +#include "rapidjson/document.h" + +using namespace rapidjson; + +TEST(Platform, GetObject) { + Document doc; + doc.Parse(" { \"object\" : { \"pi\": 3.1416} } "); + EXPECT_TRUE(doc.IsObject()); + EXPECT_TRUE(doc.HasMember("object")); + const Document::ValueType& o = doc["object"]; + EXPECT_TRUE(o.IsObject()); + auto sub = o.GetObject(); + EXPECT_TRUE(sub.HasMember("pi")); +} From d179facf90043a6704554fac262ac2b7db0a484d Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 19 Apr 2021 12:29:11 -0700 Subject: [PATCH 24/27] don't let the GetObject macro rewrite the GetObject method, add a GetObj alias --- include/rapidjson/document.h | 16 ++++++++++++++++ test/unittest/platformtest.cpp | 3 +++ 2 files changed, 19 insertions(+) diff --git a/include/rapidjson/document.h b/include/rapidjson/document.h index 54e2936..057b37a 100644 --- a/include/rapidjson/document.h +++ b/include/rapidjson/document.h @@ -42,6 +42,15 @@ RAPIDJSON_DIAG_OFF(4244) // conversion from kXxxFlags to 'uint16_t', possible lo RAPIDJSON_DIAG_OFF(effc++) #endif // __GNUC__ +#ifdef GetObject +// see https://github.com/Tencent/rapidjson/issues/1448 +// a former included windows.h might have defined a macro called GetObject, which affects +// GetObject defined here. This ensures the macro does not get applied +#pragma push_macro("GetObject") +#define RAPIDJSON_WINDOWS_GETOBJECT_WORKAROUND_APPLIED +#undef GetObject +#endif + #ifndef RAPIDJSON_NOMEMBERITERATORCLASS #include // std::random_access_iterator_tag #endif @@ -1602,7 +1611,9 @@ public: } Object GetObject() { RAPIDJSON_ASSERT(IsObject()); return Object(*this); } + Object GetObj() { RAPIDJSON_ASSERT(IsObject()); return Object(*this); } ConstObject GetObject() const { RAPIDJSON_ASSERT(IsObject()); return ConstObject(*this); } + ConstObject GetObj() const { RAPIDJSON_ASSERT(IsObject()); return ConstObject(*this); } //@} @@ -3008,4 +3019,9 @@ private: RAPIDJSON_NAMESPACE_END RAPIDJSON_DIAG_POP +#ifdef RAPIDJSON_WINDOWS_GETOBJECT_WORKAROUND_APPLIED +#pragma pop_macro("GetObject") +#undef RAPIDJSON_WINDOWS_GETOBJECT_WORKAROUND_APPLIED +#endif + #endif // RAPIDJSON_DOCUMENT_H_ diff --git a/test/unittest/platformtest.cpp b/test/unittest/platformtest.cpp index 5f9f539..ec7abc3 100644 --- a/test/unittest/platformtest.cpp +++ b/test/unittest/platformtest.cpp @@ -22,6 +22,7 @@ #endif #include "rapidjson/document.h" +#undef GetObject using namespace rapidjson; @@ -34,4 +35,6 @@ TEST(Platform, GetObject) { EXPECT_TRUE(o.IsObject()); auto sub = o.GetObject(); EXPECT_TRUE(sub.HasMember("pi")); + auto sub2 = o.GetObj(); + EXPECT_TRUE(sub2.HasMember("pi")); } From 3cdfde14d682466b2ad550f1c47825a686a3db23 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 19 Apr 2021 13:15:11 -0700 Subject: [PATCH 25/27] replace auto with concrete type --- test/unittest/platformtest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unittest/platformtest.cpp b/test/unittest/platformtest.cpp index ec7abc3..05eba3f 100644 --- a/test/unittest/platformtest.cpp +++ b/test/unittest/platformtest.cpp @@ -33,8 +33,8 @@ TEST(Platform, GetObject) { EXPECT_TRUE(doc.HasMember("object")); const Document::ValueType& o = doc["object"]; EXPECT_TRUE(o.IsObject()); - auto sub = o.GetObject(); + Value::ConstObject sub = o.GetObject(); EXPECT_TRUE(sub.HasMember("pi")); - auto sub2 = o.GetObj(); + Value::ConstObject sub2 = o.GetObj(); EXPECT_TRUE(sub2.HasMember("pi")); } From 3aa8d04b749436c5cc3722e633b8ec20763a90f8 Mon Sep 17 00:00:00 2001 From: "N. Kolotov" Date: Mon, 3 May 2021 01:56:41 +0300 Subject: [PATCH 26/27] Fixed -Wshadow warning. --- include/rapidjson/allocators.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/rapidjson/allocators.h b/include/rapidjson/allocators.h index 3ec83c1..12bc5ba 100644 --- a/include/rapidjson/allocators.h +++ b/include/rapidjson/allocators.h @@ -660,9 +660,9 @@ public: { } /* implicit */ - StdAllocator(const BaseAllocator& allocator) RAPIDJSON_NOEXCEPT : + StdAllocator(const BaseAllocator& baseAllocator) RAPIDJSON_NOEXCEPT : allocator_type(), - baseAllocator_(allocator) + baseAllocator_(baseAllocator) { } ~StdAllocator() RAPIDJSON_NOEXCEPT From cd737fb545f67b7edf9dc18a839007a1e86d9293 Mon Sep 17 00:00:00 2001 From: Saurabh Charde Date: Fri, 7 May 2021 14:13:33 +0530 Subject: [PATCH 27/27] Improve documentation Signed-off-by: Saurabh Charde --- doc/stream.md | 12 ++++++------ doc/tutorial.md | 30 +++++++++++++++--------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/doc/stream.md b/doc/stream.md index 0573549..5d0b0f3 100644 --- a/doc/stream.md +++ b/doc/stream.md @@ -1,6 +1,6 @@ # Stream -In RapidJSON, `rapidjson::Stream` is a concept for reading/writing JSON. Here we first show how to use streams provided. And then see how to create a custom stream. +In RapidJSON, `rapidjson::Stream` is a concept for reading/writing JSON. Here we'll first show you how to use provided streams. And then see how to create a custom stream. [TOC] @@ -51,7 +51,7 @@ d.Accept(writer); const char* output = buffer.GetString(); ~~~~~~~~~~ -When the buffer is full, it will increases the capacity automatically. The default capacity is 256 characters (256 bytes for UTF8, 512 bytes for UTF16, etc.). User can provide an allocator and a initial capacity. +When the buffer is full, it will increases the capacity automatically. The default capacity is 256 characters (256 bytes for UTF8, 512 bytes for UTF16, etc.). User can provide an allocator and an initial capacity. ~~~~~~~~~~cpp StringBuffer buffer1(0, 1024); // Use its allocator, initial size = 1024 @@ -89,7 +89,7 @@ d.ParseStream(is); fclose(fp); ~~~~~~~~~~ -Different from string streams, `FileReadStream` is byte stream. It does not handle encodings. If the file is not UTF-8, the byte stream can be wrapped in a `EncodedInputStream`. It will be discussed very soon. +Different from string streams, `FileReadStream` is byte stream. It does not handle encodings. If the file is not UTF-8, the byte stream can be wrapped in a `EncodedInputStream`. We will discuss more about this later in this tutorial. Apart from reading file, user can also use `FileReadStream` to read `stdin`. @@ -119,11 +119,11 @@ d.Accept(writer); fclose(fp); ~~~~~~~~~~ -It can also directs the output to `stdout`. +It can also redirect the output to `stdout`. # iostream Wrapper {#iostreamWrapper} -Due to users' requests, RapidJSON provided official wrappers for `std::basic_istream` and `std::basic_ostream`. However, please note that the performance will be much lower than the other streams above. +Due to users' requests, RapidJSON also provides official wrappers for `std::basic_istream` and `std::basic_ostream`. However, please note that the performance will be much lower than the other streams above. ## IStreamWrapper {#IStreamWrapper} @@ -181,7 +181,7 @@ As mentioned above, UTF-8 byte streams can be read directly. However, UTF-16 and Besides, it also need to handle [byte order mark (BOM)](http://en.wikipedia.org/wiki/Byte_order_mark). When reading from a byte stream, it is needed to detect or just consume the BOM if exists. When writing to a byte stream, it can optionally write BOM. -If the encoding of stream is known in compile-time, you may use `EncodedInputStream` and `EncodedOutputStream`. If the stream can be UTF-8, UTF-16LE, UTF-16BE, UTF-32LE, UTF-32BE JSON, and it is only known in runtime, you may use `AutoUTFInputStream` and `AutoUTFOutputStream`. These streams are defined in `rapidjson/encodedstream.h`. +If the encoding of stream is known during compile-time, you may use `EncodedInputStream` and `EncodedOutputStream`. If the stream can be UTF-8, UTF-16LE, UTF-16BE, UTF-32LE, UTF-32BE JSON, and it is only known in runtime, you may use `AutoUTFInputStream` and `AutoUTFOutputStream`. These streams are defined in `rapidjson/encodedstream.h`. Note that, these encoded streams can be applied to streams other than file. For example, you may have a file in memory, or a custom byte stream, be wrapped in encoded streams. diff --git a/doc/tutorial.md b/doc/tutorial.md index 4bde2fa..a86aafd 100644 --- a/doc/tutorial.md +++ b/doc/tutorial.md @@ -12,7 +12,7 @@ Each JSON value is stored in a type called `Value`. A `Document`, representing t # Query Value {#QueryValue} -In this section, we will use excerpt of `example/tutorial/tutorial.cpp`. +In this section, we will use excerpt from `example/tutorial/tutorial.cpp`. Assume we have the following JSON stored in a C string (`const char* json`): ~~~~~~~~~~js @@ -85,7 +85,7 @@ assert(document["i"].IsNumber()); // In this case, IsUint()/IsInt64()/IsUint64() also return true. assert(document["i"].IsInt()); printf("i = %d\n", document["i"].GetInt()); -// Alternative (int)document["i"] +// Alternatively (int)document["i"] assert(document["pi"].IsNumber()); assert(document["pi"].IsDouble()); @@ -113,7 +113,7 @@ a[2] = 3 a[3] = 4 ~~~~~~~~~~ -Note that, RapidJSON does not automatically convert values between JSON types. If a value is a string, it is invalid to call `GetInt()`, for example. In debug mode it will fail an assertion. In release mode, the behavior is undefined. +Note that, RapidJSON does not automatically convert values between JSON types. For example, if a value is a string, it is invalid to call `GetInt()`. In debug mode it will fail on assertion. In release mode, the behavior is undefined. In the following sections we discuss details about querying individual types. @@ -168,9 +168,9 @@ Type of member pi is Number Type of member a is Array ~~~~~~~~~~ -Note that, when `operator[](const char*)` cannot find the member, it will fail an assertion. +Note that, when `operator[](const char*)` cannot find the member, it will fail on assertion. -If we are unsure whether a member exists, we need to call `HasMember()` before calling `operator[](const char*)`. However, this incurs two lookup. A better way is to call `FindMember()`, which can check the existence of member and obtain its value at once: +If we are unsure whether a member exists, we need to call `HasMember()` before calling `operator[](const char*)`. However, this incurs two lookup. A better way is to call `FindMember()`, which can check the existence of a member and obtain its value at once: ~~~~~~~~~~cpp Value::ConstMemberIterator itr = document.FindMember("hello"); @@ -221,18 +221,18 @@ When obtaining the numeric values, `GetDouble()` will convert internal integer r ## Query String {#QueryString} -In addition to `GetString()`, the `Value` class also contains `GetStringLength()`. Here explains why. +In addition to `GetString()`, the `Value` class also contains `GetStringLength()`. Here explains why: -According to RFC 4627, JSON strings can contain Unicode character `U+0000`, which must be escaped as `"\u0000"`. The problem is that, C/C++ often uses null-terminated string, which treats ``\0'` as the terminator symbol. +According to RFC 4627, JSON strings can contain Unicode character `U+0000`, which must be escaped as `"\u0000"`. The problem is that, C/C++ often uses null-terminated string, which treats `\0` as the terminator symbol. -To conform RFC 4627, RapidJSON supports string containing `U+0000`. If you need to handle this, you can use `GetStringLength()` to obtain the correct string length. +To conform with RFC 4627, RapidJSON supports string containing `U+0000` character. If you need to handle this, you can use `GetStringLength()` to obtain the correct string length. -For example, after parsing a the following JSON to `Document d`: +For example, after parsing the following JSON to `Document d`: ~~~~~~~~~~js { "s" : "a\u0000b" } ~~~~~~~~~~ -The correct length of the value `"a\u0000b"` is 3. But `strlen()` returns 1. +The correct length of the string `"a\u0000b"` is 3, as returned by `GetStringLength()`. But `strlen()` returns 1. `GetStringLength()` can also improve performance, as user may often need to call `strlen()` for allocating buffer. @@ -246,7 +246,7 @@ which accepts the length of string as parameter. This constructor supports stori ## Comparing values -You can use `==` and `!=` to compare values. Two values are equal if and only if they are have same type and contents. You can also compare values with primitive types. Here is an example. +You can use `==` and `!=` to compare values. Two values are equal if and only if they have same type and contents. You can also compare values with primitive types. Here is an example: ~~~~~~~~~~cpp if (document["hello"] == document["n"]) /*...*/; // Compare values @@ -264,7 +264,7 @@ Note that, currently if an object contains duplicated named member, comparing eq There are several ways to create values. After a DOM tree is created and/or modified, it can be saved as JSON again using `Writer`. ## Change Value Type {#ChangeValueType} -When creating a Value or Document by default constructor, its type is Null. To change its type, call `SetXXX()` or assignment operator, for example: +When creating a `Value` or `Document` by default constructor, its type is Null. To change its type, call `SetXXX()` or assignment operator, for example: ~~~~~~~~~~cpp Document d; // Null @@ -285,7 +285,7 @@ Value u(123u); // calls Value(unsigned) Value d(1.5); // calls Value(double) ~~~~~~~~~~ -To create empty object or array, you may use `SetObject()`/`SetArray()` after default constructor, or using the `Value(Type)` in one shot: +To create empty object or array, you may use `SetObject()`/`SetArray()` after default constructor, or using the `Value(Type)` in one call: ~~~~~~~~~~cpp Value o(kObjectType); @@ -299,7 +299,7 @@ A very special decision during design of RapidJSON is that, assignment of value ~~~~~~~~~~cpp Value a(123); Value b(456); -b = a; // a becomes a Null value, b becomes number 123. +a = b; // b becomes a Null value, a becomes number 456. ~~~~~~~~~~ ![Assignment with move semantics.](diagram/move1.png) @@ -367,7 +367,7 @@ RapidJSON provides two strategies for storing string. Copy-string is always safe because it owns a copy of the data. Const-string can be used for storing a string literal, and for in-situ parsing which will be mentioned in the DOM section. -To make memory allocation customizable, RapidJSON requires users to pass an instance of allocator, whenever an operation may require allocation. This design is needed to prevent storing a allocator (or Document) pointer per Value. +To make memory allocation customizable, RapidJSON requires users to pass an instance of allocator, whenever an operation may require allocation. This design is needed to prevent storing an allocator (or Document) pointer per Value. Therefore, when we assign a copy-string, we call this overloaded `SetString()` with allocator: