From 8aab3db129585d81d74ed5108450c874622d46fd Mon Sep 17 00:00:00 2001 From: ylavic Date: Thu, 6 Dec 2018 00:21:05 +0100 Subject: [PATCH] Base buffered BasicIStreamWrapper on the original (better performing) FileReadStream algorithm. --- include/rapidjson/filereadstream.h | 81 +++++++++---------------- include/rapidjson/istreamwrapper.h | 95 ++++++++++++++---------------- test/perftest/rapidjsontest.cpp | 21 ------- 3 files changed, 73 insertions(+), 124 deletions(-) diff --git a/include/rapidjson/filereadstream.h b/include/rapidjson/filereadstream.h index d468ba1..f1bfb7d 100644 --- a/include/rapidjson/filereadstream.h +++ b/include/rapidjson/filereadstream.h @@ -17,7 +17,6 @@ #include "stream.h" #include -#include #ifdef __clang__ RAPIDJSON_DIAG_PUSH @@ -36,42 +35,21 @@ class FileReadStream { public: typedef char Ch; //!< Character type (byte). - //! Constructor. - /*! - \param fp File pointer opened for read. - */ - FileReadStream(std::FILE* fp) : fp_(fp), buffer_(peekBuffer_), size_(sizeof(peekBuffer_) / sizeof(Ch)), pos_(), len_(), count_() - { - RAPIDJSON_ASSERT(fp_ != 0); - } - //! Constructor. /*! \param fp File pointer opened for read. \param buffer user-supplied buffer. \param bufferSize size of buffer in bytes. Must >=4 bytes. */ - FileReadStream(std::FILE* fp, Ch *buffer, size_t size) : fp_(fp), buffer_(buffer), size_(size), pos_(), len_(), count_() { - RAPIDJSON_ASSERT(fp_ != 0 && buffer_ != 0 && size_ > 0); - if (RAPIDJSON_UNLIKELY(size_ < sizeof(peekBuffer_) / sizeof(Ch))) { - size_ = sizeof(peekBuffer_) / sizeof(Ch); - buffer_ = peekBuffer_; - } + FileReadStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferSize_(bufferSize), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) { + RAPIDJSON_ASSERT(fp_ != 0); + RAPIDJSON_ASSERT(bufferSize >= 4); + Read(); } - Ch Peek() const { - if (RAPIDJSON_UNLIKELY(pos_ == len_) && !Read()) - return static_cast('\0'); - return buffer_[pos_]; - } - - Ch Take() { - if (RAPIDJSON_UNLIKELY(pos_ == len_) && !Read()) - return static_cast('\0'); - return buffer_[pos_++]; - } - - size_t Tell() const { return count_ + pos_; } + Ch Peek() const { return *current_; } + Ch Take() { Ch c = *current_; Read(); return c; } + size_t Tell() const { return count_ + static_cast(current_ - buffer_); } // Not implemented void Put(Ch) { RAPIDJSON_ASSERT(false); } @@ -81,36 +59,35 @@ public: // For encoding detection only. const Ch* Peek4() const { - if (len_ - pos_ < 4) { - if (pos_) { - len_ -= pos_; - std::memmove(buffer_, buffer_ + pos_, len_); - count_ += pos_; - pos_ = 0; - } - len_ += std::fread(buffer_ + len_, sizeof(Ch), size_ - len_, fp_); - if (len_ < 4) - return 0; - } - return &buffer_[pos_]; + return (current_ + 4 <= bufferLast_) ? current_ : 0; } private: - FileReadStream(); - FileReadStream(const FileReadStream&); - FileReadStream& operator=(const FileReadStream&); + void Read() { + if (current_ < bufferLast_) + ++current_; + else if (!eof_) { + count_ += readCount_; + readCount_ = std::fread(buffer_, 1, bufferSize_, fp_); + bufferLast_ = buffer_ + readCount_ - 1; + current_ = buffer_; - size_t Read() const { - count_ += pos_; - pos_ = 0; - len_ = std::fread(buffer_, sizeof(Ch), size_, fp_); - return len_; + if (readCount_ < bufferSize_) { + buffer_[readCount_] = '\0'; + ++bufferLast_; + eof_ = true; + } + } } std::FILE* fp_; - Ch peekBuffer_[4], *buffer_; - size_t size_; - mutable size_t pos_, len_, count_; + Ch *buffer_; + size_t bufferSize_; + Ch *bufferLast_; + Ch *current_; + size_t readCount_; + size_t count_; //!< Number of characters read + bool eof_; }; RAPIDJSON_NAMESPACE_END diff --git a/include/rapidjson/istreamwrapper.h b/include/rapidjson/istreamwrapper.h index 0eac7f4..f304fb0 100644 --- a/include/rapidjson/istreamwrapper.h +++ b/include/rapidjson/istreamwrapper.h @@ -17,7 +17,6 @@ #include "stream.h" #include -#include #ifdef __clang__ RAPIDJSON_DIAG_PUSH @@ -50,76 +49,70 @@ class BasicIStreamWrapper { public: typedef typename StreamType::char_type Ch; - BasicIStreamWrapper(StreamType& stream) : stream_(stream), buffer_(peekBuffer_), size_(sizeof(peekBuffer_) / sizeof(Ch)), pos_(), len_(), count_() {} - - BasicIStreamWrapper(StreamType& stream, Ch *buffer, size_t size) : stream_(stream), buffer_(buffer), size_(size), pos_(), len_(), count_() { - RAPIDJSON_ASSERT(buffer_ != 0 && static_cast(size_) > 0); - if (RAPIDJSON_UNLIKELY(size_ < sizeof(peekBuffer_) / sizeof(Ch))) { - size_ = sizeof(peekBuffer_) / sizeof(Ch); - buffer_ = peekBuffer_; - } + //! Constructor. + /*! + \param stream stream opened for read. + */ + BasicIStreamWrapper(StreamType &stream) : stream_(stream), buffer_(peekBuffer_), bufferSize_(4), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) { + Read(); } - Ch Peek() const { - if (RAPIDJSON_UNLIKELY(pos_ == len_) && !Read()) - return static_cast('\0'); - return buffer_[pos_]; + //! Constructor. + /*! + \param stream stream opened for read. + \param buffer user-supplied buffer. + \param bufferSize size of buffer in bytes. Must >=4 bytes. + */ + BasicIStreamWrapper(StreamType &stream, char* buffer, size_t bufferSize) : stream_(stream), buffer_(buffer), bufferSize_(bufferSize), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) { + RAPIDJSON_ASSERT(bufferSize >= 4); + Read(); } - Ch Take() { - if (RAPIDJSON_UNLIKELY(pos_ == len_) && !Read()) - return static_cast('\0'); - return buffer_[pos_++]; - } - - // tellg() may return -1 when failed. So we count by ourself. - size_t Tell() const { return count_ + pos_; } + Ch Peek() const { return *current_; } + Ch Take() { Ch c = *current_; Read(); return c; } + size_t Tell() const { return count_ + static_cast(current_ - buffer_); } // Not implemented - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } void Put(Ch) { RAPIDJSON_ASSERT(false); } - void Flush() { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } // For encoding detection only. const Ch* Peek4() const { - RAPIDJSON_ASSERT(sizeof(Ch) == 1); // Only usable for byte stream. - if (len_ - pos_ < 4) { - if (pos_) { - len_ -= pos_; - std::memmove(buffer_, buffer_ + pos_, len_); - count_ += pos_; - pos_ = 0; - } - if (!stream_.read(buffer_ + len_, static_cast(size_ - len_))) { - len_ += static_cast(stream_.gcount()); - if (len_ < 4) - return 0; - } - else - len_ = size_; - } - return &buffer_[pos_]; + return (current_ + 4 - !eof_ <= bufferLast_) ? current_ : 0; } private: + BasicIStreamWrapper(); BasicIStreamWrapper(const BasicIStreamWrapper&); BasicIStreamWrapper& operator=(const BasicIStreamWrapper&); - size_t Read() const { - count_ += pos_; - pos_ = 0; - if (!stream_.read(buffer_, static_cast(size_))) - len_ = static_cast(stream_.gcount()); - else - len_ = size_; - return len_; + void Read() { + if (current_ < bufferLast_) + ++current_; + else if (!eof_) { + count_ += readCount_; + readCount_ = bufferSize_; + bufferLast_ = buffer_ + readCount_ - 1; + current_ = buffer_; + + if (!stream_.read(buffer_, static_cast(bufferSize_))) { + readCount_ = static_cast(stream_.gcount()); + *(bufferLast_ = buffer_ + readCount_) = '\0'; + eof_ = true; + } + } } - StreamType& stream_; + StreamType &stream_; Ch peekBuffer_[4], *buffer_; - size_t size_; - mutable size_t pos_, len_, count_; + size_t bufferSize_; + Ch *bufferLast_; + Ch *current_; + size_t readCount_; + size_t count_; //!< Number of characters read + bool eof_; }; typedef BasicIStreamWrapper IStreamWrapper; diff --git a/test/perftest/rapidjsontest.cpp b/test/perftest/rapidjsontest.cpp index 785c1a2..9492cc5 100644 --- a/test/perftest/rapidjsontest.cpp +++ b/test/perftest/rapidjsontest.cpp @@ -454,16 +454,6 @@ TEST_F(RapidJson, FileReadStream) { } } -TEST_F(RapidJson, FileReadStream_Unbuffered) { - for (size_t i = 0; i < kTrialCount; i++) { - FILE *fp = fopen(filename_, "rb"); - FileReadStream s(fp); - while (s.Take() != '\0') - ; - fclose(fp); - } -} - TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler_FileReadStream)) { for (size_t i = 0; i < kTrialCount; i++) { FILE *fp = fopen(filename_, "rb"); @@ -476,17 +466,6 @@ TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler_FileReadStream)) { } } -TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler_FileReadStream_Unbuffered)) { - for (size_t i = 0; i < kTrialCount; i++) { - FILE *fp = fopen(filename_, "rb"); - FileReadStream s(fp); - BaseReaderHandler<> h; - Reader reader; - reader.Parse(s, h); - fclose(fp); - } -} - TEST_F(RapidJson, IStreamWrapper) { for (size_t i = 0; i < kTrialCount; i++) { std::ifstream is(filename_, std::ios::in | std::ios::binary);