gvsafronov/rapidjson
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1424 from ylavic/file_input_streams

Browse Source 
Optimize FileReadStream and BasicIStreamWrapper.
This commit is contained in:
Milo Yip 2018-12-06 23:43:48 +08:00 committed by GitHub
commit b4b0e13b4d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 122 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
include/rapidjson/istreamwrapper.h
View File

@ -48,57 +48,71 @@ template <typename StreamType>
class BasicIStreamWrapper {
public:
typedef typename StreamType::char_type Ch;
BasicIStreamWrapper(StreamType& stream) : stream_(stream), count_(), peekBuffer_() {}
Ch Peek() const {
typename StreamType::int_type c = stream_.peek();
return RAPIDJSON_LIKELY(c != StreamType::traits_type::eof()) ? static_cast<Ch>(c) : static_cast<Ch>('\0');
//! Constructor.
/*!
\param stream stream opened for read.
*/
BasicIStreamWrapper(StreamType &stream) : stream_(stream), buffer_(peekBuffer_), bufferSize_(4), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) {
Read();
}
Ch Take() {
typename StreamType::int_type c = stream_.get();
if (RAPIDJSON_LIKELY(c != StreamType::traits_type::eof())) {
count_++;
return static_cast<Ch>(c);
}
else
return '\0';
//! Constructor.
/*!
\param stream stream opened for read.
\param buffer user-supplied buffer.
\param bufferSize size of buffer in bytes. Must >=4 bytes.
*/
BasicIStreamWrapper(StreamType &stream, char* buffer, size_t bufferSize) : stream_(stream), buffer_(buffer), bufferSize_(bufferSize), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) {
RAPIDJSON_ASSERT(bufferSize >= 4);
Read();
}
// tellg() may return -1 when failed. So we count by ourself.
size_t Tell() const { return count_; }
Ch Peek() const { return *current_; }
Ch Take() { Ch c = *current_; Read(); return c; }
size_t Tell() const { return count_ + static_cast<size_t>(current_ - buffer_); }
Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; }
// Not implemented
void Put(Ch) { RAPIDJSON_ASSERT(false); }
void Flush() { RAPIDJSON_ASSERT(false); }
void Flush() { RAPIDJSON_ASSERT(false); }
Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; }
size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; }
// For encoding detection only.
const Ch* Peek4() const {
RAPIDJSON_ASSERT(sizeof(Ch) == 1); // Only usable for byte stream.
int i;
bool hasError = false;
for (i = 0; i < 4; ++i) {
typename StreamType::int_type c = stream_.get();
if (c == StreamType::traits_type::eof()) {
hasError = true;
stream_.clear();
break;
}
peekBuffer_[i] = static_cast<Ch>(c);
}
for (--i; i >= 0; --i)
stream_.putback(peekBuffer_[i]);
return !hasError ? peekBuffer_ : 0;
return (current_ + 4 - !eof_ <= bufferLast_) ? current_ : 0;
}
private:
BasicIStreamWrapper();
BasicIStreamWrapper(const BasicIStreamWrapper&);
BasicIStreamWrapper& operator=(const BasicIStreamWrapper&);
StreamType& stream_;
size_t count_; //!< Number of characters read. Note:
mutable Ch peekBuffer_[4];
void Read() {
if (current_ < bufferLast_)
++current_;
else if (!eof_) {
count_ += readCount_;
readCount_ = bufferSize_;
bufferLast_ = buffer_ + readCount_ - 1;
current_ = buffer_;
if (!stream_.read(buffer_, static_cast<std::streamsize>(bufferSize_))) {
readCount_ = static_cast<size_t>(stream_.gcount());
*(bufferLast_ = buffer_ + readCount_) = '\0';
eof_ = true;
}
}
}
StreamType &stream_;
Ch peekBuffer_[4], *buffer_;
size_t bufferSize_;
Ch *bufferLast_;
Ch *current_;
size_t readCount_;
size_t count_; //!< Number of characters read
bool eof_;
};
typedef BasicIStreamWrapper<std::istream> IStreamWrapper;

74
test/perftest/rapidjsontest.cpp
View File

@ -21,9 +21,12 @@
#include "rapidjson/prettywriter.h"
#include "rapidjson/stringbuffer.h"
#include "rapidjson/filereadstream.h"
#include "rapidjson/istreamwrapper.h"
#include "rapidjson/encodedstream.h"
#include "rapidjson/memorystream.h"
#include <fstream>
#ifdef RAPIDJSON_SSE2
#define SIMD_SUFFIX(name) name##_SSE2
#elif defined(RAPIDJSON_SSE42)
@ -463,6 +466,77 @@ TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler_FileReadStream)) {
}
}
TEST_F(RapidJson, IStreamWrapper) {
for (size_t i = 0; i < kTrialCount; i++) {
std::ifstream is(filename_, std::ios::in | std::ios::binary);
char buffer[65536];
IStreamWrapper isw(is, buffer, sizeof(buffer));
while (isw.Take() != '\0')
;
is.close();
}
}
TEST_F(RapidJson, IStreamWrapper_Unbuffered) {
for (size_t i = 0; i < kTrialCount; i++) {
std::ifstream is(filename_, std::ios::in | std::ios::binary);
IStreamWrapper isw(is);
while (isw.Take() != '\0')
;
is.close();
}
}
TEST_F(RapidJson, IStreamWrapper_Setbuffered) {
for (size_t i = 0; i < kTrialCount; i++) {
std::ifstream is;
char buffer[65536];
is.rdbuf()->pubsetbuf(buffer, sizeof(buffer));
is.open(filename_, std::ios::in | std::ios::binary);
IStreamWrapper isw(is);
while (isw.Take() != '\0')
;
is.close();
}
}
TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler_IStreamWrapper)) {
for (size_t i = 0; i < kTrialCount; i++) {
std::ifstream is(filename_, std::ios::in | std::ios::binary);
char buffer[65536];
IStreamWrapper isw(is, buffer, sizeof(buffer));
BaseReaderHandler<> h;
Reader reader;
reader.Parse(isw, h);
is.close();
}
}
TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler_IStreamWrapper_Unbuffered)) {
for (size_t i = 0; i < kTrialCount; i++) {
std::ifstream is(filename_, std::ios::in | std::ios::binary);
IStreamWrapper isw(is);
BaseReaderHandler<> h;
Reader reader;
reader.Parse(isw, h);
is.close();
}
}
TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler_IStreamWrapper_Setbuffered)) {
for (size_t i = 0; i < kTrialCount; i++) {
std::ifstream is;
char buffer[65536];
is.rdbuf()->pubsetbuf(buffer, sizeof(buffer));
is.open(filename_, std::ios::in | std::ios::binary);
IStreamWrapper isw(is);
BaseReaderHandler<> h;
Reader reader;
reader.Parse(isw, h);
is.close();
}
}
TEST_F(RapidJson, StringBuffer) {
StringBuffer sb;
for (int i = 0; i < 32 * 1024 * 1024; i++)