diff --git a/simgear/io/iostreams/CMakeLists.txt b/simgear/io/iostreams/CMakeLists.txt index 6c87ae22..37be39c7 100644 --- a/simgear/io/iostreams/CMakeLists.txt +++ b/simgear/io/iostreams/CMakeLists.txt @@ -4,12 +4,14 @@ set(HEADERS sgstream.hxx gzfstream.hxx gzcontainerfile.hxx + zlibstream.hxx ) set(SOURCES sgstream.cxx gzfstream.cxx gzcontainerfile.cxx + zlibstream.cxx ) simgear_component(IOStreams io/iostreams "${SOURCES}" "${HEADERS}") @@ -20,4 +22,8 @@ if(ENABLE_TESTS) target_link_libraries(test_streams ${TEST_LIBS}) add_test(streams ${EXECUTABLE_OUTPUT_PATH}/test_streams) + add_executable(test_zlibstream zlibstream_test.cxx) + target_link_libraries(test_zlibstream ${TEST_LIBS}) + add_test(zlibstream ${EXECUTABLE_OUTPUT_PATH}/test_zlibstream) + endif(ENABLE_TESTS) diff --git a/simgear/io/iostreams/zlibstream.cxx b/simgear/io/iostreams/zlibstream.cxx new file mode 100644 index 00000000..3b308521 --- /dev/null +++ b/simgear/io/iostreams/zlibstream.cxx @@ -0,0 +1,695 @@ +// -*- coding: utf-8 -*- +// +// zlibstream.cxx --- IOStreams classes for working with RFC 1950 and RFC 1952 +// compression formats (respectively known as the zlib and +// gzip formats) +// +// Copyright (C) 2017 Florent Rougon +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +#include +#include // std::streamsize +#include +#include +#include +#include +#include // std::numeric_limits +#include // std::size_t, std::ptrdiff_t +#include + +#include + +#include +#include +#include +#include +#include + +using std::string; +using traits = std::char_traits; + + +// Private utility function +static string zlibErrorMessage(const z_stream& zstream, int errorCode) +{ + string res; + std::unordered_map errorCodeToMessageMap = { + {Z_OK, "zlib: no error (code Z_OK)"}, + {Z_STREAM_END, "zlib stream end"}, + {Z_NEED_DICT, "zlib: Z_NEED_DICT"}, + {Z_STREAM_ERROR, "zlib stream error"}, + {Z_DATA_ERROR, "zlib data error"}, + {Z_MEM_ERROR, "zlib memory error"}, + {Z_BUF_ERROR, "zlib buffer error"}, + {Z_VERSION_ERROR, "zlib version error"} + }; + + if (errorCode == Z_ERRNO) { + res = simgear::strutils::error_string(errno); + } else if (zstream.msg != nullptr) { + // Caution: this only works if the zstream structure hasn't been + // deallocated! + res = "zlib: " + string(zstream.msg); + } else { + try { + res = errorCodeToMessageMap.at(errorCode); + } catch (const std::out_of_range&) { + res = string("unknown zlib error (code " + std::to_string(errorCode) + + ")"); + } + } + + return res; +} + +// Return the largest value that can be represented with zlib's uInt type, +// which is the type of z_stream.avail_in and z_stream.avail_out, hence the +// function name. +static std::size_t zlibMaxChunkSize() +{ + uLong flags = ::zlibCompileFlags(); + std::size_t res; + + switch (flags & 0x3) { + case 0x3: + SG_LOG(SG_IO, SG_WARN, + "Unknown size for zlib's uInt type (code 3). Will assume 64 bits, " + "but the actual value is probably higher."); + // No 'break' here, this is intentional. + case 0x2: + res = 0xFFFFFFFFFFFFFFFF; // 2^64 - 1 + break; + case 0x1: + res = 0xFFFFFFFF; // 2^32 - 1 + break; + case 0x0: + res = 0xFFFF; // 2^16 - 1 + break; + default: + throw std::logic_error("It should be impossible to get here."); + } + + return res; +} + +static const std::size_t zlibMaxChunk = ::zlibMaxChunkSize(); + + +namespace simgear +{ + +// *************************************************************************** +// * ZlibAbstractIStreambuf class * +// *************************************************************************** + +// Common initialization. Subclasses must complete the z_stream struct +// initialization with a call to deflateInit2() or inflateInit2(), typically. +ZlibAbstractIStreambuf::ZlibAbstractIStreambuf(std::istream& iStream, + const SGPath& path, + char* inBuf, + std::size_t inBufSize, + char *outBuf, + std::size_t outBufSize, + std::size_t putbackSize) + : _iStream(iStream), + _path(path), + _inBuf(inBuf), + _inBufSize(inBufSize), + _outBuf(outBuf), + _outBufSize(outBufSize), + _putbackSize(putbackSize) + +{ + assert(_inBufSize > 0); + assert(_putbackSize >= 0); // guaranteed unless the type is changed... + assert(_putbackSize < _outBufSize); + + if (_inBuf == nullptr) { + _inBuf = new char[_inBufSize]; + _inBufMustBeFreed = true; + } + + if (_outBuf == nullptr) { + _outBuf = new char[_outBufSize]; + _outBufMustBeFreed = true; + } + + _inBufEndPtr = _inBuf; + // The input buffer is empty. + _zstream.next_in = reinterpret_cast(_inBuf); + _zstream.avail_in = 0; + // ZLib's documentation says its init functions such as inflateInit2() might + // consume stream input, therefore let's fill the input buffer now. This + // way, constructors of derived classes just have to call the appropriate + // ZLib init function: the data will already be in place. + getInputData(); + + // Force underflow() of the stream buffer on the first read. We could use + // some other value, but I avoid nullptr in order to be sure we can always + // reliably compare the three pointers with < and >, as well as compute the + // difference between any two of them. + setg(_outBuf, _outBuf, _outBuf); +} + +ZlibAbstractIStreambuf::~ZlibAbstractIStreambuf() +{ + if (_inBufMustBeFreed) { + delete[] _inBuf; + } + + if (_outBufMustBeFreed) { + delete[] _outBuf; + } +} + +// Fill or refill the output buffer, and update the three pointers +// corresponding to eback(), gptr() and egptr(). +int ZlibAbstractIStreambuf::underflow() +{ + if (_allFinished) { + return traits::eof(); + } + + // According to the C++11 standard: “The public members of basic_streambuf + // call this virtual function only if gptr() is null or gptr() >= egptr()”. + // Still, it seems some people do the following or similar (maybe in case + // underflow() is called “incorrectly”?). See for instance N. Josuttis, The + // C++ Standard Library (1st edition), p. 584. One sure thing is that it + // can't hurt (except performance, marginally), so let's do it. + if (gptr() < egptr()) { + return traits::to_int_type(*gptr()); + } + + assert(gptr() == egptr()); + assert(egptr() - eback() >= 0); + std::size_t nbPutbackChars = std::min( + static_cast(egptr() - eback()), // OK because egptr() >= eback() + _putbackSize); + std::copy(egptr() - nbPutbackChars, egptr(), + _outBuf + _putbackSize - nbPutbackChars); + + setg(_outBuf + _putbackSize - nbPutbackChars, // start of putback area + _outBuf + _putbackSize, // start of obtained data + fillOutputBuffer()); // one-past-end of obtained data + + return (gptr() == egptr()) ? traits::eof() : traits::to_int_type(*gptr()); +} + +// Simple utility method for fillOutputBuffer(), used to improve readability. +// Return the remaining space available in the output buffer, where zlib can +// write. +std::size_t +ZlibAbstractIStreambuf::fOB_remainingSpace(unsigned char* nextOutPtr) const +{ + std::ptrdiff_t remainingSpaceInOutBuf = _outBuf + _outBufSize - + reinterpret_cast(nextOutPtr); + assert(remainingSpaceInOutBuf >= 0); + + return static_cast(remainingSpaceInOutBuf); +} + +// Simple method for dealing with the Z_BUF_ERROR code that may be returned +// by zlib's deflate() and inflate() methods. +[[ noreturn ]] void ZlibAbstractIStreambuf::handleZ_BUF_ERROR() const +{ + switch (operationType()) { + case OPERATION_TYPE_DECOMPRESSION: + { + string message = (_path.isNull()) ? + "Got Z_BUF_ERROR from zlib while decompressing a stream. The stream " + "was probably incomplete." + : + "Got Z_BUF_ERROR from zlib during decompression. The compressed stream " + "was probably incomplete."; + // When _path.isNull(), sg_location(_path) is equivalent to sg_location() + throw sg_io_exception(message, sg_location(_path)); + } + case OPERATION_TYPE_COMPRESSION: + throw std::logic_error( + "Called ZlibAbstractIStreambuf::handleZ_BUF_ERROR() with " + "operationType() == OPERATION_TYPE_DECOMPRESSION"); + default: + throw std::logic_error( + "Unexpected operationType() in " + "ZlibAbstractIStreambuf::handleZ_BUF_ERROR(): " + + std::to_string(operationType())); + } +} + +// Fill or refill the output buffer. Return a pointer to the first unused char +// in _outBuf (i.e., right after the data written by this method, if any; +// otherwise: _outBuf + _putbackSize). +char* ZlibAbstractIStreambuf::fillOutputBuffer() +{ + bool allInputRead = false; + std::size_t remainingSpaceInOutBuf; + int retCode; + + // We have to do these unpleasant casts, because zlib uses pointers to + // unsigned char for its input and output buffers, while the IOStreams + // library in the C++ standard uses plain char pointers... + _zstream.next_out = reinterpret_cast(_outBuf + _putbackSize); + remainingSpaceInOutBuf = fOB_remainingSpace(_zstream.next_out); + + while (remainingSpaceInOutBuf > 0) { + // This does fit in a zlib uInt: that's the whole point of zlibMaxChunk. + _zstream.avail_out = static_cast( + std::min(remainingSpaceInOutBuf, zlibMaxChunk)); + + if (_zstream.avail_in == 0 && !allInputRead) { + // Get data from _iStream, store it in _inBuf + allInputRead = getInputData(); + } + + // Make zlib process some data (compress or decompress it). This updates + // _zstream.{avail,next}_{in,out} (4 fields of the z_stream struct). + retCode = zlibProcessData(); + + if (retCode == Z_BUF_ERROR) { + handleZ_BUF_ERROR(); // doesn't return + } else if (retCode == Z_STREAM_END) { + assert(_zstream.avail_in == 0); // all of _inBuf must have been used + _allFinished = true; + break; + } else if (retCode < 0) { // negative codes are errors + throw sg_io_exception(::zlibErrorMessage(_zstream, retCode), + sg_location(_path)); + } + + remainingSpaceInOutBuf = fOB_remainingSpace(_zstream.next_out); + } + + return reinterpret_cast(_zstream.next_out); +} + +// This method provides input data to zlib: +// - if data is already available in _inBuf, it updates _zstream.avail_in +// accordingly (using the largest possible amount given _zstream.avail_in's +// type, i.e., uInt); +// - otherwise, it reads() as much data as possible into _inBuf from +// _iStream and updates _zstream.avail_in to tell zlib about this new +// available data (again: largest possible amount given the uInt type). +// +// This method must be called only when _zstream.avail_in == 0, which means +// zlib has read everything we fed it (and does *not* mean, by the way, that +// the input buffer starting at _inBuf is empty; this is because the buffer +// size might exceed what can be represented by an uInt). +// +// On return: +// - either EOF has been reached for _iStream, and the return value is true; +// - or _zstream.avail_in > 0, and the return value is false. +// +// Note: don't read more in the previous paragraph concerning the state on +// return. In the first case, there is *no guarantee* about +// _zstream.avail_in's value; in the second case, there is *no guarantee* +// about whether EOF has been reached for _iStream. +bool ZlibAbstractIStreambuf::getInputData() +{ + bool allInputRead = false; + + assert(_zstream.avail_in == 0); + std::ptrdiff_t alreadyAvailable = + _inBufEndPtr - reinterpret_cast(_zstream.next_in); + assert(alreadyAvailable >= 0); + + // Data already available? + if (alreadyAvailable > 0) { + _zstream.avail_in = static_cast( + std::min(static_cast(alreadyAvailable), + zlibMaxChunk)); + return allInputRead; + } + + if (_inBufEndPtr == _inBuf + _inBufSize) { // buffer full, rewind + _inBufEndPtr = _inBuf; + _zstream.next_in = reinterpret_cast(_inBuf); + } + + // Fill the input buffer (as much as possible) + while (_inBufEndPtr < _inBuf + _inBufSize && !_iStream.eof()) { + std::streamsize nbCharsToRead = std::min( + _inBuf + _inBufSize - _inBufEndPtr, + std::numeric_limits::max()); // max we can pass to read() + _iStream.read(_inBufEndPtr, nbCharsToRead); + + if (_iStream.bad()) { + string errMsg = simgear::strutils::error_string(errno); + string msgStart = (_path.isNull()) ? + "Error while reading from a stream" : "Read error"; + throw sg_io_exception(msgStart + ": " + errMsg, sg_location(_path)); + } + + // Could be zero if at EOF + std::streamsize nbCharsRead = _iStream.gcount(); + // std::streamsize is a signed integral type! + assert(0 <= nbCharsRead); + _inBufEndPtr += nbCharsRead; + } + + std::ptrdiff_t availableChars = + _inBufEndPtr - reinterpret_cast(_zstream.next_in); + assert(availableChars >= 0); + _zstream.avail_in = static_cast( + std::min(static_cast(availableChars), + zlibMaxChunk)); + + if (_iStream.eof()) { + allInputRead = true; + } else { + // Trying to rewind a fully read std::istringstream with seekg() can lead + // to a weird state, where the stream doesn't return any character but + // doesn't report EOF either. Make sure we are not in this situation. + assert(_zstream.avail_in > 0); + } + + return allInputRead; +} + +// Implementing this method is optional, but should provide better +// performance. It makes zlib write the data directly in the buffer starting +// at 'dest'. Without it, the data would go through an intermediate buffer +// (_outBuf) before being copied to its (hopefully) final destination. +std::streamsize ZlibAbstractIStreambuf::xsgetn(char* dest, std::streamsize n) +{ + std::size_t remaining = static_cast(n); + std::size_t chunkSize; + std::ptrdiff_t avail; + const char* origGptr = gptr(); + char* writePtr = dest; // we'll need dest later -> work with a copy + + // First, let's take data present in our internal buffer (_outBuf) + while (remaining > 0) { + avail = egptr() - gptr(); // number of available chars in _outBuf + if (avail == 0) { // our internal buffer is empty + break; + } + + chunkSize = std::min(remaining, static_cast(avail)); + std::copy(gptr(), gptr() + chunkSize, writePtr); + gbump(chunkSize); + writePtr += chunkSize; + remaining -= chunkSize; + } + + if (remaining == 0) { + // Everything we needed was already in _outBuf. The putback area is set up + // as it should between eback() and the current gptr(), so we are fine to + // return. + return n; + } + + // Now, let's make it so that the remaining data we need is directly written + // to the destination area, without going through _outBuf. + _zstream.next_out = reinterpret_cast(writePtr); + bool allInputRead = false; + int retCode; + + while (remaining > 0) { + chunkSize = std::min(remaining, zlibMaxChunk); + // It does fit in a zlib uInt: that's the whole point of zlibMaxChunk. + _zstream.avail_out = static_cast(chunkSize); + + if (_zstream.avail_in == 0 && !allInputRead) { + allInputRead = getInputData(); + } + + // Make zlib process some data (compress or decompress). This updates + // _zstream.{avail,next}_{in,out} (4 fields of the z_stream struct). + retCode = zlibProcessData(); + // chunkSize - _zstream.avail_out is the nb of chars written by zlib + remaining -= chunkSize - _zstream.avail_out; + + if (retCode == Z_BUF_ERROR) { + handleZ_BUF_ERROR(); // doesn't return + } else if (retCode == Z_STREAM_END) { + assert(_zstream.avail_in == 0); // all of _inBuf must have been used + _allFinished = true; + break; + } else if (retCode < 0) { // negative codes are errors + throw sg_io_exception(::zlibErrorMessage(_zstream, retCode), + sg_location(_path)); + } + } + + // Finally, prepare the putback area. + // + // We could use reinterpret_cast(_zstream.next_out) everywhere below, + // except it is hardly readable. This points after the latest data we wrote. + writePtr = reinterpret_cast(_zstream.next_out); + + // There are two buffers containing characters we potentially have to copy + // to the putback area: the one starting at _outBuf and the one starting at + // dest. In the following diagram, ***** represents those from _outBuf, + // which are right-aligned at origGptr[1], and the series of # represents + // those in the [dest, writePtr) address range: + // + // |_outBuf |eback() *****|origGptr |_outBuf + _outBufSize + // + // |dest #################|writePtr + // + // Together, these two memory blocks logically form a contiguous stream of + // chars we gave to the “client”: *****#################. All we have to do + // now is copy the appropriate amount of chars from this logical stream to + // the putback area, according to _putbackSize. These chars are the last + // ones in said stream (i.e., right portion of *****#################), but + // we have to copy them in order of increasing address to avoid possible + // overlapping problems in _outBuf. This is because some of the chars to + // copy may be located before _outBuf + _putbackSize (i.e., already be in + // the putback area). + // + // [1] This means that the last char represented by a star is at address + // origGptr-1. + assert(writePtr - dest >= 0); + std::size_t inDestBuffer = static_cast(writePtr - dest); + assert(origGptr - eback() >= 0); + std::size_t nbPutbackChars = std::min( + static_cast(origGptr - eback()) + inDestBuffer, + _putbackSize); + std::size_t nbPutbackCharsToGo = nbPutbackChars; + + // chunkSize has an unsigned type; precomputing it before the following test + // wouldn't work. + // Are there chars in _outBuf that need to be copied to the putback area? + if (nbPutbackChars > inDestBuffer) { + chunkSize = nbPutbackChars - inDestBuffer; // yes, this number + std::copy(origGptr - chunkSize, origGptr, + _outBuf + _putbackSize - nbPutbackChars); + nbPutbackCharsToGo -= chunkSize; + } + + std::copy(writePtr - nbPutbackCharsToGo, writePtr, + _outBuf + _putbackSize - nbPutbackCharsToGo); + setg(_outBuf + _putbackSize - nbPutbackChars, // start of putback area + _outBuf + _putbackSize, // the buffer for pending, + _outBuf + _putbackSize); // available data is empty + + std::streamsize rem = static_cast(remaining); + // This is guaranteed because n is of type std::streamsize and the whole + // algorithm ensures that 0 <= remaining <= n. + assert(rem >= 0); + assert(static_cast(rem) == remaining); + assert(n - rem >= 0); + // Total number of chars copied. + return n - rem; +} + +// *************************************************************************** +// * ZlibCompressorIStreambuf class * +// *************************************************************************** + +ZlibCompressorIStreambuf::ZlibCompressorIStreambuf( + std::istream& iStream, + const SGPath& path, + int compressionLevel, + ZLibCompressionFormat format, + ZLibMemoryStrategy memStrategy, + char* inBuf, + std::size_t inBufSize, + char *outBuf, + std::size_t outBufSize, + std::size_t putbackSize) + : ZlibAbstractIStreambuf(iStream, path, inBuf, inBufSize, outBuf, outBufSize, + putbackSize) +{ + zStreamInit(compressionLevel, format, memStrategy); +} + +ZlibCompressorIStreambuf::~ZlibCompressorIStreambuf() +{ + int retCode = deflateEnd(&_zstream); // deallocate the z_stream struct + if (retCode != Z_OK) { + // In C++11, we can't throw exceptions from a destructor. + SG_LOG(SG_IO, SG_ALERT, "ZlibCompressorIStreambuf: " << + ::zlibErrorMessage(_zstream, retCode)); + } +} + +ZlibAbstractIStreambuf::OperationType +ZlibCompressorIStreambuf::operationType() const +{ + return OPERATION_TYPE_COMPRESSION; +} + +void ZlibCompressorIStreambuf::zStreamInit(int compressionLevel, + ZLibCompressionFormat format, + ZLibMemoryStrategy memStrategy) +{ + // Intentionally not listing ZLIB_COMPRESSION_FORMAT_AUTODETECT here + std::unordered_map compressionFormatMap = { + {ZLIB_COMPRESSION_FORMAT_ZLIB, 15}, + {ZLIB_COMPRESSION_FORMAT_GZIP, 31} + }; + std::unordered_map memoryStrategyMap = { + {ZLIB_FAVOR_MEMORY_OVER_SPEED, 8}, + {ZLIB_FAVOR_SPEED_OVER_MEMORY, 9} + }; + + // ZLIB_COMPRESSION_FORMAT_AUTODETECT is only for decompression! + assert(format != ZLIB_COMPRESSION_FORMAT_AUTODETECT); + int windowBits = compressionFormatMap.at(format); + int memLevel = memoryStrategyMap.at(memStrategy); + + _zstream.zalloc = Z_NULL; // No custom memory allocation routines + _zstream.zfree = Z_NULL; // Ditto. Therefore, the 'opaque' field won't + _zstream.opaque = Z_NULL; // be used, actually. + + int retCode = deflateInit2(&_zstream, compressionLevel, Z_DEFLATED, + windowBits, memLevel, Z_DEFAULT_STRATEGY); + if (retCode != Z_OK) { + throw sg_io_exception(::zlibErrorMessage(_zstream, retCode), + sg_location(_path)); + } +} + +int ZlibCompressorIStreambuf::zlibProcessData() +{ + // Compress as much data as possible given _zstream.avail_in and + // _zstream.avail_out. The input data starts at _zstream.next_in, the output + // at _zstream.next_out, and these four fields are all updated by this call + // to reflect exactly the amount of data zlib has consumed and produced. + return deflate(&_zstream, _zstream.avail_in ? Z_NO_FLUSH : Z_FINISH); +} + +// *************************************************************************** +// * ZlibDecompressorIStreambuf class * +// *************************************************************************** + +ZlibDecompressorIStreambuf::ZlibDecompressorIStreambuf( + std::istream& iStream, + const SGPath& path, + ZLibCompressionFormat format, + char* inBuf, + std::size_t inBufSize, + char *outBuf, + std::size_t outBufSize, + std::size_t putbackSize) + : ZlibAbstractIStreambuf(iStream, path, inBuf, inBufSize, outBuf, outBufSize, + putbackSize) +{ + zStreamInit(format); +} + +ZlibDecompressorIStreambuf::~ZlibDecompressorIStreambuf() +{ + int retCode = inflateEnd(&_zstream); // deallocate the z_stream struct + if (retCode != Z_OK) { + // In C++11, we can't throw exceptions from a destructor. + SG_LOG(SG_IO, SG_ALERT, "ZlibDecompressorIStreambuf: " << + ::zlibErrorMessage(_zstream, retCode)); + } +} + +ZlibAbstractIStreambuf::OperationType +ZlibDecompressorIStreambuf::operationType() const +{ + return OPERATION_TYPE_DECOMPRESSION; +} + +void ZlibDecompressorIStreambuf::zStreamInit(ZLibCompressionFormat format) +{ + std::unordered_map compressionFormatMap = { + {ZLIB_COMPRESSION_FORMAT_ZLIB, 15}, + {ZLIB_COMPRESSION_FORMAT_GZIP, 31}, + {ZLIB_COMPRESSION_FORMAT_AUTODETECT, 47} // 47 = 32 + 15 + }; + + int windowBits = compressionFormatMap.at(format); + + _zstream.zalloc = Z_NULL; // No custom memory allocation routines + _zstream.zfree = Z_NULL; // Ditto. Therefore, the 'opaque' field won't + _zstream.opaque = Z_NULL; // be used, actually. + + int retCode = inflateInit2(&_zstream, windowBits); + if (retCode != Z_OK) { + throw sg_io_exception(::zlibErrorMessage(_zstream, retCode), + sg_location(_path)); + } +} + +int ZlibDecompressorIStreambuf::zlibProcessData() +{ + // Decompress as much data as possible given _zstream.avail_in and + // _zstream.avail_out. The input data starts at _zstream.next_in, the output + // at _zstream.next_out, and these four fields are all updated by this call + // to reflect exactly the amount of data zlib has consumed and produced. + return inflate(&_zstream, _zstream.avail_in ? Z_NO_FLUSH : Z_FINISH); +} + +// *************************************************************************** +// * ZlibCompressorIStream class * +// *************************************************************************** + +ZlibCompressorIStream::ZlibCompressorIStream(std::istream& iStream, + const SGPath& path, + int compressionLevel, + ZLibCompressionFormat format, + ZLibMemoryStrategy memStrategy, + char* inBuf, + std::size_t inBufSize, + char *outBuf, + std::size_t outBufSize, + std::size_t putbackSize) + : _streamBuf(iStream, path, compressionLevel, format, memStrategy, inBuf, + inBufSize, outBuf, outBufSize, putbackSize) +{ + rdbuf(&_streamBuf); // Associate _streamBuf to 'this' +} + +ZlibCompressorIStream::~ZlibCompressorIStream() +{ } + +// *************************************************************************** +// * ZlibDecompressorIStream class * +// *************************************************************************** + +ZlibDecompressorIStream::ZlibDecompressorIStream(std::istream& iStream, + const SGPath& path, + ZLibCompressionFormat format, + char* inBuf, + std::size_t inBufSize, + char *outBuf, + std::size_t outBufSize, + std::size_t putbackSize) + : _streamBuf(iStream, path, format, inBuf, inBufSize, outBuf, outBufSize, + putbackSize) +{ + rdbuf(&_streamBuf); // Associate _streamBuf to 'this' +} + +ZlibDecompressorIStream::~ZlibDecompressorIStream() +{ } + +} // of namespace simgear diff --git a/simgear/io/iostreams/zlibstream.hxx b/simgear/io/iostreams/zlibstream.hxx new file mode 100644 index 00000000..d268acc3 --- /dev/null +++ b/simgear/io/iostreams/zlibstream.hxx @@ -0,0 +1,401 @@ +// -*- coding: utf-8 -*- +// +// zlibstream.hxx --- IOStreams classes for working with RFC 1950 and RFC 1952 +// compression formats (respectively known as the zlib and +// gzip formats) +// +// Copyright (C) 2017 Florent Rougon +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +#ifndef SG_ZLIBSTREAM_HXX +#define SG_ZLIBSTREAM_HXX + +#include +#include // std::streamsize +#include // struct z_stream + +#include + +// This file contains: +// +// - two stream buffer classes (ZlibCompressorIStreambuf and +// ZlibDecompressorIStreambuf), both based on the same abstract class: +// ZlibAbstractIStreambuf; +// +// - two std::istream subclasses (ZlibCompressorIStream and +// ZlibDecompressorIStream), each creating and using the corresponding +// stream buffer class from the previous item. +// +// All these allow one to work with RFC 1950 and RFC 1952 compression +// formats, respectively known as the zlib and gzip formats. +// +// These classes are *input* streaming classes, which means they can +// efficiently handle arbitrary amounts of data without using any disk +// space nor increasing amounts of memory, and allow “client code” to pull +// exactly as much data as it wants at any given time, resuming later +// when it is ready to handle the next chunk. +// +// So, for example, assuming you've created an instance of +// ZlibCompressorIStream (bound to some input stream of your choice, let's +// call it iStream), you could read 512 bytes of data from it, and you +// would get the first 512 bytes of *compressed* data corresponding to what +// iStream provided. Then you could resume at any time and ask for the next +// 512 bytes of compressed data (or any other amount), etc. +// +// Therefore, these classes are well suited, among others, to compress or +// decompress data streams while at the same time packing the result into +// discrete chunks or packets with size constraints (you can think of the +// process as making sausages :). +// +// The input being in each case an std::istream (for compressing as well as +// for decompressing), it can be tied to an arbitrary source: a file with +// sg_ifstream or std::ifstream, a memory buffer with std::istringstream or +// std::stringstream, a TCP socket with a custom std::streambuf subclass[1] +// to interface with the sockets API, etc. +// +// [1] Possibly wrapped in an std::istream. +// +// The stream buffer classes upon which ZlibCompressorIStream and +// ZlibDecompressorIStream are built have an xsgetn() implementation that +// avoids useless copies of data by asking zlib to write directly to the +// destination buffer. This xsgetn() method is used when calling read() on +// the std::istream subclasses, or sgetn() if you are using the stream +// buffer classes directly (i.e., ZlibCompressorIStreambuf and +// ZlibDecompressorIStreambuf). Other std::istream methods may instead rely +// only on the internal buffer and the underflow() method, and therefore be +// less efficient for large amounts of data. You may want to take a look at +// zlibstream_test.cxx to see various ways of using these classes. +// +// In case you use std::istream& operator>>(std::istream&, std::string&) or +// its overload friends, beware that it splits fields at spaces, and by +// default ignores spaces at the beginning of a field (cf. std::skipws, +// std::noskipws and friends). As far as I understand it, most of these +// operators are mainly intended in the IOStreams library to be used to +// read an int here, a double there, a space-delimited string afterwards, +// etc. (the exception could be the overload writing to a stream buffer, +// however it doesn't seem to be very efficient on my system with GNU +// libstdc++ [it is not using xsgetn()], so beware also of this one if you +// are handling large amounts of data). For moderately complex or large +// input handling, I'd suggest to use std::istream methods such as read(), +// gcount() and getline() (std::getline() can be useful too). Or directly +// use the stream buffer classes, in particular with sgetn(). + + +namespace simgear +{ + +enum ZLibCompressionFormat { + ZLIB_COMPRESSION_FORMAT_ZLIB = 0, + ZLIB_COMPRESSION_FORMAT_GZIP, + ZLIB_COMPRESSION_FORMAT_AUTODETECT +}; + +enum ZLibMemoryStrategy { + ZLIB_FAVOR_MEMORY_OVER_SPEED = 0, + ZLIB_FAVOR_SPEED_OVER_MEMORY +}; + +// Abstract base class for both the compressor and decompressor stream buffers. +class ZlibAbstractIStreambuf: public std::streambuf +{ +public: + /** + * @brief Constructor for ZlibAbstractIStreambuf. + * @param iStream Input stream to read from. + * @param path Optional path to the file corresponding to iStream, + * if any. Only used for error messages. + * @param inBuf Pointer to the input buffer (data read from iStream is + * written there before being compressed or decompressed). + * If nullptr, the buffer is allocated on the heap in the + * constructor and deallocated in the destructor. + * @param inBufSize Size of the input buffer, in chars. + * @param outBuf Pointer to the output buffer. Data is read by zlib + * from the input buffer, compressed or decompressed, and + * the result is directly written to the output buffer. + * If nullptr, the buffer is allocated on the heap in the + * constructor and deallocated in the destructor. + * @param outBufSize Size of the output buffer, in chars. + * @param putbackSize Size of the putback area inside the output buffer, in + * chars. + * + * It is required that putbackSize < outBufSize. It is guaranteed that, + * if at least putbackSize chars have been read without any putback (or + * unget) operation intermixed, then at least putbackSize chars can be + * put back in sequence. If you don't need this feature, use zero for the + * putbackSize value (the default) for best performance. + */ + explicit ZlibAbstractIStreambuf(std::istream& iStream, + const SGPath& path = SGPath(), + char* inBuf = nullptr, + std::size_t inBufSize = 262144, + char* outBuf = nullptr, + std::size_t outBufSize = 262144, + std::size_t putbackSize = 0); + ZlibAbstractIStreambuf(const ZlibAbstractIStreambuf&) = delete; + ZlibAbstractIStreambuf& operator=(const ZlibAbstractIStreambuf&) = delete; + ~ZlibAbstractIStreambuf(); + +protected: + enum OperationType { + OPERATION_TYPE_COMPRESSION = 0, + OPERATION_TYPE_DECOMPRESSION + }; + + virtual OperationType operationType() const = 0; + + // Either compress or decompress a chunk of data (depending on the + // particular subclass implementation). The semantics are the same as for + // zlib's inflate() and deflate() functions applied to member _zstream, + // concerning 1) the return value and 2) where, and how much to read and + // write (which thus depends on _zstream.avail_in, _zstream.next_in, + // _zstream.avail_out and _zstream.next_out). + virtual int zlibProcessData() = 0; + + // The input stream, from which data is read before being processed by zlib + std::istream& _iStream; + // Corresponding path, if any (default-constructed SGPath instance otherwise) + const SGPath _path; + // Structure used to communicate with zlib + z_stream _zstream; + +private: + // Callback whose role is to refill the output buffer when it's empty and + // the “client” tries to read more. + int underflow() override; + // Optional override when subclassing std::streambuf. This is the most + // efficient way of reading several characters (as soon as we've emptied the + // output buffer, data is written by zlib directly to the destination + // buffer). + std::streamsize xsgetn(char* dest, std::streamsize n) override; + // Make sure there is data to read in the input buffer, or signal EOF. + bool getInputData(); + // Utility method for fillOutputBuffer() + std::size_t fOB_remainingSpace(unsigned char* nextOutPtr) const; + // Fill the output buffer (using zlib functions) as much as possible. + char* fillOutputBuffer(); + // Utility method + [[ noreturn ]] void handleZ_BUF_ERROR() const; + + bool _allFinished = false; + + // The buffers + // ~~~~~~~~~~~ + // + // The input buffer receives data obtained from _iStream, before it is + // processed by zlib. In underflow(), zlib reads from this buffer it and + // writes the resulting data(*) to the output buffer. Then we point the + // standard std::streambuf pointers (gptr() and friends) directly towards + // the data inside that output buffer. xsgetn() is even more optimized: it + // first empties the output buffer, then makes zlib write the remaining data + // directly to the destination area. + // + // (*) Compressed or decompressed, depending on the particular + // implementation of zlibProcessData() in each subclass. + char* _inBuf; + const std::size_t _inBufSize; + // _inBufEndPtr points right after the last data retrieved from _iStream and + // stored into _inBuf. When zlib has read all such data, _zstream.next_in is + // equal to _inBufEndPtr (after proper casting). Except in this particular + // situation, only _zstream.next_in <= _inBufEndPtr is guaranteed. + char* _inBufEndPtr; + // Layout of the _outBuf buffer: + // + // |_outBuf |_outBuf + _putbackSize |_outBuf + _outBufSize + // + // The first _putbackSize chars in _outBuf are reserved for the putback area + // (right-aligned at _outBuf + _putbackSize). The actual output buffer thus + // starts at _outBuf + _putbackSize. At any given time for callers of this + // class, the number of characters that can be put back is gptr() - eback(). + // It may be lower than _putbackSize if we haven't read that many characters + // yet. It may also be larger if gptr() > _outBuf + _putbackSize, i.e., + // when the buffer for pending data is non-empty. + // + // At any given time, callers should see: + // + // _outBuf <= eback() <= _outBuf + _putbackSize <= gptr() <= egptr() + // <= _outBuf + _outBufSize + // + // (hoping this won't get out of sync with the code!) + char *_outBuf; + const std::size_t _outBufSize; + // Space reserved for characters to be put back into the stream. Must be + // strictly smaller than _outBufSize (this is checked in the constructor). + // It is guaranteed that this number of chars can be put back, except of + // course if we haven't read that many characters from the input stream yet. + // If characters are buffered in _outBuf[2], then it may be that more + // characters than _putbackSize can be put back (it is essentially a matter + // for std::streambuf of decreasing the “next pointer for the input + // sequence”, i.e., the one returned by gptr()). + // + // [2] In the [_outBuf + _putbackSize, _outBuf + _outBufSize) area. + const std::size_t _putbackSize; + + // Since the constructor optionally allocates memory for the input and + // output buffers, these members allow the destructor to know which buffers + // have to be deallocated, if any. + bool _inBufMustBeFreed = false; + bool _outBufMustBeFreed = false; +}; + + +// Stream buffer class for compressing data. Input data is obtained from an +// std::istream instance; the corresponding compressed data can be read using +// the standard std::streambuf read interface (mainly: sbumpc(), sgetc(), +// snextc(), sgetn(), sputbackc(), sungetc()). Input, uncompressed data is +// “pulled” as needed for the amount of compressed data requested by the +// “client” using the methods I just listed. +class ZlibCompressorIStreambuf: public ZlibAbstractIStreambuf +{ +public: + // Same parameters as for ZlibAbstractIStreambuf, except: + // + // compressionLevel: in the [0,9] range. 0 means no compression at all. + // Levels 1 to 9 yield compressed data, with 1 giving + // the highest compression speed but worst compression + // ratio, and 9 the highest compression ratio but lowest + // compression speed. + // format either ZLIB_COMPRESSION_FORMAT_ZLIB or + // ZLIB_COMPRESSION_FORMAT_GZIP + // memStrategy either ZLIB_FAVOR_MEMORY_OVER_SPEED or + // ZLIB_FAVOR_SPEED_OVER_MEMORY + explicit ZlibCompressorIStreambuf( + std::istream& iStream, + const SGPath& path = SGPath(), + int compressionLevel = Z_DEFAULT_COMPRESSION, + ZLibCompressionFormat format = ZLIB_COMPRESSION_FORMAT_ZLIB, + ZLibMemoryStrategy memStrategy = ZLIB_FAVOR_SPEED_OVER_MEMORY, + char* inBuf = nullptr, + std::size_t inBufSize = 262144, + char* outBuf = nullptr, + std::size_t outBufSize = 262144, + std::size_t putbackSize = 0); + ZlibCompressorIStreambuf(const ZlibCompressorIStreambuf&) = delete; + ZlibCompressorIStreambuf& operator=(const ZlibCompressorIStreambuf&) = delete; + ~ZlibCompressorIStreambuf(); + +protected: + OperationType operationType() const override; + // Initialize the z_stream struct used by zlib + void zStreamInit(int compressionLevel, ZLibCompressionFormat format, + ZLibMemoryStrategy memStrategy); + // Call zlib's deflate() function to compress data. + int zlibProcessData() override; +}; + + +// Stream buffer class for decompressing data. Input data is obtained from an +// std::istream instance; the corresponding decompressed data can be read +// using the standard std::streambuf read interface (mainly: sbumpc(), +// sgetc(), snextc(), sgetn(), sputbackc(), sungetc()). Input, compressed data +// is “pulled” as needed for the amount of uncompressed data requested by the +// “client” using the methods I just listed. +class ZlibDecompressorIStreambuf: public ZlibAbstractIStreambuf +{ +public: + // Same parameters as for ZlibAbstractIStreambuf, except: + // + // format ZLIB_COMPRESSION_FORMAT_ZLIB, + // ZLIB_COMPRESSION_FORMAT_GZIP or + // ZLIB_COMPRESSION_FORMAT_AUTODETECT + explicit ZlibDecompressorIStreambuf( + std::istream& iStream, + const SGPath& path = SGPath(), + ZLibCompressionFormat format = ZLIB_COMPRESSION_FORMAT_ZLIB, + char* inBuf = nullptr, + std::size_t inBufSize = 262144, + char* outBuf = nullptr, + std::size_t outBufSize = 262144, + std::size_t putbackSize = 0); // default optimized for speed + ZlibDecompressorIStreambuf(const ZlibDecompressorIStreambuf&) = delete; + ZlibDecompressorIStreambuf& operator=(const ZlibDecompressorIStreambuf&) + = delete; + ~ZlibDecompressorIStreambuf(); + +protected: + OperationType operationType() const override; + void zStreamInit(ZLibCompressionFormat format); + int zlibProcessData() override; +}; + +// std::istream subclass for compressing data. Input data is obtained from an +// std::istream instance; the corresponding compressed data can be read using +// the standard std::istream interface (read(), readsome(), gcount(), get(), +// getline(), operator>>(), peek(), putback(), ignore(), unget()... plus +// operator overloads such as istream& operator>>(istream&, string&) as +// defined in , and std::getline()). Input, uncompressed data is +// “pulled” as needed for the amount of compressed data requested by the +// “client”. +// +// To get data efficiently from an instance of this class, use its read() +// method (typically in conjunction with gcount(), inside a loop). +class ZlibCompressorIStream: public std::istream +{ +public: + // Same parameters as for ZlibCompressorIStreambuf + explicit ZlibCompressorIStream( + std::istream& iStream, + const SGPath& path = SGPath(), + int compressionLevel = Z_DEFAULT_COMPRESSION, + ZLibCompressionFormat format = ZLIB_COMPRESSION_FORMAT_ZLIB, + ZLibMemoryStrategy memStrategy = ZLIB_FAVOR_SPEED_OVER_MEMORY, + char* inBuf = nullptr, + std::size_t inBufSize = 262144, + char* outBuf = nullptr, + std::size_t outBufSize = 262144, + std::size_t putbackSize = 0); // default optimized for speed + ZlibCompressorIStream(const ZlibCompressorIStream&) = delete; + ZlibCompressorIStream& operator=(const ZlibCompressorIStream&) = delete; + ~ZlibCompressorIStream(); + +private: + ZlibCompressorIStreambuf _streamBuf; +}; + +// std::istream subclass for decompressing data. Input data is obtained from +// an std::istream instance; the corresponding decompressed data can be read +// using the standard std::istream interface (read(), readsome(), gcount(), +// get(), getline(), operator>>(), peek(), putback(), ignore(), unget()... +// plus operator overloads such as istream& operator>>(istream&, string&) as +// defined in , and std::getline()). Input, compressed data is +// “pulled” as needed for the amount of uncompressed data requested by the +// “client”. +// +// To get data efficiently from an instance of this class, use its read() +// method (typically in conjunction with gcount(), inside a loop). +class ZlibDecompressorIStream: public std::istream +{ +public: + // Same parameters as for ZlibDecompressorIStreambuf + explicit ZlibDecompressorIStream( + std::istream& iStream, + const SGPath& path = SGPath(), + ZLibCompressionFormat format = ZLIB_COMPRESSION_FORMAT_ZLIB, + char* inBuf = nullptr, + std::size_t inBufSize = 262144, + char* outBuf = nullptr, + std::size_t outBufSize = 262144, + std::size_t putbackSize = 0); // default optimized for speed + ZlibDecompressorIStream(const ZlibDecompressorIStream&) = delete; + ZlibDecompressorIStream& operator=(const ZlibDecompressorIStream&) = delete; + ~ZlibDecompressorIStream(); + +private: + ZlibDecompressorIStreambuf _streamBuf; +}; + +} // of namespace simgear + +#endif // of SG_ZLIBSTREAM_HXX diff --git a/simgear/io/iostreams/zlibstream_test.cxx b/simgear/io/iostreams/zlibstream_test.cxx new file mode 100644 index 00000000..dd3094f4 --- /dev/null +++ b/simgear/io/iostreams/zlibstream_test.cxx @@ -0,0 +1,562 @@ +// -*- coding: utf-8 -*- +// +// zlibstream_test.cxx --- Automated tests for zlibstream.cxx / zlibstream.hxx +// +// Copyright (C) 2017 Florent Rougon +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +#include // std::basic_ios, std::streamsize... +#include // std::ios_base, std::cerr, etc. +#include +#include +#include +#include // std::numeric_limits +#include // std::bind() +#include +#include // EXIT_SUCCESS +#include // std::size_t +#include // strcmp() + +using std::string; +using std::cout; +using std::cerr; +using traits = std::char_traits; + +#include +#include +#include +#include +#include + +// In many tests below, I use very small buffer sizes. Of course, this is bad +// for performance. The reason I do it this way is simply because it better +// exercises the code we want to *test* here (we are more likely to find bugs +// if the buffer(s) has/have to be refilled one or more times). Similarly, if +// you don't need the putback feature in non-test code, best performance is +// achieved with putback size = 0. +// +// I suggest you read roundTripWithIStreams() below to see how to use the +// classes most efficiently (especially the comments!). + +static std::default_random_engine randomNumbersGenerator; + +// Sample string for tests +static const string lipsum = "\ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque congue ornare\n\ +congue. Mauris mollis est et porttitor condimentum. Vivamus laoreet blandit\n\ +odio eget consectetur. Etiam quis magna eu enim luctus pretium. In et\n\ +tristique nunc, non efficitur metus. Nullam efficitur tristique velit.\n\ +Praesent et luctus nunc. Mauris eros eros, rutrum at molestie quis, egestas et\n\ +lorem. Ut nulla turpis, eleifend sed mauris ac, faucibus molestie nulla.\n\ +Quisque viverra vel turpis nec efficitur. Proin non rutrum velit. Nam sodales\n\ +metus felis, eu pharetra velit posuere ut.\n\ +\n\ +Suspendisse pellentesque tincidunt ligula et pretium. Etiam id justo mauris.\n\ +Aenean porta, sapien in suscipit tristique, metus diam malesuada dui, in porta\n\ +felis mi non felis. Etiam vel aliquam leo, non vehicula magna. Proin justo\n\ +massa, ultrices at porta eu, tempor in ligula. Duis enim ipsum, dictum quis\n\ +ultricies et, tempus eu libero. Morbi vulputate libero ut dolor rutrum, a\n\ +imperdiet nibh egestas. Phasellus finibus massa vel tempus hendrerit. Nulla\n\ +lobortis est non ligula viverra, quis egestas ante hendrerit. Pellentesque\n\ +finibus mollis blandit. In ac sollicitudin mauris, eget dignissim mauris."; + +// Utility function: generate a random string whose length is in the +// [minLen, maxLen] range. +string randomString(string::size_type minLen, string::size_type maxLen) +{ + std::uniform_int_distribution sLenDist(minLen, maxLen); + std::uniform_int_distribution byteDist(0, 255); + auto randomByte = std::bind(byteDist, randomNumbersGenerator); + + string::size_type len = sLenDist(randomNumbersGenerator); + string str; + + while (str.size() < len) { + str += traits::to_char_type(randomByte()); + } + + return str; +} + +// Utility function: perform a compression-decompression cycle on the input +// string using the stream buffer classes: +// +// 1) Compress the input string, write the result to a temporary file using +// std::ostream& operator<<(streambuf* sb), where 'sb' points to a +// ZlibCompressorIStreambuf instance. +// +// 2) Close the temporary file to make sure all the data is flushed. +// +// 3) Create a ZlibDecompressorIStreambuf instance reading from the +// temporary file, then decompress to an std::ostringstream using +// std::ostream& operator<<(streambuf* sb), where 'sb' points to our +// ZlibDecompressorIStreambuf instance. +// +// 4) Compare the result with the input string. +// +// Of course, it is possible to do this without any temporary file, even +// without any std::stringstream or such to hold the intermediate, compressed +// data. See the other tests for actual compressor + decompressor pipelines. +void pipeCompOrDecompIStreambufIntoOStream(const string& input) +{ + // Test “std::ostream << &ZlibCompressorIStreambuf” + std::istringstream input_ss(input); + simgear::ZlibCompressorIStreambuf compSBuf(input_ss, SGPath(), 9); + simgear::Dir tmpDir = simgear::Dir::tempDir("FlightGear"); + tmpDir.setRemoveOnDestroy(); + SGPath path = tmpDir.path() / "testFile.dat"; + sg_ofstream oFile(path, + std::ios::binary | std::ios::trunc | std::ios::out); + + oFile << &compSBuf; + // Make sure the compressed stream is flushed, otherwise when we read the + // file back, decompression is likely to fail with Z_BUF_ERROR. + oFile.close(); + SG_CHECK_EQUAL(compSBuf.sgetc(), EOF); + + // Read back and decompress the data we've written to 'path', testing + // “std::ostream << &ZlibDecompressorIStreambuf” + sg_ifstream iFile(path, std::ios::binary | std::ios::in); + simgear::ZlibDecompressorIStreambuf decompSBuf(iFile, path); + std::ostringstream roundTripResult_ss; + // This is also possible, though maybe not as good for error detection: + // + // decompSBuf >> roundTripResult_ss.rdbuf(); + roundTripResult_ss << &decompSBuf; + SG_CHECK_EQUAL(decompSBuf.sgetc(), EOF); + SG_CHECK_EQUAL(roundTripResult_ss.str(), input); +} + +// Perform a compression-decompression cycle on bunch of strings, using the +// stream buffer classes directly (not the std::istream subclasses). +void test_pipeCompOrDecompIStreambufIntoOStream() +{ + cerr << "Compression-decompression cycles using the stream buffer classes " + "directly\n"; + + pipeCompOrDecompIStreambufIntoOStream(""); // empty string + pipeCompOrDecompIStreambufIntoOStream("a"); + pipeCompOrDecompIStreambufIntoOStream("lorem ipsum"); + + for (int i=0; i < 10; i++) { + pipeCompOrDecompIStreambufIntoOStream(randomString(0, 20)); + } + + for (int i=0; i < 10; i++) { + pipeCompOrDecompIStreambufIntoOStream(randomString(21, 1000)); + } + + assert(std::numeric_limits::max() >= 65535); + for (int i=0; i < 10; i++) { + pipeCompOrDecompIStreambufIntoOStream(randomString(1000, 65535)); + } +} + +void test_StreambufBasicOperations() +{ + cerr << "Testing basic operations on ZlibDecompressorIStreambuf\n"; + + const string text = "0123456789abcdefghijklmnopqrstuvwxyz\nABCDEF\nGHIJK " + "LMNOPQ"; + std::istringstream text_ss(text); + + static constexpr std::size_t compInBufSize = 6; + static constexpr std::size_t compOutBufSize = 4; + static constexpr std::size_t compPutbackSize = 0; + simgear::ZlibCompressorIStreambuf compSBuf( + text_ss, SGPath(), 8, simgear::ZLIB_COMPRESSION_FORMAT_ZLIB, + simgear::ZLIB_FAVOR_SPEED_OVER_MEMORY, nullptr, compInBufSize, nullptr, + compOutBufSize, compPutbackSize); + std::stringstream compressedOutput_ss; + compressedOutput_ss << &compSBuf; + + static constexpr std::size_t decompInBufSize = 5; + static constexpr std::size_t decompOutBufSize = 4; + static constexpr std::size_t decompPutbackSize = 2; + simgear::ZlibDecompressorIStreambuf decompSBuf( + compressedOutput_ss, SGPath(), simgear::ZLIB_COMPRESSION_FORMAT_ZLIB, + nullptr, decompInBufSize, nullptr, decompOutBufSize, decompPutbackSize); + + int ch = decompSBuf.sgetc(); + SG_VERIFY(ch != EOF && traits::to_char_type(ch) == '0'); + ch = decompSBuf.sbumpc(); + SG_VERIFY(ch != EOF && traits::to_char_type(ch) == '0'); + ch = decompSBuf.sbumpc(); + SG_VERIFY(ch != EOF && traits::to_char_type(ch) == '1'); + ch = decompSBuf.snextc(); + SG_VERIFY(ch != EOF && traits::to_char_type(ch) == '3'); + ch = decompSBuf.sbumpc(); + SG_VERIFY(ch != EOF && traits::to_char_type(ch) == '3'); + ch = decompSBuf.sbumpc(); + SG_VERIFY(ch != EOF && traits::to_char_type(ch) == '4'); + ch = decompSBuf.sputbackc('4'); + SG_VERIFY(ch != EOF && traits::to_char_type(ch) == '4'); + ch = decompSBuf.sputbackc('u'); // doesn't match what we read from the stream + SG_VERIFY(ch == EOF); + ch = decompSBuf.sputbackc('3'); // this one does + SG_VERIFY(ch != EOF && traits::to_char_type(ch) == '3'); + + static constexpr std::streamsize bufSize = 10; + char buf[bufSize]; + // Most efficient way (with the underlying xsgetn()) to read several chars + // at once. + std::streamsize n = decompSBuf.sgetn(buf, bufSize); + SG_VERIFY(n == bufSize && string(buf, bufSize) == "3456789abc"); + + ch = decompSBuf.sungetc(); // same as sputbackc(), except no value to check + SG_VERIFY(ch != EOF && traits::to_char_type(ch) == 'c'); + ch = decompSBuf.sungetc(); + SG_VERIFY(ch != EOF && traits::to_char_type(ch) == 'b'); + ch = decompSBuf.sbumpc(); + SG_VERIFY(ch != EOF && traits::to_char_type(ch) == 'b'); + ch = decompSBuf.sputbackc('b'); // this one does + SG_VERIFY(ch != EOF && traits::to_char_type(ch) == 'b'); + + n = decompSBuf.sgetn(buf, bufSize); + SG_VERIFY(n == bufSize && string(buf, bufSize) == "bcdefghijk"); + + ch = decompSBuf.sungetc(); + SG_VERIFY(ch != EOF && traits::to_char_type(ch) == 'k'); + + static char buf2[64]; + n = decompSBuf.sgetn(buf2, sizeof(buf2)); + SG_VERIFY(n == 36 && string(buf2, n) == "klmnopqrstuvwxyz\nABCDEF\nGHIJK " + "LMNOPQ"); + + ch = decompSBuf.sbumpc(); + SG_CHECK_EQUAL(ch, EOF); +} + +// Utility function: take a string as input to compress and return the +// compressed output as a string. +string compress(const string& dataToCompress, + simgear::ZLibCompressionFormat compressionFormat, + int compressionLevel, + simgear::ZLibMemoryStrategy memStrategy, + std::size_t putbackSize, + SGPath path = SGPath()) +{ + // Static storage is only okay for very small sizes like these, and because + // we won't call the function from several threads at the same time. Plain + // char arrays would be fine here, but I'll use std::array to show it can be + // used here too. + static std::array inBuf; + static std::array outBuf; + + std::istringstream iss(dataToCompress); + simgear::ZlibCompressorIStream compressor( + iss, path, compressionLevel, compressionFormat, memStrategy, + &inBuf.front(), inBuf.size(), &outBuf.front(), outBuf.size(), + putbackSize); + compressor.exceptions(std::ios_base::badbit); // throw if badbit is set + std::ostringstream compressedData_ss; + compressor >> compressedData_ss.rdbuf(); + + return compressedData_ss.str(); +} + +void test_formattedInputFromDecompressor() +{ + cerr << "Testing ZlibDecompressorIStream >> std::string\n"; + + static char inBuf[6]; + static char outBuf[15]; + string compressed = compress( + lipsum, simgear::ZLIB_COMPRESSION_FORMAT_ZLIB, Z_BEST_COMPRESSION, + simgear::ZLIB_FAVOR_MEMORY_OVER_SPEED, /* putback size */ 0); + std::istringstream compressed_ss(compressed); + + simgear::ZlibDecompressorIStream decompressor( + compressed_ss, SGPath(), simgear::ZLIB_COMPRESSION_FORMAT_ZLIB, + inBuf, sizeof(inBuf), outBuf, sizeof(outBuf), /* putback size */ 1); + decompressor.exceptions(std::ios_base::badbit); // throw if badbit is set + + int count = 0; + string word; + + while (decompressor >> word) { + count++; + } + + SG_CHECK_EQUAL(count, 175); // Number of words in 'lipsum' + // If set, badbit would have caused an exception to be raised + SG_VERIFY(!decompressor.bad()); + + if (!decompressor.eof()) { + assert(decompressor.fail()); + cerr << "Decompressor: stream extraction (operator>>) failed before " + "reaching EOF.\n"; + SG_TEST_FAIL("Did not expect operator>> to fail with an std::string " + "argument."); + } +} + +void test_ZlibDecompressorIStream_readPutbackEtc() +{ + cerr << "Testing many operations on ZlibDecompressorIStream (read(), " + "putback(), etc.\n"; + + static char compInBuf[4]; + static char compOutBuf[6]; + static char decompInBuf[8]; + static char decompOutBuf[5]; + const string text = "0123456789abcdefghijklmnopqrstuvwxyz\nABCDEF\nGHIJK " + "LMNOPQ"; + std::istringstream text_ss(text); + + simgear::ZlibCompressorIStream compressor( + text_ss, SGPath(), Z_BEST_COMPRESSION, + simgear::ZLIB_COMPRESSION_FORMAT_ZLIB, + simgear::ZLIB_FAVOR_MEMORY_OVER_SPEED, + compInBuf, sizeof(compInBuf), compOutBuf, sizeof(compOutBuf), + /* putback size */ 0); + compressor.exceptions(std::ios_base::badbit); // throw if badbit is set + + // Use the compressor (subclass of std::istream) as input to the decompressor + simgear::ZlibDecompressorIStream decompressor( + compressor, SGPath(), simgear::ZLIB_COMPRESSION_FORMAT_ZLIB, + decompInBuf, sizeof(decompInBuf), decompOutBuf, sizeof(decompOutBuf), + /* putback size */ 3); + decompressor.exceptions(std::ios_base::badbit); + + { + static std::array buf; + decompressor.read(&buf.front(), 10); + SG_VERIFY(decompressor.good() && decompressor.gcount() == 10); + SG_CHECK_EQUAL(string(&buf.front(), 10), "0123456789"); + + SG_VERIFY(decompressor.putback('9')); + SG_VERIFY(decompressor.putback('8')); + SG_VERIFY(decompressor.putback('7')); + + SG_VERIFY(decompressor.get(buf[10])); + SG_VERIFY(decompressor.read(&buf[11], 6)); + SG_CHECK_EQUAL(string(&buf.front(), 17), "0123456789789abcd"); + } + + { + bool gotException = false; + try { + // 'Z' is not the last character read from the stream + decompressor.putback('Z'); + } catch (std::ios_base::failure) { + gotException = true; + } + SG_VERIFY(gotException && decompressor.bad()); + } + + { + int c; + + decompressor.clear(); + // Check we can resume normally now that we've cleared the error state flags + c = decompressor.get(); + SG_VERIFY(c != traits::eof() && traits::to_char_type(c) == 'e'); + + // unget() and get() in sequence + SG_VERIFY(decompressor.unget()); + c = decompressor.get(); + SG_VERIFY(c != traits::eof() && traits::to_char_type(c)== 'e'); + + // peek() looks at, but doesn't consume + c = decompressor.peek(); + SG_VERIFY(c != traits::eof() && traits::to_char_type(c) == 'f'); + c = decompressor.get(); + SG_VERIFY(c != traits::eof() && traits::to_char_type(c)== 'f'); + } + + { + static std::array buf; // 20 chars + terminating NUL char + + // std::istream::getline() will be stopped by \n after reading 20 chars + SG_VERIFY(decompressor.getline(&buf.front(), 25)); + SG_VERIFY(decompressor.gcount() == 21 && + !strcmp(&buf.front(), "ghijklmnopqrstuvwxyz")); + + string str; + // std::getline() will be stopped by \n after 7 chars have been extracted, + // namely 6 chars plus the \n which is extracted and discarded. + SG_VERIFY(std::getline(decompressor, str)); + SG_CHECK_EQUAL(str, "ABCDEF"); + + SG_VERIFY(decompressor.ignore(2)); + SG_VERIFY(decompressor >> str); + SG_CHECK_EQUAL(str, "IJK"); + + static char buf2[5]; + // Read up to sizeof(buf) chars, without waiting if not that many are + // immediately avaiable. + int nbCharsRead = decompressor.readsome(buf2, sizeof(buf2)); + + string rest(buf2, nbCharsRead); + do { + decompressor.read(buf2, sizeof(buf2)); + rest += string(buf2, decompressor.gcount()); + } while (decompressor); + + SG_CHECK_EQUAL(rest, " LMNOPQ"); + } + + // If set, badbit would have caused an exception to be raised, anyway + SG_VERIFY(decompressor.eof() && !decompressor.bad()); +} + +// Utility function: parametrized round-trip test with a compressor + +// decompressor pipeline. +// +// +// Note: this is nice conceptually, allows to keep memory use constant even in +// case an arbitrary amount of data is passed through, and exercises the +// stream buffer classes well, however this technique is more than twice +// as slow on my computer than using an intermediate buffer like this: +// +// std::stringstream compressedData_ss; +// compressor >> compressedData_ss.rdbuf(); +// +// simgear::ZlibDecompressorIStream decompressor(compressedData_ss, +// SGPath(), ... +void roundTripWithIStreams( + simgear::ZLibCompressionFormat compressionFormat, + int compressionLevel, + simgear::ZLibMemoryStrategy memStrategy, + std::size_t compInBufSize, + std::size_t compOutBufSize, + std::size_t decompInBufSize, + std::size_t decompOutBufSize, + std::size_t compPutbackSize, + std::size_t decompPutbackSize, + bool useAutoFormatForDecompression = false) +{ + const simgear::ZLibCompressionFormat decompFormat = + (useAutoFormatForDecompression) ? + simgear::ZLIB_COMPRESSION_FORMAT_AUTODETECT : compressionFormat; + + std::istringstream lipsum_ss(lipsum); + // This tests the optional dynamic buffer allocation in ZlibAbstractIStreambuf + simgear::ZlibCompressorIStream compressor( + lipsum_ss, SGPath(), compressionLevel, compressionFormat, memStrategy, + nullptr, compInBufSize, nullptr, compOutBufSize, compPutbackSize); + compressor.exceptions(std::ios_base::badbit); // throw if badbit is set + + // Use the compressor as input to the decompressor (pipeline). The + // decompressor uses read() with chunks that are as large as possible given + // the available space in its input buffer. These read() calls are served by + // ZlibCompressorIStreambuf::xsgetn(), which is efficient. + simgear::ZlibDecompressorIStream decompressor( + compressor, SGPath(), decompFormat, + nullptr, decompInBufSize, nullptr, decompOutBufSize, decompPutbackSize); + decompressor.exceptions(std::ios_base::badbit); + std::ostringstream roundTripResult; + // This, on the other hand, appears not to use xsgetn() (tested with the GNU + // libstdc++ from g++ 6.3.0, 20170124). This causes useless copying of the + // data to the decompressor output buffer, instead of writing it directly to + // roundTripResult's internal buffer. This is simple and nice in appearance, + // but if you are after performance, better use decompressor.read() + // (typically in conjunction with gcount(), inside a loop). + decompressor >> roundTripResult.rdbuf(); + SG_CHECK_EQUAL(roundTripResult.str(), lipsum); +} + +// Round-trip conversion with simgear::ZlibCompressorIStream and +// simgear::ZlibDecompressorIStream, using various parameter combinations. +void test_RoundTripMultiWithIStreams() +{ + cerr << + "Compression-decompression cycles using the std::istream subclasses (many\n" + "combinations of buffer sizes and compression parameters tested)\n"; + + { // More variations on these later + const std::size_t compPutbackSize = 1; + const std::size_t decompPutbackSize = 1; + + for (auto format: {simgear::ZLIB_COMPRESSION_FORMAT_ZLIB, + simgear::ZLIB_COMPRESSION_FORMAT_GZIP}) { + for (int compressionLevel: {1, 4, 7, 9}) { + for (auto memStrategy: {simgear::ZLIB_FAVOR_MEMORY_OVER_SPEED, + simgear::ZLIB_FAVOR_SPEED_OVER_MEMORY}) { + for (std::size_t compInBufSize: {3, 4}) { + for (std::size_t compOutBufSize: {3, 5}) { + for (std::size_t decompInBufSize: {3, 4}) { + for (std::size_t decompOutBufSize: {3, 4,}) { + roundTripWithIStreams( + format, compressionLevel, memStrategy, compInBufSize, + compOutBufSize, decompInBufSize, decompOutBufSize, + compPutbackSize, decompPutbackSize); + } + } + } + } + } + } + } + } + + { + const auto format = simgear::ZLIB_COMPRESSION_FORMAT_ZLIB; + const int compressionLevel = Z_DEFAULT_COMPRESSION; + const auto memStrategy = simgear::ZLIB_FAVOR_SPEED_OVER_MEMORY; + + for (std::size_t compInBufSize: {3, 4, 31, 256, 19475}) { + for (std::size_t compOutBufSize: {3, 5, 9, 74, 4568}) { + for (std::size_t decompInBufSize: {3, 4, 256, 24568}) { + for (std::size_t decompOutBufSize: {3, 5, 42, 4568}) { + for (std::size_t compPutbackSize: {0, 1, 2}) { + for (std::size_t decompPutbackSize: {0, 1, 2}) { + roundTripWithIStreams( + format, compressionLevel, memStrategy, compInBufSize, + compOutBufSize, decompInBufSize, decompOutBufSize, + compPutbackSize, decompPutbackSize); + } + } + } + } + } + } + } + + { + const std::size_t compInBufSize = 5; + const std::size_t compOutBufSize = 107; + const std::size_t decompInBufSize = 65536; + const std::size_t decompOutBufSize = 84; + int i = 0; + + for (std::size_t compPutbackSize: {25, 40, 105}) { + for (std::size_t decompPutbackSize: {30, 60, 81}) { + const simgear::ZLibCompressionFormat compFormat = (i++ % 2) ? + simgear::ZLIB_COMPRESSION_FORMAT_ZLIB : + simgear::ZLIB_COMPRESSION_FORMAT_GZIP; + + roundTripWithIStreams( + compFormat, Z_BEST_COMPRESSION, simgear::ZLIB_FAVOR_MEMORY_OVER_SPEED, + compInBufSize, compOutBufSize, decompInBufSize, decompOutBufSize, + compPutbackSize, decompPutbackSize, + /* automatic format detection for decompression */ true); + } + } + } +} + +int main(int argc, char** argv) +{ + test_pipeCompOrDecompIStreambufIntoOStream(); + test_StreambufBasicOperations(); + test_RoundTripMultiWithIStreams(); + test_formattedInputFromDecompressor(); + test_ZlibDecompressorIStream_readPutbackEtc(); + + return EXIT_SUCCESS; +}