FGFS-4.1/simgear
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add stream buffers and std::istream subclasses to deal with zlib compression

Browse Source 
Add:
 - two stream buffer classes (ZlibCompressorIStreambuf and
   ZlibDecompressorIStreambuf), both based on the same abstract class:
   ZlibAbstractIStreambuf;
 - two std::istream subclasses (ZlibCompressorIStream and
   ZlibDecompressorIStream), each creating and using the corresponding
   stream buffer class from the previous item.

All these allow one to work with RFC 1950 and RFC 1952 compression
formats, respectively known as the zlib and gzip formats.

These classes are *input* streaming classes, which means they can
efficiently handle arbitrary amounts of data without using any disk
space nor increasing amounts of memory, and allow "client code" to pull
exactly as much data as it wants at any given time, resuming later when
it is ready to handle the next chunk.

See comments in simgear/io/iostreams/zlibstream.hxx for more details.
This commit is contained in:
Florent Rougon
2017-01-26 09:28:28 +01:00
parent 79f869a7f3
commit c9611fc45b
4 changed files with 1664 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
simgear/io/iostreams/CMakeLists.txt
View File

@@ -4,12 +4,14 @@ set(HEADERS
sgstream.hxx
gzfstream.hxx
gzcontainerfile.hxx
zlibstream.hxx
)
set(SOURCES
sgstream.cxx
gzfstream.cxx
gzcontainerfile.cxx
zlibstream.cxx
)
simgear_component(IOStreams io/iostreams "${SOURCES}" "${HEADERS}")
@@ -20,4 +22,8 @@ if(ENABLE_TESTS)
target_link_libraries(test_streams ${TEST_LIBS})
add_test(streams ${EXECUTABLE_OUTPUT_PATH}/test_streams)
add_executable(test_zlibstream zlibstream_test.cxx)
target_link_libraries(test_zlibstream ${TEST_LIBS})
add_test(zlibstream ${EXECUTABLE_OUTPUT_PATH}/test_zlibstream)
endif(ENABLE_TESTS)

695
simgear/io/iostreams/zlibstream.cxx Normal file
View File

@@ -0,0 +1,695 @@
// -*- coding: utf-8 -*-
//
// zlibstream.cxx --- IOStreams classes for working with RFC 1950 and RFC 1952
// compression formats (respectively known as the zlib and
// gzip formats)
//
// Copyright (C) 2017 Florent Rougon
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with this program; if not, write to the Free Software Foundation, Inc.,
// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#include <string>
#include <ios> // std::streamsize
#include <istream>
#include <algorithm>
#include <stdexcept>
#include <unordered_map>
#include <limits> // std::numeric_limits
#include <cstddef> // std::size_t, std::ptrdiff_t
#include <cassert>
#include <zlib.h>
#include <simgear/io/iostreams/zlibstream.hxx>
#include <simgear/misc/strutils.hxx>
#include <simgear/misc/sg_path.hxx>
#include <simgear/structure/exception.hxx>
#include <simgear/debug/logstream.hxx>
using std::string;
using traits = std::char_traits<char>;
// Private utility function
static string zlibErrorMessage(const z_stream& zstream, int errorCode)
{
string res;
std::unordered_map<int, string> errorCodeToMessageMap = {
{Z_OK, "zlib: no error (code Z_OK)"},
{Z_STREAM_END, "zlib stream end"},
{Z_NEED_DICT, "zlib: Z_NEED_DICT"},
{Z_STREAM_ERROR, "zlib stream error"},
{Z_DATA_ERROR, "zlib data error"},
{Z_MEM_ERROR, "zlib memory error"},
{Z_BUF_ERROR, "zlib buffer error"},
{Z_VERSION_ERROR, "zlib version error"}
};
if (errorCode == Z_ERRNO) {
res = simgear::strutils::error_string(errno);
} else if (zstream.msg != nullptr) {
// Caution: this only works if the zstream structure hasn't been
// deallocated!
res = "zlib: " + string(zstream.msg);
} else {
try {
res = errorCodeToMessageMap.at(errorCode);
} catch (const std::out_of_range&) {
res = string("unknown zlib error (code " + std::to_string(errorCode) +
")");
}
}
return res;
}
// Return the largest value that can be represented with zlib's uInt type,
// which is the type of z_stream.avail_in and z_stream.avail_out, hence the
// function name.
static std::size_t zlibMaxChunkSize()
{
uLong flags = ::zlibCompileFlags();
std::size_t res;
switch (flags & 0x3) {
case 0x3:
SG_LOG(SG_IO, SG_WARN,
"Unknown size for zlib's uInt type (code 3). Will assume 64 bits, "
"but the actual value is probably higher.");
// No 'break' here, this is intentional.
case 0x2:
res = 0xFFFFFFFFFFFFFFFF; // 2^64 - 1
break;
case 0x1:
res = 0xFFFFFFFF; // 2^32 - 1
break;
case 0x0:
res = 0xFFFF; // 2^16 - 1
break;
default:
throw std::logic_error("It should be impossible to get here.");
}
return res;
}
static const std::size_t zlibMaxChunk = ::zlibMaxChunkSize();
namespace simgear
{
// ***************************************************************************
// * ZlibAbstractIStreambuf class *
// ***************************************************************************
// Common initialization. Subclasses must complete the z_stream struct
// initialization with a call to deflateInit2() or inflateInit2(), typically.
ZlibAbstractIStreambuf::ZlibAbstractIStreambuf(std::istream& iStream,
const SGPath& path,
char* inBuf,
std::size_t inBufSize,
char *outBuf,
std::size_t outBufSize,
std::size_t putbackSize)
: _iStream(iStream),
_path(path),
_inBuf(inBuf),
_inBufSize(inBufSize),
_outBuf(outBuf),
_outBufSize(outBufSize),
_putbackSize(putbackSize)
{
assert(_inBufSize > 0);
assert(_putbackSize >= 0); // guaranteed unless the type is changed...
assert(_putbackSize < _outBufSize);
if (_inBuf == nullptr) {
_inBuf = new char[_inBufSize];
_inBufMustBeFreed = true;
}
if (_outBuf == nullptr) {
_outBuf = new char[_outBufSize];
_outBufMustBeFreed = true;
}
_inBufEndPtr = _inBuf;
// The input buffer is empty.
_zstream.next_in = reinterpret_cast<unsigned char *>(_inBuf);
_zstream.avail_in = 0;
// ZLib's documentation says its init functions such as inflateInit2() might
// consume stream input, therefore let's fill the input buffer now. This
// way, constructors of derived classes just have to call the appropriate
// ZLib init function: the data will already be in place.
getInputData();
// Force underflow() of the stream buffer on the first read. We could use
// some other value, but I avoid nullptr in order to be sure we can always
// reliably compare the three pointers with < and >, as well as compute the
// difference between any two of them.
setg(_outBuf, _outBuf, _outBuf);
}
ZlibAbstractIStreambuf::~ZlibAbstractIStreambuf()
{
if (_inBufMustBeFreed) {
delete[] _inBuf;
}
if (_outBufMustBeFreed) {
delete[] _outBuf;
}
}
// Fill or refill the output buffer, and update the three pointers
// corresponding to eback(), gptr() and egptr().
int ZlibAbstractIStreambuf::underflow()
{
if (_allFinished) {
return traits::eof();
}
// According to the C++11 standard: “The public members of basic_streambuf
// call this virtual function only if gptr() is null or gptr() >= egptr()”.
// Still, it seems some people do the following or similar (maybe in case
// underflow() is called “incorrectly”?). See for instance N. Josuttis, The
// C++ Standard Library (1st edition), p. 584. One sure thing is that it
// can't hurt (except performance, marginally), so let's do it.
if (gptr() < egptr()) {
return traits::to_int_type(*gptr());
}
assert(gptr() == egptr());
assert(egptr() - eback() >= 0);
std::size_t nbPutbackChars = std::min(
static_cast<std::size_t>(egptr() - eback()), // OK because egptr() >= eback()
_putbackSize);
std::copy(egptr() - nbPutbackChars, egptr(),
_outBuf + _putbackSize - nbPutbackChars);
setg(_outBuf + _putbackSize - nbPutbackChars, // start of putback area
_outBuf + _putbackSize, // start of obtained data
fillOutputBuffer()); // one-past-end of obtained data
return (gptr() == egptr()) ? traits::eof() : traits::to_int_type(*gptr());
}
// Simple utility method for fillOutputBuffer(), used to improve readability.
// Return the remaining space available in the output buffer, where zlib can
// write.
std::size_t
ZlibAbstractIStreambuf::fOB_remainingSpace(unsigned char* nextOutPtr) const
{
std::ptrdiff_t remainingSpaceInOutBuf = _outBuf + _outBufSize -
reinterpret_cast<char*>(nextOutPtr);
assert(remainingSpaceInOutBuf >= 0);
return static_cast<std::size_t>(remainingSpaceInOutBuf);
}
// Simple method for dealing with the Z_BUF_ERROR code that may be returned
// by zlib's deflate() and inflate() methods.
[[ noreturn ]] void ZlibAbstractIStreambuf::handleZ_BUF_ERROR() const
{
switch (operationType()) {
case OPERATION_TYPE_DECOMPRESSION:
{
string message = (_path.isNull()) ?
"Got Z_BUF_ERROR from zlib while decompressing a stream. The stream "
"was probably incomplete."
:
"Got Z_BUF_ERROR from zlib during decompression. The compressed stream "
"was probably incomplete.";
// When _path.isNull(), sg_location(_path) is equivalent to sg_location()
throw sg_io_exception(message, sg_location(_path));
}
case OPERATION_TYPE_COMPRESSION:
throw std::logic_error(
"Called ZlibAbstractIStreambuf::handleZ_BUF_ERROR() with "
"operationType() == OPERATION_TYPE_DECOMPRESSION");
default:
throw std::logic_error(
"Unexpected operationType() in "
"ZlibAbstractIStreambuf::handleZ_BUF_ERROR(): " +
std::to_string(operationType()));
}
}
// Fill or refill the output buffer. Return a pointer to the first unused char
// in _outBuf (i.e., right after the data written by this method, if any;
// otherwise: _outBuf + _putbackSize).
char* ZlibAbstractIStreambuf::fillOutputBuffer()
{
bool allInputRead = false;
std::size_t remainingSpaceInOutBuf;
int retCode;
// We have to do these unpleasant casts, because zlib uses pointers to
// unsigned char for its input and output buffers, while the IOStreams
// library in the C++ standard uses plain char pointers...
_zstream.next_out = reinterpret_cast<unsigned char*>(_outBuf + _putbackSize);
remainingSpaceInOutBuf = fOB_remainingSpace(_zstream.next_out);
while (remainingSpaceInOutBuf > 0) {
// This does fit in a zlib uInt: that's the whole point of zlibMaxChunk.
_zstream.avail_out = static_cast<uInt>(
std::min(remainingSpaceInOutBuf, zlibMaxChunk));
if (_zstream.avail_in == 0 && !allInputRead) {
// Get data from _iStream, store it in _inBuf
allInputRead = getInputData();
}
// Make zlib process some data (compress or decompress it). This updates
// _zstream.{avail,next}_{in,out} (4 fields of the z_stream struct).
retCode = zlibProcessData();
if (retCode == Z_BUF_ERROR) {
handleZ_BUF_ERROR(); // doesn't return
} else if (retCode == Z_STREAM_END) {
assert(_zstream.avail_in == 0); // all of _inBuf must have been used
_allFinished = true;
break;
} else if (retCode < 0) { // negative codes are errors
throw sg_io_exception(::zlibErrorMessage(_zstream, retCode),
sg_location(_path));
}
remainingSpaceInOutBuf = fOB_remainingSpace(_zstream.next_out);
}
return reinterpret_cast<char*>(_zstream.next_out);
}
// This method provides input data to zlib:
// - if data is already available in _inBuf, it updates _zstream.avail_in
// accordingly (using the largest possible amount given _zstream.avail_in's
// type, i.e., uInt);
// - otherwise, it reads() as much data as possible into _inBuf from
// _iStream and updates _zstream.avail_in to tell zlib about this new
// available data (again: largest possible amount given the uInt type).
//
// This method must be called only when _zstream.avail_in == 0, which means
// zlib has read everything we fed it (and does *not* mean, by the way, that
// the input buffer starting at _inBuf is empty; this is because the buffer
// size might exceed what can be represented by an uInt).
//
// On return:
// - either EOF has been reached for _iStream, and the return value is true;
// - or _zstream.avail_in > 0, and the return value is false.
//
// Note: don't read more in the previous paragraph concerning the state on
// return. In the first case, there is *no guarantee* about
// _zstream.avail_in's value; in the second case, there is *no guarantee*
// about whether EOF has been reached for _iStream.
bool ZlibAbstractIStreambuf::getInputData()
{
bool allInputRead = false;
assert(_zstream.avail_in == 0);
std::ptrdiff_t alreadyAvailable =
_inBufEndPtr - reinterpret_cast<char*>(_zstream.next_in);
assert(alreadyAvailable >= 0);
// Data already available?
if (alreadyAvailable > 0) {
_zstream.avail_in = static_cast<uInt>(
std::min(static_cast<std::size_t>(alreadyAvailable),
zlibMaxChunk));
return allInputRead;
}
if (_inBufEndPtr == _inBuf + _inBufSize) { // buffer full, rewind
_inBufEndPtr = _inBuf;
_zstream.next_in = reinterpret_cast<unsigned char*>(_inBuf);
}
// Fill the input buffer (as much as possible)
while (_inBufEndPtr < _inBuf + _inBufSize && !_iStream.eof()) {
std::streamsize nbCharsToRead = std::min(
_inBuf + _inBufSize - _inBufEndPtr,
std::numeric_limits<std::streamsize>::max()); // max we can pass to read()
_iStream.read(_inBufEndPtr, nbCharsToRead);
if (_iStream.bad()) {
string errMsg = simgear::strutils::error_string(errno);
string msgStart = (_path.isNull()) ?
"Error while reading from a stream" : "Read error";
throw sg_io_exception(msgStart + ": " + errMsg, sg_location(_path));
}
// Could be zero if at EOF
std::streamsize nbCharsRead = _iStream.gcount();
// std::streamsize is a signed integral type!
assert(0 <= nbCharsRead);
_inBufEndPtr += nbCharsRead;
}
std::ptrdiff_t availableChars =
_inBufEndPtr - reinterpret_cast<char*>(_zstream.next_in);
assert(availableChars >= 0);
_zstream.avail_in = static_cast<uInt>(
std::min(static_cast<std::size_t>(availableChars),
zlibMaxChunk));
if (_iStream.eof()) {
allInputRead = true;
} else {
// Trying to rewind a fully read std::istringstream with seekg() can lead
// to a weird state, where the stream doesn't return any character but
// doesn't report EOF either. Make sure we are not in this situation.
assert(_zstream.avail_in > 0);
}
return allInputRead;
}
// Implementing this method is optional, but should provide better
// performance. It makes zlib write the data directly in the buffer starting
// at 'dest'. Without it, the data would go through an intermediate buffer
// (_outBuf) before being copied to its (hopefully) final destination.
std::streamsize ZlibAbstractIStreambuf::xsgetn(char* dest, std::streamsize n)
{
std::size_t remaining = static_cast<std::size_t>(n);
std::size_t chunkSize;
std::ptrdiff_t avail;
const char* origGptr = gptr();
char* writePtr = dest; // we'll need dest later -> work with a copy
// First, let's take data present in our internal buffer (_outBuf)
while (remaining > 0) {
avail = egptr() - gptr(); // number of available chars in _outBuf
if (avail == 0) { // our internal buffer is empty
break;
}
chunkSize = std::min(remaining, static_cast<std::size_t>(avail));
std::copy(gptr(), gptr() + chunkSize, writePtr);
gbump(chunkSize);
writePtr += chunkSize;
remaining -= chunkSize;
}
if (remaining == 0) {
// Everything we needed was already in _outBuf. The putback area is set up
// as it should between eback() and the current gptr(), so we are fine to
// return.
return n;
}
// Now, let's make it so that the remaining data we need is directly written
// to the destination area, without going through _outBuf.
_zstream.next_out = reinterpret_cast<unsigned char*>(writePtr);
bool allInputRead = false;
int retCode;
while (remaining > 0) {
chunkSize = std::min(remaining, zlibMaxChunk);
// It does fit in a zlib uInt: that's the whole point of zlibMaxChunk.
_zstream.avail_out = static_cast<uInt>(chunkSize);
if (_zstream.avail_in == 0 && !allInputRead) {
allInputRead = getInputData();
}
// Make zlib process some data (compress or decompress). This updates
// _zstream.{avail,next}_{in,out} (4 fields of the z_stream struct).
retCode = zlibProcessData();
// chunkSize - _zstream.avail_out is the nb of chars written by zlib
remaining -= chunkSize - _zstream.avail_out;
if (retCode == Z_BUF_ERROR) {
handleZ_BUF_ERROR(); // doesn't return
} else if (retCode == Z_STREAM_END) {
assert(_zstream.avail_in == 0); // all of _inBuf must have been used
_allFinished = true;
break;
} else if (retCode < 0) { // negative codes are errors
throw sg_io_exception(::zlibErrorMessage(_zstream, retCode),
sg_location(_path));
}
}
// Finally, prepare the putback area.
//
// We could use reinterpret_cast<char*>(_zstream.next_out) everywhere below,
// except it is hardly readable. This points after the latest data we wrote.
writePtr = reinterpret_cast<char*>(_zstream.next_out);
// There are two buffers containing characters we potentially have to copy
// to the putback area: the one starting at _outBuf and the one starting at
// dest. In the following diagram, ***** represents those from _outBuf,
// which are right-aligned at origGptr[1], and the series of # represents
// those in the [dest, writePtr) address range:
//
// |_outBuf |eback() *****|origGptr |_outBuf + _outBufSize
//
// |dest #################|writePtr
//
// Together, these two memory blocks logically form a contiguous stream of
// chars we gave to the “client”: *****#################. All we have to do
// now is copy the appropriate amount of chars from this logical stream to
// the putback area, according to _putbackSize. These chars are the last
// ones in said stream (i.e., right portion of *****#################), but
// we have to copy them in order of increasing address to avoid possible
// overlapping problems in _outBuf. This is because some of the chars to
// copy may be located before _outBuf + _putbackSize (i.e., already be in
// the putback area).
//
// [1] This means that the last char represented by a star is at address
// origGptr-1.
assert(writePtr - dest >= 0);
std::size_t inDestBuffer = static_cast<std::size_t>(writePtr - dest);
assert(origGptr - eback() >= 0);
std::size_t nbPutbackChars = std::min(
static_cast<std::size_t>(origGptr - eback()) + inDestBuffer,
_putbackSize);
std::size_t nbPutbackCharsToGo = nbPutbackChars;
// chunkSize has an unsigned type; precomputing it before the following test
// wouldn't work.
// Are there chars in _outBuf that need to be copied to the putback area?
if (nbPutbackChars > inDestBuffer) {
chunkSize = nbPutbackChars - inDestBuffer; // yes, this number
std::copy(origGptr - chunkSize, origGptr,
_outBuf + _putbackSize - nbPutbackChars);
nbPutbackCharsToGo -= chunkSize;
}
std::copy(writePtr - nbPutbackCharsToGo, writePtr,
_outBuf + _putbackSize - nbPutbackCharsToGo);
setg(_outBuf + _putbackSize - nbPutbackChars, // start of putback area
_outBuf + _putbackSize, // the buffer for pending,
_outBuf + _putbackSize); // available data is empty
std::streamsize rem = static_cast<std::streamsize>(remaining);
// This is guaranteed because n is of type std::streamsize and the whole
// algorithm ensures that 0 <= remaining <= n.
assert(rem >= 0);
assert(static_cast<std::size_t>(rem) == remaining);
assert(n - rem >= 0);
// Total number of chars copied.
return n - rem;
}
// ***************************************************************************
// * ZlibCompressorIStreambuf class *
// ***************************************************************************
ZlibCompressorIStreambuf::ZlibCompressorIStreambuf(
std::istream& iStream,
const SGPath& path,
int compressionLevel,
ZLibCompressionFormat format,
ZLibMemoryStrategy memStrategy,
char* inBuf,
std::size_t inBufSize,
char *outBuf,
std::size_t outBufSize,
std::size_t putbackSize)
: ZlibAbstractIStreambuf(iStream, path, inBuf, inBufSize, outBuf, outBufSize,
putbackSize)
{
zStreamInit(compressionLevel, format, memStrategy);
}
ZlibCompressorIStreambuf::~ZlibCompressorIStreambuf()
{
int retCode = deflateEnd(&_zstream); // deallocate the z_stream struct
if (retCode != Z_OK) {
// In C++11, we can't throw exceptions from a destructor.
SG_LOG(SG_IO, SG_ALERT, "ZlibCompressorIStreambuf: " <<
::zlibErrorMessage(_zstream, retCode));
}
}
ZlibAbstractIStreambuf::OperationType
ZlibCompressorIStreambuf::operationType() const
{
return OPERATION_TYPE_COMPRESSION;
}
void ZlibCompressorIStreambuf::zStreamInit(int compressionLevel,
ZLibCompressionFormat format,
ZLibMemoryStrategy memStrategy)
{
// Intentionally not listing ZLIB_COMPRESSION_FORMAT_AUTODETECT here
std::unordered_map<ZLibCompressionFormat, int> compressionFormatMap = {
{ZLIB_COMPRESSION_FORMAT_ZLIB, 15},
{ZLIB_COMPRESSION_FORMAT_GZIP, 31}
};
std::unordered_map<ZLibMemoryStrategy, int> memoryStrategyMap = {
{ZLIB_FAVOR_MEMORY_OVER_SPEED, 8},
{ZLIB_FAVOR_SPEED_OVER_MEMORY, 9}
};
// ZLIB_COMPRESSION_FORMAT_AUTODETECT is only for decompression!
assert(format != ZLIB_COMPRESSION_FORMAT_AUTODETECT);
int windowBits = compressionFormatMap.at(format);
int memLevel = memoryStrategyMap.at(memStrategy);
_zstream.zalloc = Z_NULL; // No custom memory allocation routines
_zstream.zfree = Z_NULL; // Ditto. Therefore, the 'opaque' field won't
_zstream.opaque = Z_NULL; // be used, actually.
int retCode = deflateInit2(&_zstream, compressionLevel, Z_DEFLATED,
windowBits, memLevel, Z_DEFAULT_STRATEGY);
if (retCode != Z_OK) {
throw sg_io_exception(::zlibErrorMessage(_zstream, retCode),
sg_location(_path));
}
}
int ZlibCompressorIStreambuf::zlibProcessData()
{
// Compress as much data as possible given _zstream.avail_in and
// _zstream.avail_out. The input data starts at _zstream.next_in, the output
// at _zstream.next_out, and these four fields are all updated by this call
// to reflect exactly the amount of data zlib has consumed and produced.
return deflate(&_zstream, _zstream.avail_in ? Z_NO_FLUSH : Z_FINISH);
}
// ***************************************************************************
// * ZlibDecompressorIStreambuf class *
// ***************************************************************************
ZlibDecompressorIStreambuf::ZlibDecompressorIStreambuf(
std::istream& iStream,
const SGPath& path,
ZLibCompressionFormat format,
char* inBuf,
std::size_t inBufSize,
char *outBuf,
std::size_t outBufSize,
std::size_t putbackSize)
: ZlibAbstractIStreambuf(iStream, path, inBuf, inBufSize, outBuf, outBufSize,
putbackSize)
{
zStreamInit(format);
}
ZlibDecompressorIStreambuf::~ZlibDecompressorIStreambuf()
{
int retCode = inflateEnd(&_zstream); // deallocate the z_stream struct
if (retCode != Z_OK) {
// In C++11, we can't throw exceptions from a destructor.
SG_LOG(SG_IO, SG_ALERT, "ZlibDecompressorIStreambuf: " <<
::zlibErrorMessage(_zstream, retCode));
}
}
ZlibAbstractIStreambuf::OperationType
ZlibDecompressorIStreambuf::operationType() const
{
return OPERATION_TYPE_DECOMPRESSION;
}
void ZlibDecompressorIStreambuf::zStreamInit(ZLibCompressionFormat format)
{
std::unordered_map<ZLibCompressionFormat, int> compressionFormatMap = {
{ZLIB_COMPRESSION_FORMAT_ZLIB, 15},
{ZLIB_COMPRESSION_FORMAT_GZIP, 31},
{ZLIB_COMPRESSION_FORMAT_AUTODETECT, 47} // 47 = 32 + 15
};
int windowBits = compressionFormatMap.at(format);
_zstream.zalloc = Z_NULL; // No custom memory allocation routines
_zstream.zfree = Z_NULL; // Ditto. Therefore, the 'opaque' field won't
_zstream.opaque = Z_NULL; // be used, actually.
int retCode = inflateInit2(&_zstream, windowBits);
if (retCode != Z_OK) {
throw sg_io_exception(::zlibErrorMessage(_zstream, retCode),
sg_location(_path));
}
}
int ZlibDecompressorIStreambuf::zlibProcessData()
{
// Decompress as much data as possible given _zstream.avail_in and
// _zstream.avail_out. The input data starts at _zstream.next_in, the output
// at _zstream.next_out, and these four fields are all updated by this call
// to reflect exactly the amount of data zlib has consumed and produced.
return inflate(&_zstream, _zstream.avail_in ? Z_NO_FLUSH : Z_FINISH);
}
// ***************************************************************************
// * ZlibCompressorIStream class *
// ***************************************************************************
ZlibCompressorIStream::ZlibCompressorIStream(std::istream& iStream,
const SGPath& path,
int compressionLevel,
ZLibCompressionFormat format,
ZLibMemoryStrategy memStrategy,
char* inBuf,
std::size_t inBufSize,
char *outBuf,
std::size_t outBufSize,
std::size_t putbackSize)
: _streamBuf(iStream, path, compressionLevel, format, memStrategy, inBuf,
inBufSize, outBuf, outBufSize, putbackSize)
{
rdbuf(&_streamBuf); // Associate _streamBuf to 'this'
}
ZlibCompressorIStream::~ZlibCompressorIStream()
{ }
// ***************************************************************************
// * ZlibDecompressorIStream class *
// ***************************************************************************
ZlibDecompressorIStream::ZlibDecompressorIStream(std::istream& iStream,
const SGPath& path,
ZLibCompressionFormat format,
char* inBuf,
std::size_t inBufSize,
char *outBuf,
std::size_t outBufSize,
std::size_t putbackSize)
: _streamBuf(iStream, path, format, inBuf, inBufSize, outBuf, outBufSize,
putbackSize)
{
rdbuf(&_streamBuf); // Associate _streamBuf to 'this'
}
ZlibDecompressorIStream::~ZlibDecompressorIStream()
{ }
} // of namespace simgear

401
simgear/io/iostreams/zlibstream.hxx Normal file
View File

@@ -0,0 +1,401 @@
// -*- coding: utf-8 -*-
//
// zlibstream.hxx --- IOStreams classes for working with RFC 1950 and RFC 1952
// compression formats (respectively known as the zlib and
// gzip formats)
//
// Copyright (C) 2017 Florent Rougon
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with this program; if not, write to the Free Software Foundation, Inc.,
// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#ifndef SG_ZLIBSTREAM_HXX
#define SG_ZLIBSTREAM_HXX
#include <iosfwd>
#include <ios> // std::streamsize
#include <zlib.h> // struct z_stream
#include <simgear/misc/sg_path.hxx>
// This file contains:
//
// - two stream buffer classes (ZlibCompressorIStreambuf and
// ZlibDecompressorIStreambuf), both based on the same abstract class:
// ZlibAbstractIStreambuf;
//
// - two std::istream subclasses (ZlibCompressorIStream and
// ZlibDecompressorIStream), each creating and using the corresponding
// stream buffer class from the previous item.
//
// All these allow one to work with RFC 1950 and RFC 1952 compression
// formats, respectively known as the zlib and gzip formats.
//
// These classes are *input* streaming classes, which means they can
// efficiently handle arbitrary amounts of data without using any disk
// space nor increasing amounts of memory, and allow “client code” to pull
// exactly as much data as it wants at any given time, resuming later
// when it is ready to handle the next chunk.
//
// So, for example, assuming you've created an instance of
// ZlibCompressorIStream (bound to some input stream of your choice, let's
// call it iStream), you could read 512 bytes of data from it, and you
// would get the first 512 bytes of *compressed* data corresponding to what
// iStream provided. Then you could resume at any time and ask for the next
// 512 bytes of compressed data (or any other amount), etc.
//
// Therefore, these classes are well suited, among others, to compress or
// decompress data streams while at the same time packing the result into
// discrete chunks or packets with size constraints (you can think of the
// process as making sausages :).
//
// The input being in each case an std::istream (for compressing as well as
// for decompressing), it can be tied to an arbitrary source: a file with
// sg_ifstream or std::ifstream, a memory buffer with std::istringstream or
// std::stringstream, a TCP socket with a custom std::streambuf subclass[1]
// to interface with the sockets API, etc.
//
// [1] Possibly wrapped in an std::istream.
//
// The stream buffer classes upon which ZlibCompressorIStream and
// ZlibDecompressorIStream are built have an xsgetn() implementation that
// avoids useless copies of data by asking zlib to write directly to the
// destination buffer. This xsgetn() method is used when calling read() on
// the std::istream subclasses, or sgetn() if you are using the stream
// buffer classes directly (i.e., ZlibCompressorIStreambuf and
// ZlibDecompressorIStreambuf). Other std::istream methods may instead rely
// only on the internal buffer and the underflow() method, and therefore be
// less efficient for large amounts of data. You may want to take a look at
// zlibstream_test.cxx to see various ways of using these classes.
//
// In case you use std::istream& operator>>(std::istream&, std::string&) or
// its overload friends, beware that it splits fields at spaces, and by
// default ignores spaces at the beginning of a field (cf. std::skipws,
// std::noskipws and friends). As far as I understand it, most of these
// operators are mainly intended in the IOStreams library to be used to
// read an int here, a double there, a space-delimited string afterwards,
// etc. (the exception could be the overload writing to a stream buffer,
// however it doesn't seem to be very efficient on my system with GNU
// libstdc++ [it is not using xsgetn()], so beware also of this one if you
// are handling large amounts of data). For moderately complex or large
// input handling, I'd suggest to use std::istream methods such as read(),
// gcount() and getline() (std::getline() can be useful too). Or directly
// use the stream buffer classes, in particular with sgetn().
namespace simgear
{
enum ZLibCompressionFormat {
ZLIB_COMPRESSION_FORMAT_ZLIB = 0,
ZLIB_COMPRESSION_FORMAT_GZIP,
ZLIB_COMPRESSION_FORMAT_AUTODETECT
};
enum ZLibMemoryStrategy {
ZLIB_FAVOR_MEMORY_OVER_SPEED = 0,
ZLIB_FAVOR_SPEED_OVER_MEMORY
};
// Abstract base class for both the compressor and decompressor stream buffers.
class ZlibAbstractIStreambuf: public std::streambuf
{
public:
/**
* @brief Constructor for ZlibAbstractIStreambuf.
* @param iStream Input stream to read from.
* @param path Optional path to the file corresponding to iStream,
* if any. Only used for error messages.
* @param inBuf Pointer to the input buffer (data read from iStream is
* written there before being compressed or decompressed).
* If nullptr, the buffer is allocated on the heap in the
* constructor and deallocated in the destructor.
* @param inBufSize Size of the input buffer, in chars.
* @param outBuf Pointer to the output buffer. Data is read by zlib
* from the input buffer, compressed or decompressed, and
* the result is directly written to the output buffer.
* If nullptr, the buffer is allocated on the heap in the
* constructor and deallocated in the destructor.
* @param outBufSize Size of the output buffer, in chars.
* @param putbackSize Size of the putback area inside the output buffer, in
* chars.
*
* It is required that putbackSize < outBufSize. It is guaranteed that,
* if at least putbackSize chars have been read without any putback (or
* unget) operation intermixed, then at least putbackSize chars can be
* put back in sequence. If you don't need this feature, use zero for the
* putbackSize value (the default) for best performance.
*/
explicit ZlibAbstractIStreambuf(std::istream& iStream,
const SGPath& path = SGPath(),
char* inBuf = nullptr,
std::size_t inBufSize = 262144,
char* outBuf = nullptr,
std::size_t outBufSize = 262144,
std::size_t putbackSize = 0);
ZlibAbstractIStreambuf(const ZlibAbstractIStreambuf&) = delete;
ZlibAbstractIStreambuf& operator=(const ZlibAbstractIStreambuf&) = delete;
~ZlibAbstractIStreambuf();
protected:
enum OperationType {
OPERATION_TYPE_COMPRESSION = 0,
OPERATION_TYPE_DECOMPRESSION
};
virtual OperationType operationType() const = 0;
// Either compress or decompress a chunk of data (depending on the
// particular subclass implementation). The semantics are the same as for
// zlib's inflate() and deflate() functions applied to member _zstream,
// concerning 1) the return value and 2) where, and how much to read and
// write (which thus depends on _zstream.avail_in, _zstream.next_in,
// _zstream.avail_out and _zstream.next_out).
virtual int zlibProcessData() = 0;
// The input stream, from which data is read before being processed by zlib
std::istream& _iStream;
// Corresponding path, if any (default-constructed SGPath instance otherwise)
const SGPath _path;
// Structure used to communicate with zlib
z_stream _zstream;
private:
// Callback whose role is to refill the output buffer when it's empty and
// the “client” tries to read more.
int underflow() override;
// Optional override when subclassing std::streambuf. This is the most
// efficient way of reading several characters (as soon as we've emptied the
// output buffer, data is written by zlib directly to the destination
// buffer).
std::streamsize xsgetn(char* dest, std::streamsize n) override;
// Make sure there is data to read in the input buffer, or signal EOF.
bool getInputData();
// Utility method for fillOutputBuffer()
std::size_t fOB_remainingSpace(unsigned char* nextOutPtr) const;
// Fill the output buffer (using zlib functions) as much as possible.
char* fillOutputBuffer();
// Utility method
[[ noreturn ]] void handleZ_BUF_ERROR() const;
bool _allFinished = false;
// The buffers
// ~~~~~~~~~~~
//
// The input buffer receives data obtained from _iStream, before it is
// processed by zlib. In underflow(), zlib reads from this buffer it and
// writes the resulting data(*) to the output buffer. Then we point the
// standard std::streambuf pointers (gptr() and friends) directly towards
// the data inside that output buffer. xsgetn() is even more optimized: it
// first empties the output buffer, then makes zlib write the remaining data
// directly to the destination area.
//
// (*) Compressed or decompressed, depending on the particular
// implementation of zlibProcessData() in each subclass.
char* _inBuf;
const std::size_t _inBufSize;
// _inBufEndPtr points right after the last data retrieved from _iStream and
// stored into _inBuf. When zlib has read all such data, _zstream.next_in is
// equal to _inBufEndPtr (after proper casting). Except in this particular
// situation, only _zstream.next_in <= _inBufEndPtr is guaranteed.
char* _inBufEndPtr;
// Layout of the _outBuf buffer:
//
// |_outBuf <putback area> |_outBuf + _putbackSize |_outBuf + _outBufSize
//
// The first _putbackSize chars in _outBuf are reserved for the putback area
// (right-aligned at _outBuf + _putbackSize). The actual output buffer thus
// starts at _outBuf + _putbackSize. At any given time for callers of this
// class, the number of characters that can be put back is gptr() - eback().
// It may be lower than _putbackSize if we haven't read that many characters
// yet. It may also be larger if gptr() > _outBuf + _putbackSize, i.e.,
// when the buffer for pending data is non-empty.
//
// At any given time, callers should see:
//
// _outBuf <= eback() <= _outBuf + _putbackSize <= gptr() <= egptr()
// <= _outBuf + _outBufSize
//
// (hoping this won't get out of sync with the code!)
char *_outBuf;
const std::size_t _outBufSize;
// Space reserved for characters to be put back into the stream. Must be
// strictly smaller than _outBufSize (this is checked in the constructor).
// It is guaranteed that this number of chars can be put back, except of
// course if we haven't read that many characters from the input stream yet.
// If characters are buffered in _outBuf[2], then it may be that more
// characters than _putbackSize can be put back (it is essentially a matter
// for std::streambuf of decreasing the “next pointer for the input
// sequence”, i.e., the one returned by gptr()).
//
// [2] In the [_outBuf + _putbackSize, _outBuf + _outBufSize) area.
const std::size_t _putbackSize;
// Since the constructor optionally allocates memory for the input and
// output buffers, these members allow the destructor to know which buffers
// have to be deallocated, if any.
bool _inBufMustBeFreed = false;
bool _outBufMustBeFreed = false;
};
// Stream buffer class for compressing data. Input data is obtained from an
// std::istream instance; the corresponding compressed data can be read using
// the standard std::streambuf read interface (mainly: sbumpc(), sgetc(),
// snextc(), sgetn(), sputbackc(), sungetc()). Input, uncompressed data is
// “pulled” as needed for the amount of compressed data requested by the
// “client” using the methods I just listed.
class ZlibCompressorIStreambuf: public ZlibAbstractIStreambuf
{
public:
// Same parameters as for ZlibAbstractIStreambuf, except:
//
// compressionLevel: in the [0,9] range. 0 means no compression at all.
// Levels 1 to 9 yield compressed data, with 1 giving
// the highest compression speed but worst compression
// ratio, and 9 the highest compression ratio but lowest
// compression speed.
// format either ZLIB_COMPRESSION_FORMAT_ZLIB or
// ZLIB_COMPRESSION_FORMAT_GZIP
// memStrategy either ZLIB_FAVOR_MEMORY_OVER_SPEED or
// ZLIB_FAVOR_SPEED_OVER_MEMORY
explicit ZlibCompressorIStreambuf(
std::istream& iStream,
const SGPath& path = SGPath(),
int compressionLevel = Z_DEFAULT_COMPRESSION,
ZLibCompressionFormat format = ZLIB_COMPRESSION_FORMAT_ZLIB,
ZLibMemoryStrategy memStrategy = ZLIB_FAVOR_SPEED_OVER_MEMORY,
char* inBuf = nullptr,
std::size_t inBufSize = 262144,
char* outBuf = nullptr,
std::size_t outBufSize = 262144,
std::size_t putbackSize = 0);
ZlibCompressorIStreambuf(const ZlibCompressorIStreambuf&) = delete;
ZlibCompressorIStreambuf& operator=(const ZlibCompressorIStreambuf&) = delete;
~ZlibCompressorIStreambuf();
protected:
OperationType operationType() const override;
// Initialize the z_stream struct used by zlib
void zStreamInit(int compressionLevel, ZLibCompressionFormat format,
ZLibMemoryStrategy memStrategy);
// Call zlib's deflate() function to compress data.
int zlibProcessData() override;
};
// Stream buffer class for decompressing data. Input data is obtained from an
// std::istream instance; the corresponding decompressed data can be read
// using the standard std::streambuf read interface (mainly: sbumpc(),
// sgetc(), snextc(), sgetn(), sputbackc(), sungetc()). Input, compressed data
// is “pulled” as needed for the amount of uncompressed data requested by the
// “client” using the methods I just listed.
class ZlibDecompressorIStreambuf: public ZlibAbstractIStreambuf
{
public:
// Same parameters as for ZlibAbstractIStreambuf, except:
//
// format ZLIB_COMPRESSION_FORMAT_ZLIB,
// ZLIB_COMPRESSION_FORMAT_GZIP or
// ZLIB_COMPRESSION_FORMAT_AUTODETECT
explicit ZlibDecompressorIStreambuf(
std::istream& iStream,
const SGPath& path = SGPath(),
ZLibCompressionFormat format = ZLIB_COMPRESSION_FORMAT_ZLIB,
char* inBuf = nullptr,
std::size_t inBufSize = 262144,
char* outBuf = nullptr,
std::size_t outBufSize = 262144,
std::size_t putbackSize = 0); // default optimized for speed
ZlibDecompressorIStreambuf(const ZlibDecompressorIStreambuf&) = delete;
ZlibDecompressorIStreambuf& operator=(const ZlibDecompressorIStreambuf&)
= delete;
~ZlibDecompressorIStreambuf();
protected:
OperationType operationType() const override;
void zStreamInit(ZLibCompressionFormat format);
int zlibProcessData() override;
};
// std::istream subclass for compressing data. Input data is obtained from an
// std::istream instance; the corresponding compressed data can be read using
// the standard std::istream interface (read(), readsome(), gcount(), get(),
// getline(), operator>>(), peek(), putback(), ignore(), unget()... plus
// operator overloads such as istream& operator>>(istream&, string&) as
// defined in <string>, and std::getline()). Input, uncompressed data is
// “pulled” as needed for the amount of compressed data requested by the
// “client”.
//
// To get data efficiently from an instance of this class, use its read()
// method (typically in conjunction with gcount(), inside a loop).
class ZlibCompressorIStream: public std::istream
{
public:
// Same parameters as for ZlibCompressorIStreambuf
explicit ZlibCompressorIStream(
std::istream& iStream,
const SGPath& path = SGPath(),
int compressionLevel = Z_DEFAULT_COMPRESSION,
ZLibCompressionFormat format = ZLIB_COMPRESSION_FORMAT_ZLIB,
ZLibMemoryStrategy memStrategy = ZLIB_FAVOR_SPEED_OVER_MEMORY,
char* inBuf = nullptr,
std::size_t inBufSize = 262144,
char* outBuf = nullptr,
std::size_t outBufSize = 262144,
std::size_t putbackSize = 0); // default optimized for speed
ZlibCompressorIStream(const ZlibCompressorIStream&) = delete;
ZlibCompressorIStream& operator=(const ZlibCompressorIStream&) = delete;
~ZlibCompressorIStream();
private:
ZlibCompressorIStreambuf _streamBuf;
};
// std::istream subclass for decompressing data. Input data is obtained from
// an std::istream instance; the corresponding decompressed data can be read
// using the standard std::istream interface (read(), readsome(), gcount(),
// get(), getline(), operator>>(), peek(), putback(), ignore(), unget()...
// plus operator overloads such as istream& operator>>(istream&, string&) as
// defined in <string>, and std::getline()). Input, compressed data is
// “pulled” as needed for the amount of uncompressed data requested by the
// “client”.
//
// To get data efficiently from an instance of this class, use its read()
// method (typically in conjunction with gcount(), inside a loop).
class ZlibDecompressorIStream: public std::istream
{
public:
// Same parameters as for ZlibDecompressorIStreambuf
explicit ZlibDecompressorIStream(
std::istream& iStream,
const SGPath& path = SGPath(),
ZLibCompressionFormat format = ZLIB_COMPRESSION_FORMAT_ZLIB,
char* inBuf = nullptr,
std::size_t inBufSize = 262144,
char* outBuf = nullptr,
std::size_t outBufSize = 262144,
std::size_t putbackSize = 0); // default optimized for speed
ZlibDecompressorIStream(const ZlibDecompressorIStream&) = delete;
ZlibDecompressorIStream& operator=(const ZlibDecompressorIStream&) = delete;
~ZlibDecompressorIStream();
private:
ZlibDecompressorIStreambuf _streamBuf;
};
} // of namespace simgear
#endif // of SG_ZLIBSTREAM_HXX

562
simgear/io/iostreams/zlibstream_test.cxx Normal file
View File

@@ -0,0 +1,562 @@
// -*- coding: utf-8 -*-
//
// zlibstream_test.cxx --- Automated tests for zlibstream.cxx / zlibstream.hxx
//
// Copyright (C) 2017 Florent Rougon
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with this program; if not, write to the Free Software Foundation, Inc.,
// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#include <ios> // std::basic_ios, std::streamsize...
#include <iostream> // std::ios_base, std::cerr, etc.
#include <sstream>
#include <array>
#include <random>
#include <limits> // std::numeric_limits
#include <functional> // std::bind()
#include <cassert>
#include <cstdlib> // EXIT_SUCCESS
#include <cstddef> // std::size_t
#include <cstring> // strcmp()
using std::string;
using std::cout;
using std::cerr;
using traits = std::char_traits<char>;
#include <simgear/misc/test_macros.hxx>
#include <simgear/io/iostreams/sgstream.hxx>
#include <simgear/io/iostreams/zlibstream.hxx>
#include <simgear/misc/sg_path.hxx>
#include <simgear/misc/sg_dir.hxx>
// In many tests below, I use very small buffer sizes. Of course, this is bad
// for performance. The reason I do it this way is simply because it better
// exercises the code we want to *test* here (we are more likely to find bugs
// if the buffer(s) has/have to be refilled one or more times). Similarly, if
// you don't need the putback feature in non-test code, best performance is
// achieved with putback size = 0.
//
// I suggest you read roundTripWithIStreams() below to see how to use the
// classes most efficiently (especially the comments!).
static std::default_random_engine randomNumbersGenerator;
// Sample string for tests
static const string lipsum = "\
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque congue ornare\n\
congue. Mauris mollis est et porttitor condimentum. Vivamus laoreet blandit\n\
odio eget consectetur. Etiam quis magna eu enim luctus pretium. In et\n\
tristique nunc, non efficitur metus. Nullam efficitur tristique velit.\n\
Praesent et luctus nunc. Mauris eros eros, rutrum at molestie quis, egestas et\n\
lorem. Ut nulla turpis, eleifend sed mauris ac, faucibus molestie nulla.\n\
Quisque viverra vel turpis nec efficitur. Proin non rutrum velit. Nam sodales\n\
metus felis, eu pharetra velit posuere ut.\n\
\n\
Suspendisse pellentesque tincidunt ligula et pretium. Etiam id justo mauris.\n\
Aenean porta, sapien in suscipit tristique, metus diam malesuada dui, in porta\n\
felis mi non felis. Etiam vel aliquam leo, non vehicula magna. Proin justo\n\
massa, ultrices at porta eu, tempor in ligula. Duis enim ipsum, dictum quis\n\
ultricies et, tempus eu libero. Morbi vulputate libero ut dolor rutrum, a\n\
imperdiet nibh egestas. Phasellus finibus massa vel tempus hendrerit. Nulla\n\
lobortis est non ligula viverra, quis egestas ante hendrerit. Pellentesque\n\
finibus mollis blandit. In ac sollicitudin mauris, eget dignissim mauris.";
// Utility function: generate a random string whose length is in the
// [minLen, maxLen] range.
string randomString(string::size_type minLen, string::size_type maxLen)
{
std::uniform_int_distribution<string::size_type> sLenDist(minLen, maxLen);
std::uniform_int_distribution<int> byteDist(0, 255);
auto randomByte = std::bind(byteDist, randomNumbersGenerator);
string::size_type len = sLenDist(randomNumbersGenerator);
string str;
while (str.size() < len) {
str += traits::to_char_type(randomByte());
}
return str;
}
// Utility function: perform a compression-decompression cycle on the input
// string using the stream buffer classes:
//
// 1) Compress the input string, write the result to a temporary file using
// std::ostream& operator<<(streambuf* sb), where 'sb' points to a
// ZlibCompressorIStreambuf instance.
//
// 2) Close the temporary file to make sure all the data is flushed.
//
// 3) Create a ZlibDecompressorIStreambuf instance reading from the
// temporary file, then decompress to an std::ostringstream using
// std::ostream& operator<<(streambuf* sb), where 'sb' points to our
// ZlibDecompressorIStreambuf instance.
//
// 4) Compare the result with the input string.
//
// Of course, it is possible to do this without any temporary file, even
// without any std::stringstream or such to hold the intermediate, compressed
// data. See the other tests for actual compressor + decompressor pipelines.
void pipeCompOrDecompIStreambufIntoOStream(const string& input)
{
// Test “std::ostream << &ZlibCompressorIStreambuf”
std::istringstream input_ss(input);
simgear::ZlibCompressorIStreambuf compSBuf(input_ss, SGPath(), 9);
simgear::Dir tmpDir = simgear::Dir::tempDir("FlightGear");
tmpDir.setRemoveOnDestroy();
SGPath path = tmpDir.path() / "testFile.dat";
sg_ofstream oFile(path,
std::ios::binary | std::ios::trunc | std::ios::out);
oFile << &compSBuf;
// Make sure the compressed stream is flushed, otherwise when we read the
// file back, decompression is likely to fail with Z_BUF_ERROR.
oFile.close();
SG_CHECK_EQUAL(compSBuf.sgetc(), EOF);
// Read back and decompress the data we've written to 'path', testing
// “std::ostream << &ZlibDecompressorIStreambuf”
sg_ifstream iFile(path, std::ios::binary | std::ios::in);
simgear::ZlibDecompressorIStreambuf decompSBuf(iFile, path);
std::ostringstream roundTripResult_ss;
// This is also possible, though maybe not as good for error detection:
//
// decompSBuf >> roundTripResult_ss.rdbuf();
roundTripResult_ss << &decompSBuf;
SG_CHECK_EQUAL(decompSBuf.sgetc(), EOF);
SG_CHECK_EQUAL(roundTripResult_ss.str(), input);
}
// Perform a compression-decompression cycle on bunch of strings, using the
// stream buffer classes directly (not the std::istream subclasses).
void test_pipeCompOrDecompIStreambufIntoOStream()
{
cerr << "Compression-decompression cycles using the stream buffer classes "
"directly\n";
pipeCompOrDecompIStreambufIntoOStream(""); // empty string
pipeCompOrDecompIStreambufIntoOStream("a");
pipeCompOrDecompIStreambufIntoOStream("lorem ipsum");
for (int i=0; i < 10; i++) {
pipeCompOrDecompIStreambufIntoOStream(randomString(0, 20));
}
for (int i=0; i < 10; i++) {
pipeCompOrDecompIStreambufIntoOStream(randomString(21, 1000));
}
assert(std::numeric_limits<string::size_type>::max() >= 65535);
for (int i=0; i < 10; i++) {
pipeCompOrDecompIStreambufIntoOStream(randomString(1000, 65535));
}
}
void test_StreambufBasicOperations()
{
cerr << "Testing basic operations on ZlibDecompressorIStreambuf\n";
const string text = "0123456789abcdefghijklmnopqrstuvwxyz\nABCDEF\nGHIJK "
"LMNOPQ";
std::istringstream text_ss(text);
static constexpr std::size_t compInBufSize = 6;
static constexpr std::size_t compOutBufSize = 4;
static constexpr std::size_t compPutbackSize = 0;
simgear::ZlibCompressorIStreambuf compSBuf(
text_ss, SGPath(), 8, simgear::ZLIB_COMPRESSION_FORMAT_ZLIB,
simgear::ZLIB_FAVOR_SPEED_OVER_MEMORY, nullptr, compInBufSize, nullptr,
compOutBufSize, compPutbackSize);
std::stringstream compressedOutput_ss;
compressedOutput_ss << &compSBuf;
static constexpr std::size_t decompInBufSize = 5;
static constexpr std::size_t decompOutBufSize = 4;
static constexpr std::size_t decompPutbackSize = 2;
simgear::ZlibDecompressorIStreambuf decompSBuf(
compressedOutput_ss, SGPath(), simgear::ZLIB_COMPRESSION_FORMAT_ZLIB,
nullptr, decompInBufSize, nullptr, decompOutBufSize, decompPutbackSize);
int ch = decompSBuf.sgetc();
SG_VERIFY(ch != EOF && traits::to_char_type(ch) == '0');
ch = decompSBuf.sbumpc();
SG_VERIFY(ch != EOF && traits::to_char_type(ch) == '0');
ch = decompSBuf.sbumpc();
SG_VERIFY(ch != EOF && traits::to_char_type(ch) == '1');
ch = decompSBuf.snextc();
SG_VERIFY(ch != EOF && traits::to_char_type(ch) == '3');
ch = decompSBuf.sbumpc();
SG_VERIFY(ch != EOF && traits::to_char_type(ch) == '3');
ch = decompSBuf.sbumpc();
SG_VERIFY(ch != EOF && traits::to_char_type(ch) == '4');
ch = decompSBuf.sputbackc('4');
SG_VERIFY(ch != EOF && traits::to_char_type(ch) == '4');
ch = decompSBuf.sputbackc('u'); // doesn't match what we read from the stream
SG_VERIFY(ch == EOF);
ch = decompSBuf.sputbackc('3'); // this one does
SG_VERIFY(ch != EOF && traits::to_char_type(ch) == '3');
static constexpr std::streamsize bufSize = 10;
char buf[bufSize];
// Most efficient way (with the underlying xsgetn()) to read several chars
// at once.
std::streamsize n = decompSBuf.sgetn(buf, bufSize);
SG_VERIFY(n == bufSize && string(buf, bufSize) == "3456789abc");
ch = decompSBuf.sungetc(); // same as sputbackc(), except no value to check
SG_VERIFY(ch != EOF && traits::to_char_type(ch) == 'c');
ch = decompSBuf.sungetc();
SG_VERIFY(ch != EOF && traits::to_char_type(ch) == 'b');
ch = decompSBuf.sbumpc();
SG_VERIFY(ch != EOF && traits::to_char_type(ch) == 'b');
ch = decompSBuf.sputbackc('b'); // this one does
SG_VERIFY(ch != EOF && traits::to_char_type(ch) == 'b');
n = decompSBuf.sgetn(buf, bufSize);
SG_VERIFY(n == bufSize && string(buf, bufSize) == "bcdefghijk");
ch = decompSBuf.sungetc();
SG_VERIFY(ch != EOF && traits::to_char_type(ch) == 'k');
static char buf2[64];
n = decompSBuf.sgetn(buf2, sizeof(buf2));
SG_VERIFY(n == 36 && string(buf2, n) == "klmnopqrstuvwxyz\nABCDEF\nGHIJK "
"LMNOPQ");
ch = decompSBuf.sbumpc();
SG_CHECK_EQUAL(ch, EOF);
}
// Utility function: take a string as input to compress and return the
// compressed output as a string.
string compress(const string& dataToCompress,
simgear::ZLibCompressionFormat compressionFormat,
int compressionLevel,
simgear::ZLibMemoryStrategy memStrategy,
std::size_t putbackSize,
SGPath path = SGPath())
{
// Static storage is only okay for very small sizes like these, and because
// we won't call the function from several threads at the same time. Plain
// char arrays would be fine here, but I'll use std::array to show it can be
// used here too.
static std::array<char, 7> inBuf;
static std::array<char, 7> outBuf;
std::istringstream iss(dataToCompress);
simgear::ZlibCompressorIStream compressor(
iss, path, compressionLevel, compressionFormat, memStrategy,
&inBuf.front(), inBuf.size(), &outBuf.front(), outBuf.size(),
putbackSize);
compressor.exceptions(std::ios_base::badbit); // throw if badbit is set
std::ostringstream compressedData_ss;
compressor >> compressedData_ss.rdbuf();
return compressedData_ss.str();
}
void test_formattedInputFromDecompressor()
{
cerr << "Testing ZlibDecompressorIStream >> std::string\n";
static char inBuf[6];
static char outBuf[15];
string compressed = compress(
lipsum, simgear::ZLIB_COMPRESSION_FORMAT_ZLIB, Z_BEST_COMPRESSION,
simgear::ZLIB_FAVOR_MEMORY_OVER_SPEED, /* putback size */ 0);
std::istringstream compressed_ss(compressed);
simgear::ZlibDecompressorIStream decompressor(
compressed_ss, SGPath(), simgear::ZLIB_COMPRESSION_FORMAT_ZLIB,
inBuf, sizeof(inBuf), outBuf, sizeof(outBuf), /* putback size */ 1);
decompressor.exceptions(std::ios_base::badbit); // throw if badbit is set
int count = 0;
string word;
while (decompressor >> word) {
count++;
}
SG_CHECK_EQUAL(count, 175); // Number of words in 'lipsum'
// If set, badbit would have caused an exception to be raised
SG_VERIFY(!decompressor.bad());
if (!decompressor.eof()) {
assert(decompressor.fail());
cerr << "Decompressor: stream extraction (operator>>) failed before "
"reaching EOF.\n";
SG_TEST_FAIL("Did not expect operator>> to fail with an std::string "
"argument.");
}
}
void test_ZlibDecompressorIStream_readPutbackEtc()
{
cerr << "Testing many operations on ZlibDecompressorIStream (read(), "
"putback(), etc.\n";
static char compInBuf[4];
static char compOutBuf[6];
static char decompInBuf[8];
static char decompOutBuf[5];
const string text = "0123456789abcdefghijklmnopqrstuvwxyz\nABCDEF\nGHIJK "
"LMNOPQ";
std::istringstream text_ss(text);
simgear::ZlibCompressorIStream compressor(
text_ss, SGPath(), Z_BEST_COMPRESSION,
simgear::ZLIB_COMPRESSION_FORMAT_ZLIB,
simgear::ZLIB_FAVOR_MEMORY_OVER_SPEED,
compInBuf, sizeof(compInBuf), compOutBuf, sizeof(compOutBuf),
/* putback size */ 0);
compressor.exceptions(std::ios_base::badbit); // throw if badbit is set
// Use the compressor (subclass of std::istream) as input to the decompressor
simgear::ZlibDecompressorIStream decompressor(
compressor, SGPath(), simgear::ZLIB_COMPRESSION_FORMAT_ZLIB,
decompInBuf, sizeof(decompInBuf), decompOutBuf, sizeof(decompOutBuf),
/* putback size */ 3);
decompressor.exceptions(std::ios_base::badbit);
{
static std::array<char, 17> buf;
decompressor.read(&buf.front(), 10);
SG_VERIFY(decompressor.good() && decompressor.gcount() == 10);
SG_CHECK_EQUAL(string(&buf.front(), 10), "0123456789");
SG_VERIFY(decompressor.putback('9'));
SG_VERIFY(decompressor.putback('8'));
SG_VERIFY(decompressor.putback('7'));
SG_VERIFY(decompressor.get(buf[10]));
SG_VERIFY(decompressor.read(&buf[11], 6));
SG_CHECK_EQUAL(string(&buf.front(), 17), "0123456789789abcd");
}
{
bool gotException = false;
try {
// 'Z' is not the last character read from the stream
decompressor.putback('Z');
} catch (std::ios_base::failure) {
gotException = true;
}
SG_VERIFY(gotException && decompressor.bad());
}
{
int c;
decompressor.clear();
// Check we can resume normally now that we've cleared the error state flags
c = decompressor.get();
SG_VERIFY(c != traits::eof() && traits::to_char_type(c) == 'e');
// unget() and get() in sequence
SG_VERIFY(decompressor.unget());
c = decompressor.get();
SG_VERIFY(c != traits::eof() && traits::to_char_type(c)== 'e');
// peek() looks at, but doesn't consume
c = decompressor.peek();
SG_VERIFY(c != traits::eof() && traits::to_char_type(c) == 'f');
c = decompressor.get();
SG_VERIFY(c != traits::eof() && traits::to_char_type(c)== 'f');
}
{
static std::array<char, 21> buf; // 20 chars + terminating NUL char
// std::istream::getline() will be stopped by \n after reading 20 chars
SG_VERIFY(decompressor.getline(&buf.front(), 25));
SG_VERIFY(decompressor.gcount() == 21 &&
!strcmp(&buf.front(), "ghijklmnopqrstuvwxyz"));
string str;
// std::getline() will be stopped by \n after 7 chars have been extracted,
// namely 6 chars plus the \n which is extracted and discarded.
SG_VERIFY(std::getline(decompressor, str));
SG_CHECK_EQUAL(str, "ABCDEF");
SG_VERIFY(decompressor.ignore(2));
SG_VERIFY(decompressor >> str);
SG_CHECK_EQUAL(str, "IJK");
static char buf2[5];
// Read up to sizeof(buf) chars, without waiting if not that many are
// immediately avaiable.
int nbCharsRead = decompressor.readsome(buf2, sizeof(buf2));
string rest(buf2, nbCharsRead);
do {
decompressor.read(buf2, sizeof(buf2));
rest += string(buf2, decompressor.gcount());
} while (decompressor);
SG_CHECK_EQUAL(rest, " LMNOPQ");
}
// If set, badbit would have caused an exception to be raised, anyway
SG_VERIFY(decompressor.eof() && !decompressor.bad());
}
// Utility function: parametrized round-trip test with a compressor +
// decompressor pipeline.
//
//
// Note: this is nice conceptually, allows to keep memory use constant even in
// case an arbitrary amount of data is passed through, and exercises the
// stream buffer classes well, however this technique is more than twice
// as slow on my computer than using an intermediate buffer like this:
//
// std::stringstream compressedData_ss;
// compressor >> compressedData_ss.rdbuf();
//
// simgear::ZlibDecompressorIStream decompressor(compressedData_ss,
// SGPath(), ...
void roundTripWithIStreams(
simgear::ZLibCompressionFormat compressionFormat,
int compressionLevel,
simgear::ZLibMemoryStrategy memStrategy,
std::size_t compInBufSize,
std::size_t compOutBufSize,
std::size_t decompInBufSize,
std::size_t decompOutBufSize,
std::size_t compPutbackSize,
std::size_t decompPutbackSize,
bool useAutoFormatForDecompression = false)
{
const simgear::ZLibCompressionFormat decompFormat =
(useAutoFormatForDecompression) ?
simgear::ZLIB_COMPRESSION_FORMAT_AUTODETECT : compressionFormat;
std::istringstream lipsum_ss(lipsum);
// This tests the optional dynamic buffer allocation in ZlibAbstractIStreambuf
simgear::ZlibCompressorIStream compressor(
lipsum_ss, SGPath(), compressionLevel, compressionFormat, memStrategy,
nullptr, compInBufSize, nullptr, compOutBufSize, compPutbackSize);
compressor.exceptions(std::ios_base::badbit); // throw if badbit is set
// Use the compressor as input to the decompressor (pipeline). The
// decompressor uses read() with chunks that are as large as possible given
// the available space in its input buffer. These read() calls are served by
// ZlibCompressorIStreambuf::xsgetn(), which is efficient.
simgear::ZlibDecompressorIStream decompressor(
compressor, SGPath(), decompFormat,
nullptr, decompInBufSize, nullptr, decompOutBufSize, decompPutbackSize);
decompressor.exceptions(std::ios_base::badbit);
std::ostringstream roundTripResult;
// This, on the other hand, appears not to use xsgetn() (tested with the GNU
// libstdc++ from g++ 6.3.0, 20170124). This causes useless copying of the
// data to the decompressor output buffer, instead of writing it directly to
// roundTripResult's internal buffer. This is simple and nice in appearance,
// but if you are after performance, better use decompressor.read()
// (typically in conjunction with gcount(), inside a loop).
decompressor >> roundTripResult.rdbuf();
SG_CHECK_EQUAL(roundTripResult.str(), lipsum);
}
// Round-trip conversion with simgear::ZlibCompressorIStream and
// simgear::ZlibDecompressorIStream, using various parameter combinations.
void test_RoundTripMultiWithIStreams()
{
cerr <<
"Compression-decompression cycles using the std::istream subclasses (many\n"
"combinations of buffer sizes and compression parameters tested)\n";
{ // More variations on these later
const std::size_t compPutbackSize = 1;
const std::size_t decompPutbackSize = 1;
for (auto format: {simgear::ZLIB_COMPRESSION_FORMAT_ZLIB,
simgear::ZLIB_COMPRESSION_FORMAT_GZIP}) {
for (int compressionLevel: {1, 4, 7, 9}) {
for (auto memStrategy: {simgear::ZLIB_FAVOR_MEMORY_OVER_SPEED,
simgear::ZLIB_FAVOR_SPEED_OVER_MEMORY}) {
for (std::size_t compInBufSize: {3, 4}) {
for (std::size_t compOutBufSize: {3, 5}) {
for (std::size_t decompInBufSize: {3, 4}) {
for (std::size_t decompOutBufSize: {3, 4,}) {
roundTripWithIStreams(
format, compressionLevel, memStrategy, compInBufSize,
compOutBufSize, decompInBufSize, decompOutBufSize,
compPutbackSize, decompPutbackSize);
}
}
}
}
}
}
}
}
{
const auto format = simgear::ZLIB_COMPRESSION_FORMAT_ZLIB;
const int compressionLevel = Z_DEFAULT_COMPRESSION;
const auto memStrategy = simgear::ZLIB_FAVOR_SPEED_OVER_MEMORY;
for (std::size_t compInBufSize: {3, 4, 31, 256, 19475}) {
for (std::size_t compOutBufSize: {3, 5, 9, 74, 4568}) {
for (std::size_t decompInBufSize: {3, 4, 256, 24568}) {
for (std::size_t decompOutBufSize: {3, 5, 42, 4568}) {
for (std::size_t compPutbackSize: {0, 1, 2}) {
for (std::size_t decompPutbackSize: {0, 1, 2}) {
roundTripWithIStreams(
format, compressionLevel, memStrategy, compInBufSize,
compOutBufSize, decompInBufSize, decompOutBufSize,
compPutbackSize, decompPutbackSize);
}
}
}
}
}
}
}
{
const std::size_t compInBufSize = 5;
const std::size_t compOutBufSize = 107;
const std::size_t decompInBufSize = 65536;
const std::size_t decompOutBufSize = 84;
int i = 0;
for (std::size_t compPutbackSize: {25, 40, 105}) {
for (std::size_t decompPutbackSize: {30, 60, 81}) {
const simgear::ZLibCompressionFormat compFormat = (i++ % 2) ?
simgear::ZLIB_COMPRESSION_FORMAT_ZLIB :
simgear::ZLIB_COMPRESSION_FORMAT_GZIP;
roundTripWithIStreams(
compFormat, Z_BEST_COMPRESSION, simgear::ZLIB_FAVOR_MEMORY_OVER_SPEED,
compInBufSize, compOutBufSize, decompInBufSize, decompOutBufSize,
compPutbackSize, decompPutbackSize,
/* automatic format detection for decompression */ true);
}
}
}
}
int main(int argc, char** argv)
{
test_pipeCompOrDecompIStreambufIntoOStream();
test_StreambufBasicOperations();
test_RoundTripMultiWithIStreams();
test_formattedInputFromDecompressor();
test_ZlibDecompressorIStream_readPutbackEtc();
return EXIT_SUCCESS;
}