From 78d731a316688fac35d0730edde913c97c3f6300 Mon Sep 17 00:00:00 2001 From: Robert Osfield Date: Tue, 14 Oct 2008 14:37:11 +0000 Subject: [PATCH] Added zlib support to curl and ive plugins --- src/osgPlugins/curl/CMakeLists.txt | 14 ++- src/osgPlugins/curl/ReaderWriterCURL.cpp | 119 +++++++++++++++++++- src/osgPlugins/curl/ReaderWriterCURL.h | 2 + src/osgPlugins/gz/ReaderWriterGZ.cpp | 2 - src/osgPlugins/ive/CMakeLists.txt | 10 +- src/osgPlugins/ive/DataInputStream.cpp | 124 ++++++++++++++++++--- src/osgPlugins/ive/DataInputStream.h | 9 +- src/osgPlugins/ive/DataOutputStream.cpp | 131 ++++++++++++++++++++--- src/osgPlugins/ive/DataOutputStream.h | 11 +- src/osgPlugins/ive/IveVersion.h | 3 +- src/osgPlugins/ive/ReaderWriterIVE.cpp | 13 +-- 11 files changed, 388 insertions(+), 50 deletions(-) diff --git a/src/osgPlugins/curl/CMakeLists.txt b/src/osgPlugins/curl/CMakeLists.txt index f1094a4ae..83b61e9f9 100644 --- a/src/osgPlugins/curl/CMakeLists.txt +++ b/src/osgPlugins/curl/CMakeLists.txt @@ -11,7 +11,13 @@ IF(WIN32) ENDIF(CURL_IS_STATIC) ENDIF(WIN32) -INCLUDE_DIRECTORIES( ${CURL_INCLUDE_DIRS} ) +IF(ZLIB_FOUND) + ADD_DEFINITIONS(-DUSE_ZLIB) + INCLUDE_DIRECTORIES( ${CURL_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS}) +ELSE(ZLIB_FOUND) + INCLUDE_DIRECTORIES( ${CURL_INCLUDE_DIRS} ) +ENDIF(ZLIB_FOUND) + SET(TARGET_SRC ReaderWriterCURL.cpp @@ -21,7 +27,11 @@ SET(TARGET_H ReaderWriterCURL.h ) -SET(TARGET_LIBRARIES_VARS CURL_LIBRARY ) +IF(ZLIB_FOUND) + SET(TARGET_LIBRARIES_VARS CURL_LIBRARY ZLIB_LIBRARY) +ELSE(ZLIB_FOUND) + SET(TARGET_LIBRARIES_VARS CURL_LIBRARY ) +ENDIF(ZLIB_FOUND) #### end var setup ### diff --git a/src/osgPlugins/curl/ReaderWriterCURL.cpp b/src/osgPlugins/curl/ReaderWriterCURL.cpp index f12215421..67230f55e 100644 --- a/src/osgPlugins/curl/ReaderWriterCURL.cpp +++ b/src/osgPlugins/curl/ReaderWriterCURL.cpp @@ -27,6 +27,7 @@ using namespace osg_curl; + // // StreamObject // @@ -260,15 +261,46 @@ osgDB::ReaderWriter::ReadResult ReaderWriterCURL::readFile(ObjectType objectType if (ext=="curl") { fileName = osgDB::getNameLessExtension(fullFileName); + ext = osgDB::getFileExtension(fileName); } else { fileName = fullFileName; } + + bool uncompress = false; + + if (ext=="gz" || ext=="osgz" || ext=="ivez") + { + osg::notify(osg::NOTICE)<<"Compressed file type "<getReaderWriterForExtension( osgDB::getFileExtension(fileName)); + osgDB::Registry::instance()->getReaderWriterForExtension( ext ); if (!reader) { @@ -295,12 +327,24 @@ osgDB::ReaderWriter::ReadResult ReaderWriterCURL::readFile(ObjectType objectType if (curlResult.status()==ReadResult::FILE_LOADED) { + osg::ref_ptr local_opt = options ? static_cast(options->clone(osg::CopyOp::SHALLOW_COPY)) : new Options; local_opt->getDatabasePathList().push_front(osgDB::getFilePath(fileName)); + if (uncompress) + { + std::string uncompressed; + if (!read(buffer, uncompressed)) + { + return ReadResult::FILE_NOT_HANDLED; + } + + buffer.str(uncompressed); + } + ReadResult readResult = readFile(objectType, reader, buffer, local_opt.get() ); local_opt->getDatabasePathList().pop_front(); @@ -313,6 +357,79 @@ osgDB::ReaderWriter::ReadResult ReaderWriterCURL::readFile(ObjectType objectType } } +#ifdef USE_ZLIB + +#include + +bool ReaderWriterCURL::read(std::istream& fin, std::string& destination) const +{ + #define CHUNK 16384 + + int ret; + unsigned have; + z_stream strm; + unsigned char in[CHUNK]; + unsigned char out[CHUNK]; + + /* allocate inflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit2(&strm, + 15 + 32 // autodected zlib or gzip header + ); + if (ret != Z_OK) + return ret; + + /* decompress until deflate stream ends or end of file */ + do { + + strm.avail_in = fin.readsome((char*)in, CHUNK); + + if (fin.fail()) + { + (void)inflateEnd(&strm); + return false; + } + if (strm.avail_in == 0) + break; + strm.next_in = in; + + /* run inflate() on input until output buffer not full */ + do { + strm.avail_out = CHUNK; + strm.next_out = out; + ret = inflate(&strm, Z_NO_FLUSH); + + switch (ret) { + case Z_NEED_DICT: + case Z_DATA_ERROR: + case Z_MEM_ERROR: + (void)inflateEnd(&strm); + return false; + } + have = CHUNK - strm.avail_out; + + destination.append((char*)out, have); + + } while (strm.avail_out == 0); + + /* done when inflate() says it's done */ + } while (ret != Z_STREAM_END); + + /* clean up and return */ + (void)inflateEnd(&strm); + return ret == Z_STREAM_END ? true : false; +} +#else +bool ReaderWriterCURL::read(std::istream& fin, std::string& destination) const +{ + return false; +} +#endif + // now register with Registry to instantiate the above // reader/writer. REGISTER_OSGPLUGIN(curl, ReaderWriterCURL) diff --git a/src/osgPlugins/curl/ReaderWriterCURL.h b/src/osgPlugins/curl/ReaderWriterCURL.h index 8335bd8f5..271b7d42b 100644 --- a/src/osgPlugins/curl/ReaderWriterCURL.h +++ b/src/osgPlugins/curl/ReaderWriterCURL.h @@ -123,6 +123,8 @@ class ReaderWriterCURL : public osgDB::ReaderWriter return *ec; } + bool read(std::istream& fin, std::string& destination) const; + protected: typedef std::map< OpenThreads::Thread*, osg::ref_ptr > ThreadCurlMap; diff --git a/src/osgPlugins/gz/ReaderWriterGZ.cpp b/src/osgPlugins/gz/ReaderWriterGZ.cpp index d548320a0..f3503f484 100644 --- a/src/osgPlugins/gz/ReaderWriterGZ.cpp +++ b/src/osgPlugins/gz/ReaderWriterGZ.cpp @@ -257,8 +257,6 @@ bool ReaderWriterGZ::read(std::istream& fin, std::string& destination) const /* decompress until deflate stream ends or end of file */ do { - osg::notify(osg::NOTICE)<<"Doing readsome"<getOptionString().find("noLoadExternalReferenceFiles")==std::string::npos); - osg::notify(osg::DEBUG_INFO) << "ive::DataInputStream.setLoadExternalReferenceFiles()=" << getLoadExternalReferenceFiles() << std::endl; - } -} - -DataInputStream::DataInputStream(std::istream* istream) +DataInputStream::DataInputStream(std::istream* istream, const osgDB::ReaderWriter::Options* options) { unsigned int endianType ; @@ -143,10 +132,19 @@ DataInputStream::DataInputStream(std::istream* istream) _verboseOutput = false; _istream = istream; + _owns_istream = false; _peeking = false; _peekValue = 0; _byteswap = 0; + _options = options; + + if (_options.get()) + { + setLoadExternalReferenceFiles(_options->getOptionString().find("noLoadExternalReferenceFiles")==std::string::npos); + osg::notify(osg::DEBUG_INFO) << "ive::DataInputStream.setLoadExternalReferenceFiles()=" << getLoadExternalReferenceFiles() << std::endl; + } + if(!istream){ throw Exception("DataInputStream::DataInputStream(): null pointer exception in argument."); } @@ -168,10 +166,108 @@ DataInputStream::DataInputStream(std::istream* istream) if(_version>VERSION){ throw Exception("DataInputStream::DataInputStream(): The version found in the file is newer than this library can handle."); } - + + if (_version>=VERSION_0033) + { + int compressionLevel = readInt(); + + if (compressionLevel>0) + { + osg::notify(osg::INFO)<<"compressed ive stream"< + +bool DataInputStream::uncompress(std::istream& fin, std::string& destination) const +{ + //#define CHUNK 16384 + #define CHUNK 32768 + + int ret; + unsigned have; + z_stream strm; + unsigned char in[CHUNK]; + unsigned char out[CHUNK]; + + /* allocate inflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit2(&strm, + 15 + 32 // autodected zlib or gzip header + ); + if (ret != Z_OK) + return ret; + + /* decompress until deflate stream ends or end of file */ + do { + strm.avail_in = fin.readsome((char*)in, CHUNK); + + if (fin.fail()) + { + (void)inflateEnd(&strm); + return false; + } + if (strm.avail_in == 0) + break; + strm.next_in = in; + + /* run inflate() on input until output buffer not full */ + do { + strm.avail_out = CHUNK; + strm.next_out = out; + ret = inflate(&strm, Z_NO_FLUSH); + + switch (ret) { + case Z_NEED_DICT: + case Z_DATA_ERROR: + case Z_MEM_ERROR: + (void)inflateEnd(&strm); + return false; + } + have = CHUNK - strm.avail_out; + + destination.append((char*)out, have); + + } while (strm.avail_out == 0); + + /* done when inflate() says it's done */ + } while (ret != Z_STREAM_END); + + /* clean up and return */ + (void)inflateEnd(&strm); + return ret == Z_STREAM_END ? true : false; +} +#else +bool DataInputStream::uncompress(std::istream& fin, std::string& destination) const +{ + return false; +} +#endif bool DataInputStream::readBool(){ char c; diff --git a/src/osgPlugins/ive/DataInputStream.h b/src/osgPlugins/ive/DataInputStream.h index 628f1edd3..09fba6ab1 100644 --- a/src/osgPlugins/ive/DataInputStream.h +++ b/src/osgPlugins/ive/DataInputStream.h @@ -36,10 +36,9 @@ namespace ive{ class DataInputStream{ public: - DataInputStream(std::istream* istream); + DataInputStream(std::istream* istream, const osgDB::ReaderWriter::Options* options); ~DataInputStream(); - void setOptions(const osgDB::ReaderWriter::Options* options); const osgDB::ReaderWriter::Options* getOptions() const { return _options.get(); } inline unsigned int getVersion() const { return _version; } @@ -123,8 +122,13 @@ public: std::istream* _istream; int _byteswap; + bool _owns_istream; + + bool uncompress(std::istream& fin, std::string& destination) const; + private: + int _version; bool _peeking; int _peekValue; @@ -142,6 +146,7 @@ private: bool _loadExternalReferenceFiles; osg::ref_ptr _options; + }; diff --git a/src/osgPlugins/ive/DataOutputStream.cpp b/src/osgPlugins/ive/DataOutputStream.cpp index f6c05f4f4..003fc2e04 100644 --- a/src/osgPlugins/ive/DataOutputStream.cpp +++ b/src/osgPlugins/ive/DataOutputStream.cpp @@ -119,10 +119,20 @@ using namespace ive; -void DataOutputStream::setOptions(const osgDB::ReaderWriter::Options* options) +DataOutputStream::DataOutputStream(std::ostream * ostream, const osgDB::ReaderWriter::Options* options) { + _verboseOutput = false; + + _includeImageMode = IMAGE_INCLUDE_DATA; + + _includeExternalReferences = false; + _writeExternalReferenceFiles = false; + _useOriginalExternalReferences = true; + _options = options; + _compressionLevel = 0; + if (_options.get()) { if(_options->getOptionString().find("noTexturesInIVEFile")!=std::string::npos) { @@ -142,28 +152,117 @@ void DataOutputStream::setOptions(const osgDB::ReaderWriter::Options* options) setUseOriginalExternalReferences(_options->getOptionString().find("useOriginalExternalReferences")!=std::string::npos); osg::notify(osg::DEBUG_INFO) << "ive::DataOutpouStream.setUseOriginalExternalReferences()=" << getUseOriginalExternalReferences() << std::endl; + + _compressionLevel = (_options->getOptionString().find("compressed")!=std::string::npos) ? 1 : 0; + osg::notify(osg::DEBUG_INFO) << "ive::DataOutpouStream._compressionLevel=" << _compressionLevel << std::endl; + } + + #ifndef USE_ZLIB + if (_compressionLevel>0) + { + osg::notify(osg::NOTICE) << "Compression not supported in this .ive version." << std::endl; + _compressionLevel = 0; + } + #endif + + _output_ostream = _ostream = ostream; + + if(!_ostream) + throw Exception("DataOutputStream::DataOutputStream(): null pointer exception in argument."); + + writeUInt(ENDIAN_TYPE) ; + writeUInt(getVersion()); + + writeInt(_compressionLevel); + + if (_compressionLevel>0) + { + + _ostream = &_compressionStream; } } -DataOutputStream::DataOutputStream(std::ostream * ostream) +DataOutputStream::~DataOutputStream() { - _verboseOutput = false; + if (_compressionLevel>0) + { + _ostream = _output_ostream; - _includeImageMode = IMAGE_INCLUDE_DATA; - - _includeExternalReferences = false; - _writeExternalReferenceFiles = false; - _useOriginalExternalReferences = true; - - - _ostream = ostream; - if(!_ostream) - throw Exception("DataOutputStream::DataOutputStream(): null pointer exception in argument."); - writeUInt(ENDIAN_TYPE) ; - writeUInt(getVersion()); + std::string compressionString(_compressionStream.str()); + writeUInt(compressionString.size()); + + compress(*_output_ostream, compressionString); + } } -DataOutputStream::~DataOutputStream(){} +#ifdef USE_ZLIB + +#include + +#define CHUNK 16384 +bool DataOutputStream::compress(std::ostream& fout, const std::string& source) const +{ + int ret, flush = Z_FINISH; + unsigned have; + z_stream strm; + unsigned char out[CHUNK]; + + int level = 6; + int stategy = Z_DEFAULT_STRATEGY; // looks to be the best for .osg/.ive files + //int stategy = Z_FILTERED; + //int stategy = Z_HUFFMAN_ONLY; + //int stategy = Z_RLE; + + /* allocate deflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + ret = deflateInit2(&strm, + level, + Z_DEFLATED, + 15+16, // +16 to use gzip encoding + 8, // default + stategy); + if (ret != Z_OK) + return false; + + strm.avail_in = source.size(); + strm.next_in = (Bytef*)(&(*source.begin())); + + /* run deflate() on input until output buffer not full, finish + compression if all of source has been read in */ + do { + strm.avail_out = CHUNK; + strm.next_out = out; + ret = deflate(&strm, flush); /* no bad return value */ + + if (ret == Z_STREAM_ERROR) + { + osg::notify(osg::NOTICE)<<"Z_STREAM_ERROR"<0) fout.write((const char*)out, have); + + if (fout.fail()) + { + (void)deflateEnd(&strm); + return false; + } + } while (strm.avail_out == 0); + + /* clean up and return */ + (void)deflateEnd(&strm); + return true; +} +#else +bool DataOutputStream::compress(std::ostream& fout, const std::string& source) const +{ + return false; +} +#endif void DataOutputStream::writeBool(bool b) { diff --git a/src/osgPlugins/ive/DataOutputStream.h b/src/osgPlugins/ive/DataOutputStream.h index 1d1d662b5..611a69a12 100644 --- a/src/osgPlugins/ive/DataOutputStream.h +++ b/src/osgPlugins/ive/DataOutputStream.h @@ -5,6 +5,8 @@ #include // for ofstream #include +#include + #include #include #include @@ -30,10 +32,9 @@ namespace ive { class DataOutputStream{ public: - DataOutputStream(std::ostream* ostream); + DataOutputStream(std::ostream* ostream, const osgDB::ReaderWriter::Options* options); ~DataOutputStream(); - void setOptions(const osgDB::ReaderWriter::Options* options); const osgDB::ReaderWriter::Options* getOptions() const { return _options.get(); } unsigned int getVersion() { return VERSION; } @@ -123,9 +124,15 @@ public: bool _verboseOutput; + bool compress(std::ostream& fout, const std::string& source) const; + private: std::ostream* _ostream; + std::ostream* _output_ostream; + + std::stringstream _compressionStream; + int _compressionLevel; // Container to map stateset uniques to their respective stateset. typedef std::map StateSetMap; diff --git a/src/osgPlugins/ive/IveVersion.h b/src/osgPlugins/ive/IveVersion.h index 1e61b3cdc..4391bee06 100644 --- a/src/osgPlugins/ive/IveVersion.h +++ b/src/osgPlugins/ive/IveVersion.h @@ -41,8 +41,9 @@ #define VERSION_0030 30 #define VERSION_0031 31 #define VERSION_0032 32 +#define VERSION_0033 33 -#define VERSION VERSION_0032 +#define VERSION VERSION_0033 /* The BYTE_SEX tag is used to check the endian of the IVE file being read in. The IVE format diff --git a/src/osgPlugins/ive/ReaderWriterIVE.cpp b/src/osgPlugins/ive/ReaderWriterIVE.cpp index fa0c33465..33a472449 100644 --- a/src/osgPlugins/ive/ReaderWriterIVE.cpp +++ b/src/osgPlugins/ive/ReaderWriterIVE.cpp @@ -78,8 +78,7 @@ class ReaderWriterIVE : public ReaderWriter virtual ReadResult readImage(std::istream& fin, const Options* options) const { try{ - ive::DataInputStream in(&fin); - in.setOptions(options); + ive::DataInputStream in(&fin, options); return in.readImage(ive::IMAGE_INCLUDE_DATA); } catch(ive::Exception e) @@ -93,8 +92,7 @@ class ReaderWriterIVE : public ReaderWriter { try{ // Create datainputstream. - ive::DataInputStream in(&fin); - in.setOptions(options); + ive::DataInputStream in(&fin, options); return in.readNode(); } @@ -162,8 +160,7 @@ class ReaderWriterIVE : public ReaderWriter { try { - ive::DataOutputStream out(&fout); - out.setOptions(options); + ive::DataOutputStream out(&fout, options); out.writeImage(ive::IMAGE_INCLUDE_DATA, const_cast(&image)); if (fout.fail()) return WriteResult::ERROR_IN_WRITING_FILE; return WriteResult::FILE_SAVED; @@ -179,9 +176,7 @@ class ReaderWriterIVE : public ReaderWriter { try { - ive::DataOutputStream out(&fout); - - out.setOptions(options); + ive::DataOutputStream out(&fout, options); out.writeNode(const_cast(&node));