#include #include #include #include #include "OSGA_Archive.h" using namespace osgDB; /* Functions to convert between std::streampos ( typedef'ed as iostream::pos_type used as seekp,seekg argument and tellp,tellg return value ) and OSGA_Archive::pos_type (64 bit file position index) Purpose: To allow using OSGA files larger than 4GiB in Windows. std::streampos is used as argument to iostreams seekp and seekg methods and is returned as result from iostream tellp and tellg methods. std::streampos can be implicitly cast from/to std::streamoff as std::streampos class defines appropriate constructor and cast operator. Since std::streamoff is usually defined as simple int, it is possible to call seekp( ), seekg( ) with int argument and assign tellp(), tellg() result to int type. But this implicit methods fail when std::streamoff is 32 bit and std::streampos actually points past 32 bit addressable range (2 GiB). Even if std::streamoff is 32 bit and incapable of representing 64 bit file positions, original std::streampos may be perfectly able to handle them. But, when such situation occurs more elaborate conversion methods from/to std::streampos are needed. Functions below employ such methods. I made this fix for use with 32 bit Windows OSG. Actually this solution is not dependent on OS but standard C++ library. Microsoft SDKs always use some version of Dinkumware libs. Practically this tweak is made for Dinkumware C++ libs. I hope it can be easily extended to other 32bit systems supporting 64bit files, provided their std::streampos implementations are similar. I based my solution on a small portion of boost iostreams code. For additional reference look at: http://boost.org/boost/iostreams/positioning.hpp */ /* Recognize Dinkumware std C++ lib implementation. Its used by Microsoft, but method is more generic - should work in all Dinkumware environments. Complex condition below was taken from http://boost.org/boost/iostreams/positioning.hpp Great thanks to J.Tukanis and G. Sylvester-Bradley for figuring it out. */ #if ((defined(_YVALS) && !defined(__IBMCPP__)) || defined(_CPPLIB_VER)) && \ !defined(__SGI_STL_PORT) && !defined(_STLPORT_VERSION) \ && !defined(__QNX__) inline std::streampos STREAM_POS( const OSGA_Archive::pos_type pos ) { return std::streampos( std::mbstate_t(), pos ); } inline OSGA_Archive::pos_type ARCHIVE_POS( const std::streampos & pos ) { #if (defined(_CPPLIB_VER) && defined(_MSC_VER) && _MSC_VER > 1914) // VC++ 2017 version 15.8 or later fpos_t position = pos; #elif (defined(_CPPLIB_VER) && defined(_MSC_VER)) // Dinkumware (eg: one included with VC++ 2003, 2005...) fpos_t position = pos.seekpos(); #else // older Dinkumware (eg: one included in Win Server 2003 Platform SDK ) fpos_t position = pos.get_fpos_t(); #endif std::streamoff offset = pos.operator std::streamoff( ) - _FPOSOFF( position ); return OSGA_Archive::pos_type( position + offset ); } #else // non Dinkumware std C++ lib implementations // do the old school streampos <-> streamoff casts inline std::streampos STREAM_POS( const OSGA_Archive::pos_type pos ) { return std::streampos( pos ); } inline OSGA_Archive::pos_type ARCHIVE_POS( const std::streampos & pos ) { return OSGA_Archive::pos_type( pos ); } #endif // Dinkumware std C++ lib //////////////////////////////////////////////////////////////////////////////// float OSGA_Archive::s_currentSupportedVersion = 0.0; const unsigned int ENDIAN_TEST_NUMBER = 0x00000001; OSGA_Archive::IndexBlock::IndexBlock(unsigned int blockSize): _requiresWrite(false), _filePosition(0), _blockSize(0), _filePositionNextIndexBlock(0), _offsetOfNextAvailableSpace(0), _data(0) { allocateData(blockSize); } OSGA_Archive::IndexBlock::~IndexBlock() { delete [] _data; } void OSGA_Archive::IndexBlock::allocateData(unsigned int blockSize) { _data = (blockSize!=0) ? new char[blockSize] : 0; if (_data) { _blockSize = blockSize; // initialize the array char* end = _data + _blockSize; for(char* ptr=_data; ptr < end; ++ptr) *ptr = 0; } else { _blockSize = 0; } } OSGA_Archive::IndexBlock* OSGA_Archive::IndexBlock::read(std::istream& in, bool doEndianSwap) { if (!in) return 0; osg::ref_ptr indexBlock = new IndexBlock; indexBlock->_filePosition = ARCHIVE_POS( in.tellg() ); in.read(reinterpret_cast(&indexBlock->_blockSize), sizeof(indexBlock->_blockSize)); in.read(reinterpret_cast(&indexBlock->_filePositionNextIndexBlock), sizeof(indexBlock->_filePositionNextIndexBlock)); in.read(reinterpret_cast(&indexBlock->_offsetOfNextAvailableSpace), sizeof(indexBlock-> _offsetOfNextAvailableSpace)); if (doEndianSwap) { osg::swapBytes(reinterpret_cast(&indexBlock->_blockSize), sizeof(indexBlock->_blockSize)); osg::swapBytes(reinterpret_cast(&indexBlock->_filePositionNextIndexBlock), sizeof(indexBlock->_filePositionNextIndexBlock)); osg::swapBytes(reinterpret_cast(&indexBlock->_offsetOfNextAvailableSpace), sizeof(indexBlock-> _offsetOfNextAvailableSpace)); } // OSG_INFO<<"indexBlock->_blockSize="<_blockSize<_filePositionNextIndexBlock="<_filePositionNextIndexBlock<_offsetOfNextAvailableSpace="<_offsetOfNextAvailableSpace<allocateData(indexBlock->_blockSize); if (indexBlock->_data) { in.read(reinterpret_cast(indexBlock->_data),indexBlock->_blockSize); if (doEndianSwap) { char* ptr = indexBlock->_data; char* end_ptr = indexBlock->_data + indexBlock->_offsetOfNextAvailableSpace; while (ptr(ptr)); _read(ptr, filename_size); ptr += sizeof(unsigned int); ptr += filename_size; OSG_INFO<<"filename size="<(ptr)); _read(ptr, filename_size); ptr += sizeof(unsigned int); return std::string(ptr, ptr+filename_size); } else { return std::string(); } } bool OSGA_Archive::IndexBlock::getFileReferences(FileNamePositionMap& indexMap) const { if (!_data || _offsetOfNextAvailableSpace==0) return false; bool valuesAdded = false; char* ptr = _data; char* end_ptr = _data + _offsetOfNextAvailableSpace; while (ptr(ptr)); _read(ptr, position); ptr += sizeof(pos_type); size_type size; // = *(reinterpret_cast(ptr)); _read(ptr, size); ptr += sizeof(size_type); unsigned int filename_size; // = *(reinterpret_cast(ptr)); _read(ptr, filename_size); ptr += sizeof(unsigned int); std::string filename(ptr, ptr+filename_size); // record this entry into the FileNamePositionMap. // Requests for files will be in unix style even on Win32 so need unix style keys in map. indexMap[osgDB::convertFileNameToUnixStyle(filename)] = PositionSizePair(position,size); ptr += filename_size; valuesAdded = true; } return valuesAdded; } void OSGA_Archive::IndexBlock::write(std::ostream& out) { pos_type currentPos = ARCHIVE_POS( out.tellp() ); if (_filePosition==pos_type(0)) { OSG_INFO<<"OSGA_Archive::IndexBlock::write() setting _filePosition"<(&_blockSize), sizeof(_blockSize)); out.write(reinterpret_cast(&_filePositionNextIndexBlock), sizeof(_filePositionNextIndexBlock)); out.write(reinterpret_cast(&_offsetOfNextAvailableSpace), sizeof(_offsetOfNextAvailableSpace)); out.write(reinterpret_cast(_data),_blockSize); if( _filePosition < currentPos ) // move file ptr to the end of file out.seekp( STREAM_POS( currentPos ) ); OSG_INFO<<"OSGA_Archive::IndexBlock::write() end"<(ptr)) = position; _write(ptr, position); ptr += sizeof(pos_type); //*(reinterpret_cast(ptr)) = size; _write(ptr, size); ptr += sizeof(size_type); //*(reinterpret_cast(ptr)) = filename.size(); _write(ptr, static_cast(filename.size())); ptr += sizeof(unsigned int); for(unsigned int i=0;i 4 GiB size_t BlockHeaderSize = sizeof( unsigned int /*_blockSize*/ ) + sizeof( pos_type /*_filePositionNextIndexBlock*/ ) + sizeof( unsigned int /*_offsetOfNextAvailableSpace*/ ); for(IndexBlockList::iterator itr=_indexBlockList.begin(); itr!=_indexBlockList.end(); ++itr) { pos_type end = (*itr)->getPosition() + BlockHeaderSize + (*itr)->getBlockSize(); if( file_size < end ) file_size = end; } for(FileNamePositionMap::iterator mitr=_indexMap.begin(); mitr!=_indexMap.end(); ++mitr) { pos_type end = mitr->second.first + mitr->second.second; if( file_size < end ) file_size = end; } } _input.close(); _status = WRITE; osgDB::open(_output, filename.c_str(), std::ios_base::binary | std::ios_base::in | std::ios_base::out); OSG_INFO<<"File position after open = "<(&ENDIAN_TEST_NUMBER),4); _output.write(reinterpret_cast(&s_currentSupportedVersion),sizeof(float)); IndexBlock *indexBlock = new IndexBlock(indexBlockSize); if (indexBlock) { indexBlock->write(_output); _indexBlockList.push_back(indexBlock); } OSG_INFO<<"File position after write = "<(_input).rdbuf(fin.rdbuf()); return _open(_input); } bool OSGA_Archive::_open(std::istream& input) { if (input) { char identifier[4]; input.read(identifier,4); bool validArchive = (identifier[0]=='o' && identifier[1]=='s' && identifier[2]=='g' && identifier[3]=='a'); if (validArchive) { unsigned int endianTestWord=0; input.read(reinterpret_cast(&endianTestWord),4); bool doEndianSwap = (endianTestWord!=ENDIAN_TEST_NUMBER); input.read(reinterpret_cast(&_version),sizeof(_version)); if (doEndianSwap) { osg::swapBytes(reinterpret_cast(&_version),sizeof(_version)); } OSG_INFO<<"OSGA_Archive::open() doEndianSwap="<first)<<" pos="<<(int)((mitr->second).first)<<" size="<<(int)((mitr->second).second)<first); } return !fileNameList.empty(); } void OSGA_Archive::writeIndexBlocks() { SERIALIZER(); if (_status==WRITE) { for(IndexBlockList::iterator itr=_indexBlockList.begin(); itr!=_indexBlockList.end(); ++itr) { if ((*itr)->requiresWrite()) { (*itr)->write(_output); } } } } bool OSGA_Archive::fileExists(const std::string& filename) const { return (_indexMap.count(filename)!=0); } bool OSGA_Archive::addFileReference(pos_type position, size_type size, const std::string& fileName) { SERIALIZER(); if (_status==READ) { OSG_INFO<<"OSGA_Archive::getPositionForNewEntry("< indexBlock = _indexBlockList.empty() ? 0 : _indexBlockList.back().get(); osg::ref_ptr previousBlock = indexBlock; if (indexBlock.valid()) { blockSize = indexBlock->getBlockSize(); if (!(indexBlock->spaceAvailable(position, size, fileName))) { previousBlock = indexBlock; indexBlock = 0; } } // if not one available create a new block. if (!indexBlock) { if (previousBlock.valid()) previousBlock->setPositionNextIndexBlock( ARCHIVE_POS( _output.tellp() ) ); indexBlock = new IndexBlock(blockSize); indexBlock->write(_output); _indexBlockList.push_back(indexBlock.get()); } if (indexBlock.valid()) { return indexBlock->addFileReference(position, size, fileName); } return false; } // streambuffer class to give access to a portion of the archive stream, for numChars onwards // from the current position in the archive. class proxy_streambuf : public std::streambuf { public: proxy_streambuf(std::streambuf* streambuf, std::streamoff numChars): _streambuf(streambuf),_oneChar(0), _curPos(0),_numChars(numChars) { _startPos = ARCHIVE_POS(_streambuf->pubseekoff(0, std::ios_base::cur, std::ios_base::in)); setg(&_oneChar, (&_oneChar)+1, (&_oneChar)+1); } // Destructor deallocates no buffer space. virtual ~proxy_streambuf() {} std::streambuf* _streambuf; // Underlying archive stream protected: char_type _oneChar; // Single character buffer std::streamoff _curPos, _numChars; OSGA_Archive::pos_type _startPos; // Set internal position pointer to relative position. Virtual function called by the public // member function pubseekoff to alter the stream position. virtual std::streampos seekoff (std::streamoff off, std::ios_base::seekdir way, std::ios_base::openmode which = std::ios_base::in) { std::streamoff newpos; if ( way == std::ios_base::beg ) { newpos = off; } else if ( way == std::ios_base::cur ) { newpos = _curPos + off; } else if ( way == std::ios_base::end ) { newpos = _numChars + off; } else { return -1; } if ( newpos<0 || newpos>_numChars ) return -1; if ( ARCHIVE_POS(_streambuf->pubseekpos( STREAM_POS(_startPos+newpos), which)) < 0 ) return -1; _curPos = newpos; return _curPos; } // Set internal position pointer to absolute position. Virtual function called by the public // member function pubseekpos to alter the stream positions virtual std::streampos seekpos (std::streampos sp, std::ios_base::openmode which = std::ios_base::in) { return seekoff(sp, std::ios_base::beg, which); } // Virtual function called by other member functions to get the current character. It is called // by streambuf public member functions such as sgetc to request a new character when there are // no read positions available at the get pointer (gptr). virtual int_type underflow() { // Return current character. if ( gptr() == &_oneChar ) return traits_type::to_int_type(_oneChar); // Get another character from the archive stream, if available. if ( _curPos==_numChars ) return traits_type::eof(); _curPos += 1; int_type next_value = _streambuf->sbumpc(); if ( !traits_type::eq_int_type(next_value,traits_type::eof()) ) { setg(&_oneChar, &_oneChar, (&_oneChar)+1); _oneChar = traits_type::to_char_type(next_value); } return next_value; } }; struct OSGA_Archive::ReadObjectFunctor : public OSGA_Archive::ReadFunctor { ReadObjectFunctor(const std::string& filename, const ReaderWriter::Options* options):ReadFunctor(filename,options) {} virtual ReaderWriter::ReadResult doRead(ReaderWriter& rw, std::istream& input) const { return rw.readObject(input, _options); } }; struct OSGA_Archive::ReadImageFunctor : public OSGA_Archive::ReadFunctor { ReadImageFunctor(const std::string& filename, const ReaderWriter::Options* options):ReadFunctor(filename,options) {} virtual ReaderWriter::ReadResult doRead(ReaderWriter& rw, std::istream& input)const { return rw.readImage(input, _options); } }; struct OSGA_Archive::ReadHeightFieldFunctor : public OSGA_Archive::ReadFunctor { ReadHeightFieldFunctor(const std::string& filename, const ReaderWriter::Options* options):ReadFunctor(filename,options) {} virtual ReaderWriter::ReadResult doRead(ReaderWriter& rw, std::istream& input) const { return rw.readHeightField(input, _options); } }; struct OSGA_Archive::ReadNodeFunctor : public OSGA_Archive::ReadFunctor { ReadNodeFunctor(const std::string& filename, const ReaderWriter::Options* options):ReadFunctor(filename,options) {} virtual ReaderWriter::ReadResult doRead(ReaderWriter& rw, std::istream& input) const { return rw.readNode(input, _options); } }; struct OSGA_Archive::ReadShaderFunctor : public OSGA_Archive::ReadFunctor { ReadShaderFunctor(const std::string& filename, const ReaderWriter::Options* options):ReadFunctor(filename,options) {} virtual ReaderWriter::ReadResult doRead(ReaderWriter& rw, std::istream& input) const { return rw.readShader(input, _options); } }; ReaderWriter::ReadResult OSGA_Archive::read(const ReadFunctor& readFunctor) { SERIALIZER(); if (_status!=READ) { OSG_INFO<<"OSGA_Archive::readObject(obj, "<getReaderWriterForExtension(getLowerCaseFileExtension(readFunctor._filename)); if (!rw) { OSG_INFO<<"OSGA_Archive::readObject(obj, "<second.first ) ); // set up proxy stream buffer to provide the faked ending. std::istream& ins = _input; proxy_streambuf mystreambuf(ins.rdbuf(),itr->second.second); ins.rdbuf(&mystreambuf); ReaderWriter::ReadResult result = readFunctor.doRead(*rw, _input); ins.rdbuf(mystreambuf._streambuf); return result; } ReaderWriter::ReadResult OSGA_Archive::readObject(const std::string& fileName,const Options* options) const { return const_cast(this)->read(ReadObjectFunctor(fileName, options)); } ReaderWriter::ReadResult OSGA_Archive::readImage(const std::string& fileName,const Options* options) const { return const_cast(this)->read(ReadImageFunctor(fileName, options)); } ReaderWriter::ReadResult OSGA_Archive::readHeightField(const std::string& fileName,const Options* options) const { return const_cast(this)->read(ReadHeightFieldFunctor(fileName, options)); } ReaderWriter::ReadResult OSGA_Archive::readNode(const std::string& fileName,const Options* options) const { return const_cast(this)->read(ReadNodeFunctor(fileName, options)); } ReaderWriter::ReadResult OSGA_Archive::readShader(const std::string& fileName,const Options* options) const { return const_cast(this)->read(ReadShaderFunctor(fileName, options)); } struct OSGA_Archive::WriteObjectFunctor : public OSGA_Archive::WriteFunctor { WriteObjectFunctor(const osg::Object& object, const std::string& filename, const ReaderWriter::Options* options): WriteFunctor(filename,options), _object(object) {} const osg::Object& _object; virtual ReaderWriter::WriteResult doWrite(ReaderWriter& rw, std::ostream& output) const { return rw.writeObject(_object, output, _options); } }; struct OSGA_Archive::WriteImageFunctor : public OSGA_Archive::WriteFunctor { WriteImageFunctor(const osg::Image& object, const std::string& filename, const ReaderWriter::Options* options): WriteFunctor(filename,options), _object(object) {} const osg::Image& _object; virtual ReaderWriter::WriteResult doWrite(ReaderWriter& rw, std::ostream& output) const { OSG_NOTICE<<"doWrite() rw.writeImage(), "<getReaderWriterForExtension(getLowerCaseFileExtension(writeFunctor._filename)); if (!rw) { OSG_INFO<<"OSGA_Archive::write(obj, "<(this)->write(WriteObjectFunctor(obj, fileName, options)); } ReaderWriter::WriteResult OSGA_Archive::writeImage(const osg::Image& image,const std::string& fileName,const Options* options) const { OSG_INFO<<"OSGA_Archive::writeImage(obj, "<(this)->write(WriteImageFunctor(image, fileName, options)); } ReaderWriter::WriteResult OSGA_Archive::writeHeightField(const osg::HeightField& heightField,const std::string& fileName,const Options* options) const { OSG_INFO<<"OSGA_Archive::writeHeightField(obj, "<(this)->write(WriteHeightFieldFunctor(heightField, fileName, options)); } ReaderWriter::WriteResult OSGA_Archive::writeNode(const osg::Node& node,const std::string& fileName,const Options* options) const { OSG_INFO<<"OSGA_Archive::writeNode(obj, "<(this)->write(WriteNodeFunctor(node, fileName, options)); } ReaderWriter::WriteResult OSGA_Archive::writeShader(const osg::Shader& shader,const std::string& fileName,const Options* options) const { OSG_INFO<<"OSGA_Archive::writeShader(obj, "<(this)->write(WriteShaderFunctor(shader, fileName, options)); }