#include #include #include #include #include "OSGA_Archive.h" using namespace osgDB; /* Functions to convert between std::streampos ( typedef'ed as iostream::pos_type used as seekp,seekg argument and tellp,tellg return value ) and OSGA_Archive::pos_type (64 bit file position index) Purpose: To allow using OSGA files larger than 4GiB in Windows. std::streampos is used as argument to iostreams seekp and seekg methods and is returned as result from iostream tellp and tellg methods. std::streampos can be implicitly cast from/to std::streamoff as std::streampos class defines appropriate constructor and cast operator. Since std::streamoff is usually defined as simple int, it is possible to call seekp( ), seekg( ) with int argument and assign tellp(), tellg() result to int type. But this implicit methods fail when std::streamoff is 32 bit and std::streampos actually points past 32 bit addressable range (2 GiB). Even if std::streamoff is 32 bit and incapable of representing 64 bit file positions, original std::streampos may be prefectly able to handle them. But, when such situation occurs more elaborate conversion methods from/to std::streampos are needed. Functions below employ such methods. I made this fix for use with 32 bit Windows OSG. Acutally this solution is not dependent on OS but standard C++ library. Microsoft SDKs always use some version of Dinkumware libs. Practically this tweak is made for Dinkumware C++ libs. I hope it can be easily extended to other 32bit systems supporting 64bit files, provided their std::streampos implementations are similar. I based my solution on a small portion of boost iostreams code. For additional reference look at: http://boost.org/boost/iostreams/positioning.hpp */ /* Recognize Dinkumware std C++ lib implementation. Its used by Microsoft, but method is more generic - should work in all Dinkumware environments. Complex condition below was taken from http://boost.org/boost/iostreams/positioning.hpp Great thanks to J.Tukanis and G. Sylvester-Bradley for figuring it out. */ #if ((defined(_YVALS) && !defined(__IBMCPP__)) || defined(_CPPLIB_VER)) && \ !defined(__SGI_STL_PORT) && !defined(_STLPORT_VERSION) \ && !defined(__QNX__) inline std::streampos STREAM_POS( const OSGA_Archive::pos_type pos ) { return std::streampos( std::mbstate_t(), pos ); } inline OSGA_Archive::pos_type ARCHIVE_POS( const std::streampos & pos ) { #if defined(_CPPLIB_VER)//newer Dinkumware(eg: one included with VC++ 2003,2005) fpos_t position = pos.seekpos(); #else // older Dinkumware (eg: one included in Win Server 2003 Platform SDK ) fpos_t position = pos.get_fpos_t(); #endif std::streamoff offset = pos.operator std::streamoff( ) - _FPOSOFF( position ); return OSGA_Archive::pos_type( position + offset ); } #else // non Dinkumware std C++ lib implementations // do the old school streampos <-> streamoff casts inline std::streampos STREAM_POS( const OSGA_Archive::pos_type pos ) { return std::streampos( pos ); } inline OSGA_Archive::pos_type ARCHIVE_POS( const std::streampos & pos ) { return OSGA_Archive::pos_type( pos ); } #endif // Dinkumware std C++ lib //////////////////////////////////////////////////////////////////////////////// float OSGA_Archive::s_currentSupportedVersion = 0.0; const unsigned int ENDIAN_TEST_NUMBER = 0x00000001; OSGA_Archive::IndexBlock::IndexBlock(unsigned int blockSize): _requiresWrite(false), _filePosition(0), _blockSize(0), _filePositionNextIndexBlock(0), _offsetOfNextAvailableSpace(0), _data(0) { allocateData(blockSize); } OSGA_Archive::IndexBlock::~IndexBlock() { delete [] _data; } void OSGA_Archive::IndexBlock::allocateData(unsigned int blockSize) { _data = (blockSize!=0) ? new char[blockSize] : 0; if (_data) { _blockSize = blockSize; // initialize the array char* end = _data + _blockSize; for(char* ptr=_data; ptr < end; ++ptr) *ptr = 0; } else { _blockSize = 0; } } OSGA_Archive::IndexBlock* OSGA_Archive::IndexBlock::read(std::istream& in, bool doEndianSwap) { if (!in) return 0; osg::ref_ptr indexBlock = new IndexBlock; indexBlock->_filePosition = ARCHIVE_POS( in.tellg() ); in.read(reinterpret_cast(&indexBlock->_blockSize), sizeof(indexBlock->_blockSize)); in.read(reinterpret_cast(&indexBlock->_filePositionNextIndexBlock), sizeof(indexBlock->_filePositionNextIndexBlock)); in.read(reinterpret_cast(&indexBlock->_offsetOfNextAvailableSpace), sizeof(indexBlock-> _offsetOfNextAvailableSpace)); if (doEndianSwap) { osg::swapBytes(reinterpret_cast(&indexBlock->_blockSize), sizeof(indexBlock->_blockSize)); osg::swapBytes(reinterpret_cast(&indexBlock->_filePositionNextIndexBlock), sizeof(indexBlock->_filePositionNextIndexBlock)); osg::swapBytes(reinterpret_cast(&indexBlock->_offsetOfNextAvailableSpace), sizeof(indexBlock-> _offsetOfNextAvailableSpace)); } // osg::notify(osg::INFO)<<"indexBlock->_blockSize="<_blockSize<_filePositionNextIndexBlock="<_filePositionNextIndexBlock<_offsetOfNextAvailableSpace="<_offsetOfNextAvailableSpace<allocateData(indexBlock->_blockSize); if (indexBlock->_data) { in.read(reinterpret_cast(indexBlock->_data),indexBlock->_blockSize); if (doEndianSwap) { char* ptr = indexBlock->_data; char* end_ptr = indexBlock->_data + indexBlock->_offsetOfNextAvailableSpace; while (ptr(ptr)); _read(ptr, filename_size); ptr += sizeof(unsigned int); ptr += filename_size; osg::notify(osg::INFO)<<"filename size="<(ptr)); _read(ptr, filename_size); ptr += sizeof(unsigned int); return std::string(ptr, ptr+filename_size); } else { return std::string(); } } bool OSGA_Archive::IndexBlock::getFileReferences(FileNamePositionMap& indexMap) const { if (!_data || _offsetOfNextAvailableSpace==0) return false; bool valuesAdded = false; char* ptr = _data; char* end_ptr = _data + _offsetOfNextAvailableSpace; while (ptr(ptr)); _read(ptr, position); ptr += sizeof(pos_type); size_type size; // = *(reinterpret_cast(ptr)); _read(ptr, size); ptr += sizeof(size_type); unsigned int filename_size; // = *(reinterpret_cast(ptr)); _read(ptr, filename_size); ptr += sizeof(unsigned int); std::string filename(ptr, ptr+filename_size); // record this entry into the FileNamePositionMap. // Requests for files will be in unix style even on Win32 so need unix style keys in map. indexMap[osgDB::convertFileNameToUnixStyle(filename)] = PositionSizePair(position,size); ptr += filename_size; valuesAdded = true; } return valuesAdded; } void OSGA_Archive::IndexBlock::write(std::ostream& out) { pos_type currentPos = ARCHIVE_POS( out.tellp() ); if (_filePosition==pos_type(0)) { osg::notify(osg::INFO)<<"OSGA_Archive::IndexBlock::write() setting _filePosition"<(&_blockSize), sizeof(_blockSize)); out.write(reinterpret_cast(&_filePositionNextIndexBlock), sizeof(_filePositionNextIndexBlock)); out.write(reinterpret_cast(&_offsetOfNextAvailableSpace), sizeof(_offsetOfNextAvailableSpace)); out.write(reinterpret_cast(_data),_blockSize); if( _filePosition < currentPos ) // move file ptr to the end of file out.seekp( STREAM_POS( currentPos ) ); osg::notify(osg::INFO)<<"OSGA_Archive::IndexBlock::write() end"<(ptr)) = position; _write(ptr, position); ptr += sizeof(pos_type); //*(reinterpret_cast(ptr)) = size; _write(ptr, size); ptr += sizeof(size_type); //*(reinterpret_cast(ptr)) = filename.size(); _write(ptr, static_cast(filename.size())); ptr += sizeof(unsigned int); for(unsigned int i=0;i 4 GiB size_t BlockHeaderSize = sizeof( unsigned int /*_blockSize*/ ) + sizeof( pos_type /*_filePositionNextIndexBlock*/ ) + sizeof( unsigned int /*_offsetOfNextAvailableSpace*/ ); for(IndexBlockList::iterator itr=_indexBlockList.begin(); itr!=_indexBlockList.end(); ++itr) { pos_type end = (*itr)->getPosition() + BlockHeaderSize + (*itr)->getBlockSize(); if( file_size < end ) file_size = end; } for(FileNamePositionMap::iterator mitr=_indexMap.begin(); mitr!=_indexMap.end(); ++mitr) { pos_type end = mitr->second.first + mitr->second.second; if( file_size < end ) file_size = end; } } _input.close(); _status = WRITE; _output.open(filename.c_str(), std::ios_base::binary | std::ios_base::in | std::ios_base::out); osg::notify(osg::INFO)<<"File position after open = "<(&ENDIAN_TEST_NUMBER),4); _output.write(reinterpret_cast(&s_currentSupportedVersion),sizeof(float)); IndexBlock *indexBlock = new IndexBlock(indexBlockSize); if (indexBlock) { indexBlock->write(_output); _indexBlockList.push_back(indexBlock); } osg::notify(osg::INFO)<<"File position after write = "<(_input).rdbuf(fin.rdbuf()); return _open(_input); } bool OSGA_Archive::_open(std::istream& input) { if (input) { char identifier[4]; input.read(identifier,4); bool validArchive = (identifier[0]=='o' && identifier[1]=='s' && identifier[2]=='g' && identifier[3]=='a'); if (validArchive) { unsigned int endianTestWord=0; input.read(reinterpret_cast(&endianTestWord),4); bool doEndianSwap = (endianTestWord!=ENDIAN_TEST_NUMBER); input.read(reinterpret_cast(&_version),sizeof(_version)); if (doEndianSwap) { osg::swapBytes(reinterpret_cast(&_version),sizeof(_version)); } osg::notify(osg::INFO)<<"OSGA_Archive::open() doEndianSwap="<first)<<" pos="<<(int)((mitr->second).first)<<" size="<<(int)((mitr->second).second)<first); } return !fileNameList.empty(); } void OSGA_Archive::writeIndexBlocks() { SERIALIZER(); if (_status==WRITE) { for(IndexBlockList::iterator itr=_indexBlockList.begin(); itr!=_indexBlockList.end(); ++itr) { if ((*itr)->requiresWrite()) { (*itr)->write(_output); } } } } bool OSGA_Archive::fileExists(const std::string& filename) const { return (_indexMap.count(filename)!=0); } bool OSGA_Archive::addFileReference(pos_type position, size_type size, const std::string& fileName) { SERIALIZER(); if (_status==READ) { osg::notify(osg::INFO)<<"OSGA_Archive::getPositionForNewEntry("< indexBlock = _indexBlockList.empty() ? 0 : _indexBlockList.back(); osg::ref_ptr previousBlock = indexBlock; if (indexBlock.valid()) { blockSize = indexBlock->getBlockSize(); if (!(indexBlock->spaceAvailable(position, size, fileName))) { previousBlock = indexBlock; indexBlock = 0; } } // if not one available create a new block. if (!indexBlock) { if (previousBlock.valid()) previousBlock->setPositionNextIndexBlock( ARCHIVE_POS( _output.tellp() ) ); indexBlock = new IndexBlock(blockSize); indexBlock->write(_output); _indexBlockList.push_back(indexBlock.get()); } if (indexBlock.valid()) { return indexBlock->addFileReference(position, size, fileName); } return false; } class proxy_streambuf : public std::streambuf { public: proxy_streambuf(std::streambuf* streambuf, unsigned int numChars): _streambuf(streambuf), _numChars(numChars) { setg(&oneChar, (&oneChar)+1, (&oneChar)+1); } /// Destructor deallocates no buffer space. virtual ~proxy_streambuf() {} std::streambuf* _streambuf; protected: unsigned int _numChars; char_type oneChar; virtual int_type underflow() { if ( gptr() == &oneChar ) return traits_type::to_int_type(oneChar); if ( _numChars==0 ) return traits_type::eof(); --_numChars; int_type next_value = _streambuf->sbumpc(); if ( !traits_type::eq_int_type(next_value,traits_type::eof()) ) { setg(&oneChar, &oneChar, (&oneChar)+1); oneChar = traits_type::to_char_type(next_value); } return next_value; } }; struct OSGA_Archive::ReadObjectFunctor : public OSGA_Archive::ReadFunctor { ReadObjectFunctor(const std::string& filename, const ReaderWriter::Options* options):ReadFunctor(filename,options) {} virtual ReaderWriter::ReadResult doRead(ReaderWriter& rw, std::istream& input) const { return rw.readObject(input, _options); } }; struct OSGA_Archive::ReadImageFunctor : public OSGA_Archive::ReadFunctor { ReadImageFunctor(const std::string& filename, const ReaderWriter::Options* options):ReadFunctor(filename,options) {} virtual ReaderWriter::ReadResult doRead(ReaderWriter& rw, std::istream& input)const { return rw.readImage(input, _options); } }; struct OSGA_Archive::ReadHeightFieldFunctor : public OSGA_Archive::ReadFunctor { ReadHeightFieldFunctor(const std::string& filename, const ReaderWriter::Options* options):ReadFunctor(filename,options) {} virtual ReaderWriter::ReadResult doRead(ReaderWriter& rw, std::istream& input) const { return rw.readHeightField(input, _options); } }; struct OSGA_Archive::ReadNodeFunctor : public OSGA_Archive::ReadFunctor { ReadNodeFunctor(const std::string& filename, const ReaderWriter::Options* options):ReadFunctor(filename,options) {} virtual ReaderWriter::ReadResult doRead(ReaderWriter& rw, std::istream& input) const { return rw.readNode(input, _options); } }; ReaderWriter::ReadResult OSGA_Archive::read(const ReadFunctor& readFunctor) { SERIALIZER(); if (_status!=READ) { osg::notify(osg::INFO)<<"OSGA_Archive::readObject(obj, "<getReaderWriterForExtension(getLowerCaseFileExtension(readFunctor._filename)); if (!rw) { osg::notify(osg::INFO)<<"OSGA_Archive::readObject(obj, "<second.first ) ); // set up proxy stream buffer to provide the faked ending. std::istream& ins = _input; proxy_streambuf mystreambuf(ins.rdbuf(),itr->second.second); ins.rdbuf(&mystreambuf); ReaderWriter::ReadResult result = readFunctor.doRead(*rw, _input); ins.rdbuf(mystreambuf._streambuf); return result; } ReaderWriter::ReadResult OSGA_Archive::readObject(const std::string& fileName,const Options* options) const { return const_cast(this)->read(ReadObjectFunctor(fileName, options)); } ReaderWriter::ReadResult OSGA_Archive::readImage(const std::string& fileName,const Options* options) const { return const_cast(this)->read(ReadImageFunctor(fileName, options)); } ReaderWriter::ReadResult OSGA_Archive::readHeightField(const std::string& fileName,const Options* options) const { return const_cast(this)->read(ReadHeightFieldFunctor(fileName, options)); } ReaderWriter::ReadResult OSGA_Archive::readNode(const std::string& fileName,const Options* options) const { return const_cast(this)->read(ReadNodeFunctor(fileName, options)); } struct OSGA_Archive::WriteObjectFunctor : public OSGA_Archive::WriteFunctor { WriteObjectFunctor(const osg::Object& object, const std::string& filename, const ReaderWriter::Options* options): WriteFunctor(filename,options), _object(object) {} const osg::Object& _object; virtual ReaderWriter::WriteResult doWrite(ReaderWriter& rw, std::ostream& output) const { return rw.writeObject(_object, output, _options); } }; struct OSGA_Archive::WriteImageFunctor : public OSGA_Archive::WriteFunctor { WriteImageFunctor(const osg::Image& object, const std::string& filename, const ReaderWriter::Options* options): WriteFunctor(filename,options), _object(object) {} const osg::Image& _object; virtual ReaderWriter::WriteResult doWrite(ReaderWriter& rw, std::ostream& output) const { return rw.writeImage(_object, output, _options); } }; struct OSGA_Archive::WriteHeightFieldFunctor : public OSGA_Archive::WriteFunctor { WriteHeightFieldFunctor(const osg::HeightField& object, const std::string& filename, const ReaderWriter::Options* options): WriteFunctor(filename,options), _object(object) {} const osg::HeightField& _object; virtual ReaderWriter::WriteResult doWrite(ReaderWriter& rw, std::ostream& output) const { return rw.writeHeightField(_object, output, _options); } }; struct OSGA_Archive::WriteNodeFunctor : public OSGA_Archive::WriteFunctor { WriteNodeFunctor(const osg::Node& object, const std::string& filename, const ReaderWriter::Options* options): WriteFunctor(filename,options), _object(object) {} const osg::Node& _object; virtual ReaderWriter::WriteResult doWrite(ReaderWriter& rw, std::ostream& output) const { return rw.writeNode(_object, output, _options); } }; ReaderWriter::WriteResult OSGA_Archive::write(const WriteFunctor& writeFunctor) { SERIALIZER(); if (_status!=WRITE) { osg::notify(osg::INFO)<<"OSGA_Archive::write(obj, "<getReaderWriterForExtension(getLowerCaseFileExtension(writeFunctor._filename)); if (!rw) { osg::notify(osg::INFO)<<"OSGA_Archive::write(obj, "<(this)->write(WriteObjectFunctor(obj, fileName, options)); } ReaderWriter::WriteResult OSGA_Archive::writeImage(const osg::Image& image,const std::string& fileName,const Options* options) const { osg::notify(osg::INFO)<<"OSGA_Archive::writeImage(obj, "<(this)->write(WriteImageFunctor(image, fileName, options)); } ReaderWriter::WriteResult OSGA_Archive::writeHeightField(const osg::HeightField& heightField,const std::string& fileName,const Options* options) const { osg::notify(osg::INFO)<<"OSGA_Archive::writeHeightField(obj, "<(this)->write(WriteHeightFieldFunctor(heightField, fileName, options)); } ReaderWriter::WriteResult OSGA_Archive::writeNode(const osg::Node& node,const std::string& fileName,const Options* options) const { osg::notify(osg::INFO)<<"OSGA_Archive::writeNode(obj, "<(this)->write(WriteNodeFunctor(node, fileName, options)); }