FGFS-4.1/OpenSceneGraph
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implemented updating of revision files as new data is writing to the FileCache

Browse Source
This commit is contained in:
Robert Osfield
2009-06-04 14:07:12 +00:00
parent cf976e956d
commit 40155d59b4
7 changed files with 1690 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1379
ChangeLog
View File

File diff suppressed because it is too large Load Diff

17
examples/osgdatabaserevisions/osgdatabaserevisions.cpp
View File

@@ -66,19 +66,6 @@ int main(int argc, char** argv)
}
#endif
std::string url, username, password;
while(arguments.read("--login",url, username, password))
{
if (!osgDB::Registry::instance()->getAuthenticationMap())
{
osgDB::Registry::instance()->setAuthenticationMap(new osgDB::AuthenticationMap);
osgDB::Registry::instance()->getAuthenticationMap()->addAuthenticationDetails(
url,
new osgDB::AuthenticationDetails(username, password)
);
}
}
// set up the camera manipulators.
{
osg::ref_ptr<osgGA::KeySwitchMatrixManipulator> keyswitchManipulator = new osgGA::KeySwitchMatrixManipulator;
@@ -135,10 +122,12 @@ int main(int argc, char** argv)
osgDB::FileCache* fileCache = osgDB::Registry::instance()->getFileCache();
if (fileCache)
{
osg::notify(osg::NOTICE)<<"We have FileCache "<<fileCache<<std::endl;
fileCache->loadDatabaseRevisionsForFile(file);
// fileCache->loadDatabaseRevisionsForFile(file); // test to make sure that repeated loads of same revision file doesn't cause problems
}
// load the data
osg::ref_ptr<osg::Node> loadedModel = osgDB::readNodeFile(file);
if (!loadedModel)

8
include/osgDB/DatabaseRevisions
View File

@@ -37,6 +37,8 @@ class OSGDB_EXPORT FileList : public osg::Object
bool contains(const std::string& filename) const { return _files.count(filename)!=0; }
bool removeFile(const std::string& filename);
protected:
virtual ~FileList();
@@ -73,6 +75,8 @@ class OSGDB_EXPORT DatabaseRevision : public osg::Object
bool isFileBlackListed(const std::string& filename) const;
bool removeFile(const std::string& filename);
protected:
virtual ~DatabaseRevision();
@@ -101,11 +105,15 @@ class OSGDB_EXPORT DatabaseRevisions : public osg::Object
void addRevision(DatabaseRevision* revision);
void removeRevision(DatabaseRevision* revision);
DatabaseRevision* getDatabaseRevision(unsigned int i) { return i<_revisionList.size() ? _revisionList[i] : 0; }
DatabaseRevisionList& getDatabaseRevisionList() { return _revisionList; }
const DatabaseRevisionList& getDatabaseRevisionList() const { return _revisionList; }
bool isFileBlackListed(const std::string& filename) const;
bool removeFile(const std::string& filename);
protected:
virtual ~DatabaseRevisions();

3
include/osgDB/FileCache
View File

@@ -67,6 +67,9 @@ class OSGDB_EXPORT FileCache : public osg::Referenced
DatabaseRevisionsList _databaseRevisionsList;
FileList* readFileList(const std::string& originalFileName) const;
bool removeFileFromBlackListed(const std::string& originalFileName) const;
};
}

36
src/osgDB/DatabaseRevisions.cpp
View File

@@ -36,6 +36,14 @@ FileList::~FileList()
{
}
bool FileList::removeFile(const std::string& filename)
{
FileNames::iterator itr = _files.find(filename);
if (itr==_files.end()) return false;
_files.erase(itr);
return true;
}
////////////////////////////////////////////////////////////////////////////////////////////
//
@@ -59,7 +67,7 @@ DatabaseRevision::~DatabaseRevision()
bool DatabaseRevision::isFileBlackListed(const std::string& filename) const
{
osg::notify(osg::NOTICE)<<"DatabaseRevision("<<getName()<<")::isFileBlackListed("<<filename<<")"<<std::endl;
osg::notify(osg::INFO)<<"DatabaseRevision("<<getName()<<")::isFileBlackListed("<<filename<<")"<<std::endl;
if (_databasePath.length()>=filename.length()) return false;
if (filename.compare(0,_databasePath.length(), _databasePath)!=0) return false;
@@ -68,13 +76,20 @@ bool DatabaseRevision::isFileBlackListed(const std::string& filename) const
_databasePath.empty() ? 0 : _databasePath.length()+1,
std::string::npos);
osg::notify(osg::NOTICE)<<" localPath = "<<localPath<<std::endl;
return (_filesRemoved.valid() && _filesRemoved->contains(localPath)) ||
(_filesModified.valid() && _filesModified->contains(localPath));
}
bool DatabaseRevision::removeFile(const std::string& filename)
{
bool removed = false;
if (_filesAdded.valid()) removed = _filesAdded->removeFile(filename) | removed;
if (_filesRemoved.valid()) removed = _filesRemoved->removeFile(filename) | removed;
if (_filesModified.valid()) removed = _filesModified->removeFile(filename) | removed;
return removed;
}
////////////////////////////////////////////////////////////////////////////////////////////
//
// DatabaseRevisions
@@ -134,10 +149,23 @@ bool DatabaseRevisions::isFileBlackListed(const std::string& filename) const
{
if ((*itr)->isFileBlackListed(filename))
{
osg::notify(osg::NOTICE)<<"File is black listed "<<filename<<std::endl;
osg::notify(osg::INFO)<<"File is black listed "<<filename<<std::endl;
return true;
}
}
return false;
}
bool DatabaseRevisions::removeFile(const std::string& filename)
{
osg::notify(osg::INFO)<<"Remove file "<<filename<<std::endl;
bool removed = false;
for(DatabaseRevisionList::iterator itr = _revisionList.begin();
itr != _revisionList.end();
++itr)
{
removed = (*itr)->removeFile(filename) | removed;
}
return removed;
}

244
src/osgDB/FileCache.cpp
View File

@@ -87,7 +87,12 @@ ReaderWriter::WriteResult FileCache::writeObject(const osg::Object& object, cons
}
osg::notify(osg::INFO)<<"FileCache::writeObjectToCache("<<originalFileName<<") as "<<cacheFileName<<std::endl;
return osgDB::Registry::instance()->writeObject(object, cacheFileName, options);
ReaderWriter::WriteResult result = osgDB::Registry::instance()->writeObject(object, cacheFileName, options);
if (result.success())
{
removeFileFromBlackListed(originalFileName);
}
return result;
}
return ReaderWriter::WriteResult::FILE_NOT_HANDLED;
}
@@ -120,7 +125,12 @@ ReaderWriter::WriteResult FileCache::writeImage(const osg::Image& image, const s
}
osg::notify(osg::INFO)<<"FileCache::writeImageToCache("<<originalFileName<<") as "<<cacheFileName<<std::endl;
return osgDB::Registry::instance()->writeImage(image, cacheFileName, options);
ReaderWriter::WriteResult result = osgDB::Registry::instance()->writeImage(image, cacheFileName, options);
if (result.success())
{
removeFileFromBlackListed(originalFileName);
}
return result;
}
return ReaderWriter::WriteResult::FILE_NOT_HANDLED;
}
@@ -153,7 +163,12 @@ ReaderWriter::WriteResult FileCache::writeHeightField(const osg::HeightField& hf
}
osg::notify(osg::INFO)<<"FileCache::writeHeightFieldToCache("<<originalFileName<<") as "<<cacheFileName<<std::endl;
return osgDB::Registry::instance()->writeHeightField(hf, cacheFileName, options);
ReaderWriter::WriteResult result = osgDB::Registry::instance()->writeHeightField(hf, cacheFileName, options);
if (result.success())
{
removeFileFromBlackListed(originalFileName);
}
return result;
}
return ReaderWriter::WriteResult::FILE_NOT_HANDLED;
}
@@ -186,7 +201,12 @@ ReaderWriter::WriteResult FileCache::writeNode(const osg::Node& node, const std:
}
osg::notify(osg::INFO)<<"FileCache::writeNodeToCache("<<originalFileName<<") as "<<cacheFileName<<std::endl;
return osgDB::Registry::instance()->writeNode(node, cacheFileName, options);
ReaderWriter::WriteResult result = osgDB::Registry::instance()->writeNode(node, cacheFileName, options);
if (result.success())
{
removeFileFromBlackListed(originalFileName);
}
return result;
}
return ReaderWriter::WriteResult::FILE_NOT_HANDLED;
}
@@ -220,7 +240,12 @@ ReaderWriter::WriteResult FileCache::writeShader(const osg::Shader& shader, cons
}
osg::notify(osg::INFO)<<"FileCache::writeShaderToCache("<<originalFileName<<") as "<<cacheFileName<<std::endl;
return osgDB::Registry::instance()->writeShader(shader, cacheFileName, options);
ReaderWriter::WriteResult result = osgDB::Registry::instance()->writeShader(shader, cacheFileName, options);
if (result.success())
{
removeFileFromBlackListed(originalFileName);
}
return result;
}
return ReaderWriter::WriteResult::FILE_NOT_HANDLED;
}
@@ -228,7 +253,6 @@ ReaderWriter::WriteResult FileCache::writeShader(const osg::Shader& shader, cons
bool FileCache::isCachedFileBlackListed(const std::string& originalFileName) const
{
osg::notify(osg::NOTICE)<<"FileCache::isCachedFileBlackListed("<<originalFileName<<")"<<std::endl;
for(DatabaseRevisionsList::const_iterator itr = _databaseRevisionsList.begin();
itr != _databaseRevisionsList.end();
++itr)
@@ -238,22 +262,189 @@ bool FileCache::isCachedFileBlackListed(const std::string& originalFileName) con
return false;
}
bool FileCache::removeFileFromBlackListed(const std::string& originalFileName) const
{
for(DatabaseRevisionsList::const_iterator dr_itr = _databaseRevisionsList.begin();
dr_itr != _databaseRevisionsList.end();
++dr_itr)
{
DatabaseRevisions* dr = dr_itr->get();
if (dr->getDatabasePath().length()>=originalFileName.length()) continue;
if (originalFileName.compare(0,dr->getDatabasePath().length(), dr->getDatabasePath())!=0) continue;
std::string localPath(originalFileName,
dr->getDatabasePath().empty() ? 0 : dr->getDatabasePath().length()+1,
std::string::npos);
for(DatabaseRevisions::DatabaseRevisionList::const_iterator itr = dr->getDatabaseRevisionList().begin();
itr != dr->getDatabaseRevisionList().end();
++itr)
{
DatabaseRevision* revision = const_cast<DatabaseRevision*>(itr->get());
if (revision->getFilesAdded() && revision->getFilesAdded()->removeFile(localPath))
{
std::string cacheFileName = revision->getFilesAdded()->getName();
if (containsServerAddress(cacheFileName)) cacheFileName = createCacheFileName(cacheFileName);
if (!cacheFileName.empty()) writeObjectFile(*(revision->getFilesAdded()), cacheFileName);
}
if (revision->getFilesRemoved() && revision->getFilesRemoved()->removeFile(localPath))
{
std::string cacheFileName = revision->getFilesRemoved()->getName();
if (containsServerAddress(cacheFileName)) cacheFileName = createCacheFileName(cacheFileName);
if (!cacheFileName.empty()) writeObjectFile(*(revision->getFilesRemoved()), cacheFileName);
}
if (revision->getFilesModified() && revision->getFilesModified()->removeFile(localPath))
{
std::string cacheFileName = revision->getFilesModified()->getName();
if (containsServerAddress(cacheFileName)) cacheFileName = createCacheFileName(cacheFileName);
if (!cacheFileName.empty()) writeObjectFile(*(revision->getFilesModified()), cacheFileName);
}
}
}
return false;
}
bool FileCache::loadDatabaseRevisionsForFile(const std::string& originalFileName)
{
osg::notify(osg::NOTICE)<<"FileCache::loadDatabaseRevisionsForFile("<<originalFileName<<")"<<std::endl;
osg::notify(osg::INFO)<<"FileCache::loadDatabaseRevisionsForFile("<<originalFileName<<")"<<std::endl;
std::string revisionsFileName = originalFileName;
if (getLowerCaseFileExtension(revisionsFileName)!="revisions") revisionsFileName += ".revisions";
osg::notify(osg::NOTICE)<<" revisionsFileName("<<revisionsFileName<<")"<<std::endl;
osg::notify(osg::INFO)<<" revisionsFileName("<<revisionsFileName<<")"<<std::endl;
osg::ref_ptr<osg::Object> object = osgDB::readObjectFile(revisionsFileName);
DatabaseRevisions* dr = dynamic_cast<DatabaseRevisions*>(object.get());
osg::ref_ptr<DatabaseRevisions> dr_local;
if (dr)
std::string cacheFileName = createCacheFileName(revisionsFileName);
// check to see if revion file is already loaded.
DatabaseRevisionsList::iterator ritr = _databaseRevisionsList.begin();
for(;
ritr != _databaseRevisionsList.end() && !dr_local;
++ritr)
{
osg::notify(osg::NOTICE)<<" loaded revisions File("<<revisionsFileName<<")"<<std::endl;
_databaseRevisionsList.push_back(dr);
osg::notify(osg::INFO)<<" comparing "<<(*ritr)->getName()<<" to "<<revisionsFileName<<std::endl;
if ((*ritr)->getName()==revisionsFileName)
{
osg::notify(osg::INFO)<<"Already loaded"<<std::endl;
dr_local = *ritr;
}
}
if (!dr_local)
{
if (!cacheFileName.empty() && osgDB::fileExists(cacheFileName))
{
osg::notify(osg::INFO)<<" found revisions file in local cache, now loading it"<<std::endl;
osg::ref_ptr<osg::Object> object = osgDB::readObjectFile(cacheFileName);
dr_local = dynamic_cast<DatabaseRevisions*>(object.get());
if (dr_local)
{
osg::notify(osg::INFO)<<" loaded local revisions File("<<cacheFileName<<")"<<std::endl;
}
}
else
{
osg::notify(osg::INFO)<<" could not load found revisions file from local cache."<<std::endl;
}
}
// now load revision file from remote server
osg::ref_ptr<osg::Object> object = osgDB::readObjectFile(revisionsFileName+".curl");
osg::ref_ptr<DatabaseRevisions> dr_remote = dynamic_cast<DatabaseRevisions*>(object.get());
if (dr_remote.valid())
{
bool needToWriteRevisionsFileToDisk = true;
if (dr_local.valid())
{
if (dr_local->getDatabaseRevisionList().size()==dr_remote->getDatabaseRevisionList().size())
{
unsigned int i;
for(i=0; i<dr_local->getDatabaseRevisionList().size(); ++i)
{
DatabaseRevision* revision_local = dr_local->getDatabaseRevision(i);
DatabaseRevision* revision_remote = dr_remote->getDatabaseRevision(i);
osg::notify(osg::INFO)<<" Comparing local "<<revision_local->getName()<<" to remote "<<revision_remote->getName()<<std::endl;
if (revision_local->getName()!=revision_remote->getName()) break;
}
needToWriteRevisionsFileToDisk = (i!=dr_local->getDatabaseRevisionList().size());
osg::notify(osg::INFO)<<"Local and remote revisions are different "<<needToWriteRevisionsFileToDisk<<std::endl;
}
}
if (needToWriteRevisionsFileToDisk)
{
osg::notify(osg::INFO)<<"Need to write DatabaseRevions "<<revisionsFileName<<" to local FileCache"<<std::endl;
if (!cacheFileName.empty()) writeObjectFile(*dr_remote, cacheFileName);
}
else
{
osg::notify(osg::INFO)<<"No need to write DatabaseRevions "<<revisionsFileName<<" to local FileCache"<<std::endl;
}
}
osg::ref_ptr<DatabaseRevisions> dr = dr_remote.valid() ? dr_remote : dr_local;
if (dr.valid())
{
osg::notify(osg::INFO)<<" loaded remote revisions File("<<revisionsFileName<<")"<<std::endl;
if (ritr != _databaseRevisionsList.end())
{
// replace already loaded DatabaseRevisions object
osg::notify(osg::INFO)<<"Replacing already loaded DatabaseRevisions object"<<std::endl;
*ritr = dr;
}
else
{
osg::notify(osg::INFO)<<"Added newly loaded DatabaseRevisions object "<<dr->getName()<<std::endl;
_databaseRevisionsList.push_back(dr);
}
// now need to load the individual FileLists
for(DatabaseRevisions::DatabaseRevisionList::iterator itr = dr->getDatabaseRevisionList().begin();
itr != dr->getDatabaseRevisionList().end();
++itr)
{
DatabaseRevision* revision = itr->get();
osg::notify(osg::INFO)<<" now loaded DatabaseRevisions "<<revision->getName()<<" FileList contents"<<std::endl;
if (revision->getFilesAdded())
{
FileList* fileList = readFileList(osgDB::concatPaths(revision->getDatabasePath(), revision->getFilesAdded()->getName()));
if (fileList)
{
revision->setFilesAdded(fileList);
}
}
if (revision->getFilesRemoved())
{
FileList* fileList = readFileList(osgDB::concatPaths(revision->getDatabasePath(), revision->getFilesRemoved()->getName()));
if (fileList)
{
revision->setFilesRemoved(fileList);
}
}
if (revision->getFilesModified())
{
FileList* fileList = readFileList(osgDB::concatPaths(revision->getDatabasePath(), revision->getFilesModified()->getName()));
if (fileList)
{
revision->setFilesModified(fileList);
}
}
}
return true;
}
else
@@ -262,3 +453,30 @@ bool FileCache::loadDatabaseRevisionsForFile(const std::string& originalFileName
return false;
}
}
FileList* FileCache::readFileList(const std::string& originalFileName) const
{
osg::ref_ptr<FileList> fileList;
std::string cacheFileListName = createCacheFileName(originalFileName);
if (!cacheFileListName.empty() && osgDB::fileExists(cacheFileListName))
{
osg::ref_ptr<osg::Object> object = osgDB::readObjectFile(cacheFileListName);
fileList = dynamic_cast<osgDB::FileList*>(object.get());
if (fileList) osg::notify(osg::INFO)<<" loadeded FileList from local cache "<<fileList->getName()<<std::endl;
}
if (!fileList)
{
osg::notify(osg::INFO)<<" complete_path="<<originalFileName<<std::endl;
osg::ref_ptr<osg::Object> object = osgDB::readObjectFile(originalFileName+".curl");
fileList = dynamic_cast<osgDB::FileList*>(object.get());
if (fileList)
{
osg::notify(osg::INFO)<<" loadeded FileList from remote system "<<fileList->getName()<<std::endl;
osg::notify(osg::INFO)<<" Need to write to local file cache "<<fileList->getName()<<std::endl;
if (!cacheFileListName.empty()) writeObjectFile(*fileList, cacheFileListName);
}
}
return fileList.release();
}

56
src/osgPlugins/revisions/ReaderWriterRevisions.cpp
View File

@@ -50,7 +50,7 @@ class ReaderWriterFreeType : public osgDB::ReaderWriter
ReadResult readFileList(std::istream& fin, const std::string& name, const osgDB::ReaderWriter::Options* options) const
{
osg::notify(osg::NOTICE)<<" readFileList="<<name<<std::endl;
osg::notify(osg::INFO)<<" readFileList="<<name<<std::endl;
osg::ref_ptr<osgDB::FileList> fileList = new osgDB::FileList;
fileList->setName(name);
@@ -59,7 +59,7 @@ class ReaderWriterFreeType : public osgDB::ReaderWriter
{
std::string filename;
fin >> filename;
osg::notify(osg::NOTICE)<<" ="<<filename<<std::endl;
osg::notify(osg::INFO)<<" ="<<filename<<std::endl;
if (!filename.empty()) fileList->getFileNames().insert(filename);
}
@@ -77,18 +77,21 @@ class ReaderWriterFreeType : public osgDB::ReaderWriter
std::string revisions_path;
if (options && !(options->getDatabasePathList().empty())) revisions_path = options->getDatabasePathList().front();
else revisions_path = osgDB::getFilePath(name);
revisions->setDatabasePath(revisions_path);
osg::notify(osg::NOTICE)<<"readRevisions="<<name<<std::endl;
osg::notify(osg::NOTICE)<<" revisions_path="<<revisions_path<<std::endl;
osg::notify(osg::INFO)<<"readRevisions="<<name<<std::endl;
osg::notify(osg::INFO)<<" revisions_path="<<revisions_path<<std::endl;
bool loadFileLists = false;
while(fin)
{
std::string filename;
fin >> filename;
osg::notify(osg::NOTICE)<<" filename="<<filename<<std::endl;
osg::notify(osg::INFO)<<" filename="<<filename<<std::endl;
if (!filename.empty())
{
@@ -104,25 +107,34 @@ class ReaderWriterFreeType : public osgDB::ReaderWriter
dbRevision->setDatabasePath(revisions_path);
}
std::string complete_path = osgDB::concatPaths(revisions_path, filename);
osg::notify(osg::NOTICE)<<" complete_path="<<complete_path<<std::endl;
osg::ref_ptr<osg::Object> object = osgDB::readObjectFile(complete_path, options);
osgDB::FileList* fileList = dynamic_cast<osgDB::FileList*>(object.get());
osg::ref_ptr<osgDB::FileList> fileList;
if (fileList)
if (loadFileLists)
{
if (ext=="added")
{
dbRevision->setFilesAdded(fileList);
}
else if (ext=="removed")
{
dbRevision->setFilesRemoved(fileList);
}
else if (ext=="modified")
{
dbRevision->setFilesModified(fileList);
}
std::string complete_path = osgDB::concatPaths(revisions_path, filename);
osg::notify(osg::INFO)<<" complete_path="<<complete_path<<std::endl;
osg::ref_ptr<osg::Object> object = osgDB::readObjectFile(complete_path, options);
fileList = dynamic_cast<osgDB::FileList*>(object.get());
}
if (!fileList)
{
fileList = new osgDB::FileList;
fileList->setName(filename);
}
if (ext=="added")
{
dbRevision->setFilesAdded(fileList);
}
else if (ext=="removed")
{
dbRevision->setFilesRemoved(fileList);
}
else if (ext=="modified")
{
dbRevision->setFilesModified(fileList);
}
}
}