FGFS-4.1/OpenSceneGraph
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactored DatabasePager and related classes to introduce support for

Browse Source 
multi-threaded paging, where the Pager manages threads of reading local
and http files via seperate threads.  This makes it possible to smoothly
browse large databases where parts of the data are locally cached while
others are on a remote server.  Previously with this type of dataset 
the pager would stall all paging while http requests were being served,
even when parts of the models are still loadable virtue of being in the 
local cache.

Also as part of the refactoring the DatabaseRequest are now stored in the
ProxyNode/PagedLOD nodes to facilitate quite updating in the cull traversal,
with the new code avoiding mutex locks and searches.  Previous on big 
databases the overhead involved in make database requests could accumulate
to a point where it'd cause the cull traversal to break frame.  The overhead
now is negligable.

Finally OSG_FILE_CACHE support has been moved from the curl plugin into
the DatabasePager.  Eventually this functionality will be moved out into
osgDB for more general usage.
This commit is contained in:
Robert Osfield
2008-05-21 21:09:45 +00:00
parent 100cc12ecb
commit 7b003b24ea
21 changed files with 1477 additions and 1017 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
include/osg/NodeVisitor
View File

@@ -268,7 +268,7 @@ class OSG_EXPORT NodeVisitor : public virtual Referenced
DatabaseRequestHandler():
Referenced(true) {}
virtual void requestNodeFile(const std::string& fileName,osg::Group* group, float priority, const FrameStamp* framestamp) = 0;
virtual void requestNodeFile(const std::string& fileName,osg::Group* group, float priority, const FrameStamp* framestamp, osg::ref_ptr<osg::Referenced>& databaseRequest) = 0;
protected:
virtual ~DatabaseRequestHandler() {}

19
include/osg/PagedLOD
View File

@@ -58,10 +58,11 @@ class OSG_EXPORT PagedLOD : public LOD
PerRangeData(const PerRangeData& prd);
PerRangeData& operator = (const PerRangeData& prd);
std::string _filename;
float _priorityOffset;
float _priorityScale;
double _timeStamp;
std::string _filename;
float _priorityOffset;
float _priorityScale;
double _timeStamp;
osg::ref_ptr<osg::Referenced> _databaseRequest;
};
typedef std::vector<PerRangeData> PerRangeDataList;
@@ -85,6 +86,16 @@ class OSG_EXPORT PagedLOD : public LOD
unsigned int getNumTimeStamps() const { return _perRangeDataList.size(); }
/** Return the DatabaseRequest object used by the DatabasePager to keep track of file load requests
* being carried on behalf of the DatabasePager.
* Note, in normal OSG usage you should not set this value yourself, as this will be managed by
* the osgDB::DatabasePager.*/
osg::ref_ptr<osg::Referenced>& getDatabaseRequest(unsigned int childNo) { return _perRangeDataList[childNo]._databaseRequest; }
/** Return the const DatabaseRequest object.*/
const osg::ref_ptr<osg::Referenced>& getDatabaseRequest(unsigned int childNo) const { return _perRangeDataList[childNo]._databaseRequest; }
/** Set the frame number of the last time that this PageLOD node was traversed.
* Note, this frame number is automatically set by the traverse() method for all traversals (update, cull etc.).
*/

31
include/osg/ProxyNode
View File

@@ -46,12 +46,20 @@ class OSG_EXPORT ProxyNode : public Group
/** Get the database path used to prepend to children's filenames.*/
inline const std::string& getDatabasePath() const { return _databasePath; }
typedef std::vector<std::string> FileNameList;
void setFileName(unsigned int childNo, const std::string& filename) { expandFileNameListTo(childNo); _filenameList[childNo]=filename; }
const std::string& getFileName(unsigned int childNo) const { return _filenameList[childNo]; }
void setFileName(unsigned int childNo, const std::string& filename) { expandFileNameListTo(childNo); _filenameList[childNo].first=filename; }
const std::string& getFileName(unsigned int childNo) const { return _filenameList[childNo].first; }
unsigned int getNumFileNames() const { return _filenameList.size(); }
/** Return the DatabaseRequest object used by the DatabasePager to keep track of file load requests
* being carried on behalf of the DatabasePager.
* Note, in normal OSG usage you should not set this value yourself, as this will be managed by
* the osgDB::DatabasePager.*/
osg::ref_ptr<osg::Referenced>& getDatabaseRequest(unsigned int childNo) { return _filenameList[childNo].second; }
/** Return the const DatabaseRequest object.*/
const osg::ref_ptr<osg::Referenced>& getDatabaseRequest(unsigned int childNo) const { return _filenameList[childNo].second; }
/** Modes which control how the center of object should be determined when computed which child is active.*/
enum CenterMode
{
@@ -102,14 +110,17 @@ class OSG_EXPORT ProxyNode : public Group
void expandFileNameListTo(unsigned int pos);
FileNameList _filenameList;
std::string _databasePath;
typedef std::pair< std::string, osg::ref_ptr<osg::Referenced> > FileNameDatabaseRequestPair;
typedef std::vector<FileNameDatabaseRequestPair> FileNameDatabaseRequestList;
FileNameDatabaseRequestList _filenameList;
std::string _databasePath;
LoadingExternalReferenceMode _loadingExtReference;
LoadingExternalReferenceMode _loadingExtReference;
CenterMode _centerMode;
vec_type _userDefinedCenter;
value_type _radius;
CenterMode _centerMode;
vec_type _userDefinedCenter;
value_type _radius;
};

126
include/osgDB/DatabasePager
View File

@@ -37,9 +37,10 @@
namespace osgDB {
/** Database paging class which manages the loading of files in a background thread,
* and synchronizing of loaded models with the main scene graph.*/
class OSGDB_EXPORT DatabasePager : public osg::NodeVisitor::DatabaseRequestHandler, public OpenThreads::Thread
class OSGDB_EXPORT DatabasePager : public osg::NodeVisitor::DatabaseRequestHandler
{
public :
@@ -62,21 +63,63 @@ class OSGDB_EXPORT DatabasePager : public osg::NodeVisitor::DatabaseRequestHandl
/** Add a request to load a node file to end the the database request list.*/
virtual void requestNodeFile(const std::string& fileName,osg::Group* group,
float priority, const osg::FrameStamp* framestamp);
float priority, const osg::FrameStamp* framestamp,
osg::ref_ptr<osg::Referenced>& databaseRequest);
virtual void requestNodeFile(const std::string& fileName,osg::Group* group,
float priority, const osg::FrameStamp* framestamp,
osg::ref_ptr<osg::Referenced>& databaseRequest,
ReaderWriter::Options* loadOptions);
/** Run does the database paging.*/
virtual void run();
/** Cancel the database pager thread.*/
/** Set the priority of the database pager thread(s).*/
int setSchedulePriority(OpenThreads::Thread::ThreadPriority priority);
/** Cancel the database pager thread(s).*/
virtual int cancel();
virtual bool isRunning() const;
/** Clear all internally cached structures.*/
virtual void clear();
class DatabaseThread : public osg::Referenced, public OpenThreads::Thread
{
public:
enum Mode
{
HANDLE_ALL_REQUESTS,
HANDLE_NON_HTTP,
HANDLE_ONLY_HTTP
};
DatabaseThread(DatabasePager* pager, Mode mode, const std::string& name);
DatabaseThread(const DatabaseThread& dt, DatabasePager* pager);
void setDone(bool done) { _done = done; }
bool getDone() const { return _done; }
virtual int cancel();
virtual void run();
protected:
virtual ~DatabaseThread();
bool _done;
DatabasePager* _pager;
Mode _mode;
std::string _name;
};
DatabaseThread* getDatabaseThread(unsigned int i) { return _databaseThreads[i].get(); }
const DatabaseThread* getDatabaseThread(unsigned int i) const { return _databaseThreads[i].get(); }
unsigned int getNumDatabaseThreads() const { return _databaseThreads.size(); }
/** Set whether the database pager thread should be paused or not.*/
void setDatabasePagerThreadPause(bool pause);
@@ -232,7 +275,7 @@ class OSGDB_EXPORT DatabasePager : public osg::NodeVisitor::DatabaseRequestHandl
virtual void compileAllGLObjects(osg::State& state);
/** Report how many items are in the _fileRequestList queue */
unsigned int getFileRequestListSize() const { return _fileRequestList.size(); }
unsigned int getFileRequestListSize() const { return _fileRequestQueue._requestList.size() + _httpRequestQueue._requestList.size(); }
/** Report how many items are in the _dataToCompileList queue */
unsigned int getDataToCompileListSize() const { return _dataToCompileList.size(); }
@@ -263,12 +306,21 @@ class OSGDB_EXPORT DatabasePager : public osg::NodeVisitor::DatabaseRequestHandl
virtual ~DatabasePager();
friend class DatabaseThread;
friend struct DatabaseRequest;
struct DatabaseRequest : public osg::Referenced
{
DatabaseRequest():
osg::Referenced(true),
_frameNumberFirstRequest(0),
_timestampFirstRequest(0.0),
_priorityFirstRequest(0.f),
_frameNumberLastRequest(0),
_timestampLastRequest(0.0),
_priorityLastRequest(0.0f),
_numOfRequests(0)
{}
@@ -290,10 +342,43 @@ class OSGDB_EXPORT DatabasePager : public osg::NodeVisitor::DatabaseRequestHandl
return frameNumber - _frameNumberLastRequest <= 1;
}
};
typedef std::vector< osg::ref_ptr<DatabaseRequest> > DatabaseRequestList;
typedef std::vector< osg::ref_ptr<DatabaseThread> > DatabaseThreadList;
typedef std::list< osg::ref_ptr<DatabaseRequest> > DatabaseRequestList;
typedef std::vector< osg::ref_ptr<osg::Object> > ObjectList;
struct RequestQueue : public osg::Referenced
{
RequestQueue(DatabasePager* pager, const std::string& name);
void block() { _block->block(); }
void release() { _block->release(); }
void updateBlock()
{
_block->set((!_requestList.empty() || !_childrenToDeleteList.empty()) &&
!_pager->_databasePagerThreadPaused);
}
void clear();
void add(DatabaseRequest* databaseRequest);
void takeFirst(osg::ref_ptr<DatabaseRequest>& databaseRequest);
osg::ref_ptr<osg::RefBlock> _block;
DatabasePager* _pager;
std::string _name;
OpenThreads::Mutex _requestMutex;
DatabaseRequestList _requestList;
OpenThreads::Mutex _childrenToDeleteListMutex;
ObjectList _childrenToDeleteList;
};
// forward declare inner helper classes
class FindCompileableGLObjectsVisitor;
friend class FindCompileableGLObjectsVisitor;
@@ -310,15 +395,6 @@ class OSGDB_EXPORT DatabasePager : public osg::NodeVisitor::DatabaseRequestHandl
OpenThreads::Mutex _run_mutex;
bool _startThreadCalled;
osg::ref_ptr<osg::RefBlock> _databasePagerThreadBlock;
inline void updateDatabasePagerThreadBlock()
{
_databasePagerThreadBlock->set(
(!_fileRequestList.empty() || !_childrenToDeleteList.empty()) && !_databasePagerThreadPaused);
}
// Helper functions for determining if objects need to be
// compiled.
inline static bool isCompiled(const osg::Texture* texture,
@@ -410,12 +486,17 @@ class OSGDB_EXPORT DatabasePager : public osg::NodeVisitor::DatabaseRequestHandl
bool _acceptNewRequests;
bool _databasePagerThreadPaused;
DatabaseThreadList _databaseThreads;
int _numFramesActive;
mutable OpenThreads::Mutex _numFramesActiveMutex;
int _frameNumber;
DatabaseRequestList _fileRequestList;
mutable OpenThreads::Mutex _fileRequestListMutex;
RequestQueue _fileRequestQueue;
RequestQueue _httpRequestQueue;
//DatabaseRequestList _fileRequestList;
//mutable OpenThreads::Mutex _fileRequestListMutex;
DatabaseRequestList _dataToCompileList;
mutable OpenThreads::Mutex _dataToCompileListMutex;
@@ -428,8 +509,8 @@ class OSGDB_EXPORT DatabasePager : public osg::NodeVisitor::DatabaseRequestHandl
float _valueAnisotropy;
bool _deleteRemovedSubgraphsInDatabaseThread;
ObjectList _childrenToDeleteList;
mutable OpenThreads::Mutex _childrenToDeleteListMutex;
//ObjectList _childrenToDeleteList;
//mutable OpenThreads::Mutex _childrenToDeleteListMutex;
DatabaseRequestList _dataToMergeList;
mutable OpenThreads::Mutex _dataToMergeListMutex;
@@ -451,7 +532,6 @@ class OSGDB_EXPORT DatabasePager : public osg::NodeVisitor::DatabaseRequestHandl
double _totalTimeToMergeTiles;
unsigned int _numTilesMerges;
struct CompileOperation : public osg::GraphicsOperation
{
CompileOperation(DatabasePager* databasePager);

25
include/osgDB/ReaderWriter
View File

@@ -90,21 +90,18 @@ class OSGDB_EXPORT ReaderWriter : public osg::Object
Options():
osg::Object(true),
_objectCacheHint(CACHE_ARCHIVES),
_asynchronousFileReadHint(false) {}
_objectCacheHint(CACHE_ARCHIVES) {}
Options(const std::string& str):
osg::Object(true),
_str(str),
_objectCacheHint(CACHE_ARCHIVES),
_asynchronousFileReadHint(false) {}
_objectCacheHint(CACHE_ARCHIVES) {}
Options(const Options& options,const osg::CopyOp& copyop=osg::CopyOp::SHALLOW_COPY):
osg::Object(options,copyop),
_str(options._str),
_databasePaths(options._databasePaths),
_objectCacheHint(options._objectCacheHint),
_asynchronousFileReadHint(options._asynchronousFileReadHint) {}
_objectCacheHint(options._objectCacheHint) {}
META_Object(osgDB,Options);
@@ -131,21 +128,6 @@ class OSGDB_EXPORT ReaderWriter : public osg::Object
CacheHintOptions getObjectCacheHint() const { return _objectCacheHint; }
/** Set Asynchrnous file read hint.
* This hint is used by plugins like the libcurl http reader plugin to inform them that
* they should make an internal file read requests to their background threads to load files,
* with the plugin returning immediately with a ReadResult::FILE_REQUESTED status. It is
* assumed that calls will continue to be made to the plugin until the background threads
* have read or failed to read the request file, at which point the return status which change
* to FILE_LOADED and the objects will be returned.
* Note, this facility is particular useful when using DatabasePager in conjunction with
* internet based databases where file load latency is relatively high.*/
void setAsynchronousFileReadHint(bool flag) { _asynchronousFileReadHint = flag; }
/** Get Asynchrnous file read hint. */
bool getAsynchronousFileReadHint() const { return _asynchronousFileReadHint; }
/** Sets a plugindata value PluginData with a string */
void setPluginData(const std::string& s, void* v) const { _pluginData[s] = v; }
@@ -169,7 +151,6 @@ class OSGDB_EXPORT ReaderWriter : public osg::Object
std::string _str;
FilePathList _databasePaths;
CacheHintOptions _objectCacheHint;
bool _asynchronousFileReadHint;
typedef std::map<std::string,void*> PluginDataMap;
mutable PluginDataMap _pluginData;