Files
OpenSceneGraph/include/osgDB/DatabasePager
Robert Osfield 7b003b24ea Refactored DatabasePager and related classes to introduce support for
multi-threaded paging, where the Pager manages threads of reading local
and http files via seperate threads.  This makes it possible to smoothly
browse large databases where parts of the data are locally cached while
others are on a remote server.  Previously with this type of dataset 
the pager would stall all paging while http requests were being served,
even when parts of the models are still loadable virtue of being in the 
local cache.

Also as part of the refactoring the DatabaseRequest are now stored in the
ProxyNode/PagedLOD nodes to facilitate quite updating in the cull traversal,
with the new code avoiding mutex locks and searches.  Previous on big 
databases the overhead involved in make database requests could accumulate
to a point where it'd cause the cull traversal to break frame.  The overhead
now is negligable.

Finally OSG_FILE_CACHE support has been moved from the curl plugin into
the DatabasePager.  Eventually this functionality will be moved out into
osgDB for more general usage.
2008-05-21 21:09:45 +00:00

548 lines
24 KiB
C++

/* -*-c++-*- OpenSceneGraph - Copyright (C) 1998-2006 Robert Osfield
*
* This library is open source and may be redistributed and/or modified under
* the terms of the OpenSceneGraph Public License (OSGPL) version 0.0 or
* (at your option) any later version. The full license is in LICENSE file
* included with this distribution, and on the openscenegraph.org website.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* OpenSceneGraph Public License for more details.
*/
#ifndef OSGDB_DATABASEPAGER
#define OSGDB_DATABASEPAGER 1
#include <osg/NodeVisitor>
#include <osg/Group>
#include <osg/PagedLOD>
#include <osg/Drawable>
#include <osg/GraphicsThread>
#include <OpenThreads/Thread>
#include <OpenThreads/Mutex>
#include <OpenThreads/ScopedLock>
#include <OpenThreads/Condition>
#include <osgDB/SharedStateManager>
#include <osgDB/ReaderWriter>
#include <osgDB/Export>
#include <map>
#include <list>
#include <algorithm>
#include <functional>
namespace osgDB {
/** Database paging class which manages the loading of files in a background thread,
* and synchronizing of loaded models with the main scene graph.*/
class OSGDB_EXPORT DatabasePager : public osg::NodeVisitor::DatabaseRequestHandler
{
public :
typedef OpenThreads::Thread::ThreadPriority ThreadPriority;
DatabasePager();
DatabasePager(const DatabasePager& rhs);
/** Create a shallow copy on the DatabasePager.*/
virtual DatabasePager* clone() const { return new DatabasePager(*this); }
/** get the prototype singleton used by DatabasePager::create().*/
static osg::ref_ptr<DatabasePager>& prototype();
/** create a DatabasePager by cloning DatabasePager::prototype().*/
static DatabasePager* create();
/** Add a request to load a node file to end the the database request list.*/
virtual void requestNodeFile(const std::string& fileName,osg::Group* group,
float priority, const osg::FrameStamp* framestamp,
osg::ref_ptr<osg::Referenced>& databaseRequest);
virtual void requestNodeFile(const std::string& fileName,osg::Group* group,
float priority, const osg::FrameStamp* framestamp,
osg::ref_ptr<osg::Referenced>& databaseRequest,
ReaderWriter::Options* loadOptions);
/** Set the priority of the database pager thread(s).*/
int setSchedulePriority(OpenThreads::Thread::ThreadPriority priority);
/** Cancel the database pager thread(s).*/
virtual int cancel();
virtual bool isRunning() const;
/** Clear all internally cached structures.*/
virtual void clear();
class DatabaseThread : public osg::Referenced, public OpenThreads::Thread
{
public:
enum Mode
{
HANDLE_ALL_REQUESTS,
HANDLE_NON_HTTP,
HANDLE_ONLY_HTTP
};
DatabaseThread(DatabasePager* pager, Mode mode, const std::string& name);
DatabaseThread(const DatabaseThread& dt, DatabasePager* pager);
void setDone(bool done) { _done = done; }
bool getDone() const { return _done; }
virtual int cancel();
virtual void run();
protected:
virtual ~DatabaseThread();
bool _done;
DatabasePager* _pager;
Mode _mode;
std::string _name;
};
DatabaseThread* getDatabaseThread(unsigned int i) { return _databaseThreads[i].get(); }
const DatabaseThread* getDatabaseThread(unsigned int i) const { return _databaseThreads[i].get(); }
unsigned int getNumDatabaseThreads() const { return _databaseThreads.size(); }
/** Set whether the database pager thread should be paused or not.*/
void setDatabasePagerThreadPause(bool pause);
/** Get whether the database pager thread should is paused or not.*/
bool getDatabasePagerThreadPause() const { return _databasePagerThreadPaused; }
/** Set whether new database request calls are accepted or ignored.*/
void setAcceptNewDatabaseRequests(bool acceptNewRequests) { _acceptNewRequests = acceptNewRequests; }
/** Get whether new database request calls are accepted or ignored.*/
bool getAcceptNewDatabaseRequests() const { return _acceptNewRequests; }
/** Get the number of frames that are currently active.*/
int getNumFramesActive() const { return _numFramesActive; }
/** Signal the database thread that the update, cull and draw has begun for a new frame.
* Note, this is called by the application so that the database pager can go to sleep while the CPU is busy on the main rendering threads. */
virtual void signalBeginFrame(const osg::FrameStamp* framestamp);
/** Signal the database thread that the update, cull and draw dispatch has completed.
* Note, this is called by the application so that the database pager can go to wake back up now the main rendering threads are iddle waiting for the next frame.*/
virtual void signalEndFrame();
/** Find all PagedLOD nodes in a subgraph and register them with
* the DatabasePager so it can keep track of expired nodes.
* note, should be only be called from the update thread. */
virtual void registerPagedLODs(osg::Node* subgraph);
/** Set whether the database pager should pre compile OpenGL objects before allowing
* them to be merged into the scene graph.
* Pre compilation helps reduce the chances of frame drops, but also slows the
* speed at which tiles are merged as they have to be compiled first.*/
void setDoPreCompile(bool flag) { _doPreCompile = flag; }
/** Get whether the database pager should pre compile OpenGL objects before allowing
* them to be merged into the scene graph.*/
bool getDoPreCompile() const { return _doPreCompile; }
/** Set the target frame rate that the DatabasePager should assume.
* Typically one would set this to the value refresh rate of your display system i.e. 60Hz.
* Default value is 100.
* Usage notes. The TargetFrameRate and the MinimumTimeAvailableForGLCompileAndDeletePerFrame
* parameters are not directly used by DatabasePager, but are should be used as a guide for how
* long to set aside per frame for compiling and deleting OpenGL objects - ie. the value to use
* when calling DatabasePager::compileGLObjectgs(state,availableTime,). The longer amount of
* time to set aside cthe faster databases will be paged in but with increased chance of frame drops,
* the lower the amount of time the set aside the slower databases will paged it but with better
* chance of avoid any frame drops. The default values are chosen to achieve the later when running
* on a modern mid to high end PC.
* The way to compute the amount of available time use a scheme such as :
* availableTime = maximum(1.0/targetFrameRate - timeTakenDuringUpdateCullAndDraw, minimumTimeAvailableForGLCompileAndDeletePerFrame).
*/
void setTargetFrameRate(double tfr) { _targetFrameRate = tfr; }
/** Get the target frame rate that the DatabasePager should assume.*/
double getTargetFrameRate() const { return _targetFrameRate; }
/** Set the minimum amount of time (in seconds) that should be made available for compiling and delete OpenGL objects per frame.
* Default value is 0.001 (1 millisecond).
* For usage see notes in setTargetFrameRate.*/
void setMinimumTimeAvailableForGLCompileAndDeletePerFrame(double ta) { _minimumTimeAvailableForGLCompileAndDeletePerFrame = ta; }
/** Get the minimum amount of time that should be made available for compiling and delete OpenGL objects per frame.
* For usage see notes in setTargetFrameRate.*/
double getMinimumTimeAvailableForGLCompileAndDeletePerFrame() const { return _minimumTimeAvailableForGLCompileAndDeletePerFrame; }
/** Set the maximum number of OpenGL objects that the page should attempt to compile per frame.
* Note, Lower values reduces chances of a frame drop but lower the rate that database will be paged in at.
* Default value is 8. */
void setMaximumNumOfObjectsToCompilePerFrame(unsigned int num) { _maximumNumOfObjectsToCompilePerFrame = num; }
/** Get the maximum number of OpenGL objects that the page should attempt to compile per frame.*/
unsigned int getMaximumNumOfObjectsToCompilePerFrame() const { return _maximumNumOfObjectsToCompilePerFrame; }
/** Set the amount of time that a subgraph will be kept without being visited in the cull traversal
* before being removed.*/
void setExpiryDelay(double expiryDelay) { _expiryDelay = expiryDelay; }
/** Get the amount of time that a subgraph will be kept without being visited in the cull traversal
* before being removed.*/
double getExpiryDelay() const { return _expiryDelay; }
/** Set whether the removed subgraphs should be deleted in the database thread or not.*/
void setDeleteRemovedSubgraphsInDatabaseThread(bool flag) { _deleteRemovedSubgraphsInDatabaseThread = flag; }
/** Get whether the removed subgraphs should be deleted in the database thread or not.*/
bool getDeleteRemovedSubgraphsInDatabaseThread() const { return _deleteRemovedSubgraphsInDatabaseThread; }
enum DrawablePolicy
{
DO_NOT_MODIFY_DRAWABLE_SETTINGS,
USE_DISPLAY_LISTS,
USE_VERTEX_BUFFER_OBJECTS,
USE_VERTEX_ARRAYS
};
/** Set how loaded drawables should be handled w.r.t their display list/vertex buffer object/vertex array settings.*/
void setDrawablePolicy(DrawablePolicy policy) { _drawablePolicy = policy; }
/** Get how loaded drawables should be handled w.r.t their display list/vertex buffer object/vertex array settings.*/
DrawablePolicy getDrawablePolicy() const { return _drawablePolicy; }
/** Set whether newly loaded textures should have their UnrefImageDataAfterApply set to a specified value.*/
void setUnrefImageDataAfterApplyPolicy(bool changeAutoUnRef, bool valueAutoUnRef) { _changeAutoUnRef = changeAutoUnRef; _valueAutoUnRef = valueAutoUnRef; }
/** Get whether newly loaded textures should have their UnrefImageDataAfterApply set to a specified value.*/
void getUnrefImageDataAfterApplyPolicy(bool& changeAutoUnRef, bool& valueAutoUnRef) const { changeAutoUnRef = _changeAutoUnRef; valueAutoUnRef = _valueAutoUnRef; }
/** Set whether newly loaded textures should have their MaxAnisotopy set to a specified value.*/
void setMaxAnisotropyPolicy(bool changeAnisotropy, float valueAnisotropy) { _changeAnisotropy = changeAnisotropy; _valueAnisotropy = valueAnisotropy; }
/** Set whether newly loaded textures should have their MaxAnisotopy set to a specified value.*/
void getMaxAnisotropyPolicy(bool& changeAnisotropy, float& valueAnisotropy) const { changeAnisotropy = _changeAnisotropy; valueAnisotropy = _valueAnisotropy; }
/** Return true if there are pending updates to the scene graph that require a call to updateSceneGraph(double). */
bool requiresUpdateSceneGraph() const;
/** Merge the changes to the scene graph by calling calling removeExpiredSubgraphs then addLoadedDataToSceneGraph.
* Note, must only be called from single thread update phase. */
virtual void updateSceneGraph(double currentFrameTime)
{
removeExpiredSubgraphs(currentFrameTime);
addLoadedDataToSceneGraph(currentFrameTime);
}
/** Turn the compilation of rendering objects for specified graphics context on (true) or off(false). */
void setCompileGLObjectsForContextID(unsigned int contextID, bool on);
/** Get whether the compilation of rendering objects for specified graphics context on (true) or off(false). */
bool getCompileGLObjectsForContextID(unsigned int contextID);
/** Return true if an external draw thread should call compileGLObjects(..) or not.*/
bool requiresExternalCompileGLObjects(unsigned int contextID) const;
/** Return true if there are pending compile operations that are required.
* If requiresCompileGLObjects() return true the application should call compileGLObjects() .*/
bool requiresCompileGLObjects() const;
/** Compile the rendering objects (display lists,texture objects, VBO's) on loaded subgraph.
* note, should only be called from the draw thread.
* Note, must only be called from a valid graphics context. */
virtual void compileGLObjects(osg::State& state,double& availableTime);
/** Compile the rendering objects (display lists,texture objects, VBO's) on loaded subgraph.
* note, should only be called from the draw thread.
* Note, must only be called from a valid graphics context. */
virtual void compileAllGLObjects(osg::State& state);
/** Report how many items are in the _fileRequestList queue */
unsigned int getFileRequestListSize() const { return _fileRequestQueue._requestList.size() + _httpRequestQueue._requestList.size(); }
/** Report how many items are in the _dataToCompileList queue */
unsigned int getDataToCompileListSize() const { return _dataToCompileList.size(); }
/** Get the minimum time between the first request for a tile to be loaded and the time of its merge into the main scene graph.*/
double getMinimumTimeToMergeTile() const { return _minimumTimeToMergeTile; }
/** Get the maximum time between the first request for a tile to be loaded and the time of its merge into the main scene graph.*/
double getMaximumTimeToMergeTile() const { return _maximumTimeToMergeTile; }
/** Get the average time between the first request for a tile to be loaded and the time of its merge into the main scene graph.*/
double getAverageTimeToMergeTiles() const { return _totalTimeToMergeTiles/static_cast<double>(_numTilesMerges); }
/** Reset the Stats variables.*/
void resetStats();
typedef std::list< osg::ref_ptr<osg::PagedLOD> > PagedLODList;
typedef std::set< osg::ref_ptr<osg::StateSet> > StateSetList;
typedef std::vector< osg::ref_ptr<osg::Drawable> > DrawableList;
typedef std::pair<StateSetList,DrawableList> DataToCompile;
typedef std::map< unsigned int, DataToCompile > DataToCompileMap;
typedef std::set<unsigned int> ActiveGraphicsContexts;
typedef std::vector< osg::observer_ptr<osg::GraphicsContext> > CompileGraphicsContexts;
protected:
virtual ~DatabasePager();
friend class DatabaseThread;
friend struct DatabaseRequest;
struct DatabaseRequest : public osg::Referenced
{
DatabaseRequest():
osg::Referenced(true),
_frameNumberFirstRequest(0),
_timestampFirstRequest(0.0),
_priorityFirstRequest(0.f),
_frameNumberLastRequest(0),
_timestampLastRequest(0.0),
_priorityLastRequest(0.0f),
_numOfRequests(0)
{}
std::string _fileName;
int _frameNumberFirstRequest;
double _timestampFirstRequest;
float _priorityFirstRequest;
int _frameNumberLastRequest;
double _timestampLastRequest;
float _priorityLastRequest;
unsigned int _numOfRequests;
osg::ref_ptr<osg::Group> _groupForAddingLoadedSubgraph;
osg::ref_ptr<osg::Node> _loadedModel;
DataToCompileMap _dataToCompileMap;
osg::ref_ptr<ReaderWriter::Options> _loadOptions;
bool isRequestCurrent (int frameNumber) const
{
return frameNumber - _frameNumberLastRequest <= 1;
}
};
typedef std::vector< osg::ref_ptr<DatabaseThread> > DatabaseThreadList;
typedef std::list< osg::ref_ptr<DatabaseRequest> > DatabaseRequestList;
typedef std::vector< osg::ref_ptr<osg::Object> > ObjectList;
struct RequestQueue : public osg::Referenced
{
RequestQueue(DatabasePager* pager, const std::string& name);
void block() { _block->block(); }
void release() { _block->release(); }
void updateBlock()
{
_block->set((!_requestList.empty() || !_childrenToDeleteList.empty()) &&
!_pager->_databasePagerThreadPaused);
}
void clear();
void add(DatabaseRequest* databaseRequest);
void takeFirst(osg::ref_ptr<DatabaseRequest>& databaseRequest);
osg::ref_ptr<osg::RefBlock> _block;
DatabasePager* _pager;
std::string _name;
OpenThreads::Mutex _requestMutex;
DatabaseRequestList _requestList;
OpenThreads::Mutex _childrenToDeleteListMutex;
ObjectList _childrenToDeleteList;
};
// forward declare inner helper classes
class FindCompileableGLObjectsVisitor;
friend class FindCompileableGLObjectsVisitor;
class MarkPagedLODsVisitor;
class FindPagedLODsVisitor;
friend class FindPagedLODsVisitor;
struct SortFileRequestFunctor;
friend struct SortFileRequestFunctor;
OpenThreads::Mutex _run_mutex;
bool _startThreadCalled;
// Helper functions for determining if objects need to be
// compiled.
inline static bool isCompiled(const osg::Texture* texture,
unsigned int contextID)
{
return( texture->getTextureObject(contextID) != NULL );
}
// Is texture compiled for all active contexts?
inline bool isCompiled(osg::Texture* texture) const
{
for (ActiveGraphicsContexts::const_iterator iter=_activeGraphicsContexts.begin();
iter!=_activeGraphicsContexts.end(); ++iter )
{
if ( texture->getTextureObject(*iter) == NULL ) return false;
}
return true;
}
inline static bool isCompiled(const osg::StateSet* stateSet,
unsigned int contextID)
{
for (unsigned i = 0;
i < stateSet->getTextureAttributeList().size();
++i)
{
const osg::Texture* texture
= dynamic_cast<const osg::Texture*>(stateSet->getTextureAttribute(i,osg::StateAttribute::TEXTURE));
if (texture && !isCompiled(texture, contextID))
return false;
}
return true;
}
inline bool isCompiled(osg::StateSet* stateSet)
{
for (unsigned i = 0;
i < stateSet->getTextureAttributeList().size();
++i)
{
osg::Texture* texture
= dynamic_cast<osg::Texture*>(stateSet->getTextureAttribute(i,osg::StateAttribute::TEXTURE));
if (texture)
{
for (ActiveGraphicsContexts::iterator iter=_activeGraphicsContexts.begin();
iter!=_activeGraphicsContexts.end(); ++iter )
{
if ( texture->getTextureObject(*iter) == NULL ) return false;
}
}
}
return true;
}
inline static bool isCompiled(const osg::Drawable* drawable,
unsigned int contextID)
{
// Worry about vbos later
if (drawable->getUseDisplayList())
{
return drawable->getDisplayList(contextID) != 0;
}
return true;
}
inline bool isCompiled(const osg::Drawable* drawable) const
{
if (drawable->getUseDisplayList())
{
for (ActiveGraphicsContexts::const_iterator iter=_activeGraphicsContexts.begin();
iter!=_activeGraphicsContexts.end(); ++iter )
{
if ( drawable->getDisplayList(*iter) == 0 ) return false;
}
}
return true;
}
/** Iterate through the active PagedLOD nodes children removing
* children which havn't been visited since specified expiryTime.
* note, should be only be called from the update thread. */
virtual void removeExpiredSubgraphs(double currentFrameTime);
/** Add the loaded data to the scene graph.*/
void addLoadedDataToSceneGraph(double currentFrameTime);
bool _done;
bool _acceptNewRequests;
bool _databasePagerThreadPaused;
DatabaseThreadList _databaseThreads;
int _numFramesActive;
mutable OpenThreads::Mutex _numFramesActiveMutex;
int _frameNumber;
RequestQueue _fileRequestQueue;
RequestQueue _httpRequestQueue;
//DatabaseRequestList _fileRequestList;
//mutable OpenThreads::Mutex _fileRequestListMutex;
DatabaseRequestList _dataToCompileList;
mutable OpenThreads::Mutex _dataToCompileListMutex;
DrawablePolicy _drawablePolicy;
bool _changeAutoUnRef;
bool _valueAutoUnRef;
bool _changeAnisotropy;
float _valueAnisotropy;
bool _deleteRemovedSubgraphsInDatabaseThread;
//ObjectList _childrenToDeleteList;
//mutable OpenThreads::Mutex _childrenToDeleteListMutex;
DatabaseRequestList _dataToMergeList;
mutable OpenThreads::Mutex _dataToMergeListMutex;
PagedLODList _pagedLODList;
double _expiryDelay;
ActiveGraphicsContexts _activeGraphicsContexts;
// CompileGraphicsContexts _compileGraphicsContexts;
bool _doPreCompile;
double _targetFrameRate;
double _minimumTimeAvailableForGLCompileAndDeletePerFrame;
unsigned int _maximumNumOfObjectsToCompilePerFrame;
double _minimumTimeToMergeTile;
double _maximumTimeToMergeTile;
double _totalTimeToMergeTiles;
unsigned int _numTilesMerges;
struct CompileOperation : public osg::GraphicsOperation
{
CompileOperation(DatabasePager* databasePager);
virtual void operator () (osg::GraphicsContext* context);
osg::observer_ptr<DatabasePager> _databasePager;
};
};
}
#endif