rewrite ModelRegistry callbacks as a template with pluggable policy classes

In a big effort to improve use of the object cache, provide a
ModelRegistryCallback template class with different policies for
substitution,  caching, optimization, etc.

Change SGTexDataVarianceVistor to make StateSets static too.

simgear/scene/model/ModelRegistry.cxx

@@ -1,3 +1,21 @@
+// ModelRegistry.hxx -- interface to the OSG model registry
+//
+// Copyright (C) 2005-2007 Mathias Froehlich 
+// Copyright (C) 2007  Tim Moore <timoore@redhat.com>
+//
+// This program is free software; you can redistribute it and/or
+// modify it under the terms of the GNU General Public License as
+// published by the Free Software Foundation; either version 2 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "ModelRegistry.hxx"
 
 #include <osg/observer_ptr>
@@ -27,6 +45,7 @@
 
 using namespace std;
 using namespace osg;
+using namespace osgUtil;
 using namespace osgDB;
 using namespace simgear;
 
@@ -156,6 +175,14 @@ public:
     
     texture->setDataVariance(Object::STATIC);
   }
+
+  virtual void apply(StateSet* stateSet)
+  {
+    if (!stateSet)
+      return;
+    SGTextureStateAttributeVisitor::apply(stateSet);
+    stateSet->setDataVariance(Object::STATIC);
+  }
 };
 
 class SGAcMaterialCrippleVisitor : public SGStateAttributeVisitor {
@@ -199,154 +226,108 @@ ModelRegistry::readImage(const string& fileName,
     return res;
 }
 
-ReaderWriter::ReadResult
+
-ModelRegistry::readNode(const string& fileName,
+osg::Node* DefaultCachePolicy::find(const string& fileName,
-                        const ReaderWriter::Options* opt)
+                                    const ReaderWriter::Options* opt)
 {
     Registry* registry = Registry::instance();
-    ReaderWriter::ReadResult res;
+    osg::Node* cached
-    Node* cached = 0;
+        = dynamic_cast<Node*>(registry->getFromObjectCache(fileName));
-    CallbackMap::iterator iter
+    if (cached)
-        = nodeCallbackMap.find(getFileExtension(fileName));
-    if (iter != nodeCallbackMap.end() && iter->second.valid())
-        return iter->second->readNode(fileName, opt);
-    // First, look for a file with the same name, and the extension
-    // ".osg" and, if it exists, load it instead. This allows for
-    // substitution of optimized models for ones named in the scenery.
-    bool optimizeModel = true;
-    string fileSansExtension = getNameLessExtension(fileName);
-    string osgFileName = fileSansExtension + ".osg";
-    string absFileName = findDataFile(osgFileName);
-    // The absolute file name is passed to the reader plugin, which
-    // calls findDataFile again... but that's OK, it should find the
-    // file by its absolute path straight away.
-    if (fileExists(absFileName)) {
-        optimizeModel = false;
-    } else {
-        absFileName = findDataFile(fileName);
-    }
-    if (!fileExists(absFileName)) {
-        SG_LOG(SG_IO, SG_ALERT, "Cannot find model file \""
-               << fileName << "\"");
-        return ReaderWriter::ReadResult::FILE_NOT_FOUND;
-    }
-    cached
-        = dynamic_cast<Node*>(registry->getFromObjectCache(absFileName));
-    if (cached) {
         SG_LOG(SG_IO, SG_INFO, "Got cached model \""
-               << absFileName << "\"");
+               << fileName << "\"");
-    } else {
+    else
         SG_LOG(SG_IO, SG_INFO, "Reading model \""
-               << absFileName << "\"");
+               << fileName << "\"");
-        res = registry->readNodeImplementation(absFileName, opt);
+    return cached;
-        if (!res.validNode())
+}
-            return res;
 
-        bool needTristrip = true;
+void DefaultCachePolicy::addToCache(const string& fileName,
-        if (getLowerCaseFileExtension(fileName) == "ac") {
+                                    osg::Node* node)
-            // we get optimal geometry from the loader.
+{
-            needTristrip = false;
+    Registry::instance()->addEntryToObjectCache(fileName, node);
-            Matrix m(1, 0, 0, 0,
+}
-                     0, 0, 1, 0,
-                     0, -1, 0, 0,
-                     0, 0, 0, 1);
-        
-            ref_ptr<Group> root = new Group;
-            MatrixTransform* transform = new MatrixTransform;
-            root->addChild(transform);
-        
-            transform->setDataVariance(Object::STATIC);
-            transform->setMatrix(m);
-            transform->addChild(res.getNode());
-        
-            res = ReaderWriter::ReadResult(0);
 
-            if (optimizeModel) {
+// Optimizations we don't use:
-                osgUtil::Optimizer optimizer;
+// Don't use this one. It will break animation names ...
-                unsigned opts = osgUtil::Optimizer::FLATTEN_STATIC_TRANSFORMS;
+// opts |= osgUtil::Optimizer::REMOVE_REDUNDANT_NODES;
-                optimizer.optimize(root.get(), opts);
+//
-            }
+// opts |= osgUtil::Optimizer::REMOVE_LOADED_PROXY_NODES;
+// opts |= osgUtil::Optimizer::COMBINE_ADJACENT_LODS;
+// opts |= osgUtil::Optimizer::CHECK_GEOMETRY;
+// opts |= osgUtil::Optimizer::SPATIALIZE_GROUPS;
+// opts |= osgUtil::Optimizer::COPY_SHARED_NODES;
+// opts |= osgUtil::Optimizer::TESSELATE_GEOMETRY;
+// opts |= osgUtil::Optimizer::OPTIMIZE_TEXTURE_SETTINGS;
 
-            // strip away unneeded groups
+OptimizeModelPolicy::OptimizeModelPolicy(const string& extension) :
-            if (root->getNumChildren() == 1 && root->getName().empty()) {
+    _osgOptions(Optimizer::SHARE_DUPLICATE_STATE
-                res = ReaderWriter::ReadResult(root->getChild(0));
+                | Optimizer::MERGE_GEOMETRY
-            } else
+                | Optimizer::FLATTEN_STATIC_TRANSFORMS
-                res = ReaderWriter::ReadResult(root.get());
+                | Optimizer::TRISTRIP_GEOMETRY)
-        
+{
-            // Ok, this step is questionable.
+}
-            // It is there to have the same visual appearance of ac objects for the
-            // first cut. Osg's ac3d loader will correctly set materials from the
-            // ac file. But the old plib loader used GL_AMBIENT_AND_DIFFUSE for the
-            // materials that in effect igored the ambient part specified in the
-            // file. We emulate that for the first cut here by changing all
-            // ac models here. But in the long term we should use the
-            // unchanged model and fix the input files instead ...
-            SGAcMaterialCrippleVisitor matCriple;
-            res.getNode()->accept(matCriple);
-        }
 
-        if (optimizeModel) {
+osg::Node* OptimizeModelPolicy::optimize(osg::Node* node,
-            osgUtil::Optimizer optimizer;
+                                         const string& fileName,
-            unsigned opts = 0;
+                                         const osgDB::ReaderWriter::Options* opt)
-            // Don't use this one. It will break animation names ...
+{
-            // opts |= osgUtil::Optimizer::REMOVE_REDUNDANT_NODES;
+    osgUtil::Optimizer optimizer;
+    optimizer.optimize(node, _osgOptions);
 
-            // opts |= osgUtil::Optimizer::REMOVE_LOADED_PROXY_NODES;
+    // Make sure the data variance of sharable objects is set to
-            // opts |= osgUtil::Optimizer::COMBINE_ADJACENT_LODS;
+    // STATIC so that textures will be globally shared.
-            // opts |= osgUtil::Optimizer::SHARE_DUPLICATE_STATE;
+    SGTexDataVarianceVisitor dataVarianceVisitor;
-            opts |= osgUtil::Optimizer::MERGE_GEOMETRY;
+    node->accept(dataVarianceVisitor);
-            // opts |= osgUtil::Optimizer::CHECK_GEOMETRY;
-            // opts |= osgUtil::Optimizer::SPATIALIZE_GROUPS;
-            // opts |= osgUtil::Optimizer::COPY_SHARED_NODES;
-            opts |= osgUtil::Optimizer::FLATTEN_STATIC_TRANSFORMS;
-            if (needTristrip)
-                opts |= osgUtil::Optimizer::TRISTRIP_GEOMETRY;
-            // opts |= osgUtil::Optimizer::TESSELATE_GEOMETRY;
-            // opts |= osgUtil::Optimizer::OPTIMIZE_TEXTURE_SETTINGS;
-            optimizer.optimize(res.getNode(), opts);
-        }
-        // Make sure the data variance of sharable objects is set to STATIC ...
-        SGTexDataVarianceVisitor dataVarianceVisitor;
-        res.getNode()->accept(dataVarianceVisitor);
-        // ... so that textures are now globally shared
-        registry->getSharedStateManager()->share(res.getNode());
       
-        SGTexCompressionVisitor texComp;
+    SGTexCompressionVisitor texComp;
-        res.getNode()->accept(texComp);
+    node->accept(texComp);
-        cached = res.getNode();
+    return node;
-        registry->addEntryToObjectCache(absFileName, cached);
+}
-    }
+
+osg::Node* DefaultCopyPolicy::copy(osg::Node* model, const string& fileName,
+                    const osgDB::ReaderWriter::Options* opt)
+{
     // Add an extra reference to the model stored in the database.
     // That it to avoid expiring the object from the cache even if it is still
     // in use. Note that the object cache will think that a model is unused
     // if the reference count is 1. If we clone all structural nodes here
     // we need that extra reference to the original object
     SGDatabaseReference* databaseReference;
-    databaseReference = new SGDatabaseReference(cached);
+    databaseReference = new SGDatabaseReference(model);
     CopyOp::CopyFlags flags = CopyOp::DEEP_COPY_ALL;
     flags &= ~CopyOp::DEEP_COPY_TEXTURES;
     flags &= ~CopyOp::DEEP_COPY_IMAGES;
+    flags &= ~CopyOp::DEEP_COPY_STATESETS;
+    flags &= ~CopyOp::DEEP_COPY_STATEATTRIBUTES;
     flags &= ~CopyOp::DEEP_COPY_ARRAYS;
     flags &= ~CopyOp::DEEP_COPY_PRIMITIVES;
     // This will safe display lists ...
     flags &= ~CopyOp::DEEP_COPY_DRAWABLES;
     flags &= ~CopyOp::DEEP_COPY_SHAPES;
-    res = ReaderWriter::ReadResult(CopyOp(flags)(cached));
+    osg::Node* res = CopyOp(flags)(model);
-    res.getNode()->addObserver(databaseReference);
+    res->addObserver(databaseReference);
 
     // Update liveries
     SGTextureUpdateVisitor liveryUpdate(getDataFilePathList());
-    res.getNode()->accept(liveryUpdate);
+    res->accept(liveryUpdate);
-
-    // Make sure the data variance of sharable objects is set to STATIC ...
-    SGTexDataVarianceVisitor dataVarianceVisitor;
-    res.getNode()->accept(dataVarianceVisitor);
-    // ... so that textures are now globally shared
-    registry->getOrCreateSharedStateManager()->share(res.getNode(), 0);
-
     return res;
 }
 
+string OSGSubstitutePolicy::substitute(const string& name,
+                                       const ReaderWriter::Options* opt)
+{
+    string fileSansExtension = getNameLessExtension(name);
+    string osgFileName = fileSansExtension + ".osg";
+    string absFileName = findDataFile(osgFileName);
+    return absFileName;
+}
+
+ModelRegistry::ModelRegistry() :
+    _defaultCallback(new DefaultCallback(""))
+{
+}
+
 void
 ModelRegistry::addImageCallbackForExtension(const string& extension,
                                             Registry::ReadFileCallback* callback)
@@ -371,6 +352,20 @@ ModelRegistry* ModelRegistry::getInstance()
     return instance.get();
 }
 
+ReaderWriter::ReadResult
+ModelRegistry::readNode(const string& fileName,
+                        const ReaderWriter::Options* opt)
+{
+    Registry* registry = Registry::instance();
+    ReaderWriter::ReadResult res;
+    Node* cached = 0;
+    CallbackMap::iterator iter
+        = nodeCallbackMap.find(getFileExtension(fileName));
+    if (iter != nodeCallbackMap.end() && iter->second.valid())
+        return iter->second->readNode(fileName, opt);
+    return _defaultCallback->readNode(fileName, opt);
+}
+
 class SGReadCallbackInstaller {
 public:
   SGReadCallbackInstaller()
@@ -381,20 +376,69 @@ public:
 
     Registry* registry = Registry::instance();
     ReaderWriter::Options* options = new ReaderWriter::Options;
-    // We manage node caching ourselves
+    int cacheOptions = ReaderWriter::Options::CACHE_ALL;
-    int cacheOptions = ReaderWriter::Options::CACHE_ALL
-      & ~ReaderWriter::Options::CACHE_NODES;
     options->
       setObjectCacheHint((ReaderWriter::Options::CacheHintOptions)cacheOptions);
     registry->setOptions(options);
     registry->getOrCreateSharedStateManager()->
-      setShareMode(SharedStateManager::SHARE_TEXTURES);
+      setShareMode(SharedStateManager::SHARE_STATESETS);
     registry->setReadFileCallback(ModelRegistry::getInstance());
   }
 };
 
 static SGReadCallbackInstaller readCallbackInstaller;
 
+// we get optimal geometry from the loader.
+struct ACOptimizePolicy : public OptimizeModelPolicy {
+    ACOptimizePolicy(const string& extension)  :
+        OptimizeModelPolicy(extension)
+    {
+        _osgOptions &= ~Optimizer::TRISTRIP_GEOMETRY;
+    }
+};
+
+struct ACProcessPolicy {
+    ACProcessPolicy(const string& extension) {}
+    Node* process(Node* node, const string& filename,
+                  const ReaderWriter::Options* opt)
+    {
+        Matrix m(1, 0, 0, 0,
+                 0, 0, 1, 0,
+                 0, -1, 0, 0,
+                 0, 0, 0, 1);
+        // XXX Does there need to be a Group node here to trick the
+        // optimizer into optimizing the static transform?
+        osg::Group* root = new Group;
+        MatrixTransform* transform = new MatrixTransform;
+        root->addChild(transform);
+        
+        transform->setDataVariance(Object::STATIC);
+        transform->setMatrix(m);
+        transform->addChild(node);
+        // Ok, this step is questionable.
+        // It is there to have the same visual appearance of ac objects for the
+        // first cut. Osg's ac3d loader will correctly set materials from the
+        // ac file. But the old plib loader used GL_AMBIENT_AND_DIFFUSE for the
+        // materials that in effect igored the ambient part specified in the
+        // file. We emulate that for the first cut here by changing all
+        // ac models here. But in the long term we should use the
+        // unchanged model and fix the input files instead ...
+        SGAcMaterialCrippleVisitor matCriple;
+        root->accept(matCriple);
+        return root;
+    }
+};
+
+typedef ModelRegistryCallback<ACProcessPolicy, DefaultCachePolicy,
+                              ACOptimizePolicy, DefaultCopyPolicy,
+                              OSGSubstitutePolicy> ACCallback;
+
+namespace
+{
+ModelRegistryCallbackProxy<ACCallback> g_acRegister("ac");
+}   
+
+
 ReaderWriter::ReadResult
 OSGFileCallback::readImage(const string& fileName,
                            const ReaderWriter::Options* opt)

simgear/scene/model/ModelRegistry.hxx

@@ -1,5 +1,6 @@
 // ModelRegistry.hxx -- interface to the OSG model registry
 //
+// Copyright (C) 2005-2007 Mathias Froehlich 
 // Copyright (C) 2007  Tim Moore <timoore@redhat.com>
 //
 // This program is free software; you can redistribute it and/or
@@ -19,6 +20,9 @@
 #define _SG_MODELREGISTRY_HXX 1
 
 #include <osg/ref_ptr>
+#include <osg/Node>
+#include <osgDB/FileUtils>
+#include <osgDB/FileNameUtils>
 #include <osgDB/ReaderWriter>
 #include <osgDB/Registry>
 
@@ -27,10 +31,165 @@
 #include STL_STRING
 #include <map>
 
+// Class to register per file extension read callbacks with the OSG
+// registry, mostly to control caching and post load optimization /
+// copying that happens above the level of the ReaderWriter.
 namespace simgear
 {
+
+// Different caching and optimization strategies are needed for
+// different file types. Most loaded files should be optimized and the
+// optimized version should be cached. When an .osg file is
+// substituted for another, it is assumed to be optimized already but
+// it should be cached too (under the name of the original?). .stg
+// files should not be cached (that's the pager's job) but the files
+// it causes to be loaded should be. .btg files are already optimized
+// and shouldn't be cached.
+//
+// Complicating this is the effect that removing CACHE_NODES has from
+// the ReaderWriter options: it switches the object cache with an
+// empty one, so that's not an option for the files that could be
+// loaded from a .stg file. So, we'll let
+// Registry::readNodeImplementation cache a loaded file and then add
+// the optimized version to the cache ourselves, replacing the
+// original subgraph.
+//
+// To support all these options with a minimum of duplication, the
+// readNode function is specified as a template with a bunch of
+// pluggable (and predefined) policies.
+template <typename ProcessPolicy, typename CachePolicy, typename OptimizePolicy,
+          typename CopyPolicy, typename SubstitutePolicy>
+class ModelRegistryCallback : public osgDB::Registry::ReadFileCallback {
+public:
+    ModelRegistryCallback(const std::string& extension) :
+        _processPolicy(extension), _cachePolicy(extension),
+        _optimizePolicy(extension), _copyPolicy(extension),
+        _substitutePolicy(extension)
+    {
+    }
+    virtual osgDB::ReaderWriter::ReadResult
+    readNode(const std::string& fileName,
+             const osgDB::ReaderWriter::Options* opt)
+    {
+        using namespace osg;
+        using namespace osgDB;
+        using osgDB::ReaderWriter;
+        Registry* registry = Registry::instance();
+        std::string usedFileName = _substitutePolicy.substitute(fileName, opt);
+        if (usedFileName.empty())
+            usedFileName = fileName;
+        ref_ptr<osg::Node> loadedNode = _cachePolicy.find(usedFileName, opt);
+        if (!loadedNode.valid()) {
+            ReaderWriter* rw = registry ->getReaderWriterForExtension(osgDB::getFileExtension(usedFileName));
+            if (!rw)
+                return ReaderWriter::ReadResult(); // FILE_NOT_HANDLED
+            ReaderWriter::ReadResult res = rw->readNode(usedFileName, opt);
+            if (!res.validNode())
+                return res;
+            ref_ptr<osg::Node> processedNode
+                = _processPolicy.process(res.getNode(), usedFileName, opt);
+            ref_ptr<osg::Node> optimizedNode
+                = _optimizePolicy.optimize(processedNode.get(), usedFileName,
+                                           opt);
+            _cachePolicy.addToCache(usedFileName, optimizedNode.get());
+            loadedNode = optimizedNode;
+        }
+        return ReaderWriter::ReadResult(_copyPolicy.copy(loadedNode.get(),
+                                                         usedFileName,
+                                                         opt));
+    }
+protected:
+    ProcessPolicy _processPolicy;
+    CachePolicy _cachePolicy;
+    OptimizePolicy _optimizePolicy;
+    CopyPolicy _copyPolicy;
+    SubstitutePolicy _substitutePolicy;
+    virtual ~ModelRegistryCallback() {}
+};
+
+// Predefined policies
+
+struct DefaultProcessPolicy {
+    DefaultProcessPolicy(const std::string& extension) {}
+    osg::Node* process(osg::Node* node, const std::string& filename,
+                       const osgDB::ReaderWriter::Options* opt)
+    {
+        return node;
+    }
+};
+
+struct DefaultCachePolicy {
+    DefaultCachePolicy(const std::string& extension) {}
+    osg::Node* find(const std::string& fileName,
+                    const osgDB::ReaderWriter::Options* opt);
+    void addToCache(const std::string& filename, osg::Node* node);
+};
+
+struct NoCachePolicy {
+    NoCachePolicy(const std::string& extension) {}
+    osg::Node* find(const std::string& fileName,
+                    const osgDB::ReaderWriter::Options* opt)
+    {
+        return 0;
+    }
+    void addToCache(const std::string& filename, osg::Node* node) {}
+};
+
+class OptimizeModelPolicy {
+public:
+    OptimizeModelPolicy(const std::string& extension);
+    osg::Node* optimize(osg::Node* node, const std::string& fileName,
+                        const osgDB::ReaderWriter::Options* opt);
+protected:
+    unsigned _osgOptions;
+};
+
+struct NoOptimizePolicy {
+    NoOptimizePolicy(const std::string& extension) {}
+    osg::Node* optimize(osg::Node* node, const std::string& fileName,
+                        const osgDB::ReaderWriter::Options* opt)
+    {
+        return node;
+    }
+};
+
+struct DefaultCopyPolicy {
+    DefaultCopyPolicy(const std::string& extension) {}
+    osg::Node* copy(osg::Node* node, const std::string& fileName,
+                    const osgDB::ReaderWriter::Options* opt);
+};
+
+struct NoCopyPolicy {
+    NoCopyPolicy(const std::string& extension) {}
+    osg::Node* copy(osg::Node* node, const std::string& fileName,
+                    const osgDB::ReaderWriter::Options* opt)
+    {
+        return node;
+    }
+};
+
+struct OSGSubstitutePolicy {
+    OSGSubstitutePolicy(const std::string& extension) {}
+    std::string substitute(const std::string& name,
+                           const osgDB::ReaderWriter::Options* opt);
+};
+
+struct NoSubstitutePolicy {
+    NoSubstitutePolicy(const std::string& extension) {}
+    std::string substitute(const std::string& name,
+                           const osgDB::ReaderWriter::Options* opt)
+    {
+        return std::string();
+    }
+};
+typedef ModelRegistryCallback<DefaultProcessPolicy, DefaultCachePolicy,
+                              OptimizeModelPolicy, DefaultCopyPolicy,
+                              OSGSubstitutePolicy> DefaultCallback;
+
+// The manager for the callbacks
 class ModelRegistry : public osgDB::Registry::ReadFileCallback {
 public:
+    ModelRegistry();
     virtual osgDB::ReaderWriter::ReadResult
     readImage(const std::string& fileName,
               const osgDB::ReaderWriter::Options* opt);
@@ -44,14 +203,22 @@ public:
                                      osgDB::Registry::ReadFileCallback*
                                      callback);
     static ModelRegistry* getInstance();
+    virtual ~ModelRegistry() {}
 protected:
     static osg::ref_ptr<ModelRegistry> instance;
     typedef std::map<std::string, osg::ref_ptr<osgDB::Registry::ReadFileCallback> >
     CallbackMap;
     CallbackMap imageCallbackMap;
     CallbackMap nodeCallbackMap;
+    osg::ref_ptr<DefaultCallback> _defaultCallback;
 };
 
+// Callback that only loads the file without any caching or
+// postprocessing.
+typedef ModelRegistryCallback<DefaultProcessPolicy, NoCachePolicy,
+                              NoOptimizePolicy, NoCopyPolicy,
+                              NoSubstitutePolicy> LoadOnlyCallback;
+
 // Proxy for registering extension-based callbacks
 
 template<typename T>
@@ -60,8 +227,8 @@ class ModelRegistryCallbackProxy
 public:
     ModelRegistryCallbackProxy(std::string extension)
     {
-        ModelRegistry::getInstance()->addNodeCallbackForExtension(extension,
+        ModelRegistry::getInstance()
-                                                                  new T);
+            ->addNodeCallbackForExtension(extension, new T(extension));
     }
 };
 

simgear/scene/tgdb/SGReaderWriterBTG.cxx

@@ -71,5 +71,5 @@ SGReaderWriterBTG::readNode(const std::string& fileName,
 
 namespace
 {
-ModelRegistryCallbackProxy<OSGFileCallback> g_btgCallbackProxy("btg");
+ModelRegistryCallbackProxy<LoadOnlyCallback> g_btgCallbackProxy("btg");
 }