From 8b35c9a4b121c425caa264d27d0497d93313485d Mon Sep 17 00:00:00 2001 From: Richard Harrison Date: Thu, 24 Jan 2019 18:05:37 +0100 Subject: [PATCH] Improvements to the DDS texture cache - use a file contents hash instead of filepath. - add a local lru cache for filepath->hash (for performance improvements) - calculate and resize to nearest power of two - handle normal maps and images from effects differently. - when cache is active any image that can't be converted to a dds will have a mipmap generated (which still helps the loading process); although this may be responsible for introducing purple into transparent images.. --- simgear/scene/model/ModelRegistry.cxx | 482 +++++++++++++++++++++----- 1 file changed, 394 insertions(+), 88 deletions(-) diff --git a/simgear/scene/model/ModelRegistry.cxx b/simgear/scene/model/ModelRegistry.cxx index 3756fae7..b5bbc7b9 100644 --- a/simgear/scene/model/ModelRegistry.cxx +++ b/simgear/scene/model/ModelRegistry.cxx @@ -60,6 +60,8 @@ #include #include #include +#include +#include #include "BoundingVolumeBuildVisitor.hxx" #include "model.hxx" @@ -180,9 +182,34 @@ public: } // namespace -static bool isPowerOfTwo(int width, int height) +static int nearestPowerOfTwo(unsigned int _v) { - return (((width & (width - 1)) == 0) && ((height & (height - 1))) == 0); + // uint v; // compute the next highest power of 2 of 32-bit v + unsigned int v = (unsigned int)_v; + bool neg = _v < 0; + if (neg) + v = (unsigned int)(-_v); + + v &= (2 << 16) - 1; // make +ve + + // bit twiddle to round up to nearest pot. + v--; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + v++; + + if (neg) + _v = -(int)v; + else + _v = (int)v; + return v; +} + static bool isPowerOfTwo(int v) +{ + return ((v & (v - 1)) == 0); } osg::Node* DefaultProcessPolicy::process(osg::Node* node, const std::string& filename, const Options* opt) @@ -205,6 +232,139 @@ osg::Image* getImageByName(const std::string& filename) return nullptr; } #endif +// a cache which evicts the least recently used item when it is full +#include +#include +#include + +#include +template +class lru_cache +{ +public: + SGMutex _mutex; + + typedef Key key_type; + typedef Value value_type; + typedef std::list list_type; + typedef std::map< + key_type, + std::pair + > map_type; + + lru_cache(size_t capacity) + : m_capacity(capacity) + { + } + + ~lru_cache() + { + } + + + size_t size() const + { + return m_map.size(); + } + + size_t capacity() const + { + return m_capacity; + } + + bool empty() const + { + return m_map.empty(); + } + + bool contains(const key_type &key) + { + SGGuard scopeLock(_mutex); + return m_map.find(key) != m_map.end(); + } + + void insert(const key_type &key, const value_type &value) + { + SGGuard scopeLock(_mutex); + typename map_type::iterator i = m_map.find(key); + if (i == m_map.end()) { + // insert item into the cache, but first check if it is full + if (size() >= m_capacity) { + // cache is full, evict the least recently used item + evict(); + } + + // insert the new item + m_list.push_front(key); + m_map[key] = std::make_pair(value, m_list.begin()); + } + } + boost::optional findValue(const std::string &requiredValue) + { + SGGuard scopeLock(_mutex); + for (typename map_type::iterator it = m_map.begin(); it != m_map.end(); ++it) + if (it->second.first == requiredValue) + return it->first; + return boost::none; + } + boost::optional get(const key_type &key) + { + SGGuard scopeLock(_mutex); + // lookup value in the cache + typename map_type::iterator i = m_map.find(key); + if (i == m_map.end()) { + // value not in cache + return boost::none; + } + + // return the value, but first update its place in the most + // recently used list + typename list_type::iterator j = i->second.second; + if (j != m_list.begin()) { + // move item to the front of the most recently used list + m_list.erase(j); + m_list.push_front(key); + + // update iterator in map + j = m_list.begin(); + const value_type &value = i->second.first; + m_map[key] = std::make_pair(value, j); + + // return the value + return value; + } + else { + // the item is already at the front of the most recently + // used list so just return it + return i->second.first; + } + } + + void clear() + { + SGGuard scopeLock(_mutex); + m_map.clear(); + m_list.clear(); + } + +private: + void evict() + { + SGGuard scopeLock(_mutex); + // evict item from the end of most recently used list + typename list_type::iterator i = --m_list.end(); + m_map.erase(*i); + m_list.erase(i); + } + +private: + map_type m_map; + list_type m_list; + size_t m_capacity; +}; +lru_cache < std::string, std::string> filename_hash_cache(100000); +lru_cache < std::string, bool> filesCleaned(100000); +static bool refreshCache = false; ReaderWriter::ReadResult ModelRegistry::readImage(const string& fileName, @@ -248,6 +408,8 @@ ModelRegistry::readImage(const string& fileName, if (cache_active) { if (fileExtension != "dds" && fileExtension != "gz") { + const SGReaderWriterOptions* sgoptC = dynamic_cast(opt); + std::string root = getPathRoot(absFileName); std::string prr = getPathRelative(root, absFileName); std::string cache_root = SGSceneFeatures::instance()->getTextureCompressionPath().c_str(); @@ -255,99 +417,231 @@ ModelRegistry::readImage(const string& fileName, SGPath file(absFileName); std::stringstream tstream; - tstream << std::hex << file.modTime(); - newName += "." + tstream.str(); - newName += ".cache.dds"; - if (!fileExists(newName)) { - res = registry->readImageImplementation(absFileName, opt); + // calucate and use hash for storing cached image. This also + // helps with sharing of identical images between models. + if (fileExists(absFileName)) { + SGFile f(absFileName); + std::string hash; + boost::optional cachehash = filename_hash_cache.get(absFileName); + if (cachehash) { + hash = *cachehash; +// SG_LOG(SG_IO, SG_ALERT, "Hash for " + absFileName + " in cache " + hash); + } + else { +// SG_LOG(SG_IO, SG_ALERT, "Creating hash for " + absFileName); + hash = f.computeHash(); + filename_hash_cache.insert(absFileName, hash); + boost::optional cacheFilename = filename_hash_cache.findValue(hash); - if (res.validImage()) { - osg::ref_ptr srcImage = res.getImage(); - int width = srcImage->s(); - bool transparent = srcImage->isImageTranslucent(); - int height = srcImage->t(); - - if (height >= max_texture_size) - { - SG_LOG(SG_IO, SG_WARN, "Image texture too high " << width << "," << height << absFileName); - osg::ref_ptr resizedImage; - int factor = height / max_texture_size; - if (ImageUtils::resizeImage(srcImage, width / factor, height / factor, resizedImage)) - srcImage = resizedImage; - width = srcImage->s(); - height = srcImage->t(); - } - if (width >= max_texture_size) - { - SG_LOG(SG_IO, SG_WARN, "Image texture too wide " << width << "," << height << absFileName); - osg::ref_ptr resizedImage; - int factor = width / max_texture_size; - if (ImageUtils::resizeImage(srcImage, width / factor, height / factor, resizedImage)) - srcImage = resizedImage; - width = srcImage->s(); - height = srcImage->t(); - } - - // - // only cache power of two textures that are of a reasonable size - if (width >= 64 && height >= 64 && isPowerOfTwo(width, height)) { - simgear::effect::MipMapTuple mipmapFunctions(simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE); - - SGPath filePath(newName); - filePath.create_dir(); - - // setup the options string for saving the texture as we don't want OSG to auto flip the texture - // as this complicates loading as it requires a flag to flip it back which will preclude the - // image from being cached because we will have to clone the options to set the flag and thus lose - // the link to the cache in the options from the caller. - osg::ref_ptr nopt; - nopt = opt->cloneOptions(); - std::string optionstring = nopt->getOptionString(); - - if (!optionstring.empty()) - optionstring += " "; - - nopt->setOptionString(optionstring + "ddsNoAutoFlipWrite"); - - /* - * decide if we need to compress this. - */ - bool compress = (transparent && compress_transparent) || (!transparent && compress_solid); - - if (compress) { - if (processor) - { - if (transparent) - processor->compress(*srcImage, osg::Texture::USE_S3TC_DXT5_COMPRESSION, true, true, osgDB::ImageProcessor::USE_CPU, osgDB::ImageProcessor::PRODUCTION); - else - processor->compress(*srcImage, osg::Texture::USE_S3TC_DXT1_COMPRESSION, true, true, osgDB::ImageProcessor::USE_CPU, osgDB::ImageProcessor::PRODUCTION); - //processor->generateMipMap(*srcImage, true, osgDB::ImageProcessor::USE_CPU); - } - else { - simgear::effect::MipMapTuple mipmapFunctions(simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE); - SG_LOG(SG_IO, SG_WARN, "Texture compression plugin (osg_nvtt) not available; storing uncompressed image: " << newName); - srcImage = simgear::effect::computeMipmap(srcImage, mipmapFunctions); - } - } - else { - if (processor) { - processor->generateMipMap(*srcImage, true, osgDB::ImageProcessor::USE_CPU); - } - else { - simgear::effect::MipMapTuple mipmapFunctions(simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE); - srcImage = simgear::effect::computeMipmap(srcImage, mipmapFunctions); - } - } - registry->writeImage(*srcImage, newName, nopt); - absFileName = newName; - } - } + // possibly a shared texture - but warn the user to allow investigation. + if (cacheFilename && *cacheFilename != absFileName) { + SG_LOG(SG_IO, SG_ALERT, " Already have " + hash + " : " + *cacheFilename + " not "+absFileName); + } +// SG_LOG(SG_IO, SG_ALERT, " >>>> " + hash + " :: " + newName); + } + newName = cache_root + "/" + hash.substr(0,2) + "/" + hash + ".cache.dds"; } else + { + tstream << std::hex << file.modTime(); + newName += "." + tstream.str(); + newName += ".cache.dds"; + } + bool doRefresh = refreshCache; + //if (fileExists(newName) && sgoptC && sgoptC->getLoadOriginHint() == SGReaderWriterOptions::LoadOriginHint::ORIGIN_EFFECTS) { + // doRefresh = true; + // + //} + + if (fileExists(newName) && doRefresh) { + if (!filesCleaned.contains(newName)) { + SG_LOG(SG_IO, SG_ALERT, "Removing previously cached effects image " + newName); + SGPath(newName).remove(); + filesCleaned.insert(newName, true); + } + + } + + if (!fileExists(newName)) { + res = registry->readImageImplementation(absFileName, opt); + if (res.validImage()) { + osg::ref_ptr srcImage = res.getImage(); + int width = srcImage->s(); + bool transparent = srcImage->isImageTranslucent(); + bool isNormalMap = false; + bool isEffect = false; + /* + * decide if we need to compress this. + */ + bool can_compress = (transparent && compress_transparent) || (!transparent && compress_solid); + + int height = srcImage->t(); + + // use the new file origin to determine any special processing + // we handle the following + // - normal maps + // - images loaded from effects + if (sgoptC && transparent && sgoptC->getLoadOriginHint() == SGReaderWriterOptions::LoadOriginHint::ORIGIN_EFFECTS_NORMALIZED) { + isNormalMap = true; + } + else if (sgoptC && transparent && sgoptC->getLoadOriginHint() == SGReaderWriterOptions::LoadOriginHint::ORIGIN_EFFECTS) { + SG_LOG(SG_IO, SG_ALERT, "From effects transparent " + absFileName); + isEffect = true; +// can_compress = false; + } + else if (sgoptC && transparent && sgoptC->getLoadOriginHint() == SGReaderWriterOptions::LoadOriginHint::ORIGIN_EFFECTS) { + SG_LOG(SG_IO, SG_ALERT, "From effects " + absFileName); + isEffect = true; + } + if (can_compress) + { + std::string pot_message; + bool resize = false; + if (!isPowerOfTwo(width)) { + width = nearestPowerOfTwo(width); + resize = true; + pot_message += std::string(" not POT: resized width to ") + std::to_string(width); + } + if (!isPowerOfTwo(height)) { + height = nearestPowerOfTwo(height); + resize = true; + pot_message += std::string(" not POT: resized height to ") + std::to_string(height); + } +if (pot_message.size()) +SG_LOG(SG_IO, SG_WARN, pot_message << " " << absFileName); + + // unlikely that after resizing in height the width will still be outside of the max texture size. + if (height > max_texture_size) + { + SG_LOG(SG_IO, SG_WARN, "Image texture too high (max " << max_texture_size << ") " << width << "," << height << " " << absFileName); + int factor = height / max_texture_size; + height /= factor; + width /= factor; + resize = true; + } + if (width > max_texture_size) + { + SG_LOG(SG_IO, SG_WARN, "Image texture too wide (max " << max_texture_size << ") " << width << "," << height << " " << absFileName); + int factor = width / max_texture_size; + height /= factor; + width /= factor; + resize = true; + } + if (resize) { + osg::ref_ptr resizedImage; + + if (ImageUtils::resizeImage(srcImage, width, height, resizedImage)) + srcImage = resizedImage; + } + + // + // only cache power of two textures that are of a reasonable size + if (width >= 4 && height >= 4) { + + simgear::effect::MipMapTuple mipmapFunctions(simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE); + + SGPath filePath(newName); + filePath.create_dir(); + + // setup the options string for saving the texture as we don't want OSG to auto flip the texture + // as this complicates loading as it requires a flag to flip it back which will preclude the + // image from being cached because we will have to clone the options to set the flag and thus lose + // the link to the cache in the options from the caller. + osg::ref_ptr nopt; + nopt = opt->cloneOptions(); + std::string optionstring = nopt->getOptionString(); + + if (!optionstring.empty()) + optionstring += " "; + + nopt->setOptionString(optionstring + "ddsNoAutoFlipWrite"); + + //GLenum srcImageType = srcImage->getDataType(); + // printf("--- %-80s --> f=%8x t=%8x\n", newName.c_str(), srcImage->getPixelFormat(), srcImageType); + + try + { + if (can_compress) { + osg::Texture::InternalFormatMode targetFormat = osg::Texture::USE_S3TC_DXT1_COMPRESSION; + if (isNormalMap) { + if (transparent) { + targetFormat = osg::Texture::USE_S3TC_DXT5_COMPRESSION; + } + else + targetFormat = osg::Texture::USE_S3TC_DXT5_COMPRESSION; + } + else if (isEffect) + { + if (transparent) { + targetFormat = osg::Texture::USE_S3TC_DXT5_COMPRESSION; + } + else + targetFormat = osg::Texture::USE_S3TC_DXT1_COMPRESSION; + } + else{ + if (transparent) { + targetFormat = osg::Texture::USE_S3TC_DXT3_COMPRESSION; + } + else + targetFormat = osg::Texture::USE_S3TC_DXT1_COMPRESSION; + } + + if (processor) + { + SG_LOG(SG_IO, SG_ALERT, "Creating " << targetFormat << " for " + absFileName); + // normal maps: + // nvdxt.exe - quality_highest - rescaleKaiser - Kaiser - dxt5nm - norm + processor->compress(*srcImage, targetFormat, true, true, osgDB::ImageProcessor::USE_CPU, osgDB::ImageProcessor::PRODUCTION); + SG_LOG(SG_IO, SG_ALERT, "-- finished creating DDS: " + newName); + //processor->generateMipMap(*srcImage, true, osgDB::ImageProcessor::USE_CPU); + } + else { + simgear::effect::MipMapTuple mipmapFunctions(simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE); + SG_LOG(SG_IO, SG_WARN, "Texture compression plugin (osg_nvtt) not available; storing uncompressed image: " << absFileName); + srcImage = simgear::effect::computeMipmap(srcImage, mipmapFunctions); + } + } + else { + SG_LOG(SG_IO, SG_ALERT, "Creating uncompressed DDS for " + absFileName); + if (processor) { + processor->generateMipMap(*srcImage, true, osgDB::ImageProcessor::USE_CPU); + } + else { + simgear::effect::MipMapTuple mipmapFunctions(simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE); + srcImage = simgear::effect::computeMipmap(srcImage, mipmapFunctions); + } + } + //} + //else + // printf("--- no compress or mipmap of format %s\n", newName.c_str()); + registry->writeImage(*srcImage, newName, nopt); + { + std::string mdlDirectory = cache_root + "/cache-i ndex.txt"; + FILE *f = ::fopen(mdlDirectory.c_str(), "a"); + if (f) + { + ::fprintf(f, "%s, %s\n", absFileName.c_str(), newName.c_str()); + ::fclose(f); + } + } + absFileName = newName; + } + catch (...) { + SG_LOG(SG_IO, SG_ALERT, "Exception processing " << absFileName << " may be corrupted"); + } + } + else + SG_LOG(SG_IO, SG_WARN, absFileName + " too small " << width << "," << height); + } + } + } + else { absFileName = newName; + } } } + res = registry->readImageImplementation(absFileName, opt); if (!res.success()) { @@ -368,6 +662,18 @@ ModelRegistry::readImage(const string& fileName, if (srcImage1->getName().empty()) { srcImage1->setName(absFileName); } + if(cache_active && getFileExtension(absFileName) != "dds") + { + if (processor) { + processor->generateMipMap(*srcImage1, true, osgDB::ImageProcessor::USE_CPU); + SG_LOG(SG_IO, SG_ALERT, "Created nvtt mipmaps DDS for " + absFileName); + } + else { + simgear::effect::MipMapTuple mipmapFunctions(simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE); + srcImage1 = simgear::effect::computeMipmap(srcImage1, mipmapFunctions); + SG_LOG(SG_IO, SG_ALERT, "Created sg mipmaps DDS for " + absFileName); + } + } if (res.loadedFromCache()) SG_LOG(SG_IO, SG_BULK, "Returning cached image \""