Improvements to the DDS texture cache

- use a file contents hash instead of filepath.
- add a local lru cache for filepath->hash (for performance improvements)
- calculate and resize to nearest power of two
- handle normal maps and images from effects differently.
- when cache is active any image that can't be converted to a dds will have a mipmap generated (which still helps the loading process); although this may be responsible for introducing purple into transparent images..
This commit is contained in:
Richard Harrison
2019-01-24 18:05:37 +01:00
parent a5d8eec25f
commit 8b35c9a4b1

View File

@@ -60,6 +60,8 @@
#include <simgear/props/props.hxx>
#include <simgear/props/props_io.hxx>
#include <simgear/props/condition.hxx>
#include <simgear/io/sg_file.hxx>
#include <simgear/threads/SGGuard.hxx>
#include "BoundingVolumeBuildVisitor.hxx"
#include "model.hxx"
@@ -180,9 +182,34 @@ public:
} // namespace
static bool isPowerOfTwo(int width, int height)
static int nearestPowerOfTwo(unsigned int _v)
{
return (((width & (width - 1)) == 0) && ((height & (height - 1))) == 0);
// uint v; // compute the next highest power of 2 of 32-bit v
unsigned int v = (unsigned int)_v;
bool neg = _v < 0;
if (neg)
v = (unsigned int)(-_v);
v &= (2 << 16) - 1; // make +ve
// bit twiddle to round up to nearest pot.
v--;
v |= v >> 1;
v |= v >> 2;
v |= v >> 4;
v |= v >> 8;
v |= v >> 16;
v++;
if (neg)
_v = -(int)v;
else
_v = (int)v;
return v;
}
static bool isPowerOfTwo(int v)
{
return ((v & (v - 1)) == 0);
}
osg::Node* DefaultProcessPolicy::process(osg::Node* node, const std::string& filename,
const Options* opt)
@@ -205,6 +232,139 @@ osg::Image* getImageByName(const std::string& filename)
return nullptr;
}
#endif
// a cache which evicts the least recently used item when it is full
#include <map>
#include <list>
#include <utility>
#include <boost/optional.hpp>
template<class Key, class Value>
class lru_cache
{
public:
SGMutex _mutex;
typedef Key key_type;
typedef Value value_type;
typedef std::list<key_type> list_type;
typedef std::map<
key_type,
std::pair<value_type, typename list_type::iterator>
> map_type;
lru_cache(size_t capacity)
: m_capacity(capacity)
{
}
~lru_cache()
{
}
size_t size() const
{
return m_map.size();
}
size_t capacity() const
{
return m_capacity;
}
bool empty() const
{
return m_map.empty();
}
bool contains(const key_type &key)
{
SGGuard<SGMutex> scopeLock(_mutex);
return m_map.find(key) != m_map.end();
}
void insert(const key_type &key, const value_type &value)
{
SGGuard<SGMutex> scopeLock(_mutex);
typename map_type::iterator i = m_map.find(key);
if (i == m_map.end()) {
// insert item into the cache, but first check if it is full
if (size() >= m_capacity) {
// cache is full, evict the least recently used item
evict();
}
// insert the new item
m_list.push_front(key);
m_map[key] = std::make_pair(value, m_list.begin());
}
}
boost::optional<key_type> findValue(const std::string &requiredValue)
{
SGGuard<SGMutex> scopeLock(_mutex);
for (typename map_type::iterator it = m_map.begin(); it != m_map.end(); ++it)
if (it->second.first == requiredValue)
return it->first;
return boost::none;
}
boost::optional<value_type> get(const key_type &key)
{
SGGuard<SGMutex> scopeLock(_mutex);
// lookup value in the cache
typename map_type::iterator i = m_map.find(key);
if (i == m_map.end()) {
// value not in cache
return boost::none;
}
// return the value, but first update its place in the most
// recently used list
typename list_type::iterator j = i->second.second;
if (j != m_list.begin()) {
// move item to the front of the most recently used list
m_list.erase(j);
m_list.push_front(key);
// update iterator in map
j = m_list.begin();
const value_type &value = i->second.first;
m_map[key] = std::make_pair(value, j);
// return the value
return value;
}
else {
// the item is already at the front of the most recently
// used list so just return it
return i->second.first;
}
}
void clear()
{
SGGuard<SGMutex> scopeLock(_mutex);
m_map.clear();
m_list.clear();
}
private:
void evict()
{
SGGuard<SGMutex> scopeLock(_mutex);
// evict item from the end of most recently used list
typename list_type::iterator i = --m_list.end();
m_map.erase(*i);
m_list.erase(i);
}
private:
map_type m_map;
list_type m_list;
size_t m_capacity;
};
lru_cache < std::string, std::string> filename_hash_cache(100000);
lru_cache < std::string, bool> filesCleaned(100000);
static bool refreshCache = false;
ReaderWriter::ReadResult
ModelRegistry::readImage(const string& fileName,
@@ -248,6 +408,8 @@ ModelRegistry::readImage(const string& fileName,
if (cache_active) {
if (fileExtension != "dds" && fileExtension != "gz") {
const SGReaderWriterOptions* sgoptC = dynamic_cast<const SGReaderWriterOptions*>(opt);
std::string root = getPathRoot(absFileName);
std::string prr = getPathRelative(root, absFileName);
std::string cache_root = SGSceneFeatures::instance()->getTextureCompressionPath().c_str();
@@ -255,99 +417,231 @@ ModelRegistry::readImage(const string& fileName,
SGPath file(absFileName);
std::stringstream tstream;
tstream << std::hex << file.modTime();
newName += "." + tstream.str();
newName += ".cache.dds";
if (!fileExists(newName)) {
res = registry->readImageImplementation(absFileName, opt);
// calucate and use hash for storing cached image. This also
// helps with sharing of identical images between models.
if (fileExists(absFileName)) {
SGFile f(absFileName);
std::string hash;
boost::optional<std::string> cachehash = filename_hash_cache.get(absFileName);
if (cachehash) {
hash = *cachehash;
// SG_LOG(SG_IO, SG_ALERT, "Hash for " + absFileName + " in cache " + hash);
}
else {
// SG_LOG(SG_IO, SG_ALERT, "Creating hash for " + absFileName);
hash = f.computeHash();
filename_hash_cache.insert(absFileName, hash);
boost::optional<std::string> cacheFilename = filename_hash_cache.findValue(hash);
if (res.validImage()) {
osg::ref_ptr<osg::Image> srcImage = res.getImage();
int width = srcImage->s();
bool transparent = srcImage->isImageTranslucent();
int height = srcImage->t();
if (height >= max_texture_size)
{
SG_LOG(SG_IO, SG_WARN, "Image texture too high " << width << "," << height << absFileName);
osg::ref_ptr<osg::Image> resizedImage;
int factor = height / max_texture_size;
if (ImageUtils::resizeImage(srcImage, width / factor, height / factor, resizedImage))
srcImage = resizedImage;
width = srcImage->s();
height = srcImage->t();
}
if (width >= max_texture_size)
{
SG_LOG(SG_IO, SG_WARN, "Image texture too wide " << width << "," << height << absFileName);
osg::ref_ptr<osg::Image> resizedImage;
int factor = width / max_texture_size;
if (ImageUtils::resizeImage(srcImage, width / factor, height / factor, resizedImage))
srcImage = resizedImage;
width = srcImage->s();
height = srcImage->t();
}
//
// only cache power of two textures that are of a reasonable size
if (width >= 64 && height >= 64 && isPowerOfTwo(width, height)) {
simgear::effect::MipMapTuple mipmapFunctions(simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE);
SGPath filePath(newName);
filePath.create_dir();
// setup the options string for saving the texture as we don't want OSG to auto flip the texture
// as this complicates loading as it requires a flag to flip it back which will preclude the
// image from being cached because we will have to clone the options to set the flag and thus lose
// the link to the cache in the options from the caller.
osg::ref_ptr<Options> nopt;
nopt = opt->cloneOptions();
std::string optionstring = nopt->getOptionString();
if (!optionstring.empty())
optionstring += " ";
nopt->setOptionString(optionstring + "ddsNoAutoFlipWrite");
/*
* decide if we need to compress this.
*/
bool compress = (transparent && compress_transparent) || (!transparent && compress_solid);
if (compress) {
if (processor)
{
if (transparent)
processor->compress(*srcImage, osg::Texture::USE_S3TC_DXT5_COMPRESSION, true, true, osgDB::ImageProcessor::USE_CPU, osgDB::ImageProcessor::PRODUCTION);
else
processor->compress(*srcImage, osg::Texture::USE_S3TC_DXT1_COMPRESSION, true, true, osgDB::ImageProcessor::USE_CPU, osgDB::ImageProcessor::PRODUCTION);
//processor->generateMipMap(*srcImage, true, osgDB::ImageProcessor::USE_CPU);
}
else {
simgear::effect::MipMapTuple mipmapFunctions(simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE);
SG_LOG(SG_IO, SG_WARN, "Texture compression plugin (osg_nvtt) not available; storing uncompressed image: " << newName);
srcImage = simgear::effect::computeMipmap(srcImage, mipmapFunctions);
}
}
else {
if (processor) {
processor->generateMipMap(*srcImage, true, osgDB::ImageProcessor::USE_CPU);
}
else {
simgear::effect::MipMapTuple mipmapFunctions(simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE);
srcImage = simgear::effect::computeMipmap(srcImage, mipmapFunctions);
}
}
registry->writeImage(*srcImage, newName, nopt);
absFileName = newName;
}
}
// possibly a shared texture - but warn the user to allow investigation.
if (cacheFilename && *cacheFilename != absFileName) {
SG_LOG(SG_IO, SG_ALERT, " Already have " + hash + " : " + *cacheFilename + " not "+absFileName);
}
// SG_LOG(SG_IO, SG_ALERT, " >>>> " + hash + " :: " + newName);
}
newName = cache_root + "/" + hash.substr(0,2) + "/" + hash + ".cache.dds";
}
else
{
tstream << std::hex << file.modTime();
newName += "." + tstream.str();
newName += ".cache.dds";
}
bool doRefresh = refreshCache;
//if (fileExists(newName) && sgoptC && sgoptC->getLoadOriginHint() == SGReaderWriterOptions::LoadOriginHint::ORIGIN_EFFECTS) {
// doRefresh = true;
//
//}
if (fileExists(newName) && doRefresh) {
if (!filesCleaned.contains(newName)) {
SG_LOG(SG_IO, SG_ALERT, "Removing previously cached effects image " + newName);
SGPath(newName).remove();
filesCleaned.insert(newName, true);
}
}
if (!fileExists(newName)) {
res = registry->readImageImplementation(absFileName, opt);
if (res.validImage()) {
osg::ref_ptr<osg::Image> srcImage = res.getImage();
int width = srcImage->s();
bool transparent = srcImage->isImageTranslucent();
bool isNormalMap = false;
bool isEffect = false;
/*
* decide if we need to compress this.
*/
bool can_compress = (transparent && compress_transparent) || (!transparent && compress_solid);
int height = srcImage->t();
// use the new file origin to determine any special processing
// we handle the following
// - normal maps
// - images loaded from effects
if (sgoptC && transparent && sgoptC->getLoadOriginHint() == SGReaderWriterOptions::LoadOriginHint::ORIGIN_EFFECTS_NORMALIZED) {
isNormalMap = true;
}
else if (sgoptC && transparent && sgoptC->getLoadOriginHint() == SGReaderWriterOptions::LoadOriginHint::ORIGIN_EFFECTS) {
SG_LOG(SG_IO, SG_ALERT, "From effects transparent " + absFileName);
isEffect = true;
// can_compress = false;
}
else if (sgoptC && transparent && sgoptC->getLoadOriginHint() == SGReaderWriterOptions::LoadOriginHint::ORIGIN_EFFECTS) {
SG_LOG(SG_IO, SG_ALERT, "From effects " + absFileName);
isEffect = true;
}
if (can_compress)
{
std::string pot_message;
bool resize = false;
if (!isPowerOfTwo(width)) {
width = nearestPowerOfTwo(width);
resize = true;
pot_message += std::string(" not POT: resized width to ") + std::to_string(width);
}
if (!isPowerOfTwo(height)) {
height = nearestPowerOfTwo(height);
resize = true;
pot_message += std::string(" not POT: resized height to ") + std::to_string(height);
}
if (pot_message.size())
SG_LOG(SG_IO, SG_WARN, pot_message << " " << absFileName);
// unlikely that after resizing in height the width will still be outside of the max texture size.
if (height > max_texture_size)
{
SG_LOG(SG_IO, SG_WARN, "Image texture too high (max " << max_texture_size << ") " << width << "," << height << " " << absFileName);
int factor = height / max_texture_size;
height /= factor;
width /= factor;
resize = true;
}
if (width > max_texture_size)
{
SG_LOG(SG_IO, SG_WARN, "Image texture too wide (max " << max_texture_size << ") " << width << "," << height << " " << absFileName);
int factor = width / max_texture_size;
height /= factor;
width /= factor;
resize = true;
}
if (resize) {
osg::ref_ptr<osg::Image> resizedImage;
if (ImageUtils::resizeImage(srcImage, width, height, resizedImage))
srcImage = resizedImage;
}
//
// only cache power of two textures that are of a reasonable size
if (width >= 4 && height >= 4) {
simgear::effect::MipMapTuple mipmapFunctions(simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE);
SGPath filePath(newName);
filePath.create_dir();
// setup the options string for saving the texture as we don't want OSG to auto flip the texture
// as this complicates loading as it requires a flag to flip it back which will preclude the
// image from being cached because we will have to clone the options to set the flag and thus lose
// the link to the cache in the options from the caller.
osg::ref_ptr<Options> nopt;
nopt = opt->cloneOptions();
std::string optionstring = nopt->getOptionString();
if (!optionstring.empty())
optionstring += " ";
nopt->setOptionString(optionstring + "ddsNoAutoFlipWrite");
//GLenum srcImageType = srcImage->getDataType();
// printf("--- %-80s --> f=%8x t=%8x\n", newName.c_str(), srcImage->getPixelFormat(), srcImageType);
try
{
if (can_compress) {
osg::Texture::InternalFormatMode targetFormat = osg::Texture::USE_S3TC_DXT1_COMPRESSION;
if (isNormalMap) {
if (transparent) {
targetFormat = osg::Texture::USE_S3TC_DXT5_COMPRESSION;
}
else
targetFormat = osg::Texture::USE_S3TC_DXT5_COMPRESSION;
}
else if (isEffect)
{
if (transparent) {
targetFormat = osg::Texture::USE_S3TC_DXT5_COMPRESSION;
}
else
targetFormat = osg::Texture::USE_S3TC_DXT1_COMPRESSION;
}
else{
if (transparent) {
targetFormat = osg::Texture::USE_S3TC_DXT3_COMPRESSION;
}
else
targetFormat = osg::Texture::USE_S3TC_DXT1_COMPRESSION;
}
if (processor)
{
SG_LOG(SG_IO, SG_ALERT, "Creating " << targetFormat << " for " + absFileName);
// normal maps:
// nvdxt.exe - quality_highest - rescaleKaiser - Kaiser - dxt5nm - norm
processor->compress(*srcImage, targetFormat, true, true, osgDB::ImageProcessor::USE_CPU, osgDB::ImageProcessor::PRODUCTION);
SG_LOG(SG_IO, SG_ALERT, "-- finished creating DDS: " + newName);
//processor->generateMipMap(*srcImage, true, osgDB::ImageProcessor::USE_CPU);
}
else {
simgear::effect::MipMapTuple mipmapFunctions(simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE);
SG_LOG(SG_IO, SG_WARN, "Texture compression plugin (osg_nvtt) not available; storing uncompressed image: " << absFileName);
srcImage = simgear::effect::computeMipmap(srcImage, mipmapFunctions);
}
}
else {
SG_LOG(SG_IO, SG_ALERT, "Creating uncompressed DDS for " + absFileName);
if (processor) {
processor->generateMipMap(*srcImage, true, osgDB::ImageProcessor::USE_CPU);
}
else {
simgear::effect::MipMapTuple mipmapFunctions(simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE);
srcImage = simgear::effect::computeMipmap(srcImage, mipmapFunctions);
}
}
//}
//else
// printf("--- no compress or mipmap of format %s\n", newName.c_str());
registry->writeImage(*srcImage, newName, nopt);
{
std::string mdlDirectory = cache_root + "/cache-i ndex.txt";
FILE *f = ::fopen(mdlDirectory.c_str(), "a");
if (f)
{
::fprintf(f, "%s, %s\n", absFileName.c_str(), newName.c_str());
::fclose(f);
}
}
absFileName = newName;
}
catch (...) {
SG_LOG(SG_IO, SG_ALERT, "Exception processing " << absFileName << " may be corrupted");
}
}
else
SG_LOG(SG_IO, SG_WARN, absFileName + " too small " << width << "," << height);
}
}
}
else {
absFileName = newName;
}
}
}
res = registry->readImageImplementation(absFileName, opt);
if (!res.success()) {
@@ -368,6 +662,18 @@ ModelRegistry::readImage(const string& fileName,
if (srcImage1->getName().empty()) {
srcImage1->setName(absFileName);
}
if(cache_active && getFileExtension(absFileName) != "dds")
{
if (processor) {
processor->generateMipMap(*srcImage1, true, osgDB::ImageProcessor::USE_CPU);
SG_LOG(SG_IO, SG_ALERT, "Created nvtt mipmaps DDS for " + absFileName);
}
else {
simgear::effect::MipMapTuple mipmapFunctions(simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE);
srcImage1 = simgear::effect::computeMipmap(srcImage1, mipmapFunctions);
SG_LOG(SG_IO, SG_ALERT, "Created sg mipmaps DDS for " + absFileName);
}
}
if (res.loadedFromCache())
SG_LOG(SG_IO, SG_BULK, "Returning cached image \""