new version: 2019.2.0

Speculative fix to address scenery chunks being unloaded
unexpectedly under high load.

CMakeLists.txt

@@ -32,6 +32,7 @@ include (GenerateExportHeader)
 
 # only relevant for building shared libs but let's set it regardless
 set(CMAKE_OSX_RPATH 1)
+set(CMAKE_OSX_DEPLOYMENT_TARGET "10.9" CACHE STRING "Minimum OS X deployment version")
 
 # let's use & require C++11 - note these are only functional with CMake 3.1
 # we do manual fallbacks for CMake 3.0 in the compilers section
@@ -44,12 +45,7 @@ string(STRIP ${versionFile} SIMGEAR_VERSION)
 
 project(SimGear VERSION ${SIMGEAR_VERSION} LANGUAGES C CXX)
 
-# using 10.7 because boost requires libc++ and 10.6 doesn't include it
-# Cmake documentation says we must set this before calling project(), but
-# it only seems to be picked up setting it /after/ the call to project()
-set(CMAKE_OSX_DEPLOYMENT_TARGET "10.7")
-
-# add a dependency on the versino file
+# add a dependency on the version file
 set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS version)
 
 set_property(GLOBAL PROPERTY FIND_LIBRARY_USE_LIB64_PATHS TRUE)
@@ -126,12 +122,17 @@ option(ENABLE_RTI       "Set to ON to build SimGear with RTI support" OFF)
 option(ENABLE_GDAL      "Set to ON to build SimGear with GDAL support" OFF)
 option(ENABLE_TESTS     "Set to OFF to disable building SimGear's test applications" ON)
 option(ENABLE_SOUND     "Set to OFF to disable building SimGear's sound support" ON)
-option(USE_AEONWAVE     "Set to ON to use AeonWave instead of OpenAL" OFF)
+option(USE_AEONWAVE     "Set to ON to use AeonWave instead of OpenAL" ON)
 option(ENABLE_PKGUTIL   "Set to ON to build the sg_pkgutil application (default)" ON)
 option(ENABLE_DNS       "Set to ON to use udns library and DNS service resolver" ON)
-option(ENABLE_SIMD      "Enable SSE/SSE2 support for x86 compilers" ON)
+option(ENABLE_SIMD      "Enable SSE/SSE2 support for compilers" ON)
+option(ENABLE_SIMD_CODE	"Enable SSE/SSE2 support code for compilers" OFF)
 option(ENABLE_OPENMP    "Enable OpenMP compiler support" OFF)
 
+if (NOT ENABLE_SIMD AND ENABLE_SIMD_CODE)
+  set(ENABLE_SIMD_CODE OFF)
+endif()
+
 include (DetectArch)
 
 # until the fstream fix is applied and generally available in OSG,
@@ -167,14 +168,22 @@ endif (MSVC)
 if (MSVC AND MSVC_3RDPARTY_ROOT)
   message(STATUS "3rdparty files located in ${MSVC_3RDPARTY_ROOT}")
 
+  string(SUBSTRING ${MSVC_VERSION} 0 2 MSVC_VERSION_MAJOR)
+  string(SUBSTRING ${MSVC_VERSION} 2 2 MSVC_VERSION_MINOR)
+
   set( OSG_MSVC "msvc" )
-  if (${MSVC_VERSION} EQUAL 1900)
+  if (${MSVC_VERSION_MAJOR} EQUAL "19")
+    if (${MSVC_VERSION_MINOR} EQUAL "00")
       set( OSG_MSVC ${OSG_MSVC}140 )
-  elseif (${MSVC_VERSION} EQUAL 1800)
-      set( OSG_MSVC ${OSG_MSVC}120 )
+    else ()
+      set( OSG_MSVC ${OSG_MSVC}141 )
+    endif ()
+  elseif (${MSVC_VERSION_MAJOR} EQUAL "18")
+    set( OSG_MSVC ${OSG_MSVC}120 )
   else ()
-      message(FATAL_ERROR "Visual Studio 2013/2015 is required now")
+    message(FATAL_ERROR "Visual Studio 2013/15/17 is required")
   endif ()
+
   if (CMAKE_CL_64)
       set( OSG_MSVC ${OSG_MSVC}-64 )
       set( MSVC_3RDPARTY_DIR 3rdParty.x64 )
@@ -194,11 +203,8 @@ if (MSVC AND MSVC_3RDPARTY_ROOT)
     message(STATUS "BOOST_INCLUDEDIR is ${BOOST_INCLUDEDIR}")
   endif()
 
-  if (NOT USE_AEONWAVE)
-    set (OPENAL_INCLUDE_DIR ${MSVC_3RDPARTY_ROOT}/${MSVC_3RDPARTY_DIR}/include)
-    set (OPENAL_LIBRARY_DIR ${MSVC_3RDPARTY_ROOT}/${MSVC_3RDPARTY_DIR}/lib)
-    message(STATUS "OPENAL_INCLUDE_DIR is ${OPENAL_INCLUDE_DIR}")
-  endif()
+  set (OPENAL_INCLUDE_DIR ${MSVC_3RDPARTY_ROOT}/${MSVC_3RDPARTY_DIR}/include)
+  set (OPENAL_LIBRARY_DIR ${MSVC_3RDPARTY_ROOT}/${MSVC_3RDPARTY_DIR}/lib)
 endif (MSVC AND MSVC_3RDPARTY_ROOT)
 
 if(APPLE)
@@ -222,12 +228,19 @@ else()
 
     if (ENABLE_SOUND)
         if (USE_AEONWAVE)
-            find_package(AAX COMPONENTS aax REQUIRED)
-        else()
+            find_package(AAX)
+        endif()
+
+        if(NOT AAX_FOUND)
+            set(USE_AEONWAVE FALSE)
             find_package(OpenAL REQUIRED)
         endif()
 
-        message(STATUS "Sound support: ENABLED")
+        if(AAX_FOUND)
+            message(STATUS "Sound support: AeonWave")
+        else()
+            message(STATUS "Sound support: OpenAL")
+        endif()
     endif(ENABLE_SOUND)
 
     find_package(OpenSceneGraph 3.2.0 REQUIRED osgText osgSim osgDB osgParticle osgGA osgViewer osgUtil)
@@ -404,8 +417,8 @@ if(CMAKE_COMPILER_IS_GNUCXX)
         "${CMAKE_CXX_FLAGS} -O0 -fno-omit-frame-pointer -fno-inline")
     elseif (ENABLE_SIMD)
       if (X86 OR X86_64)
-        set(CMAKE_C_FLAGS_RELEASE "-O3 -msse2 -mfpmath=sse")
-        set(CMAKE_CXX_FLAGS_RELEASE "-O3 -msse2 -mfpmath=sse")
+        set(CMAKE_C_FLAGS_RELEASE "-O3 -msse2 -mfpmath=sse -ftree-vectorize -ftree-slp-vectorize")
+        set(CMAKE_CXX_FLAGS_RELEASE "-O3 -msse2 -mfpmath=sse -ftree-vectorize -ftree-slp-vectorize")
       endif()
     endif()
 
@@ -432,8 +445,8 @@ if (CLANG)
         "${CMAKE_CXX_FLAGS} -O0 -fno-omit-frame-pointer -fno-inline-functions")
     elseif (ENABLE_SIMD)
       if (X86 OR X86_64)
-        set(CMAKE_C_FLAGS_RELEASE "-O3 -msse2 -mfpmath=sse")
-        set(CMAKE_CXX_FLAGS_RELEASE "-O3 -msse2 -mfpmath=sse")
+        set(CMAKE_C_FLAGS_RELEASE "-O3 -msse2 -mfpmath=sse -ftree-vectorize -ftree-slp-vectorize")
+        set(CMAKE_CXX_FLAGS_RELEASE "-O3 -msse2 -mfpmath=sse -ftree-vectorize -ftree-slp-vectorize")
       endif()
     endif()
 endif()

CMakeModules/FindAAX.cmake

@@ -1,61 +1,74 @@
-# Locate AAX
+# Try to find AAX (AeonWave)
 # This module defines
-# AAX_LIBRARIES
-# AAX_FOUND, if false, do not try to link to AAX 
-# AAX_INCLUDE_DIR, where to find the headers
+#
+# AAX_FOUND - if false, do not try to link to AAX
+# AAX_INCLUDE_DIR - where to find the headers
+# AAX_LIBRARIES - Link these to use AAX
+#
+# Copyright (C) 2016-2018 by Erik Hofman.
+# Copyright (C) 2016-2018 by Adalin B.V.
 #
 # $AAXDIR is an environment variable that would
 # correspond to the ./configure --prefix=$AAXDIR
 # used in building AAX.
 #
-# Created by Erik Hofman.
+# This file is Public Domain (www.unlicense.org)
+# This is free and unencumbered software released into the public domain.
 
-FIND_PATH(AAX_INCLUDE_DIR aax/aax.h
-  HINTS
-  $ENV{AAXDIR}
-  $ENV{ProgramFiles}/aax
-  $ENV{ProgramFiles}/AeonWave
-  $ENV{ProgramFiles}/Adalin/AeonWave
-  ${CMAKE_SOURCE_DIR}/aax
-  PATH_SUFFIXES include
-  PATHS
-  ~/Library/Frameworks
-  /Library/Frameworks
-  /usr/local
-  /usr
-  /opt
-)
+if (AAX_LIBRARY AND AAX_INCLUDE_DIR)
+  # in cache already
+  set(AAX_FOUND TRUE)
+else()
+  find_path(AAX_INCLUDE_DIR aax/aax.h
+    HINTS
+    $ENV{AAXDIR}
+    $ENV{ProgramFiles}/aax
+    $ENV{ProgramFiles}/AeonWave
+    $ENV{ProgramFiles}/Adalin/AeonWave
+    ${CMAKE_SOURCE_DIR}/aax
+    PATH_SUFFIXES include
+    PATHS
+    ~/Library/Frameworks
+    /Library/Frameworks
+    /usr/local
+    /usr
+    /opt
+  )
 
-FIND_LIBRARY(AAX_LIBRARY 
-  NAMES AAX aax AAX32
-  HINTS
-  $ENV{AAXDIR}
-  $ENV{ProgramFiles}/AAX
-  $ENV{ProgramFiles}/AeonWave
-  $ENV{ProgramFiles}/Adalin/AeonWave
-  ${CMAKE_BUILD_DIR}/aax
-  PATH_SUFFIXES bin lib lib/${CMAKE_LIBRARY_ARCHITECTURE} lib64 libs64 libs libs/Win32 libs/Win64
-  PATHS
-  ~/Library/Frameworks
-  /Library/Frameworks
-  /usr/local
-  /usr
-  /opt
-)
+  find_library(AAX_LIBRARY 
+    NAMES AAX aax libAAX
+    HINTS
+    $ENV{AAXDIR}
+    $ENV{ProgramFiles}/AAX
+    $ENV{ProgramFiles}/AeonWave
+    $ENV{ProgramFiles}/Adalin/AeonWave
+    ${CMAKE_BUILD_DIR}/aax
+    PATH_SUFFIXES lib64 lib lib/${CMAKE_LIBRARY_ARCHITECTURE} libs64 libs libs/Win32 libs/Win64 bin
+    PATHS
+    ~/Library/Frameworks
+    /Library/Frameworks
+    /usr/local
+    /usr
+    /opt
+  )
 
-IF(AAX_LIBRARY AND AAX_INCLUDE_DIR)
-  SET(AAX_FOUND "YES")
-ELSE(AAX_LIBRARY AND AAX_INCLUDE_DIR)
-  IF(NOT AAX_INCLUDE_DIR)
-    MESSAGE(FATAL_ERROR "Unable to find the AAX library development files.")
-    SET(AAX_FOUND "NO")
-  ENDIF(NOT AAX_INCLUDE_DIR)
-  IF(NOT AAX_LIBRARY)
-    IF(SINGLE_PACKAGE)
-      SET(AAX_LIBRARY "${aax_BUILD_DIR}/aax/AAX32.dll")
-      SET(AAX_FOUND "YES")
-    ELSE(SINGLE_PACKAGE)
-    ENDIF(SINGLE_PACKAGE)
-  ENDIF(NOT AAX_LIBRARY)
-ENDIF(AAX_LIBRARY AND AAX_INCLUDE_DIR)
+  set(AAX_DEFINITIONS "")
+  if (AAX_LIBRARY AND AAX_INCLUDE_DIR)
+    set(AAX_FOUND TRUE)
+  endif()
+
+  if (AAX_FOUND)
+    if (NOT Udns_FIND_QUIETLY)
+      message(STATUS "Found AeonWave: ${AAX_LIBRARIES}")
+    endif ()
+  else ()
+    if (Udns_FIND_REQUIRED)
+      message(FATAL_ERROR "Could not find AeonWave")
+    endif ()
+  endif ()
+
+  # show the AAX_INCLUDE_DIRS and AAX_LIBRARIES variables only in the advanced view
+  mark_as_advanced(AAX_INCLUDE_DIRS AAX_LIBRARIES)
+
+endif()
 

CMakeModules/SimGearComponent.cmake

@@ -12,6 +12,11 @@ macro(simgear_component_common name includePath sourcesList sources headers)
 		set_property(GLOBAL
 			APPEND PROPERTY PUBLIC_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/${h}")
         set(fh${sourcesList} "${fh${sourcesList}}#${CMAKE_CURRENT_SOURCE_DIR}/${h}")
+
+        # also append headers to the sources list, so that IDEs find the files
+        # correctly (otherwise they are not in the project)
+        set_property(GLOBAL
+            APPEND PROPERTY ${sourcesList} "${CMAKE_CURRENT_SOURCE_DIR}/${h}")
 	endforeach()
 
     set_property(GLOBAL APPEND PROPERTY FG_GROUPS_${sourcesList}_C "${fc${sourcesList}}@")

README.OpenAL

@@ -1,20 +0,0 @@
-[This file is mirrored in both the FlightGear and SimGear packages.]
-
-You *must* have the development components of OpenAL installed on your system
-to build FlightGear!"  You can get a copy here:
-
-      http://connect.creativelabs.com/openal/default.aspx
-
-Build notes:
-
-You can download a versioned release of the openal library from 
-http://www.openal.org/downloads.html. Download the openal source,
-release 0.0.8 (dated February 11, 2006) and run:  
-tar xjvf openal-soft-1.5.304.tar.bz2
-cd openal-soft-1.5.304/
-ccmake .
-
-[ While running ccmake: press 'c' to configure, press 'c' once more, and
-  then press 'g' to generate and exit ]
-
-

README.sound

@@ -0,0 +1,39 @@
+[This file is mirrored in both the FlightGear and SimGear packages.]
+
+For Sound support FlightGear requires one of the two following packages:
+- OpenAL
+- AeonWave
+
+== OpenAL ===
+
+You *must* have the development components of OpenAL installed on your system
+to build FlightGear!"  You can get a copy here:
+
+      http://connect.creativelabs.com/openal/default.aspx
+
+Build notes:
+
+You can download a versioned release of the openal library from
+http://www.openal.org/downloads.html. Download the openal source,
+release 0.0.8 (dated February 11, 2006) and run:
+tar xjvf openal-soft-1.5.304.tar.bz2
+cd openal-soft-1.5.304/
+ccmake .
+
+[ While running ccmake: press 'c' to configure, press 'c' once more, and
+  then press 'g' to generate and exit ]
+
+
+== AeonWave ===
+
+For FlightGear AeonWave has a number of advantages over OpenAL:
+* Correct Doppler effect behavior
+* Default distance attenuation frequency filtering
+* Native support for 29 types of audio formats.
+* Native support for wav, mp3, vorbis and raw file formats.
+
+The source code of AeonWave can be found on GitHub:
+https://github.com/adalinbv
+
+Optimized binary packages are available at:
+http://www.adalin.com/

simgear/canvas/Canvas.cxx

@@ -39,6 +39,75 @@ namespace simgear
 {
 namespace canvas
 {
+  static int globalinstanceid = 1;
+  /**
+   * Camera Callback for moving completed canvas images to subscribed listener.
+   */
+  class CanvasImageCallback : public osg::Camera::DrawCallback {
+    public:
+    osg::Image *_rawImage;
+
+    CanvasImageCallback(osg::Image *rawImage)
+    : _min_delta_tick(1.0 / 8.0) {
+      _previousFrameTick = osg::Timer::instance()->tick();
+      _rawImage = rawImage;
+      SG_LOG(SG_GENERAL,SG_INFO,"CanvasImageCallback created. instance is " << instanceid);
+    }
+
+    virtual void operator()(osg::RenderInfo& renderInfo) const {
+      osg::Timer_t n = osg::Timer::instance()->tick();
+      double dt = osg::Timer::instance()->delta_s(_previousFrameTick, n);
+      if (dt < _min_delta_tick)
+        return;
+      _previousFrameTick = n;
+      SG_LOG(SG_GENERAL,SG_DEBUG,"CanvasImageCallback " << instanceid << ": image available for " << _subscribers.size() << " subscribers. camera is " << renderInfo.getCurrentCamera());
+
+      bool hasSubscribers = false;
+      {
+        OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_lock);
+        hasSubscribers = !_subscribers.empty();
+      }
+      if (hasSubscribers) {
+        //Make sure image can be overwritten by next frame while it is still returned to the client
+        osg::Image* image = new osg::Image(*_rawImage, osg::CopyOp::DEEP_COPY_ALL);
+        {
+          OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_lock);
+          while (!_subscribers.empty()) {
+            try {
+              CanvasImageReadyListener *subs = _subscribers.back();
+              if (subs){
+                subs->imageReady(image);
+              }else{
+                SG_LOG(SG_GENERAL,SG_WARN,"CanvasImageCallback subscriber null");
+              }
+            } catch (...) { }
+            _subscribers.pop_back();
+          }
+        }
+      }
+    }
+
+    void subscribe(CanvasImageReadyListener * subscriber) {
+      OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_lock);
+      _subscribers.push_back(subscriber);
+    }
+
+    void unsubscribe(CanvasImageReadyListener * subscriber) {
+      OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_lock);
+      _subscribers.remove(subscriber);
+    }
+
+    int getSubscriberCount() {
+      return _subscribers.size();
+    }
+
+    private:
+      mutable list<CanvasImageReadyListener*> _subscribers;
+      mutable OpenThreads::Mutex _lock;
+      mutable double _previousFrameTick;
+      double _min_delta_tick;
+      int instanceid = globalinstanceid++;
+  };
 
   //----------------------------------------------------------------------------
   Canvas::CullCallback::CullCallback(const CanvasWeakPtr& canvas):
@@ -248,6 +317,21 @@ namespace canvas
 
       osg::Camera* camera = _texture.getCamera();
 
+      string canvasname = _node->getStringValue("name");
+      int renderToImage = _node->getBoolValue("render-to-image");
+
+      if (renderToImage){
+        CanvasImageCallback *_screenshotCallback = dynamic_cast<CanvasImageCallback*> (camera->getFinalDrawCallback());
+        if (!_screenshotCallback) {
+          // no draw callback yet
+          osg::Image* shot = new osg::Image();
+          shot->allocateImage(getSizeX(), getSizeY(), 24, GL_RGB, GL_UNSIGNED_BYTE);
+          camera->attach(osg::Camera::COLOR_BUFFER, shot);
+          camera->setFinalDrawCallback(new CanvasImageCallback(shot));
+          SG_LOG(SG_GENERAL,SG_INFO,"CanvasImage: attached image and draw callback to camera " << camera << " for canvas " << canvasname << ". Ready for subscriber now.");
+        }
+      }
+
       // TODO Allow custom render order? For now just keep in order with
       //      property tree.
       camera->setRenderOrder(osg::Camera::PRE_RENDER, _node->getIndex());
@@ -347,6 +431,41 @@ namespace canvas
     }
   }
 
+  int Canvas::subscribe(CanvasImageReadyListener * subscriber) {
+    osg::Camera* camera = _texture.getCamera();
+    const string canvasname = _node->getStringValue("name");
+
+    SG_LOG(SG_GENERAL,SG_DEBUG,"CanvasImage: subscribe to canvas " << canvasname.c_str() << ", camera ="<< camera);
+
+    if (!_node->getBoolValue("render-to-image")) {
+      SG_LOG(SG_GENERAL,SG_INFO,"CanvasImage: Setting render-to-image");
+      _node->addChild("render-to-image", 0)->setBoolValue(1);
+      setStatusFlags(STATUS_DIRTY, true);
+    }
+
+    CanvasImageCallback *_screenshotCallback = dynamic_cast<CanvasImageCallback*> (camera->getFinalDrawCallback());
+    if (_screenshotCallback) {
+      // Camera ready for subscriber. Otherwise, draw callback is created by canvas thread later.
+      SG_LOG(SG_GENERAL,SG_DEBUG,"CanvasImage: adding subscriber to camera draw callback");
+      _screenshotCallback->subscribe(subscriber);
+      // TODO: check: Is this the correct way to ensure the canvas will be available?
+      enableRendering(true);
+      return 1;
+    }
+    return 0;
+  }
+
+  int Canvas::unsubscribe(CanvasImageReadyListener * subscriber) {
+    osg::Camera* camera = _texture.getCamera();
+    SG_LOG(SG_GENERAL,SG_DEBUG,"CanvasImage: unsubscribe");
+    CanvasImageCallback *cb = dynamic_cast<CanvasImageCallback*> (camera->getFinalDrawCallback());
+    if (cb) {
+      SG_LOG(SG_GENERAL,SG_DEBUG,"CanvasImage: unsubscribe from camera " << camera);
+      cb->unsubscribe(subscriber);
+    }
+    return 0;
+  }
+
   //----------------------------------------------------------------------------
   bool Canvas::addEventListener( const std::string& type,
                                  const EventListener& cb )

simgear/canvas/Canvas.hxx

@@ -44,6 +44,17 @@ namespace canvas
   class CanvasMgr;
   class MouseEvent;
 
+  /**
+   * A listener interested in completed canvas drawing.
+   */
+  class CanvasImageReadyListener {
+  public:
+    virtual void imageReady(osg::ref_ptr<osg::Image>) = 0;
+    virtual ~CanvasImageReadyListener()
+    {
+    }
+  };
+
   /**
    * Canvas to draw onto (to an off-screen render target).
    */
@@ -160,7 +171,12 @@ namespace canvas
        */
       void enableRendering(bool force = false);
 
-      void update(double delta_time_sec);
+      void update(double delta_time_sec) override;
+
+      osg::Camera* getCamera();
+      int subscribe(CanvasImageReadyListener * subscriber);
+      int unsubscribe(CanvasImageReadyListener * subscriber);
+      int getSubscriberCount();
 
       bool addEventListener(const std::string& type, const EventListener& cb);
       bool dispatchEvent(const EventPtr& event);

simgear/canvas/ShivaVG/src/shPath.c

@@ -1193,7 +1193,7 @@ VG_API_CALL VGboolean vgInterpolatePath(VGPath dstPath, VGPath startPath,
   SHfloat *procData1, *procData2;
   SHint procSegCount1=0, procSegCount2=0;
   SHint procDataCount1=0, procDataCount2=0;
-  SHuint8 *newSegs, *newData;
+  SHuint8 *newSegs, *newData=0;
   void *userData[4];
   SHint segment1, segment2;
   SHint segindex, s,d,i;

simgear/canvas/elements/CanvasMap.cxx

@@ -97,17 +97,26 @@ namespace canvas
           || (!geo_node->isDirty() && !_projection_dirty) )
         continue;
 
-      GeoCoord lat = parseGeoCoord(geo_node->getLat());
-      if( lat.type != GeoCoord::LATITUDE )
-        continue;
-
-      GeoCoord lon = parseGeoCoord(geo_node->getLon());
-      if( lon.type != GeoCoord::LONGITUDE )
-        continue;
-
-      Projection::ScreenPosition pos =
-        _projection->worldToScreen(lat.value, lon.value);
-
+      double latD = -9999.0, lonD = -9999.0;
+      if (geo_node->isDirty()) {
+        GeoCoord lat = parseGeoCoord(geo_node->getLat());
+        if( lat.type != GeoCoord::LATITUDE )
+          continue;
+        
+        GeoCoord lon = parseGeoCoord(geo_node->getLon());
+        if( lon.type != GeoCoord::LONGITUDE )
+          continue;
+        
+        // save the parsed values so we can re-use them if only projection
+        // is changed (very common case for moving vehicle)
+        latD = lat.value;
+        lonD = lon.value;
+        geo_node->setCachedLatLon(std::make_pair(latD, lonD));
+      } else {
+        std::tie(latD, lonD) = geo_node->getCachedLatLon();
+      }
+      
+      Projection::ScreenPosition pos = _projection->worldToScreen(latD, lonD);
       geo_node->setScreenPos(pos.x, pos.y);
 
 //      geo_node->print();

simgear/canvas/elements/CanvasPath.cxx

@@ -863,6 +863,9 @@ namespace canvas
   //----------------------------------------------------------------------------
   void Path::childChanged(SGPropertyNode* child)
   {
+    if( child->getParent() != _node )
+      return;
+      
     const std::string& name = child->getNameString();
     const std::string &prName = child->getParent()->getNameString();
 
@@ -892,9 +895,6 @@ namespace canvas
       return;
     }
 
-    if( child->getParent() != _node )
-      return;
-
     if( name == "cmd" )
       _attributes_dirty |= CMDS;
     else if( name == "coord" )

simgear/canvas/elements/CanvasText.cxx

@@ -71,8 +71,12 @@ namespace canvas
 
       canvas::Text *_text_element;
 
-      void computePositions(unsigned int contextID) const override;
-  };
+#if OSG_VERSION_LESS_THAN(3,5,6)
+     void computePositions(unsigned int contextID) const override;
+#else
+     void computePositionsImplementation() override;
+#endif
+};
 
   class TextLine
   {
@@ -122,6 +126,8 @@ namespace canvas
 
     _quads = &text->_textureGlyphQuadMap.begin()->second;
 
+
+#if OSG_VERSION_LESS_THAN(3,5,6)
     GlyphQuads::LineNumbers const& line_numbers = _quads->_lineNumbers;
     GlyphQuads::LineNumbers::const_iterator begin_it =
       std::lower_bound(line_numbers.begin(), line_numbers.end(), _line);
@@ -133,6 +139,10 @@ namespace canvas
     _begin = begin_it - line_numbers.begin();
     _end = std::upper_bound(begin_it, line_numbers.end(), _line)
          - line_numbers.begin();
+#else
+//OSG:TODO: Need 3.5.6 version of this
+
+#endif
   }
 
   //----------------------------------------------------------------------------
@@ -162,33 +172,35 @@ namespace canvas
     if( empty() )
       return pos;
 #if OSG_VERSION_LESS_THAN(3,3,5)
-    GlyphQuads::Coords2 const& coords = _quads->_coords;
-#else
-    GlyphQuads::Coords2 refCoords = _quads->_coords;
-    GlyphQuads::Coords2::element_type &coords = *refCoords.get();
-#endif
-    size_t global_i = _begin + i;
+      GlyphQuads::Coords2 const& coords = _quads->_coords;
+#elif OSG_VERSION_LESS_THAN(3,5,6)
+      GlyphQuads::Coords2 refCoords = _quads->_coords;
+      GlyphQuads::Coords2::element_type &coords = *refCoords.get();
+      size_t global_i = _begin + i;
 
-    if( global_i == _begin )
-      // before first character of line
-      pos.x() = coords[_begin * 4].x();
-    else if( global_i == _end )
-      // After Last character of line
-      pos.x() = coords[(_end - 1) * 4 + 2].x();
-    else
-    {
-      float prev_l = coords[(global_i - 1) * 4].x(),
-            prev_r = coords[(global_i - 1) * 4 + 2].x(),
-            cur_l = coords[global_i * 4].x();
-
-      if( prev_l == prev_r )
-        // If previous character width is zero set to begin of next character
-        // (Happens eg. with spaces)
-        pos.x() = cur_l;
+      if (global_i == _begin)
+          // before first character of line
+          pos.x() = coords[_begin * 4].x();
+      else if (global_i == _end)
+          // After Last character of line
+          pos.x() = coords[(_end - 1) * 4 + 2].x();
       else
-        // position at center between characters
-        pos.x() = 0.5 * (prev_r + cur_l);
-    }
+      {
+          float prev_l = coords[(global_i - 1) * 4].x(),
+              prev_r = coords[(global_i - 1) * 4 + 2].x(),
+              cur_l = coords[global_i * 4].x();
+
+          if (prev_l == prev_r)
+              // If previous character width is zero set to begin of next character
+              // (Happens eg. with spaces)
+              pos.x() = cur_l;
+          else
+              // position at center between characters
+              pos.x() = 0.5 * (prev_r + cur_l);
+      }
+#else
+//OSG:TODO: need 3.5.7 version of this.
+#endif
 
     return pos;
   }
@@ -200,12 +212,12 @@ namespace canvas
       return cursorPos(0);
 
     GlyphQuads::Glyphs const& glyphs = _quads->_glyphs;
-    #if OSG_VERSION_LESS_THAN(3,3,5)
+#if OSG_VERSION_LESS_THAN(3,3,5)
     GlyphQuads::Coords2 const& coords = _quads->_coords;
-#else
+#elif OSG_VERSION_LESS_THAN(3,5,6)
+
     GlyphQuads::Coords2 refCoords = _quads->_coords;
     GlyphQuads::Coords2::element_type &coords = *refCoords.get();
-#endif
 
     float const HIT_FRACTION = 0.6;
     float const character_width = _text->getCharacterHeight()
@@ -225,6 +237,10 @@ namespace canvas
     }
 
     return cursorPos(i - _begin);
+#else
+//OSG:TODO: need 3.5.7 version of this.
+	return cursorPos(0);
+#endif
   }
 
   //----------------------------------------------------------------------------
@@ -640,7 +656,7 @@ namespace canvas
     return bb;
   }
 
-  //----------------------------------------------------------------------------
+#if OSG_VERSION_LESS_THAN(3,5,6)
   void Text::TextOSG::computePositions(unsigned int contextID) const
   {
     if( _textureGlyphQuadMap.empty() || _layout == VERTICAL )
@@ -710,6 +726,14 @@ namespace canvas
     return osgText::Text::computePositions(contextID);
   }
 
+#else
+  void Text::TextOSG::computePositionsImplementation() 
+  {
+          TextBase::computePositionsImplementation();
+  }
+#endif
+ //----------------------------------------------------------------------------
+
   //----------------------------------------------------------------------------
   const std::string Text::TYPE_NAME = "text";
 

simgear/canvas/elements/map/geo_node_pair.hxx

@@ -67,7 +67,8 @@ namespace canvas
       {
         _node_lat = node;
         _status &= ~LAT_MISSING;
-
+        _xNode.reset();
+          
         if( node == _node_lon )
         {
           _node_lon = 0;
@@ -79,7 +80,8 @@ namespace canvas
       {
         _node_lon = node;
         _status &= ~LON_MISSING;
-
+        _yNode.reset();
+          
         if( node == _node_lat )
         {
           _node_lat = 0;
@@ -97,19 +99,34 @@ namespace canvas
         return _node_lon ? _node_lon->getStringValue() : "";
       }
 
+      void setCachedLatLon(const std::pair<double, double>& latLon)
+      { _cachedLatLon = latLon; }
+      
+      std::pair<double, double> getCachedLatLon()
+      { return _cachedLatLon; }
+      
       void setTargetName(const std::string& name)
       {
         _target_name = name;
+        _xNode.reset();
+        _yNode.reset();
       }
 
       void setScreenPos(float x, float y)
       {
         assert( isComplete() );
-        SGPropertyNode *parent = _node_lat->getParent();
-        parent->getChild(_target_name, _node_lat->getIndex(), true)
-              ->setDoubleValue(x);
-        parent->getChild(_target_name, _node_lon->getIndex(), true)
-              ->setDoubleValue(y);
+        if (!_xNode) {
+          SGPropertyNode *parent = _node_lat->getParent();
+          _xNode = parent->getChild(_target_name, _node_lat->getIndex(), true);
+        }
+          
+        if (!_yNode) {
+          SGPropertyNode *parent = _node_lat->getParent();
+          _yNode = parent->getChild(_target_name, _node_lon->getIndex(), true);
+        }
+          
+        _xNode->setDoubleValue(x);
+        _yNode->setDoubleValue(y);
       }
 
       void print()
@@ -125,6 +142,9 @@ namespace canvas
       SGPropertyNode *_node_lat,
                      *_node_lon;
       std::string   _target_name;
+      SGPropertyNode_ptr _xNode,
+          _yNode;
+      std::pair<double, double> _cachedLatLon;
 
   };
 

simgear/debug/CMakeLists.txt

@@ -1,7 +1,7 @@
 
 include (SimGearComponent)
 
-set(HEADERS debug_types.h logstream.hxx BufferedLogCallback.hxx)
+set(HEADERS debug_types.h logstream.hxx BufferedLogCallback.hxx OsgIoCapture.hxx)
 set(SOURCES logstream.cxx BufferedLogCallback.cxx)
 
-simgear_component(debug debug "${SOURCES}" "${HEADERS}")
+simgear_component(debug debug "${SOURCES}" "${HEADERS}")

simgear/debug/OsgIoCapture.hxx

@@ -0,0 +1,54 @@
+#include <cstring>
+
+#include <osg/Notify>
+
+using namespace osg;
+
+
+/**
+* merge OSG output into our logging system, so it gets recorded to file,
+* and so we can display a GUI console with renderer issues, especially
+* shader compilation warnings and errors.
+*/
+class NotifyLogger : public osg::NotifyHandler
+{
+public:
+    // note this callback will be invoked by OSG from multiple threads.
+    // fortunately our Simgear logging implementation already handles
+    // that internally, so we simply pass the message on.
+    virtual void notify(osg::NotifySeverity severity, const char* message) {
+        // Detect whether a osg::Reference derived object is deleted with a non-zero
+        // reference count. In this case trigger a segfault to get a stack trace.
+        if (strstr(message, "the final reference count was")) {
+            // as this is going to segfault ignore the translation of severity and always output the message.
+            SG_LOG(SG_GL, SG_ALERT, message);
+#ifndef DEBUG
+            throw new std::string(message);
+            //int* trigger_segfault = 0;
+            //*trigger_segfault     = 0;
+#endif
+            return;
+        }
+        char*tmessage = strdup(message);
+        char*lf = strrchr(tmessage, '\n');
+        if (lf)
+            *lf = 0;
+
+        SG_LOG(SG_OSG, translateSeverity(severity), tmessage);
+        free(tmessage);
+    }
+
+private:
+    sgDebugPriority translateSeverity(osg::NotifySeverity severity) {
+        switch (severity) {
+        case osg::ALWAYS:
+        case osg::FATAL:      return SG_ALERT;
+        case osg::WARN:       return SG_WARN;
+        case osg::NOTICE:
+        case osg::INFO:       return SG_INFO;
+        case osg::DEBUG_FP:
+        case osg::DEBUG_INFO: return SG_DEBUG;
+        default:              return SG_ALERT;
+        }
+    }
+};

simgear/debug/debug_types.h

@@ -34,7 +34,11 @@ typedef enum {
     SG_GUI         = 0x00800000,
     SG_TERRASYNC   = 0x01000000,
     SG_PARTICLES   = 0x02000000,
-    SG_UNDEFD      = 0x04000000, // For range checking
+    SG_HEADLESS    = 0x04000000,
+    // SG_OSG (OSG notify) - will always be displayed regardless of FG log settings as OSG log level is configured 
+    // separately and thus it makes more sense to allow these message through.
+    SG_OSG         = 0x08000000,
+    SG_UNDEFD      = 0x10000000, // For range checking
 
     SG_ALL         = 0xFFFFFFFF
 } sgDebugClass;

simgear/debug/logstream.cxx

@@ -38,6 +38,7 @@
 
 #include <simgear/io/iostreams/sgstream.hxx>
 #include <simgear/misc/sg_path.hxx>
+#include <simgear/timing/timestamp.hxx>
 
 #if defined (SG_WINDOWS)
 // for AllocConsole, OutputDebugString
@@ -61,7 +62,12 @@ LogCallback::LogCallback(sgDebugClass c, sgDebugPriority p) :
 
 bool LogCallback::shouldLog(sgDebugClass c, sgDebugPriority p) const
 {
-	 return ((c & m_class) != 0 && p >= m_priority);
+    
+    if ((c & m_class) != 0 && p >= m_priority)
+        return true;
+    if (c == SG_OSG) // always have OSG logging as it OSG logging is configured separately.
+        return true;
+    return false;
 }
 
 void LogCallback::setLogLevels( sgDebugClass c, sgDebugPriority p )
@@ -69,6 +75,18 @@ void LogCallback::setLogLevels( sgDebugClass c, sgDebugPriority p )
 	m_priority = p;
 	m_class = c;
 }
+const char* LogCallback::debugPriorityToString(sgDebugPriority p)
+{
+    switch (p) {
+    case SG_ALERT: return "ALRT";
+    case SG_BULK:  return "BULK";
+    case SG_DEBUG: return "DBUG";
+    case SG_INFO:  return "INFO";
+    case SG_POPUP: return "POPU";
+    case SG_WARN:  return "WARN";
+    default: return "UNKN";
+    }
+}
 
 const char* LogCallback::debugClassToString(sgDebugClass c)
 {
@@ -100,6 +118,8 @@ const char* LogCallback::debugClassToString(sgDebugClass c)
         case SG_GUI:        return "gui";
         case SG_TERRASYNC:  return "terrasync";
         case SG_PARTICLES:  return "particles";
+        case SG_HEADLESS:   return "headless";
+        case SG_OSG:        return "OSG";
         default:            return "unknown";
     }
 }
@@ -111,18 +131,41 @@ const char* LogCallback::debugClassToString(sgDebugClass c)
 class FileLogCallback : public simgear::LogCallback
 {
 public:
+    SGTimeStamp logTimer;
     FileLogCallback(const SGPath& aPath, sgDebugClass c, sgDebugPriority p) :
 	    simgear::LogCallback(c, p)
     {
         m_file.open(aPath, std::ios_base::out | std::ios_base::trunc);
+        logTimer.stamp();
     }
 
     virtual void operator()(sgDebugClass c, sgDebugPriority p,
         const char* file, int line, const std::string& message)
     {
         if (!shouldLog(c, p)) return;
-        m_file << debugClassToString(c) << ":" << (int) p
-            << ":" << file << ":" << line << ":" << message << std::endl;
+
+
+//        fprintf(stderr, "%7.2f [%.8s]:%-10s %s\n", logTimer.elapsedMSec() / 1000.0, debugPriorityToString(p), debugClassToString(c), aMessage.c_str());
+        m_file
+            << std::fixed
+            << std::setprecision(2)
+            << std::setw(8)
+            << std::right
+            << (logTimer.elapsedMSec() / 1000.0)
+            << std::setw(8)
+            << std::left
+            << " ["+std::string(debugPriorityToString(p))+"]:"
+            << std::setw(10)
+            << std::left
+            << debugClassToString(c)
+            << " " 
+            << file 
+            << ":" 
+            << line 
+            << ":" 
+            << message << std::endl;
+        //m_file << debugClassToString(c) << ":" << (int)p
+        //    << ":" << file << ":" << line << ":" << message << std::endl;
     }
 private:
     sg_ofstream m_file;
@@ -131,9 +174,12 @@ private:
 class StderrLogCallback : public simgear::LogCallback
 {
 public:
+    SGTimeStamp logTimer;
+
     StderrLogCallback(sgDebugClass c, sgDebugPriority p) :
 		simgear::LogCallback(c, p)
     {
+        logTimer.stamp();
     }
 
 #if defined (SG_WINDOWS)
@@ -147,8 +193,9 @@ public:
         const char* file, int line, const std::string& aMessage)
     {
         if (!shouldLog(c, p)) return;
-
-        fprintf(stderr, "%s\n", aMessage.c_str());
+        //fprintf(stderr, "%s\n", aMessage.c_str());
+        fprintf(stderr, "%8.2f [%.8s]:%-10s %s\n", logTimer.elapsedMSec()/1000.0, debugPriorityToString(p), debugClassToString(c), aMessage.c_str());
+        //    file, line, aMessage.c_str());
         //fprintf(stderr, "%s:%d:%s:%d:%s\n", debugClassToString(c), p,
         //    file, line, aMessage.c_str());
         fflush(stderr);
@@ -469,6 +516,10 @@ public:
         // Testing mode, so always log.
         if (m_testMode) return true;
 
+        // SG_OSG (OSG notify) - will always be displayed regardless of FG log settings as OSG log level is configured 
+        // separately and thus it makes more sense to allow these message through.
+        if (static_cast<unsigned>(p) == static_cast<unsigned>(SG_OSG)) return true;
+
         p = translatePriority(p);
         if (p >= SG_INFO) return true;
         return ((c & m_logClass) != 0 && p >= m_logPriority);

simgear/debug/logstream.hxx

@@ -52,6 +52,7 @@ protected:
 	bool shouldLog(sgDebugClass c, sgDebugPriority p) const;
 
     static const char* debugClassToString(sgDebugClass c);
+    static const char* debugPriorityToString(sgDebugPriority p);
 private:
 	sgDebugClass m_class;
 	sgDebugPriority m_priority;

simgear/environment/precipitation.cxx

@@ -44,6 +44,10 @@ SGPrecipitation::SGPrecipitation() :
 void SGPrecipitation::setEnabled( bool value )
 {
     _enabled = value;
+    if (!_enabled) {
+        _precipitationEffect->snow(0);
+        _precipitationEffect->rain(0);
+    }
 }
 
 void SGPrecipitation::setDropletExternal( bool value )
@@ -64,6 +68,9 @@ bool SGPrecipitation::getEnabled() const
  */
 osg::Group* SGPrecipitation::build(void)
 {
+    if (!_enabled) 
+        return nullptr;
+
     osg::ref_ptr<osg::Group> group = new osg::Group;
 
     _precipitationEffect->snow(0);	
@@ -227,6 +234,9 @@ void SGPrecipitation::setWindProperty(double heading, double speed)
  */
 bool SGPrecipitation::update(void)
 {
+    if (!_enabled)
+        return false;
+
     if (this->_freeze) {
         if (this->_rain_intensity > 0) {
 		this->_snow_intensity = this->_rain_intensity;

simgear/io/HTTPClient.cxx

@@ -48,6 +48,8 @@
 #include <simgear/debug/logstream.hxx>
 #include <simgear/timing/timestamp.hxx>
 #include <simgear/structure/exception.hxx>
+#include <simgear/threads/SGThread.hxx>
+#include <simgear/threads/SGGuard.hxx>
 
 #if defined( HAVE_VERSION_H ) && HAVE_VERSION_H
 #include "version.h"
@@ -123,6 +125,9 @@ Client::Client() :
     setUserAgent("SimGear-" SG_STRINGIZE(SIMGEAR_VERSION));
 
     static bool didInitCurlGlobal = false;
+    static SGMutex initMutex;
+    
+    SGGuard<SGMutex> g(initMutex);
     if (!didInitCurlGlobal) {
       curl_global_init(CURL_GLOBAL_ALL);
       didInitCurlGlobal = true;
@@ -273,6 +278,10 @@ void Client::makeRequest(const Request_ptr& r)
     curl_easy_setopt(curlRequest, CURLOPT_USERAGENT, d->userAgent.c_str());
     curl_easy_setopt(curlRequest, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
 
+    if (sglog().would_log(SG_TERRASYNC, SG_DEBUG)) {
+        curl_easy_setopt(curlRequest, CURLOPT_VERBOSE, 1);
+    }
+
     curl_easy_setopt(curlRequest, CURLOPT_FOLLOWLOCATION, 1);
 
     if (!d->proxy.empty()) {
@@ -472,12 +481,26 @@ size_t Client::requestReadCallback(char *ptr, size_t size, size_t nmemb, void *u
   return actualBytes;
 }
 
+bool isRedirectStatus(int code)
+{
+    return ((code >= 300) && (code < 400));
+}
+    
 size_t Client::requestHeaderCallback(char *rawBuffer, size_t size, size_t nitems, void *userdata)
 {
   size_t byteSize = size * nitems;
   Request* req = static_cast<Request*>(userdata);
   std::string h = strutils::simplify(std::string(rawBuffer, byteSize));
 
+  if (req->readyState() >= HTTP::Request::HEADERS_RECEIVED) {
+      // this can happen with chunked transfers (secondary chunks)
+      // or redirects
+      if (isRedirectStatus(req->responseCode())) {
+          req->responseStart(h);
+          return byteSize;
+      }
+  }
+    
   if (req->readyState() == HTTP::Request::OPENED) {
     req->responseStart(h);
     return byteSize;

simgear/io/HTTPRepository.cxx

@@ -90,8 +90,10 @@ namespace simgear
         case HTTPRepository::REPO_ERROR_CANCELLED:   return "cancelled";
         case HTTPRepository::REPO_PARTIAL_UPDATE:   return "partial update (incomplete)";
         }
+
+        return "Unknown response code";
     }
-    
+
 class HTTPRepoPrivate
 {
 public:
@@ -180,17 +182,11 @@ class HTTPDirectory
         ChildInfo(Type ty, const std::string & nameData, const std::string & hashData) :
             type(ty),
             name(nameData),
-            hash(hashData),
-            sizeInBytes(0)
+            hash(hashData)
         {
         }
 
-        ChildInfo(const ChildInfo& other) :
-            type(other.type),
-            name(other.name),
-            hash(other.hash),
-            sizeInBytes(other.sizeInBytes)
-        { }
+		ChildInfo(const ChildInfo& other) = default;
 
         void setSize(const std::string & sizeData)
         {
@@ -204,7 +200,8 @@ class HTTPDirectory
 
         Type type;
         std::string name, hash;
-        size_t sizeInBytes;
+		size_t sizeInBytes = 0;
+		SGPath path; // absolute path on disk
     };
 
     typedef std::vector<ChildInfo> ChildInfoList;
@@ -271,44 +268,45 @@ public:
             return;
         }
 
-        string_list indexNames = indexChildren();
-        const_string_list_iterator nameIt = indexNames.begin();
-        for (; nameIt != indexNames.end(); ++nameIt) {
-            SGPath p(absolutePath());
-            p.append(*nameIt);
-            if (p.exists()) {
-                continue; // only copy if the file is missing entirely
-            }
+		char* buf = nullptr;
+		size_t bufSize = 0;
 
-            ChildInfoList::iterator c = findIndexChild(*nameIt);
-            if (c->type == ChildInfo::DirectoryType) {
-                continue; // only care about files
-            }
+		for (const auto& child : children) {
+			if (child.type != ChildInfo::FileType)
+				continue;
 
-            SGPath cp = _repository->installedCopyPath;
-            cp.append(relativePath());
-            cp.append(*nameIt);
-            if (!cp.exists()) {
-                continue;
-            }
-            
-            SGBinaryFile src(cp);
-            SGBinaryFile dst(p);
-            src.open(SG_IO_IN);
-            dst.open(SG_IO_OUT);
+			if (child.path.exists())
+				continue;
 
-            char* buf = (char*) malloc(cp.sizeInBytes());
-            if (!buf) {
-                continue;
-            }
+			SGPath cp = _repository->installedCopyPath;
+			cp.append(relativePath());
+			cp.append(child.name);
+			if (!cp.exists()) {
+				continue;
+			}
 
-            src.read(buf, cp.sizeInBytes());
-            dst.write(buf, cp.sizeInBytes());
-            src.close();
-            dst.close();
+			SGBinaryFile src(cp);
+			SGBinaryFile dst(child.path);
+			src.open(SG_IO_IN);
+			dst.open(SG_IO_OUT);
 
-            free(buf);
-        }
+			if (bufSize < cp.sizeInBytes()) {
+				bufSize = cp.sizeInBytes();
+				free(buf);
+				buf = (char*) malloc(bufSize);
+				if (!buf) {
+					continue;
+				}
+			}
+
+			src.read(buf, cp.sizeInBytes());
+			dst.write(buf, cp.sizeInBytes());
+			src.close();
+			dst.close();
+
+		}
+
+		free(buf);
     }
 
     void updateChildrenBasedOnHash()
@@ -317,36 +315,40 @@ public:
 
         copyInstalledChildren();
 
-        string_list indexNames = indexChildren(),
-            toBeUpdated, orphans;
+		string_list toBeUpdated, orphans,
+			indexNames = indexChildren();
         simgear::Dir d(absolutePath());
         PathList fsChildren = d.children(0);
-        PathList::const_iterator it = fsChildren.begin();
 
+        for (const auto& child : fsChildren) {
+			const auto& fileName = child.file();
+			if ((fileName == ".dirindex") || (fileName == ".hashes")) {
+				continue;
+			}
 
-        for (; it != fsChildren.end(); ++it) {
-            ChildInfo info(it->isDir() ? ChildInfo::DirectoryType : ChildInfo::FileType,
-                           it->file(), "");
+            ChildInfo info(child.isDir() ? ChildInfo::DirectoryType : ChildInfo::FileType, 
+				fileName, "");
+			info.path = child;
             std::string hash = hashForChild(info);
 
-            ChildInfoList::iterator c = findIndexChild(it->file());
+            ChildInfoList::iterator c = findIndexChild(fileName);
             if (c == children.end()) {
-                orphans.push_back(it->file());
+                orphans.push_back(fileName);
             } else if (c->hash != hash) {
 #if 0
-                SG_LOG(SG_TERRASYNC, SG_DEBUG, "hash mismatch'" << it->file() );
+                SG_LOG(SG_TERRASYNC, SG_DEBUG, "hash mismatch'" << fileName);
                 // file exists, but hash mismatch, schedule update
                 if (!hash.empty()) {
-                    SG_LOG(SG_TERRASYNC, SG_DEBUG, "file exists but hash is wrong for:" << it->file() );
+                    SG_LOG(SG_TERRASYNC, SG_DEBUG, "file exists but hash is wrong for:" << fileName);
                     SG_LOG(SG_TERRASYNC, SG_DEBUG, "on disk:" << hash << " vs in info:" << c->hash);
                 }
 #endif
-                toBeUpdated.push_back(it->file() );
+                toBeUpdated.push_back(fileName);
             } else {
                 // file exists and hash is valid. If it's a directory,
                 // perform a recursive check.
                 if (c->type == ChildInfo::DirectoryType) {
-                    HTTPDirectory* childDir = childDirectory(it->file());
+                    HTTPDirectory* childDir = childDirectory(fileName);
                     childDir->updateChildrenBasedOnHash();
                 }
             }
@@ -354,7 +356,7 @@ public:
             // remove existing file system children from the index list,
             // so we can detect new children
             // https://en.wikibooks.org/wiki/More_C%2B%2B_Idioms/Erase-Remove
-            indexNames.erase(std::remove(indexNames.begin(), indexNames.end(), it->file()), indexNames.end());
+            indexNames.erase(std::remove(indexNames.begin(), indexNames.end(), fileName), indexNames.end());
         } // of real children iteration
 
         // all remaining names in indexChilden are new children
@@ -423,19 +425,16 @@ public:
     void didUpdateFile(const std::string& file, const std::string& hash, size_t sz)
     {
         // check hash matches what we expected
-        ChildInfoList::iterator it = findIndexChild(file);
+        auto it = findIndexChild(file);
         if (it == children.end()) {
             SG_LOG(SG_TERRASYNC, SG_WARN, "updated file but not found in dir:" << _relativePath << " " << file);
         } else {
-            SGPath fpath(absolutePath());
-            fpath.append(file);
-
             if (it->hash != hash) {
                 // we don't erase the file on a hash mismatch, becuase if we're syncing during the
                 // middle of a server-side update, the downloaded file may actually become valid.
                 _repository->failedToUpdateChild(_relativePath, HTTPRepository::REPO_ERROR_CHECKSUM);
             } else {
-                _repository->updatedFileContents(fpath, hash);
+                _repository->updatedFileContents(it->path, hash);
                 _repository->totalDownloaded += sz;
             } // of hash matches
         } // of found in child list
@@ -534,8 +533,8 @@ private:
                 continue;
             }
 
-            children.push_back(ChildInfo(typeData == "f" ? ChildInfo::FileType : ChildInfo::DirectoryType, tokens[1], tokens[2]));
-
+            children.emplace_back(ChildInfo(typeData == "f" ? ChildInfo::FileType : ChildInfo::DirectoryType, tokens[1], tokens[2]));
+			children.back().path = absolutePath() / tokens[1];
             if (tokens.size() > 3) {
                 children.back().setSize(tokens[3]);
             }
@@ -567,8 +566,7 @@ private:
 
     std::string hashForChild(const ChildInfo& child) const
     {
-        SGPath p(absolutePath());
-        p.append(child.name);
+		SGPath p(child.path);
         if (child.type == ChildInfo::DirectoryType) {
             p.append(".dirindex");
         }
@@ -677,7 +675,7 @@ std::string HTTPRepository::resultCodeAsString(ResultCode code)
 {
     return innerResultCodeAsString(code);
 }
-    
+
 HTTPRepository::ResultCode
 HTTPRepository::failure() const
 {
@@ -746,7 +744,11 @@ HTTPRepository::failure() const
             if (responseCode() == -1) {
                 code = HTTPRepository::REPO_ERROR_CANCELLED;
             }
-            
+
+			if (file) {
+				file->close();
+			}
+
             file.reset();
             if (pathInRepo.exists()) {
                 pathInRepo.remove();
@@ -818,7 +820,7 @@ HTTPRepository::failure() const
 
                     // dir index data has changed, so write to disk and update
                     // the hash accordingly
-                    sg_ofstream of(pathInRepo(), std::ios::trunc | std::ios::out);
+                    sg_ofstream of(pathInRepo(), std::ios::trunc | std::ios::out | std::ios::binary);
                     if (!of.is_open()) {
                         throw sg_io_exception("Failed to open directory index file for writing", pathInRepo());
                     }
@@ -997,11 +999,11 @@ HTTPRepository::failure() const
 
         SGPath cachePath = basePath;
         cachePath.append(".hashes");
-        sg_ofstream stream(cachePath, std::ios::out | std::ios::trunc);
+        sg_ofstream stream(cachePath, std::ios::out | std::ios::trunc | std::ios::binary);
         HashCache::const_iterator it;
         for (it = hashes.begin(); it != hashes.end(); ++it) {
-            stream << it->filePath << ":" << it->modTime << ":"
-            << it->lengthBytes << ":" << it->hashHex << "\n";
+            stream << it->filePath << "*" << it->modTime << "*"
+            << it->lengthBytes << "*" << it->hashHex << "\n";
         }
         stream.close();
         hashCacheDirty = false;
@@ -1025,7 +1027,7 @@ HTTPRepository::failure() const
             if( line.empty() || line[0] == '#' )
                 continue;
 
-            string_list tokens = simgear::strutils::split( line, ":" );
+			string_list tokens = simgear::strutils::split(line, "*");
             if( tokens.size() < 4 ) {
                 SG_LOG(SG_TERRASYNC, SG_WARN, "invalid entry in '" << cachePath << "': '" << line << "' (ignoring line)");
                 continue;
@@ -1084,7 +1086,7 @@ HTTPRepository::failure() const
 		} else {
 			// we encounter this code path when deleting an orphaned directory
 		}
-		
+
 		Dir dir(absPath);
 		bool result = dir.remove(true);
 
@@ -1152,7 +1154,7 @@ HTTPRepository::failure() const
             }
 
             SG_LOG(SG_TERRASYNC, SG_WARN, "failed to update repository:" << baseUrl
-                   << "\n\tchecksum failure for: " << relativePath 
+                   << "\n\tchecksum failure for: " << relativePath
                    << "\n\tthis typically indicates the remote repository is corrupt or was being updated during the sync");
         } else if (fileStatus == HTTPRepository::REPO_ERROR_CANCELLED) {
             // if we were cancelled, don't report or log

simgear/io/HTTPRequest.cxx

@@ -328,6 +328,16 @@ unsigned int Request::responseLength() const
   return _responseLength;
 }
 
+//------------------------------------------------------------------------------
+void Request::setSuccess(int code)
+{
+    _responseStatus = code;
+    _responseReason.clear();
+    if( !isComplete() ) {
+        setReadyState(DONE);
+    }
+}
+    
 //------------------------------------------------------------------------------
 void Request::setFailure(int code, const std::string& reason)
 {

simgear/io/HTTPRequest.hxx

@@ -224,7 +224,7 @@ protected:
     virtual void onAlways();
 
     void setFailure(int code, const std::string& reason);
-
+    void setSuccess(int code);
 private:
     friend class Client;
     friend class Connection;

simgear/io/sg_binobj.cxx

@@ -160,7 +160,7 @@ public:
         }
     }
 
-    float readInt()
+    int32_t readInt()
     {
         unsigned int* p = reinterpret_cast<unsigned int*>(ptr + offset);
         if ( sgIsBigEndian() ) {

simgear/io/sg_file.cxx

@@ -64,6 +64,31 @@ SGFile::SGFile( int existingFd ) :
 SGFile::~SGFile() {
 }
 
+#include <simgear/misc/sg_hash.hxx>
+#include <simgear/structure/exception.hxx>
+#include "simgear/misc/strutils.hxx"
+
+std::string SGFile::computeHash()
+{
+    if (!file_name.exists())
+        return std::string();
+    simgear::sha1nfo info;
+    sha1_init(&info);
+    char* buf = static_cast<char*>(malloc(1024 * 1024));
+    size_t readLen;
+    SGBinaryFile f(file_name);
+    if (!f.open(SG_IO_IN)) {
+        throw sg_io_exception("Couldn't open file for compute hash", file_name);
+    }
+    while ((readLen = f.read(buf, 1024 * 1024)) > 0) {
+        sha1_write(&info, buf, readLen);
+    }
+
+    f.close();
+    free(buf);
+    std::string hashBytes((char*)sha1_result(&info), HASH_LENGTH);
+    return simgear::strutils::encodeHex(hashBytes);
+}
 
 // open the file based on specified direction
 bool SGFile::open( const SGProtocolDir d ) {

simgear/io/sg_file.hxx

@@ -62,7 +62,7 @@ public:
     SGFile( int existingFd );
 
     /** Destructor */
-    ~SGFile();
+    virtual ~SGFile();
 
     // open the file based on specified direction
     bool open( const SGProtocolDir dir );
@@ -87,6 +87,9 @@ public:
 
     /** @return true of eof conditions exists */
     virtual bool eof() const { return eof_flag; };
+
+    std::string computeHash();
+
 };
 
 class SGBinaryFile : public SGFile {

simgear/io/test_HTTP.cxx

@@ -273,7 +273,23 @@ public:
             d << "\r\n"; // final CRLF to terminate the headers
             d << contentStr;
             push(d.str().c_str());
-
+        } else if (path == "/test_redirect") {
+            string contentStr("<html>See <a href=\"wibble\">Here</a></html>");
+            stringstream d;
+            d << "HTTP/1.1 " << 302 << " " << "Found" << "\r\n";
+            d << "Location:" << " http://localhost:2000/was_redirected" << "\r\n";
+            d << "Content-Length:" << contentStr.size() << "\r\n";
+            d << "\r\n"; // final CRLF to terminate the headers
+            d << contentStr;
+            push(d.str().c_str());
+        } else if (path == "/was_redirected") {
+            string contentStr(BODY1);
+            stringstream d;
+            d << "HTTP/1.1 " << 200 << " " << reasonForCode(200) << "\r\n";
+            d << "Content-Length:" << contentStr.size() << "\r\n";
+            d << "\r\n"; // final CRLF to terminate the headers
+            d << contentStr;
+            push(d.str().c_str());
         } else {
           TestServerChannel::processRequestHeaders();
         }
@@ -773,6 +789,24 @@ cout << "testing proxy close" << endl;
         SG_CHECK_EQUAL(tr2->bodyData, string(BODY1));
         SG_CHECK_EQUAL(tr2->responseBytesReceived(), strlen(BODY1));
     }
+    
+    {
+        cout << "redirect test" << endl;
+        // redirect test
+        testServer.disconnectAll();
+        cl.clearAllConnections();
+        
+        TestRequest* tr = new TestRequest("http://localhost:2000/test_redirect");
+        HTTP::Request_ptr own(tr);
+        cl.makeRequest(tr);
+        
+        waitForComplete(&cl, tr);
+        SG_CHECK_EQUAL(tr->responseCode(), 200);
+        SG_CHECK_EQUAL(tr->responseReason(), string("OK"));
+        SG_CHECK_EQUAL(tr->responseLength(), strlen(BODY1));
+        SG_CHECK_EQUAL(tr->responseBytesReceived(), strlen(BODY1));
+        SG_CHECK_EQUAL(tr->bodyData, string(BODY1));
+    }
 
     cout << "all tests passed ok" << endl;
     return EXIT_SUCCESS;

simgear/io/test_HTTP.hxx

@@ -30,7 +30,6 @@ public:
 
     virtual ~TestServerChannel()
     {
-        std::cerr << "dtor test server channel" << std::endl;
     }
 
     virtual void collectIncomingData(const char* s, int n)
@@ -139,8 +138,8 @@ public:
 
     void sendErrorResponse(int code, bool close, std::string content)
     {
-        std::cerr << "sending error " << code << " for " << path << std::endl;
-        std::cerr << "\tcontent:" << content << std::endl;
+     //   std::cerr << "sending error " << code << " for " << path << std::endl;
+       // std::cerr << "\tcontent:" << content << std::endl;
 
         std::stringstream headerData;
         headerData << "HTTP/1.1 " << code << " " << reasonForCode(code) << "\r\n";
@@ -168,7 +167,6 @@ public:
 
     virtual void handleClose (void)
     {
-        std::cerr << "channel close" << std::endl;
         NetBufferChannel::handleClose();
     }
 

simgear/io/test_repository.cxx

@@ -309,7 +309,8 @@ std::string test_computeHashForPath(const SGPath& p)
         return std::string();
     sha1nfo info;
     sha1_init(&info);
-    char* buf = static_cast<char*>(alloca(1024 * 1024));
+    char* buf = static_cast<char*>(malloc(1024 * 1024));
+	assert(buf);
     size_t readLen;
 
     SGBinaryFile f(p);
@@ -319,6 +320,9 @@ std::string test_computeHashForPath(const SGPath& p)
         sha1_write(&info, buf, readLen);
     }
 
+	f.close();
+	free(buf);
+
     std::string hashBytes((char*) sha1_result(&info), HASH_LENGTH);
     return strutils::encodeHex(hashBytes);
 }
@@ -433,6 +437,34 @@ void testBasicClone(HTTP::Client* cl)
     std::cout << "Passed test: basic clone and update" << std::endl;
 }
 
+void testUpdateNoChanges(HTTP::Client* cl)
+{
+	std::unique_ptr<HTTPRepository> repo;
+	SGPath p(simgear::Dir::current().path());
+	p.append("http_repo_basic"); // same as before
+
+	global_repo->clearRequestCounts();
+
+	repo.reset(new HTTPRepository(p, cl));
+	repo->setBaseUrl("http://localhost:2000/repo");
+	repo->update();
+
+	waitForUpdateComplete(cl, repo.get());
+
+	verifyFileState(p, "fileA");
+	verifyFileState(p, "dirC/subdirA/subsubA/fileCAAA");
+
+	verifyRequestCount("dirA", 0);
+	verifyRequestCount("dirB", 0);
+	verifyRequestCount("dirB/subdirA", 0);
+	verifyRequestCount("dirB/subdirA/fileBAA", 0);
+	verifyRequestCount("dirC", 0);
+	verifyRequestCount("dirC/fileCA", 0);
+
+	std::cout << "Passed test:no changes update" << std::endl;
+
+}
+
 void testModifyLocalFiles(HTTP::Client* cl)
 {
     std::unique_ptr<HTTPRepository> repo;
@@ -469,10 +501,6 @@ void testModifyLocalFiles(HTTP::Client* cl)
     std::cout << "Passed test: identify and fix locally modified files" << std::endl;
 }
 
-void testNoChangesUpdate()
-{
-
-}
 
 void testMergeExistingFileWithoutDownload(HTTP::Client* cl)
 {
@@ -725,7 +753,7 @@ void testCopyInstalledChildren(HTTP::Client* cl)
     verifyRequestCount("dirJ/fileJC", 1);
     verifyRequestCount("dirJ/fileJD", 1);
 
-    std::cout << "Copy installed children" << std::endl;
+    std::cout << "passed Copy installed children" << std::endl;
 }
 
 int main(int argc, char* argv[])
@@ -751,6 +779,7 @@ int main(int argc, char* argv[])
     global_repo->defineFile("dirC/subdirA/subsubA/fileCAAA");
 
     testBasicClone(&cl);
+	testUpdateNoChanges(&cl);
 
     testModifyLocalFiles(&cl);
 

simgear/io/test_untar.cxx

@@ -11,6 +11,7 @@
 
 #include <simgear/misc/test_macros.hxx>
 #include <simgear/misc/sg_path.hxx>
+#include <simgear/misc/sg_dir.hxx>
 #include <simgear/io/sg_file.hxx>
 
 
@@ -31,7 +32,7 @@ void testTarGz()
     uint8_t* buf = (uint8_t*) alloca(8192);
     size_t bufSize = f.read((char*) buf, 8192);
 
-    SG_VERIFY(TarExtractor::isTarData(buf, bufSize));
+    SG_VERIFY(ArchiveExtractor::determineType(buf, bufSize) == ArchiveExtractor::TarData);
 
     f.close();
 }
@@ -44,18 +45,146 @@ void testPlainTar()
     SGBinaryFile f(p);
     f.open(SG_IO_IN);
 
-    uint8_t* buf = (uint8_t*) alloca(8192);
-    size_t bufSize = f.read((char*) buf, 8192);
+	uint8_t* buf = (uint8_t*)alloca(8192);
+	size_t bufSize = f.read((char*) buf, 8192);
 
-    SG_VERIFY(TarExtractor::isTarData(buf, bufSize));
+	SG_VERIFY(ArchiveExtractor::determineType(buf, bufSize) == ArchiveExtractor::TarData);
 
     f.close();
 }
 
+void testExtractStreamed()
+{
+	SGPath p = SGPath(SRC_DIR);
+	p.append("test.tar.gz");
+
+	SGBinaryFile f(p);
+	f.open(SG_IO_IN);
+
+	SGPath extractDir = simgear::Dir::current().path() / "test_extract_streamed";
+	simgear::Dir pd(extractDir);
+	pd.removeChildren();
+
+	ArchiveExtractor ex(extractDir);
+
+	uint8_t* buf = (uint8_t*) alloca(128);
+	while (!f.eof()) {
+		size_t bufSize = f.read((char*) buf, 128);
+		ex.extractBytes(buf, bufSize);
+	}
+
+	ex.flush();
+	SG_VERIFY(ex.isAtEndOfArchive());
+	SG_VERIFY(ex.hasError() == false);
+
+	SG_VERIFY((extractDir / "testDir/hello.c").exists());
+	SG_VERIFY((extractDir / "testDir/foo.txt").exists());
+}
+
+void testExtractLocalFile()
+{
+
+}
+
+void testFilterTar()
+{
+    SGPath p = SGPath(SRC_DIR);
+    p.append("badTar.tgz");
+    
+    SGBinaryFile f(p);
+    f.open(SG_IO_IN);
+    
+    SGPath extractDir = simgear::Dir::current().path() / "test_filter_tar";
+    simgear::Dir pd(extractDir);
+    pd.removeChildren();
+    
+    ArchiveExtractor ex(extractDir);
+    
+    uint8_t* buf = (uint8_t*) alloca(128);
+    while (!f.eof()) {
+        size_t bufSize = f.read((char*) buf, 128);
+        ex.extractBytes(buf, bufSize);
+    }
+    
+    ex.flush();
+    SG_VERIFY(ex.isAtEndOfArchive());
+    SG_VERIFY(ex.hasError() == false);
+    
+    SG_VERIFY((extractDir / "tarWithBadContent/regular-file.txt").exists());
+    SG_VERIFY(!(extractDir / "tarWithBadContent/symbolic-linked.png").exists());
+    SG_VERIFY((extractDir / "tarWithBadContent/screenshot.png").exists());
+    SG_VERIFY((extractDir / "tarWithBadContent/dirOne/subDirA").exists());
+    SG_VERIFY(!(extractDir / "tarWithBadContent/dirOne/subDirA/linked.txt").exists());
+
+
+}
+
+void testExtractZip()
+{
+	SGPath p = SGPath(SRC_DIR);
+	p.append("zippy.zip");
+
+	SGBinaryFile f(p);
+	f.open(SG_IO_IN);
+
+	SGPath extractDir = simgear::Dir::current().path() / "test_extract_zip";
+	simgear::Dir pd(extractDir);
+	pd.removeChildren();
+
+	ArchiveExtractor ex(extractDir);
+
+	uint8_t* buf = (uint8_t*)alloca(128);
+	while (!f.eof()) {
+		size_t bufSize = f.read((char*)buf, 128);
+		ex.extractBytes(buf, bufSize);
+	}
+
+	ex.flush();
+	SG_VERIFY(ex.isAtEndOfArchive());
+	SG_VERIFY(ex.hasError() == false);
+
+	SG_VERIFY((extractDir / "zippy/dirA/hello.c").exists());
+	SG_VERIFY((extractDir / "zippy/bar.xml").exists());
+	SG_VERIFY((extractDir / "zippy/long-named.json").exists());
+}
+
+void testPAXAttributes()
+{
+    SGPath p = SGPath(SRC_DIR);
+    p.append("pax-extended.tar");
+    
+    SGBinaryFile f(p);
+    f.open(SG_IO_IN);
+    
+    SGPath extractDir = simgear::Dir::current().path() / "test_pax_extended";
+    simgear::Dir pd(extractDir);
+    pd.removeChildren();
+    
+    ArchiveExtractor ex(extractDir);
+    
+    uint8_t* buf = (uint8_t*) alloca(128);
+    while (!f.eof()) {
+        size_t bufSize = f.read((char*) buf, 128);
+        ex.extractBytes(buf, bufSize);
+    }
+    
+    ex.flush();
+    SG_VERIFY(ex.isAtEndOfArchive());
+    SG_VERIFY(ex.hasError() == false);
+
+}
+
 int main(int ac, char ** av)
 {
     testTarGz();
     testPlainTar();
-
+    testFilterTar();
+	testExtractStreamed();
+	testExtractZip();
+   
+    // disabled to avoiding checking in large PAX archive
+    // testPAXAttributes();
+    
+	std::cout << "all tests passed" << std::endl;
     return 0;
 }

simgear/io/untar.cxx

@@ -31,12 +31,80 @@
 #include <simgear/sg_inlines.h>
 #include <simgear/io/sg_file.hxx>
 #include <simgear/misc/sg_dir.hxx>
-
+#include <simgear/io/iostreams/sgstream.hxx>
 #include <simgear/debug/logstream.hxx>
+#include <simgear/package/unzip.h>
+#include <simgear/structure/exception.hxx>
 
 namespace simgear
 {
 
+	class ArchiveExtractorPrivate
+	{
+	public:
+		ArchiveExtractorPrivate(ArchiveExtractor* o) :
+			outer(o)
+		{
+			assert(outer);
+		}
+
+		typedef enum {
+			INVALID = 0,
+			READING_HEADER,
+			READING_FILE,
+			READING_PADDING,
+            READING_PAX_GLOBAL_ATTRIBUTES,
+            READING_PAX_FILE_ATTRIBUTES,
+			PRE_END_OF_ARCHVE,
+			END_OF_ARCHIVE,
+			ERROR_STATE, ///< states above this are error conditions
+			BAD_ARCHIVE,
+			BAD_DATA,
+			FILTER_STOPPED
+		} State;
+
+		State state = INVALID;
+		ArchiveExtractor* outer = nullptr;
+
+		virtual void extractBytes(const uint8_t* bytes, size_t count) = 0;
+
+		virtual void flush() = 0;
+
+		SGPath extractRootPath()
+		{
+			return outer->_rootPath;
+		}
+
+		ArchiveExtractor::PathResult filterPath(std::string& pathToExtract)
+		{
+			return outer->filterPath(pathToExtract);
+		}
+
+
+		bool isSafePath(const std::string& p) const
+		{
+			if (p.empty()) {
+				return false;
+			}
+
+			// reject absolute paths
+			if (p.at(0) == '/') {
+				return false;
+			}
+
+			// reject paths containing '..'
+			size_t doubleDot = p.find("..");
+			if (doubleDot != std::string::npos) {
+				return false;
+			}
+
+			// on POSIX could use realpath to sanity check
+			return true;
+		}
+	};
+
+///////////////////////////////////////////////////////////////////////////////////////////////////
+
 const int ZLIB_DECOMPRESS_BUFFER_SIZE = 32 * 1024;
 const int ZLIB_INFLATE_WINDOW_BITS = MAX_WBITS;
 const int ZLIB_DECODE_GZIP_HEADER = 16;
@@ -81,24 +149,14 @@ typedef struct
 #define FIFOTYPE '6'            /* FIFO special */
 #define CONTTYPE '7'            /* reserved */
 
-class TarExtractorPrivate
+	const char PAX_GLOBAL_HEADER = 'g';
+	const char PAX_FILE_ATTRIBUTES = 'x';
+
+
+class TarExtractorPrivate : public ArchiveExtractorPrivate
 {
 public:
-    typedef enum {
-        INVALID = 0,
-        READING_HEADER,
-        READING_FILE,
-        READING_PADDING,
-        PRE_END_OF_ARCHVE,
-        END_OF_ARCHIVE,
-        ERROR_STATE, ///< states above this are error conditions
-        BAD_ARCHIVE,
-        BAD_DATA,
-        FILTER_STOPPED
-    } State;
-
-    SGPath path;
-    State state;
+    
     union {
         UstarHeaderBlock header;
         uint8_t headerBytes[TAR_HEADER_BLOCK_SIZE];
@@ -109,17 +167,22 @@ public:
     size_t currentFileSize;
     z_stream zlibStream;
     uint8_t* zlibOutput;
-    bool haveInitedZLib;
-    bool uncompressedData; // set if reading a plain .tar (not tar.gz)
+    bool haveInitedZLib = false;
+    bool uncompressedData = false; // set if reading a plain .tar (not tar.gz)
     uint8_t* headerPtr;
-    TarExtractor* outer;
     bool skipCurrentEntry = false;
+    std::string paxAttributes;
+    std::string paxPathName;
     
-    TarExtractorPrivate(TarExtractor* o) :
-        haveInitedZLib(false),
-        uncompressedData(false),
-        outer(o)
+    TarExtractorPrivate(ArchiveExtractor* o) :
+		ArchiveExtractorPrivate(o)
     {
+		memset(&zlibStream, 0, sizeof(z_stream));
+		zlibOutput = (unsigned char*)malloc(ZLIB_DECOMPRESS_BUFFER_SIZE);
+		zlibStream.zalloc = Z_NULL;
+		zlibStream.zfree = Z_NULL;
+		zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE;
+		zlibStream.next_out = zlibOutput;
     }
 
     ~TarExtractorPrivate()
@@ -127,6 +190,17 @@ public:
         free(zlibOutput);
     }
 
+    void readPaddingIfRequired()
+    {
+        size_t pad = currentFileSize % TAR_HEADER_BLOCK_SIZE;
+        if (pad) {
+            bytesRemaining = TAR_HEADER_BLOCK_SIZE - pad;
+            setState(READING_PADDING);
+        } else {
+            setState(READING_HEADER);
+        }
+    }
+    
     void checkEndOfState()
     {
         if (bytesRemaining > 0) {
@@ -138,13 +212,7 @@ public:
                 currentFile->close();
                 currentFile.reset();
             }
-            size_t pad = currentFileSize % TAR_HEADER_BLOCK_SIZE;
-            if (pad) {
-                bytesRemaining = TAR_HEADER_BLOCK_SIZE - pad;
-                setState(READING_PADDING);
-            } else {
-                setState(READING_HEADER);
-            }
+            readPaddingIfRequired();
         } else if (state == READING_HEADER) {
             processHeader();
         } else if (state == PRE_END_OF_ARCHVE) {
@@ -153,6 +221,12 @@ public:
             } else {
                 // what does the spec say here?
             }
+        } else if (state == READING_PAX_GLOBAL_ATTRIBUTES) {
+            parsePAXAttributes(true);
+            readPaddingIfRequired();
+        } else if (state == READING_PAX_FILE_ATTRIBUTES) {
+            parsePAXAttributes(false);
+            readPaddingIfRequired();
         } else if (state == READING_PADDING) {
             setState(READING_HEADER);
         }
@@ -168,6 +242,77 @@ public:
         state = newState;
     }
 
+	void extractBytes(const uint8_t* bytes, size_t count) override
+	{
+		zlibStream.next_in = (uint8_t*) bytes;
+		zlibStream.avail_in = count;
+
+		if (!haveInitedZLib) {
+			// now we have data, see if we're dealing with GZ-compressed data or not
+			if ((bytes[0] == 0x1f) && (bytes[1] == 0x8b)) {
+				// GZIP identification bytes
+				if (inflateInit2(&zlibStream, ZLIB_INFLATE_WINDOW_BITS | ZLIB_DECODE_GZIP_HEADER) != Z_OK) {
+					SG_LOG(SG_IO, SG_WARN, "inflateInit2 failed");
+					state = TarExtractorPrivate::BAD_DATA;
+					return;
+				}
+			} else {
+				UstarHeaderBlock* header = (UstarHeaderBlock*)bytes;
+				if (strncmp(header->magic, TMAGIC, TMAGLEN) != 0) {
+					SG_LOG(SG_IO, SG_WARN, "didn't find tar magic in header");
+					state = TarExtractorPrivate::BAD_DATA;
+					return;
+				}
+
+				uncompressedData = true;
+			}
+
+			haveInitedZLib = true;
+			setState(TarExtractorPrivate::READING_HEADER);
+		} // of init on first-bytes case
+
+		if (uncompressedData) {
+			processBytes((const char*) bytes, count);
+		} else {
+			size_t writtenSize;
+			// loop, running zlib() inflate and sending output bytes to
+			// our request body handler. Keep calling inflate until no bytes are
+			// written, and ZLIB has consumed all available input
+			do {
+				zlibStream.next_out = zlibOutput;
+				zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE;
+				int result = inflate(&zlibStream, Z_NO_FLUSH);
+				if (result == Z_OK || result == Z_STREAM_END) {
+					// nothing to do
+
+				}
+				else if (result == Z_BUF_ERROR) {
+					// transient error, fall through
+				}
+				else {
+					//  _error = result;
+					SG_LOG(SG_IO, SG_WARN, "Permanent ZLib error:" << zlibStream.msg);
+					state = TarExtractorPrivate::BAD_DATA;
+					return;
+				}
+
+				writtenSize = ZLIB_DECOMPRESS_BUFFER_SIZE - zlibStream.avail_out;
+				if (writtenSize > 0) {
+					processBytes((const char*) zlibOutput, writtenSize);
+				}
+
+				if (result == Z_STREAM_END) {
+					break;
+				}
+			} while ((zlibStream.avail_in > 0) || (writtenSize > 0));
+		} // of Zlib-compressed data
+	}
+
+	void flush() override
+	{
+		// no-op for tar files, we process everything greedily
+	}
+
     void processHeader()
     {
         if (headerIsAllZeros()) {
@@ -180,28 +325,32 @@ public:
         }
 
         if (strncmp(header.magic, TMAGIC, TMAGLEN) != 0) {
-            SG_LOG(SG_IO, SG_WARN, "magic is wrong");
+            SG_LOG(SG_IO, SG_WARN, "Untar: magic is wrong");
             state = BAD_ARCHIVE;
             return;
         }
 
         skipCurrentEntry = false;
         std::string tarPath = std::string(header.prefix) + std::string(header.fileName);
-
+        if (!paxPathName.empty()) {
+            tarPath = paxPathName;
+            paxPathName.clear(); // clear for next file
+        }
+        
         if (!isSafePath(tarPath)) {
             SG_LOG(SG_IO, SG_WARN, "bad tar path:" << tarPath);
             skipCurrentEntry = true;
         }
 
-        auto result = outer->filterPath(tarPath);
-        if (result == TarExtractor::Stop) {
+        auto result = filterPath(tarPath);
+        if (result == ArchiveExtractor::Stop) {
             setState(FILTER_STOPPED);
             return;
-        } else if (result == TarExtractor::Skipped) {
+        } else if (result == ArchiveExtractor::Skipped) {
             skipCurrentEntry = true;
         }
         
-        SGPath p = path / tarPath;
+        SGPath p = extractRootPath() / tarPath;
         if (header.typeflag == DIRTYPE) {
             if (!skipCurrentEntry) {
                 Dir dir(p);
@@ -216,6 +365,20 @@ public:
                 currentFile->open(SG_IO_OUT);
             }
             setState(READING_FILE);
+        } else if (header.typeflag == PAX_GLOBAL_HEADER) {
+            setState(READING_PAX_GLOBAL_ATTRIBUTES);
+            currentFileSize = ::strtol(header.size, NULL, 8);
+            bytesRemaining = currentFileSize;
+            paxAttributes.clear();
+        } else if (header.typeflag == PAX_FILE_ATTRIBUTES) {
+            setState(READING_PAX_FILE_ATTRIBUTES);
+            currentFileSize = ::strtol(header.size, NULL, 8);
+            bytesRemaining = currentFileSize;
+            paxAttributes.clear();
+        } else if ((header.typeflag == SYMTYPE) || (header.typeflag == LNKTYPE)) {
+            SG_LOG(SG_IO, SG_WARN, "Tarball contains a link or symlink, will be skipped:" << tarPath);
+            skipCurrentEntry = true;
+            setState(READING_HEADER);
         } else {
             SG_LOG(SG_IO, SG_WARN, "Unsupported tar file type:" << header.typeflag);
             state = BAD_ARCHIVE;
@@ -240,6 +403,9 @@ public:
             headerPtr += curBytes;
         } else if (state == READING_PADDING) {
             bytesRemaining -= curBytes;
+        } else if ((state == READING_PAX_FILE_ATTRIBUTES) || (state == READING_PAX_GLOBAL_ATTRIBUTES)) {
+            bytesRemaining -= curBytes;
+            paxAttributes.append(bytes, curBytes);
         }
 
         checkEndOfState();
@@ -261,132 +427,283 @@ public:
         return true;
     }
 
-    bool isSafePath(const std::string& p) const
+    // https://www.ibm.com/support/knowledgecenter/en/SSLTBW_2.3.0/com.ibm.zos.v2r3.bpxa500/paxex.htm#paxex
+    void parsePAXAttributes(bool areGlobal)
     {
-        if (p.empty()) {
-            return false;
+        auto lineStart = 0;
+        for (;;) {
+            auto firstSpace = paxAttributes.find(' ', lineStart);
+            auto firstEq = paxAttributes.find('=', lineStart);
+            if ((firstEq == std::string::npos) || (firstSpace == std::string::npos)) {
+                SG_LOG(SG_IO, SG_WARN, "Malfroemd PAX attributes in tarfile");
+                break;
+            }
+            
+            uint32_t lengthBytes = std::stoul(paxAttributes.substr(lineStart, firstSpace));
+            uint32_t dataBytes = lengthBytes - (firstEq + 1);
+            std::string name = paxAttributes.substr(firstSpace+1, firstEq - (firstSpace + 1));
+            
+            // dataBytes - 1 here to trim off the trailing newline
+            std::string data = paxAttributes.substr(firstEq+1, dataBytes - 1);
+            
+            processPAXAttribute(areGlobal, name, data);
+            
+            lineStart += lengthBytes;
         }
-
-        // reject absolute paths
-        if (p.at(0) == '/') {
-            return false;
+    }
+    
+    void processPAXAttribute(bool isGlobalAttr, const std::string& attrName, const std::string& data)
+    {
+        if (!isGlobalAttr && (attrName == "path")) {
+            // data is UTF-8 encoded path name
+            paxPathName = data;
         }
-
-        // reject paths containing '..'
-        size_t doubleDot = p.find("..");
-        if (doubleDot != std::string::npos) {
-            return false;
-        }
-
-        // on POSIX could use realpath to sanity check
-        return true;
     }
 };
 
-TarExtractor::TarExtractor(const SGPath& rootPath) :
-    d(new TarExtractorPrivate(this))
-{
+///////////////////////////////////////////////////////////////////////////////
 
-    d->path = rootPath;
-    d->state = TarExtractorPrivate::INVALID;
-
-    memset(&d->zlibStream, 0, sizeof(z_stream));
-    d->zlibOutput = (unsigned char*) malloc(ZLIB_DECOMPRESS_BUFFER_SIZE);
-    d->zlibStream.zalloc = Z_NULL;
-    d->zlibStream.zfree = Z_NULL;
-
-    d->zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE;
-    d->zlibStream.next_out = d->zlibOutput;
+extern "C" {
+	void fill_memory_filefunc(zlib_filefunc_def*);
 }
 
-TarExtractor::~TarExtractor()
+class ZipExtractorPrivate : public ArchiveExtractorPrivate
+{
+public:
+	std::string m_buffer;
+
+	ZipExtractorPrivate(ArchiveExtractor* outer) :
+		ArchiveExtractorPrivate(outer)
+	{
+
+	}
+
+	~ZipExtractorPrivate()
+	{
+
+	}
+
+	void extractBytes(const uint8_t* bytes, size_t count) override
+	{
+		// becuase the .zip central directory is at the end of the file,
+		// we have no choice but to simply buffer bytes here until flush()
+		// is called
+		m_buffer.append((const char*) bytes, count);
+	}
+
+	void flush() override
+	{
+		zlib_filefunc_def memoryAccessFuncs;
+		fill_memory_filefunc(&memoryAccessFuncs);
+
+		char bufferName[128];
+#if defined(SG_WINDOWS)
+        ::snprintf(bufferName, 128, "%p+%llx", m_buffer.data(), m_buffer.size());
+#else
+		::snprintf(bufferName, 128, "%p+%lx", m_buffer.data(), m_buffer.size());
+#endif
+		unzFile zip = unzOpen2(bufferName, &memoryAccessFuncs);
+
+		const size_t BUFFER_SIZE = 32 * 1024;
+		void* buf = malloc(BUFFER_SIZE);
+
+		try {
+			int result = unzGoToFirstFile(zip);
+			if (result != UNZ_OK) {
+				throw sg_exception("failed to go to first file in archive");
+			}
+
+			while (true) {
+				extractCurrentFile(zip, (char*)buf, BUFFER_SIZE);
+				if (state == FILTER_STOPPED) {
+					break;
+				}
+
+				result = unzGoToNextFile(zip);
+				if (result == UNZ_END_OF_LIST_OF_FILE) {
+					break;
+				}
+				else if (result != UNZ_OK) {
+					throw sg_io_exception("failed to go to next file in the archive");
+				}
+			}
+			state = END_OF_ARCHIVE;
+		}
+		catch (sg_exception&) {
+			state = BAD_ARCHIVE;
+		}
+
+		free(buf);
+		unzClose(zip);
+	}
+
+	void extractCurrentFile(unzFile zip, char* buffer, size_t bufferSize)
+	{
+		unz_file_info fileInfo;
+		unzGetCurrentFileInfo(zip, &fileInfo,
+			buffer, bufferSize,
+			NULL, 0,  /* extra field */
+			NULL, 0 /* comment field */);
+
+		std::string name(buffer);
+		if (!isSafePath(name)) {
+			throw sg_format_exception("Bad zip path", name);
+		}
+
+		auto filterResult = filterPath(name);
+		if (filterResult == ArchiveExtractor::Stop) {
+			state = FILTER_STOPPED;
+			return;
+		}
+		else if (filterResult == ArchiveExtractor::Skipped) {
+			return;
+		}
+
+		if (fileInfo.uncompressed_size == 0) {
+			// assume it's a directory for now
+			// since we create parent directories when extracting
+			// a path, we're done here
+			return;
+		}
+
+		int result = unzOpenCurrentFile(zip);
+		if (result != UNZ_OK) {
+			throw sg_io_exception("opening current zip file failed", sg_location(name));
+		}
+
+		sg_ofstream outFile;
+		bool eof = false;
+		SGPath path = extractRootPath() / name;
+
+		// create enclosing directory heirarchy as required
+		Dir parentDir(path.dir());
+		if (!parentDir.exists()) {
+			bool ok = parentDir.create(0755);
+			if (!ok) {
+				throw sg_io_exception("failed to create directory heirarchy for extraction", path);
+			}
+		}
+
+		outFile.open(path, std::ios::binary | std::ios::trunc | std::ios::out);
+		if (outFile.fail()) {
+			throw sg_io_exception("failed to open output file for writing", path);
+		}
+
+		while (!eof) {
+			int bytes = unzReadCurrentFile(zip, buffer, bufferSize);
+			if (bytes < 0) {
+				throw sg_io_exception("unzip failure reading curent archive", sg_location(name));
+			}
+			else if (bytes == 0) {
+				eof = true;
+			}
+			else {
+				outFile.write(buffer, bytes);
+			}
+		}
+
+		outFile.close();
+		unzCloseCurrentFile(zip);
+	}
+};
+
+//////////////////////////////////////////////////////////////////////////////
+
+ArchiveExtractor::ArchiveExtractor(const SGPath& rootPath) :
+	_rootPath(rootPath)
+{
+}
+
+ArchiveExtractor::~ArchiveExtractor()
 {
 
 }
 
-void TarExtractor::extractBytes(const char* bytes, size_t count)
+void ArchiveExtractor::extractBytes(const uint8_t* bytes, size_t count)
 {
-    if (d->state >= TarExtractorPrivate::ERROR_STATE) {
+	if (!d) {
+		_prebuffer.append((char*) bytes, count);
+		auto r = determineType((uint8_t*) _prebuffer.data(), _prebuffer.size());
+		if (r == InsufficientData) {
+			return;
+		}
+
+		if (r == TarData) {
+			d.reset(new TarExtractorPrivate(this));
+		}
+		else if (r == ZipData) {
+			d.reset(new ZipExtractorPrivate(this));
+		}
+		else {
+			SG_LOG(SG_IO, SG_ALERT, "Invcalid archive type");
+			_invalidDataType = true;
+			return;
+		}
+
+		// if hit here, we created the extractor. Feed the prefbuffer
+		// bytes through it
+		d->extractBytes((uint8_t*) _prebuffer.data(), _prebuffer.size());
+		_prebuffer.clear();
+		return;
+	}
+
+    if (d->state >= ArchiveExtractorPrivate::ERROR_STATE) {
         return;
     }
 
-    d->zlibStream.next_in = (uint8_t*) bytes;
-    d->zlibStream.avail_in = count;
-
-    if (!d->haveInitedZLib) {
-        // now we have data, see if we're dealing with GZ-compressed data or not
-        uint8_t* ubytes = (uint8_t*) bytes;
-        if ((ubytes[0] == 0x1f) && (ubytes[1] == 0x8b)) {
-            // GZIP identification bytes
-            if (inflateInit2(&d->zlibStream, ZLIB_INFLATE_WINDOW_BITS | ZLIB_DECODE_GZIP_HEADER) != Z_OK) {
-                SG_LOG(SG_IO, SG_WARN, "inflateInit2 failed");
-                d->state = TarExtractorPrivate::BAD_DATA;
-                return;
-            }
-        } else {
-            UstarHeaderBlock* header = (UstarHeaderBlock*) bytes;
-            if (strncmp(header->magic, TMAGIC, TMAGLEN) != 0) {
-                SG_LOG(SG_IO, SG_WARN, "didn't find tar magic in header");
-                d->state = TarExtractorPrivate::BAD_DATA;
-                return;
-            }
-
-            d->uncompressedData = true;
-        }
-
-        d->haveInitedZLib = true;
-        d->setState(TarExtractorPrivate::READING_HEADER);
-    } // of init on first-bytes case
-
-    if (d->uncompressedData) {
-        d->processBytes(bytes, count);
-    } else {
-        size_t writtenSize;
-        // loop, running zlib() inflate and sending output bytes to
-        // our request body handler. Keep calling inflate until no bytes are
-        // written, and ZLIB has consumed all available input
-        do {
-            d->zlibStream.next_out = d->zlibOutput;
-            d->zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE;
-            int result = inflate(&d->zlibStream, Z_NO_FLUSH);
-            if (result == Z_OK || result == Z_STREAM_END) {
-                // nothing to do
-
-            } else if (result == Z_BUF_ERROR) {
-                // transient error, fall through
-            } else {
-                //  _error = result;
-                SG_LOG(SG_IO, SG_WARN, "Permanent ZLib error:" << d->zlibStream.msg);
-                d->state = TarExtractorPrivate::BAD_DATA;
-                return;
-            }
-
-            writtenSize = ZLIB_DECOMPRESS_BUFFER_SIZE - d->zlibStream.avail_out;
-            if (writtenSize > 0) {
-                d->processBytes((const char*) d->zlibOutput, writtenSize);
-            }
-
-            if (result == Z_STREAM_END) {
-                break;
-            }
-        } while ((d->zlibStream.avail_in > 0) || (writtenSize > 0));
-    } // of Zlib-compressed data
+	d->extractBytes(bytes, count);
 }
 
-bool TarExtractor::isAtEndOfArchive() const
+void ArchiveExtractor::flush()
 {
-    return (d->state == TarExtractorPrivate::END_OF_ARCHIVE);
+	if (!d)
+		return;
+
+	d->flush();
 }
 
-bool TarExtractor::hasError() const
+bool ArchiveExtractor::isAtEndOfArchive() const
 {
-    return (d->state >= TarExtractorPrivate::ERROR_STATE);
+	if (!d)
+		return false;
+
+    return (d->state == ArchiveExtractorPrivate::END_OF_ARCHIVE);
 }
 
-bool TarExtractor::isTarData(const uint8_t* bytes, size_t count)
+bool ArchiveExtractor::hasError() const
+{
+	if (_invalidDataType)
+		return true;
+
+	if (!d)
+		return false;
+
+    return (d->state >= ArchiveExtractorPrivate::ERROR_STATE);
+}
+
+ArchiveExtractor::DetermineResult ArchiveExtractor::determineType(const uint8_t* bytes, size_t count)
+{
+	// check for ZIP
+	if (count < 4) {
+		return InsufficientData;
+	}
+
+	if (memcmp(bytes, "PK\x03\x04", 4) == 0) {
+		return ZipData;
+	}
+
+	auto r = isTarData(bytes, count);
+	if ((r == TarData) || (r == InsufficientData))
+		return r;
+
+	return Invalid;
+}
+
+
+ArchiveExtractor::DetermineResult ArchiveExtractor::isTarData(const uint8_t* bytes, size_t count)
 {
     if (count < 2) {
-        return false;
+        return InsufficientData;
     }
 
     UstarHeaderBlock* header = 0;
@@ -404,21 +721,20 @@ bool TarExtractor::isTarData(const uint8_t* bytes, size_t count)
 
         if (inflateInit2(&z, ZLIB_INFLATE_WINDOW_BITS | ZLIB_DECODE_GZIP_HEADER) != Z_OK) {
             inflateEnd(&z);
-            return false;
+            return Invalid;
         }
 
         int result = inflate(&z, Z_SYNC_FLUSH);
         if (result != Z_OK) {
             SG_LOG(SG_IO, SG_WARN, "inflate failed:" << result);
             inflateEnd(&z);
-            return false; // not tar data
+            return Invalid; // not tar data
         }
 
         size_t written = 4096 - z.avail_out;
         if (written < TAR_HEADER_BLOCK_SIZE) {
-            SG_LOG(SG_IO, SG_WARN, "insufficient data for header");
             inflateEnd(&z);
-            return false;
+            return InsufficientData;
         }
 
         header = reinterpret_cast<UstarHeaderBlock*>(zlibOutput);
@@ -426,22 +742,25 @@ bool TarExtractor::isTarData(const uint8_t* bytes, size_t count)
     } else {
         // uncompressed tar
         if (count < TAR_HEADER_BLOCK_SIZE) {
-            SG_LOG(SG_IO, SG_WARN, "insufficient data for header");
-            return false;
+            return InsufficientData;
         }
 
         header = (UstarHeaderBlock*) bytes;
     }
 
     if (strncmp(header->magic, TMAGIC, TMAGLEN) != 0) {
-        SG_LOG(SG_IO, SG_WARN, "not a tar file");
-        return false;
+        return Invalid;
     }
 
-    return true;
+    return TarData;
 }
 
-auto TarExtractor::filterPath(std::string& pathToExtract)
+void ArchiveExtractor::extractLocalFile(const SGPath& archiveFile)
+{
+
+}
+
+auto ArchiveExtractor::filterPath(std::string& pathToExtract)
   -> PathResult
 {
     SG_UNUSED(pathToExtract);

simgear/io/untar.hxx

@@ -21,40 +21,68 @@
 #include <memory>
 
 #include <cstdlib>
-#include <stdint.h> // for uint8_t
+#include <cstdint>
+
 #include <simgear/misc/sg_path.hxx>
 
 namespace simgear
 {
 
-class TarExtractorPrivate;
+class ArchiveExtractorPrivate;
 
-class TarExtractor
+class ArchiveExtractor
 {
 public:
-    TarExtractor(const SGPath& rootPath);
-	~TarExtractor();
+	ArchiveExtractor(const SGPath& rootPath);
+	~ArchiveExtractor();
 
-    static bool isTarData(const uint8_t* bytes, size_t count);
+	enum DetermineResult
+	{
+		Invalid,
+		InsufficientData,
+		TarData,
+		ZipData
+	};
 
-    void extractBytes(const char* bytes, size_t count);
+	static DetermineResult determineType(const uint8_t* bytes, size_t count);
+
+	/**
+	 * @brief API to extract a local zip or tar.gz 
+	 */
+	void extractLocalFile(const SGPath& archiveFile);
+
+	/**
+	 * @brief API to extract from memory - this can be called multiple
+	 * times for streamking from a network socket etc
+	 */
+    void extractBytes(const uint8_t* bytes, size_t count);
+
+	void flush();
 
     bool isAtEndOfArchive() const;
 
     bool hasError() const;
 
+	enum PathResult {
+		Accepted,
+		Skipped,
+		Modified,
+		Stop
+	};
+
 protected:
-    enum PathResult {
-      Accepted,
-      Skipped,
-      Modified,
-      Stop
-    };
+
 
     virtual PathResult filterPath(std::string& pathToExtract);
 private:
-    friend class TarExtractorPrivate;
-    std::unique_ptr<TarExtractorPrivate> d;
+	static DetermineResult isTarData(const uint8_t* bytes, size_t count);
+
+    friend class ArchiveExtractorPrivate;
+    std::unique_ptr<ArchiveExtractorPrivate> d;
+
+	SGPath _rootPath;
+	std::string _prebuffer; // store bytes before type is determined
+	bool _invalidDataType = false;
 };
 
 } // of namespace simgear

simgear/math/SGMatrix.hxx

@@ -142,10 +142,10 @@ public:
   T (&sg(void))[4][4]
   { return _data.ptr(); }
   /// Readonly raw storage interface
-  const simd4x4_t<T,4> (&simd4x4(void) const)
+  const simd4x4_t<T,4> &simd4x4(void) const
   { return _data; }
   /// Readonly raw storage interface
-  simd4x4_t<T,4> (&simd4x4(void))
+  simd4x4_t<T,4> &simd4x4(void)
   { return _data; }
 
 

simgear/math/SGVec2.hxx

@@ -87,10 +87,10 @@ public:
   /// Access raw data
   T (&data(void))[2]
   { return _data.ptr(); }
-  const simd4_t<T,2> (&simd2(void) const)
+  const simd4_t<T,2> &simd2(void) const
   { return _data; }
   /// Readonly raw storage interface
-  simd4_t<T,2> (&simd2(void))
+  simd4_t<T,2> &simd2(void)
   { return _data; }
 
   /// Inplace addition

simgear/math/SGVec3.hxx

@@ -106,10 +106,10 @@ public:
   T (&data(void))[3]
   { return _data.ptr(); }
   /// Readonly raw storage interface
-  const simd4_t<T,3> (&simd3(void) const)
+  const simd4_t<T,3> &simd3(void) const
   { return _data; }
   /// Readonly raw storage interface
-  simd4_t<T,3> (&simd3(void))
+  simd4_t<T,3> &simd3(void)
   { return _data; }
 
   /// Inplace addition

simgear/math/SGVec4.hxx

@@ -99,10 +99,10 @@ public:
   T (&data(void))[4]
   { return _data.ptr(); }
   /// Readonly raw storage interface
-  const simd4_t<T,4> (&simd4(void) const)
+  const simd4_t<T,4> &simd4(void) const
   { return _data; }
   /// Readonly raw storage interface
-  simd4_t<T,4> (&simd4(void))
+  simd4_t<T,4> &simd4(void)
   { return _data; }
 
   /// Inplace addition

simgear/math/simd.hxx

@@ -309,7 +309,7 @@ inline simd4_t<T,N> operator*(simd4_t<T,N> v, T f) {
     return v;
 }
 
-#ifdef ENABLE_SIMD
+#ifdef ENABLE_SIMD_CODE
 
 # ifdef __SSE__
 namespace simd4
@@ -352,10 +352,10 @@ public:
     simd4_t(const simd4_t<float,2>& v) { simd4 = v.v4(); }
     simd4_t(const __m128& v) { simd4 = v; }
 
-    inline const __m128 (&v4(void) const) {
+    inline const __m128 &v4(void) const {
         return simd4;
     }
-    inline __m128 (&v4(void)) {
+    inline __m128 &v4(void) {
         return simd4;
     }
 
@@ -1120,11 +1120,11 @@ public:
     simd4_t(const simd4_t<int,2>& v) { simd4 = v.v4(); }
     simd4_t(const __m128i& v) { simd4 = v; }
 
-    inline __m128i (&v4(void)) {
+    inline __m128i &v4(void) {
         return simd4;
     }
 
-    inline const __m128i (&v4(void) const) {
+    inline const __m128i &v4(void) const {
         return simd4;
     }
 
@@ -1305,7 +1305,7 @@ inline simd4_t<int,N> max(simd4_t<int,N> v1, const simd4_t<int,N>& v2) {
 
 # endif
 
-#endif /* ENABLE_SIMD */
+#endif /* ENABLE_SIMD_CODE */
 
 #endif /* __SIMD_H__ */
 

simgear/math/simd4x4.hxx

@@ -289,7 +289,7 @@ inline simd4x4_t<T,N> operator*(const simd4x4_t<T,N>& m1, const simd4x4_t<T,N>&
 }
 
 
-#ifdef ENABLE_SIMD
+#ifdef ENABLE_SIMD_CODE
 
 # ifdef __SSE__
 template<>
@@ -1191,7 +1191,7 @@ inline simd4_t<int,3> transform<int>(const simd4x4_t<int,4>& m, const simd4_t<in
 } /* namespace simd4x */
 # endif
 
-#endif /* ENABLE_SIMD */
+#endif /* ENABLE_SIMD_CODE */
 
 #endif /* __SIMD4X4_H__ */
 

simgear/misc/ResourceManager.cxx

@@ -17,14 +17,23 @@
 //
 // $Id$
 
+#include <assert.h>
+#include <algorithm>
+
 #include <simgear_config.h>
 
 #include <simgear/misc/ResourceManager.hxx>
+#include <simgear/debug/logstream.hxx>
 
 namespace simgear
 {
     
-static ResourceManager* static_manager = NULL;
+static ResourceManager* static_manager = nullptr;
+
+ResourceProvider::~ResourceProvider()
+{
+    // pin to this compilation unit
+}
 
 ResourceManager::ResourceManager()
 {
@@ -40,13 +49,20 @@ ResourceManager* ResourceManager::instance()
     return static_manager;
 }    
 
+ResourceManager::~ResourceManager()
+{
+    assert(this == static_manager);
+    static_manager = nullptr;
+    std::for_each(_providers.begin(), _providers.end(), 
+        [](ResourceProvider* p) { delete p; });
+}
 /**
  * trivial provider using a fixed base path
  */
 class BasePathProvider : public ResourceProvider
 {
 public:
-    BasePathProvider(const SGPath& aBase, int aPriority) :
+    BasePathProvider(const SGPath& aBase, ResourceManager::Priority aPriority) :
         ResourceProvider(aPriority),
         _base(aBase)
     {
@@ -69,6 +85,8 @@ void ResourceManager::addBasePath(const SGPath& aPath, Priority aPriority)
 
 void ResourceManager::addProvider(ResourceProvider* aProvider)
 {
+    assert(aProvider);
+
     ProviderVec::iterator it = _providers.begin();
     for (; it != _providers.end(); ++it) {
       if (aProvider->priority() > (*it)->priority()) {
@@ -81,6 +99,16 @@ void ResourceManager::addProvider(ResourceProvider* aProvider)
     _providers.push_back(aProvider);
 }
 
+void ResourceManager::removeProvider(ResourceProvider* aProvider)
+{
+    assert(aProvider);
+    auto it = std::find(_providers.begin(), _providers.end(), aProvider);
+    if (it == _providers.end()) {
+        SG_LOG(SG_GENERAL, SG_DEV_ALERT, "unknown provider doing remove");
+        return;
+    }
+}
+
 SGPath ResourceManager::findPath(const std::string& aResource, SGPath aContext)
 {
     if (!aContext.isNull()) {
@@ -90,9 +118,8 @@ SGPath ResourceManager::findPath(const std::string& aResource, SGPath aContext)
         }
     }
     
-    ProviderVec::iterator it = _providers.begin();
-    for (; it != _providers.end(); ++it) {
-      SGPath path = (*it)->resolve(aResource, aContext);
+    for (auto provider : _providers) {
+      SGPath path = provider->resolve(aResource, aContext);
       if (!path.isNull()) {
         return path;
       }

simgear/misc/ResourceManager.hxx

@@ -36,6 +36,8 @@ class ResourceProvider;
 class ResourceManager
 {
 public:
+    ~ResourceManager();
+
     typedef enum {
       PRIORITY_DEFAULT = 0,
       PRIORITY_FALLBACK = -100,
@@ -55,6 +57,8 @@ public:
      */
     void addProvider(ResourceProvider* aProvider);
     
+    void removeProvider(ResourceProvider* aProvider);
+
     /**
      * given a resource name (or path), find the appropriate real resource
      * path.
@@ -75,17 +79,19 @@ class ResourceProvider
 public:
     virtual SGPath resolve(const std::string& aResource, SGPath& aContext) const = 0;
     
-    virtual int priority() const
+    virtual ~ResourceProvider();
+
+    virtual ResourceManager::Priority priority() const
     {
       return _priority;
     }
     
 protected:
-    ResourceProvider(int aPriority) :
+    ResourceProvider(ResourceManager::Priority aPriority) :
       _priority(aPriority)
     {}
     
-    int _priority;
+    ResourceManager::Priority _priority = ResourceManager::PRIORITY_DEFAULT;
 };
     
 } // of simgear namespace

simgear/misc/path_test.cxx

@@ -160,7 +160,7 @@ void test_path_dir()
 	SG_VERIFY(sub2.isFile());
 	SG_CHECK_EQUAL(sub2.sizeInBytes(), 250);
 
-    SGPath sub3 = p / "subß" / "file𝕽";
+    SGPath sub3 = p / "subß" / u8"file𝕽";
     sub3.create_dir(0755);
 
     {
@@ -174,7 +174,7 @@ void test_path_dir()
     sub3.set_cached(false);
     SG_VERIFY(sub3.exists());
     SG_CHECK_EQUAL(sub3.sizeInBytes(), 100);
-    SG_CHECK_EQUAL(sub3.file(), "file𝕽");
+    SG_CHECK_EQUAL(sub3.file(), u8"file𝕽");
 
 	simgear::Dir subD(p / "subA");
 	simgear::PathList dirChildren = subD.children(simgear::Dir::TYPE_DIR | simgear::Dir::NO_DOT_OR_DOTDOT);
@@ -188,7 +188,7 @@ void test_path_dir()
     simgear::Dir subS(sub3.dirPath());
     fileChildren = subS.children(simgear::Dir::TYPE_FILE | simgear::Dir::NO_DOT_OR_DOTDOT);
     SG_CHECK_EQUAL(fileChildren.size(), 1);
-    SG_CHECK_EQUAL(fileChildren[0], subS.path() / "file𝕽");
+    SG_CHECK_EQUAL(fileChildren[0], subS.path() / u8"file𝕽");
 
 }
 

simgear/misc/sg_path.cxx

@@ -107,7 +107,7 @@ static SGPath pathForKnownFolder(REFKNOWNFOLDERID folderId, const SGPath& def)
             // system call will allocate dynamic memory... which we must release when done
             wchar_t* localFolder = 0;
 
-            if (pSHGetKnownFolderPath(folderId, KF_FLAG_DEFAULT_PATH, NULL, &localFolder) == S_OK) {
+			if (pSHGetKnownFolderPath(folderId, KF_FLAG_DONT_VERIFY, NULL, &localFolder) == S_OK) {
                 SGPath folder_path = SGPath(localFolder, def.getPermissionChecker());
                 // release dynamic memory
                 CoTaskMemFree(static_cast<void*>(localFolder));

simgear/misc/stdint.hxx

@@ -12,65 +12,50 @@
 // $Id$
 
 
-//////////////////////////////////////////////////////////////////////
-//
-//  There are many sick systems out there:
-//
-//  check for sizeof(float) and sizeof(double)
-//  if sizeof(float) != 4 this code must be patched
-//  if sizeof(double) != 8 this code must be patched
-//
-//  Those are comments I fetched out of glibc source:
-//  - s390 is big-endian
-//  - Sparc is big-endian, but v9 supports endian conversion
-//    on loads/stores and GCC supports such a mode.  Be prepared.
-//  - The MIPS architecture has selectable endianness.
-//  - x86_64 is little-endian.
-//  - CRIS is little-endian.
-//  - m68k is big-endian.
-//  - Alpha is little-endian.
-//  - PowerPC can be little or big endian.
-//  - SH is bi-endian but with a big-endian FPU.
-//  - hppa1.1 big-endian.
-//  - ARM is (usually) little-endian but with a big-endian FPU.
-//
-//////////////////////////////////////////////////////////////////////
+#include <cstdint>
+#include <cstdlib> // for _byteswap_foo on Win32
 
-
-#ifdef _MSC_VER
-typedef signed char      int8_t;
-typedef signed short     int16_t;
-typedef signed int       int32_t;
-typedef signed __int64   int64_t;
-typedef unsigned char    uint8_t;
-typedef unsigned short   uint16_t;
-typedef unsigned int     uint32_t;
-typedef unsigned __int64 uint64_t;
-
-typedef int ssize_t;
-#elif defined(sgi) || defined(__sun)
-# include <sys/types.h>
-#else
-# include <stdint.h>
+#if defined(_MSC_VER)
+using ssize_t = int64_t; // this is a POSIX type, not a C one
 #endif
 
-
 inline uint16_t sg_bswap_16(uint16_t x) {
+#if defined(__llvm__) || \
+ (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) && !defined(__ICC)
+    return __builtin_bswap16(x);
+#elif defined(_MSC_VER) && !defined(_DEBUG)
+   return _byteswap_ushort(x);
+#else
     x = (x >> 8) | (x << 8);
     return x;
+#endif
 }
 
 inline uint32_t sg_bswap_32(uint32_t x) {
+#if defined(__llvm__) || \
+ (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) && !defined(__ICC)
+    return __builtin_bswap32(x);
+#elif defined(_MSC_VER) && !defined(_DEBUG)
+   return _byteswap_ulong(x);
+#else
     x = ((x >>  8) & 0x00FF00FFL) | ((x <<  8) & 0xFF00FF00L);
     x = (x >> 16) | (x << 16);
     return x;
+#endif
 }
 
 inline uint64_t sg_bswap_64(uint64_t x) {
+#if defined(__llvm__) || \
+ (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) && !defined(__ICC)
+   return __builtin_bswap64(x);
+#elif defined(_MSC_VER) && !defined(_DEBUG)
+   return _byteswap_uint64(x);
+#else
     x = ((x >>  8) & 0x00FF00FF00FF00FFLL) | ((x <<  8) & 0xFF00FF00FF00FF00LL);
     x = ((x >> 16) & 0x0000FFFF0000FFFFLL) | ((x << 16) & 0xFFFF0000FFFF0000LL);
     x = (x >> 32) | (x << 32);
     return x;
+#endif
 }
 
 

simgear/misc/strutils.cxx

@@ -37,7 +37,7 @@
 #include <simgear/package/md5.h>
 #include <simgear/compiler.h>   // SG_WINDOWS
 #include <simgear/structure/exception.hxx>
-
+#include <simgear/math/SGGeod.hxx>
 
 #if defined(SG_WINDOWS)
 	#include <windows.h>
@@ -623,6 +623,22 @@ namespace simgear {
         *p = tolower(*p);
       }
     }
+        
+        
+bool iequals(const std::string& a, const std::string& b)
+{
+    const auto lenA = a.length();
+    const auto lenB = b.length();
+    if (lenA != lenB) return false;
+    
+    const char* aPtr = a.data();
+    const char* bPtr = b.data();
+    for (size_t i = 0; i < lenA; ++i) {
+        if (tolower(*aPtr++) != tolower(*bPtr++)) return false;
+    }
+    
+    return true;
+}
 
 #if defined(SG_WINDOWS)
 static std::wstring convertMultiByteToWString(DWORD encoding, const std::string& a)
@@ -1133,6 +1149,306 @@ bool matchPropPathToTemplate(const std::string& path, const std::string& templat
 
     // unreachable
 }
+ 
+bool parseStringAsLatLonValue(const std::string& s, double& degrees)
+{
+    try {
+            string ss = simplify(s);
+            auto spacePos = ss.find_first_of(" *");
+        
+            if (spacePos == std::string::npos) {
+                degrees = std::stod(ss);
+            } else {
+                degrees = std::stod(ss.substr(0, spacePos));
+                
+                double minutes = 0.0, seconds = 0.0;
+                
+                // check for minutes marker
+                auto quotePos = ss.find('\'');
+                if (quotePos == std::string::npos) {
+                    const auto minutesStr = ss.substr(spacePos+1);
+                    if (!minutesStr.empty()) {
+                        minutes = std::stod(minutesStr);
+                    }
+                } else {
+                    minutes = std::stod(ss.substr(spacePos+1, quotePos - spacePos));
+                    const auto secondsStr = ss.substr(quotePos+1);
+                    if (!secondsStr.empty()) {
+                        seconds = std::stod(secondsStr);
+                    }
+                }
+                
+                if ((seconds < 0.0) || (minutes < 0.0)) {
+                    // don't allow sign information in minutes or seconds
+                    return false;
+                }
+                
+                double offset = (minutes / 60.0) + (seconds / 3600.0);
+                degrees += (degrees >= 0.0) ? offset : -offset;
+            }
+        
+            // since we simplified, any trailing N/S/E/W must be the last char
+            const char lastChar = ::toupper(ss.back());
+            if ((lastChar == 'W') || (lastChar == 'S')) {
+                degrees = -degrees;
+            }
+    } catch (std::exception&) {
+        // std::stdo can throw
+        return false;
+    }
+    
+    return true;
+}
+        
+namespace {
+    bool isLatString(const std::string &s)
+    {
+        const char lastChar = ::toupper(s.back());
+        return (lastChar == 'N') || (lastChar == 'S');
+    }
+    
+    bool isLonString(const std::string &s)
+    {
+        const char lastChar = ::toupper(s.back());
+        return (lastChar == 'E') || (lastChar == 'W');
+    }
+} // of anonymous namespace
+        
+bool parseStringAsGeod(const std::string& s, SGGeod* result, bool assumeLonLatOrder)
+{
+    if (s.empty())
+        return false;
+    
+    const auto commaPos = s.find(',');
+    if (commaPos == string::npos) {
+        return false;
+    }
+    
+    auto termA = simplify(s.substr(0, commaPos)),
+        termB = simplify(s.substr(commaPos+1));
+    double valueA, valueB;
+    if (!parseStringAsLatLonValue(termA, valueA) || !parseStringAsLatLonValue(termB, valueB)) {
+        return false;
+    }
+    
+    if (result) {
+        // explicit ordering
+        if (isLatString(termA) && isLonString(termB)) {
+            *result = SGGeod::fromDeg(valueB, valueA);
+        } else if (isLonString(termA) && isLatString(termB)) {
+            *result = SGGeod::fromDeg(valueA, valueB);
+        } else {
+            // implicit ordering
+            // SGGeod wants longitude, latitude
+            *result = assumeLonLatOrder ? SGGeod::fromDeg(valueA, valueB)
+                                        : SGGeod::fromDeg(valueB, valueA);
+        }
+    }
+    
+    return true;
+}
+        
+namespace {
+    const char* static_degreeSymbols[] = {
+        "*",
+        " ",
+        "\xB0",     // Latin-1 B0 codepoint
+        "\xC2\xB0"  // UTF-8 equivalent
+    };
+} // of anonymous namespace
+        
+std::string formatLatLonValueAsString(double deg, LatLonFormat format,
+                                      char c,
+                                      DegreeSymbol degreeSymbol)
+{
+    double min, sec;
+    const int sign = deg < 0.0 ? -1 : 1;
+    deg = fabs(deg);
+    char buf[128];
+    const char* degSym = static_degreeSymbols[static_cast<int>(degreeSymbol)];
+    
+    switch (format) {
+    case LatLonFormat::DECIMAL_DEGREES:
+        ::snprintf(buf, sizeof(buf), "%3.6f%c", deg, c);
+        break;
+        
+    case LatLonFormat::DEGREES_MINUTES:
+        // d mm.mmm' (DMM format) -- uses a round-off factor tailored to the
+        // required precision of the minutes field (three decimal places),
+        // preventing minute values of 60.
+        min = (deg - int(deg)) * 60.0;
+        if (min >= 59.9995) {
+            min -= 60.0;
+            deg += 1.0;
+        }
+        snprintf(buf, sizeof(buf), "%d%s%06.3f'%c", int(deg), degSym, fabs(min), c);
+        break;
+        
+    case LatLonFormat::DEGREES_MINUTES_SECONDS:
+        // d mm'ss.s" (DMS format) -- uses a round-off factor tailored to the
+        // required precision of the seconds field (one decimal place),
+        // preventing second values of 60.
+        min = (deg - int(deg)) * 60.0;
+        sec = (min - int(min)) * 60.0;
+        if (sec >= 59.95) {
+            sec -= 60.0;
+            min += 1.0;
+            if (min >= 60.0) {
+                min -= 60.0;
+                deg += 1.0;
+            }
+        }
+        ::snprintf(buf, sizeof(buf), "%d%s%02d'%04.1f\"%c", int(deg), degSym,
+                   int(min), fabs(sec), c);
+        break;
+            
+    case LatLonFormat::SIGNED_DECIMAL_DEGREES:
+        // d.dddddd' (signed DDD format).
+        ::snprintf(buf, sizeof(buf), "%3.6f", sign*deg);
+        break;
+    
+    case LatLonFormat::SIGNED_DEGREES_MINUTES:
+        // d mm.mmm' (signed DMM format).
+        min = (deg - int(deg)) * 60.0;
+        if (min >= 59.9995) {
+            min -= 60.0;
+            deg += 1.0;
+        }
+        if (sign == 1) {
+            snprintf(buf, sizeof(buf), "%d%s%06.3f'", int(deg), degSym, fabs(min));
+        } else {
+            snprintf(buf, sizeof(buf), "-%d%s%06.3f'", int(deg), degSym, fabs(min));
+        }
+        break;
+        
+            
+    case LatLonFormat::SIGNED_DEGREES_MINUTES_SECONDS:
+        // d mm'ss.s" (signed DMS format).
+        min = (deg - int(deg)) * 60.0;
+        sec = (min - int(min)) * 60.0;
+        if (sec >= 59.95) {
+            sec -= 60.0;
+            min += 1.0;
+            if (min >= 60.0) {
+                min -= 60.0;
+                deg += 1.0;
+            }
+        }
+        if (sign == 1) {
+            snprintf(buf, sizeof(buf), "%d%s%02d'%04.1f\"", int(deg), degSym, int(min), fabs(sec));
+        } else {
+            snprintf(buf, sizeof(buf), "-%d%s%02d'%04.1f\"", int(deg), degSym, int(min), fabs(sec));
+        }
+        break;
+            
+    case LatLonFormat::ZERO_PAD_DECIMAL_DEGRESS:
+        // dd.dddddd X, ddd.dddddd X (zero padded DDD format).
+        if (c == 'N' || c == 'S') {
+            snprintf(buf, sizeof(buf), "%09.6f%c", deg, c);
+        } else {
+            snprintf(buf, sizeof(buf), "%010.6f%c", deg, c);
+        }
+        break;
+            
+    case LatLonFormat::ZERO_PAD_DEGREES_MINUTES:
+        // dd mm.mmm' X, ddd mm.mmm' X (zero padded DMM format).
+        min = (deg - int(deg)) * 60.0;
+        if (min >= 59.9995) {
+            min -= 60.0;
+            deg += 1.0;
+        }
+        if (c == 'N' || c == 'S') {
+            snprintf(buf, sizeof(buf), "%02d%s%06.3f'%c", int(deg), degSym, fabs(min), c);
+        } else {
+            snprintf(buf, sizeof(buf), "%03d%s%06.3f'%c", int(deg), degSym, fabs(min), c);
+        }
+        break;
+            
+    case LatLonFormat::ZERO_PAD_DEGREES_MINUTES_SECONDS:
+        // dd mm'ss.s" X, dd mm'ss.s" X (zero padded DMS format).
+        min = (deg - int(deg)) * 60.0;
+        sec = (min - int(min)) * 60.0;
+        if (sec >= 59.95) {
+            sec -= 60.0;
+            min += 1.0;
+            if (min >= 60.0) {
+                min -= 60.0;
+                deg += 1.0;
+            }
+        }
+        if (c == 'N' || c == 'S') {
+            snprintf(buf, sizeof(buf), "%02d%s%02d'%04.1f\"%c", int(deg), degSym, int(min), fabs(sec), c);
+        } else {
+            snprintf(buf, sizeof(buf), "%03d%s%02d'%04.1f\"%c", int(deg), degSym, int(min), fabs(sec), c);
+        }
+        break;
+            
+    case LatLonFormat::TRINITY_HOUSE:
+        // dd* mm'.mmm X, ddd* mm'.mmm X (Trinity House Navigation standard).
+        min = (deg - int(deg)) * 60.0;
+        if (min >= 59.9995) {
+            min -= 60.0;
+            deg += 1.0;
+        }
+        if (c == 'N' || c == 'S') {
+            snprintf(buf, sizeof(buf), "%02d* %02d'.%03d%c", int(deg), int(min), int(SGMisc<double>::round((min-int(min))*1000)), c);
+        } else {
+            snprintf(buf, sizeof(buf), "%03d* %02d'.%03d%c", int(deg), int(min), int(SGMisc<double>::round((min-int(min))*1000)), c);
+        }
+        break;
+            
+    case LatLonFormat::DECIMAL_DEGREES_SYMBOL:
+        ::snprintf(buf, sizeof(buf), "%3.6f%s%c", deg, degSym, c);
+        break;
+            
+    case LatLonFormat::ICAO_ROUTE_DEGREES:
+    {
+        min = (deg - int(deg)) * 60.0;
+        if (min >= 59.9995) {
+            min -= 60.0;
+            deg += 1.0;
+        }
+        
+        if (static_cast<int>(min) == 0) {
+            // 7-digit mode
+            if (c == 'N' || c == 'S') {
+                snprintf(buf, sizeof(buf), "%02d%c", int(deg), c);
+            } else {
+                snprintf(buf, sizeof(buf), "%03d%c", int(deg), c);
+            }
+        } else {
+            // 11-digit mode
+            if (c == 'N' || c == 'S') {
+                snprintf(buf, sizeof(buf), "%02d%02d%c", int(deg), int(min), c);
+            } else {
+                snprintf(buf, sizeof(buf), "%03d%02d%c", int(deg), int(min), c);
+            }
+        }
+        break;
+    }
+            
+    default:
+        break;
+    }
+ 
+    return std::string(buf);
+}
+
+std::string formatGeodAsString(const SGGeod& geod, LatLonFormat format,
+                               DegreeSymbol degreeSymbol)
+{
+    const char ns = (geod.getLatitudeDeg() > 0.0) ? 'N' : 'S';
+    const char ew = (geod.getLongitudeDeg() > 0.0) ? 'E' : 'W';
+    
+    // no comma seperator
+    if (format == LatLonFormat::ICAO_ROUTE_DEGREES) {
+        return formatLatLonValueAsString(geod.getLatitudeDeg(), format, ns, degreeSymbol) +
+            formatLatLonValueAsString(geod.getLongitudeDeg(), format, ew, degreeSymbol);
+    }
+    
+    return formatLatLonValueAsString(geod.getLatitudeDeg(), format, ns, degreeSymbol) + ","
+        + formatLatLonValueAsString(geod.getLongitudeDeg(), format, ew, degreeSymbol);
+}
 
 } // end namespace strutils
 

simgear/misc/strutils.hxx

@@ -36,6 +36,9 @@
 
 typedef std::vector < std::string > string_list;
 
+// forward decls
+class SGGeod;
+
 namespace simgear {
   namespace strutils {
 
@@ -261,6 +264,11 @@ namespace simgear {
    */
   void lowercase(std::string &s);
   
+      /**
+       * case-insensitive string comparisom
+       */
+      bool iequals(const std::string& a, const std::string& b);
+      
 	/**
      * convert a string in the local Windows 8-bit encoding to UTF-8
      * (no-op on other platforms)
@@ -354,6 +362,68 @@ namespace simgear {
        *      /views[0]/view[4]/fig, /views[0]/view[1000]/flight
        */
       bool matchPropPathToTemplate(const std::string& path, const std::string& templatePath);
+      
+      bool parseStringAsLatLonValue(const std::string& s, double& result);
+      
+      /**
+       * Attempt to parse a string as a latitude,longitude input. Returns true
+       * or false based on success, and returns the SGGeod by pointer. Leading,
+       * trailing and internal white-space is skipped / ignored.
+       *
+       * Supported formats:
+       *   <signed decimal degrees latitude>,<signed decimal degress longitude>
+       *   <unsigned decimal degrees>[NS],<unsigned decimal degrees>[EW]
+       *   <degrees>*<decimal minutes>'[NS],<degrees>*<decimal minutes>'[EW]
+       *
+       * Latitude and longitude are parsed seperately so the formats for each
+       * do not need to agree. Latitude is assumed to precede longitude
+       * unless assumeLonLatOrder = true
+       *
+       * When NSEW characters are used, the order can be swapped and will be
+       * fixed correctly (longitude then latitude).
+       */
+      bool parseStringAsGeod(const std::string& string,
+                             SGGeod* result = nullptr,
+                             bool assumeLonLatOrder = false);
+      
+      // enum values here correspond to existing lon-lat format codes inside
+      // FlightGear (property: /sim/lon-lat-format )
+      // Don't re-order, just add new ones, or things may break
+      enum class LatLonFormat
+      {
+          DECIMAL_DEGREES = 0,          ///< 88.4N,4.54W,
+          DEGREES_MINUTES,             ///< 88 24.6'N, 4 30.5'W
+          DEGREES_MINUTES_SECONDS,
+          SIGNED_DECIMAL_DEGREES,          ///< 88.4,-4.54
+          SIGNED_DEGREES_MINUTES,
+          SIGNED_DEGREES_MINUTES_SECONDS,
+          ZERO_PAD_DECIMAL_DEGRESS,
+          ZERO_PAD_DEGREES_MINUTES,
+          ZERO_PAD_DEGREES_MINUTES_SECONDS,
+          TRINITY_HOUSE,            ///< dd* mm'.mmm X, ddd* mm'.mmm X (Trinity House Navigation standard).
+          DECIMAL_DEGREES_SYMBOL,        ///< 88.4*N,4.54*W
+          ICAO_ROUTE_DEGREES,              ///< 52N045W or 5212N04512W - precision auto-selected
+      };
+      
+      enum class DegreeSymbol
+      {
+          ASTERISK = 0,
+          SPACE,
+          LATIN1_DEGREE,
+          UTF8_DEGREE
+      };
+      
+      std::string formatLatLonValueAsString(double deg,
+                                            LatLonFormat format, char c,
+                                            DegreeSymbol degreeSymbol = DegreeSymbol::ASTERISK);
+
+      /**
+       * Format an SGGeod as a string according to the provided rule.
+       * if the SGGeod is invalid (default constructed), will return an empty string
+       */
+      std::string formatGeodAsString(const SGGeod& geod,
+                                     LatLonFormat format = LatLonFormat::DECIMAL_DEGREES,
+                                     DegreeSymbol degreeSymbol = DegreeSymbol::ASTERISK);
   } // end namespace strutils
 } // end namespace simgear
 

simgear/misc/strutils_test.cxx

@@ -20,6 +20,7 @@
 #include <simgear/misc/strutils.hxx>
 #include <simgear/structure/exception.hxx>
 #include <simgear/constants.h>
+#include <simgear/math/SGGeod.hxx>
 
 using std::string;
 using std::vector;
@@ -98,6 +99,16 @@ void test_to_int()
   SG_CHECK_EQUAL(strutils::to_int("-10000"), -10000);
 }
 
+void test_iequals()
+{
+    SG_VERIFY(strutils::iequals("abcdef", "AbCDeF"));
+    SG_VERIFY(strutils::iequals("", ""));
+    SG_VERIFY(!strutils::iequals("abcdE", "ABCD"));
+    SG_VERIFY(strutils::iequals("%$abcdef12", "%$AbCDeF12"));
+    SG_VERIFY(strutils::iequals("VOR-DME", "vor-dme"));
+    SG_VERIFY(!strutils::iequals("VOR-DME", "vor_dme"));
+}
+
 // Auxiliary function for test_readNonNegativeInt()
 void aux_readNonNegativeInt_setUpOStringStream(std::ostringstream& oss, int base)
 {
@@ -619,6 +630,101 @@ void test_utf8Convert()
     SG_VERIFY(a == aRoundTrip);
 }
 
+void test_parseGeod()
+{
+    SGGeod a;
+    SG_VERIFY(strutils::parseStringAsGeod("56.12,-3.0", &a));
+    SG_CHECK_EQUAL_EP(a.getLongitudeDeg(), -3.0);
+    SG_CHECK_EQUAL_EP(a.getLatitudeDeg(), 56.12);
+    
+    SG_VERIFY(strutils::parseStringAsGeod("56.12345678s,3.12345678w", &a));
+    SG_CHECK_EQUAL_EP(a.getLongitudeDeg(), -3.12345678);
+    SG_CHECK_EQUAL_EP(a.getLatitudeDeg(), -56.12345678);
+    
+    
+    // trailing degrees
+    SG_VERIFY(strutils::parseStringAsGeod("56.12*,-3.0*", &a));
+    SG_CHECK_EQUAL_EP(a.getLongitudeDeg(), -3.0);
+    SG_CHECK_EQUAL_EP(a.getLatitudeDeg(), 56.12);
+    
+    // embedded whitepace, DMS notation, NSEW notation
+    SG_VERIFY(strutils::parseStringAsGeod("\t40 30'50\"S,  12 34'56\"W ", &a));
+    SG_CHECK_EQUAL_EP(a.getLongitudeDeg(), -12.58222222);
+    SG_CHECK_EQUAL_EP(a.getLatitudeDeg(), -40.5138888);
+    
+    // embedded whitepace, DMS notation, NSEW notation, degrees symbol
+    SG_VERIFY(strutils::parseStringAsGeod("\t40*30'50\"S,  12*34'56\"W ", &a));
+    SG_CHECK_EQUAL_EP(a.getLongitudeDeg(), -12.58222222);
+    SG_CHECK_EQUAL_EP(a.getLatitudeDeg(), -40.5138888);
+    
+    // signed degrees-minutes
+    SG_VERIFY(strutils::parseStringAsGeod("-45 27.89,-12 34.56", &a));
+    SG_CHECK_EQUAL_EP(a.getLongitudeDeg(), -12.576);
+    SG_CHECK_EQUAL_EP(a.getLatitudeDeg(), -45.464833333);
+    
+    SG_VERIFY(strutils::parseStringAsGeod("") == false);
+    SG_VERIFY(strutils::parseStringAsGeod("aaaaaaaa") == false);
+    
+    // ordering tests
+    
+    // normal default order, but explicitly pass as lon,lat
+    // (should work)
+    SG_VERIFY(strutils::parseStringAsGeod("3.12345678w, 56.12345678s", &a));
+    SG_CHECK_EQUAL_EP(a.getLongitudeDeg(), -3.12345678);
+    SG_CHECK_EQUAL_EP(a.getLatitudeDeg(), -56.12345678);
+    
+
+    // different default order
+    // also some embedded whitespace for fun
+    SG_VERIFY(strutils::parseStringAsGeod(" -12 34.56,\n-45 27.89 ", &a, true));
+    SG_CHECK_EQUAL_EP(a.getLongitudeDeg(), -12.576);
+    SG_CHECK_EQUAL_EP(a.getLatitudeDeg(), -45.464833333);
+    
+    
+    // differnet default order, but still set explicitly so should
+    // use the lat,lon order
+    SG_VERIFY(strutils::parseStringAsGeod("\t40 30'50\"S,  12 34'56\"W ", &a, true));
+    SG_CHECK_EQUAL_EP(a.getLongitudeDeg(), -12.58222222);
+    SG_CHECK_EQUAL_EP(a.getLatitudeDeg(), -40.5138888);
+    
+    
+    // malformed inputs
+    
+    SG_VERIFY(strutils::parseStringAsGeod("12.345,", &a, true) == false);
+    double d;
+    SG_VERIFY(strutils::parseStringAsLatLonValue("", d) == false);
+}
+
+void test_formatGeod()
+{
+    SGGeod a = SGGeod::fromDeg(-3.46, 55.45);
+    SG_CHECK_EQUAL(strutils::formatGeodAsString(a, strutils::LatLonFormat::SIGNED_DECIMAL_DEGREES), "55.450000,-3.460000");
+    SG_CHECK_EQUAL(strutils::formatGeodAsString(a, strutils::LatLonFormat::DEGREES_MINUTES_SECONDS),
+                   "55*27'00.0\"N,3*27'36.0\"W");
+
+    
+    SG_CHECK_EQUAL(strutils::formatGeodAsString(a, strutils::LatLonFormat::ICAO_ROUTE_DEGREES),
+                   "5527N00327W");
+    SGGeod shortA = SGGeod::fromDeg(106, -34);
+    SG_CHECK_EQUAL(strutils::formatGeodAsString(shortA, strutils::LatLonFormat::ICAO_ROUTE_DEGREES),
+                   "34S106E");
+    
+    
+    const auto s = strutils::formatGeodAsString(a,
+                                                strutils::LatLonFormat::ZERO_PAD_DEGREES_MINUTES,
+                                                strutils::DegreeSymbol::LATIN1_DEGREE);
+    SG_CHECK_EQUAL(s, "55\xB0" "27.000'N,003\xB0" "27.600'W");
+    
+    // Jakarta, if you care
+    SGGeod b = SGGeod::fromDeg(106.8278, -6.1568);
+    const auto s2 = strutils::formatGeodAsString(b,
+                                                strutils::LatLonFormat::DECIMAL_DEGREES_SYMBOL,
+                                                strutils::DegreeSymbol::UTF8_DEGREE);
+    SG_CHECK_EQUAL(s2, "6.156800\xC2\xB0S,106.827800\xC2\xB0" "E");
+    
+    
+}
+
 int main(int argc, char* argv[])
 {
     test_strip();
@@ -639,6 +745,9 @@ int main(int argc, char* argv[])
     test_propPathMatch();
     test_readTime();
     test_utf8Convert();
+    test_parseGeod();
+    test_formatGeod();
+    test_iequals();
     
     return EXIT_SUCCESS;
 }

simgear/nasal/cppbind/NasalHash.hxx

@@ -24,6 +24,11 @@
 
 #include <simgear/structure/map.hxx>
 #include <boost/iterator/iterator_facade.hpp>
+#if BOOST_VERSION >= 105600
+#include <boost/core/enable_if.hpp>
+#else
+#include <boost/utility/enable_if.hpp>
+#endif
 
 namespace nasal
 {

simgear/package/Catalog.cxx

@@ -81,17 +81,27 @@ bool checkVersion(const std::string& aVersion, SGPropertyNode_ptr props)
     }
     return false;
 }
+    
+SGPropertyNode_ptr alternateForVersion(const std::string& aVersion, SGPropertyNode_ptr props)
+{
+    for (auto v : props->getChildren("alternate-version")) {
+        for (auto versionSpec : v->getChildren("version")) {
+            if (checkVersionString(aVersion, versionSpec->getStringValue())) {
+                return v;
+            }
+        }
+    }
+    
+    return {};
+}
 
 std::string redirectUrlForVersion(const std::string& aVersion, SGPropertyNode_ptr props)
 {
-    for (SGPropertyNode* v : props->getChildren("alternate-version")) {
-        std::string s(v->getStringValue("version"));
-        if (checkVersionString(aVersion, s)) {
-            return  v->getStringValue("url");;
-        }
-    }
+    auto node = alternateForVersion(aVersion, props);
+    if (node)
+        return node->getStringValue("url");;
 
-    return std::string();
+    return {};
 }
 
 //////////////////////////////////////////////////////////////////////////////
@@ -139,19 +149,15 @@ protected:
 
         std::string ver(m_owner->root()->applicationVersion());
         if (!checkVersion(ver, props)) {
-            SG_LOG(SG_GENERAL, SG_WARN, "downloaded catalog " << m_owner->url() << ", but version required " << ver);
-
             // check for a version redirect entry
-            std::string url = redirectUrlForVersion(ver, props);
-            if (!url.empty()) {
-                SG_LOG(SG_GENERAL, SG_WARN, "redirecting from " << m_owner->url() <<
-                       " to \n\t" << url);
-
-                // update the URL and kick off a new request
-                m_owner->setUrl(url);
-                Downloader* dl = new Downloader(m_owner, url);
-                m_owner->root()->makeHTTPRequest(dl);
+            auto alt = alternateForVersion(ver, props);
+            if (alt) {
+                SG_LOG(SG_GENERAL, SG_WARN, "have alternate version of package at:"
+                       << alt->getStringValue("url"));
+                m_owner->processAlternate(alt);
             } else {
+                SG_LOG(SG_GENERAL, SG_WARN, "downloaded catalog " << m_owner->url()
+                       << ", but app version " << ver << " is not comaptible");
                 m_owner->refreshComplete(Delegate::FAIL_VERSION);
             }
 
@@ -228,7 +234,7 @@ CatalogRef Catalog::createFromPath(Root* aRoot, const SGPath& aPath)
 
     bool versionCheckOk = checkVersion(aRoot->applicationVersion(), props);
     if (!versionCheckOk) {
-        SG_LOG(SG_GENERAL, SG_INFO, "catalog at:" << aPath << " failed version check: need version: "
+        SG_LOG(SG_GENERAL, SG_INFO, "catalog at:" << aPath << " failed version check: app version: "
                << aRoot->applicationVersion());
         // keep the catalog but mark it as needing an update
     } else {
@@ -598,6 +604,50 @@ bool Catalog::isEnabled() const
     }
 }
     
+void Catalog::processAlternate(SGPropertyNode_ptr alt)
+{
+    std::string altId;
+    const auto idPtr = alt->getStringValue("id");
+    if (idPtr) {
+        altId = std::string(idPtr);
+    }
+    
+    std::string altUrl;
+    if (alt->getStringValue("url")) {
+        altUrl = std::string(alt->getStringValue("url"));
+    }
+    
+    CatalogRef existing;
+    if (!altId.empty()) {
+        existing = root()->getCatalogById(altId);
+    } else {
+        existing = root()->getCatalogByUrl(altUrl);
+    }
+    
+    if (existing && (existing != this)) {
+        // we already have the alternate, so just go quiet here
+        changeStatus(Delegate::FAIL_VERSION);
+        return;
+    }
+
+    // we have an alternate ID, and it's differnt from our ID, so let's
+    // define a new catalog
+    if (!altId.empty()) {
+        SG_LOG(SG_GENERAL, SG_INFO, "Adding new catalog:" << altId << " as version alternate for " << id());
+        // new catalog being added
+        createFromUrl(root(), altUrl);
+
+        // and we can go idle now
+        changeStatus(Delegate::FAIL_VERSION);
+        return;
+    }
+    
+    SG_LOG(SG_GENERAL, SG_INFO, "Migrating catalog " << id() << " to new URL:" << altUrl);
+    setUrl(altUrl);
+    Downloader* dl = new Downloader(this, altUrl);
+    root()->makeHTTPRequest(dl);
+}
+    
 } // of namespace pkg
 
 } // of namespace simgear

simgear/package/Catalog.hxx

@@ -179,6 +179,8 @@ private:
 
     void changeStatus(Delegate::StatusCode newStatus);
 
+    void processAlternate(SGPropertyNode_ptr alt);
+    
     Root* m_root;
     SGPropertyNode_ptr m_props;
     SGPath m_installRoot;

simgear/package/CatalogTest.cxx

@@ -148,6 +148,7 @@ int parseTest()
     SGPath rootPath = simgear::Dir::current().path();
     rootPath.append("testRoot");
     pkg::Root* root = new pkg::Root(rootPath, "8.1.12");
+    root->setLocale("de");
     pkg::CatalogRef cat = pkg::Catalog::createFromPath(root, SGPath(SRC_DIR "/catalogTest1"));
 
     SG_VERIFY(cat.valid());
@@ -172,7 +173,7 @@ int parseTest()
     pkg::PackageRef p2 = cat->getPackageById("c172p");
     SG_VERIFY(p2.valid());
     SG_CHECK_EQUAL(p2->qualifiedId(), "org.flightgear.test.catalog1.c172p");
-    SG_CHECK_EQUAL(p2->description(), "A plane made by Cessna on Jupiter");
+    SG_CHECK_EQUAL(p2->description(), "German description of C172");
 
     pkg::Package::PreviewVec thumbs = p2->previewsForVariant(0);
     SG_CHECK_EQUAL(thumbs.size(), 3);
@@ -217,7 +218,7 @@ int parseTest()
     SG_CHECK_EQUAL(skisVariant, skisVariantFull);
 
     SG_CHECK_EQUAL(p2->getLocalisedProp("description", skisVariant),
-                   "A plane with skis");
+                   "German description of C172 with skis");
     SG_CHECK_EQUAL(p2->getLocalisedProp("author", skisVariant),
                    "Standard author");
 
@@ -225,6 +226,7 @@ int parseTest()
     SG_VERIFY(floatsVariant > 0);
     SG_CHECK_EQUAL(p2->parentIdForVariant(floatsVariant), "c172p");
 
+    // no DE localisation description provided for the floats variant
     SG_CHECK_EQUAL(p2->getLocalisedProp("description", floatsVariant),
                    "A plane with floats");
     SG_CHECK_EQUAL(p2->getLocalisedProp("author", floatsVariant),
@@ -257,6 +259,12 @@ int parseTest()
     string_list primaries = {"c172p", "c172r"};
     SG_VERIFY(p2->primaryVariants() == primaries);
 
+///////////
+    pkg::PackageRef p3 = cat->getPackageById("b737-NG");
+    SG_VERIFY(p3.valid());
+    SG_CHECK_EQUAL(p3->description(), "German description of B737NG XYZ");
+    
+    
 // test filtering / searching too
     string_set tags(p2->tags());
     SG_CHECK_EQUAL(tags.size(), 4);
@@ -305,6 +313,16 @@ int parseTest()
         queryText->setStringValue("any-of/description", "float");
         SG_VERIFY(p2->matches(queryText.ptr()));
     }
+    
+    // match localized variant descriptions
+    {
+        SGPropertyNode_ptr queryText(new SGPropertyNode);
+        queryText->setStringValue("any-of/description", "XYZ");
+        SG_VERIFY(p3->matches(queryText.ptr()));
+    }
+    
+    
+
     delete root;
     return EXIT_SUCCESS;
 }
@@ -723,6 +741,130 @@ void testVersionMigrate(HTTP::Client* cl)
     }
 }
 
+void testVersionMigrateToId(HTTP::Client* cl)
+{
+    global_catalogVersion = 2; // version which has migration info
+    SGPath rootPath(simgear::Dir::current().path());
+    rootPath.append("cat_migrate_version_id");
+    simgear::Dir pd(rootPath);
+    pd.removeChildren();
+    
+    {
+        pkg::RootRef root(new pkg::Root(rootPath, "8.1.2"));
+        root->setHTTPClient(cl);
+        
+        pkg::CatalogRef c = pkg::Catalog::createFromUrl(root.ptr(), "http://localhost:2000/catalogTest1/catalog.xml");
+        waitForUpdateComplete(cl, root);
+        SG_VERIFY(c->isEnabled());
+        
+        // install a package
+        pkg::PackageRef p1 = root->getPackageById("org.flightgear.test.catalog1.b737-NG");
+        SG_CHECK_EQUAL(p1->id(), "b737-NG");
+        pkg::InstallRef ins = p1->install();
+        SG_VERIFY(ins->isQueued());
+        waitForUpdateComplete(cl, root);
+        SG_VERIFY(p1->isInstalled());
+    }
+    
+    // change version to an alternate one
+    {
+        pkg::RootRef root(new pkg::Root(rootPath, "7.5"));
+        root->setHTTPClient(cl);
+        
+        // this should cause the alternate package to be loaded
+        root->refresh(true);
+        waitForUpdateComplete(cl, root);
+        
+        pkg::CatalogRef cat = root->getCatalogById("org.flightgear.test.catalog1");
+        SG_VERIFY(!cat->isEnabled());
+        SG_CHECK_EQUAL(cat->status(), pkg::Delegate::FAIL_VERSION);
+        SG_CHECK_EQUAL(cat->id(), "org.flightgear.test.catalog1");
+        SG_CHECK_EQUAL(cat->url(), "http://localhost:2000/catalogTest1/catalog.xml");
+        
+        auto enabledCats = root->catalogs();
+        auto it = std::find(enabledCats.begin(), enabledCats.end(), cat);
+        SG_VERIFY(it == enabledCats.end());
+                
+        // ensure existing package is still installed
+        pkg::PackageRef p1 = root->getPackageById("org.flightgear.test.catalog1.b737-NG");
+        SG_VERIFY(p1 !=  pkg::PackageRef());
+        SG_CHECK_EQUAL(p1->id(), "b737-NG");
+        
+        // but not listed
+        auto packs = root->allPackages();
+        auto k = std::find(packs.begin(), packs.end(), p1);
+        SG_VERIFY(k == packs.end());
+        
+        // check the new catalog
+        auto altCat = root->getCatalogById("org.flightgear.test.catalog-alt");
+        SG_VERIFY(altCat->isEnabled());
+        SG_CHECK_EQUAL(altCat->status(), pkg::Delegate::STATUS_REFRESHED);
+        SG_CHECK_EQUAL(altCat->id(), "org.flightgear.test.catalog-alt");
+        SG_CHECK_EQUAL(altCat->url(), "http://localhost:2000/catalogTest1/catalog-alt.xml");
+        
+        it = std::find(enabledCats.begin(), enabledCats.end(), altCat);
+        SG_VERIFY(it != enabledCats.end());
+        
+        // install a parallel package from the new catalog
+        pkg::PackageRef p2 = root->getPackageById("org.flightgear.test.catalog-alt.b737-NG");
+        SG_CHECK_EQUAL(p2->id(), "b737-NG");
+        pkg::InstallRef ins = p2->install();
+        SG_VERIFY(ins->isQueued());
+        waitForUpdateComplete(cl, root);
+        SG_VERIFY(p2->isInstalled());
+        
+        // do a non-scoped lookup, we should get the new one
+        pkg::PackageRef p3 = root->getPackageById("b737-NG");
+        SG_CHECK_EQUAL(p2, p3);
+    }
+    
+    // test that re-init-ing doesn't mirgate again
+    {
+        pkg::RootRef root(new pkg::Root(rootPath, "7.5"));
+        root->setHTTPClient(cl);
+        
+        root->refresh(true);
+        waitForUpdateComplete(cl, root);
+        
+        pkg::CatalogRef cat = root->getCatalogById("org.flightgear.test.catalog1");
+        SG_VERIFY(!cat->isEnabled());
+        SG_CHECK_EQUAL(cat->status(), pkg::Delegate::FAIL_VERSION);
+        
+        auto altCat = root->getCatalogById("org.flightgear.test.catalog-alt");
+        SG_VERIFY(altCat->isEnabled());
+        
+        auto packs = root->allPackages();
+        SG_CHECK_EQUAL(packs.size(), 4);
+    }
+    
+    // and now switch back to the older version
+    {
+        pkg::RootRef root(new pkg::Root(rootPath, "8.1.0"));
+        root->setHTTPClient(cl);
+        root->refresh(true);
+        waitForUpdateComplete(cl, root);
+        
+        pkg::CatalogRef cat = root->getCatalogById("org.flightgear.test.catalog1");
+        SG_VERIFY(cat->isEnabled());
+        SG_CHECK_EQUAL(cat->status(), pkg::Delegate::STATUS_REFRESHED);
+        
+        auto altCat = root->getCatalogById("org.flightgear.test.catalog-alt");
+        SG_VERIFY(!altCat->isEnabled());
+
+        // verify the original aircraft is still installed and available
+        pkg::PackageRef p1 = root->getPackageById("org.flightgear.test.catalog1.b737-NG");
+        SG_VERIFY(p1 !=  pkg::PackageRef());
+        SG_CHECK_EQUAL(p1->id(), "b737-NG");
+        SG_VERIFY(p1->isInstalled());
+        
+        // verify the alt package is still installed,
+        pkg::PackageRef p2 = root->getPackageById("org.flightgear.test.catalog-alt.b737-NG");
+        SG_VERIFY(p2 !=  pkg::PackageRef());
+        SG_CHECK_EQUAL(p2->id(), "b737-NG");
+        SG_VERIFY(p2->isInstalled());
+    }
+}
+
 void testOfflineMode(HTTP::Client* cl)
 {
     global_catalogVersion = 0;
@@ -972,6 +1114,8 @@ int main(int argc, char* argv[])
     
     removeInvalidCatalog(&cl);
     
+    testVersionMigrateToId(&cl);
+    
     SG_LOG(SG_GENERAL, SG_INFO, "Successfully passed all tests!");
     return EXIT_SUCCESS;
 }

simgear/package/Install.cxx

@@ -36,10 +36,6 @@
 #include <simgear/misc/strutils.hxx>
 #include <simgear/io/iostreams/sgstream.hxx>
 
-extern "C" {
-    void fill_memory_filefunc (zlib_filefunc_def*);
-}
-
 namespace simgear {
 
 namespace pkg {
@@ -49,17 +45,16 @@ class Install::PackageArchiveDownloader : public HTTP::Request
 public:
     PackageArchiveDownloader(InstallRef aOwner) :
         HTTP::Request("" /* dummy URL */),
-        m_owner(aOwner),
-        m_downloaded(0)
+        m_owner(aOwner)
     {
         m_urls = m_owner->package()->downloadUrls();
         if (m_urls.empty()) {
             throw sg_exception("no package download URLs");
         }
 
-        if (m_owner->package()->properties()->hasChild("archive-type")) {
-            setArchiveTypeFromExtension(m_owner->package()->properties()->getStringValue("archive-type"));
-        }
+       // if (m_owner->package()->properties()->hasChild("archive-type")) {
+       //     setArchiveTypeFromExtension(m_owner->package()->properties()->getStringValue("archive-type"));
+        //}
         
         // TODO randomise order of m_urls
 
@@ -110,17 +105,18 @@ protected:
         Dir d(m_extractPath);
         d.create(0755);
 
+		m_extractor.reset(new ArchiveExtractor(m_extractPath));
         memset(&m_md5, 0, sizeof(SG_MD5_CTX));
         SG_MD5Init(&m_md5);
     }
 
     virtual void gotBodyData(const char* s, int n)
     {
-        m_buffer += std::string(s, n);
-        SG_MD5Update(&m_md5, (unsigned char*) s, n);
-
-        m_downloaded = m_buffer.size();
-        m_owner->installProgress(m_buffer.size(), responseLength());
+		const uint8_t* ubytes = (uint8_t*) s;
+        SG_MD5Update(&m_md5, ubytes, n);
+        m_downloaded += n;
+        m_owner->installProgress(m_downloaded, responseLength());
+		m_extractor->extractBytes(ubytes, n);
     }
 
     virtual void onDone()
@@ -151,7 +147,8 @@ protected:
             return;
         }
 
-        if (!extract()) {
+		m_extractor->flush();
+        if (m_extractor->hasError() || !m_extractor->isAtEndOfArchive()) {
             SG_LOG(SG_GENERAL, SG_WARN, "archive extraction failed");
             doFailure(Delegate::FAIL_EXTRACT);
             return;
@@ -207,151 +204,6 @@ protected:
     }
 
 private:
-    void setArchiveTypeFromExtension(const std::string& ext)
-    {
-        if (ext.empty())
-            return;
-        
-        if (ext == "zip") {
-            m_archiveType = ZIP;
-            return;
-        }
-        
-        if ((ext == "tar.gz") || (ext == "tgz")) {
-            m_archiveType = TAR_GZ;
-            return;
-        }
-    }
-    
-    void extractCurrentFile(unzFile zip, char* buffer, size_t bufferSize)
-    {
-        unz_file_info fileInfo;
-        unzGetCurrentFileInfo(zip, &fileInfo,
-            buffer, bufferSize,
-            NULL, 0,  /* extra field */
-            NULL, 0 /* comment field */);
-
-        std::string name(buffer);
-    // no absolute paths, no 'up' traversals
-    // we could also look for suspicious file extensions here (forbid .dll, .exe, .so)
-        if ((name[0] == '/') || (name.find("../") != std::string::npos) || (name.find("..\\") != std::string::npos)) {
-            throw sg_format_exception("Bad zip path", name);
-        }
-
-        if (fileInfo.uncompressed_size == 0) {
-            // assume it's a directory for now
-            // since we create parent directories when extracting
-            // a path, we're done here
-            return;
-        }
-
-        int result = unzOpenCurrentFile(zip);
-        if (result != UNZ_OK) {
-            throw sg_io_exception("opening current zip file failed", sg_location(name));
-        }
-
-        sg_ofstream outFile;
-        bool eof = false;
-        SGPath path(m_extractPath);
-        path.append(name);
-
-    // create enclosing directory heirarchy as required
-        Dir parentDir(path.dir());
-        if (!parentDir.exists()) {
-            bool ok = parentDir.create(0755);
-            if (!ok) {
-                throw sg_io_exception("failed to create directory heirarchy for extraction", path);
-            }
-        }
-
-        outFile.open(path, std::ios::binary | std::ios::trunc | std::ios::out);
-        if (outFile.fail()) {
-            throw sg_io_exception("failed to open output file for writing", path);
-        }
-
-        while (!eof) {
-            int bytes = unzReadCurrentFile(zip, buffer, bufferSize);
-            if (bytes < 0) {
-                throw sg_io_exception("unzip failure reading curent archive", sg_location(name));
-            } else if (bytes == 0) {
-                eof = true;
-            } else {
-                outFile.write(buffer, bytes);
-            }
-        }
-
-        outFile.close();
-        unzCloseCurrentFile(zip);
-    }
-
-    bool extract()
-    {
-        const std::string u(url());
-        const size_t ul(u.length());
-        
-        if (m_archiveType == AUTO_DETECT) {
-            if (u.rfind(".zip") == (ul - 4)) {
-                m_archiveType = ZIP;
-            } else if (u.rfind(".tar.gz") == (ul - 7)) {
-                m_archiveType = TAR_GZ;
-            }
-            // we will fall through to the error case now
-        }
-        
-        if (m_archiveType == ZIP) {
-            return extractUnzip();
-        } else if (m_archiveType == TAR_GZ) {
-            return extractTar();
-        }
-        
-        SG_LOG(SG_IO, SG_WARN, "unsupported archive format:" << u);
-        return false;
-    }
-
-    bool extractUnzip()
-    {
-        bool result = true;
-        zlib_filefunc_def memoryAccessFuncs;
-        fill_memory_filefunc(&memoryAccessFuncs);
-
-        char bufferName[128];
-        snprintf(bufferName, 128, "%p+%lx", m_buffer.data(), m_buffer.size());
-        unzFile zip = unzOpen2(bufferName, &memoryAccessFuncs);
-
-        const size_t BUFFER_SIZE = 32 * 1024;
-        void* buf = malloc(BUFFER_SIZE);
-
-        try {
-            int result = unzGoToFirstFile(zip);
-            if (result != UNZ_OK) {
-                throw sg_exception("failed to go to first file in archive");
-            }
-
-            while (true) {
-                extractCurrentFile(zip, (char*) buf, BUFFER_SIZE);
-                result = unzGoToNextFile(zip);
-                if (result == UNZ_END_OF_LIST_OF_FILE) {
-                    break;
-                } else if (result != UNZ_OK) {
-                    throw sg_io_exception("failed to go to next file in the archive");
-                }
-            }
-        } catch (sg_exception& ) {
-            result = false;
-        }
-
-        free(buf);
-        unzClose(zip);
-        return result;
-    }
-
-    bool extractTar()
-    {
-        TarExtractor tx(m_extractPath);
-        tx.extractBytes(m_buffer.data(), m_buffer.size());
-        return !tx.hasError() && tx.isAtEndOfArchive();
-    }
-
     void doFailure(Delegate::StatusCode aReason)
     {
         Dir dir(m_extractPath);
@@ -364,19 +216,13 @@ private:
         m_owner->installResult(aReason);
     }
 
-    enum ArchiveType {
-        AUTO_DETECT = 0,
-        ZIP,
-        TAR_GZ
-    };
     
     InstallRef m_owner;
-    ArchiveType m_archiveType = AUTO_DETECT;
     string_list m_urls;
     SG_MD5_CTX m_md5;
-    std::string m_buffer;
     SGPath m_extractPath;
-    size_t m_downloaded;
+    size_t m_downloaded = 0;
+	std::unique_ptr<ArchiveExtractor> m_extractor;
 };
 
 ////////////////////////////////////////////////////////////////////

simgear/package/Package.cxx

@@ -44,7 +44,7 @@ void Package::initWithProps(const SGPropertyNode* aProps)
 {
     m_props = const_cast<SGPropertyNode*>(aProps);
 // cache tag values
-    BOOST_FOREACH(const SGPropertyNode* c, aProps->getChildren("tag")) {
+    for (auto c : aProps->getChildren("tag")) {
       std::string t(c->getStringValue());
       m_tags.insert(boost::to_lower_copy(t));
     }
@@ -133,22 +133,8 @@ bool Package::matches(const SGPropertyNode* aFilter) const
 
     if ((filter_name == "text") || (filter_name == "description")) {
         handled = true;
-      std::string n(aFilter->getStringValue());
-      boost::to_lower(n);
-      size_t pos = boost::to_lower_copy(description()).find(n);
-      if (pos != std::string::npos) {
-        return true;
-      }
-
-        for (auto var : m_props->getChildren("variant")) {
-            if (var->hasChild("description")) {
-                std::string variantDesc(var->getStringValue("description"));
-                boost::to_lower(variantDesc);
-                size_t pos = variantDesc.find(n);
-                if (pos != std::string::npos) {
-                    return true;
-                }
-            }
+        if (matchesDescription(aFilter->getStringValue())) {
+            return true;
         }
     }
 
@@ -158,6 +144,50 @@ bool Package::matches(const SGPropertyNode* aFilter) const
 
     return false;
 }
+    
+bool Package::matchesDescription(const std::string &search) const
+{
+    std::string n(search);
+    boost::to_lower(n);
+    
+    bool localized;
+    auto d = getLocalisedString(m_props, "description", &localized);
+    boost::to_lower(d);
+    if (d.find(n) != std::string::npos) {
+        return true;
+    }
+    
+    // try non-localized description too, if the abovce was a localized one
+    if (localized) {
+        const std::string baseDesc = m_props->getStringValue("description");
+        auto pos = boost::to_lower_copy(baseDesc).find(n);
+        if (pos != std::string::npos) {
+            return true;
+        }
+    }
+    
+    // try each variant's description
+    for (auto var : m_props->getChildren("variant")) {
+        auto vd = getLocalisedString(var, "description", &localized);
+        if (!vd.empty()) {
+            boost::to_lower(vd);
+            if (vd.find(n) != std::string::npos) {
+                return true;
+            }
+        }
+        
+        if (localized) {
+            // try non-localized variant description
+            std::string vd = var->getStringValue("description");
+            boost::to_lower(vd);
+            if (vd.find(n) != std::string::npos) {
+                return true;
+            }
+        }
+    } // of variant iteration
+    
+    return false;
+}
 
 bool Package::isInstalled() const
 {
@@ -307,14 +337,29 @@ std::string Package::getLocalisedProp(const std::string& aName, const unsigned i
     return getLocalisedString(propsForVariant(vIndex, aName.c_str()), aName.c_str());
 }
 
-std::string Package::getLocalisedString(const SGPropertyNode* aRoot, const char* aName) const
+std::string Package::getLocalisedString(const SGPropertyNode* aRoot, const char* aName, bool* isLocalized) const
 {
-    std::string locale = m_catalog->root()->getLocale();
-    if (aRoot->hasChild(locale)) {
-        const SGPropertyNode* localeRoot = aRoot->getChild(locale.c_str());
-        if (localeRoot->hasChild(aName)) {
-            return localeRoot->getStringValue(aName);
-        }
+    // we used to place localised strings under /sim/<locale>/name - but this
+    // potentially pollutes the /sim namespace
+    // we now check first in /sim/localized/<locale>/name first
+    const auto& locale = m_catalog->root()->getLocale();
+    if (isLocalized) *isLocalized = false;
+    
+    if (locale.empty()) {
+        return aRoot->getStringValue(aName);
+    }
+    
+    const SGPropertyNode* localeRoot;
+    if (aRoot->hasChild("localized")) {
+        localeRoot = aRoot->getChild("localized")->getChild(locale);
+    } else {
+        // old behaviour where locale nodes are directly beneath /sim
+        localeRoot = aRoot->getChild(locale);
+    }
+
+    if (localeRoot && localeRoot->hasChild(aName)) {
+        if (isLocalized) *isLocalized = true;
+        return localeRoot->getStringValue(aName);
     }
 
     return aRoot->getStringValue(aName);

simgear/package/Package.hxx

@@ -225,12 +225,14 @@ private:
      */
     bool validate() const;
     
-    std::string getLocalisedString(const SGPropertyNode* aRoot, const char* aName) const;
+    std::string getLocalisedString(const SGPropertyNode* aRoot, const char* aName, bool* isLocalized = nullptr) const;
 
     PreviewVec previewsFromProps(const SGPropertyNode_ptr& ptr) const;
 
     SGPropertyNode_ptr propsForVariant(const unsigned int vIndex, const char* propName = nullptr) const;
 
+    bool matchesDescription(const std::string &search) const;
+    
     SGPropertyNode_ptr m_props;
     std::string m_id;
     string_set m_tags;

simgear/package/Root.cxx

@@ -414,7 +414,7 @@ Root::Root(const SGPath& aPath, const std::string& aVersion) :
         if (cat && cat->isEnabled()) {
             d->catalogs.insert({cat->id(), cat});
         } else if (cat) {
-            SG_LOG(SG_GENERAL, SG_WARN, "Package-Root init: catalog is disabled: " << cat->id());
+            SG_LOG(SG_GENERAL, SG_DEBUG, "Package-Root init: catalog is disabled: " << cat->id());
         }
     } // of child directories iteration
 }
@@ -450,6 +450,17 @@ CatalogRef Root::getCatalogById(const std::string& aId) const
     return it->second;
 }
 
+CatalogRef Root::getCatalogByUrl(const std::string& aUrl) const
+{
+    auto it = std::find_if(d->catalogs.begin(), d->catalogs.end(),
+                           [aUrl](const CatalogDict::value_type& v)
+                           { return (v.second->url() == aUrl); });
+    if (it == d->catalogs.end())
+        return {};
+    
+    return it->second;
+}
+    
 PackageRef Root::getPackageById(const std::string& aName) const
 {
     size_t lastDot = aName.rfind('.');
@@ -594,6 +605,12 @@ void Root::scheduleToUpdate(InstallRef aInstall)
     if (!aInstall) {
         throw sg_exception("missing argument to scheduleToUpdate");
     }
+    
+    auto it = std::find(d->updateDeque.begin(), d->updateDeque.end(), aInstall);
+    if (it != d->updateDeque.end()) {
+        // already scheduled to update
+        return;
+    }
 
     PackageList deps = aInstall->package()->dependencies();
     for (Package* dep : deps) {

simgear/package/Root.hxx

@@ -132,6 +132,8 @@ public:
     
     CatalogRef getCatalogById(const std::string& aId) const;
     
+    CatalogRef getCatalogByUrl(const std::string& aUrl) const;
+    
     void scheduleToUpdate(InstallRef aInstall);
     
     /**

simgear/package/catalogTest1/catalog-alt.xml

@@ -0,0 +1,141 @@
+<?xml version="1.0"?>
+
+<PropertyList>
+    <id>org.flightgear.test.catalog-alt</id>
+    <description>Alternate test catalog</description>
+    <url>http://localhost:2000/catalogTest1/catalog-alt.xml</url>
+    <catalog-version>4</catalog-version>
+
+    <version>7.*</version>
+
+
+    <package>
+        <id>alpha</id>
+        <name>Alpha package</name>
+        <revision type="int">9</revision>
+        <file-size-bytes type="int">593</file-size-bytes>
+
+        <md5>a469c4b837f0521db48616cfe65ac1ea</md5>
+        <url>http://localhost:2000/catalogTest1/alpha.zip</url>
+
+        <dir>alpha</dir>
+    </package>
+
+    <package>
+        <id>c172p</id>
+        <name>Cessna 172-P</name>
+        <dir>c172p</dir>
+        <description>A plane made by Cessna</description>
+        <revision type="int">42</revision>
+        <file-size-bytes type="int">860</file-size-bytes>
+
+        <tag>cessna</tag>
+        <tag>ga</tag>
+        <tag>piston</tag>
+        <tag>ifr</tag>
+
+        <rating>
+          <FDM type="int">3</FDM>
+          <systems type="int">4</systems>
+          <model type="int">5</model>
+          <cockpit type="int">4</cockpit>
+        </rating>
+
+
+        <preview>
+          <type>exterior</type>
+          <path>thumb-exterior.png</path>
+          <url>http://foo.bar.com/thumb-exterior.png</url>
+        </preview>
+
+        <preview>
+          <type>panel</type>
+          <path>thumb-panel.png</path>
+          <url>http://foo.bar.com/thumb-panel.png</url>
+        </preview>
+
+        <preview>
+          <path>thumb-something.png</path>
+          <url>http://foo.bar.com/thumb-something.png</url>
+        </preview>
+
+        <variant>
+            <id>c172p-2d-panel</id>
+            <name>C172 with 2d panel only</name>
+        </variant>
+
+        <variant>
+            <id>c172p-floats</id>
+            <name>C172 with floats</name>
+
+            <preview>
+              <type>exterior</type>
+              <path>thumb-exterior-floats.png</path>
+              <url>http://foo.bar.com/thumb-exterior-floats.png</url>
+            </preview>
+
+            <preview>
+              <type>panel</type>
+              <path>thumb-panel.png</path>
+              <url>http://foo.bar.com/thumb-panel.png</url>
+            </preview>
+        </variant>
+
+        <variant>
+            <id>c172p-skis</id>
+            <name>C172 with skis</name>
+
+            <preview>
+              <type>exterior</type>
+              <path>thumb-exterior-skis.png</path>
+              <url>http://foo.bar.com/thumb-exterior-skis.png</url>
+            </preview>
+
+            <preview>
+              <type>panel</type>
+              <path>thumb-panel.png</path>
+              <url>http://foo.bar.com/thumb-panel.png</url>
+            </preview>
+        </variant>
+
+        <md5>ec0e2ffdf98d6a5c05c77445e5447ff5</md5>
+        <url>http://localhost:2000/catalogTest1/c172p.zip</url>
+
+    </package>
+
+    <package>
+        <id>b737-NG</id>
+        <name>Boeing 737 NG</name>
+        <dir>b737NG</dir>
+        <description>A popular twin-engined narrow body jet</description>
+        <revision type="int">112</revision>
+        <file-size-bytes type="int">860</file-size-bytes>
+
+        <tag>boeing</tag>
+        <tag>jet</tag>
+        <tag>ifr</tag>
+
+        <rating>
+          <FDM type="int">5</FDM>
+          <systems type="int">5</systems>
+          <model type="int">4</model>
+          <cockpit type="int">4</cockpit>
+        </rating>
+
+        <md5>a94ca5704f305b90767f40617d194ed6</md5>
+        <url>http://localhost:2000/catalogTest1/b737.tar.gz</url>
+
+      </package>
+
+    <package>
+        <id>dc3</id>
+        <name>DC-3</name>
+        <revision type="int">9</revision>
+        <file-size-bytes type="int">593</file-size-bytes>
+
+        <md5>a469c4b837f0521db48616cfe65ac1ea</md5>
+        <url>http://localhost:2000/catalogTest1/alpha.zip</url>
+
+        <dir>dc3</dir>
+    </package>
+</PropertyList>

simgear/package/catalogTest1/catalog-v2.xml

@@ -15,6 +15,12 @@
       <url>http://localhost:2000/catalogTest1/catalog-v10.xml</url>
     </alternate-version>
     
+      <alternate-version>
+        <version>7.*</version>
+        <id>org.flightgear.test.catalog-alt</id>
+        <url>http://localhost:2000/catalogTest1/catalog-alt.xml</url>
+      </alternate-version>
+
     <package>
         <id>alpha</id>
         <name>Alpha package</name>

simgear/package/catalogTest1/catalog.xml

@@ -32,6 +32,15 @@
         <file-size-bytes type="int">860</file-size-bytes>
         <author>Standard author</author>
 
+        <localized>
+          <de>
+            <description>German description of C172</description>
+          </de>
+          <fr>
+            <description>French description of C172</description>
+          </fr>
+        </localized>
+
         <tag>cessna</tag>
         <tag>ga</tag>
         <tag>piston</tag>
@@ -99,6 +108,15 @@
             <name>C172 with skis</name>
             <description>A plane with skis</description>
 
+            <localized>
+            <de>
+              <description>German description of C172 with skis</description>
+            </de>
+            <fr>
+              <description>French description of C172 with skis</description>
+            </fr>
+          </localized>
+
             <variant-of>c172p</variant-of>
 
             <preview>
@@ -157,6 +175,14 @@
         <tag>jet</tag>
         <tag>ifr</tag>
 
+      <!-- not within a localized element -->
+          <de>
+            <description>German description of B737NG XYZ</description>
+          </de>
+          <fr>
+            <description>French description of B737NG</description>
+          </fr>
+
         <rating>
           <FDM type="int">5</FDM>
           <systems type="int">5</systems>

simgear/props/propertyObject.hxx

@@ -92,6 +92,12 @@ public:
     return p;
   }
   
+  static PropertyObject<T> create(SGPropertyNode_ptr aNode)
+  {
+    PropertyObject<T> p(aNode);
+    return p;
+  }
+
   static PropertyObject<T> create(SGPropertyNode* aNode, T aValue)
   {
     PropertyObject<T> p(aNode);
@@ -157,6 +163,8 @@ template <>
 class PropertyObject<std::string> : PropertyObjectBase
 {
 public:
+  PropertyObject() = default;
+
   explicit PropertyObject(const char* aChild) :
     PropertyObjectBase(aChild)
   { }
@@ -183,6 +191,12 @@ public:
     return p;
   }
   
+  static PropertyObject<std::string> create(SGPropertyNode_ptr aNode)
+  {
+    PropertyObject<std::string> p(aNode);
+    return p;
+  }
+
   static PropertyObject<std::string> create(SGPropertyNode* aNode, const std::string& aValue)
   {
     PropertyObject<std::string> p(aNode);

simgear/props/propertyObject_test.cxx

@@ -205,6 +205,34 @@ void testCreate()
   
 }
 
+void testDeclare()
+{
+    PropertyObject<bool>        a;
+    PropertyObject<int>         b;
+    PropertyObject<double>      c;
+    PropertyObject<std::string> d;
+}
+
+void testDeclareThenDefine()
+{
+    // Declare.
+    PropertyObject<bool>        a;
+    PropertyObject<std::string> b;
+
+    // Property nodes.
+    SGPropertyNode_ptr propsA = new SGPropertyNode;
+    SGPropertyNode_ptr propsB = new SGPropertyNode;
+    propsA->setValue(true);
+    propsB->setValue("test");
+
+    // Now define.
+    a = PropertyObject<bool>::create(propsA);
+    b = PropertyObject<std::string>::create(propsB);
+
+    assert(a == true);
+    assert(b == "test");
+}
+
 int main(int argc, char* argv[])
 {
 	testRoot = new SGPropertyNode();
@@ -227,6 +255,8 @@ int main(int argc, char* argv[])
   testAssignment();
   testSTLContainer();
   testCreate();
+  testDeclare();
+  testDeclareThenDefine();
 
   return EXIT_SUCCESS;
 }

simgear/props/props.cxx

@@ -118,7 +118,7 @@ parse_name (const SGPropertyNode *node, const Range &path)
       } else {
         std::string err = "'";
         err.push_back(*i);
-        err.append("' found in propertyname after '"+node->getNameString()+"'");
+        err.append("' found in propertyname after '"+node->getPath()+"'");
         err.append("\nname may contain only ._- and alphanumeric characters");
 	throw err;
       }
@@ -130,7 +130,7 @@ parse_name (const SGPropertyNode *node, const Range &path)
     if (path.begin() == i) {
       std::string err = "'";
       err.push_back(*i);
-      err.append("' found in propertyname after '"+node->getNameString()+"'");
+      err.append("' found in propertyname after '"+node->getPath()+"'");
       err.append("\nname must begin with alpha or '_'");
       throw err;
     }
@@ -536,7 +536,7 @@ find_node (SGPropertyNode * current,
   using namespace boost;
   typedef split_iterator<typename range_result_iterator<Range>::type>
     PathSplitIterator;
-  
+
   PathSplitIterator itr
     = make_split_iterator(path, first_finder("/", is_equal()));
   if (*path.begin() == '/')
@@ -874,7 +874,7 @@ SGPropertyNode::SGPropertyNode (const SGPropertyNode &node)
   }
   switch (_type) {
   case props::BOOL:
-    set_bool(node.get_bool());    
+    set_bool(node.get_bool());
     break;
   case props::INT:
     set_int(node.get_int());
@@ -963,6 +963,11 @@ SGPropertyNode::alias (SGPropertyNode * target)
 {
   if (target && (_type != props::ALIAS) && (!_tied))
   {
+   /* loop protection: check alias chain; must not contain self */
+    for (auto p = target; p; p = ((p->_type == props::ALIAS) ? p->_value.alias : nullptr)) {
+      if (p == this) return false;
+    }
+
     clearValue();
     get(target);
     _value.alias = target;
@@ -973,7 +978,7 @@ SGPropertyNode::alias (SGPropertyNode * target)
   if (!target)
   {
     SG_LOG(SG_GENERAL, SG_ALERT,
-           "Failed to create alias for " << getPath() << ". "
+           "Failed to set alias " << getPath() << ". "
            "The target property does not exist.");
   }
   else
@@ -981,15 +986,15 @@ SGPropertyNode::alias (SGPropertyNode * target)
   {
     if (_value.alias == target)
         return true; // ok, identical alias requested
-    SG_LOG(SG_GENERAL, SG_ALERT,
-           "Failed to create alias at " << target->getPath() << ". "
-           "Source "<< getPath() << " is already aliasing another property.");
+    SG_LOG(SG_GENERAL, SG_ALERT, "alias(): "<< getPath() <<
+        " is already pointing to " << _value.alias->getPath() <<
+        " so it cannot alias '" << target->getPath() << ". Use unalias() first.");
   }
   else
   if (_tied)
   {
-    SG_LOG(SG_GENERAL, SG_ALERT, "Failed to create alias at " << target->getPath() << ". "
-           "Source " << getPath() << " is a tied property.");
+    SG_LOG(SG_GENERAL, SG_ALERT, "alias(): " << getPath() <<
+        " is a tied property. It cannot alias " << target->getPath() << ".");
   }
 
   return false;
@@ -1311,7 +1316,7 @@ SGPropertyNode::getType () const
 }
 
 
-bool 
+bool
 SGPropertyNode::getBoolValue () const
 {
 				// Shortcut for common case
@@ -1344,7 +1349,7 @@ SGPropertyNode::getBoolValue () const
   }
 }
 
-int 
+int
 SGPropertyNode::getIntValue () const
 {
 				// Shortcut for common case
@@ -1377,7 +1382,7 @@ SGPropertyNode::getIntValue () const
   }
 }
 
-long 
+long
 SGPropertyNode::getLongValue () const
 {
 				// Shortcut for common case
@@ -1410,7 +1415,7 @@ SGPropertyNode::getLongValue () const
   }
 }
 
-float 
+float
 SGPropertyNode::getFloatValue () const
 {
 				// Shortcut for common case
@@ -1443,7 +1448,7 @@ SGPropertyNode::getFloatValue () const
   }
 }
 
-double 
+double
 SGPropertyNode::getDoubleValue () const
 {
 				// Shortcut for common case
@@ -2569,7 +2574,7 @@ template<>
 std::ostream& SGRawBase<SGVec4d>::printOn(std::ostream& stream) const
 {
     const SGVec4d vec
-        = static_cast<const SGRawValue<SGVec4d>*>(this)->getValue();    
+        = static_cast<const SGRawValue<SGVec4d>*>(this)->getValue();
     for (int i = 0; i < 4; ++i) {
         stream << vec[i];
         if (i < 3)

simgear/props/props.hxx

@@ -55,13 +55,18 @@ namespace boost {
   struct disable_if : public disable_if_c<Cond::value, T> {};
 }
 #else
-# include <boost/utility.hpp>
 # include <boost/type_traits/is_enum.hpp>
+#if BOOST_VERSION >= 105600
+#include <boost/core/enable_if.hpp>
+#else
+#include <boost/utility/enable_if.hpp>
+#endif
 
 # include <simgear/debug/logstream.hxx>
 # include <simgear/math/SGMathFwd.hxx>
 # include <simgear/math/sg_types.hxx>
 #endif
+
 #include <simgear/structure/SGReferenced.hxx>
 #include <simgear/structure/SGSharedPtr.hxx>
 
@@ -751,7 +756,7 @@ typedef SGSharedPtr<const SGPropertyNode> SGConstPropertyNode_ptr;
 
 namespace simgear
 {
-typedef std::vector<SGPropertyNode_ptr> PropertyList;
+    using PropertyList = std::vector<SGPropertyNode_ptr>;
 }
 
 /**

simgear/props/propsfwd.hxx

@@ -1,3 +1,20 @@
+// Copyright (C) 2018  James Turner - <james@flightgear>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Library General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Library General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//
+
 /** \file
  *
  * Forward declarations for properties (and related structures)
@@ -7,12 +24,18 @@
 #define SG_PROPS_FWD_HXX
 
 #include <simgear/structure/SGSharedPtr.hxx>
+#include <vector>
 
 class SGPropertyNode;
 
 typedef SGSharedPtr<SGPropertyNode> SGPropertyNode_ptr;
 typedef SGSharedPtr<const SGPropertyNode> SGConstPropertyNode_ptr;
 
+namespace simgear
+{
+    using PropertyList = std::vector<SGPropertyNode_ptr>;
+}
+
 class SGCondition; 
 
 #endif // of SG_PROPS_FWD_HXX

simgear/scene/CMakeLists.txt

@@ -9,6 +9,7 @@ foreach( mylibfolder
         tgdb
         util
         tsync
+        viewer
     )
 
     add_subdirectory(${mylibfolder})

simgear/scene/dem/SGMesh.cxx

@@ -125,7 +125,7 @@ SGMesh::SGMesh( const SGDemPtr dem,
 
     ::std::vector<SGGeod> geodes(grid_width*grid_height);
 
-    // session can't be paralell yet - save alts in geode array
+    // session can't be parallel yet - save alts in geode array
     fprintf( stderr, "SGMesh::SGMesh - create session - num dem roots is %d\n", dem->getNumRoots() );
     SGDemSession s = dem->openSession( wo, so, eo, no, lvl, true );
     s.getGeods( wo, so, eo, no, grid_width, grid_height, skipx, skipy, geodes, Debug1, Debug2 );
@@ -159,10 +159,10 @@ SGMesh::SGMesh( const SGDemPtr dem,
         index.push_back( src_idx );
     }
 
-    // we can convert to cartesian in paralell
-    unsigned int nv = geodes.size();
+    // we can convert to cartesian in parallel
+    long nv = geodes.size();
 #pragma omp parallel for
-    for (unsigned int i = 0; i < nv; i++) {
+    for (long i = 0; i < nv; i++) {
         (*vertices)[i].set( toOsg( SGVec3f::fromGeod( geodes[i] ) ) );
     }
 
@@ -195,7 +195,7 @@ SGMesh::SGMesh( const SGDemPtr dem,
 
     // translate pos after normals computed
 #pragma omp parallel for
-    for ( unsigned int i=0; i < nv; i++ ) {
+    for ( long i=0; i < nv; i++ ) {
         (*vertices)[i].set( transform.preMult( (*vertices)[i]) );
     }
 
@@ -245,7 +245,7 @@ SGMesh::SGMesh( const SGDemPtr dem,
     osg::Geometry* geometry = new osg::Geometry;
 
     char geoName[64];
-    snprintf( geoName, sizeof(geoName), "tilemesh (%u,%u)-(%u,%u):level%d,%d", 
+    snprintf( geoName, sizeof(geoName), "tilemesh (%u,%u)-(%u,%u):level%u,%u", 
                 wo, so, eo, no,
                 widthLevel, heightLevel );
     geometry->setName(geoName);
@@ -364,9 +364,9 @@ void SGMesh::need_normals()
         // need_faces();
         if ( !faces.empty() ) {
             // Compute from faces
-            int nf = faces.size();
+            long nf = faces.size();
 #pragma omp parallel for
-            for (int i = 0; i < nf; i++) {
+            for (long i = 0; i < nf; i++) {
                 const osg::Vec3 &p0 = (*vertices)[faces[i][0]];
                 const osg::Vec3 &p1 = (*vertices)[faces[i][1]];
                 const osg::Vec3 &p2 = (*vertices)[faces[i][2]];
@@ -384,9 +384,9 @@ void SGMesh::need_normals()
         }
 
         // Make them all unit-length
-        unsigned int nn = normals->size();
+        long nn = normals->size();
 #pragma omp parallel for
-        for (unsigned int i = 0; i < nn; i++)
+        for (long i = 0; i < nn; i++)
             (*normals)[i].normalize();
     }
 }

simgear/scene/material/Effect.cxx

@@ -830,6 +830,9 @@ void reload_shaders()
         if (!fileName.empty()) {
             shader->loadShaderSourceFromFile(fileName);
         }
+        else
+            SG_LOG(SG_INPUT, SG_ALERT, "Could not locate shader: " << fileName);
+
     }
 }
 
@@ -917,8 +920,13 @@ void ShaderProgramBuilder::buildAttribute(Effect* effect, Pass* pass,
         Shader::Type stype = (Shader::Type)shaderKey.second;
         string fileName = SGModelLib::findDataFile(shaderName, options);
         if (fileName.empty())
+        {
+            SG_LOG(SG_INPUT, SG_ALERT, "Could not locate shader" << shaderName);
+
+
             throw BuilderException(string("couldn't find shader ") +
-                                   shaderName);
+                shaderName);
+        }
         resolvedKey.shaders.push_back(ShaderKey(fileName, stype));
     }
     ProgramMap::iterator resitr = resolvedProgramMap.find(resolvedKey);
@@ -1405,8 +1413,11 @@ bool makeParametersFromStateSet(SGPropertyNode* effectRoot, const StateSet* ss)
 
 // Walk the techniques property tree, building techniques and
 // passes.
+static SGMutex  realizeTechniques_lock;
 bool Effect::realizeTechniques(const SGReaderWriterOptions* options)
 {
+    SGGuard<SGMutex> g(realizeTechniques_lock);
+
     if (_isRealized)
         return true;
     PropertyList tniqList = root->getChildren("technique");

simgear/scene/material/EffectCullVisitor.cxx

@@ -34,8 +34,9 @@ namespace simgear
 
 using osgUtil::CullVisitor;
 
-EffectCullVisitor::EffectCullVisitor(bool collectLights) :
-    _collectLights(collectLights)
+EffectCullVisitor::EffectCullVisitor(bool collectLights, Effect *effectOverride) :
+    _collectLights(collectLights),
+    _effectOverride(effectOverride)
 {
 }
 
@@ -61,12 +62,18 @@ void EffectCullVisitor::apply(osg::Geode& node)
     if (_collectLights && ( eg->getNodeMask() & MODELLIGHT_BIT ) ) {
         _lightList.push_back( eg );
     }
-    Effect* effect = eg->getEffect();
+    Effect *effect;
+    if (_effectOverride) {
+        effect = _effectOverride;
+    } else {
+        effect = eg->getEffect();
+        if (!effect) {
+            CullVisitor::apply(node);
+            return;
+        }
+    }
     Technique* technique = 0;
-    if (!effect) {
-        CullVisitor::apply(node);
-        return;
-    } else if (!(technique = effect->chooseTechnique(&getRenderInfo()))) {
+    if (!(technique = effect->chooseTechnique(&getRenderInfo()))) {
         return;
     }
     // push the node's state.

simgear/scene/material/EffectCullVisitor.hxx

@@ -29,11 +29,12 @@ class Texture2D;
 
 namespace simgear
 {
+class Effect;
 class EffectGeode;
 class EffectCullVisitor : public osgUtil::CullVisitor
 {
 public:
-    EffectCullVisitor(bool collectLights = false);
+    EffectCullVisitor(bool collectLights = false, Effect *effectOverride = 0);
     EffectCullVisitor(const EffectCullVisitor&);
     virtual osgUtil::CullVisitor* clone() const;
     using osgUtil::CullVisitor::apply;
@@ -48,6 +49,7 @@ private:
     std::map<std::string,osg::ref_ptr<osg::Texture2D> > _bufferList;
     std::vector<osg::ref_ptr<EffectGeode> > _lightList;
     bool _collectLights;
+    osg::ref_ptr<Effect> _effectOverride;
 };
 }
 #endif

simgear/scene/material/TextureBuilder.cxx

@@ -23,6 +23,7 @@
 
 #include "Pass.hxx"
 
+#include <osg/Version>
 #include <osg/PointSprite>
 #include <osg/TexEnv>
 #include <osg/TexEnvCombine>
@@ -76,7 +77,7 @@ osg::Texture* TextureBuilder::buildFromType(Effect* effect, Pass* pass, const st
 
 typedef boost::tuple<string, Texture::FilterMode, Texture::FilterMode,
                      Texture::WrapMode, Texture::WrapMode, Texture::WrapMode,
-                     string, MipMapTuple> TexTuple;
+                     string, MipMapTuple, ImageInternalFormat> TexTuple;
 
 EffectNameValue<TexEnv::Mode> texEnvModesInit[] =
 {
@@ -239,6 +240,17 @@ TexTuple makeTexTuple(Effect* effect, const SGPropertyNode* props,
         }
     }
 
+    const SGPropertyNode* pInternalFormat = getEffectPropertyChild(effect, props, "internal-format");
+    pInternalFormat = props->getChild("internal-format");
+    ImageInternalFormat iformat = ImageInternalFormat::Unspecified;
+    if (pInternalFormat) {
+        std::string internalFormat = pInternalFormat->getStringValue();
+        if (internalFormat == "normalized") {
+            iformat = ImageInternalFormat::Normalized;
+            SG_LOG(SG_INPUT, SG_ALERT, "internal-format normalized '" << imageName << "'");
+        }
+    }
+
     const SGPropertyNode* pMipmapControl
         = getEffectPropertyChild(effect, props, "mipmap-control");
     MipMapTuple mipmapFunctions( AUTOMATIC, AUTOMATIC, AUTOMATIC, AUTOMATIC );
@@ -246,7 +258,7 @@ TexTuple makeTexTuple(Effect* effect, const SGPropertyNode* props,
         mipmapFunctions = makeMipMapTuple(effect, pMipmapControl, options);
 
     return TexTuple(absFileName, minFilter, magFilter, sWrap, tWrap, rWrap,
-                    texType, mipmapFunctions);
+                    texType, mipmapFunctions, iformat);
 }
 
 bool setAttrs(const TexTuple& attrs, Texture* tex,
@@ -257,7 +269,19 @@ bool setAttrs(const TexTuple& attrs, Texture* tex,
         return false;
 
     osgDB::ReaderWriter::ReadResult result;
+
+    // load texture for effect
+    SGReaderWriterOptions::LoadOriginHint origLOH = options->getLoadOriginHint();
+    if(attrs.get<8>() == ImageInternalFormat::Normalized)
+        options->setLoadOriginHint(SGReaderWriterOptions::LoadOriginHint::ORIGIN_EFFECTS_NORMALIZED);
+    else
+        options->setLoadOriginHint(SGReaderWriterOptions::LoadOriginHint::ORIGIN_EFFECTS);
+#if OSG_VERSION_LESS_THAN(3,4,2)
     result = osgDB::readImageFile(imageName, options);
+#else
+    result = osgDB::readRefImageFile(imageName, options);
+#endif
+    options->setLoadOriginHint(origLOH);
     osg::ref_ptr<osg::Image> image;
     if (result.success())
         image = result.getImage();
@@ -272,11 +296,9 @@ bool setAttrs(const TexTuple& attrs, Texture* tex,
         } else if (t < s && 32 <= t) {
             SGSceneFeatures::instance()->setTextureCompression(tex);
         }
-        tex->setMaxAnisotropy(SGSceneFeatures::instance()
-                              ->getTextureFilter());
+        tex->setMaxAnisotropy(SGSceneFeatures::instance()->getTextureFilter());
     } else {
-        SG_LOG(SG_INPUT, SG_ALERT, "failed to load effect texture file "
-               << imageName);
+        SG_LOG(SG_INPUT, SG_ALERT, "failed to load effect texture file " << imageName);
         return false;
     }
 
@@ -585,36 +607,64 @@ Texture* CubeMapBuilder::build(Effect* effect, Pass* pass, const SGPropertyNode*
         cubeTexture->setWrap(osg::Texture3D::WRAP_R, osg::Texture::CLAMP_TO_EDGE);
 
         osgDB::ReaderWriter::ReadResult result;
+        SGReaderWriterOptions* wOpts = (SGReaderWriterOptions*)options;
+        SGReaderWriterOptions::LoadOriginHint origLOH = wOpts->getLoadOriginHint();
+        wOpts->setLoadOriginHint(SGReaderWriterOptions::LoadOriginHint::ORIGIN_EFFECTS);
+#if OSG_VERSION_LESS_THAN(3,4,1)
         result = osgDB::readImageFile(_tuple.get<0>(), options);
+#else
+        result = osgDB::readRefImageFile(_tuple.get<0>(), options);
+#endif
         if(result.success()) {
             osg::Image* image = result.getImage();
             cubeTexture->setImage(TextureCubeMap::POSITIVE_X, image);
         }
+#if OSG_VERSION_LESS_THAN(3,4,1)
         result = osgDB::readImageFile(_tuple.get<1>(), options);
+#else
+        result = osgDB::readRefImageFile(_tuple.get<1>(), options);
+#endif
         if(result.success()) {
             osg::Image* image = result.getImage();
             cubeTexture->setImage(TextureCubeMap::NEGATIVE_X, image);
         }
+#if OSG_VERSION_LESS_THAN(3,4,1)
         result = osgDB::readImageFile(_tuple.get<2>(), options);
+#else
+        result = osgDB::readRefImageFile(_tuple.get<2>(), options);
+#endif
         if(result.success()) {
             osg::Image* image = result.getImage();
             cubeTexture->setImage(TextureCubeMap::POSITIVE_Y, image);
         }
+#if OSG_VERSION_LESS_THAN(3,4,1)
         result = osgDB::readImageFile(_tuple.get<3>(), options);
+#else
+        result = osgDB::readRefImageFile(_tuple.get<3>(), options);
+#endif
         if(result.success()) {
             osg::Image* image = result.getImage();
             cubeTexture->setImage(TextureCubeMap::NEGATIVE_Y, image);
         }
+#if OSG_VERSION_LESS_THAN(3,4,1)
         result = osgDB::readImageFile(_tuple.get<4>(), options);
+#else
+        result = osgDB::readRefImageFile(_tuple.get<4>(), options);
+#endif
         if(result.success()) {
             osg::Image* image = result.getImage();
             cubeTexture->setImage(TextureCubeMap::POSITIVE_Z, image);
         }
+#if OSG_VERSION_LESS_THAN(3,4,1)
         result = osgDB::readImageFile(_tuple.get<5>(), options);
+#else
+        result = osgDB::readRefImageFile(_tuple.get<5>(), options);
+#endif
         if(result.success()) {
             osg::Image* image = result.getImage();
             cubeTexture->setImage(TextureCubeMap::NEGATIVE_Z, image);
         }
+        wOpts->setLoadOriginHint(origLOH);
 
         if (itr == _cubemaps.end())
             _cubemaps[_tuple] = cubeTexture;
@@ -634,7 +684,11 @@ Texture* CubeMapBuilder::build(Effect* effect, Pass* pass, const SGPropertyNode*
             return cubeTexture.release();
 
         osgDB::ReaderWriter::ReadResult result;
+#if OSG_VERSION_LESS_THAN(3,4,1)
         result = osgDB::readImageFile(texname, options);
+#else
+        result = osgDB::readRefImageFile(texname, options);
+#endif
         if(result.success()) {
             osg::Image* image = result.getImage();
             image->flipVertical();   // Seems like the image coordinates are somewhat funny, flip to get better ones

simgear/scene/material/mat.cxx

@@ -227,7 +227,11 @@ SGMaterial::read_properties(const SGReaderWriterOptions* options,
             }
             else
             {
+#if OSG_VERSION_LESS_THAN(3,4,0)
                 osg::Image* image = osgDB::readImageFile(fullMaskPath, options);
+#else
+                osg::Image* image = osgDB::readRefImageFile(fullMaskPath, options);
+#endif
                 if (image && image->valid())
                 {
                     Texture2DRef object_mask = new osg::Texture2D;

simgear/scene/material/mipmap.cxx

@@ -392,6 +392,8 @@ osg::Image* computeMipmap( osg::Image* image, MipMapTuple attrs )
             image->getInternalTextureFormat(), image->getPixelFormat(),
             image->getDataType(), data, osg::Image::USE_NEW_DELETE, image->getPacking() );
         mipmaps->setMipmapLevels( mipmapOffsets );
+        mipmaps->setName(image->getName());
+        mipmaps->setFileName(image->getFileName());
 
         return mipmaps.release();
     }

simgear/scene/material/mipmap.hxx

@@ -31,14 +31,18 @@ class Effect;
 class SGReaderWriterOptions;
 
 namespace effect {
-enum MipMapFunction {
-    AUTOMATIC,
-    AVERAGE,
-    SUM,
-    PRODUCT,
-    MIN,
-    MAX
-};
+    enum MipMapFunction {
+        AUTOMATIC,
+        AVERAGE,
+        SUM,
+        PRODUCT,
+        MIN,
+        MAX
+    };
+    enum ImageInternalFormat {
+        Unspecified,
+        Normalized,
+    };
 
 typedef boost::tuple<MipMapFunction, MipMapFunction, MipMapFunction, MipMapFunction> MipMapTuple;
 

simgear/scene/model/BVHDebugCollectVisitor.hxx

@@ -111,6 +111,11 @@ public:
         if (transform->getNumChildren())
             _group->addChild(transform.get());
     }
+    virtual void apply(BVHPageNode& leaf)
+    {
+        leaf.traverse(*this);
+    }
+
     virtual void apply(BVHLineGeometry&)
     {
     }

simgear/scene/model/BVHPageNodeOSG.cxx

@@ -24,6 +24,7 @@
 #include "../../bvh/BVHPageRequest.hxx"
 #include "../../bvh/BVHPager.hxx"
 
+#include <osg/Version>
 #include <osg/io_utils>
 #include <osg/Camera>
 #include <osg/Drawable>
@@ -215,7 +216,11 @@ public:
                 if (pagedLOD.getMinRange(i) <= 0) {
                     osg::ref_ptr<const osgDB::Options> options;
                     options = getOptions(pagedLOD.getDatabaseOptions(), pagedLOD.getDatabasePath());
+#if OSG_VERSION_LESS_THAN(3,4,0)
+                    node = osgDB::readNodeFile(pagedLOD.getFileName(i), options.get());
+#else
                     node = osgDB::readRefNodeFile(pagedLOD.getFileName(i), options.get());
+#endif
                 }
                 if (!node.valid())
                     node = new osg::Group;
@@ -256,7 +261,11 @@ public:
             osg::ref_ptr<const osgDB::Options> options;
             options = getOptions(proxyNode.getDatabaseOptions(), proxyNode.getDatabasePath());
             osg::ref_ptr<osg::Node> node;
+#if OSG_VERSION_LESS_THAN(3,4,0)
+            node = osgDB::readNodeFile(proxyNode.getFileName(i), options.get());
+#else
             node = osgDB::readRefNodeFile(proxyNode.getFileName(i), options.get());
+#endif
             if (!node.valid())
                 node = new osg::Group;
             if (i < proxyNode.getNumChildren())
@@ -353,7 +362,11 @@ SGSharedPtr<BVHNode>
 BVHPageNodeOSG::load(const std::string& name, const osg::ref_ptr<const osg::Referenced>& options)
 {
     osg::ref_ptr<osg::Node> node;
+#if OSG_VERSION_LESS_THAN(3,4,0)
+    node = osgDB::readNodeFile(name, dynamic_cast<const osgDB::Options*>(options.get()));
+#else
     node = osgDB::readRefNodeFile(name, dynamic_cast<const osgDB::Options*>(options.get()));
+#endif
     if (!node.valid())
         return SGSharedPtr<BVHNode>();
 

simgear/scene/model/ModelRegistry.cxx

@@ -22,6 +22,8 @@
 #endif
 
 #include "ModelRegistry.hxx"
+#include <simgear/scene/util/SGImageUtils.hxx>
+#include "../material/mipmap.hxx"
 
 #include <algorithm>
 #include <utility>
@@ -44,6 +46,7 @@
 #include <osgDB/Registry>
 #include <osgDB/SharedStateManager>
 #include <osgUtil/Optimizer>
+#include <osg/Texture>
 
 #include <simgear/sg_inlines.h>
 
@@ -57,10 +60,16 @@
 #include <simgear/props/props.hxx>
 #include <simgear/props/props_io.hxx>
 #include <simgear/props/condition.hxx>
+#include <simgear/io/sg_file.hxx>
+#include <simgear/threads/SGGuard.hxx>
 
 #include "BoundingVolumeBuildVisitor.hxx"
 #include "model.hxx"
 
+#include <boost/lexical_cast.hpp>
+#include <boost/tuple/tuple.hpp>
+#include <boost/tuple/tuple_comparison.hpp>
+
 using namespace std;
 using namespace osg;
 using namespace osgUtil;
@@ -173,126 +182,601 @@ public:
 
 } // namespace
 
-Node* DefaultProcessPolicy::process(Node* node, const string& filename,
-                                    const Options* opt)
+static int nearestPowerOfTwo(unsigned int _v)
+{
+    //    uint v; // compute the next highest power of 2 of 32-bit v
+    unsigned int v = (unsigned int)_v;
+    bool neg = _v < 0;
+    if (neg)
+        v = (unsigned int)(-_v);
+
+    v &= (2 << 16) - 1; // make +ve
+
+    // bit twiddle to round up to nearest pot.
+    v--;
+    v |= v >> 1;
+    v |= v >> 2;
+    v |= v >> 4;
+    v |= v >> 8;
+    v |= v >> 16;
+    v++;
+
+    if (neg)
+        _v = -(int)v;
+    else
+        _v = (int)v;
+    return v;
+}
+    static bool isPowerOfTwo(int v)
+{
+    return ((v & (v - 1)) == 0);
+}
+osg::Node* DefaultProcessPolicy::process(osg::Node* node, const std::string& filename,
+    const Options* opt)
 {
     TextureNameVisitor nameVisitor;
     node->accept(nameVisitor);
     return node;
 }
 
+//#define LOCAL_IMAGE_CACHE
+#ifdef LOCAL_IMAGE_CACHE
+typedef std::map<std::string, osg::ref_ptr<Image>> ImageMap;
+ImageMap _imageMap;
+//typedef std::map<std::string, osg::ref_ptr<osg::Image> >    ImageMap;
+//ImageMap _imageMap;
+osg::Image* getImageByName(const std::string& filename)
+{
+    ImageMap::iterator itr = _imageMap.find(filename);
+    if (itr != _imageMap.end()) return itr->second.get();
+    return nullptr;
+}
+#endif
+// a cache which evicts the least recently used item when it is full
+#include <map>
+#include <list>
+#include <utility>
+
+#include <boost/optional.hpp>
+template<class Key, class Value>
+class lru_cache
+{
+public:
+    SGMutex _mutex;
+
+    typedef Key key_type;
+    typedef Value value_type;
+    typedef std::list<key_type> list_type;
+    typedef std::map<
+        key_type,
+        std::pair<value_type, typename list_type::iterator>
+    > map_type;
+
+    lru_cache(size_t capacity)
+        : m_capacity(capacity)
+    {
+    }
+
+    ~lru_cache()
+    {
+    }
+
+
+    size_t size() const
+    {
+        return m_map.size();
+    }
+
+    size_t capacity() const
+    {
+        return m_capacity;
+    }
+
+    bool empty() const
+    {
+        return m_map.empty();
+    }
+
+    bool contains(const key_type &key)
+    {
+        SGGuard<SGMutex> scopeLock(_mutex);
+        return m_map.find(key) != m_map.end();
+    }
+
+    void insert(const key_type &key, const value_type &value)
+    {
+        SGGuard<SGMutex> scopeLock(_mutex);
+        typename map_type::iterator i = m_map.find(key);
+        if (i == m_map.end()) {
+            // insert item into the cache, but first check if it is full
+            if (size() >= m_capacity) {
+                // cache is full, evict the least recently used item
+                evict();
+            }
+
+            // insert the new item
+            m_list.push_front(key);
+            m_map[key] = std::make_pair(value, m_list.begin());
+        }
+    }
+    boost::optional<key_type> findValue(const std::string &requiredValue)
+    {
+        SGGuard<SGMutex> scopeLock(_mutex);
+        for (typename map_type::iterator it = m_map.begin(); it != m_map.end(); ++it)
+            if (it->second.first == requiredValue)
+                return it->first;
+        return boost::none;
+    }
+    boost::optional<value_type> get(const key_type &key)
+    {
+        SGGuard<SGMutex> scopeLock(_mutex);
+        // lookup value in the cache
+        typename map_type::iterator i = m_map.find(key);
+        if (i == m_map.end()) {
+            // value not in cache
+            return boost::none;
+        }
+
+        // return the value, but first update its place in the most
+        // recently used list
+        typename list_type::iterator j = i->second.second;
+        if (j != m_list.begin()) {
+            // move item to the front of the most recently used list
+            m_list.erase(j);
+            m_list.push_front(key);
+
+            // update iterator in map
+            j = m_list.begin();
+            const value_type &value = i->second.first;
+            m_map[key] = std::make_pair(value, j);
+
+            // return the value
+            return value;
+        }
+        else {
+            // the item is already at the front of the most recently
+            // used list so just return it
+            return i->second.first;
+        }
+    }
+
+    void clear()
+    {
+        SGGuard<SGMutex> scopeLock(_mutex);
+        m_map.clear();
+        m_list.clear();
+    }
+
+private:
+    void evict()
+    {
+        SGGuard<SGMutex> scopeLock(_mutex);
+        // evict item from the end of most recently used list
+        typename list_type::iterator i = --m_list.end();
+        m_map.erase(*i);
+        m_list.erase(i);
+    }
+
+private:
+    map_type m_map;
+    list_type m_list;
+    size_t m_capacity;
+};
+lru_cache < std::string, std::string> filename_hash_cache(100000);
+lru_cache < std::string, bool> filesCleaned(100000);
+static bool refreshCache = false;
+
 ReaderWriter::ReadResult
 ModelRegistry::readImage(const string& fileName,
-                         const Options* opt)
+    const Options* opt)
 {
-    CallbackMap::iterator iter
-        = imageCallbackMap.find(getFileExtension(fileName));
+    /*
+     * processor is the interface to the osg_nvtt plugin
+     */
+    static bool init = false;
+    static osgDB::ImageProcessor *processor = 0;
+    int max_texture_size = SGSceneFeatures::instance()->getMaxTextureSize();
+    if (!init) {
+        processor = osgDB::Registry::instance()->getImageProcessor();
+        init = true;
+    }
+
+    bool cache_active = SGSceneFeatures::instance()->getTextureCacheActive();
+    bool compress_solid = SGSceneFeatures::instance()->getTextureCacheCompressionActive();
+    bool compress_transparent = SGSceneFeatures::instance()->getTextureCacheCompressionActiveTransparent();
+
+    //
+    // heuristically less than 2048 is more likely to be a badly reported size rather than
+    // something that is valid so we'll have a minimum size of 2048.
+    if (max_texture_size < 2048)
+        max_texture_size = 2048;
+
+    std::string fileExtension = getFileExtension(fileName);
+    CallbackMap::iterator iter = imageCallbackMap.find(fileExtension);
+
+    if (iter != imageCallbackMap.end() && iter->second.valid())
+        return iter->second->readImage(fileName, opt);
+    string absFileName = SGModelLib::findDataFile(fileName, opt);
+    string originalFileName = absFileName;
+    if (!fileExists(absFileName)) {
+        SG_LOG(SG_IO, SG_ALERT, "Cannot find image file \""
+            << fileName << "\"");
+        return ReaderWriter::ReadResult::FILE_NOT_FOUND;
+    }
+    Registry* registry = Registry::instance();
+    ReaderWriter::ReadResult res;
+
+    if (cache_active) {
+        if (fileExtension != "dds" && fileExtension != "gz") {
+            const SGReaderWriterOptions* sgoptC = dynamic_cast<const SGReaderWriterOptions*>(opt);
+
+            std::string root = getPathRoot(absFileName);
+            std::string prr = getPathRelative(root, absFileName);
+            std::string cache_root = SGSceneFeatures::instance()->getTextureCompressionPath().c_str();
+            std::string newName = cache_root + "/" + prr;
+
+            SGPath file(absFileName);
+            std::stringstream tstream;
+
+            // calucate and use hash for storing cached image. This also
+            // helps with sharing of identical images between models.
+            if (fileExists(absFileName)) {
+                SGFile f(absFileName);
+                std::string hash;
+                boost::optional<std::string> cachehash = filename_hash_cache.get(absFileName);
+                if (cachehash) {
+                    hash = *cachehash;
+                    //                    SG_LOG(SG_IO, SG_ALERT, "Hash for " + absFileName + " in cache " + hash);
+                }
+                else {
+                    //                  SG_LOG(SG_IO, SG_ALERT, "Creating hash for " + absFileName);
+                    try {
+                        hash = f.computeHash();
+                    }
+                    catch (sg_io_exception &e) {
+                        SG_LOG(SG_INPUT, SG_ALERT, "Modelregistry::failed to compute filehash '" << absFileName << "' " << e.getFormattedMessage());
+                        hash = std::string();
+                    }
+                }
+                if (hash != std::string()) {
+                    filename_hash_cache.insert(absFileName, hash);
+                    boost::optional<std::string> cacheFilename = filename_hash_cache.findValue(hash);
+
+                    // possibly a shared texture - but warn the user to allow investigation.
+                    if (cacheFilename && *cacheFilename != absFileName) {
+                        SG_LOG(SG_IO, SG_ALERT, " Already have " + hash + " : " + *cacheFilename + " not " + absFileName);
+                    }
+                    //                    SG_LOG(SG_IO, SG_ALERT, " >>>> " + hash + " :: " + newName);
+                }
+                newName = cache_root + "/" + hash.substr(0, 2) + "/" + hash + ".cache.dds";
+            }
+            else
+            {
+                tstream << std::hex << file.modTime();
+                newName += "." + tstream.str();
+                newName += ".cache.dds";
+            }
+            bool doRefresh = refreshCache;
+            //if (fileExists(newName) && sgoptC && sgoptC->getLoadOriginHint() == SGReaderWriterOptions::LoadOriginHint::ORIGIN_EFFECTS) {
+            //    doRefresh = true;
+            //    
+            //}
+
+            if (newName != std::string() && fileExists(newName) && doRefresh) {
+                if (!filesCleaned.contains(newName)) {
+                    SG_LOG(SG_IO, SG_ALERT, "Removing previously cached effects image " + newName);
+                    SGPath(newName).remove();
+                    filesCleaned.insert(newName, true);
+                }
+
+            }
+
+            if (newName != std::string() && !fileExists(newName)) {
+                res = registry->readImageImplementation(absFileName, opt);
+                if (res.validImage()) {
+                    osg::ref_ptr<osg::Image> srcImage = res.getImage();
+                    int width = srcImage->s();
+                    bool transparent = srcImage->isImageTranslucent();
+                    bool isNormalMap = false;
+                    bool isEffect = false;
+                    /*
+                    * decide if we need to compress this.
+                    */
+                    bool can_compress = (transparent && compress_transparent) || (!transparent && compress_solid);
+
+                    int height = srcImage->t();
+
+                    // use the new file origin to determine any special processing
+                    // we handle the following
+                    // - normal maps
+                    // - images loaded from effects
+                    if (sgoptC && transparent && sgoptC->getLoadOriginHint() == SGReaderWriterOptions::LoadOriginHint::ORIGIN_EFFECTS_NORMALIZED) {
+                        isNormalMap = true;
+                    }
+                    else if (sgoptC && transparent && sgoptC->getLoadOriginHint() == SGReaderWriterOptions::LoadOriginHint::ORIGIN_EFFECTS) {
+                        SG_LOG(SG_IO, SG_ALERT, "From effects transparent " + absFileName);
+                        isEffect = true;
+//                        can_compress = false;
+                    }
+                    else if (sgoptC && transparent && sgoptC->getLoadOriginHint() == SGReaderWriterOptions::LoadOriginHint::ORIGIN_EFFECTS) {
+                        SG_LOG(SG_IO, SG_ALERT, "From effects " + absFileName);
+                        isEffect = true;
+                    }
+                    if (can_compress)
+                    {
+                        std::string pot_message;
+                        bool resize = false;
+                        if (!isPowerOfTwo(width)) {
+                            width = nearestPowerOfTwo(width);
+                            resize = true;
+                            pot_message += std::string(" not POT: resized width to ") + std::to_string(width);
+                        }
+                        if (!isPowerOfTwo(height)) {
+                            height = nearestPowerOfTwo(height);
+                            resize = true;
+                            pot_message += std::string(" not POT: resized height to ") + std::to_string(height);
+                        }
+if (pot_message.size())
+SG_LOG(SG_IO, SG_WARN, pot_message << " " << absFileName);
+
+                        // unlikely that after resizing in height the width will still be outside of the max texture size.
+                        if (height > max_texture_size)
+                        {
+                            SG_LOG(SG_IO, SG_WARN, "Image texture too high (max " << max_texture_size << ") " << width << "," << height << " " << absFileName);
+                            int factor = height / max_texture_size;
+                            height /= factor;
+                            width /= factor;
+                            resize = true;
+                        }
+                        if (width > max_texture_size)
+                        {
+                            SG_LOG(SG_IO, SG_WARN, "Image texture too wide (max " << max_texture_size << ") " << width << "," << height << " " << absFileName);
+                            int factor = width / max_texture_size;
+                            height /= factor;
+                            width /= factor;
+                            resize = true;
+                        }
+                        if (resize) {
+                            osg::ref_ptr<osg::Image> resizedImage;
+
+                            if (ImageUtils::resizeImage(srcImage, width, height, resizedImage))
+                                srcImage = resizedImage;
+                        }
+
+                        //
+                        // only cache power of two textures that are of a reasonable size
+                        if (width >= 4 && height >= 4) {
+
+                            simgear::effect::MipMapTuple mipmapFunctions(simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE);
+
+                            SGPath filePath(newName);
+                            filePath.create_dir();
+
+                            // setup the options string for saving the texture as we don't want OSG to auto flip the texture
+                            // as this complicates loading as it requires a flag to flip it back which will preclude the
+                            // image from being cached because we will have to clone the options to set the flag and thus lose
+                            // the link to the cache in the options from the caller.
+                            osg::ref_ptr<Options> nopt;
+                            nopt = opt->cloneOptions();
+                            std::string optionstring = nopt->getOptionString();
+
+                            if (!optionstring.empty())
+                                optionstring += " ";
+
+                            nopt->setOptionString(optionstring + "ddsNoAutoFlipWrite");
+
+                            //GLenum srcImageType = srcImage->getDataType();
+                            //                            printf("--- %-80s --> f=%8x t=%8x\n", newName.c_str(), srcImage->getPixelFormat(), srcImageType);
+
+                            try
+                            {
+                                if (can_compress) {
+                                    osg::Texture::InternalFormatMode targetFormat = osg::Texture::USE_S3TC_DXT1_COMPRESSION;
+                                    if (isNormalMap) {
+                                        if (transparent) {
+                                            targetFormat = osg::Texture::USE_S3TC_DXT5_COMPRESSION;
+                                        }
+                                        else
+                                            targetFormat = osg::Texture::USE_S3TC_DXT5_COMPRESSION;
+                                    }
+                                    else if (isEffect)
+                                    {
+                                        if (transparent) {
+                                            targetFormat = osg::Texture::USE_S3TC_DXT5_COMPRESSION;
+                                        }
+                                        else
+                                            targetFormat = osg::Texture::USE_S3TC_DXT1_COMPRESSION;
+                                    }
+                                    else{
+                                        if (transparent) {
+                                            targetFormat = osg::Texture::USE_S3TC_DXT3_COMPRESSION;
+                                        }
+                                        else
+                                            targetFormat = osg::Texture::USE_S3TC_DXT1_COMPRESSION;
+                                    }
+
+                                    if (processor)
+                                    {
+                                        SG_LOG(SG_IO, SG_ALERT, "Creating " << targetFormat << " for " + absFileName);
+                                        // normal maps:
+                                        // nvdxt.exe - quality_highest - rescaleKaiser - Kaiser - dxt5nm - norm
+                                        processor->compress(*srcImage, targetFormat, true, true, osgDB::ImageProcessor::USE_CPU, osgDB::ImageProcessor::PRODUCTION);
+                                        SG_LOG(SG_IO, SG_ALERT, "-- finished creating DDS: " + newName);
+                                        //processor->generateMipMap(*srcImage, true, osgDB::ImageProcessor::USE_CPU);
+                                    }
+                                    else {
+                                        simgear::effect::MipMapTuple mipmapFunctions(simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE);
+                                        SG_LOG(SG_IO, SG_WARN, "Texture compression plugin (osg_nvtt) not available; storing uncompressed image: " << absFileName);
+                                        srcImage = simgear::effect::computeMipmap(srcImage, mipmapFunctions);
+                                    }
+                                }
+                                else {
+                                    SG_LOG(SG_IO, SG_ALERT, "Creating uncompressed DDS for " + absFileName);
+                                    if (processor) {
+                                        processor->generateMipMap(*srcImage, true, osgDB::ImageProcessor::USE_CPU);
+                                    }
+                                    else {
+                                        simgear::effect::MipMapTuple mipmapFunctions(simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE);
+                                        srcImage = simgear::effect::computeMipmap(srcImage, mipmapFunctions);
+                                    }
+                                }
+                                //}
+                                //else
+                                //    printf("--- no compress or mipmap of format %s\n", newName.c_str());
+                                registry->writeImage(*srcImage, newName, nopt);
+                                {
+                                    std::string mdlDirectory = cache_root + "/cache-index.txt";
+                                    FILE *f = ::fopen(mdlDirectory.c_str(), "a");
+                                    if (f)
+                                    {
+                                        ::fprintf(f, "%s, %s\n", absFileName.c_str(), newName.c_str());
+                                        ::fclose(f);
+                                    }
+                                }
+                                absFileName = newName;
+                            }
+                            catch (...) {
+                                SG_LOG(SG_IO, SG_ALERT, "Exception processing " << absFileName << " may be corrupted");
+                            }
+                        }
+                        else
+                            SG_LOG(SG_IO, SG_WARN, absFileName + " too small " << width << "," << height);
+                    }
+                }
+            }
+            else {
+                if (newName != std::string())
+                    absFileName = newName;
+            }
+        }
+    }
+
+    res = registry->readImageImplementation(absFileName, opt);
+
+    if (!res.success()) {
+        SG_LOG(SG_IO, SG_WARN, "Image loading failed:" << res.message());
+        return res;
+    }
+
+    osg::ref_ptr<osg::Image> srcImage1 = res.getImage();
+
+    //printf(" --> finished loading %s [%s] (%s) %d\n", absFileName.c_str(), srcImage1->getFileName().c_str(), res.loadedFromCache() ? "from cache" : "from disk", res.getImage()->getOrigin());
+    /*
+    * Fixup the filename - as when loading from eg. dds.gz the originating filename is lost in the conversion due to the way the OSG loader works
+    */
+    if (srcImage1->getFileName().empty()) {
+        srcImage1->setFileName(absFileName);
+    }
+    srcImage1->setFileName(originalFileName);
+
+    if(cache_active && getFileExtension(absFileName) != "dds")
     {
-        if (iter != imageCallbackMap.end() && iter->second.valid())
-            return iter->second->readImage(fileName, opt);
-        string absFileName = SGModelLib::findDataFile(fileName, opt);
-        if (!fileExists(absFileName)) {
-            SG_LOG(SG_IO, SG_ALERT, "Cannot find image file \""
-                   << fileName << "\"");
-            return ReaderWriter::ReadResult::FILE_NOT_FOUND;
+        if (processor) {
+            processor->generateMipMap(*srcImage1, true, osgDB::ImageProcessor::USE_CPU);
+            SG_LOG(SG_IO, SG_ALERT, "Created nvtt mipmaps DDS for " + absFileName);
         }
-
-        Registry* registry = Registry::instance();
-        ReaderWriter::ReadResult res;
-        res = registry->readImageImplementation(absFileName, opt);
-        if (!res.success()) {
-          SG_LOG(SG_IO, SG_WARN, "Image loading failed:" << res.message());
-          return res;
+        else {
+            simgear::effect::MipMapTuple mipmapFunctions(simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE, simgear::effect::AVERAGE);
+            srcImage1 = simgear::effect::computeMipmap(srcImage1, mipmapFunctions);
+            SG_LOG(SG_IO, SG_ALERT, "Created sg mipmaps DDS for " + absFileName);
         }
+    }
 
-        if (res.loadedFromCache())
-            SG_LOG(SG_IO, SG_BULK, "Returning cached image \""
-                   << res.getImage()->getFileName() << "\"");
-        else
-            SG_LOG(SG_IO, SG_BULK, "Reading image \""
-                   << res.getImage()->getFileName() << "\"");
+    if (res.loadedFromCache())
+        SG_LOG(SG_IO, SG_BULK, "Returning cached image \""
+            << res.getImage()->getFileName() << "\"");
+    else
+        SG_LOG(SG_IO, SG_BULK, "Reading image \""
+            << res.getImage()->getFileName() << "\"");
 
+    // as of March 2018 all patents have expired, https://en.wikipedia.org/wiki/S3_Texture_Compression#Patent
+    // there is support for S3TC DXT1..5 in MESA  https://www.phoronix.com/scan.php?page=news_item&px=S3TC-Lands-In-Mesa
+    // so it seems that there isn't a valid reason to warn any longer; and beside this is one of those cases where it should
+    // really only be a developer message
+#ifdef WARN_DDS_TEXTURES
         // Check for precompressed textures that depend on an extension
-        switch (res.getImage()->getPixelFormat()) {
+    switch (res.getImage()->getPixelFormat()) {
 
-            // GL_EXT_texture_compression_s3tc
-            // patented, no way to decompress these
+        // GL_EXT_texture_compression_s3tc
+        // patented, no way to decompress these
 #ifndef GL_EXT_texture_compression_s3tc
 #define GL_COMPRESSED_RGB_S3TC_DXT1_EXT   0x83F0
 #define GL_COMPRESSED_RGBA_S3TC_DXT1_EXT  0x83F1
 #define GL_COMPRESSED_RGBA_S3TC_DXT3_EXT  0x83F2
 #define GL_COMPRESSED_RGBA_S3TC_DXT5_EXT  0x83F3
 #endif
-        case GL_COMPRESSED_RGB_S3TC_DXT1_EXT:
-        case GL_COMPRESSED_RGBA_S3TC_DXT1_EXT:
-        case GL_COMPRESSED_RGBA_S3TC_DXT3_EXT:
-        case GL_COMPRESSED_RGBA_S3TC_DXT5_EXT:
+    case GL_COMPRESSED_RGB_S3TC_DXT1_EXT:
+    case GL_COMPRESSED_RGBA_S3TC_DXT1_EXT:
+    case GL_COMPRESSED_RGBA_S3TC_DXT3_EXT:
+    case GL_COMPRESSED_RGBA_S3TC_DXT5_EXT:
 
-            // GL_EXT_texture_sRGB
-            // patented, no way to decompress these
+        // GL_EXT_texture_sRGB
+        // patented, no way to decompress these
 #ifndef GL_EXT_texture_sRGB
 #define GL_COMPRESSED_SRGB_S3TC_DXT1_EXT  0x8C4C
 #define GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT1_EXT 0x8C4D
 #define GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT3_EXT 0x8C4E
 #define GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT5_EXT 0x8C4F
 #endif
-        case GL_COMPRESSED_SRGB_S3TC_DXT1_EXT:
-        case GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT1_EXT:
-        case GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT3_EXT:
-        case GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT5_EXT:
+    case GL_COMPRESSED_SRGB_S3TC_DXT1_EXT:
+    case GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT1_EXT:
+    case GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT3_EXT:
+    case GL_COMPRESSED_SRGB_ALPHA_S3TC_DXT5_EXT:
 
-            // GL_TDFX_texture_compression_FXT1
-            // can decompress these in software but
-            // no code present in simgear.
+        // GL_TDFX_texture_compression_FXT1
+        // can decompress these in software but
+        // no code present in simgear.
 #ifndef GL_3DFX_texture_compression_FXT1
 #define GL_COMPRESSED_RGB_FXT1_3DFX       0x86B0
 #define GL_COMPRESSED_RGBA_FXT1_3DFX      0x86B1
 #endif
-        case GL_COMPRESSED_RGB_FXT1_3DFX:
-        case GL_COMPRESSED_RGBA_FXT1_3DFX:
+    case GL_COMPRESSED_RGB_FXT1_3DFX:
+    case GL_COMPRESSED_RGBA_FXT1_3DFX:
 
-            // GL_EXT_texture_compression_rgtc
-            // can decompress these in software but
-            // no code present in simgear.
+        // GL_EXT_texture_compression_rgtc
+        // can decompress these in software but
+        // no code present in simgear.
 #ifndef GL_EXT_texture_compression_rgtc
 #define GL_COMPRESSED_RED_RGTC1_EXT       0x8DBB
 #define GL_COMPRESSED_SIGNED_RED_RGTC1_EXT 0x8DBC
 #define GL_COMPRESSED_RED_GREEN_RGTC2_EXT 0x8DBD
 #define GL_COMPRESSED_SIGNED_RED_GREEN_RGTC2_EXT 0x8DBE
 #endif
-        case GL_COMPRESSED_RED_RGTC1_EXT:
-        case GL_COMPRESSED_SIGNED_RED_RGTC1_EXT:
-        case GL_COMPRESSED_RED_GREEN_RGTC2_EXT:
-        case GL_COMPRESSED_SIGNED_RED_GREEN_RGTC2_EXT:
+    case GL_COMPRESSED_RED_RGTC1_EXT:
+    case GL_COMPRESSED_SIGNED_RED_RGTC1_EXT:
+    case GL_COMPRESSED_RED_GREEN_RGTC2_EXT:
+    case GL_COMPRESSED_SIGNED_RED_GREEN_RGTC2_EXT:
 
-            SG_LOG(SG_IO, SG_WARN, "Image \"" << fileName << "\"\n"
-                   "uses compressed textures which cannot be supported on "
-                   "some systems.\n"
-                   "Please decompress this texture for improved portability.");
-            break;
+        SG_LOG(SG_IO, SG_WARN, "Image \"" << fileName << "\"\n"
+            "uses compressed textures which cannot be supported on "
+            "some systems.\n"
+            "Please decompress this texture for improved portability.");
+        break;
 
-        default:
-            break;
-        }
-
-        return res;
+    default:
+        break;
     }
+#endif
+
+    return res;
 }
 
 
-osg::Node* DefaultCachePolicy::find(const string& fileName,
-                                    const Options* opt)
+osg::ref_ptr<osg::Node> DefaultCachePolicy::find(const string& fileName, const Options* opt)
 {
     Registry* registry = Registry::instance();
-    osg::Node* cached
-        = dynamic_cast<Node*>(registry->getFromObjectCache(fileName));
-    if (cached)
-        SG_LOG(SG_IO, SG_BULK, "Got cached model \""
-               << fileName << "\"");
+#if OSG_VERSION_LESS_THAN(3,4,0)
+    osg::ref_ptr<osg::Object> cachedObject = registry->getFromObjectCache(fileName);
+#else
+    osg::ref_ptr<osg::Object> cachedObject = registry->getRefFromObjectCache(fileName);
+#endif
+
+    ref_ptr<osg::Node> cachedNode = dynamic_cast<osg::Node*>(cachedObject.get());
+    if (cachedNode.valid())
+        SG_LOG(SG_IO, SG_BULK, "Got cached model \"" << fileName << "\"");
     else
-        SG_LOG(SG_IO, SG_BULK, "Reading model \""
-               << fileName << "\"");
-    return cached;
+        SG_LOG(SG_IO, SG_BULK, "Reading model \"" << fileName << "\"");
+    return cachedNode;
 }
 
 void DefaultCachePolicy::addToCache(const string& fileName,
@@ -430,7 +914,7 @@ public:
       setObjectCacheHint((Options::CacheHintOptions)cacheOptions);
     registry->setOptions(options);
     registry->getOrCreateSharedStateManager()->
-      setShareMode(SharedStateManager::SHARE_STATESETS);
+      setShareMode(SharedStateManager::SHARE_ALL);
     registry->setReadFileCallback(ModelRegistry::instance());
   }
 };
@@ -520,9 +1004,79 @@ typedef ModelRegistryCallback<OBJProcessPolicy,
                               OSGSubstitutePolicy, BuildLeafBVHPolicy>
 OBJCallback;
 
+
+// we get optimal geometry from the loader (Hah!).
+struct IVEOptimizePolicy : public OptimizeModelPolicy {
+    IVEOptimizePolicy(const string& extension) :
+        OptimizeModelPolicy(extension)
+    {
+        _osgOptions &= ~Optimizer::TRISTRIP_GEOMETRY;
+    }
+    Node* optimize(Node* node, const string& fileName,
+        const Options* opt)
+    {
+        ref_ptr<Node> optimized
+            = OptimizeModelPolicy::optimize(node, fileName, opt);
+        Group* group = dynamic_cast<Group*>(optimized.get());
+        MatrixTransform* transform
+            = dynamic_cast<MatrixTransform*>(optimized.get());
+        if (((transform && transform->getMatrix().isIdentity()) || group)
+            && group->getName().empty()
+            && group->getNumChildren() == 1) {
+            optimized = static_cast<Node*>(group->getChild(0));
+            group = dynamic_cast<Group*>(optimized.get());
+            if (group && group->getName().empty()
+                && group->getNumChildren() == 1)
+                optimized = static_cast<Node*>(group->getChild(0));
+        }
+        const SGReaderWriterOptions* sgopt
+            = dynamic_cast<const SGReaderWriterOptions*>(opt);
+
+        if (sgopt && sgopt->getInstantiateMaterialEffects()) {
+            optimized = instantiateMaterialEffects(optimized.get(), sgopt);
+        }
+        else if (sgopt && sgopt->getInstantiateEffects()) {
+            optimized = instantiateEffects(optimized.get(), sgopt);
+        }
+
+        return optimized.release();
+    }
+};
+
+struct IVEProcessPolicy {
+    IVEProcessPolicy(const string& extension) {}
+    Node* process(Node* node, const string& filename,
+        const Options* opt)
+    {
+        Matrix m(1, 0, 0, 0,
+            0, 0, 1, 0,
+            0, -1, 0, 0,
+            0, 0, 0, 1);
+        // XXX Does there need to be a Group node here to trick the
+        // optimizer into optimizing the static transform?
+        osg::Group* root = new Group;
+        MatrixTransform* transform = new MatrixTransform;
+        root->addChild(transform);
+
+        transform->setDataVariance(Object::STATIC);
+        transform->setMatrix(m);
+        transform->addChild(node);
+
+        return root;
+    }
+};
+
+typedef ModelRegistryCallback<IVEProcessPolicy, DefaultCachePolicy,
+    IVEOptimizePolicy,
+    OSGSubstitutePolicy, BuildLeafBVHPolicy>
+    IVECallback;
+
 namespace
 {
 ModelRegistryCallbackProxy<ACCallback> g_acRegister("ac");
 ModelRegistryCallbackProxy<OBJCallback> g_objRegister("obj");
+ModelRegistryCallbackProxy<IVECallback> g_iveRegister("ive");
+ModelRegistryCallbackProxy<IVECallback> g_osgtRegister("osgt");
+ModelRegistryCallbackProxy<IVECallback> g_osgbRegister("osgb");
 
 }

simgear/scene/model/ModelRegistry.hxx

@@ -132,7 +132,7 @@ struct DefaultProcessPolicy {
 
 struct DefaultCachePolicy {
     DefaultCachePolicy(const std::string& extension) {}
-    osg::Node* find(const std::string& fileName,
+    osg::ref_ptr<osg::Node> find(const std::string& fileName,
                     const osgDB::Options* opt);
     void addToCache(const std::string& filename, osg::Node* node);
 };

simgear/scene/model/SGReaderWriterXML.cxx

@@ -27,6 +27,7 @@
 
 #include <boost/bind.hpp>
 
+#include <osg/Version>
 #include <osg/Geode>
 #include <osg/MatrixTransform>
 #include <osgDB/ReadFile>
@@ -56,7 +57,7 @@ using namespace std;
 using namespace simgear;
 using namespace osg;
 
-static osg::Node *
+static std::tuple<int, osg::Node *>
 sgLoad3DModel_internal(const SGPath& path,
                        const osgDB::Options* options,
                        SGPropertyNode *overlay = 0);
@@ -88,8 +89,10 @@ SGReaderWriterXML::readNode(const std::string& name,
         if (!p.exists()) {
           return ReadResult::FILE_NOT_FOUND;
         }
-        
-        result=sgLoad3DModel_internal(p, options);
+
+        static std::tuple<int, osg::Node *> retval;
+        int num_anims;
+        std::tie(num_anims, result) = sgLoad3DModel_internal(p, options);
     } catch (const sg_exception &t) {
         SG_LOG(SG_INPUT, SG_ALERT, "Failed to load model: " << t.getFormattedMessage()
           << "\n\tfrom:" << fileName);
@@ -160,13 +163,13 @@ void makeEffectAnimations(PropertyList& animation_nodes,
         SGPropertyNode* typeProp = animProp->getChild("type");
         if (!typeProp)
             continue;
-        
+
         const char* typeString = typeProp->getStringValue();
         if (!strcmp(typeString, "material")) {
             effectProp
                 = SGMaterialAnimation::makeEffectProperties(animProp);
         } else if (!strcmp(typeString, "shader")) {
-            
+
             SGPropertyNode* shaderProp = animProp->getChild("shader");
             if (!shaderProp || strcmp(shaderProp->getStringValue(), "chrome"))
                 continue;
@@ -215,7 +218,7 @@ private:
     osg::Node::NodeMask nodeMaskSet;
     osg::Node::NodeMask nodeMaskClear;
 };
-    
+
 }
 
 namespace {
@@ -229,7 +232,7 @@ namespace {
             return (typeString == "pick") || (typeString == "knob") || (typeString == "slider") || (typeString == "touch");
         }
     };
-    
+
     bool removeNamedNode(osg::Group* aGroup, const std::string& aName)
     {
         int nKids = aGroup->getNumChildren();
@@ -240,28 +243,28 @@ namespace {
                 return true;
             }
         }
-        
+
         for (int i = 0; i < nKids; i++) {
             osg::Group* childGroup = aGroup->getChild(i)->asGroup();
             if (!childGroup)
                 continue;
-            
+
             if (removeNamedNode(childGroup, aName))
                 return true;
         } // of child groups traversal
-        
+
         return false;
     }
 }
 
-static osg::Node *
+static std::tuple<int, osg::Node *>
 sgLoad3DModel_internal(const SGPath& path,
                        const osgDB::Options* dbOptions,
                        SGPropertyNode *overlay)
 {
     if (!path.exists()) {
       SG_LOG(SG_INPUT, SG_ALERT, "Failed to load file: \"" << path << "\"");
-      return NULL;
+      return std::make_tuple(0, (osg::Node *) NULL);
     }
 
     osg::ref_ptr<SGReaderWriterOptions> options;
@@ -270,19 +273,20 @@ sgLoad3DModel_internal(const SGPath& path,
     SGPath modelpath(path);
     SGPath texturepath(path);
     SGPath modelDir(modelpath.dir());
-    
+    int animationcount = 0;
+
     SGSharedPtr<SGPropertyNode> prop_root = options->getPropertyNode();
     if (!prop_root.valid())
         prop_root = new SGPropertyNode;
     // The model data appear to be only used in the topmost model
     osg::ref_ptr<SGModelData> data = options->getModelData();
     options->setModelData(0);
-    
+
     osg::ref_ptr<osg::Node> model;
     osg::ref_ptr<osg::Group> group;
     SGPropertyNode_ptr props = new SGPropertyNode;
     bool previewMode = (dbOptions->getPluginStringData("SimGear::PREVIEW") == "ON");
-    
+
     // Check for an XML wrapper
     if (modelpath.extension() == "xml") {
        try {
@@ -296,9 +300,9 @@ sgLoad3DModel_internal(const SGPath& path,
             copyProperties(overlay, props);
 
         if (previewMode && props->hasChild("nopreview")) {
-            return NULL;
+            return std::make_tuple(0, (osg::Node *) NULL);
         }
-        
+
         if (props->hasValue("/path")) {
             string modelPathStr = props->getStringValue("/path");
             modelpath = SGModelLib::findDataFile(modelPathStr, NULL, modelDir);
@@ -335,7 +339,11 @@ sgLoad3DModel_internal(const SGPath& path,
 
         options->setDatabasePath(texturepath.local8BitStr());
         osgDB::ReaderWriter::ReadResult modelResult;
+#if OSG_VERSION_LESS_THAN(3,4,1)
         modelResult = osgDB::readNodeFile(modelpath.local8BitStr(), options.get());
+#else
+        modelResult = osgDB::readRefNodeFile(modelpath.local8BitStr(), options.get());
+#endif
         if (!modelResult.validNode())
             throw sg_io_exception("Failed to load 3D model:" + modelResult.message(),
                                   modelpath);
@@ -399,9 +407,9 @@ sgLoad3DModel_internal(const SGPath& path,
 
         SGPath submodelpath;
         osg::ref_ptr<osg::Node> submodel;
-        
+
         string subPathStr = sub_props->getStringValue("path");
-        SGPath submodelPath = SGModelLib::findDataFile(subPathStr, 
+        SGPath submodelPath = SGModelLib::findDataFile(subPathStr,
           NULL, modelDir);
 
         if (submodelPath.isNull()) {
@@ -419,14 +427,16 @@ sgLoad3DModel_internal(const SGPath& path,
         }
 
         try {
-            submodel = sgLoad3DModel_internal(submodelPath, options.get(),
+            int num_anims;
+            std::tie(num_anims, submodel) = sgLoad3DModel_internal(submodelPath, options.get(),
                                               sub_props->getNode("overlay"));
+            animationcount += num_anims;
         } catch (const sg_exception &t) {
             SG_LOG(SG_INPUT, SG_ALERT, "Failed to load submodel: " << t.getFormattedMessage()
               << "\n\tfrom:" << t.getOrigin());
             continue;
         }
-        
+
         if (!submodel)
             continue;
 
@@ -481,7 +491,7 @@ sgLoad3DModel_internal(const SGPath& path,
         }
     }
 
-    if (dbOptions->getPluginStringData("SimGear::PARTICLESYSTEM") != "OFF") {
+    if (GlobalParticleCallback::getEnabled()){//dbOptions->getPluginStringData("SimGear::PARTICLESYSTEM") != "OFF") {
         std::vector<SGPropertyNode_ptr> particle_nodes;
         particle_nodes = props->getChildren("particlesystem");
         for (unsigned i = 0; i < particle_nodes.size(); ++i) {
@@ -491,7 +501,7 @@ sgLoad3DModel_internal(const SGPath& path,
             if (i==0) {
                 if (!texturepath.extension().empty())
                     texturepath = texturepath.dir();
-                
+
                 options2->setDatabasePath(texturepath.local8BitStr());
             }
             group->addChild(Particles::appendParticles(particle_nodes[i],
@@ -510,13 +520,13 @@ sgLoad3DModel_internal(const SGPath& path,
 
     PropertyList effect_nodes = props->getChildren("effect");
     PropertyList animation_nodes = props->getChildren("animation");
-    
+
     if (previewMode) {
         PropertyList::iterator it;
         it = std::remove_if(animation_nodes.begin(), animation_nodes.end(), ExcludeInPreview());
         animation_nodes.erase(it, animation_nodes.end());
     }
-    
+
     // Some material animations (eventually all) are actually effects.
     makeEffectAnimations(animation_nodes, effect_nodes);
     {
@@ -543,13 +553,15 @@ sgLoad3DModel_internal(const SGPath& path,
         /// OSGFIXME: duh, why not only model?????
         SGAnimation::animate(modelData);
     }
-    
+
+    animationcount += animation_nodes.size();
+
     if (!needTransform && group->getNumChildren() < 2) {
         model = group->getChild(0);
         group->removeChild(model.get());
         if (data.valid())
             data->modelLoaded(modelpath.utf8Str(), props, model.get());
-        return model.release();
+        return std::make_tuple(animationcount, model.release());
     }
     if (data.valid())
         data->modelLoaded(modelpath.utf8Str(), props, group.get());
@@ -559,6 +571,7 @@ sgLoad3DModel_internal(const SGPath& path,
         osgDB::writeNodeFile(*group, outputfile);
     }
 
-    return group.release();
-}
+    SG_LOG(SG_GENERAL, SG_DEBUG, "Model " << path << " animation count: " << animationcount);
 
+    return std::make_tuple(animationcount, group.release());
+}

simgear/scene/model/model.cxx

@@ -41,13 +41,24 @@ SGLoadTexture2D(bool staticTexture, const std::string& path,
                 const osgDB::Options* options,
                 bool wrapu, bool wrapv, int)
 {
-  osg::Image* image;
+  osg::ref_ptr<osg::Image> image;
   if (options)
-    image = osgDB::readImageFile(path, options);
+#if OSG_VERSION_LESS_THAN(3,4,0)
+      image = osgDB::readImageFile(path, options);
+#else
+      image = osgDB::readRefImageFile(path, options);
+#endif
   else
-    image = osgDB::readImageFile(path);
+#if OSG_VERSION_LESS_THAN(3,4,0)
+      image = osgDB::readImageFile(path);
+#else
+      image = osgDB::readRefImageFile(path);
+#endif
+
   osg::ref_ptr<osg::Texture2D> texture = new osg::Texture2D;
   texture->setImage(image);
+  texture->setMaxAnisotropy(SGSceneFeatures::instance()->getTextureFilter());
+
   if (staticTexture)
     texture->setDataVariance(osg::Object::STATIC);
   if (wrapu)
@@ -128,30 +139,45 @@ Texture2D* TextureUpdateVisitor::textureReplace(int unit, const StateAttribute*
     if (image) {
         // The currently loaded file name
         fullFilePath = &image->getFileName();
-
     } else {
         fullFilePath = &texture->getName();
     }
+
     // The short name
     string fileName = getSimpleFileName(*fullFilePath);
     if (fileName.empty())
         return 0;
+
     // The name that should be found with the current database path
     string fullLiveryFile = findFileInPath(fileName, _pathList);
     // If it is empty or they are identical then there is nothing to do
     if (fullLiveryFile.empty() || fullLiveryFile == *fullFilePath)
         return 0;
+
+#if OSG_VERSION_LESS_THAN(3,4,0)
     Image* newImage = readImageFile(fullLiveryFile);
+#else
+    osg::ref_ptr<Image> newImage = readRefImageFile(fullLiveryFile);
+#endif
     if (!newImage)
         return 0;
+
     CopyOp copyOp(CopyOp::DEEP_COPY_ALL & ~CopyOp::DEEP_COPY_IMAGES);
     Texture2D* newTexture = static_cast<Texture2D*>(copyOp(texture));
-    if (!newTexture) {
+    if (!newTexture)
         return 0;
-    } else {
-        newTexture->setImage(newImage);
-        return newTexture;
+
+    newTexture->setImage(newImage);
+#if OSG_VERSION_LESS_THAN(3,4,0)
+    if (newImage->valid())
+#else
+    if (newImage.valid())
+#endif
+    {
+        newTexture->setMaxAnisotropy(SGSceneFeatures::instance()->getTextureFilter());
     }
+
+    return newTexture;
 }
 
 StateSet* TextureUpdateVisitor::cloneStateSet(const StateSet* stateSet)

simgear/scene/model/modellib.cxx

@@ -21,6 +21,7 @@
 
 #include <boost/algorithm/string.hpp>
 
+#include <osg/Version>
 #include <osg/PagedLOD>
 #include <osg/ProxyNode>
 #include <osgDB/ReadFile>
@@ -101,7 +102,11 @@ osg::Node* loadFile(const string& path, SGReaderWriterOptions* options)
         options->setInstantiateEffects(true);
     }
 
+#if OSG_VERSION_LESS_THAN(3,4,0)
+    ref_ptr<Node> model = readNodeFile(path, options);
+#else
     ref_ptr<Node> model = readRefNodeFile(path, options);
+#endif
     if (!model)
         return 0;
     else
@@ -161,33 +166,69 @@ SGModelLib::loadDeferredModel(const string &path, SGPropertyNode *prop_root,
     return proxyNode;
 }
 
+
+/*
+ * Load a set of models at different LOD range_nearest
+ *
+ */
 osg::PagedLOD*
-SGModelLib::loadPagedModel(const string &path, SGPropertyNode *prop_root,
-                           SGModelData *data)
+SGModelLib::loadPagedModel(SGPropertyNode *prop_root, SGModelData *data, SGModelLOD model_lods)
 {
+    unsigned int simple_models = 0;
     osg::PagedLOD *plod = new osg::PagedLOD;
-    plod->setName("Paged LOD for \"" + path + "\"");
-    plod->setFileName(0, path);
-    plod->setRange(0, 0.0, 50.0*SG_NM_TO_METER);
-    plod->setMinimumExpiryTime( 0, prop_root->getDoubleValue("/sim/rendering/plod-minimum-expiry-time-secs", 180.0 ) );
 
     osg::ref_ptr<SGReaderWriterOptions> opt;
     opt = SGReaderWriterOptions::copyOrCreate(osgDB::Registry::instance()->getOptions());
     opt->setPropertyNode(prop_root ? prop_root: static_propRoot.get());
     opt->setModelData(data);
     opt->setLoadPanel(static_panelFunc);
-    std::string lext = SGPath(path).lower_extension();
-
-    if ((lext == "ac") || (lext == "obj")) {
-        opt->setInstantiateEffects(true);
-    }
-
     if (!prop_root || prop_root->getBoolValue("/sim/rendering/cache", true))
         opt->setObjectCacheHint(osgDB::Options::CACHE_ALL);
     else
         opt->setObjectCacheHint(osgDB::Options::CACHE_NONE);
+
+    for(unsigned int i = 0; i < model_lods.getNumLODs(); i++) {
+      SGModelLOD::ModelLOD lod = model_lods.getModelLOD(i);
+      plod->setName("Paged LOD for \"" + lod.path + "\"");
+      plod->setFileName(i, lod.path);
+      plod->setRange(i, lod.min_range, lod.max_range);
+      plod->setMinimumExpiryTime(i, prop_root->getDoubleValue("/sim/rendering/plod-minimum-expiry-time-secs", 180.0 ) );
+
+      std::string lext = SGPath(lod.path).lower_extension();
+      if ((lext == "ac") || (lext == "obj")) {
+        simple_models++;
+      }
+    }
+
+    // If all we have are simple models, then we can instantiate effects in
+    // the loader.
+    if (simple_models == model_lods.getNumLODs()) opt->setInstantiateEffects(true);
+
     plod->setDatabaseOptions(opt.get());
+
     return plod;
 }
 
+osg::PagedLOD*
+SGModelLib::loadPagedModel(const string &path, SGPropertyNode *prop_root,
+                           SGModelData *data)
+{
+    SGModelLOD model_lods;
+    model_lods.insert(path, 0.0, 50.0*SG_NM_TO_METER);
+    return SGModelLib::loadPagedModel(prop_root, data, model_lods);
+}
+
+osg::PagedLOD*
+SGModelLib::loadPagedModel(std::vector<string> paths, SGPropertyNode *prop_root,
+                           SGModelData *data)
+{
+    SGModelLOD model_lods;
+    for(unsigned int i = 0; i < paths.size(); i++) {
+      // We don't have any range data, so simply set them all up to full range.
+      // Some other code will update the LoD ranges later.  (AIBase::updateLOD)
+      model_lods.insert(paths[i], 0.0, 50.0*SG_NM_TO_METER);
+    }
+    return SGModelLib::loadPagedModel(prop_root, data, model_lods);
+}
+
 // end of modellib.cxx

simgear/scene/model/modellib.hxx

@@ -39,6 +39,7 @@ namespace osg {
 namespace simgear {
 
 class SGModelData; // defined below
+class SGModelLOD;  // defined below
 
 /**
  * Class for loading and managing models with XML wrappers.
@@ -51,9 +52,9 @@ public:
     static void init(const std::string &root_dir, SGPropertyNode* root);
 
     static void resetPropertyRoot();
-    
+
     static void setPanelFunc(panel_func pf);
-    
+
     // Load a 3D model (any format)
     // data->modelLoaded() will be called after the model is loaded
     static osg::Node* loadModel(const std::string &path,
@@ -72,17 +73,24 @@ public:
     // the model file. Once the viewer steps onto that node the
     // model will be loaded. When the viewer does no longer reference this
     // node for a long time the node is unloaded again.
+    static osg::PagedLOD* loadPagedModel(SGPropertyNode *prop_root,
+                                      SGModelData *data,
+                                      SGModelLOD model_lods);
     static osg::PagedLOD* loadPagedModel(const std::string &path,
                                      SGPropertyNode *prop_root = NULL,
                                      SGModelData *data=0);
 
-    static std::string findDataFile(const std::string& file, 
+   static osg::PagedLOD* loadPagedModel(std::vector<string> paths,
+                                    SGPropertyNode *prop_root = NULL,
+                                    SGModelData *data=0);
+
+    static std::string findDataFile(const std::string& file,
       const osgDB::Options* opts = NULL,
-      SGPath currentDir = SGPath()); 
+      SGPath currentDir = SGPath());
 protected:
     SGModelLib();
     ~SGModelLib ();
-    
+
 private:
   static SGPropertyNode_ptr static_propRoot;
   static panel_func static_panelFunc;
@@ -102,6 +110,39 @@ public:
     virtual SGModelData* clone() const = 0;
 };
 
+/*
+ * Data for a model with multiple LoD versions
+ */
+
+class SGModelLOD {
+public:
+  struct ModelLOD {
+    ModelLOD(const string &p, float minrange, float maxrange) :
+      path(p), min_range(minrange), max_range(maxrange)
+      { }
+    const string &path;
+    float min_range;
+    float max_range;
+  };
+typedef std::vector<ModelLOD> ModelLODList;
+
+void insert(const ModelLOD& model)
+{
+  _models.push_back(model);
+}
+
+void insert(const string &p, float minrange, float maxrange)
+{
+  insert(ModelLOD(p, minrange, maxrange));
+}
+
+unsigned getNumLODs() const { return _models.size(); }
+const ModelLOD& getModelLOD(unsigned i) const { return _models[i]; }
+
+private:
+  ModelLODList _models;
+};
+
 }
 
 #endif // _SG_MODEL_LIB_HXX

simgear/scene/sky/cloud.cxx

@@ -485,9 +485,11 @@ SGCloudLayer::rebuild()
 // repaint the cloud layer colors
 bool SGCloudLayer::repaint( const SGVec3f& fog_color ) {
     osg::Vec4f combineColor(toOsg(fog_color), cloud_alpha);
-    osg::TexEnvCombine* combiner
-        = dynamic_cast<osg::TexEnvCombine*>(layer_root->getStateSet()
-                                            ->getTextureAttribute(1, osg::StateAttribute::TEXENV));
+    osg::StateAttribute* textureAtt = layer_root->getStateSet()->getTextureAttribute(1, osg::StateAttribute::TEXENV);
+    osg::TexEnvCombine* combiner = dynamic_cast<osg::TexEnvCombine*>(textureAtt);
+
+    if (combiner == nullptr)
+        return false;
     combiner->setConstantColor(combineColor);
 
     // Set the fog color for the 3D clouds too.

simgear/scene/sky/cloudfield.cxx

@@ -1,4 +1,4 @@
-// a layer of 3d clouds
+  // a layer of 3d clouds
 //
 // Written by Harald JOHNSEN, started April 2005.
 //
@@ -78,56 +78,57 @@ bool SGCloudField::reposition( const SGVec3f& p, const SGVec3f& up, double lon,
 
     SGVec3<double> cart;
     SGGeod new_pos = SGGeod::fromRadFt(lon, lat, 0.0f);
-                            
+
     SGGeodesy::SGGeodToCart(new_pos, cart);
     osg::Vec3f osg_pos = toOsg(cart);
     osg::Quat orient = toOsg(SGQuatd::fromLonLatRad(lon, lat) * SGQuatd::fromRealImag(0, SGVec3d(0, 1, 0)));
-    
+
     // Always update the altitude transform, as this allows
     // the clouds to rise and fall smoothly depending on environment updates.
     altitude_transform->setPosition(osg::Vec3f(0.0f, 0.0f, (float) asl));
-    
+
     // Similarly, always determine the effects of the wind
     osg::Vec3f wind = osg::Vec3f(-cos((direction + 180)* SGD_DEGREES_TO_RADIANS) * speed * dt,
                                  sin((direction + 180)* SGD_DEGREES_TO_RADIANS) * speed * dt,
                                  0.0f);
     osg::Vec3f windosg = field_transform->getAttitude() * wind;
     field_transform->setPosition(field_transform->getPosition() + windosg);
-    
+
     if (!wrap) {
         // If we're not wrapping the cloudfield, then we make no effort to reposition
         return false;
     }
-        
+
     if ((old_pos - osg_pos).length() > fieldSize*2) {
         // Big movement - reposition centered to current location.
         field_transform->setPosition(osg_pos);
         field_transform->setAttitude(orient);
         old_pos = osg_pos;
-    } else {        
+    } else if ((old_pos - osg_pos).length() > fieldSize*0.1) {
+        // Smaller, but non-trivial movement - check if any clouds need to be moved
         osg::Quat fta =  field_transform->getAttitude();
         osg::Quat ftainv = field_transform->getAttitude().inverse();
-        
+
         // delta is the vector from the old position to the new position in cloud-coords
         // osg::Vec3f delta = ftainv * (osg_pos - old_pos);
         old_pos = osg_pos;
-                
+
         // Check if any of the clouds should be moved.
         for(CloudHash::const_iterator itr = cloud_hash.begin(), end = cloud_hash.end();
             itr != end;
             ++itr) {
-              
+
              osg::ref_ptr<osg::PositionAttitudeTransform> pat = itr->second;
-            
+
              if (pat == 0) {
                 continue;
              }
-             
+
              osg::Vec3f currpos = field_transform->getPosition() + fta * pat->getPosition();
-                                      
+
              // Determine the vector from the new position to the cloud in cloud-space.
-             osg::Vec3f w =  ftainv * (currpos - toOsg(cart));               
-              
+             osg::Vec3f w =  ftainv * (currpos - toOsg(cart));
+
              // Determine a course if required. Note that this involves some axis translation.
              float x = 0.0;
              float y = 0.0;
@@ -135,16 +136,16 @@ bool SGCloudField::reposition( const SGVec3f& p, const SGVec3f& up, double lon,
              if (w.x() < -0.6*fieldSize) { y = -fieldSize; }
              if (w.y() >  0.6*fieldSize) { x = -fieldSize; }
              if (w.y() < -0.6*fieldSize) { x =  fieldSize; }
-              
+
              if ((x != 0.0) || (y != 0.0)) {
                  removeCloudFromTree(pat);
-                 SGGeod p = SGGeod::fromCart(toSG(field_transform->getPosition() + 
+                 SGGeod p = SGGeod::fromCart(toSG(field_transform->getPosition() +
                                                   fta * pat->getPosition()));
-                 addCloudToTree(pat, p, x, y);                
+                 addCloudToTree(pat, p, x, y);
             }
-        }        
+        }
     }
-    
+
     // Render the clouds in order from farthest away layer to nearest one.
     field_root->getStateSet()->setRenderBinDetails(CLOUDS_BIN, "DepthSortedBin");
     return true;
@@ -161,7 +162,7 @@ SGCloudField::SGCloudField() :
     osg::StateSet *rootSet = field_root->getOrCreateStateSet();
     rootSet->setRenderBinDetails(CLOUDS_BIN, "DepthSortedBin");
     rootSet->setAttributeAndModes(getFog());
-    
+
     field_transform->addChild(altitude_transform.get());
     placed_root = new osg::Group();
     altitude_transform->addChild(placed_root);
@@ -169,7 +170,7 @@ SGCloudField::SGCloudField() :
     lodcount = 0;
     cloudcount = 0;
 }
-    
+
 SGCloudField::~SGCloudField() {
 }
 
@@ -181,7 +182,7 @@ void SGCloudField::clear(void) {
         ++itr) {
         removeCloudFromTree(itr->second);
     }
-    
+
     cloud_hash.clear();
 }
 
@@ -198,7 +199,7 @@ void SGCloudField::applyVisAndLoDRange(void)
         }
     }
 }
-            
+
 bool SGCloudField::addCloud(float lon, float lat, float alt, int index, osg::ref_ptr<EffectGeode> geode) {
   return addCloud(lon, lat, alt, 0.0f, 0.0f, index, geode);
 }
@@ -244,34 +245,34 @@ void SGCloudField::removeCloudFromTree(osg::ref_ptr<osg::PositionAttitudeTransfo
 
 void SGCloudField::addCloudToTree(osg::ref_ptr<osg::PositionAttitudeTransform> transform,
                                   float lon, float lat, float alt, float x, float y) {
-    
+
     // Get the base position
     SGGeod loc = SGGeod::fromDegFt(lon, lat, alt);
-    addCloudToTree(transform, loc, x, y);      
+    addCloudToTree(transform, loc, x, y);
 }
 
 
 void SGCloudField::addCloudToTree(osg::ref_ptr<osg::PositionAttitudeTransform> transform,
                                   SGGeod loc, float x, float y) {
-        
+
     float alt = loc.getElevationFt();
     // Determine any shift by x/y
     if ((x != 0.0f) || (y != 0.0f)) {
-        double crs = 90.0 - SG_RADIANS_TO_DEGREES * atan2(y, x); 
+        double crs = 90.0 - SG_RADIANS_TO_DEGREES * atan2(y, x);
         double dst = sqrt(x*x + y*y);
         double endcrs;
-        
-        SGGeod base_pos = SGGeod::fromGeodFt(loc, 0.0f);            
+
+        SGGeod base_pos = SGGeod::fromGeodFt(loc, 0.0f);
         SGGeodesy::direct(base_pos, crs, dst, loc, endcrs);
     }
-    
-    // The direct call provides the position at 0 alt, so adjust as required.      
-    loc.setElevationFt(alt);    
-    addCloudToTree(transform, loc);    
+
+    // The direct call provides the position at 0 alt, so adjust as required.
+    loc.setElevationFt(alt);
+    addCloudToTree(transform, loc);
 }
-        
-        
-void SGCloudField::addCloudToTree(osg::ref_ptr<osg::PositionAttitudeTransform> transform, SGGeod loc) {        
+
+
+void SGCloudField::addCloudToTree(osg::ref_ptr<osg::PositionAttitudeTransform> transform, SGGeod loc) {
     // Work out where this cloud should go in OSG coordinates.
     SGVec3<double> cart;
     SGGeodesy::SGGeodToCart(loc, cart);
@@ -285,17 +286,17 @@ void SGCloudField::addCloudToTree(osg::ref_ptr<osg::PositionAttitudeTransform> t
         // Convert to the scenegraph orientation where we just rotate around
         // the y axis 180 degrees.
         osg::Quat orient = toOsg(SGQuatd::fromLonLatDeg(loc.getLongitudeDeg(), loc.getLatitudeDeg()) * SGQuatd::fromRealImag(0, SGVec3d(0, 1, 0)));
-        
-        osg::Vec3f osg_pos = toOsg(fieldcenter);            
-        
+
+        osg::Vec3f osg_pos = toOsg(fieldcenter);
+
         field_transform->setPosition(osg_pos);
         field_transform->setAttitude(orient);
         old_pos = osg_pos;
     }
-    
+
     // Convert position to cloud-coordinates
     pos = pos - field_transform->getPosition();
-    pos = field_transform->getAttitude().inverse() * pos;    
+    pos = field_transform->getAttitude().inverse() * pos;
 
     // We have a two level dynamic quad tree which the cloud will be added
     // to. If there are no appropriate nodes in the quad tree, they are
@@ -310,7 +311,7 @@ void SGCloudField::addCloudToTree(osg::ref_ptr<osg::PositionAttitudeTransform> t
         if ((lodnode1->getCenter() - pos).length2() < lod1_range*lod1_range) {
           // New cloud is within RADIUS_LEVEL_1 of the center of the LOD node.
           found = true;
-        }         
+        }
     }
 
     if (!found) {
@@ -318,7 +319,7 @@ void SGCloudField::addCloudToTree(osg::ref_ptr<osg::PositionAttitudeTransform> t
           impostornode = new osgSim::Impostor();
           impostornode->setImpostorThreshold(impostor_distance);
           //impostornode->setImpostorThresholdToBound();
-          //impostornode->setCenter(pos);                
+          //impostornode->setCenter(pos);
           placed_root->addChild(impostornode.get());
           lodnode1 = (osg::ref_ptr<osg::LOD>) impostornode;
         } else {
@@ -330,8 +331,8 @@ void SGCloudField::addCloudToTree(osg::ref_ptr<osg::PositionAttitudeTransform> t
 
     // Now check if there is a second level LOD node at an appropriate distance
     found = false;
-    
-    for (unsigned int j = 0; (!found) && (j < lodnode1->getNumChildren()); j++) {      
+
+    for (unsigned int j = 0; (!found) && (j < lodnode1->getNumChildren()); j++) {
         lodnode = (osg::LOD*) lodnode1->getChild(j);
         if ((lodnode->getCenter() - pos).length2() < lod2_range*lod2_range) {
             // We've found the right leaf LOD node
@@ -344,37 +345,37 @@ void SGCloudField::addCloudToTree(osg::ref_ptr<osg::PositionAttitudeTransform> t
         lodnode = new osg::LOD();
         lodnode1->addChild(lodnode, 0.0f, lod1_range + lod2_range + view_distance + MAX_CLOUD_DEPTH);
         lodcount++;
-    } 
-    
+    }
+
     transform->setPosition(pos);
     lodnode->addChild(transform.get(), 0.0f, view_distance + MAX_CLOUD_DEPTH);
     cloudcount++;
     SG_LOG(SG_ENVIRONMENT, SG_DEBUG, "Impostors: " << impostorcount <<
-                                     " LoD: " << lodcount << 
+                                     " LoD: " << lodcount <<
                                      " Clouds: " << cloudcount);
 
     lodnode->dirtyBound();
     lodnode1->dirtyBound();
     field_root->dirtyBound();
 }
-        
+
 bool SGCloudField::deleteCloud(int identifier) {
     osg::ref_ptr<osg::PositionAttitudeTransform> transform = cloud_hash[identifier];
     if (transform == 0) return false;
-        
+
     removeCloudFromTree(transform);
     cloud_hash.erase(identifier);
 
     return true;
 }
-        
+
 bool SGCloudField::repositionCloud(int identifier, float lon, float lat, float alt) {
     return repositionCloud(identifier, lon, lat, alt, 0.0f, 0.0f);
 }
 
 bool SGCloudField::repositionCloud(int identifier, float lon, float lat, float alt, float x, float y) {
     osg::ref_ptr<osg::PositionAttitudeTransform> transform = cloud_hash[identifier];
-    
+
     if (transform == NULL) return false;
 
     removeCloudFromTree(transform);
@@ -398,4 +399,3 @@ void SGCloudField::updateFog(double visibility, const osg::Vec4f& color) {
     fog->setColor(color);
     fog->setDensity(sqrt_m_log01 / visibility);
 }
-

simgear/scene/sky/sky.cxx

@@ -94,7 +94,7 @@ void SGSky::build( double h_radius_m,
     planets = new SGStars;
     _ephTransform->addChild( planets->build(eph.getNumPlanets(), eph.getPlanets(), h_radius_m) );
 
-    stars = new SGStars;
+    stars = new SGStars(property_tree_node);
     _ephTransform->addChild( stars->build(eph.getNumStars(), eph.getStars(), h_radius_m) );
     
     moon = new SGMoon;

simgear/scene/sky/stars.cxx

@@ -30,9 +30,11 @@
 #include <simgear/compiler.h>
 #include <simgear/constants.h>
 #include <simgear/debug/logstream.hxx>
+#include <simgear/props/props.hxx>
 
 #include <stdio.h>
 #include <iostream>
+#include <algorithm>
 
 #include <osg/AlphaFunc>
 #include <osg/BlendFunc>
@@ -47,9 +49,14 @@
 #include "stars.hxx"
 
 // Constructor
-SGStars::SGStars( void ) :
-old_phase(-1)
+SGStars::SGStars( SGPropertyNode* props ) :
+    old_phase(-1)
 {
+    if (props) {
+        // don't create here - if it's not defined, we won't use the cutoff
+        // from a property
+        _cutoffProperty = props->getNode("star-magnitude-cutoff");
+    }
 }
 
 
@@ -117,69 +124,102 @@ SGStars::build( int num, const SGVec3d star_data[], double star_dist ) {
 // 0 degrees = high noon
 // 90 degrees = sun rise/set
 // 180 degrees = darkest midnight
-bool SGStars::repaint( double sun_angle, int num, const SGVec3d star_data[] ) {
-    // cout << "repainting stars" << endl;
-    // double min = 100;
-    // double max = -100;
+
+bool SGStars::repaint( double sun_angle, int num, const SGVec3d star_data[] )
+{
     double mag, nmag, alpha, factor, cutoff;
 
+    /*
+    maximal magnitudes under dark sky on Earth, from Eq.(90) and (91) of astro-ph/1405.4209
+	For (18 < musky < 20)
+	mmax = 0.27 musky + 0.8 - 2.5 * log(F)
+
+	For (19.5  µsky  22)
+	mmax = 0.383 musky - 1.44 - 2.5 * log(F)
+
+    
+	Let's take F = 1.4 for healthy young pilot
+	mudarksky ~ 22 mag/arcsec^2 => mmax=6.2
+	muastrotwilight ~ 20 mag/arsec^2 => mmax=5.4
+	mu99deg ~ 17.5 mag/arcsec^2 => mmax=4.7
+	mu97.5deg ~ 16 mag/arcsec^2 => ? let's keep it rough
+    */
+
+    double mag_nakedeye = 6.2;
+    double mag_twilight_astro = 5.4;
+    double mag_twilight_nautic = 4.7;
+
+    // sirius, brightest star (not brightest object)
+    double mag_min = -1.46;
+    
     int phase;
 
     // determine which star structure to draw
-    if ( sun_angle > (SGD_PI_2 + 10.0 * SGD_DEGREES_TO_RADIANS ) ) {
-        // deep night
+    if ( sun_angle > (SGD_PI_2 + 18.0 * SGD_DEGREES_TO_RADIANS ) ) {
+        // deep night, atmosphere is not lighten by the sun
         factor = 1.0;
-        cutoff = 4.5;
+        cutoff = mag_nakedeye;
         phase = 0;
-    } else if ( sun_angle > (SGD_PI_2 + 8.8 * SGD_DEGREES_TO_RADIANS ) ) {
+    } else if ( sun_angle > (SGD_PI_2 + 12.0 * SGD_DEGREES_TO_RADIANS ) ) {
+        // less than 18deg and more than 12deg is astronomical twilight
         factor = 1.0;
-        cutoff = 3.8;
+        cutoff = mag_twilight_astro;
         phase = 1;
+    } else if ( sun_angle > (SGD_PI_2 + 9.0 * SGD_DEGREES_TO_RADIANS ) ) {
+        // less 12deg and more than 6deg is is nautical twilight 
+        factor = 1.0;
+        cutoff = mag_twilight_nautic;
+        phase = 2;
     } else if ( sun_angle > (SGD_PI_2 + 7.5 * SGD_DEGREES_TO_RADIANS ) ) {
         factor = 0.95;
         cutoff = 3.1;
-        phase = 2;
+        phase = 3;
     } else if ( sun_angle > (SGD_PI_2 + 7.0 * SGD_DEGREES_TO_RADIANS ) ) {
         factor = 0.9;
         cutoff = 2.4;
-        phase = 3;
+        phase = 4;
     } else if ( sun_angle > (SGD_PI_2 + 6.5 * SGD_DEGREES_TO_RADIANS ) ) {
         factor = 0.85;
         cutoff = 1.8;
-        phase = 4;
+        phase = 5;
     } else if ( sun_angle > (SGD_PI_2 + 6.0 * SGD_DEGREES_TO_RADIANS ) ) {
         factor = 0.8;
         cutoff = 1.2;
-        phase = 5;
+        phase = 6;
     } else if ( sun_angle > (SGD_PI_2 + 5.5 * SGD_DEGREES_TO_RADIANS ) ) {
         factor = 0.75;
         cutoff = 0.6;
-        phase = 6;
+        phase = 7;
     } else {
         // early dusk or late dawn
         factor = 0.7;
         cutoff = 0.0;
-        phase = 7;
+        phase = 8;
     }
-
-    if( phase != old_phase ) {
+    
+    if (_cutoffProperty) {
+        double propCutoff = _cutoffProperty->getDoubleValue();
+        cutoff = std::min(propCutoff, cutoff);
+    }
+    
+    if ((phase != old_phase) || (cutoff != _cachedCutoff)) {
 	// cout << "  phase change, repainting stars, num = " << num << endl;
         old_phase = phase;
+        _cachedCutoff = cutoff;
+
         for ( int i = 0; i < num; ++i ) {
             // if ( star_data[i][2] < min ) { min = star_data[i][2]; }
             // if ( star_data[i][2] > max ) { max = star_data[i][2]; }
 
-            // magnitude ranges from -1 (bright) to 4 (dim).  The
+            // magnitude ranges from -1 (bright) to 6 (dim).  The
             // range of star and planet magnitudes can actually go
             // outside of this, but for our purpose, if it is brighter
-            // that -1, we'll color it full white/alpha anyway and 4
-            // is a convenient cutoff point which keeps the number of
-            // stars drawn at about 500.
+            // that magmin, we'll color it full white/alpha anyway
 
             // color (magnitude)
             mag = star_data[i][2];
             if ( mag < cutoff ) {
-                nmag = ( 4.5 - mag ) / 5.5; // translate to 0 ... 1.0 scale
+                nmag = ( cutoff - mag ) / (cutoff - mag_min); // translate to 0 ... 1.0 scale
                 alpha = nmag * 0.85 + 0.15; // translate to a 0.15 ... 1.0 scale
                 alpha *= factor;          // dim when the sun is brighter
             } else {
@@ -190,11 +230,8 @@ bool SGStars::repaint( double sun_angle, int num, const SGVec3d star_data[] ) {
             if (alpha < 0.0) { alpha = 0.0; }
 
             (*cl)[i] = osg::Vec4(1, 1, 1, alpha);
-            // cout << "alpha[" << i << "] = " << alpha << endl;
         }
         cl->dirty();
-    } else {
-	// cout << "  no phase change, skipping" << endl;
     }
 
     // cout << "min = " << min << " max = " << max << " count = " << num 

simgear/scene/sky/stars.hxx

@@ -33,18 +33,20 @@
 
 #include <simgear/math/SGMath.hxx>
 #include <simgear/structure/SGReferenced.hxx>
-
+#include <simgear/props/propsfwd.hxx>
 
 class SGStars : public SGReferenced {
 
     osg::ref_ptr<osg::Vec4Array> cl;
 
     int old_phase;		// data for optimization
-
+    
+    double _cachedCutoff = 0.0;
+    SGPropertyNode_ptr _cutoffProperty;
 public:
 
     // Constructor
-    SGStars( void );
+    SGStars( SGPropertyNode* props = nullptr);
 
     // Destructor
     ~SGStars( void );

simgear/scene/tgdb/ReaderWriterSPT.cxx

@@ -25,6 +25,7 @@
 
 #include <cassert>
 
+#include <osg/Version>
 #include <osg/CullFace>
 #include <osg/PagedLOD>
 #include <osg/MatrixTransform>
@@ -208,7 +209,11 @@ ReaderWriterSPT::readObject(const std::string& fileName, const osgDB::Options* o
         imageFileName = osgDB::concatPaths(imageFileName, "Globe");
         imageFileName = osgDB::concatPaths(imageFileName, "world.topo.bathy.200407.3x4096x2048.png");
     }
+#if OSG_VERSION_LESS_THAN(3,4,0)
     if (osg::Image* image = osgDB::readImageFile(imageFileName, options)) {
+#else
+    if (osg::Image* image = osgDB::readRefImageFile(imageFileName, options)) {
+#endif
         osg::Texture2D* texture = new osg::Texture2D;
         texture->setImage(image);
         texture->setWrap(osg::Texture2D::WRAP_S, osg::Texture2D::REPEAT);
@@ -256,7 +261,11 @@ ReaderWriterSPT::createTree(const BucketBox& bucketBox, const LocalOptions& opti
     if (bucketBox.getIsBucketSize()) {
         std::string fileName;
         fileName = bucketBox.getBucket().gen_index_str() + std::string(".stg");
+#if OSG_VERSION_LESS_THAN(3,4,0)
+        return osgDB::readNodeFile(fileName, options._options);
+#else
         return osgDB::readRefNodeFile(fileName, options._options);
+#endif
     } else if (!topLevel && options.isPageLevel(bucketBox.getStartLevel())) {
         return createPagedLOD(bucketBox, options);
     } else {
@@ -314,7 +323,11 @@ ReaderWriterSPT::createPagedLOD(const BucketBox& bucketBox, const LocalOptions&
         std::string fileName = osgDB::findDataFile(lodPath + extensions[i], options._options);
         if (fileName.empty())
             continue;
+#if OSG_VERSION_LESS_THAN(3,4,0)
+        osg::ref_ptr<osg::Node> node = osgDB::readNodeFile(fileName, options._options);
+#else
         osg::ref_ptr<osg::Node> node = osgDB::readRefNodeFile(fileName, options._options);
+#endif
         if (!node.valid())
             continue;
         pagedLOD->addChild(node.get(), range, std::numeric_limits<float>::max());
@@ -412,7 +425,11 @@ ReaderWriterSPT::getLowLODStateSet(const LocalOptions& options) const
     localOptions = static_cast<osgDB::Options*>(options._options->clone(osg::CopyOp()));
     localOptions->setObjectCacheHint(osgDB::Options::CACHE_ALL);
 
+#if OSG_VERSION_LESS_THAN(3,4,0)
+    osg::ref_ptr<osg::Object> object = osgDB::readObjectFile("state.spt", localOptions.get());
+#else
     osg::ref_ptr<osg::Object> object = osgDB::readRefObjectFile("state.spt", localOptions.get());
+#endif
     if (!dynamic_cast<osg::StateSet*>(object.get()))
         return 0;
 

simgear/scene/tgdb/ReaderWriterSTG.cxx

@@ -50,6 +50,8 @@
 #include <simgear/scene/util/SGReaderWriterOptions.hxx>
 #include <simgear/scene/tgdb/apt_signs.hxx>
 #include <simgear/scene/tgdb/obj.hxx>
+#include <simgear/scene/material/matlib.hxx>
+#include <simgear/scene/tgdb/SGBuildingBin.hxx>
 
 #include "SGOceanTile.hxx"
 
@@ -59,6 +61,7 @@
 #define ROAD_DETAILED "OBJECT_ROAD_DETAILED"
 #define RAILWAY_ROUGH "OBJECT_RAILWAY_ROUGH"
 #define RAILWAY_DETAILED "OBJECT_RAILWAY_DETAILED"
+#define BUILDING_LIST "BUILDING_LIST"
 
 namespace simgear {
 
@@ -121,6 +124,12 @@ struct ReaderWriterSTG::_ModelBin {
         double _hdg;
         int _size;
     };
+    struct _BuildingList {
+      _BuildingList() : _lon(0), _lat(0), _elev(0) { }
+      std::string _filename;
+      std::string _material_name;
+      double _lon, _lat, _elev;
+    };
 
     class DelayLoadReadFileCallback : public OptionsReadFileCallback {
 
@@ -147,7 +156,11 @@ struct ReaderWriterSTG::_ModelBin {
                 proxy->setCenterMode(osg::ProxyNode::UNION_OF_BOUNDING_SPHERE_AND_USER_DEFINED);
                 node = proxy;
             } else {
+#if OSG_VERSION_LESS_THAN(3,4,0)
+                node = osgDB::readNodeFile(o._name, o._options.get());
+#else
                 node = osgDB::readRefNodeFile(o._name, o._options.get());
+#endif
                 if (!node.valid()) {
                     SG_LOG(SG_TERRAIN, SG_ALERT, o._errorLocation << ": Failed to load "
                            << o._token << " '" << o._name << "'");
@@ -185,8 +198,6 @@ struct ReaderWriterSTG::_ModelBin {
       };
       typedef QuadTreeBuilder<osg::LOD*, _ObjectStatic, MakeQuadLeaf, AddModelLOD,
                               GetModelLODCoord>  STGObjectsQuadtree;
-
-
     public:
         virtual osgDB::ReaderWriter::ReadResult
         readNode(const std::string&, const osgDB::Options*)
@@ -203,12 +214,40 @@ struct ReaderWriterSTG::_ModelBin {
             if (signBuilder.getSignsGroup())
                 group->addChild(signBuilder.getSignsGroup());
 
+            if (_buildingList.size() > 0) {
+                SGMaterialLibPtr matlib = _options->getMaterialLib();
+                bool useVBOs = (_options->getPluginStringData("SimGear::USE_VBOS") == "ON");
+
+                if (!matlib) {
+                    SG_LOG( SG_TERRAIN, SG_ALERT, "Unable to get materials definition for buildings");
+                } else {
+                    for (std::list<_BuildingList>::iterator i = _buildingList.begin(); i != _buildingList.end(); ++i) {
+                        // Build buildings for each list of buildings
+                        SGGeod geodPos = SGGeod::fromDegM(i->_lon, i->_lat, 0.0);
+                        SGMaterial* mat = matlib->find(i->_material_name, geodPos);
+                        SGPath path = SGPath(i->_filename);
+                        SGBuildingBin* buildingBin = new SGBuildingBin(path, mat, useVBOs);
+
+                        SGBuildingBinList buildingBinList;
+                        buildingBinList.push_back(buildingBin);
+
+                        osg::MatrixTransform* matrixTransform;
+                        matrixTransform = new osg::MatrixTransform(makeZUpFrame(SGGeod::fromDegM(i->_lon, i->_lat, i->_elev)));
+                        matrixTransform->setName("rotateBuildings");
+                        matrixTransform->setDataVariance(osg::Object::STATIC);
+                        matrixTransform->addChild(createRandomBuildings(buildingBinList, osg::Matrix::identity(), _options));
+                        group->addChild(matrixTransform);
+                    }
+                }
+            }
+
             return group.release();
         }
 
         mt _seed;
         std::list<_ObjectStatic> _objectStaticList;
         std::list<_Sign> _signList;
+        std::list<_BuildingList> _buildingList;
 
         /// The original options to use for this bunch of models
         osg::ref_ptr<SGReaderWriterOptions> _options;
@@ -313,10 +352,16 @@ struct ReaderWriterSTG::_ModelBin {
             return false;
         }
 
+        // starting with 2018.3 we will use deltas rather than absolutes as it is more intuitive for the user
+        // and somewhat easier to visualise
+        double detailedRange = atof(options->getPluginStringData("SimGear::LOD_RANGE_DETAILED").c_str());
+        double bareRangeDelta = atof(options->getPluginStringData("SimGear::LOD_RANGE_BARE").c_str());
+        double roughRangeDelta = atof(options->getPluginStringData("SimGear::LOD_RANGE_ROUGH").c_str());
+
         // Determine object ranges.  Mesh size of 2000mx2000m needs to be accounted for.
-        _object_range_bare = 1414.0f + atof(options->getPluginStringData("SimGear::LOD_RANGE_BARE").c_str());
-        _object_range_rough = 1414.0f + atof(options->getPluginStringData("SimGear::LOD_RANGE_ROUGH").c_str());
-        _object_range_detailed = 1414.0f + atof(options->getPluginStringData("SimGear::LOD_RANGE_DETAILED").c_str());
+        _object_range_detailed = 1414.0f + detailedRange;
+        _object_range_bare = _object_range_detailed + bareRangeDelta;
+        _object_range_rough = _object_range_detailed + roughRangeDelta;
 
         SG_LOG(SG_TERRAIN, SG_INFO, "Loading stg file " << absoluteFileName);
 
@@ -483,6 +528,13 @@ struct ReaderWriterSTG::_ModelBin {
                     obj._options = opt;
                     checkInsideBucket(absoluteFileName, obj._lon, obj._lat);
                     _objectStaticList.push_back(obj);
+                } else if (token == BUILDING_LIST) {
+                  _BuildingList buildinglist;
+                  buildinglist._filename = path.local8BitStr();
+                  in >> buildinglist._material_name >> buildinglist._lon >> buildinglist._lat >> buildinglist._elev;
+                  checkInsideBucket(absoluteFileName, buildinglist._lon, buildinglist._lat);
+                  _buildingListList.push_back(buildinglist);
+                  //SG_LOG(SG_TERRAIN, SG_ALERT, "Building list: " << buildinglist._filename << " " << buildinglist._material_name << " " << buildinglist._lon << " " << buildinglist._lat);
                 } else {
                     SG_LOG( SG_TERRAIN, SG_ALERT, absoluteFileName
                             << ": Unknown token '" << token << "'" );
@@ -505,7 +557,11 @@ struct ReaderWriterSTG::_ModelBin {
         if (_foundBase) {
             for (auto stgObject : _objectList) {
                 osg::ref_ptr<osg::Node> node;
+#if OSG_VERSION_LESS_THAN(3,4,0)
+                node = osgDB::readNodeFile(stgObject._name, stgObject._options.get());
+#else
                 node = osgDB::readRefNodeFile(stgObject._name, stgObject._options.get());
+#endif
                 if (!node.valid()) {
                     SG_LOG(SG_TERRAIN, SG_ALERT, stgObject._errorLocation << ": Failed to load "
                            << stgObject._token << " '" << stgObject._name << "'");
@@ -538,7 +594,7 @@ struct ReaderWriterSTG::_ModelBin {
             i->_elev += elevation(*terrainGroup, SGGeod::fromDeg(i->_lon, i->_lat));
         }
 
-        if (_objectStaticList.empty() && _signList.empty()) {
+        if (_objectStaticList.empty() && _signList.empty() && (_buildingListList.size() == 0)) {
             // The simple case, just return the terrain group
             return terrainGroup.release();
         } else {
@@ -553,6 +609,7 @@ struct ReaderWriterSTG::_ModelBin {
             // we just need to know about the read file callback that itself holds the data
             osg::ref_ptr<DelayLoadReadFileCallback> readFileCallback = new DelayLoadReadFileCallback;
             readFileCallback->_objectStaticList = _objectStaticList;
+            readFileCallback->_buildingList = _buildingListList;
             readFileCallback->_signList = _signList;
             readFileCallback->_options = options;
             readFileCallback->_bucket = bucket;
@@ -563,9 +620,11 @@ struct ReaderWriterSTG::_ModelBin {
             pagedLOD->setFileName(pagedLOD->getNumChildren(), "Dummy name - use the stored data in the read file callback");
 
             // Objects may end up displayed up to 2x the object range.
-            pagedLOD->setRange(pagedLOD->getNumChildren(), 0, 2.0 * _object_range_rough + SG_TILE_RADIUS);
+            pagedLOD->setRange(pagedLOD->getNumChildren(), 0, 2.0 * _object_range_rough);
+            pagedLOD->setRadius(SG_TILE_RADIUS);
             SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile PagedLOD Center: " << pagedLOD->getCenter().x() << "," << pagedLOD->getCenter().y() << "," << pagedLOD->getCenter().z() );
-            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile PagedLOD Range: " << (2.0 * _object_range_rough + SG_TILE_RADIUS));
+            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile PagedLOD Range: " << (2.0 * _object_range_rough));
+            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile PagedLOD Radius: " << SG_TILE_RADIUS);
             return pagedLOD;
         }
     }
@@ -577,6 +636,7 @@ struct ReaderWriterSTG::_ModelBin {
     std::list<_Object> _objectList;
     std::list<_ObjectStatic> _objectStaticList;
     std::list<_Sign> _signList;
+    std::list<_BuildingList> _buildingListList;
 };
 
 ReaderWriterSTG::ReaderWriterSTG()

simgear/scene/tgdb/SGBuildingBin.cxx

@@ -44,12 +44,11 @@
 #include <osgDB/FileUtils>
 
 #include <simgear/debug/logstream.hxx>
+#include <simgear/io/iostreams/sgstream.hxx>
 #include <simgear/math/SGLimits.hxx>
 #include <simgear/math/SGMisc.hxx>
 #include <simgear/math/sg_random.h>
 #include <simgear/misc/sg_path.hxx>
-#include <simgear/scene/material/Effect.hxx>
-#include <simgear/scene/material/EffectGeode.hxx>
 #include <simgear/scene/model/model.hxx>
 #include <simgear/props/props.hxx>
 
@@ -107,6 +106,59 @@ BuildingBoundingBoxCallback::computeBound(const Drawable& drawable) const
     return bb;
 }
 
+  // Set up a BuildingBin from a file containing a list of individual building
+  // positions.
+  SGBuildingBin::SGBuildingBin(const SGPath& absoluteFileName, const SGMaterial *mat, bool useVBOs) :
+    SGBuildingBin::SGBuildingBin(mat, useVBOs)
+  {
+    if (!absoluteFileName.exists()) {
+      SG_LOG(SG_TERRAIN, SG_ALERT, "Building list file " << absoluteFileName << " does not exist.");
+      return;
+    }
+
+    sg_gzifstream stream(absoluteFileName);
+    if (!stream.is_open()) {
+      SG_LOG(SG_TERRAIN, SG_ALERT, "Unable to open " << absoluteFileName << " does not exist.");
+      return;
+    }
+
+    while (!stream.eof()) {
+      // read a line.  Each line defines a single builing position, and may have
+      // a comment, starting with #
+      std::string line;
+      std::getline(stream, line);
+
+      // strip comments
+      std::string::size_type hash_pos = line.find('#');
+      if (hash_pos != std::string::npos)
+          line.resize(hash_pos);
+
+      // and process further
+      std::stringstream in(line);
+
+      // Line format is X Y Z R T
+      // where:
+      // X,Y,Z are the cartesian coordinates of the bottom SW corner of the building.  +X is East, +Y is North
+      // R is the building rotation in degrees centered on the SW corner
+      // T is the building type [0, 1, 2] for SMALL, MEDIUM, LARGE
+      float x, y, z, r;
+      int t;
+      in >> x >> y >> z >> r >> t;
+
+      //SG_LOG(SG_TERRAIN, SG_ALERT, "Building entry " << x << " " << y << " " << z << " " << t );
+      SGVec3f p = SGVec3f(x,y,z);
+      BuildingType type = BuildingType::SMALL;
+      if (t == 1)  type = BuildingType::MEDIUM;
+      if (t == 2)  type = BuildingType::LARGE;
+
+      // Rotation is in the file as degrees, but in the datastructure normalized
+      // to 0.0 - 1.0
+      insert(p, (float) (r / 360.0f), type);
+    }
+
+    stream.close();
+  };
+
   // Set up the building set based on the material definitions
   SGBuildingBin::SGBuildingBin(const SGMaterial *mat, bool useVBOs) {
 
@@ -165,7 +217,7 @@ BuildingBoundingBoxCallback::computeBound(const Drawable& drawable) const
       if (useVBOs) {
           sharedGeometry->setUseVertexBufferObjects(true);
       }
-      
+
       for (unsigned int j = 0; j < BUILDING_SET_SIZE; j++) {
         float width;
         float depth;
@@ -820,8 +872,8 @@ BuildingBoundingBoxCallback::computeBound(const Drawable& drawable) const
   };
 
   // This actually returns a MatrixTransform node. If we rotate the whole
-  // forest into the local Z-up coordinate system we can reuse the
-  // primitive building geometry for all the forests of the same type.
+  // set of buildings into the local Z-up coordinate system we can reuse the
+  // primitive building geometry for all the buildings of the same type.
   osg::Group* createRandomBuildings(SGBuildingBinList& buildings, const osg::Matrix& transform,
                            const SGReaderWriterOptions* options)
   {

simgear/scene/tgdb/SGBuildingBin.hxx

@@ -36,10 +36,12 @@
 #include <osg/Material>
 #include <osg/CullFace>
 
-
 #include <simgear/scene/util/OsgMath.hxx>
 #include <simgear/scene/material/mat.hxx>
 
+#include <simgear/scene/material/Effect.hxx>
+#include <simgear/scene/material/EffectGeode.hxx>
+
 #include <simgear/scene/util/QuadTreeBuilder.hxx>
 #include <simgear/scene/util/RenderConstants.hxx>
 #include <simgear/scene/util/StateAttributeFactory.hxx>
@@ -169,6 +171,7 @@ private:
 public:
 
   SGBuildingBin(const SGMaterial *mat, bool useVBOs);
+  SGBuildingBin(const SGPath& absoluteFileName, const SGMaterial *mat, bool useVBOs);
 
   ~SGBuildingBin() {
     smallBuildings.clear();

simgear/scene/tgdb/SGTileDetailsCallback.hxx

@@ -689,10 +689,13 @@ public:
                 triangleBuildingList.clear();
             }
             
-            SG_LOG(SG_TERRAIN, SG_DEBUG, "Random Buildings: " << ((bin) ? bin->getNumBuildings() : 0));
-            SG_LOG(SG_TERRAIN, SG_DEBUG, "  Dropped due to mask: " << mask_dropped);
-            SG_LOG(SG_TERRAIN, SG_DEBUG, "  Dropped due to random object: " << random_dropped);
-            SG_LOG(SG_TERRAIN, SG_DEBUG, "  Dropped due to other buildings: " << building_dropped);
+            const int numBuildings = (bin) ? bin->getNumBuildings() : 0;
+            if (numBuildings > 0) {
+                SG_LOG(SG_TERRAIN, SG_DEBUG, "computed Random Buildings: " << numBuildings);
+                SG_LOG(SG_TERRAIN, SG_DEBUG, "  Dropped due to mask: " << mask_dropped);
+                SG_LOG(SG_TERRAIN, SG_DEBUG, "  Dropped due to random object: " << random_dropped);
+                SG_LOG(SG_TERRAIN, SG_DEBUG, "  Dropped due to other buildings: " << building_dropped);
+            }
         }
     }
     

simgear/scene/tgdb/TreeBin.cxx

@@ -230,12 +230,14 @@ void addTreeToLeafGeode(Geode* geode, const SGVec3f& p, const SGVec3f& t)
         posArray->insert(posArray->end(), 4, pos);
 
         size_t numVerts = posArray->size();
-        int imax = 2;
+        unsigned int imax = 2;
         if (use_tree_shadows) { imax = 3; }
-        for (int i = 0; i < imax; ++i) {
-            DrawArrays* primSet = static_cast<DrawArrays*>(geom->getPrimitiveSet(i));
-            if(primSet != nullptr)
-                primSet->setCount(numVerts);
+        for (unsigned int i = 0; i < imax; ++i) {
+            if (i < geom->getNumPrimitiveSets()) {
+                DrawArrays* primSet = static_cast<DrawArrays*>(geom->getPrimitiveSet(i));
+                if (primSet != nullptr)
+                    primSet->setCount(numVerts);
+            }
         }
     }
 }

simgear/scene/util/CMakeLists.txt

@@ -16,6 +16,7 @@ set(HEADERS
     RenderConstants.hxx
     SGDebugDrawCallback.hxx
     SGEnlargeBoundingBox.hxx
+    SGImageUtils.hxx
     SGNodeMasks.hxx
     SGPickCallback.hxx
     SGReaderWriterOptions.hxx
@@ -44,6 +45,7 @@ set(SOURCES
     PrimitiveUtils.cxx
     QuadTreeBuilder.cxx
     SGEnlargeBoundingBox.cxx
+    SGImageUtils.cxx
     SGReaderWriterOptions.cxx
     SGSceneFeatures.cxx
     SGSceneUserData.cxx

simgear/scene/util/SGImageUtils.hxx

@@ -0,0 +1,542 @@
+  /* -*-c++-*- */
+  /* ImageUtils: copied from  osgEarth - Geospatial SDK for OpenSceneGraph
+  * Copyright 2018 Pelican Mapping
+  * http://osgearth.org
+  *
+  * osgEarth is free software; you can redistribute it and/or modify
+  * it under the terms of the GNU Lesser General Public License as published by
+  * the Free Software Foundation; either version 2 of the License, or
+  * (at your option) any later version.
+  *
+  * This program is distributed in the hope that it will be useful,
+  * but WITHOUT ANY WARRANTY; without even the implied warranty of
+  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  * GNU Lesser General Public License for more details.
+  *
+  * You should have received a copy of the GNU Lesser General Public License
+  * along with this program.  If not, see <http://www.gnu.org/licenses/>
+  */
+
+#ifndef SIMGEAR_IMAGEUTILS_H
+#define SIMGEAR_IMAGEUTILS_H
+
+#include <osg/Image>
+#include <osg/Texture>
+#include <osg/GL>
+#include <osg/NodeVisitor>
+#include <osgDB/ReaderWriter>
+#include <vector>
+
+  //These formats were not added to OSG until after 2.8.3 so we need to define them to use them.
+#ifndef GL_EXT_texture_compression_rgtc
+#define GL_COMPRESSED_RED_RGTC1_EXT                0x8DBB
+#define GL_COMPRESSED_SIGNED_RED_RGTC1_EXT         0x8DBC
+#define GL_COMPRESSED_RED_GREEN_RGTC2_EXT          0x8DBD
+#define GL_COMPRESSED_SIGNED_RED_GREEN_RGTC2_EXT   0x8DBE
+#endif
+
+#ifndef GL_IMG_texture_compression_pvrtc
+#define GL_COMPRESSED_RGB_PVRTC_4BPPV1_IMG      0x8C00
+#define GL_COMPRESSED_RGB_PVRTC_2BPPV1_IMG      0x8C01
+#define GL_COMPRESSED_RGBA_PVRTC_4BPPV1_IMG     0x8C02
+#define GL_COMPRESSED_RGBA_PVRTC_2BPPV1_IMG     0x8C03
+#endif
+
+namespace simgear
+{
+    class  ImageUtils
+    {
+    public:
+        /**
+        * Clones an image.
+        *
+        * Use this instead of the osg::Image copy construtor, which keeps the referenced to
+        * its underlying BufferObject around. Calling dirty() on the new clone appears to
+        * help, but just call this method instead to be sure.
+        */
+        static osg::Image* cloneImage(const osg::Image* image);
+
+        /**
+        * Tweaks an image for consistency. OpenGL allows enums like "GL_RGBA" et.al. to be
+        * used in the internal texture format, when really "GL_RGBA8" is the proper things
+        * to use. This method accounts for that. Some parts of osgEarth (like the texture-
+        * array compositor) rely on the internal texture format being correct.
+        * (http://http.download.nvidia.com/developer/Papers/2005/Fast_Texture_Transfers/Fast_Texture_Transfers.pdf)
+        */
+        static void fixInternalFormat(osg::Image* image);
+
+        /**
+        * Marks an image as containing un-normalized data values.
+        *
+        * Normally the values in an image are "normalized", i.e. scaled so they are in the
+        * range [0..1]. This is normal for color values. But when the image is being used
+        * for coverage data (a value lookup table) it is desireable to store the raw
+        * values instead.
+        */
+        static void markAsUnNormalized(osg::Image* image, bool value);
+
+        /** Inverse of above. */
+        static void markAsNormalized(osg::Image* image, bool value) { markAsUnNormalized(image, !value); }
+
+        /**
+        * Whether the image has been marked as containing un-normalized values.
+        */
+        static bool isUnNormalized(const osg::Image* image);
+
+        /**
+        * Whether the image has been marked as containing normalized values.
+        */
+        static bool isNormalized(const osg::Image* image) { return !isUnNormalized(image); }
+
+        /**
+        * Copys a portion of one image into another.
+        */
+        static bool copyAsSubImage(
+            const osg::Image* src,
+            osg::Image*       dst,
+            int dst_start_col, int dst_start_row);
+
+        /**
+        * Resizes an image. Returns a new image, leaving the input image unaltered.
+        *
+        * Note. If the output parameter is NULL, this method will allocate a new image and
+        * resize into that new image. If the output parameter is non-NULL, this method will
+        * assume that the output image is already allocated to the proper size, and will
+        * do a resize+copy into that image. In the latter case, it is your responsibility
+        * to make sure the output image is allocated to the proper size.
+        *
+        * If the output parameter is non-NULL, then the mipmapLevel is also considered.
+        * This lets you resize directly into a particular mipmap level of the output image.
+        */
+        static bool resizeImage(
+            const osg::Image* input,
+            unsigned int new_s, unsigned int new_t,
+            osg::ref_ptr<osg::Image>& output,
+            unsigned int mipmapLevel = 0, bool bilinear = true);
+
+        /**
+        * Crops the input image to the dimensions provided and returns a
+        * new image. Returns a new image, leaving the input image unaltered.
+        * Note:  The input destination bounds are modified to reflect the bounds of the
+        *        actual output image.  Due to the fact that you cannot crop in the middle of a pixel
+        *        The specified destination extents and the output extents may vary slightly.
+        *@param src_minx
+        *       The minimum x coordinate of the input image.
+        *@param src_miny
+        *       The minimum y coordinate of the input image.
+        *@param src_maxx
+        *       The maximum x coordinate of the input image.
+        *@param src_maxy
+        *       The maximum y coordinate of the input image.
+        *@param dst_minx
+        *       The desired minimum x coordinate of the cropped image.
+        *@param dst_miny
+        *       The desired minimum y coordinate of the cropped image.
+        *@param dst_maxx
+        *       The desired maximum x coordinate of the cropped image.
+        *@param dst_maxy
+        *       The desired maximum y coordinate of the cropped image.
+        */
+        static osg::Image* cropImage(
+            const osg::Image* image,
+            double src_minx, double src_miny, double src_maxx, double src_maxy,
+            double &dst_minx, double &dst_miny, double &dst_maxx, double &dst_maxy);
+
+        /**
+        * Creates an Image that "blends" two images into a new image in which "primary"
+        * occupies mipmap level 0, and "secondary" occupies all the other mipmap levels.
+        *
+        * WARNING: this method assumes that primary and seconday are the same exact size
+        * and the same exact format.
+        */
+        static osg::Image* createMipmapBlendedImage(
+            const osg::Image* primary,
+            const osg::Image* secondary);
+
+        /**
+        * Creates a new image containing mipmaps built with nearest-neighbor
+        * sampling.
+        */
+        static osg::Image* buildNearestNeighborMipmaps(
+            const osg::Image* image);
+
+        /**
+        * Blends the "src" image into the "dest" image, based on the "a" value.
+        * The two images must be the same.
+        */
+        static bool mix(osg::Image* dest, const osg::Image* src, float a);
+
+        /**
+        * Creates and returns a copy of the input image after applying a
+        * sharpening filter. Returns a new image, leaving the input image unaltered.
+        */
+        static osg::Image* createSharpenedImage(const osg::Image* image);
+
+        /**
+        * For each "layer" in the input image (each bitmap in the "r" dimension),
+        * create a new, separate image with r=1. If the input image is r=1, it is
+        * simply placed onto the output vector (no copy).
+        * Returns true upon sucess, false upon failure
+        */
+        static bool flattenImage(osg::Image* image, std::vector<osg::ref_ptr<osg::Image> >& output);
+
+        /**
+        * Gets whether the input image's dimensions are powers of 2.
+        */
+        static bool isPowerOfTwo(const osg::Image* image);
+
+        /**
+        * Gets a transparent, single pixel image used for a placeholder
+        */
+        static osg::Image* createEmptyImage();
+
+        /**
+        * Gets a transparent image used for a placeholder with the specified dimensions
+        */
+        static osg::Image* createEmptyImage(unsigned int s, unsigned int t);
+
+        /**
+        * Creates a one-pixel image.
+        */
+        static osg::Image* createOnePixelImage(const osg::Vec4& color);
+
+        /**
+        * Tests an image to see whether it's "empty", i.e. completely transparent,
+        * within an alpha threshold.
+        */
+        static bool isEmptyImage(const osg::Image* image, float alphaThreshold = 0.01);
+
+        /**
+        * Tests an image to see whether it's "single color", i.e. completely filled with a single color,
+        * within an threshold (threshold is tested on each channel).
+        */
+        static bool isSingleColorImage(const osg::Image* image, float threshold = 0.01);
+
+        /**
+        * Returns true if it is possible to convert the image to the specified
+        * format/datatype specification.
+        */
+        static bool canConvert(const osg::Image* image, GLenum pixelFormat, GLenum dataType);
+
+        /**
+        * Converts an image to the specified format.
+        */
+        static osg::Image* convert(const osg::Image* image, GLenum pixelFormat, GLenum dataType);
+
+        /**
+        *Converts the given image to RGB8
+        */
+        static osg::Image* convertToRGB8(const osg::Image* image);
+
+        /**
+        *Converts the given image to RGBA8
+        */
+        static osg::Image* convertToRGBA8(const osg::Image* image);
+
+        /**
+        * True if the two images are of the same format (pixel format, data type, etc.)
+        * though not necessarily the same size, depth, etc.
+        */
+        static bool sameFormat(const osg::Image* lhs, const osg::Image* rhs);
+
+        /**
+        * True if the two images have the same format AND size, and can therefore
+        * be used together in a texture array.
+        */
+        static bool textureArrayCompatible(const osg::Image* lhs, const osg::Image* rhs);
+
+        /**
+        *Compares the image data of two images and determines if they are equivalent
+        */
+        static bool areEquivalent(const osg::Image *lhs, const osg::Image *rhs);
+
+        /**
+        * Whether two colors are roughly equivalent.
+        */
+        static bool areRGBEquivalent(const osg::Vec4& lhs, const osg::Vec4& rhs, float epsilon = 0.01f) {
+            return
+                fabs(lhs.r() - rhs.r()) < epsilon &&
+                fabs(lhs.g() - rhs.g()) < epsilon &&
+                fabs(lhs.b() - rhs.b()) < epsilon;
+        }
+
+        /**
+        * Checks whether the image has an alpha component
+        */
+        static bool hasAlphaChannel(const osg::Image* image);
+
+        /**
+        * Checks whether an image has transparency; i.e. whether
+        * there are any pixels with an alpha component whole value
+        * falls below the specified threshold.
+        */
+        static bool hasTransparency(const osg::Image* image, float alphaThreshold = 1.0f);
+
+        /**
+        * Finds pixels with alpha less than [maxAlpha] and sets their color
+        * to match that or neighboring non-alpha pixels. This facilitates multipass
+        * blending or abutting tiles by overlapping them slightly. Specify "maxAlpha"
+        * as the maximum value to consider when searching for fully-transparent pixels.
+        *
+        * Returns false if there is no reader or writer for the image's format.
+        */
+        static bool featherAlphaRegions(osg::Image* image, float maxAlpha = 0.0f);
+
+        /**
+        * Converts an image (in place) to premultiplied-alpha format.
+        * Returns False is the conversion fails, e.g., if there is no reader
+        * or writer for the image format.
+        */
+        static bool convertToPremultipliedAlpha(osg::Image* image);
+
+        /**
+        * Checks whether the given image is compressed
+        */
+        static bool isCompressed(const osg::Image* image);
+
+        /**
+        * Generated a bump map image for the input image
+        */
+        static osg::Image* createBumpMap(const osg::Image* input);
+
+        /**
+        * Is it a floating-point texture format?
+        */
+        static bool isFloatingPointInternalFormat(GLint internalFormat);
+
+        /**
+        * Compute a texture compression format suitable for the image.
+        */
+        static bool computeTextureCompressionMode(
+            const osg::Image* image,
+            osg::Texture::InternalFormatMode& out_mode);
+
+
+        /**
+        * Bicubic upsampling in a quadrant. Target image is already allocated.
+        */
+        static bool bicubicUpsample(
+            const osg::Image* source,
+            osg::Image* target,
+            unsigned quadrant,
+            unsigned stride);
+
+        /**
+        *
+        */
+        static osg::Image* upSampleNN(const osg::Image* src, int quadrant);
+
+        /**
+        * Activates mipmapping for a texture image if the correct filters exist.
+        *
+        * If OSG has an ImageProcessor service installed, this method will use that
+        * to generate mipmaps. If not, the method will be a NOP and the GPU wil
+        * generate mipmaps (if necessary) upon GPU transfer.
+        */
+        static void activateMipMaps(osg::Texture* texture);
+
+        /**
+        * Gets an osgDB::ReaderWriter for the given input stream.
+        * Returns NULL if no ReaderWriter can be found.
+        */
+        static osgDB::ReaderWriter* getReaderWriterForStream(std::istream& stream);
+
+        /**
+        * Reads an osg::Image from the given input stream.
+        * Returns NULL if the image could not be read.
+        */
+        static osg::Image* readStream(std::istream& stream, const osgDB::Options* options);
+
+        /**
+        * Reads color data out of an image, regardles of its internal pixel format.
+        */
+        class  PixelReader
+        {
+        public:
+            /**
+            * Constructs a pixel reader. "Normalized" means that the values in the source
+            * image have been scaled to [0..1] and should be denormalized upon reading.
+            */
+            PixelReader(const osg::Image* image);
+
+            /** Sets an image to read. */
+            void setImage(const osg::Image* image);
+
+            /** Whether to use bilinear interpolation when reading with u,v coords (default=true) */
+            void setBilinear(bool value) { _bilinear = value; }
+
+            /** Whether PixelReader supports a given format/datatype combiniation. */
+            static bool supports(GLenum pixelFormat, GLenum dataType);
+
+            /** Whether PixelReader can read from the specified image. */
+            static bool supports(const osg::Image* image) {
+                return image && supports(image->getPixelFormat(), image->getDataType());
+            }
+
+            /** Reads a color from the image */
+            osg::Vec4 operator()(int s, int t, int r = 0, int m = 0) const {
+                return (*_reader)(this, s, t, r, m);
+            }
+
+            /** Reads a color from the image */
+            osg::Vec4 operator()(unsigned s, unsigned t, unsigned r = 0, int m = 0) const {
+                return (*_reader)(this, s, t, r, m);
+            }
+
+            /** Reads a color from the image by unit coords [0..1] */
+            osg::Vec4 operator()(float u, float v, int r = 0, int m = 0) const;
+            osg::Vec4 operator()(double u, double v, int r = 0, int m = 0) const;
+
+            // internals:
+            const unsigned char* data(int s = 0, int t = 0, int r = 0, int m = 0) const {
+                return m == 0 ?
+                    _image->data() + s*_colMult + t*_rowMult + r*_imageSize :
+                    _image->getMipmapData(m) + s*_colMult + t*(_rowMult >> m) + r*(_imageSize >> m);
+            }
+
+            typedef osg::Vec4(*ReaderFunc)(const PixelReader* ia, int s, int t, int r, int m);
+            ReaderFunc _reader;
+            const osg::Image* _image;
+            unsigned _colMult;
+            unsigned _rowMult;
+            unsigned _imageSize;
+            bool     _normalized;
+            bool     _bilinear;
+        };
+
+        /**
+        * Writes color data to an image, regardles of its internal pixel format.
+        */
+        class  PixelWriter
+        {
+        public:
+            /**
+            * Constructs a pixel writer. "Normalized" means the values are scaled to [0..1]
+            * before writing.
+            */
+            PixelWriter(osg::Image* image);
+
+            /** Whether PixelWriter can write to an image with the given format/datatype combo. */
+            static bool supports(GLenum pixelFormat, GLenum dataType);
+
+            /** Whether PixelWriter can write to non-const version of an image. */
+            static bool supports(const osg::Image* image) {
+                return image && supports(image->getPixelFormat(), image->getDataType());
+            }
+
+            /** Writes a color to a pixel. */
+            void operator()(const osg::Vec4& c, int s, int t, int r = 0, int m = 0) {
+                (*_writer)(this, c, s, t, r, m);
+            }
+
+            void f(const osg::Vec4& c, float s, float t, int r = 0, int m = 0) {
+                this->operator()(c,
+                    (int)(s * (float)(_image->s() - 1)),
+                    (int)(t * (float)(_image->t() - 1)),
+                    r, m);
+            }
+
+            // internals:
+            osg::Image* _image;
+            unsigned _colMult;
+            unsigned _rowMult;
+            unsigned _imageSize;
+            bool     _normalized;
+
+            unsigned char* data(int s = 0, int t = 0, int r = 0, int m = 0) const {
+                return m == 0 ?
+                    _image->data() + s*_colMult + t*_rowMult + r*_imageSize :
+                    _image->getMipmapData(m) + s*_colMult + t*(_rowMult >> m) + r*(_imageSize >> m);
+            }
+
+            typedef void(*WriterFunc)(const PixelWriter* iw, const osg::Vec4& c, int s, int t, int r, int m);
+            WriterFunc _writer;
+        };
+
+        /**
+        * Functor that visits every pixel in an image
+        */
+        template<typename T>
+        struct PixelVisitor : public T
+        {
+            /**
+            * Traverse an image, and call this method on the superclass:
+            *
+            *   bool operator(osg::Vec4& pixel);
+            *
+            * If that method returns true, write the value back at the same location.
+            */
+            void accept(osg::Image* image) {
+                PixelReader _reader(image);
+                PixelWriter _writer(image);
+                for (int r = 0; r<image->r(); ++r) {
+                    for (int t = 0; t<image->t(); ++t) {
+                        for (int s = 0; s<image->s(); ++s) {
+                            osg::Vec4f pixel = _reader(s, t, r);
+                            if ((*this)(pixel))
+                                _writer(pixel, s, t, r);
+                        }
+                    }
+                }
+            }
+
+            /**
+            * Traverse an image, and call this method on the superclass:
+            *
+            *   bool operator(const osg::Vec4& srcPixel, osg::Vec4& destPixel);
+            *
+            * If that method returns true, write destPixel back at the same location
+            * in the destination image.
+            */
+            void accept(const osg::Image* src, osg::Image* dest) {
+                PixelReader _readerSrc(src);
+                PixelReader _readerDest(dest);
+                PixelWriter _writerDest(dest);
+                for (int r = 0; r<src->r(); ++r) {
+                    for (int t = 0; t<src->t(); ++t) {
+                        for (int s = 0; s<src->s(); ++s) {
+                            const osg::Vec4f pixelSrc = _readerSrc(s, t, r);
+                            osg::Vec4f pixelDest = _readerDest(s, t, r);
+                            if ((*this)(pixelSrc, pixelDest))
+                                _writerDest(pixelDest, s, t, r);
+                        }
+                    }
+                }
+            }
+        };
+
+        /**
+        * Simple functor to copy pixels from one image to another.
+        *
+        * Usage:
+        *    PixelVisitor<CopyImage>().accept( fromImage, toImage );
+        */
+        struct CopyImage {
+            bool operator()(const osg::Vec4f& src, osg::Vec4f& dest) {
+                dest = src;
+                return true;
+            }
+        };
+    };
+
+    /** Visitor that finds and operates on textures and images */
+    class  TextureAndImageVisitor : public osg::NodeVisitor
+    {
+    public:
+        TextureAndImageVisitor();
+        virtual ~TextureAndImageVisitor() { }
+
+    public:
+        /** Visits a texture and, by default, all its components images */
+        virtual void apply(osg::Texture& texture);
+
+        /** Visits an image inside a texture */
+        virtual void apply(osg::Image& image) { }
+
+    public: // osg::NodeVisitor
+        virtual void apply(osg::Node& node);
+        virtual void apply(osg::StateSet& stateSet);
+    };
+}
+
+#endif //SIMGEAR_IMAGEUTILS_H

simgear/scene/util/SGReaderWriterOptions.hxx

@@ -38,12 +38,22 @@ namespace simgear
 
 class SGReaderWriterOptions : public osgDB::Options {
 public:
+    enum LoadOriginHint
+    {
+        ORIGIN_MODEL,
+        ORIGIN_EFFECTS,
+        ORIGIN_EFFECTS_NORMALIZED,
+    };
+
+    //SGReaderWriterOptions* cloneOptions(const osg::CopyOp& copyop = osg::CopyOp::SHALLOW_COPY) const { return static_cast<SGReaderWriterOptions*>(clone(copyop)); }
+
     SGReaderWriterOptions() :
         _materialLib(0),
         _load_panel(0),
         _model_data(0),
         _instantiateEffects(false),
-        _instantiateMaterialEffects(false)
+        _instantiateMaterialEffects(false),
+        _LoadOriginHint(ORIGIN_MODEL)
     { }
     SGReaderWriterOptions(const std::string& str) :
         osgDB::Options(str),
@@ -51,7 +61,8 @@ public:
         _load_panel(0),
         _model_data(0),
         _instantiateEffects(false),
-        _instantiateMaterialEffects(false)
+        _instantiateMaterialEffects(false),
+        _LoadOriginHint(ORIGIN_MODEL)
     { }
     SGReaderWriterOptions(const osgDB::Options& options,
                           const osg::CopyOp& copyop = osg::CopyOp::SHALLOW_COPY) :
@@ -60,7 +71,8 @@ public:
         _load_panel(0),
         _model_data(0),
         _instantiateEffects(false),
-        _instantiateMaterialEffects(false)
+        _instantiateMaterialEffects(false),
+        _LoadOriginHint(ORIGIN_MODEL)
     { }
     SGReaderWriterOptions(const SGReaderWriterOptions& options,
                           const osg::CopyOp& copyop = osg::CopyOp::SHALLOW_COPY) :
@@ -75,7 +87,8 @@ public:
         _instantiateEffects(options._instantiateEffects),
         _instantiateMaterialEffects(options._instantiateMaterialEffects),
         _materialName(options._materialName),
-        _sceneryPathSuffixes(options._sceneryPathSuffixes)
+        _sceneryPathSuffixes(options._sceneryPathSuffixes),
+        _LoadOriginHint(ORIGIN_MODEL)
     { }
 
     META_Object(simgear, SGReaderWriterOptions);
@@ -139,6 +152,13 @@ public:
     const SGGeod& getLocation() const
     { return _geod; }
 
+    // the load origin defines where the load request has come from.
+    // example usage; to allow the DDS Texture Cache (DTC) to ignore 
+    // any texture that is used in a shader, as these often have special values
+    // encoded into the channels that aren't suitable for conversion.
+    void setLoadOriginHint(LoadOriginHint _v) const { _LoadOriginHint = _v; } 
+    LoadOriginHint getLoadOriginHint() const { return _LoadOriginHint; } 
+
 protected:
     virtual ~SGReaderWriterOptions();
 
@@ -157,6 +177,7 @@ private:
     string _materialName;
     string_list _sceneryPathSuffixes;
     SGGeod _geod;
+    mutable LoadOriginHint _LoadOriginHint;
 };
 
 }

simgear/scene/util/SGSceneFeatures.cxx

@@ -40,6 +40,10 @@
 
 SGSceneFeatures::SGSceneFeatures() :
   _textureCompression(UseARBCompression),
+  _MaxTextureSize(4096),
+  _TextureCacheCompressionActive(true),
+  _TextureCacheCompressionActiveTransparent(true),
+  _TextureCacheActive(false),
   _shaderLights(true),
   _pointSpriteLights(true),
   _distanceAttenuationLights(true),

simgear/scene/util/SGSceneFeatures.hxx

@@ -1,102 +1,133 @@
 /* -*-c++-*-
- *
- * Copyright (C) 2006-2007 Mathias Froehlich 
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- *
- */
+*
+* Copyright (C) 2006-2007 Mathias Froehlich
+*
+* This program is free software; you can redistribute it and/or
+* modify it under the terms of the GNU General Public License as
+* published by the Free Software Foundation; either version 2 of the
+* License, or (at your option) any later version.
+*
+* This program is distributed in the hope that it will be useful, but
+* WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+* General Public License for more details.
+*
+* You should have received a copy of the GNU General Public License
+* along with this program; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+* MA 02110-1301, USA.
+*
+*/
 
 #ifndef SG_SCENE_FEATURES_HXX
 #define SG_SCENE_FEATURES_HXX
 
 #include <simgear/structure/SGReferenced.hxx>
+#include <string>
+#include <simgear/misc/sg_path.hxx>
 
 namespace osg { class Texture; }
 
 class SGSceneFeatures : public SGReferenced {
 public:
-  static SGSceneFeatures* instance();
+    static SGSceneFeatures* instance();
 
-  enum TextureCompression {
-    DoNotUseCompression,
-    UseARBCompression,
-    UseDXT1Compression,
-    UseDXT3Compression,
-    UseDXT5Compression
-  };
+    enum TextureCompression {
+        DoNotUseCompression,
+        UseARBCompression,
+        UseDXT1Compression,
+        UseDXT3Compression,
+        UseDXT5Compression
+    };
+    int getMaxTextureSize() const { return _MaxTextureSize; }
+    void setMaxTextureSize(const int maxTextureSize) { _MaxTextureSize = maxTextureSize; }
 
-  void setTextureCompression(TextureCompression textureCompression)
-  { _textureCompression = textureCompression; }
-  TextureCompression getTextureCompression() const
-  { return _textureCompression; }
-  void setTextureCompression(osg::Texture* texture) const;
+    SGPath getTextureCompressionPath() const { return _TextureCompressionPath; }
+    void setTextureCompressionPath(const SGPath path) { _TextureCompressionPath = path; }
 
-  void setEnablePointSpriteLights(bool enable)
-  { _pointSpriteLights = enable; }
-  bool getEnablePointSpriteLights() const
-  {
-      return _pointSpriteLights;
-  }
-  bool getEnablePointSpriteLights(unsigned contextId) const
-  {
-    if (!_pointSpriteLights)
-      return false;
-    return getHavePointSprites(contextId);
-  }
+    bool getTextureCacheActive() const { return _TextureCacheActive; }
+    void setTextureCacheActive(const bool val) { _TextureCacheActive = val; }
 
-  void setEnableDistanceAttenuationLights(bool enable)
-  { _distanceAttenuationLights = enable; }
-  bool getEnableDistanceAttenuationLights(unsigned contextId) const
-  {
-    if (!_distanceAttenuationLights)
-      return false;
-    return getHavePointParameters(contextId);
-  }
+    bool getTextureCacheCompressionActive() const { return _TextureCacheCompressionActive; }
+    void setTextureCacheCompressionActive(const bool val) { _TextureCacheCompressionActive = val; }
 
-  void setEnableShaderLights(bool enable)
-  { _shaderLights = enable; }
-  bool getEnableShaderLights(unsigned contextId) const
-  {
-    if (!_shaderLights)
-      return false;
-    return getHaveShaderPrograms(contextId);
-  }
-  
-  void setTextureFilter(int max) 
-  { _textureFilter = max; }
-  int getTextureFilter() const
-  { return _textureFilter; }
+    bool getTextureCacheCompressionActiveTransparent() const { return _TextureCacheCompressionActiveTransparent; }
+    void setTextureCacheCompressionActiveTransparent(const bool val) { _TextureCacheCompressionActiveTransparent = val; }
+
+    void setTextureCompression(TextureCompression textureCompression) { _textureCompression = textureCompression; }
+    TextureCompression getTextureCompression() const { return _textureCompression; }
+
+    // modify the texture compression on the texture parameter
+    void setTextureCompression(osg::Texture* texture) const;
+
+    void setEnablePointSpriteLights(bool enable)
+    {
+        _pointSpriteLights = enable;
+    }
+    bool getEnablePointSpriteLights() const
+    {
+        return _pointSpriteLights;
+    }
+    bool getEnablePointSpriteLights(unsigned contextId) const
+    {
+        if (!_pointSpriteLights)
+            return false;
+        return getHavePointSprites(contextId);
+    }
+
+    void setEnableDistanceAttenuationLights(bool enable)
+    {
+        _distanceAttenuationLights = enable;
+    }
+    bool getEnableDistanceAttenuationLights(unsigned contextId) const
+    {
+        if (!_distanceAttenuationLights)
+            return false;
+        return getHavePointParameters(contextId);
+    }
+
+    void setEnableShaderLights(bool enable)
+    {
+        _shaderLights = enable;
+    }
+    bool getEnableShaderLights(unsigned contextId) const
+    {
+        if (!_shaderLights)
+            return false;
+        return getHaveShaderPrograms(contextId);
+    }
+
+    void setTextureFilter(int max)
+    {
+        _textureFilter = max;
+    }
+    int getTextureFilter() const
+    {
+        return _textureFilter;
+    }
 
 protected:
-  bool getHavePointSprites(unsigned contextId) const;
-  bool getHaveFragmentPrograms(unsigned contextId) const;
-  bool getHaveVertexPrograms(unsigned contextId) const;
-  bool getHaveShaderPrograms(unsigned contextId) const;
-  bool getHavePointParameters(unsigned contextId) const;
+    bool getHavePointSprites(unsigned contextId) const;
+    bool getHaveFragmentPrograms(unsigned contextId) const;
+    bool getHaveVertexPrograms(unsigned contextId) const;
+    bool getHaveShaderPrograms(unsigned contextId) const;
+    bool getHavePointParameters(unsigned contextId) const;
 
 private:
-  SGSceneFeatures();
-  SGSceneFeatures(const SGSceneFeatures&);
-  SGSceneFeatures& operator=(const SGSceneFeatures&);
+    SGSceneFeatures();
+    SGSceneFeatures(const SGSceneFeatures&);
+    SGSceneFeatures& operator=(const SGSceneFeatures&);
 
-  TextureCompression _textureCompression;
-  bool _shaderLights;
-  bool _pointSpriteLights;
-  bool _distanceAttenuationLights;
-  int  _textureFilter;
+    TextureCompression _textureCompression;
+    int _MaxTextureSize;
+    SGPath _TextureCompressionPath;
+    bool _TextureCacheCompressionActive;
+    bool _TextureCacheCompressionActiveTransparent;
+    bool _TextureCacheActive;
+    bool _shaderLights;
+    bool _pointSpriteLights;
+    bool _distanceAttenuationLights;
+    int  _textureFilter;
 };
 
 #endif

simgear/scene/viewer/CMakeLists.txt

@@ -0,0 +1,17 @@
+set(HEADERS
+  ClusteredForward.hxx
+  Compositor.hxx
+  CompositorBuffer.hxx
+  CompositorPass.hxx
+  CompositorUtil.hxx
+  )
+
+set(SOURCES
+  ClusteredForward.cxx
+  Compositor.cxx
+  CompositorBuffer.cxx
+  CompositorPass.cxx
+  CompositorUtil.cxx
+  )
+
+simgear_scene_component(viewer scene/viewer "${SOURCES}" "${HEADERS}")

simgear/scene/viewer/ClusteredForward.cxx

@@ -0,0 +1,216 @@
+// Copyright (C) 2018  Fernando García Liñán <fernandogarcialinan@gmail.com>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Library General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301, USA
+
+#include "ClusteredForward.hxx"
+
+#include <osg/BufferIndexBinding>
+#include <osg/BufferObject>
+#include <osg/RenderInfo>
+#include <osg/Texture3D>
+#include <osg/TextureBuffer>
+#include <osg/Version>
+
+namespace simgear {
+namespace compositor {
+
+///// BEGIN DEBUG
+#define DATA_SIZE 24
+const GLfloat LIGHT_DATA[DATA_SIZE] = {
+    0.0, 0.0, -10.0, 1.0,   1.0, 0.0, 0.0, 1.0,
+    0.0, 0.0, 10.0, 1.0,   0.0, 1.0, 0.0, 1.0,
+    0.0, 1.0, 1.0, 1.0,   1.0, 1.0, 0.0, 1.0
+};
+
+#define MAX_LIGHT_INDICES 4096
+#define MAX_POINT_LIGHTS 256
+
+struct Light {
+    osg::Vec3 position;
+    float range;
+};
+
+#define NUM_LIGHTS 2
+Light LIGHT_LIST[NUM_LIGHTS] = {
+    {osg::Vec3(0.0f, 0.0f, -10.0f), 10.0f},
+    {osg::Vec3(0.0f, 0.0f,  5.0f), 1000.0f}
+};
+///// END DEBUG
+
+ClusteredForwardDrawCallback::ClusteredForwardDrawCallback() :
+    _initialized(false),
+    _tile_size(64),
+    _light_grid(new osg::Image),
+    _light_indices(new osg::Image),
+    _light_data(new osg::FloatArray(MAX_POINT_LIGHTS))
+{
+}
+
+void
+ClusteredForwardDrawCallback::operator()(osg::RenderInfo &renderInfo) const
+{
+    osg::Camera *camera = renderInfo.getCurrentCamera();
+    const osg::Viewport *vp = camera->getViewport();
+    const int width = vp->width();
+    const int height = vp->height();
+
+    // Round up
+    int n_htiles = (width  + _tile_size - 1) / _tile_size;
+    int n_vtiles = (height + _tile_size - 1) / _tile_size;
+
+    if (!_initialized) {
+        // Create and associate the light grid 3D texture
+        _light_grid->allocateImage(n_htiles, n_vtiles, 1,
+                                   GL_RGB_INTEGER_EXT, GL_UNSIGNED_SHORT);
+        _light_grid->setInternalTextureFormat(GL_RGB16UI_EXT);
+
+        osg::ref_ptr<osg::Texture3D> light_grid_tex = new osg::Texture3D;
+        light_grid_tex->setResizeNonPowerOfTwoHint(false);
+        light_grid_tex->setWrap(osg::Texture3D::WRAP_R, osg::Texture3D::CLAMP_TO_BORDER);
+        light_grid_tex->setWrap(osg::Texture3D::WRAP_S, osg::Texture3D::CLAMP_TO_BORDER);
+        light_grid_tex->setWrap(osg::Texture3D::WRAP_T, osg::Texture3D::CLAMP_TO_BORDER);
+        light_grid_tex->setFilter(osg::Texture3D::MIN_FILTER, osg::Texture3D::NEAREST);
+        light_grid_tex->setFilter(osg::Texture3D::MAG_FILTER, osg::Texture3D::NEAREST);
+        light_grid_tex->setImage(0, _light_grid.get());
+
+        camera->getOrCreateStateSet()->setTextureAttributeAndModes(
+            10, light_grid_tex.get(), osg::StateAttribute::ON);
+
+        // Create and associate the light indices TBO
+        _light_indices->allocateImage(4096, 1, 1, GL_RED_INTEGER_EXT, GL_UNSIGNED_SHORT);
+
+        osg::ref_ptr<osg::TextureBuffer> light_indices_tbo =
+            new osg::TextureBuffer;
+        light_indices_tbo->setInternalFormat(GL_R16UI);
+        light_indices_tbo->setImage(_light_indices.get());
+
+        camera->getOrCreateStateSet()->setTextureAttribute(
+            11, light_indices_tbo.get());
+
+        // Create and associate the light data UBO
+        osg::ref_ptr<osg::UniformBufferObject> light_data_ubo =
+            new osg::UniformBufferObject;
+        _light_data->setBufferObject(light_data_ubo.get());
+
+#if OSG_VERSION_LESS_THAN(3,6,0)
+         osg::ref_ptr<osg::UniformBufferBinding> light_data_ubb =
+            new osg::UniformBufferBinding(0, light_data_ubo.get(),
+                                          0, MAX_POINT_LIGHTS * 8 * sizeof(GLfloat));
+#else
+         osg::ref_ptr<osg::UniformBufferBinding> light_data_ubb =
+            new osg::UniformBufferBinding(0, _light_data.get(),
+                                          0, MAX_POINT_LIGHTS * 8 * sizeof(GLfloat));
+#endif
+light_data_ubb->setDataVariance(osg::Object::DYNAMIC);
+
+        camera->getOrCreateStateSet()->setAttribute(
+            light_data_ubb.get(), osg::StateAttribute::ON);
+
+        _initialized = true;
+    }
+
+    std::vector<osg::Polytope> subfrustums;
+    const osg::Matrix &view_matrix = camera->getViewMatrix();
+    const osg::Matrix &proj_matrix = camera->getProjectionMatrix();
+    osg::Matrix view_proj_inverse = osg::Matrix::inverse(view_matrix * proj_matrix);
+
+    double x_step = (_tile_size / width) * 2.0;
+    double y_step = (_tile_size / height) * 2.0;
+    for (int y = 0; y < n_vtiles; ++y) {
+        for (int x = 0; x < n_htiles; ++x) {
+            // Create the subfrustum in clip space
+            double x_min = -1.0 + x_step * x; double x_max = x_min + x_step;
+            double y_min = -1.0 + y_step * y; double y_max = y_min + y_step;
+            double z_min = 1.0;               double z_max = -1.0;
+            osg::BoundingBox subfrustum_bb(
+                x_min, y_min, z_min, x_max, y_max, z_max);
+            osg::Polytope subfrustum;
+            subfrustum.setToBoundingBox(subfrustum_bb);
+
+            // Transform it to world space
+            subfrustum.transformProvidingInverse(view_proj_inverse);
+
+            subfrustums.push_back(subfrustum);
+        }
+    }
+
+    GLushort *grid_data = reinterpret_cast<GLushort *>
+        (_light_grid->data());
+    GLushort *index_data = reinterpret_cast<GLushort *>
+        (_light_indices->data());
+
+    GLushort global_light_count = 0;
+    for (size_t i = 0; i < subfrustums.size(); ++i) {
+        GLushort start_offset = global_light_count;
+        GLushort local_light_count = 0;
+
+        for (GLushort light_list_index = 0;
+             light_list_index < NUM_LIGHTS;
+             ++light_list_index) {
+            const Light &light = LIGHT_LIST[light_list_index];
+            osg::BoundingSphere bs(light.position, light.range);
+
+            if (subfrustums[i].contains(bs)) {
+                index_data[global_light_count] = light_list_index;
+                ++local_light_count;
+                ++global_light_count;
+            }
+        }
+        grid_data[i * 3 + 0] = start_offset;
+        grid_data[i * 3 + 1] = local_light_count;
+        grid_data[i * 3 + 2] = 0;
+    }
+
+    _light_grid->dirty();
+    _light_indices->dirty();
+
+    // Upload light data
+    for (int i = 0; i < DATA_SIZE; ++i) {
+        (*_light_data)[i] = LIGHT_DATA[i];
+    }
+
+            // DEBUG
+        /*
+        if (!_debug) {
+            for (int y = 0; y < num_vtiles; ++y) {
+                for (int x = 0; x < num_htiles; ++x) {
+                    std::cout << grid_data[(y * num_htiles + x) * 3 + 0] << ","
+                              << grid_data[(y * num_htiles + x) * 3 + 1] << " ";
+                }
+                std::cout << std::endl;
+            }
+            std::cout << "\n\n";
+
+            for (int i = 0; i < num_vtiles * num_htiles; ++i) {
+                std::cout << index_data[i] << " ";
+            }
+            std::cout << "\n";
+            _debug = true;
+        }
+        */
+/*
+        for (int y = 0; y < num_vtiles; ++y) {
+            for (int x = 0; x < num_htiles; ++x) {
+                data[(y * num_htiles + x) * 3 + 0] = (unsigned short)x;
+                data[(y * num_htiles + x) * 3 + 1] = (unsigned short)y;
+                data[(y * num_htiles + x) * 3 + 2] = 0;
+            }
+        }
+        _light_grid->dirty();
+*/
+}
+
+} // namespace compositor
+} // namespace simgear