Effects: pass the model XML path through to makeEffects

This gives knowledge of how to resolve relative paths, to makeEffects
invocation of findDataFile, and hence gives a more familiar lookup
ordering, for Aircraft-supplied effects.

SF-ID: https://sourceforge.net/p/flightgear/codetickets/2678/

CMakeLists.txt

@@ -286,6 +286,7 @@ else()
     find_package(ZLIB 1.2.4 REQUIRED)
 endif()
 
+find_package(LibLZMA REQUIRED)
 find_package(CURL REQUIRED)
 
 if (SYSTEM_EXPAT)

CMakeModules/FindLibLZMA.cmake

@@ -0,0 +1,124 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+#[=======================================================================[.rst:
+FindLibLZMA
+-----------
+
+Find LZMA compression algorithm headers and library.
+
+
+Imported Targets
+^^^^^^^^^^^^^^^^
+
+This module defines :prop_tgt:`IMPORTED` target ``LibLZMA::LibLZMA``, if
+liblzma has been found.
+
+Result variables
+^^^^^^^^^^^^^^^^
+
+This module will set the following variables in your project:
+
+``LIBLZMA_FOUND``
+  True if liblzma headers and library were found.
+``LIBLZMA_INCLUDE_DIRS``
+  Directory where liblzma headers are located.
+``LIBLZMA_LIBRARIES``
+  Lzma libraries to link against.
+``LIBLZMA_HAS_AUTO_DECODER``
+  True if lzma_auto_decoder() is found (required).
+``LIBLZMA_HAS_EASY_ENCODER``
+  True if lzma_easy_encoder() is found (required).
+``LIBLZMA_HAS_LZMA_PRESET``
+  True if lzma_lzma_preset() is found (required).
+``LIBLZMA_VERSION_MAJOR``
+  The major version of lzma
+``LIBLZMA_VERSION_MINOR``
+  The minor version of lzma
+``LIBLZMA_VERSION_PATCH``
+  The patch version of lzma
+``LIBLZMA_VERSION_STRING``
+  version number as a string (ex: "5.0.3")
+#]=======================================================================]
+
+find_path(LIBLZMA_INCLUDE_DIR lzma.h )
+if(NOT LIBLZMA_LIBRARY)
+  find_library(LIBLZMA_LIBRARY_RELEASE NAMES lzma liblzma NAMES_PER_DIR PATH_SUFFIXES lib)
+  find_library(LIBLZMA_LIBRARY_DEBUG NAMES lzmad liblzmad NAMES_PER_DIR PATH_SUFFIXES lib)
+  include(SelectLibraryConfigurations)
+  select_library_configurations(LIBLZMA)
+else()
+  file(TO_CMAKE_PATH "${LIBLZMA_LIBRARY}" LIBLZMA_LIBRARY)
+endif()
+
+if(LIBLZMA_INCLUDE_DIR AND EXISTS "${LIBLZMA_INCLUDE_DIR}/lzma/version.h")
+    file(STRINGS "${LIBLZMA_INCLUDE_DIR}/lzma/version.h" LIBLZMA_HEADER_CONTENTS REGEX "#define LZMA_VERSION_[A-Z]+ [0-9]+")
+
+    string(REGEX REPLACE ".*#define LZMA_VERSION_MAJOR ([0-9]+).*" "\\1" LIBLZMA_VERSION_MAJOR "${LIBLZMA_HEADER_CONTENTS}")
+    string(REGEX REPLACE ".*#define LZMA_VERSION_MINOR ([0-9]+).*" "\\1" LIBLZMA_VERSION_MINOR "${LIBLZMA_HEADER_CONTENTS}")
+    string(REGEX REPLACE ".*#define LZMA_VERSION_PATCH ([0-9]+).*" "\\1" LIBLZMA_VERSION_PATCH "${LIBLZMA_HEADER_CONTENTS}")
+
+    set(LIBLZMA_VERSION_STRING "${LIBLZMA_VERSION_MAJOR}.${LIBLZMA_VERSION_MINOR}.${LIBLZMA_VERSION_PATCH}")
+    unset(LIBLZMA_HEADER_CONTENTS)
+endif()
+
+# We're using new code known now as XZ, even library still been called LZMA
+# it can be found in http://tukaani.org/xz/
+# Avoid using old codebase
+if (LIBLZMA_LIBRARY)
+  include(CheckLibraryExists)
+  set(CMAKE_REQUIRED_QUIET_SAVE ${CMAKE_REQUIRED_QUIET})
+  set(CMAKE_REQUIRED_QUIET ${LibLZMA_FIND_QUIETLY})
+  if(NOT LIBLZMA_LIBRARY_RELEASE AND NOT LIBLZMA_LIBRARY_DEBUG)
+    set(LIBLZMA_LIBRARY_check ${LIBLZMA_LIBRARY})
+  elseif(LIBLZMA_LIBRARY_RELEASE)
+    set(LIBLZMA_LIBRARY_check ${LIBLZMA_LIBRARY_RELEASE})
+  elseif(LIBLZMA_LIBRARY_DEBUG)
+    set(LIBLZMA_LIBRARY_check ${LIBLZMA_LIBRARY_DEBUG})
+  endif()
+  CHECK_LIBRARY_EXISTS(${LIBLZMA_LIBRARY_check} lzma_auto_decoder "" LIBLZMA_HAS_AUTO_DECODER)
+  CHECK_LIBRARY_EXISTS(${LIBLZMA_LIBRARY_check} lzma_easy_encoder "" LIBLZMA_HAS_EASY_ENCODER)
+  CHECK_LIBRARY_EXISTS(${LIBLZMA_LIBRARY_check} lzma_lzma_preset "" LIBLZMA_HAS_LZMA_PRESET)
+  unset(LIBLZMA_LIBRARY_check)
+  set(CMAKE_REQUIRED_QUIET ${CMAKE_REQUIRED_QUIET_SAVE})
+endif ()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(LibLZMA  REQUIRED_VARS  LIBLZMA_LIBRARY
+                                                          LIBLZMA_INCLUDE_DIR
+                                                          LIBLZMA_HAS_AUTO_DECODER
+                                                          LIBLZMA_HAS_EASY_ENCODER
+                                                          LIBLZMA_HAS_LZMA_PRESET
+                                           VERSION_VAR    LIBLZMA_VERSION_STRING
+                                 )
+mark_as_advanced( LIBLZMA_INCLUDE_DIR LIBLZMA_LIBRARY )
+
+if (LIBLZMA_FOUND)
+    set(LIBLZMA_LIBRARIES ${LIBLZMA_LIBRARY})
+    set(LIBLZMA_INCLUDE_DIRS ${LIBLZMA_INCLUDE_DIR})
+    if(NOT TARGET LibLZMA::LibLZMA)
+        add_library(LibLZMA::LibLZMA UNKNOWN IMPORTED)
+        set_target_properties(LibLZMA::LibLZMA PROPERTIES
+                              INTERFACE_INCLUDE_DIRECTORIES ${LIBLZMA_INCLUDE_DIR}
+                              IMPORTED_LINK_INTERFACE_LANGUAGES C)
+
+        if(LIBLZMA_LIBRARY_RELEASE)
+            set_property(TARGET LibLZMA::LibLZMA APPEND PROPERTY
+                IMPORTED_CONFIGURATIONS RELEASE)
+            set_target_properties(LibLZMA::LibLZMA PROPERTIES
+                IMPORTED_LOCATION_RELEASE "${LIBLZMA_LIBRARY_RELEASE}")
+        endif()
+
+        if(LIBLZMA_LIBRARY_DEBUG)
+            set_property(TARGET LibLZMA::LibLZMA APPEND PROPERTY
+                IMPORTED_CONFIGURATIONS DEBUG)
+            set_target_properties(LibLZMA::LibLZMA PROPERTIES
+                IMPORTED_LOCATION_DEBUG "${LIBLZMA_LIBRARY_DEBUG}")
+        endif()
+
+        if(NOT LIBLZMA_LIBRARY_RELEASE AND NOT LIBLZMA_LIBRARY_DEBUG)
+            set_target_properties(LibLZMA::LibLZMA PROPERTIES
+                IMPORTED_LOCATION "${LIBLZMA_LIBRARY}")
+        endif()
+    endif()
+endif ()

SimGearConfig.cmake.in

@@ -1,6 +1,7 @@
 include(CMakeFindDependencyMacro)
 
 find_dependency(ZLIB)
+find_dependency(LibLZMA)
 find_dependency(Threads)
 
 # OSG

simgear-version

@@ -1 +1 @@
-2020.3.2
+2020.3.13

simgear/CMakeLists.txt

@@ -168,7 +168,9 @@ target_link_libraries(SimGearCore PRIVATE
     ${COCOA_LIBRARY}
     ${CURL_LIBRARIES}
     ${WINSOCK_LIBRARY}
-    ${SHLWAPI_LIBRARY})
+    ${SHLWAPI_LIBRARY}
+    LibLZMA::LibLZMA
+)
 
 if(SYSTEM_EXPAT)
     target_link_libraries(SimGearCore PRIVATE ${EXPAT_LIBRARIES})

simgear/canvas/Canvas.cxx

@@ -691,6 +691,10 @@ namespace canvas
       {
         _sampling_dirty = true;
       }
+      else if( name == "anisotropy" )
+      {
+        _texture.setMaxAnisotropy( node->getFloatValue() );
+      }
       else if( name == "additive-blend" )
       {
         _texture.useAdditiveBlend( node->getBoolValue() );

simgear/canvas/ODGauge.cxx

@@ -202,6 +202,12 @@ namespace canvas
     updateSampling();
   }
 
+  //----------------------------------------------------------------------------
+  void ODGauge::setMaxAnisotropy(float anis)
+  {
+    texture->setMaxAnisotropy(anis);
+  }
+
   //----------------------------------------------------------------------------
   void ODGauge::setRender(bool render)
   {
@@ -292,8 +298,8 @@ namespace canvas
   {
     if( camera.valid() && Canvas::getSystemAdapter() )
       Canvas::getSystemAdapter()->removeCamera(camera.get());
-    camera.release();
-    texture.release();
+    camera = nullptr;
+    texture = nullptr;
 
     _flags &= ~AVAILABLE;
   }

simgear/canvas/ODGauge.hxx

@@ -109,6 +109,8 @@ namespace canvas
                         int coverage_samples = 0,
                         int color_samples = 0 );
 
+      void setMaxAnisotropy(float anis);
+
       /**
        * Enable/Disable updating the texture (If disabled the contents of the
        * texture remains with the outcome of the last rendering pass)

simgear/constants.h

@@ -218,6 +218,9 @@ const float SG_RADIANS_TO_DEGREES = 180.0f / SG_PI;
 #define SG_OBJECT_RANGE_ROUGH     9000.0
 #define SG_OBJECT_RANGE_DETAILED  1500.0
 
+/** Minimum expiry time of PagedLOD within the Tile.  Overridden by /sim/rendering/plod-minimum-expiry-time-secs **/
+#define SG_TILE_MIN_EXPIRY 180.0
+
 /** Radius of scenery tiles in m **/
 #define SG_TILE_RADIUS      14000.0
 

simgear/debug/CMakeLists.txt

@@ -3,8 +3,12 @@ include (SimGearComponent)
 
 set(HEADERS debug_types.h 
     logstream.hxx BufferedLogCallback.hxx OsgIoCapture.hxx
-    LogCallback.hxx LogEntry.hxx)
+    LogCallback.hxx LogEntry.hxx
+    ErrorReportingCallback.hxx)
+
 set(SOURCES logstream.cxx BufferedLogCallback.cxx
-    LogCallback.cxx LogEntry.cxx)
+    LogCallback.cxx LogEntry.cxx
+    ErrorReportingCallback.cxx
+    )
 
 simgear_component(debug debug "${SOURCES}" "${HEADERS}")

simgear/debug/ErrorReportingCallback.cxx

@@ -0,0 +1,150 @@
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Library General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Library General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//
+
+#include <simgear_config.h>
+
+#include "ErrorReportingCallback.hxx"
+#include "logstream.hxx"
+
+#include <simgear/misc/sg_path.hxx>
+
+using std::string;
+
+namespace simgear {
+
+static ErrorReportCallback static_callback;
+static ContextCallback static_contextCallback;
+
+
+string_list static_errorNames = {
+    "shader/effect",
+    "loading texture",
+    "XML/animation load",
+    "3D model load",
+    "loading BTG",
+    "sceanrio load",
+    "UI dialog load",
+    "loading Audio/FX",
+    "Nasal/commnad load of XML",
+    "aircraft-systems",
+    "loading input/joystick config",
+    "load AI/traffic",
+    "from TerraSync"};
+
+string_list static_errorTypeNames = {
+    "unknown",
+    "not found",
+    "out of memory",
+    "bad header",
+    "bad data",
+    "misconfigured",
+    "disk IO/permissions",
+    "network IO"};
+
+void setErrorReportCallback(ErrorReportCallback cb)
+{
+    static_callback = cb;
+}
+
+
+void reportError(const std::string& msg, const std::string& more)
+{
+    if (!static_callback)
+        return;
+
+    static_callback(msg, more, false);
+}
+
+void reportFatalError(const std::string& msg, const std::string& more)
+{
+    if (!static_callback)
+        return;
+    static_callback(msg, more, true);
+}
+
+static FailureCallback static_failureCallback;
+
+void reportFailure(LoadFailure type, ErrorCode code, const std::string& details, sg_location loc)
+{
+    if (!static_failureCallback) {
+        SG_LOG(SG_GENERAL, SG_WARN, "Error:" << static_errorTypeNames.at(static_cast<int>(type)) << " from " << static_errorNames.at(static_cast<int>(code)) << "::" << details << "\n\t" << loc.asString());
+        return;
+    }
+
+    static_failureCallback(type, code, details, loc);
+}
+
+void reportFailure(LoadFailure type, ErrorCode code, const std::string& details, const SGPath& path)
+{
+    if (!static_failureCallback) {
+        // if no callback is registered, log here instead
+        SG_LOG(SG_GENERAL, SG_WARN, "Error:" << static_errorTypeNames.at(static_cast<int>(type)) << " from " << static_errorNames.at(static_cast<int>(code)) << "::" << details << "\n\t" << path);
+
+        return;
+    }
+
+    static_failureCallback(type, code, details, sg_location{path});
+}
+
+void setFailureCallback(FailureCallback cb)
+{
+    static_failureCallback = cb;
+}
+
+void setErrorContextCallback(ContextCallback cb)
+{
+    static_contextCallback = cb;
+}
+
+ErrorReportContext::ErrorReportContext(const std::string& key, const std::string& value)
+{
+    add(key, value);
+}
+
+void ErrorReportContext::add(const std::string& key, const std::string& value)
+{
+    if (static_contextCallback) {
+        _keys.push_back(key);
+        static_contextCallback(key, value);
+    }
+}
+
+ErrorReportContext::ErrorReportContext(const ContextMap& context)
+{
+    addFromMap(context);
+}
+
+void ErrorReportContext::addFromMap(const ContextMap& context)
+{
+    if (static_contextCallback) {
+        for (const auto& p : context) {
+            _keys.push_back(p.first);
+            static_contextCallback(p.first, p.second);
+        }
+    }
+}
+
+
+ErrorReportContext::~ErrorReportContext()
+{
+    if (static_contextCallback) {
+        // pop all our keys
+        for (const auto& k : _keys) {
+            static_contextCallback(k, "POP");
+        }
+    }
+}
+
+} // namespace simgear

simgear/debug/ErrorReportingCallback.hxx

@@ -0,0 +1,128 @@
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Library General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Library General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//
+
+#pragma once
+
+#include <functional>
+#include <map>
+#include <string>
+#include <vector>
+
+#include <simgear/structure/exception.hxx>
+
+//forward decls
+class SGPath;
+
+namespace simgear {
+
+void reportError(const std::string& msg, const std::string& more = {});
+
+void reportFatalError(const std::string& msg, const std::string& more = {});
+
+using ErrorReportCallback = std::function<void(const std::string& msg, const std::string& more, bool isFatal)>;
+
+void setErrorReportCallback(ErrorReportCallback cb);
+
+/** kinds of failures we can report. This is *how* (or why) something failed. Extend
+  as necessary but update the correponsdings string translations if you do. More detail isn't
+ necessariyl useful here: better to provide that in the 'details' string
+ */
+enum class LoadFailure {
+    Unknown,
+    NotFound,
+    OutOfMemory,
+    BadHeader,
+    BadData,
+    Misconfigured,
+    IOError, // disk full, permissions error, etc
+    NetworkError
+};
+
+/**
+ @brief enum of the operations which can fail. This should be extended as necessary: it maps to
+ translated error messages for the user. This describes what failed, the enum above gives why/how. The
+ combination of what+why should be something at the user level: use details for debug-level information.
+ */
+enum class ErrorCode {
+    LoadEffectsShaders,
+    LoadingTexture,
+    XMLModelLoad,
+    ThreeDModelLoad, // AC3D, OBJ, etc
+    BTGLoad,
+    ScenarioLoad,
+    GUIDialog,
+    AudioFX,
+    XMLLoadCommand,
+    AircraftSystems, // autopilot, hydrualics, instruments
+    InputDeviceConfig,
+    AITrafficSchedule,
+    TerraSync
+};
+/**
+ @brief Define an error-reporting context value, for the duration of this
+ object's lifetime. The context value will be active for any errors occuring on the same thread, while
+ this object exists.
+ */
+class ErrorReportContext
+{
+public:
+    ErrorReportContext(const std::string& key, const std::string& value);
+
+    using ContextMap = std::map<std::string, std::string>;
+
+    /**
+    Allow establishing multiple context values in a single operation
+     */
+    ErrorReportContext(const ContextMap& context = {});
+
+    void add(const std::string& key, const std::string& value);
+
+    /**
+     @brief allowed delayed add of values
+     */
+    void addFromMap(const ContextMap& context);
+
+    ~ErrorReportContext();
+
+private:
+    std::vector<std::string> _keys;
+};
+
+/**
+ * @brief Report failure to load a resource, so they can be collated for reporting
+ * to the user.
+ * 
+ * @param type - the reason for the failure, if it can be determined
+ * @param msg - an informational message about what caused the failure
+ * @param path - path on disk to the resource. In some cases this may be a relative path;
+ *  especially in the case of a resource not found, we cannot report a file path.
+ */
+
+void reportFailure(LoadFailure type, ErrorCode code, const std::string& detailedMessage = {}, sg_location loc = {});
+
+/**
+  overload taking a path as the location
+ */
+void reportFailure(LoadFailure type, ErrorCode code, const std::string& detailedMessage, const SGPath& p);
+
+using FailureCallback = std::function<void(LoadFailure type, ErrorCode code, const std::string& details, const sg_location& location)>;
+
+void setFailureCallback(FailureCallback cb);
+
+using ContextCallback = std::function<void(const std::string& key, const std::string& value)>;
+
+void setErrorContextCallback(ContextCallback cb);
+
+} // namespace simgear

simgear/emesary/Transmitter.hxx

@@ -45,14 +45,6 @@ namespace simgear
             std::atomic<int> receiveDepth;
             std::atomic<int> sentMessageCount;
 
-            void UnlockList()
-            {
-                _lock.unlock();
-            }
-            void LockList()
-            {
-                _lock.lock();
-            }
         public:
             Transmitter() : receiveDepth(0), sentMessageCount(0)
             {
@@ -69,20 +61,17 @@ namespace simgear
             // most recently registered recipients should process the messages/events first.
             virtual void Register(IReceiver& r)
             {
-                LockList();
+                std::lock_guard<std::mutex> scopeLock(_lock);
                 recipient_list.push_back(&r);
                 r.OnRegisteredAtTransmitter(this);
                 if (std::find(deleted_recipients.begin(), deleted_recipients.end(), &r) != deleted_recipients.end())
                     deleted_recipients.remove(&r);
-
-                UnlockList();
             }
 
             //  Removes an object from receving message from this transmitter
             virtual void DeRegister(IReceiver& R)
             {
-                LockList();
-                //printf("Remove %x\n", &R);
+                std::lock_guard<std::mutex> scopeLock(_lock);
                 if (recipient_list.size())
                 {
                     if (std::find(recipient_list.begin(), recipient_list.end(), &R) != recipient_list.end())
@@ -93,7 +82,6 @@ namespace simgear
                             deleted_recipients.push_back(&R);
                     }
                 }
-                UnlockList();
             }
 
             // Notify all registered recipients. Stop when receipt status of abort or finished are received.
@@ -107,69 +95,68 @@ namespace simgear
                 ReceiptStatus return_status = ReceiptStatusNotProcessed;
 
                 sentMessageCount++;
-                try
+
+                std::vector<IReceiver*> temp;
                 {
-                    LockList();
+                    std::lock_guard<std::mutex> scopeLock(_lock);
                     if (receiveDepth == 0)
                         deleted_recipients.clear();
                     receiveDepth++;
-                    std::vector<IReceiver*> temp(recipient_list.size());
+
                     int idx = 0;
                     for (RecipientList::iterator i = recipient_list.begin(); i != recipient_list.end(); i++)
                     {
-                        temp[idx++] = *i;
+                        temp.push_back(*i);
                     }
-                    UnlockList();
-                    int tempSize = temp.size();
-                    for (int index = 0; index < tempSize; index++)
+                }
+                int tempSize = temp.size();
+                for (int index = 0; index < tempSize; index++)
+                {
+                    IReceiver* R = temp[index];
                     {
-                        IReceiver* R = temp[index];
-                        LockList();
+                        std::lock_guard<std::mutex> scopeLock(_lock);
                         if (deleted_recipients.size())
                         {
                             if (std::find(deleted_recipients.begin(), deleted_recipients.end(), R) != deleted_recipients.end())
                             {
-                                UnlockList();
                                 continue;
                             }
                         }
-                        UnlockList();
-                        if (R)
-                        {
-                            ReceiptStatus rstat = R->Receive(M);
-                            switch (rstat)
-                            {
-                            case ReceiptStatusFail:
-                                return_status = ReceiptStatusFail;
-                                break;
-                            case ReceiptStatusPending:
-                                return_status = ReceiptStatusPending;
-                                break;
-                            case ReceiptStatusPendingFinished:
-                                return rstat;
-
-                            case ReceiptStatusNotProcessed:
-                                break;
-                            case ReceiptStatusOK:
-                                if (return_status == ReceiptStatusNotProcessed)
-                                    return_status = rstat;
-                                break;
-
-                            case ReceiptStatusAbort:
-                                return ReceiptStatusAbort;
-
-                            case ReceiptStatusFinished:
-                                return ReceiptStatusOK;
-                            }
-                        }
-
                     }
+                    if (R)
+                    {
+                        ReceiptStatus rstat = R->Receive(M);
+                        switch (rstat)
+                        {
+                        case ReceiptStatusFail:
+                            return_status = ReceiptStatusFail;
+                            break;
+
+                        case ReceiptStatusPending:
+                            return_status = ReceiptStatusPending;
+                            break;
+
+                        case ReceiptStatusPendingFinished:
+                            return rstat;
+
+                        case ReceiptStatusNotProcessed:
+                            break;
+
+                        case ReceiptStatusOK:
+                            if (return_status == ReceiptStatusNotProcessed)
+                                return_status = rstat;
+                            break;
+
+                        case ReceiptStatusAbort:
+                            return ReceiptStatusAbort;
+
+                        case ReceiptStatusFinished:
+                            return ReceiptStatusOK;
+                        }
+                    }
+
                 }
-                catch (...)
-                {
-                    throw;
-                    // return_status = ReceiptStatusAbort;
-                }
+
                 receiveDepth--;
                 return return_status;
             }
@@ -177,9 +164,8 @@ namespace simgear
             // number of currently registered recipients
             virtual int Count()
             {
-                LockList();
+                std::lock_guard<std::mutex> scopeLock(_lock);
                 return recipient_list.size();
-                UnlockList();
             }
 
             // number of sent messages.

simgear/environment/metar.cxx

@@ -46,6 +46,7 @@
 #include <sstream>
 
 #include <simgear/debug/logstream.hxx>
+#include <simgear/math/sg_random.h>
 #include <simgear/structure/exception.hxx>
 
 #include "metar.hxx"
@@ -646,16 +647,21 @@ bool SGMetar::scanWind()
 	double gust = NaN;
 	if (*m == 'G') {
 		m++;
-		if (!scanNumber(&m, &i, 2, 3))
+		if (!strncmp(m, "//", 2))	// speed not measurable
+			m += 2, i = -1;
+		else if (!scanNumber(&m, &i, 2, 3))
 			return false;
-		gust = i;
+
+		if (i != -1)
+			gust = i;
 	}
+
 	double factor;
 	if (!strncmp(m, "KT", 2))
 		m += 2, factor = SG_KT_TO_MPS;
-	else if (!strncmp(m, "KMH", 3))
+	else if (!strncmp(m, "KMH", 3))		// invalid Km/h
 		m += 3, factor = SG_KMH_TO_MPS;
-	else if (!strncmp(m, "KPH", 3))		// ??
+	else if (!strncmp(m, "KPH", 3))		// invalid Km/h
 		m += 3, factor = SG_KMH_TO_MPS;
 	else if (!strncmp(m, "MPS", 3))
 		m += 3, factor = 1.0;
@@ -680,18 +686,28 @@ bool SGMetar::scanVariability()
 {
 	char *m = _m;
 	int from, to;
-	if (!scanNumber(&m, &from, 3))
+
+	if (!strncmp(m, "///", 3))	// direction not measurable
+		m += 3, from = -1;
+	else if (!scanNumber(&m, &from, 3))
 		return false;
+
 	if (*m++ != 'V')
 		return false;
-	if (!scanNumber(&m, &to, 3))
+
+	if (!strncmp(m, "///", 3))	// direction not measurable
+		m += 3, to = -1;
+	else if (!scanNumber(&m, &to, 3))
 		return false;
+
 	if (!scanBoundary(&m))
 		return false;
+
 	_m = m;
 	_wind_range_from = from;
 	_wind_range_to = to;
 	_grpcount++;
+
 	return true;
 }
 
@@ -1053,9 +1069,12 @@ bool SGMetar::scanSkyCondition()
 		return false;
 	m += i;
 
-	if (!strncmp(m, "///", 3))	// vis not measurable (e.g. because of heavy snowing)
+	if (!strncmp(m, "///", 3))	{ // vis not measurable (e.g. because of heavy snowing)
 		m += 3, i = -1;
-	else if (scanBoundary(&m)) {
+		sg_srandom_time();
+		// randomize the base height to avoid the black sky issue
+		i = 50 + static_cast<int>(sg_random() * 250.0);		// range [5,000, 30,000]
+	} else if (scanBoundary(&m)) {
 		_m = m;
 		return true;				// ignore single OVC/BKN/...
 	} else if (!scanNumber(&m, &i, 3))

simgear/environment/test_metar.cxx

@@ -81,6 +81,15 @@ void test_sensor_failure_wind()
     SGMetar m1("2020/10/23 16:55 LIVD 231655Z /////KT 9999 OVC025 10/08 Q1020 RMK OVC VIS MIN 9999 BLU");
     SG_CHECK_EQUAL(m1.getWindDir(), -1);
     SG_CHECK_EQUAL_EP2(m1.getWindSpeed_kt(), -1, TEST_EPSILON);
+
+    SGMetar m2("2020/10/21 16:55 LIVD 211655Z /////KT CAVOK 07/03 Q1023 RMK SKC VIS MIN 9999 BLU");
+    SG_CHECK_EQUAL(m2.getWindDir(), -1);
+    SG_CHECK_EQUAL_EP2(m2.getWindSpeed_kt(), -1, TEST_EPSILON);
+
+    SGMetar m3("2020/11/17 16:00 CYAZ 171600Z 14040G//KT 10SM -RA OVC012 12/11 A2895 RMK NS8 VIA CYXY SLP806 DENSITY ALT 900FT");
+    SG_CHECK_EQUAL(m3.getWindDir(), 140);
+    SG_CHECK_EQUAL_EP2(m3.getWindSpeed_kt(), 40, TEST_EPSILON);
+    SG_CHECK_EQUAL_EP2(m3.getGustSpeed_kt(), SGMetarNaN, TEST_EPSILON);
 }
 
 void test_wind_unit_not_specified()

simgear/io/ArchiveExtractor_private.hxx

@@ -0,0 +1,90 @@
+// Copyright (C) 2021  James Turner - <james@flightgear.org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Library General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Library General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//
+
+
+#pragma once
+
+#include "untar.hxx"
+
+namespace simgear {
+
+class ArchiveExtractorPrivate
+{
+public:
+    ArchiveExtractorPrivate(ArchiveExtractor* o) : outer(o)
+    {
+        assert(outer);
+    }
+
+    virtual ~ArchiveExtractorPrivate() = default;
+
+    typedef enum {
+        INVALID = 0,
+        READING_HEADER,
+        READING_FILE,
+        READING_PADDING,
+        READING_PAX_GLOBAL_ATTRIBUTES,
+        READING_PAX_FILE_ATTRIBUTES,
+        PRE_END_OF_ARCHVE,
+        END_OF_ARCHIVE,
+        ERROR_STATE, ///< states above this are error conditions
+        BAD_ARCHIVE,
+        BAD_DATA,
+        FILTER_STOPPED
+    } State;
+
+    State state = INVALID;
+    ArchiveExtractor* outer = nullptr;
+
+    virtual void extractBytes(const uint8_t* bytes, size_t count) = 0;
+
+    virtual void flush() = 0;
+
+    SGPath extractRootPath()
+    {
+        return outer->_rootPath;
+    }
+
+    ArchiveExtractor::PathResult filterPath(std::string& pathToExtract)
+    {
+        return outer->filterPath(pathToExtract);
+    }
+
+
+    bool isSafePath(const std::string& p) const
+    {
+        if (p.empty()) {
+            return false;
+        }
+
+        // reject absolute paths
+        if (p.at(0) == '/') {
+            return false;
+        }
+
+        // reject paths containing '..'
+        size_t doubleDot = p.find("..");
+        if (doubleDot != std::string::npos) {
+            return false;
+        }
+
+        // on POSIX could use realpath to sanity check
+        return true;
+    }
+};
+
+} // namespace simgear

simgear/io/DNSClient.cxx

@@ -61,6 +61,10 @@ public:
 
     struct dns_ctx * ctx;
     static size_t instanceCounter;
+
+    using RequestVec = std::vector<Request_ptr>;
+
+    RequestVec _activeRequests;
 };
 
 size_t Client::ClientPrivate::instanceCounter = 0;
@@ -78,6 +82,11 @@ Request::~Request()
 {
 }
 
+void Request::cancel()
+{
+    _cancelled = true;
+}
+
 bool Request::isTimeout() const
 {
     return (time(NULL) - _start) > _timeout_secs;
@@ -114,18 +123,20 @@ static void dnscbSRV(struct dns_ctx *ctx, struct dns_rr_srv *result, void *data)
 {
     SRVRequest * r = static_cast<SRVRequest*>(data);
     if (result) {
-        r->cname = result->dnssrv_cname;
-        r->qname = result->dnssrv_qname;
-        r->ttl = result->dnssrv_ttl;
-        for (int i = 0; i < result->dnssrv_nrr; i++) {
-            SRVRequest::SRV_ptr srv(new SRVRequest::SRV);
-            r->entries.push_back(srv);
-            srv->priority = result->dnssrv_srv[i].priority;
-            srv->weight = result->dnssrv_srv[i].weight;
-            srv->port = result->dnssrv_srv[i].port;
-            srv->target = result->dnssrv_srv[i].name;
+        if (!r->isCancelled()) {
+            r->cname = result->dnssrv_cname;
+            r->qname = result->dnssrv_qname;
+            r->ttl = result->dnssrv_ttl;
+            for (int i = 0; i < result->dnssrv_nrr; i++) {
+                SRVRequest::SRV_ptr srv(new SRVRequest::SRV);
+                r->entries.push_back(srv);
+                srv->priority = result->dnssrv_srv[i].priority;
+                srv->weight = result->dnssrv_srv[i].weight;
+                srv->port = result->dnssrv_srv[i].port;
+                srv->target = result->dnssrv_srv[i].name;
+            }
+            std::sort(r->entries.begin(), r->entries.end(), sortSRV);
         }
-        std::sort( r->entries.begin(), r->entries.end(), sortSRV );
         free(result);
     }
     r->setComplete();
@@ -134,11 +145,16 @@ static void dnscbSRV(struct dns_ctx *ctx, struct dns_rr_srv *result, void *data)
 void SRVRequest::submit( Client * client )
 {
     // if service is defined, pass service and protocol
-    if (!dns_submit_srv(client->d->ctx, getDn().c_str(), _service.empty() ? NULL : _service.c_str(), _service.empty() ? NULL : _protocol.c_str(), 0, dnscbSRV, this )) {
+    auto q = dns_submit_srv(client->d->ctx, getDn().c_str(), _service.empty() ? NULL : _service.c_str(),
+                            _service.empty() ? NULL : _protocol.c_str(),
+                            0, dnscbSRV, this);
+
+    if (!q) {
         SG_LOG(SG_IO, SG_ALERT, "Can't submit dns request for " << getDn());
         return;
     }
     _start = time(NULL);
+    _query = q;
 }
 
 TXTRequest::TXTRequest( const std::string & dn ) :
@@ -151,22 +167,24 @@ static void dnscbTXT(struct dns_ctx *ctx, struct dns_rr_txt *result, void *data)
 {
     TXTRequest * r = static_cast<TXTRequest*>(data);
     if (result) {
-        r->cname = result->dnstxt_cname;
-        r->qname = result->dnstxt_qname;
-        r->ttl = result->dnstxt_ttl;
-        for (int i = 0; i < result->dnstxt_nrr; i++) {
-          //TODO: interprete the .len field of dnstxt_txt?
-          auto rawTxt = reinterpret_cast<char*>(result->dnstxt_txt[i].txt);
-          if (!rawTxt) {
-              continue;
-          }
+        if (!r->isCancelled()) {
+            r->cname = result->dnstxt_cname;
+            r->qname = result->dnstxt_qname;
+            r->ttl = result->dnstxt_ttl;
+            for (int i = 0; i < result->dnstxt_nrr; i++) {
+                //TODO: interprete the .len field of dnstxt_txt?
+                auto rawTxt = reinterpret_cast<char*>(result->dnstxt_txt[i].txt);
+                if (!rawTxt) {
+                    continue;
+                }
 
-          const string txt{rawTxt};
-          r->entries.push_back(txt);
-          string_list tokens = simgear::strutils::split( txt, "=", 1 );
-          if( tokens.size() == 2 ) {
-            r->attributes[tokens[0]] = tokens[1];
-          }
+                const string txt{rawTxt};
+                r->entries.push_back(txt);
+                string_list tokens = simgear::strutils::split(txt, "=", 1);
+                if (tokens.size() == 2) {
+                    r->attributes[tokens[0]] = tokens[1];
+                }
+            }
         }
         free(result);
     }
@@ -176,11 +194,13 @@ static void dnscbTXT(struct dns_ctx *ctx, struct dns_rr_txt *result, void *data)
 void TXTRequest::submit( Client * client )
 {
     // protocol and service an already encoded in DN so pass in NULL for both
-    if (!dns_submit_txt(client->d->ctx, getDn().c_str(), DNS_C_IN, 0, dnscbTXT, this )) {
+    auto q = dns_submit_txt(client->d->ctx, getDn().c_str(), DNS_C_IN, 0, dnscbTXT, this);
+    if (!q) {
         SG_LOG(SG_IO, SG_ALERT, "Can't submit dns request for " << getDn());
         return;
     }
     _start = time(NULL);
+    _query = q;
 }
 
 
@@ -195,27 +215,29 @@ static void dnscbNAPTR(struct dns_ctx *ctx, struct dns_rr_naptr *result, void *d
 {
     NAPTRRequest * r = static_cast<NAPTRRequest*>(data);
     if (result) {
-        r->cname = result->dnsnaptr_cname;
-        r->qname = result->dnsnaptr_qname;
-        r->ttl = result->dnsnaptr_ttl;
-        for (int i = 0; i < result->dnsnaptr_nrr; i++) {
-            if( !r->qservice.empty() && r->qservice != result->dnsnaptr_naptr[i].service )
-                continue;
+        if (!r->isCancelled()) {
+            r->cname = result->dnsnaptr_cname;
+            r->qname = result->dnsnaptr_qname;
+            r->ttl = result->dnsnaptr_ttl;
+            for (int i = 0; i < result->dnsnaptr_nrr; i++) {
+                if (!r->qservice.empty() && r->qservice != result->dnsnaptr_naptr[i].service)
+                    continue;
 
-            //TODO: case ignore and result flags may have more than one flag
-            if( !r->qflags.empty() && r->qflags != result->dnsnaptr_naptr[i].flags )
-                continue;
+                //TODO: case ignore and result flags may have more than one flag
+                if (!r->qflags.empty() && r->qflags != result->dnsnaptr_naptr[i].flags)
+                    continue;
 
-            NAPTRRequest::NAPTR_ptr naptr(new NAPTRRequest::NAPTR);
-            r->entries.push_back(naptr);
-            naptr->order = result->dnsnaptr_naptr[i].order;
-            naptr->preference = result->dnsnaptr_naptr[i].preference;
-            naptr->flags = result->dnsnaptr_naptr[i].flags;
-            naptr->service = result->dnsnaptr_naptr[i].service;
-            naptr->regexp = result->dnsnaptr_naptr[i].regexp;
-            naptr->replacement = result->dnsnaptr_naptr[i].replacement;
+                NAPTRRequest::NAPTR_ptr naptr(new NAPTRRequest::NAPTR);
+                r->entries.push_back(naptr);
+                naptr->order = result->dnsnaptr_naptr[i].order;
+                naptr->preference = result->dnsnaptr_naptr[i].preference;
+                naptr->flags = result->dnsnaptr_naptr[i].flags;
+                naptr->service = result->dnsnaptr_naptr[i].service;
+                naptr->regexp = result->dnsnaptr_naptr[i].regexp;
+                naptr->replacement = result->dnsnaptr_naptr[i].replacement;
+            }
+            std::sort(r->entries.begin(), r->entries.end(), sortNAPTR);
         }
-        std::sort( r->entries.begin(), r->entries.end(), sortNAPTR );
         free(result);
     }
     r->setComplete();
@@ -223,11 +245,13 @@ static void dnscbNAPTR(struct dns_ctx *ctx, struct dns_rr_naptr *result, void *d
 
 void NAPTRRequest::submit( Client * client )
 {
-    if (!dns_submit_naptr(client->d->ctx, getDn().c_str(), 0, dnscbNAPTR, this )) {
+    auto q = dns_submit_naptr(client->d->ctx, getDn().c_str(), 0, dnscbNAPTR, this);
+    if (!q) {
         SG_LOG(SG_IO, SG_ALERT, "Can't submit dns request for " << getDn());
         return;
     }
     _start = time(NULL);
+    _query = q;
 }
 
 
@@ -242,6 +266,7 @@ Client::Client() :
 
 void Client::makeRequest(const Request_ptr& r)
 {
+    d->_activeRequests.push_back(r);
     r->submit(this);
 }
 
@@ -252,6 +277,19 @@ void Client::update(int waitTimeout)
         return;
 
     dns_ioevent(d->ctx, now);
+
+    // drop our owning ref to completed requests,
+    // and cancel any which timed out
+    auto it = std::remove_if(d->_activeRequests.begin(), d->_activeRequests.end(),
+                             [this](const Request_ptr& r) {
+                                 if (r->isTimeout()) {
+                                     dns_cancel(d->ctx, reinterpret_cast<struct dns_query*>(r->_query));
+                                     return true;
+                                 }
+
+                                 return r->isComplete();
+                             });
+    d->_activeRequests.erase(it, d->_activeRequests.end());
 }
 
 } // of namespace DNS

simgear/io/DNSClient.hxx

@@ -40,28 +40,38 @@ namespace DNS
 {
 
 class Client;
+
+using UDNSQueryPtr = void*;
+
 class Request : public SGReferenced
 {
 public:
     Request( const std::string & dn );
     virtual ~Request();
-    std::string getDn() const { return _dn; }
+    const std::string& getDn() const { return _dn; }
     int getType() const { return _type; }
     bool isComplete() const { return _complete; }
     bool isTimeout() const;
     void setComplete( bool b = true ) { _complete = b; }
+    bool isCancelled() const { return _cancelled; }
 
     virtual void submit( Client * client) = 0;
 
+    void cancel();
+
     std::string cname;
     std::string qname;
     unsigned ttl;
 protected:
+    friend class Client;
+
+    UDNSQueryPtr _query = nullptr;
     std::string _dn;
     int _type;
     bool _complete;
     time_t _timeout_secs;
     time_t _start;
+    bool _cancelled = false;
 };
 typedef SGSharedPtr<Request> Request_ptr;
 
@@ -69,7 +79,7 @@ class NAPTRRequest : public Request
 {
 public:
     NAPTRRequest( const std::string & dn );
-    virtual void submit( Client * client );
+    void submit(Client* client) override;
 
     struct NAPTR : SGReferenced {
         int order;
@@ -92,7 +102,7 @@ class SRVRequest : public Request
 public:
     SRVRequest( const std::string & dn );
     SRVRequest( const std::string & dn, const string & service, const string & protocol );
-    virtual void submit( Client * client );
+    void submit(Client* client) override;
 
     struct SRV : SGReferenced {
       int priority;
@@ -112,7 +122,7 @@ class TXTRequest : public Request
 {
 public:
     TXTRequest( const std::string & dn );
-    virtual void submit( Client * client );
+    void submit(Client* client) override;
 
     typedef std::vector<string> TXT_list;
     typedef std::map<std::string,std::string> TXT_Attribute_map;

simgear/io/HTTPClient.cxx

@@ -81,21 +81,8 @@ void Client::ClientPrivate::createCurlMulti() {
 #endif
 }
 
-Client::Client() :
-    d(new ClientPrivate)
+Client::Client()
 {
-    d->proxyPort = 0;
-    d->maxConnections = 4;
-    d->maxHostConnections = 4;
-    d->bytesTransferred = 0;
-    d->lastTransferRate = 0;
-    d->timeTransferSample.stamp();
-    d->totalBytesDownloaded = 0;
-    d->maxPipelineDepth = 5;
-    setUserAgent("SimGear-" SG_STRINGIZE(SIMGEAR_VERSION));
-
-    d->tlsCertificatePath = SGPath::fromEnv("SIMGEAR_TLS_CERT_PATH");
-
     static bool didInitCurlGlobal = false;
     static std::mutex initMutex;
 
@@ -105,7 +92,7 @@ Client::Client() :
       didInitCurlGlobal = true;
     }
 
-    d->createCurlMulti();
+    reset();
 }
 
 Client::~Client()
@@ -139,8 +126,21 @@ void Client::setMaxPipelineDepth(unsigned int depth)
 
 void Client::reset()
 {
-    curl_multi_cleanup(d->curlMulti);
+    if (d.get()) {
+        curl_multi_cleanup(d->curlMulti);
+    }
+
     d.reset(new ClientPrivate);
+
+    d->proxyPort = 0;
+    d->maxConnections = 4;
+    d->maxHostConnections = 4;
+    d->bytesTransferred = 0;
+    d->lastTransferRate = 0;
+    d->timeTransferSample.stamp();
+    d->totalBytesDownloaded = 0;
+    d->maxPipelineDepth = 5;
+    setUserAgent("SimGear-" SG_STRINGIZE(SIMGEAR_VERSION));
     d->tlsCertificatePath = SGPath::fromEnv("SIMGEAR_TLS_CERT_PATH");
     d->createCurlMulti();
 }

simgear/io/HTTPRepository.cxx

@@ -20,26 +20,27 @@
 
 #include "HTTPRepository.hxx"
 
-#include <iostream>
-#include <cassert>
 #include <algorithm>
-#include <sstream>
+#include <cassert>
+#include <cstdlib>
+#include <fstream>
+#include <iostream>
+#include <limits>
 #include <map>
 #include <set>
-#include <fstream>
-#include <limits>
-#include <cstdlib>
+#include <sstream>
 
 #include <fcntl.h>
 
 #include "simgear/debug/logstream.hxx"
 #include "simgear/misc/strutils.hxx"
 
-#include <simgear/misc/sg_dir.hxx>
+#include <simgear/debug/ErrorReportingCallback.hxx>
 #include <simgear/io/HTTPClient.hxx>
+#include <simgear/io/iostreams/sgstream.hxx>
 #include <simgear/io/sg_file.hxx>
 #include <simgear/io/untar.hxx>
-#include <simgear/io/iostreams/sgstream.hxx>
+#include <simgear/misc/sg_dir.hxx>
 #include <simgear/structure/exception.hxx>
 #include <simgear/timing/timestamp.hxx>
 
@@ -143,8 +144,8 @@ class HTTPDirectory
 
         HTTPRepository::EntryType type;
         std::string name, hash;
-		size_t sizeInBytes = 0;
-		SGPath path; // absolute path on disk
+        size_t sizeInBytes = 0;
+        SGPath path; // absolute path on disk
     };
 
     typedef std::vector<ChildInfo> ChildInfoList;
@@ -218,51 +219,78 @@ public:
             return;
         }
 
-		char* buf = nullptr;
-		size_t bufSize = 0;
+        std::vector<char> buf;
 
-                for (auto &child : children) {
-                  if (child.type != HTTPRepository::FileType)
-                    continue;
+        for (auto& child : children) {
+            if (child.type != HTTPRepository::FileType)
+                continue;
 
-                  if (child.path.exists())
-                    continue;
+            if (child.path.exists())
+                continue;
 
-                  SGPath cp = _repository->installedCopyPath;
-                  cp.append(relativePath());
-                  cp.append(child.name);
-                  if (!cp.exists()) {
-                    continue;
-                  }
+            SGPath cp = _repository->installedCopyPath;
+            cp.append(relativePath());
+            cp.append(child.name);
+            if (!cp.exists()) {
+                continue;
+            }
 
-                  SGBinaryFile src(cp);
-                  SGBinaryFile dst(child.path);
-                  src.open(SG_IO_IN);
-                  dst.open(SG_IO_OUT);
+            SGBinaryFile src(cp);
+            SGBinaryFile dst(child.path);
+            src.open(SG_IO_IN);
+            dst.open(SG_IO_OUT);
 
-                  if (bufSize < cp.sizeInBytes()) {
-                    bufSize = cp.sizeInBytes();
-                    free(buf);
-                    buf = (char *)malloc(bufSize);
-                    if (!buf) {
-                      continue;
-                    }
-                  }
-
-                  src.read(buf, cp.sizeInBytes());
-                  dst.write(buf, cp.sizeInBytes());
-                  src.close();
-                  dst.close();
-
-                  // reset caching
-                  child.path.set_cached(false);
-                  child.path.set_cached(true);
-
-                  std::string hash = computeHashForPath(child.path);
-                  updatedFileContents(child.path, hash);
+            const auto sizeToCopy = cp.sizeInBytes();
+            if (buf.size() < sizeToCopy) {
+                try {
+                    buf.resize(sizeToCopy);
+                } catch (std::bad_alloc) {
+                    simgear::reportFailure(simgear::LoadFailure::OutOfMemory, simgear::ErrorCode::TerraSync,
+                                           "copyInstalledChildren: couldn't allocation copy buffer of size:" + std::to_string(sizeToCopy),
+                                           child.path);
+                    return;
                 }
+            }
 
-                free(buf);
+            const auto r = src.read(buf.data(), sizeToCopy);
+            if (r != sizeToCopy) {
+                simgear::reportFailure(simgear::LoadFailure::IOError, simgear::ErrorCode::TerraSync,
+                                       "copyInstalledChildren: read underflow, got:" + std::to_string(r),
+                                       cp);
+                return;
+            }
+
+            const auto written = dst.write(buf.data(), sizeToCopy);
+            if (written != sizeToCopy) {
+                simgear::reportFailure(simgear::LoadFailure::IOError, simgear::ErrorCode::TerraSync,
+                                       "copyInstalledChildren: write underflow, wrote:" + std::to_string(r),
+                                       child.path);
+                return;
+            }
+
+            src.close();
+            dst.close();
+
+            // reset caching
+            child.path.set_cached(false);
+            child.path.set_cached(true);
+
+            std::string hash = computeHashForPath(child.path);
+            updatedFileContents(child.path, hash);
+        }
+    }
+
+    /// helper to check and erase 'fooBar' from paths, if passed fooBar.zip, fooBar.tgz, etc.
+    void removeExtractedDirectoryFromList(PathList& paths, const std::string& tarballName)
+    {
+        const auto directoryName = SGPath::fromUtf8(tarballName).file_base();
+        auto it = std::find_if(paths.begin(), paths.end(), [directoryName](const SGPath& p) {
+            return p.isDir() && (p.file() == directoryName);
+        });
+
+        if (it != paths.end()) {
+            paths.erase(it);
+        }
     }
 
     void updateChildrenBasedOnHash()
@@ -298,6 +326,11 @@ public:
                         orphans.end());
         }
 
+        // ensure the extracted directory corresponding to a tarball, is *not* considered an orphan
+        if (c.type == HTTPRepository::TarballType) {
+            removeExtractedDirectoryFromList(orphans, c.name);
+        }
+
         if (_repository->syncPredicate) {
           const auto pathOnDisk = isNew ? absolutePath() / c.name : *p;
           // never handle deletes here, do them at the end
@@ -320,7 +353,6 @@ public:
           toBeUpdated.push_back(c);
         } else {
           // File/Directory exists and hash is valid.
-
           if (c.type == HTTPRepository::DirectoryType) {
             // If it's a directory,perform a recursive check.
             HTTPDirectory *childDir = childDirectory(c.name);
@@ -360,7 +392,12 @@ public:
 
       // We now have a list of entries that need to be updated, and a list
       // of orphan files that should be removed.
-      removeOrphans(orphans);
+      try {
+          removeOrphans(orphans);
+      } catch (sg_exception& e) {
+          _repository->failedToUpdateChild(_relativePath, HTTPRepository::ResultCode::REPO_ERROR_IO);
+      }
+
       scheduleUpdates(toBeUpdated);
     }
 
@@ -372,11 +409,10 @@ public:
 
     void removeOrphans(const PathList orphans)
     {
-        PathList::const_iterator it;
-        for (it = orphans.begin(); it != orphans.end(); ++it) {
-            if (it->file() == ".dirindex") continue;
-            if (it->file() == ".hash") continue;
-            removeChild(*it);
+        for (const auto& o : orphans) {
+            if (o.file() == ".dirindex") continue;
+            if (o.file() == ".hash") continue;
+            removeChild(o);
         }
     }
 
@@ -432,41 +468,54 @@ public:
           return;
         }
 
+        compressedBytes = p.sizeInBytes();
         buffer = (uint8_t *)malloc(bufferSize);
       }
 
       ArchiveExtractTask(const ArchiveExtractTask &) = delete;
 
-      HTTPRepoPrivate::ProcessResult run(HTTPRepoPrivate *repo) {
-        size_t rd = file.read((char *)buffer, bufferSize);
-        extractor.extractBytes(buffer, rd);
-
-        if (file.eof()) {
-          extractor.flush();
-          file.close();
-
-          if (!extractor.isAtEndOfArchive()) {
-            SG_LOG(SG_TERRASYNC, SG_ALERT, "Corrupt tarball " << relativePath);
-            repo->failedToUpdateChild(relativePath,
-                                      HTTPRepository::REPO_ERROR_IO);
-            return HTTPRepoPrivate::ProcessFailed;
+      HTTPRepoPrivate::ProcessResult run(HTTPRepoPrivate* repo)
+      {
+          if (!buffer) {
+              return HTTPRepoPrivate::ProcessFailed;
           }
 
-          if (extractor.hasError()) {
-            SG_LOG(SG_TERRASYNC, SG_ALERT, "Error extracting " << relativePath);
-            repo->failedToUpdateChild(relativePath,
-                                      HTTPRepository::REPO_ERROR_IO);
-            return HTTPRepoPrivate::ProcessFailed;
+          size_t rd = file.read((char*)buffer, bufferSize);
+          repo->bytesExtracted += rd;
+          extractor.extractBytes(buffer, rd);
+
+          if (file.eof()) {
+              extractor.flush();
+              file.close();
+
+              if (!extractor.isAtEndOfArchive()) {
+                  SG_LOG(SG_TERRASYNC, SG_ALERT, "Corrupt tarball " << relativePath);
+                  repo->failedToUpdateChild(relativePath,
+                                            HTTPRepository::REPO_ERROR_IO);
+                  return HTTPRepoPrivate::ProcessFailed;
+              }
+
+              if (extractor.hasError()) {
+                  SG_LOG(SG_TERRASYNC, SG_ALERT, "Error extracting " << relativePath);
+                  repo->failedToUpdateChild(relativePath,
+                                            HTTPRepository::REPO_ERROR_IO);
+                  return HTTPRepoPrivate::ProcessFailed;
+              }
+
+              return HTTPRepoPrivate::ProcessDone;
           }
 
-          return HTTPRepoPrivate::ProcessDone;
-        }
+          return HTTPRepoPrivate::ProcessContinue;
+      }
 
-        return HTTPRepoPrivate::ProcessContinue;
+      size_t archiveSizeBytes() const
+      {
+          return compressedBytes;
       }
 
       ~ArchiveExtractTask() { free(buffer); }
 
+
     private:
       // intentionally small so we extract incrementally on Windows
       // where Defender throttles many small files, sorry
@@ -478,6 +527,7 @@ public:
       uint8_t *buffer = nullptr;
       SGBinaryFile file;
       ArchiveExtractor extractor;
+      std::size_t compressedBytes;
     };
 
     using ArchiveExtractTaskPtr = std::shared_ptr<ArchiveExtractTask>;
@@ -504,42 +554,42 @@ public:
                 _repository->totalDownloaded += sz;
                 SGPath p = SGPath(absolutePath(), file);
 
-                if ((p.extension() == "tgz") || (p.extension() == "zip")) {
-                  // We require that any compressed files have the same filename as the file or directory
-                  // they expand to, so we can remove the old file/directory before extracting the new
-                  // data.
-                  SGPath removePath = SGPath(p.base());
-                  bool pathAvailable = true;
-                  if (removePath.exists()) {
-                    if (removePath.isDir()) {
-                      simgear::Dir pd(removePath);
-                    	pathAvailable = pd.removeChildren();
-                    } else {
-                      pathAvailable = removePath.remove();
+                if (it->type == HTTPRepository::TarballType) {
+                    // We require that any compressed files have the same filename as the file or directory
+                    // they expand to, so we can remove the old file/directory before extracting the new
+                    // data.
+                    SGPath removePath = SGPath(p.base());
+                    bool pathAvailable = true;
+                    if (removePath.exists()) {
+                        if (removePath.isDir()) {
+                            simgear::Dir pd(removePath);
+                            pathAvailable = pd.removeChildren();
+                        } else {
+                            pathAvailable = removePath.remove();
+                        }
                     }
-                  }
 
-                  if (pathAvailable) {
-                    // we use a Task helper to extract tarballs incrementally.
-                    // without this, archive extraction blocks here, which
-                    // prevents other repositories downloading / updating.
-                    // Unfortunately due Windows AV (Defender, etc) we cna block
-                    // here for many minutes.
+                    if (pathAvailable) {
+                        // we use a Task helper to extract tarballs incrementally.
+                        // without this, archive extraction blocks here, which
+                        // prevents other repositories downloading / updating.
+                        // Unfortunately due Windows AV (Defender, etc) we cna block
+                        // here for many minutes.
 
-                    // use a lambda to own this shared_ptr; this means when the
-                    // lambda is destroyed, the ArchiveExtraTask will get
-                    // cleaned up.
-                    ArchiveExtractTaskPtr t =
-                        std::make_shared<ArchiveExtractTask>(p, _relativePath);
-                    auto cb = [t](HTTPRepoPrivate *repo) {
-                      return t->run(repo);
-                    };
-
-                    _repository->addTask(cb);
-                  } else {
-                    SG_LOG(SG_TERRASYNC, SG_ALERT, "Unable to remove old file/directory " << removePath);
-                  } // of pathAvailable
-                } // of handling tgz files
+                        // use a lambda to own this shared_ptr; this means when the
+                        // lambda is destroyed, the ArchiveExtraTask will get
+                        // cleaned up.
+                        ArchiveExtractTaskPtr t =
+                            std::make_shared<ArchiveExtractTask>(p, _relativePath);
+                        auto cb = [t](HTTPRepoPrivate* repo) {
+                            return t->run(repo);
+                        };
+                        _repository->bytesToExtract += t->archiveSizeBytes();
+                        _repository->addTask(cb);
+                    } else {
+                        SG_LOG(SG_TERRASYNC, SG_ALERT, "Unable to remove old file/directory " << removePath);
+                    } // of pathAvailable
+                }     // of handling archive files
             } // of hash matches
         } // of found in child list
     }
@@ -724,18 +774,16 @@ private:
 
         if (!ok) {
             SG_LOG(SG_TERRASYNC, SG_WARN, "removal failed for:" << path);
-            throw sg_io_exception("Failed to remove existing file/dir:", path);
+            throw sg_io_exception("Failed to remove existing file/dir:", path, "", false);
         }
     }
 
     std::string hashForChild(const ChildInfo& child) const
     {
       SGPath p(child.path);
-      if (child.type == HTTPRepository::DirectoryType)
-        p.append(".dirindex");
-      if (child.type == HTTPRepository::TarballType)
-        p.concat(
-            ".tgz"); // For tarballs the hash is against the tarball file itself
+      if (child.type == HTTPRepository::DirectoryType) {
+          p.append(".dirindex");
+      }
       return hashForPath(p);
     }
 
@@ -932,6 +980,11 @@ size_t HTTPRepository::bytesDownloaded() const
     return result;
 }
 
+size_t HTTPRepository::bytesToExtract() const
+{
+    return _d->bytesToExtract - _d->bytesExtracted;
+}
+
 void HTTPRepository::setInstalledCopyPath(const SGPath& copyPath)
 {
     _d->installedCopyPath = copyPath;
@@ -976,54 +1029,90 @@ HTTPRepository::failure() const
         }
 
     protected:
-      void gotBodyData(const char *s, int n) override {
-        if (!file.get()) {
-          file.reset(new SGBinaryFile(pathInRepo));
-          if (!file->open(SG_IO_OUT)) {
-            SG_LOG(SG_TERRASYNC, SG_WARN,
-                   "unable to create file " << pathInRepo);
-            _directory->repository()->http->cancelRequest(
-                this, "Unable to create output file:" + pathInRepo.utf8Str());
-          }
+        void gotBodyData(const char* s, int n) override
+        {
+            if (!file.get()) {
+                const bool ok = createOutputFile();
+                if (!ok) {
+                    ioFailureOccurred = true;
+                    _directory->repository()->http->cancelRequest(
+                        this, "Unable to create output file:" + pathInRepo.utf8Str());
+                }
+            }
 
-          sha1_init(&hashContext);
+            sha1_write(&hashContext, s, n);
+            const auto written = file->write(s, n);
+            if (written != n) {
+                SG_LOG(SG_TERRASYNC, SG_WARN, "Underflow writing to " << pathInRepo);
+                ioFailureOccurred = true;
+                _directory->repository()->http->cancelRequest(
+                    this, "Unable to write to output file:" + pathInRepo.utf8Str());
+            }
         }
 
-        sha1_write(&hashContext, s, n);
-        file->write(s, n);
-      }
+        bool createOutputFile()
+        {
+            file.reset(new SGBinaryFile(pathInRepo));
+            if (!file->open(SG_IO_OUT)) {
+                SG_LOG(SG_TERRASYNC, SG_WARN,
+                       "unable to create file " << pathInRepo);
+                return false;
+            }
 
-      void onDone() override {
-        file->close();
-        if (responseCode() == 200) {
-          std::string hash =
-              strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH);
-          _directory->didUpdateFile(fileName, hash, contentSize());
-        } else if (responseCode() == 404) {
-          SG_LOG(SG_TERRASYNC, SG_WARN,
-                 "terrasync file not found on server: "
-                     << fileName << " for " << _directory->absolutePath());
-          _directory->didFailToUpdateFile(
-              fileName, HTTPRepository::REPO_ERROR_FILE_NOT_FOUND);
-        } else {
-          SG_LOG(SG_TERRASYNC, SG_WARN,
-                 "terrasync file download error on server: "
-                     << fileName << " for " << _directory->absolutePath()
-                     << "\n\tserver responded: " << responseCode() << "/"
-                     << responseReason());
-          _directory->didFailToUpdateFile(fileName,
-                                          HTTPRepository::REPO_ERROR_HTTP);
-          // should we every retry here?
+            sha1_init(&hashContext);
+            return true;
         }
 
-        _directory->repository()->finishedRequest(
-            this, HTTPRepoPrivate::RequestFinish::Done);
-      }
+        void onDone() override
+        {
+            const bool is200Response = (responseCode() == 200);
+            if (!file && is200Response) {
+                // if the server defines a zero-byte file, we will never call
+                // gotBodyData, so create the file here
+                // this ensures all the logic below works as expected
+                createOutputFile();
+            }
+
+            if (file) {
+                file->close();
+            }
+
+            if (is200Response) {
+                std::string hash =
+                    strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH);
+                _directory->didUpdateFile(fileName, hash, contentSize());
+            } else if (responseCode() == 404) {
+                SG_LOG(SG_TERRASYNC, SG_WARN,
+                       "terrasync file not found on server: "
+                           << fileName << " for " << _directory->absolutePath());
+                _directory->didFailToUpdateFile(
+                    fileName, HTTPRepository::REPO_ERROR_FILE_NOT_FOUND);
+            } else {
+                SG_LOG(SG_TERRASYNC, SG_WARN,
+                       "terrasync file download error on server: "
+                           << fileName << " for " << _directory->absolutePath()
+                           << "\n\tserver responded: " << responseCode() << "/"
+                           << responseReason());
+                _directory->didFailToUpdateFile(fileName,
+                                                HTTPRepository::REPO_ERROR_HTTP);
+                // should we every retry here?
+            }
+
+            _directory->repository()->finishedRequest(
+                this, HTTPRepoPrivate::RequestFinish::Done);
+        }
 
       void onFail() override {
         HTTPRepository::ResultCode code = HTTPRepository::REPO_ERROR_SOCKET;
+
+        // -1 means request cancelled locally
         if (responseCode() == -1) {
-          code = HTTPRepository::REPO_ERROR_CANCELLED;
+            if (ioFailureOccurred) {
+                // cancelled by code above due to IO error
+                code = HTTPRepository::REPO_ERROR_IO;
+            } else {
+                code = HTTPRepository::REPO_ERROR_CANCELLED;
+            }
         }
 
         if (file) {
@@ -1060,6 +1149,10 @@ HTTPRepository::failure() const
         SGPath pathInRepo;
         simgear::sha1nfo hashContext;
         std::unique_ptr<SGBinaryFile> file;
+
+        /// becuase we cancel() in the case of an IO failure, we need to a way to distuinguish
+        /// user initated cancellation and IO-failure cancellation in onFail. This flag lets us do that
+        bool ioFailureOccurred = false;
     };
 
     class DirGetRequest : public HTTPRepoGetRequest
@@ -1266,7 +1359,7 @@ HTTPRepository::failure() const
         }
 
         Dir dir(absPath);
-		bool result = dir.remove(true);
+        bool result = dir.remove(true);
         return result;
     }
 
@@ -1313,6 +1406,9 @@ HTTPRepository::failure() const
         if (st == HTTPRepository::REPO_ERROR_FILE_NOT_FOUND) {
             status = HTTPRepository::REPO_ERROR_NOT_FOUND;
         } else {
+            simgear::reportFailure(simgear::LoadFailure::NetworkError, simgear::ErrorCode::TerraSync,
+                                   "failed to get TerraSync repository root:" + innerResultCodeAsString(st),
+                                   sg_location{baseUrl});
             SG_LOG(SG_TERRASYNC, SG_WARN, "Failed to get root of repo:" << baseUrl << " " << st);
             status = st;
         }
@@ -1328,7 +1424,12 @@ HTTPRepository::failure() const
         SG_LOG(SG_TERRASYNC, SG_WARN,
                "failed to update entry:" << relativePath << " status/code: "
                                          << innerResultCodeAsString(fileStatus)
-                                         << "/" << fileStatus);
+                                         << "/" << fileStatus
+                                         << "\nrepo:" << baseUrl);
+
+        simgear::reportFailure(simgear::LoadFailure::NetworkError, simgear::ErrorCode::TerraSync,
+                               "failed to update entry:" + innerResultCodeAsString(fileStatus),
+                               sg_location{relativePath});
       }
 
       HTTPRepository::Failure f;

simgear/io/HTTPRepository.hxx

@@ -73,6 +73,8 @@ public:
 
   virtual size_t bytesDownloaded() const;
 
+  virtual size_t bytesToExtract() const;
+
   /**
    * optionally provide the location of an installer copy of this
    * repository. When a file is missing it will be copied from this tree.

simgear/io/HTTPRepository_private.hxx

@@ -60,22 +60,23 @@ public:
   HTTPRepository::FailureVec failures;
   int maxPermittedFailures = 16;
 
-  HTTPRepoPrivate(HTTPRepository *parent)
-      : p(parent), isUpdating(false), status(HTTPRepository::REPO_NO_ERROR),
-        totalDownloaded(0) {
-    ;
+  HTTPRepoPrivate(HTTPRepository* parent)
+      : p(parent)
+  {
   }
 
   ~HTTPRepoPrivate();
 
   HTTPRepository *p; // link back to outer
-  HTTP::Client *http;
+  HTTP::Client* http = nullptr;
   std::string baseUrl;
   SGPath basePath;
-  bool isUpdating;
-  HTTPRepository::ResultCode status;
+  bool isUpdating = false;
+  HTTPRepository::ResultCode status = HTTPRepository::REPO_NO_ERROR;
   HTTPDirectory_ptr rootDir;
-  size_t totalDownloaded;
+  size_t totalDownloaded = 0;
+  size_t bytesToExtract = 0;
+  size_t bytesExtracted = 0;
   HTTPRepository::SyncPredicate syncPredicate;
 
   HTTP::Request_ptr updateFile(HTTPDirectory *dir, const std::string &name,

simgear/io/lowlevel.cxx

@@ -23,28 +23,43 @@
 // $Id$
 
 
-#ifdef HAVE_CONFIG_H
-#  include <simgear_config.h>
-#endif
+#include <simgear_config.h>
 
 #include <string.h>		// for memcpy()
+#include <errno.h>
 
-#include "lowlevel.hxx" 
+#include <simgear/structure/exception.hxx>
+#include <simgear/misc/strutils.hxx>
+#include <simgear/misc/sg_path.hxx>
 
+#include "lowlevel.hxx"
 
-static int  read_error = false ;
-static int write_error = false ;
+thread_local SGPath thread_gzPath;
 
-void sgClearReadError() { read_error = false; }
-void sgClearWriteError() { write_error = false; }
-int sgReadError() { return  read_error ; }
-int sgWriteError() { return write_error ; }
+void setThreadLocalSimgearReadPath(const SGPath& path)
+{
+    thread_gzPath = path;
+}
 
+static std::string gzErrorMessage(gzFile fd)
+{
+    int errNum = 0;
+    const char *gzMsg = gzerror(fd, &errNum);
+
+    if (errNum == Z_ERRNO) {
+        return simgear::strutils::error_string(errno);
+    } else if (gzMsg) {
+        return {gzMsg};
+    }
+
+    return {"GZError: no string code for code:" + std::to_string(errNum)};
+}
 
 void sgReadChar ( gzFile fd, char *var )
 {
     if ( gzread ( fd, var, sizeof(char) ) != sizeof(char) ) {
-        read_error = true ;
+        throw sg_io_exception("sgReadChar: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
 }
 
@@ -52,7 +67,7 @@ void sgReadChar ( gzFile fd, char *var )
 void sgWriteChar ( gzFile fd, const char var )
 {
     if ( gzwrite ( fd, (void *)(&var), sizeof(char) ) != sizeof(char) ) {
-        write_error = true ;
+        throw sg_io_exception("sgWriteChar: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -61,7 +76,8 @@ void sgReadFloat ( gzFile fd, float *var )
 {
     union { float v; uint32_t u; } buf;
     if ( gzread ( fd, &buf.u, sizeof(float) ) != sizeof(float) ) {
-        read_error = true ;
+        throw sg_io_exception("sgReadFloat: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         sgEndianSwap( &buf.u );
@@ -78,7 +94,7 @@ void sgWriteFloat ( gzFile fd, const float var )
         sgEndianSwap( &buf.u );
     }
     if ( gzwrite ( fd, (void *)(&buf.u), sizeof(float) ) != sizeof(float) ) {
-        write_error = true ;
+        throw sg_io_exception("sgWriteFloat: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -87,7 +103,8 @@ void sgReadDouble ( gzFile fd, double *var )
 {
     union { double v; uint64_t u; } buf;
     if ( gzread ( fd, &buf.u, sizeof(double) ) != sizeof(double) ) {
-        read_error = true ;
+        throw sg_io_exception("sgReadDouble: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         sgEndianSwap( &buf.u );
@@ -104,7 +121,7 @@ void sgWriteDouble ( gzFile fd, const double var )
         sgEndianSwap( &buf.u );
     }
     if ( gzwrite ( fd, (void *)(&buf.u), sizeof(double) ) != sizeof(double) ) {
-        write_error = true ;
+        throw sg_io_exception("sgWriteDouble: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -112,7 +129,8 @@ void sgWriteDouble ( gzFile fd, const double var )
 void sgReadUInt ( gzFile fd, unsigned int *var )
 {
     if ( gzread ( fd, var, sizeof(unsigned int) ) != sizeof(unsigned int) ) {
-        read_error = true ;
+        throw sg_io_exception("sgReadUInt: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         sgEndianSwap( (uint32_t *)var);
@@ -128,7 +146,7 @@ void sgWriteUInt ( gzFile fd, const unsigned int var )
     if ( gzwrite ( fd, (void *)(&var), sizeof(unsigned int) )
 	 != sizeof(unsigned int) )
     {
-        write_error = true ;
+        throw sg_io_exception("sgWriteUInt: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -136,7 +154,8 @@ void sgWriteUInt ( gzFile fd, const unsigned int var )
 void sgReadInt ( gzFile fd, int *var )
 {
     if ( gzread ( fd, var, sizeof(int) ) != sizeof(int) ) {
-        read_error = true ;
+        throw sg_io_exception("sgReadInt: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         sgEndianSwap( (uint32_t *)var);
@@ -150,7 +169,7 @@ void sgWriteInt ( gzFile fd, const int var )
         sgEndianSwap( (uint32_t *)&var);
     }
     if ( gzwrite ( fd, (void *)(&var), sizeof(int) ) != sizeof(int) ) {
-        write_error = true ;
+        throw sg_io_exception("sgWriteInt: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -158,7 +177,8 @@ void sgWriteInt ( gzFile fd, const int var )
 void sgReadLong ( gzFile fd, int32_t *var )
 {
     if ( gzread ( fd, var, sizeof(int32_t) ) != sizeof(int32_t) ) {
-        read_error = true ;
+        throw sg_io_exception("sgReadLong: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         sgEndianSwap( (uint32_t *)var);
@@ -174,7 +194,7 @@ void sgWriteLong ( gzFile fd, const int32_t var )
     if ( gzwrite ( fd, (void *)(&var), sizeof(int32_t) )
          != sizeof(int32_t) )
     {
-        write_error = true ;
+        throw sg_io_exception("sgWriteLong: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -182,7 +202,8 @@ void sgWriteLong ( gzFile fd, const int32_t var )
 void sgReadLongLong ( gzFile fd, int64_t *var )
 {
     if ( gzread ( fd, var, sizeof(int64_t) ) != sizeof(int64_t) ) {
-        read_error = true ;
+        throw sg_io_exception("sgReadLongLong: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         sgEndianSwap( (uint64_t *)var);
@@ -198,7 +219,7 @@ void sgWriteLongLong ( gzFile fd, const int64_t var )
     if ( gzwrite ( fd, (void *)(&var), sizeof(int64_t) )
          != sizeof(int64_t) )
     {
-        write_error = true ;
+        throw sg_io_exception("sgWriteLongLong: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -206,7 +227,8 @@ void sgWriteLongLong ( gzFile fd, const int64_t var )
 void sgReadUShort ( gzFile fd, unsigned short *var )
 {
     if ( gzread ( fd, var, sizeof(unsigned short) ) != sizeof(unsigned short) ){
-        read_error = true ;
+        throw sg_io_exception("sgReadUShort: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         sgEndianSwap( (uint16_t *)var);
@@ -222,7 +244,7 @@ void sgWriteUShort ( gzFile fd, const unsigned short var )
     if ( gzwrite ( fd, (void *)(&var), sizeof(unsigned short) )
 	 != sizeof(unsigned short) )
     {
-        write_error = true ;
+        throw sg_io_exception("sgWriteUShort: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -230,7 +252,8 @@ void sgWriteUShort ( gzFile fd, const unsigned short var )
 void sgReadShort ( gzFile fd, short *var )
 {
     if ( gzread ( fd, var, sizeof(short) ) != sizeof(short) ) {
-        read_error = true ;
+        throw sg_io_exception("sgReadShort: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         sgEndianSwap( (uint16_t *)var);
@@ -244,7 +267,7 @@ void sgWriteShort ( gzFile fd, const short var )
         sgEndianSwap( (uint16_t *)&var);
     }
     if ( gzwrite ( fd, (void *)(&var), sizeof(short) ) != sizeof(short) ) {
-        write_error = true ;
+        throw sg_io_exception("sgWriteShort: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -252,7 +275,8 @@ void sgWriteShort ( gzFile fd, const short var )
 void sgReadFloat ( gzFile fd, const unsigned int n, float *var )
 {
     if ( gzread ( fd, var, sizeof(float) * n ) != (int)(sizeof(float) * n) ) {
-        read_error = true ;
+        throw sg_io_exception("sgReadFloat array: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         for ( unsigned int i = 0; i < n; ++i ) {
@@ -276,14 +300,15 @@ void sgWriteFloat ( gzFile fd, const unsigned int n, const float *var )
     if ( gzwrite ( fd, (void *)var, sizeof(float) * n )
 	 != (int)(sizeof(float) * n) )
     {
-        write_error = true ;
+        throw sg_io_exception("sgWriteFloat array: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
 void sgReadDouble ( gzFile fd, const unsigned int n, double *var )
 {
     if ( gzread ( fd, var, sizeof(double) * n ) != (int)(sizeof(double) * n) ) {
-        read_error = true ;
+        throw sg_io_exception("sgReadDouble array: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         for ( unsigned int i = 0; i < n; ++i ) {
@@ -307,7 +332,7 @@ void sgWriteDouble ( gzFile fd, const unsigned int n, const double *var )
     if ( gzwrite ( fd, (void *)var, sizeof(double) * n )
 	 != (int)(sizeof(double) * n) )
     {
-        write_error = true ;
+        throw sg_io_exception("sgWriteDouble array: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -315,7 +340,8 @@ void sgReadBytes ( gzFile fd, const unsigned int n, void *var )
 {
     if ( n == 0) return;
     if ( gzread ( fd, var, n ) != (int)n ) {
-        read_error = true ;
+        throw sg_io_exception("sgReadBytes: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
 }
 
@@ -323,7 +349,7 @@ void sgWriteBytes ( gzFile fd, const unsigned int n, const void *var )
 {
     if ( n == 0) return;
     if ( gzwrite ( fd, (void *)var, n ) != (int)n ) {
-        write_error = true ;
+        throw sg_io_exception("sgWriteBytes: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -333,7 +359,8 @@ void sgReadUShort ( gzFile fd, const unsigned int n, unsigned short *var )
     if ( gzread ( fd, var, sizeof(unsigned short) * n )
 	 != (int)(sizeof(unsigned short) * n) )
     {
-        read_error = true ;
+        throw sg_io_exception("sgReadUShort array: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         for ( unsigned int i = 0; i < n; ++i ) {
@@ -357,7 +384,7 @@ void sgWriteUShort ( gzFile fd, const unsigned int n, const unsigned short *var
     if ( gzwrite ( fd, (void *)var, sizeof(unsigned short) * n )
 	 != (int)(sizeof(unsigned short) * n) )
     {
-        write_error = true ;
+        throw sg_io_exception("sgWriteUShort array: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -368,7 +395,8 @@ void sgReadShort ( gzFile fd, const unsigned int n, short *var )
     if ( gzread ( fd, var, sizeof(short) * n )
 	 != (int)(sizeof(short) * n) )
     {
-        read_error = true ;
+        throw sg_io_exception("sgReadShort array: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         for ( unsigned int i = 0; i < n; ++i ) {
@@ -392,7 +420,7 @@ void sgWriteShort ( gzFile fd, const unsigned int n, const short *var )
     if ( gzwrite ( fd, (void *)var, sizeof(short) * n )
 	 != (int)(sizeof(short) * n) )
     {
-        write_error = true ;
+        throw sg_io_exception("sgWriteShort array: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -402,7 +430,8 @@ void sgReadUInt ( gzFile fd, const unsigned int n, unsigned int *var )
     if ( gzread ( fd, var, sizeof(unsigned int) * n )
 	 != (int)(sizeof(unsigned int) * n) )
     {
-        read_error = true ;
+        throw sg_io_exception("sgReadUInt array: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         for ( unsigned int i = 0; i < n; ++i ) {
@@ -426,7 +455,7 @@ void sgWriteUInt ( gzFile fd, const unsigned int n, const unsigned int *var )
     if ( gzwrite ( fd, (void *)var, sizeof(unsigned int) * n )
 	 != (int)(sizeof(unsigned int) * n) )
     {
-        write_error = true ;
+        throw sg_io_exception("sgWriteUInt array: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -437,7 +466,8 @@ void sgReadInt ( gzFile fd, const unsigned int n, int *var )
     if ( gzread ( fd, var, sizeof(int) * n )
 	 != (int)(sizeof(int) * n) )
     {
-        read_error = true ;
+        throw sg_io_exception("sgReadInt array: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         for ( unsigned int i = 0; i < n; ++i ) {
@@ -461,7 +491,7 @@ void sgWriteInt ( gzFile fd, const unsigned int n, const int *var )
     if ( gzwrite ( fd, (void *)var, sizeof(int) * n )
 	 != (int)(sizeof(int) * n) )
     {
-        write_error = true ;
+        throw sg_io_exception("sgWriteInt array: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 

simgear/io/lowlevel.hxx

@@ -35,6 +35,9 @@
 
 #include <simgear/math/SGMath.hxx>
 
+// forward decls
+class SGPath;
+
 // Note that output is written in little endian form (and converted as
 // necessary for big endian machines)
 
@@ -121,9 +124,10 @@ inline void sgWriteGeod ( gzFile fd, const SGGeod& var ) {
     sgWriteDouble( fd, var.getElevationM() );
 }
 
-void sgClearReadError();
-void sgClearWriteError();
-int sgReadError();
-int sgWriteError();
+/**
+    @ error aid: allow calling code to specify which file path we're reading from, so that erros we
+ throw from sgReadXXXX can have a valid location set.
+ */
+void setThreadLocalSimgearReadPath(const SGPath& path);
 
 #endif // _SG_LOWLEVEL_HXX

simgear/io/sg_binobj.cxx

@@ -41,9 +41,10 @@
 #include <bitset>
 
 #include <simgear/bucket/newbucket.hxx>
+#include <simgear/debug/ErrorReportingCallback.hxx>
+#include <simgear/math/SGGeometry.hxx>
 #include <simgear/misc/sg_path.hxx>
 #include <simgear/misc/strutils.hxx>
-#include <simgear/math/SGGeometry.hxx>
 #include <simgear/structure/exception.hxx>
 
 #include "lowlevel.hxx"
@@ -454,10 +455,6 @@ void SGBinObject::read_object( gzFile fp,
         }
     }
 
-    if ( sgReadError() ) {
-        throw sg_exception("Error reading object properties");
-    }
-
     size_t indexCount = std::bitset<32>((int)idx_mask).count();
     if (indexCount == 0) {
         throw sg_exception("object index mask has no bits set");
@@ -465,18 +462,10 @@ void SGBinObject::read_object( gzFile fp,
 
     for ( j = 0; j < nelements; ++j ) {
         sgReadUInt( fp, &nbytes );
-        if ( sgReadError() ) {
-            throw sg_exception("Error reading element size");
-        }
-
         buf.resize( nbytes );
         char *ptr = buf.get_ptr();
         sgReadBytes( fp, nbytes, ptr );
 
-        if ( sgReadError() ) {
-            throw sg_exception("Error reading element bytes");
-        }
-
         int_list vs;
         int_list ns;
         int_list cs;
@@ -503,312 +492,311 @@ void SGBinObject::read_object( gzFile fp,
 
 
 // read a binary file and populate the provided structures.
-bool SGBinObject::read_bin( const SGPath& file ) {
-    SGVec3d p;
-    int i, k;
-    size_t j;
-    unsigned int nbytes;
-    sgSimpleBuffer buf( 32768 );  // 32 Kb
+bool SGBinObject::read_bin( const SGPath& file )
+{
+    gzFile fp = NULL;
+    
+    try {
+        SGVec3d p;
+        int i, k;
+        size_t j;
+        unsigned int nbytes;
+        sgSimpleBuffer buf( 32768 );  // 32 Kb
 
-    // zero out structures
-    gbs_center = SGVec3d(0, 0, 0);
-    gbs_radius = 0.0;
+        simgear::ErrorReportContext ec("btg", file.utf8Str());
 
-    wgs84_nodes.clear();
-    normals.clear();
-    texcoords.clear();
+        // zero out structures
+        gbs_center = SGVec3d(0, 0, 0);
+        gbs_radius = 0.0;
 
-    pts_v.clear();
-    pts_n.clear();
-    pts_c.clear();
-    pts_tcs.clear();
-    pts_vas.clear();
-    pt_materials.clear();
+        wgs84_nodes.clear();
+        normals.clear();
+        texcoords.clear();
 
-    tris_v.clear();
-    tris_n.clear();
-    tris_c.clear();
-    tris_tcs.clear();
-    tris_vas.clear();
-    tri_materials.clear();
+        pts_v.clear();
+        pts_n.clear();
+        pts_c.clear();
+        pts_tcs.clear();
+        pts_vas.clear();
+        pt_materials.clear();
 
-    strips_v.clear();
-    strips_n.clear();
-    strips_c.clear();
-    strips_tcs.clear();
-    strips_vas.clear();
-    strip_materials.clear();
+        tris_v.clear();
+        tris_n.clear();
+        tris_c.clear();
+        tris_tcs.clear();
+        tris_vas.clear();
+        tri_materials.clear();
 
-    fans_v.clear();
-    fans_n.clear();
-    fans_c.clear();
-    fans_tcs.clear();
-    fans_vas.clear();
-    fan_materials.clear();
+        strips_v.clear();
+        strips_n.clear();
+        strips_c.clear();
+        strips_tcs.clear();
+        strips_vas.clear();
+        strip_materials.clear();
 
-    gzFile fp = gzFileFromSGPath(file, "rb");
-    if ( fp == NULL ) {
-        SGPath withGZ = file;
-        withGZ.concat(".gz");
-        fp = gzFileFromSGPath(withGZ, "rb");
-        if (fp == nullptr) {
-            SG_LOG( SG_EVENT, SG_ALERT,
-               "ERROR: opening " << file << " or " << withGZ << " for reading!");
+        fans_v.clear();
+        fans_n.clear();
+        fans_c.clear();
+        fans_tcs.clear();
+        fans_vas.clear();
+        fan_materials.clear();
 
-            throw sg_io_exception("Error opening for reading (and .gz)", sg_location(file));
+        gzFile fp = gzFileFromSGPath(file, "rb");
+        if ( fp == NULL ) {
+            SGPath withGZ = file;
+            withGZ.concat(".gz");
+            fp = gzFileFromSGPath(withGZ, "rb");
+            if (fp == nullptr) {
+                throw sg_io_exception("Error opening for reading (and .gz)", sg_location(file), {}, false);
+            }
         }
-    }
+        
+        setThreadLocalSimgearReadPath(file);
 
-    sgClearReadError();
+        // read headers
+        unsigned int header;
+        sgReadUInt( fp, &header );
 
-    // read headers
-    unsigned int header;
-    sgReadUInt( fp, &header );
 
-    if (sgReadError()) {
-        gzclose(fp);
-        throw sg_io_exception("Unable to read BTG header: " + simgear::strutils::error_string(errno), sg_location(file));
-    }
+        if ( ((header & 0xFF000000) >> 24) == 'S' &&
+             ((header & 0x00FF0000) >> 16) == 'G' ) {
 
-    if ( ((header & 0xFF000000) >> 24) == 'S' &&
-         ((header & 0x00FF0000) >> 16) == 'G' ) {
+            // read file version
+            version = (header & 0x0000FFFF);
+        } else {
+            throw sg_io_exception("Bad BTG magic/version", sg_location(file), {}, false);
+        }
 
-        // read file version
-        version = (header & 0x0000FFFF);
-    } else {
-        // close the file before we return
-        gzclose(fp);
-        throw sg_io_exception("Bad BTG magic/version", sg_location(file));
-    }
+        // read creation time
+        unsigned int foo_calendar_time;
+        sgReadUInt( fp, &foo_calendar_time );
 
-    // read creation time
-    unsigned int foo_calendar_time;
-    sgReadUInt( fp, &foo_calendar_time );
+    #if 0
+        time_t calendar_time = foo_calendar_time;
+        // The following code has a global effect on the host application
+        // and can screws up the time elsewhere.  It should be avoided
+        // unless you need this for debugging in which case you should
+        // disable it again once the debugging task is finished.
+        struct tm *local_tm;
+        local_tm = localtime( &calendar_time );
+        char time_str[256];
+        strftime( time_str, 256, "%a %b %d %H:%M:%S %Z %Y", local_tm);
+        SG_LOG( SG_EVENT, SG_DEBUG, "File created on " << time_str);
+    #endif
 
-#if 0
-    time_t calendar_time = foo_calendar_time;
-    // The following code has a global effect on the host application
-    // and can screws up the time elsewhere.  It should be avoided
-    // unless you need this for debugging in which case you should
-    // disable it again once the debugging task is finished.
-    struct tm *local_tm;
-    local_tm = localtime( &calendar_time );
-    char time_str[256];
-    strftime( time_str, 256, "%a %b %d %H:%M:%S %Z %Y", local_tm);
-    SG_LOG( SG_EVENT, SG_DEBUG, "File created on " << time_str);
-#endif
-
-    // read number of top level objects
-    int nobjects;
-    if ( version >= 10) { // version 10 extends everything to be 32-bit
-        sgReadInt( fp, &nobjects );
-    } else if ( version >= 7 ) {
-        uint16_t v;
-        sgReadUShort( fp, &v );
-        nobjects = v;
-    } else {
-        int16_t v;
-        sgReadShort( fp, &v );
-        nobjects = v;
-    }
-
-    SG_LOG(SG_IO, SG_DEBUG, "SGBinObject::read_bin Total objects to read = " << nobjects);
-
-    if ( sgReadError() ) {
-        throw sg_io_exception("Error reading BTG file header", sg_location(file));
-    }
-
-    // read in objects
-    for ( i = 0; i < nobjects; ++i ) {
-        // read object header
-        char obj_type;
-        uint32_t nproperties, nelements;
-        sgReadChar( fp, &obj_type );
-        if ( version >= 10 ) {
-            sgReadUInt( fp, &nproperties );
-            sgReadUInt( fp, &nelements );
+        // read number of top level objects
+        int nobjects;
+        if ( version >= 10) { // version 10 extends everything to be 32-bit
+            sgReadInt( fp, &nobjects );
         } else if ( version >= 7 ) {
             uint16_t v;
             sgReadUShort( fp, &v );
-            nproperties = v;
-            sgReadUShort( fp, &v );
-            nelements = v;
+            nobjects = v;
         } else {
             int16_t v;
             sgReadShort( fp, &v );
-            nproperties = v;
-            sgReadShort( fp, &v );
-            nelements = v;
+            nobjects = v;
         }
 
-        SG_LOG(SG_IO, SG_DEBUG, "SGBinObject::read_bin object " << i <<
-                " = " << (int)obj_type << " props = " << nproperties <<
-                " elements = " << nelements);
+        SG_LOG(SG_IO, SG_DEBUG, "SGBinObject::read_bin Total objects to read = " << nobjects);
 
-        if ( obj_type == SG_BOUNDING_SPHERE ) {
-            // read bounding sphere properties
-            read_properties( fp, nproperties );
 
-            // read bounding sphere elements
-            for ( j = 0; j < nelements; ++j ) {
-                sgReadUInt( fp, &nbytes );
-                buf.resize( nbytes );
-                buf.reset();
-                char *ptr = buf.get_ptr();
-                sgReadBytes( fp, nbytes, ptr );
-                gbs_center = buf.readVec3d();
-                gbs_radius = buf.readFloat();
+        // read in objects
+        for ( i = 0; i < nobjects; ++i ) {
+            // read object header
+            char obj_type;
+            uint32_t nproperties, nelements;
+            sgReadChar( fp, &obj_type );
+            if ( version >= 10 ) {
+                sgReadUInt( fp, &nproperties );
+                sgReadUInt( fp, &nelements );
+            } else if ( version >= 7 ) {
+                uint16_t v;
+                sgReadUShort( fp, &v );
+                nproperties = v;
+                sgReadUShort( fp, &v );
+                nelements = v;
+            } else {
+                int16_t v;
+                sgReadShort( fp, &v );
+                nproperties = v;
+                sgReadShort( fp, &v );
+                nelements = v;
             }
-        } else if ( obj_type == SG_VERTEX_LIST ) {
-            // read vertex list properties
-            read_properties( fp, nproperties );
 
-            // read vertex list elements
-            for ( j = 0; j < nelements; ++j ) {
-                sgReadUInt( fp, &nbytes );
-                buf.resize( nbytes );
-                buf.reset();
-                char *ptr = buf.get_ptr();
-                sgReadBytes( fp, nbytes, ptr );
-                int count = nbytes / (sizeof(float) * 3);
-                wgs84_nodes.reserve( count );
-                for ( k = 0; k < count; ++k ) {
-                    SGVec3f v = buf.readVec3f();
-                    // extend from float to double, hmmm
-                    wgs84_nodes.push_back( SGVec3d(v[0], v[1], v[2]) );
+            SG_LOG(SG_IO, SG_DEBUG, "SGBinObject::read_bin object " << i <<
+                    " = " << (int)obj_type << " props = " << nproperties <<
+                    " elements = " << nelements);
+
+            if ( obj_type == SG_BOUNDING_SPHERE ) {
+                // read bounding sphere properties
+                read_properties( fp, nproperties );
+
+                // read bounding sphere elements
+                for ( j = 0; j < nelements; ++j ) {
+                    sgReadUInt( fp, &nbytes );
+                    buf.resize( nbytes );
+                    buf.reset();
+                    char *ptr = buf.get_ptr();
+                    sgReadBytes( fp, nbytes, ptr );
+                    gbs_center = buf.readVec3d();
+                    gbs_radius = buf.readFloat();
                 }
-            }
-        } else if ( obj_type == SG_COLOR_LIST ) {
-            // read color list properties
-            read_properties( fp, nproperties );
+            } else if ( obj_type == SG_VERTEX_LIST ) {
+                // read vertex list properties
+                read_properties( fp, nproperties );
 
-            // read color list elements
-            for ( j = 0; j < nelements; ++j ) {
-                sgReadUInt( fp, &nbytes );
-                buf.resize( nbytes );
-                buf.reset();
-                char *ptr = buf.get_ptr();
-                sgReadBytes( fp, nbytes, ptr );
-                int count = nbytes / (sizeof(float) * 4);
-                colors.reserve(count);
-                for ( k = 0; k < count; ++k ) {
-                    colors.push_back( buf.readVec4f() );
+                // read vertex list elements
+                for ( j = 0; j < nelements; ++j ) {
+                    sgReadUInt( fp, &nbytes );
+                    buf.resize( nbytes );
+                    buf.reset();
+                    char *ptr = buf.get_ptr();
+                    sgReadBytes( fp, nbytes, ptr );
+                    int count = nbytes / (sizeof(float) * 3);
+                    wgs84_nodes.reserve( count );
+                    for ( k = 0; k < count; ++k ) {
+                        SGVec3f v = buf.readVec3f();
+                        // extend from float to double, hmmm
+                        wgs84_nodes.push_back( SGVec3d(v[0], v[1], v[2]) );
+                    }
                 }
-            }
-        } else if ( obj_type == SG_NORMAL_LIST ) {
-            // read normal list properties
-            read_properties( fp, nproperties );
+            } else if ( obj_type == SG_COLOR_LIST ) {
+                // read color list properties
+                read_properties( fp, nproperties );
 
-            // read normal list elements
-            for ( j = 0; j < nelements; ++j ) {
-                sgReadUInt( fp, &nbytes );
-                buf.resize( nbytes );
-                buf.reset();
-                unsigned char *ptr = (unsigned char *)(buf.get_ptr());
-                sgReadBytes( fp, nbytes, ptr );
-                int count = nbytes / 3;
-                normals.reserve( count );
-
-                for ( k = 0; k < count; ++k ) {
-                    SGVec3f normal( (ptr[0]) / 127.5 - 1.0,
-                                    (ptr[1]) / 127.5 - 1.0,
-                                    (ptr[2]) / 127.5 - 1.0);
-                    normals.push_back(normalize(normal));
-                    ptr += 3;
+                // read color list elements
+                for ( j = 0; j < nelements; ++j ) {
+                    sgReadUInt( fp, &nbytes );
+                    buf.resize( nbytes );
+                    buf.reset();
+                    char *ptr = buf.get_ptr();
+                    sgReadBytes( fp, nbytes, ptr );
+                    int count = nbytes / (sizeof(float) * 4);
+                    colors.reserve(count);
+                    for ( k = 0; k < count; ++k ) {
+                        colors.push_back( buf.readVec4f() );
+                    }
                 }
-            }
-        } else if ( obj_type == SG_TEXCOORD_LIST ) {
-            // read texcoord list properties
-            read_properties( fp, nproperties );
+            } else if ( obj_type == SG_NORMAL_LIST ) {
+                // read normal list properties
+                read_properties( fp, nproperties );
 
-            // read texcoord list elements
-            for ( j = 0; j < nelements; ++j ) {
-                sgReadUInt( fp, &nbytes );
-                buf.resize( nbytes );
-                buf.reset();
-                char *ptr = buf.get_ptr();
-                sgReadBytes( fp, nbytes, ptr );
-                int count = nbytes / (sizeof(float) * 2);
-                texcoords.reserve(count);
-                for ( k = 0; k < count; ++k ) {
-                    texcoords.push_back( buf.readVec2f() );
+                // read normal list elements
+                for ( j = 0; j < nelements; ++j ) {
+                    sgReadUInt( fp, &nbytes );
+                    buf.resize( nbytes );
+                    buf.reset();
+                    unsigned char *ptr = (unsigned char *)(buf.get_ptr());
+                    sgReadBytes( fp, nbytes, ptr );
+                    int count = nbytes / 3;
+                    normals.reserve( count );
+
+                    for ( k = 0; k < count; ++k ) {
+                        SGVec3f normal( (ptr[0]) / 127.5 - 1.0,
+                                        (ptr[1]) / 127.5 - 1.0,
+                                        (ptr[2]) / 127.5 - 1.0);
+                        normals.push_back(normalize(normal));
+                        ptr += 3;
+                    }
                 }
-            }
-        } else if ( obj_type == SG_VA_FLOAT_LIST ) {
-            // read vertex attribute (float) properties
-            read_properties( fp, nproperties );
+            } else if ( obj_type == SG_TEXCOORD_LIST ) {
+                // read texcoord list properties
+                read_properties( fp, nproperties );
 
-            // read vertex attribute list elements
-            for ( j = 0; j < nelements; ++j ) {
-                sgReadUInt( fp, &nbytes );
-                buf.resize( nbytes );
-                buf.reset();
-                char *ptr = buf.get_ptr();
-                sgReadBytes( fp, nbytes, ptr );
-                int count = nbytes / (sizeof(float));
-                va_flt.reserve(count);
-                for ( k = 0; k < count; ++k ) {
-                    va_flt.push_back( buf.readFloat() );
+                // read texcoord list elements
+                for ( j = 0; j < nelements; ++j ) {
+                    sgReadUInt( fp, &nbytes );
+                    buf.resize( nbytes );
+                    buf.reset();
+                    char *ptr = buf.get_ptr();
+                    sgReadBytes( fp, nbytes, ptr );
+                    int count = nbytes / (sizeof(float) * 2);
+                    texcoords.reserve(count);
+                    for ( k = 0; k < count; ++k ) {
+                        texcoords.push_back( buf.readVec2f() );
+                    }
                 }
-            }
-        } else if ( obj_type == SG_VA_INTEGER_LIST ) {
-            // read vertex attribute (integer) properties
-            read_properties( fp, nproperties );
+            } else if ( obj_type == SG_VA_FLOAT_LIST ) {
+                // read vertex attribute (float) properties
+                read_properties( fp, nproperties );
 
-            // read vertex attribute list elements
-            for ( j = 0; j < nelements; ++j ) {
-                sgReadUInt( fp, &nbytes );
-                buf.resize( nbytes );
-                buf.reset();
-                char *ptr = buf.get_ptr();
-                sgReadBytes( fp, nbytes, ptr );
-                int count = nbytes / (sizeof(unsigned int));
-                va_int.reserve(count);
-                for ( k = 0; k < count; ++k ) {
-                    va_int.push_back( buf.readInt() );
+                // read vertex attribute list elements
+                for ( j = 0; j < nelements; ++j ) {
+                    sgReadUInt( fp, &nbytes );
+                    buf.resize( nbytes );
+                    buf.reset();
+                    char *ptr = buf.get_ptr();
+                    sgReadBytes( fp, nbytes, ptr );
+                    int count = nbytes / (sizeof(float));
+                    va_flt.reserve(count);
+                    for ( k = 0; k < count; ++k ) {
+                        va_flt.push_back( buf.readFloat() );
+                    }
                 }
-            }
-        } else if ( obj_type == SG_POINTS ) {
-            // read point elements
-            read_object( fp, SG_POINTS, nproperties, nelements,
-                         pts_v, pts_n, pts_c, pts_tcs,
-                         pts_vas, pt_materials );
-        } else if ( obj_type == SG_TRIANGLE_FACES ) {
-            // read triangle face properties
-            read_object( fp, SG_TRIANGLE_FACES, nproperties, nelements,
-                         tris_v, tris_n, tris_c, tris_tcs,
-                         tris_vas, tri_materials );
-        } else if ( obj_type == SG_TRIANGLE_STRIPS ) {
-            // read triangle strip properties
-            read_object( fp, SG_TRIANGLE_STRIPS, nproperties, nelements,
-                         strips_v, strips_n, strips_c, strips_tcs,
-                         strips_vas, strip_materials );
-        } else if ( obj_type == SG_TRIANGLE_FANS ) {
-            // read triangle fan properties
-            read_object( fp, SG_TRIANGLE_FANS, nproperties, nelements,
-                         fans_v, fans_n, fans_c, fans_tcs,
-                         fans_vas, fan_materials );
-        } else {
-            // unknown object type, just skip
-            read_properties( fp, nproperties );
+            } else if ( obj_type == SG_VA_INTEGER_LIST ) {
+                // read vertex attribute (integer) properties
+                read_properties( fp, nproperties );
 
-            // read elements
-            for ( j = 0; j < nelements; ++j ) {
-                sgReadUInt( fp, &nbytes );
-                // cout << "element size = " << nbytes << endl;
-                if ( nbytes > buf.get_size() ) { buf.resize( nbytes ); }
-                char *ptr = buf.get_ptr();
-                sgReadBytes( fp, nbytes, ptr );
+                // read vertex attribute list elements
+                for ( j = 0; j < nelements; ++j ) {
+                    sgReadUInt( fp, &nbytes );
+                    buf.resize( nbytes );
+                    buf.reset();
+                    char *ptr = buf.get_ptr();
+                    sgReadBytes( fp, nbytes, ptr );
+                    int count = nbytes / (sizeof(unsigned int));
+                    va_int.reserve(count);
+                    for ( k = 0; k < count; ++k ) {
+                        va_int.push_back( buf.readInt() );
+                    }
+                }
+            } else if ( obj_type == SG_POINTS ) {
+                // read point elements
+                read_object( fp, SG_POINTS, nproperties, nelements,
+                             pts_v, pts_n, pts_c, pts_tcs,
+                             pts_vas, pt_materials );
+            } else if ( obj_type == SG_TRIANGLE_FACES ) {
+                // read triangle face properties
+                read_object( fp, SG_TRIANGLE_FACES, nproperties, nelements,
+                             tris_v, tris_n, tris_c, tris_tcs,
+                             tris_vas, tri_materials );
+            } else if ( obj_type == SG_TRIANGLE_STRIPS ) {
+                // read triangle strip properties
+                read_object( fp, SG_TRIANGLE_STRIPS, nproperties, nelements,
+                             strips_v, strips_n, strips_c, strips_tcs,
+                             strips_vas, strip_materials );
+            } else if ( obj_type == SG_TRIANGLE_FANS ) {
+                // read triangle fan properties
+                read_object( fp, SG_TRIANGLE_FANS, nproperties, nelements,
+                             fans_v, fans_n, fans_c, fans_tcs,
+                             fans_vas, fan_materials );
+            } else {
+                // unknown object type, just skip
+                read_properties( fp, nproperties );
+
+                // read elements
+                for ( j = 0; j < nelements; ++j ) {
+                    sgReadUInt( fp, &nbytes );
+                    // cout << "element size = " << nbytes << endl;
+                    if ( nbytes > buf.get_size() ) { buf.resize( nbytes ); }
+                    char *ptr = buf.get_ptr();
+                    sgReadBytes( fp, nbytes, ptr );
+                }
             }
         }
 
-        if ( sgReadError() ) {
-            throw sg_io_exception("Error while reading object", sg_location(file, i));
+        gzclose(fp);
+        fp = NULL;
+    } catch (std::exception&) {
+        if (fp) {
+            // close the file
+            gzclose(fp);
         }
+        
+        throw; // re-throw
     }
 
-    // close the file
-    gzclose(fp);
 
     return true;
 }
@@ -982,112 +970,112 @@ bool SGBinObject::write_bin_file(const SGPath& file)
         return false;
     }
 
-    sgClearWriteError();
+    try {
+        SG_LOG(SG_IO, SG_DEBUG, "points size = " << pts_v.size()
+             << "  pt_materials = " << pt_materials.size() );
+        SG_LOG(SG_IO, SG_DEBUG, "triangles size = " << tris_v.size()
+             << "  tri_materials = " << tri_materials.size() );
+        SG_LOG(SG_IO, SG_DEBUG, "strips size = " << strips_v.size()
+             << "  strip_materials = " << strip_materials.size() );
+        SG_LOG(SG_IO, SG_DEBUG, "fans size = " << fans_v.size()
+             << "  fan_materials = " << fan_materials.size() );
 
-    SG_LOG(SG_IO, SG_DEBUG, "points size = " << pts_v.size()
-         << "  pt_materials = " << pt_materials.size() );
-    SG_LOG(SG_IO, SG_DEBUG, "triangles size = " << tris_v.size()
-         << "  tri_materials = " << tri_materials.size() );
-    SG_LOG(SG_IO, SG_DEBUG, "strips size = " << strips_v.size()
-         << "  strip_materials = " << strip_materials.size() );
-    SG_LOG(SG_IO, SG_DEBUG, "fans size = " << fans_v.size()
-         << "  fan_materials = " << fan_materials.size() );
+        SG_LOG(SG_IO, SG_DEBUG, "nodes = " << wgs84_nodes.size() );
+        SG_LOG(SG_IO, SG_DEBUG, "colors = " << colors.size() );
+        SG_LOG(SG_IO, SG_DEBUG, "normals = " << normals.size() );
+        SG_LOG(SG_IO, SG_DEBUG, "tex coords = " << texcoords.size() );
 
-    SG_LOG(SG_IO, SG_DEBUG, "nodes = " << wgs84_nodes.size() );
-    SG_LOG(SG_IO, SG_DEBUG, "colors = " << colors.size() );
-    SG_LOG(SG_IO, SG_DEBUG, "normals = " << normals.size() );
-    SG_LOG(SG_IO, SG_DEBUG, "tex coords = " << texcoords.size() );
+        version = 10;
+        bool shortMaterialsRanges =
+            (max_object_size(pt_materials) < VERSION_7_MATERIAL_LIMIT) &&
+            (max_object_size(fan_materials) < VERSION_7_MATERIAL_LIMIT) &&
+            (max_object_size(strip_materials) < VERSION_7_MATERIAL_LIMIT) &&
+            (max_object_size(tri_materials) < VERSION_7_MATERIAL_LIMIT);
 
-    version = 10;
-    bool shortMaterialsRanges =
-        (max_object_size(pt_materials) < VERSION_7_MATERIAL_LIMIT) &&
-        (max_object_size(fan_materials) < VERSION_7_MATERIAL_LIMIT) &&
-        (max_object_size(strip_materials) < VERSION_7_MATERIAL_LIMIT) &&
-        (max_object_size(tri_materials) < VERSION_7_MATERIAL_LIMIT);
+        if ((wgs84_nodes.size() < 0xffff) &&
+            (normals.size() < 0xffff) &&
+            (texcoords.size() < 0xffff) &&
+            shortMaterialsRanges) {
+            version = 7; // use smaller indices if possible
+        }
 
-    if ((wgs84_nodes.size() < 0xffff) &&
-        (normals.size() < 0xffff) &&
-        (texcoords.size() < 0xffff) &&
-        shortMaterialsRanges) {
-        version = 7; // use smaller indices if possible
-    }
+        // write header magic
 
-    // write header magic
+        /** Magic Number for our file format */
+        #define SG_FILE_MAGIC_NUMBER  ( ('S'<<24) + ('G'<<16) + version )
 
-    /** Magic Number for our file format */
-    #define SG_FILE_MAGIC_NUMBER  ( ('S'<<24) + ('G'<<16) + version )
+        sgWriteUInt( fp, SG_FILE_MAGIC_NUMBER );
+        time_t calendar_time = time(NULL);
+        sgWriteLong( fp, (int32_t)calendar_time );
 
-    sgWriteUInt( fp, SG_FILE_MAGIC_NUMBER );
-    time_t calendar_time = time(NULL);
-    sgWriteLong( fp, (int32_t)calendar_time );
+        // calculate and write number of top level objects
+        int nobjects = 5; // gbs, vertices, colors, normals, texcoords
+        nobjects += count_objects(pt_materials);
+        nobjects += count_objects(tri_materials);
+        nobjects += count_objects(strip_materials);
+        nobjects += count_objects(fan_materials);
 
-    // calculate and write number of top level objects
-    int nobjects = 5; // gbs, vertices, colors, normals, texcoords
-    nobjects += count_objects(pt_materials);
-    nobjects += count_objects(tri_materials);
-    nobjects += count_objects(strip_materials);
-    nobjects += count_objects(fan_materials);
+        SG_LOG(SG_IO, SG_DEBUG, "total top level objects = " << nobjects);
 
-    SG_LOG(SG_IO, SG_DEBUG, "total top level objects = " << nobjects);
+        if (version == 7) {
+            sgWriteUShort( fp, (uint16_t) nobjects );
+        } else {
+            sgWriteInt( fp, nobjects );
+        }
 
-    if (version == 7) {
-        sgWriteUShort( fp, (uint16_t) nobjects );
-    } else {
-        sgWriteInt( fp, nobjects );
-    }
+        // write bounding sphere
+        write_header( fp, SG_BOUNDING_SPHERE, 0, 1);
+        sgWriteUInt( fp, sizeof(double) * 3 + sizeof(float) ); // nbytes
+        sgWritedVec3( fp, gbs_center );
+        sgWriteFloat( fp, gbs_radius );
 
-    // write bounding sphere
-    write_header( fp, SG_BOUNDING_SPHERE, 0, 1);
-    sgWriteUInt( fp, sizeof(double) * 3 + sizeof(float) ); // nbytes
-    sgWritedVec3( fp, gbs_center );
-    sgWriteFloat( fp, gbs_radius );
+        // dump vertex list
+        write_header( fp, SG_VERTEX_LIST, 0, 1);
+        sgWriteUInt( fp, wgs84_nodes.size() * sizeof(float) * 3 ); // nbytes
+        for ( i = 0; i < (int)wgs84_nodes.size(); ++i ) {
+            sgWriteVec3( fp, toVec3f(wgs84_nodes[i] - gbs_center));
+        }
 
-    // dump vertex list
-    write_header( fp, SG_VERTEX_LIST, 0, 1);
-    sgWriteUInt( fp, wgs84_nodes.size() * sizeof(float) * 3 ); // nbytes
-    for ( i = 0; i < (int)wgs84_nodes.size(); ++i ) {
-        sgWriteVec3( fp, toVec3f(wgs84_nodes[i] - gbs_center));
-    }
+        // dump vertex color list
+        write_header( fp, SG_COLOR_LIST, 0, 1);
+        sgWriteUInt( fp, colors.size() * sizeof(float) * 4 ); // nbytes
+        for ( i = 0; i < (int)colors.size(); ++i ) {
+          sgWriteVec4( fp, colors[i]);
+        }
 
-    // dump vertex color list
-    write_header( fp, SG_COLOR_LIST, 0, 1);
-    sgWriteUInt( fp, colors.size() * sizeof(float) * 4 ); // nbytes
-    for ( i = 0; i < (int)colors.size(); ++i ) {
-      sgWriteVec4( fp, colors[i]);
-    }
+        // dump vertex normal list
+        write_header( fp, SG_NORMAL_LIST, 0, 1);
+        sgWriteUInt( fp, normals.size() * 3 );              // nbytes
+        char normal[3];
+        for ( i = 0; i < (int)normals.size(); ++i ) {
+            SGVec3f p = normals[i];
+            normal[0] = (unsigned char)((p.x() + 1.0) * 127.5);
+            normal[1] = (unsigned char)((p.y() + 1.0) * 127.5);
+            normal[2] = (unsigned char)((p.z() + 1.0) * 127.5);
+            sgWriteBytes( fp, 3, normal );
+        }
 
-    // dump vertex normal list
-    write_header( fp, SG_NORMAL_LIST, 0, 1);
-    sgWriteUInt( fp, normals.size() * 3 );              // nbytes
-    char normal[3];
-    for ( i = 0; i < (int)normals.size(); ++i ) {
-        SGVec3f p = normals[i];
-        normal[0] = (unsigned char)((p.x() + 1.0) * 127.5);
-        normal[1] = (unsigned char)((p.y() + 1.0) * 127.5);
-        normal[2] = (unsigned char)((p.z() + 1.0) * 127.5);
-        sgWriteBytes( fp, 3, normal );
-    }
+        // dump texture coordinates
+        write_header( fp, SG_TEXCOORD_LIST, 0, 1);
+        sgWriteUInt( fp, texcoords.size() * sizeof(float) * 2 ); // nbytes
+        for ( i = 0; i < (int)texcoords.size(); ++i ) {
+          sgWriteVec2( fp, texcoords[i]);
+        }
 
-    // dump texture coordinates
-    write_header( fp, SG_TEXCOORD_LIST, 0, 1);
-    sgWriteUInt( fp, texcoords.size() * sizeof(float) * 2 ); // nbytes
-    for ( i = 0; i < (int)texcoords.size(); ++i ) {
-      sgWriteVec2( fp, texcoords[i]);
-    }
+        write_objects(fp, SG_POINTS, pts_v, pts_n, pts_c, pts_tcs, pts_vas, pt_materials);
+        write_objects(fp, SG_TRIANGLE_FACES, tris_v, tris_n, tris_c, tris_tcs, tris_vas, tri_materials);
+        write_objects(fp, SG_TRIANGLE_STRIPS, strips_v, strips_n, strips_c, strips_tcs, strips_vas, strip_materials);
+        write_objects(fp, SG_TRIANGLE_FANS, fans_v, fans_n, fans_c, fans_tcs, fans_vas, fan_materials);
 
-    write_objects(fp, SG_POINTS, pts_v, pts_n, pts_c, pts_tcs, pts_vas, pt_materials);
-    write_objects(fp, SG_TRIANGLE_FACES, tris_v, tris_n, tris_c, tris_tcs, tris_vas, tri_materials);
-    write_objects(fp, SG_TRIANGLE_STRIPS, strips_v, strips_n, strips_c, strips_tcs, strips_vas, strip_materials);
-    write_objects(fp, SG_TRIANGLE_FANS, fans_v, fans_n, fans_c, fans_tcs, fans_vas, fan_materials);
-
-    // close the file
-    gzclose(fp);
-
-    if ( sgWriteError() ) {
-        cout << "Error while writing file " << file << endl;
+        // close the file
+        gzclose(fp);
+        fp = NULL;
+    } catch (std::exception&) {
+        if (fp) {
+            gzclose(fp);
+        }
         return false;
     }
-
     return true;
 }
 

simgear/io/sg_file.cxx

@@ -82,7 +82,7 @@ std::string SGFile::computeHash()
         [](char* p) { free(p); }};
 
     if (!buf) {
-        SG_LOG(SG_IO, SG_ALERT, "Failed to malloc buffer for SHA1 check");
+        throw sg_exception("Malloc of SHA buffer failed", {}, file_name);
     }
 
     size_t readLen;

simgear/io/test_repository.cxx

@@ -31,6 +31,10 @@ using TestApi = simgear::HTTP::TestApi;
 
 std::string dataForFile(const std::string& parentName, const std::string& name, int revision)
 {
+    if (name == "zeroByteFile") {
+        return {};
+    }
+
     std::ostringstream os;
     // random content but which definitely depends on our tree location
     // and revision.
@@ -446,6 +450,7 @@ void testBasicClone(HTTP::Client* cl)
     verifyFileState(p, "fileA");
     verifyFileState(p, "dirA/subdirA/fileAAA");
     verifyFileState(p, "dirC/subdirA/subsubA/fileCAAA");
+    verifyFileState(p, "dirA/subdirA/zeroByteFile");
 
     global_repo->findEntry("fileA")->revision++;
     global_repo->findEntry("dirB/subdirA/fileBAA")->revision++;
@@ -896,6 +901,27 @@ void testPersistentSocketFailure(HTTP::Client *cl) {
   verifyRequestCount("dirD/subdirDB/fileDBA", 1);
 }
 
+void testHashOnEmptyFile()
+{
+    std::unique_ptr<HTTPRepository> repo;
+    SGPath p(simgear::Dir::current().path());
+    p.append("sgfile_compute_hash");
+    simgear::Dir pd(p);
+    if (pd.exists()) {
+        pd.removeChildren();
+    } else {
+        pd.create(0700);
+    }
+
+    SGPath fPath = p / "test_empty_file";
+    
+    SGFile file(fPath);
+    file.open(SG_IO_OUT);
+    file.close();
+
+    const auto hash = file.computeHash();
+}
+
 int main(int argc, char* argv[])
 {
   sglog().setLogLevels( SG_ALL, SG_INFO );
@@ -911,6 +937,8 @@ int main(int argc, char* argv[])
     global_repo->defineFile("dirA/fileAC");
     global_repo->defineFile("dirA/subdirA/fileAAA");
     global_repo->defineFile("dirA/subdirA/fileAAB");
+    global_repo->defineFile("dirA/subdirA/zeroByteFile");
+
     global_repo->defineFile("dirB/subdirA/fileBAA");
     global_repo->defineFile("dirB/subdirA/fileBAB");
     global_repo->defineFile("dirB/subdirA/fileBAC");
@@ -944,6 +972,8 @@ int main(int argc, char* argv[])
     testRetryAfterSocketFailure(&cl);
     testPersistentSocketFailure(&cl);
 
+    testHashOnEmptyFile();
+
     std::cout << "all tests passed ok" << std::endl;
     return 0;
 }

simgear/io/test_untar.cxx

@@ -32,7 +32,7 @@ void testTarGz()
     uint8_t* buf = (uint8_t*) alloca(8192);
     size_t bufSize = f.read((char*) buf, 8192);
 
-    SG_VERIFY(ArchiveExtractor::determineType(buf, bufSize) == ArchiveExtractor::TarData);
+    SG_VERIFY(ArchiveExtractor::determineType(buf, bufSize) == ArchiveExtractor::GZData);
 
     f.close();
 }
@@ -174,6 +174,34 @@ void testPAXAttributes()
 
 }
 
+void testExtractXZ()
+{
+    SGPath p = SGPath(SRC_DIR);
+    p.append("test.tar.xz");
+
+    SGBinaryFile f(p);
+    f.open(SG_IO_IN);
+
+    SGPath extractDir = simgear::Dir::current().path() / "test_extract_xz";
+    simgear::Dir pd(extractDir);
+    pd.removeChildren();
+
+    ArchiveExtractor ex(extractDir);
+
+    uint8_t* buf = (uint8_t*)alloca(128);
+    while (!f.eof()) {
+        size_t bufSize = f.read((char*)buf, 128);
+        ex.extractBytes(buf, bufSize);
+    }
+
+    ex.flush();
+    SG_VERIFY(ex.isAtEndOfArchive());
+    SG_VERIFY(ex.hasError() == false);
+
+    SG_VERIFY((extractDir / "testDir/hello.c").exists());
+    SG_VERIFY((extractDir / "testDir/foo.txt").exists());
+}
+
 int main(int ac, char ** av)
 {
     testTarGz();
@@ -181,7 +209,8 @@ int main(int ac, char ** av)
     testFilterTar();
 	testExtractStreamed();
 	testExtractZip();
-   
+    testExtractXZ();
+
     // disabled to avoiding checking in large PAX archive
     // testPAXAttributes();
     

simgear/io/untar.cxx

@@ -38,82 +38,11 @@
 #include <simgear/package/unzip.h>
 #include <simgear/structure/exception.hxx>
 
+#include "ArchiveExtractor_private.hxx"
+
 namespace simgear
 {
 
-	class ArchiveExtractorPrivate
-	{
-	public:
-		ArchiveExtractorPrivate(ArchiveExtractor* o) :
-			outer(o)
-		{
-			assert(outer);
-		}
-
-        virtual ~ArchiveExtractorPrivate() = default;
-        
-		typedef enum {
-			INVALID = 0,
-			READING_HEADER,
-			READING_FILE,
-			READING_PADDING,
-            READING_PAX_GLOBAL_ATTRIBUTES,
-            READING_PAX_FILE_ATTRIBUTES,
-			PRE_END_OF_ARCHVE,
-			END_OF_ARCHIVE,
-			ERROR_STATE, ///< states above this are error conditions
-			BAD_ARCHIVE,
-			BAD_DATA,
-			FILTER_STOPPED
-		} State;
-
-		State state = INVALID;
-		ArchiveExtractor* outer = nullptr;
-
-		virtual void extractBytes(const uint8_t* bytes, size_t count) = 0;
-
-		virtual void flush() = 0;
-
-		SGPath extractRootPath()
-		{
-			return outer->_rootPath;
-		}
-
-		ArchiveExtractor::PathResult filterPath(std::string& pathToExtract)
-		{
-			return outer->filterPath(pathToExtract);
-		}
-
-
-		bool isSafePath(const std::string& p) const
-		{
-			if (p.empty()) {
-				return false;
-			}
-
-			// reject absolute paths
-			if (p.at(0) == '/') {
-				return false;
-			}
-
-			// reject paths containing '..'
-			size_t doubleDot = p.find("..");
-			if (doubleDot != std::string::npos) {
-				return false;
-			}
-
-			// on POSIX could use realpath to sanity check
-			return true;
-		}
-	};
-
-///////////////////////////////////////////////////////////////////////////////////////////////////
-
-const int ZLIB_DECOMPRESS_BUFFER_SIZE = 32 * 1024;
-const int ZLIB_INFLATE_WINDOW_BITS = MAX_WBITS;
-const int ZLIB_DECODE_GZIP_HEADER = 16;
-
-/* tar Header Block, from POSIX 1003.1-1990.  */
 
 typedef struct
 {
@@ -153,320 +82,413 @@ typedef struct
 #define FIFOTYPE '6'            /* FIFO special */
 #define CONTTYPE '7'            /* reserved */
 
-	const char PAX_GLOBAL_HEADER = 'g';
-	const char PAX_FILE_ATTRIBUTES = 'x';
+    const char PAX_GLOBAL_HEADER = 'g';
+    const char PAX_FILE_ATTRIBUTES = 'x';
 
 
-class TarExtractorPrivate : public ArchiveExtractorPrivate
+    ///////////////////////////////////////////////////////////////////////////////////////////////////
+
+    const int ZLIB_DECOMPRESS_BUFFER_SIZE = 32 * 1024;
+    const int ZLIB_INFLATE_WINDOW_BITS = MAX_WBITS;
+    const int ZLIB_DECODE_GZIP_HEADER = 16;
+
+    /* tar Header Block, from POSIX 1003.1-1990.  */
+
+
+    class TarExtractorPrivate : public ArchiveExtractorPrivate
+    {
+    public:
+        union {
+            UstarHeaderBlock header;
+            uint8_t headerBytes[TAR_HEADER_BLOCK_SIZE];
+        };
+
+        size_t bytesRemaining;
+        std::unique_ptr<SGFile> currentFile;
+        size_t currentFileSize;
+
+        uint8_t* headerPtr;
+        bool skipCurrentEntry = false;
+        std::string paxAttributes;
+        std::string paxPathName;
+
+        TarExtractorPrivate(ArchiveExtractor* o) : ArchiveExtractorPrivate(o)
+        {
+            setState(TarExtractorPrivate::READING_HEADER);
+        }
+
+        ~TarExtractorPrivate() = default;
+
+        void readPaddingIfRequired()
+        {
+            size_t pad = currentFileSize % TAR_HEADER_BLOCK_SIZE;
+            if (pad) {
+                bytesRemaining = TAR_HEADER_BLOCK_SIZE - pad;
+                setState(READING_PADDING);
+            } else {
+                setState(READING_HEADER);
+            }
+        }
+
+        void checkEndOfState()
+        {
+            if (bytesRemaining > 0) {
+                return;
+            }
+
+            if (state == READING_FILE) {
+                if (currentFile) {
+                    currentFile->close();
+                    currentFile.reset();
+                }
+                readPaddingIfRequired();
+            } else if (state == READING_HEADER) {
+                processHeader();
+            } else if (state == PRE_END_OF_ARCHVE) {
+                if (headerIsAllZeros()) {
+                    setState(END_OF_ARCHIVE);
+                } else {
+                    // what does the spec say here?
+                }
+            } else if (state == READING_PAX_GLOBAL_ATTRIBUTES) {
+                parsePAXAttributes(true);
+                readPaddingIfRequired();
+            } else if (state == READING_PAX_FILE_ATTRIBUTES) {
+                parsePAXAttributes(false);
+                readPaddingIfRequired();
+            } else if (state == READING_PADDING) {
+                setState(READING_HEADER);
+            }
+        }
+
+        void setState(State newState)
+        {
+            if ((newState == READING_HEADER) || (newState == PRE_END_OF_ARCHVE)) {
+                bytesRemaining = TAR_HEADER_BLOCK_SIZE;
+                headerPtr = headerBytes;
+            }
+
+            state = newState;
+        }
+
+        void extractBytes(const uint8_t* bytes, size_t count) override
+        {
+            // uncompressed, just pass through directly
+            processBytes((const char*)bytes, count);
+        }
+
+        void flush() override
+        {
+            // no-op for tar files, we process everything greedily
+        }
+
+        void processHeader()
+        {
+            if (headerIsAllZeros()) {
+                if (state == PRE_END_OF_ARCHVE) {
+                    setState(END_OF_ARCHIVE);
+                } else {
+                    setState(PRE_END_OF_ARCHVE);
+                }
+                return;
+            }
+
+            if (strncmp(header.magic, TMAGIC, TMAGLEN) != 0) {
+                SG_LOG(SG_IO, SG_WARN, "Untar: magic is wrong");
+                state = BAD_ARCHIVE;
+                return;
+            }
+
+            skipCurrentEntry = false;
+            std::string tarPath = std::string(header.prefix) + std::string(header.fileName);
+            if (!paxPathName.empty()) {
+                tarPath = paxPathName;
+                paxPathName.clear(); // clear for next file
+            }
+
+            if (!isSafePath(tarPath)) {
+                SG_LOG(SG_IO, SG_WARN, "unsafe tar path, skipping::" << tarPath);
+                skipCurrentEntry = true;
+            }
+
+            auto result = filterPath(tarPath);
+            if (result == ArchiveExtractor::Stop) {
+                setState(FILTER_STOPPED);
+                return;
+            } else if (result == ArchiveExtractor::Skipped) {
+                skipCurrentEntry = true;
+            }
+
+            SGPath p = extractRootPath() / tarPath;
+            if (header.typeflag == DIRTYPE) {
+                if (!skipCurrentEntry) {
+                    Dir dir(p);
+                    dir.create(0755);
+                }
+                setState(READING_HEADER);
+            } else if ((header.typeflag == REGTYPE) || (header.typeflag == AREGTYPE)) {
+                currentFileSize = ::strtol(header.size, NULL, 8);
+                bytesRemaining = currentFileSize;
+                if (!skipCurrentEntry) {
+                    currentFile.reset(new SGBinaryFile(p));
+                    currentFile->open(SG_IO_OUT);
+                }
+                setState(READING_FILE);
+            } else if (header.typeflag == PAX_GLOBAL_HEADER) {
+                setState(READING_PAX_GLOBAL_ATTRIBUTES);
+                currentFileSize = ::strtol(header.size, NULL, 8);
+                bytesRemaining = currentFileSize;
+                paxAttributes.clear();
+            } else if (header.typeflag == PAX_FILE_ATTRIBUTES) {
+                setState(READING_PAX_FILE_ATTRIBUTES);
+                currentFileSize = ::strtol(header.size, NULL, 8);
+                bytesRemaining = currentFileSize;
+                paxAttributes.clear();
+            } else if ((header.typeflag == SYMTYPE) || (header.typeflag == LNKTYPE)) {
+                SG_LOG(SG_IO, SG_WARN, "Tarball contains a link or symlink, will be skipped:" << tarPath);
+                skipCurrentEntry = true;
+                setState(READING_HEADER);
+            } else {
+                SG_LOG(SG_IO, SG_WARN, "Unsupported tar file type:" << header.typeflag);
+                state = BAD_ARCHIVE;
+            }
+        }
+
+        void processBytes(const char* bytes, size_t count)
+        {
+            if ((state >= ERROR_STATE) || (state == END_OF_ARCHIVE)) {
+                return;
+            }
+
+            size_t curBytes = std::min(bytesRemaining, count);
+            if (state == READING_FILE) {
+                if (currentFile) {
+                    currentFile->write(bytes, curBytes);
+                }
+                bytesRemaining -= curBytes;
+            } else if ((state == READING_HEADER) || (state == PRE_END_OF_ARCHVE) || (state == END_OF_ARCHIVE)) {
+                memcpy(headerPtr, bytes, curBytes);
+                bytesRemaining -= curBytes;
+                headerPtr += curBytes;
+            } else if (state == READING_PADDING) {
+                bytesRemaining -= curBytes;
+            } else if ((state == READING_PAX_FILE_ATTRIBUTES) || (state == READING_PAX_GLOBAL_ATTRIBUTES)) {
+                bytesRemaining -= curBytes;
+                paxAttributes.append(bytes, curBytes);
+            }
+
+            checkEndOfState();
+            if (count > curBytes) {
+                // recurse with unprocessed bytes
+                processBytes(bytes + curBytes, count - curBytes);
+            }
+        }
+
+        bool headerIsAllZeros() const
+        {
+            char* headerAsChar = (char*)&header;
+            for (size_t i = 0; i < offsetof(UstarHeaderBlock, magic); ++i) {
+                if (*headerAsChar++ != 0) {
+                    return false;
+                }
+            }
+
+            return true;
+        }
+
+        // https://www.ibm.com/support/knowledgecenter/en/SSLTBW_2.3.0/com.ibm.zos.v2r3.bpxa500/paxex.htm#paxex
+        void parsePAXAttributes(bool areGlobal)
+        {
+            auto lineStart = 0;
+            for (;;) {
+                auto firstSpace = paxAttributes.find(' ', lineStart);
+                auto firstEq = paxAttributes.find('=', lineStart);
+                if ((firstEq == std::string::npos) || (firstSpace == std::string::npos)) {
+                    SG_LOG(SG_IO, SG_WARN, "Malfroemd PAX attributes in tarfile");
+                    break;
+                }
+
+                uint32_t lengthBytes = std::stoul(paxAttributes.substr(lineStart, firstSpace));
+                uint32_t dataBytes = lengthBytes - (firstEq + 1);
+                std::string name = paxAttributes.substr(firstSpace + 1, firstEq - (firstSpace + 1));
+
+                // dataBytes - 1 here to trim off the trailing newline
+                std::string data = paxAttributes.substr(firstEq + 1, dataBytes - 1);
+
+                processPAXAttribute(areGlobal, name, data);
+
+                lineStart += lengthBytes;
+            }
+        }
+
+        void processPAXAttribute(bool isGlobalAttr, const std::string& attrName, const std::string& data)
+        {
+            if (!isGlobalAttr && (attrName == "path")) {
+                // data is UTF-8 encoded path name
+                paxPathName = data;
+            }
+        }
+};
+
+///////////////////////////////////////////////////////////////////////////////
+
+
+class GZTarExtractor : public TarExtractorPrivate
 {
 public:
-    
-    union {
-        UstarHeaderBlock header;
-        uint8_t headerBytes[TAR_HEADER_BLOCK_SIZE];
-    };
-
-    size_t bytesRemaining;
-    std::unique_ptr<SGFile> currentFile;
-    size_t currentFileSize;
-    z_stream zlibStream;
-    uint8_t* zlibOutput;
-    bool haveInitedZLib = false;
-    bool uncompressedData = false; // set if reading a plain .tar (not tar.gz)
-    uint8_t* headerPtr;
-    bool skipCurrentEntry = false;
-    std::string paxAttributes;
-    std::string paxPathName;
-    
-    TarExtractorPrivate(ArchiveExtractor* o) :
-		ArchiveExtractorPrivate(o)
+    GZTarExtractor(ArchiveExtractor* outer) : TarExtractorPrivate(outer)
     {
-		memset(&zlibStream, 0, sizeof(z_stream));
-		zlibOutput = (unsigned char*)malloc(ZLIB_DECOMPRESS_BUFFER_SIZE);
-		zlibStream.zalloc = Z_NULL;
-		zlibStream.zfree = Z_NULL;
-		zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE;
-		zlibStream.next_out = zlibOutput;
+        memset(&zlibStream, 0, sizeof(z_stream));
+        zlibOutput = (unsigned char*)malloc(ZLIB_DECOMPRESS_BUFFER_SIZE);
+        zlibStream.zalloc = Z_NULL;
+        zlibStream.zfree = Z_NULL;
+        zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE;
+        zlibStream.next_out = zlibOutput;
     }
 
-    ~TarExtractorPrivate()
+    ~GZTarExtractor()
     {
+        if (haveInitedZLib) {
+            inflateEnd(&zlibStream);
+        }
         free(zlibOutput);
     }
 
-    void readPaddingIfRequired()
+    void extractBytes(const uint8_t* bytes, size_t count) override
     {
-        size_t pad = currentFileSize % TAR_HEADER_BLOCK_SIZE;
-        if (pad) {
-            bytesRemaining = TAR_HEADER_BLOCK_SIZE - pad;
-            setState(READING_PADDING);
-        } else {
-            setState(READING_HEADER);
-        }
-    }
-    
-    void checkEndOfState()
-    {
-        if (bytesRemaining > 0) {
-            return;
-        }
+        zlibStream.next_in = (uint8_t*)bytes;
+        zlibStream.avail_in = count;
 
-        if (state == READING_FILE) {
-            if (currentFile) {
-                currentFile->close();
-                currentFile.reset();
-            }
-            readPaddingIfRequired();
-        } else if (state == READING_HEADER) {
-            processHeader();
-        } else if (state == PRE_END_OF_ARCHVE) {
-            if (headerIsAllZeros()) {
-                setState(END_OF_ARCHIVE);
+        if (!haveInitedZLib) {
+            // now we have data, see if we're dealing with GZ-compressed data or not
+            if ((bytes[0] == 0x1f) && (bytes[1] == 0x8b)) {
+                // GZIP identification bytes
+                if (inflateInit2(&zlibStream, ZLIB_INFLATE_WINDOW_BITS | ZLIB_DECODE_GZIP_HEADER) != Z_OK) {
+                    SG_LOG(SG_IO, SG_WARN, "inflateInit2 failed");
+                    state = TarExtractorPrivate::BAD_DATA;
+                    return;
+                }
             } else {
-                // what does the spec say here?
+                // set error state
+                state = TarExtractorPrivate::BAD_DATA;
+                return;
             }
-        } else if (state == READING_PAX_GLOBAL_ATTRIBUTES) {
-            parsePAXAttributes(true);
-            readPaddingIfRequired();
-        } else if (state == READING_PAX_FILE_ATTRIBUTES) {
-            parsePAXAttributes(false);
-            readPaddingIfRequired();
-        } else if (state == READING_PADDING) {
-            setState(READING_HEADER);
-        }
-    }
 
-    void setState(State newState)
-    {
-        if ((newState == READING_HEADER) || (newState == PRE_END_OF_ARCHVE)) {
-            bytesRemaining = TAR_HEADER_BLOCK_SIZE;
-            headerPtr = headerBytes;
-        }
+            haveInitedZLib = true;
+            setState(TarExtractorPrivate::READING_HEADER);
+        } // of init on first-bytes case
 
-        state = newState;
-    }
+        size_t writtenSize;
+        // loop, running zlib() inflate and sending output bytes to
+        // our request body handler. Keep calling inflate until no bytes are
+        // written, and ZLIB has consumed all available input
+        do {
+            zlibStream.next_out = zlibOutput;
+            zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE;
+            int result = inflate(&zlibStream, Z_NO_FLUSH);
+            if (result == Z_OK || result == Z_STREAM_END) {
+                // nothing to do
 
-	void extractBytes(const uint8_t* bytes, size_t count) override
-	{
-		zlibStream.next_in = (uint8_t*) bytes;
-		zlibStream.avail_in = count;
-
-		if (!haveInitedZLib) {
-			// now we have data, see if we're dealing with GZ-compressed data or not
-			if ((bytes[0] == 0x1f) && (bytes[1] == 0x8b)) {
-				// GZIP identification bytes
-				if (inflateInit2(&zlibStream, ZLIB_INFLATE_WINDOW_BITS | ZLIB_DECODE_GZIP_HEADER) != Z_OK) {
-					SG_LOG(SG_IO, SG_WARN, "inflateInit2 failed");
-					state = TarExtractorPrivate::BAD_DATA;
-					return;
-				}
-			} else {
-				UstarHeaderBlock* header = (UstarHeaderBlock*)bytes;
-				if (strncmp(header->magic, TMAGIC, TMAGLEN) != 0) {
-					SG_LOG(SG_IO, SG_WARN, "didn't find tar magic in header");
-					state = TarExtractorPrivate::BAD_DATA;
-					return;
-				}
-
-				uncompressedData = true;
-			}
-
-			haveInitedZLib = true;
-			setState(TarExtractorPrivate::READING_HEADER);
-		} // of init on first-bytes case
-
-		if (uncompressedData) {
-			processBytes((const char*) bytes, count);
-		} else {
-			size_t writtenSize;
-			// loop, running zlib() inflate and sending output bytes to
-			// our request body handler. Keep calling inflate until no bytes are
-			// written, and ZLIB has consumed all available input
-			do {
-				zlibStream.next_out = zlibOutput;
-				zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE;
-				int result = inflate(&zlibStream, Z_NO_FLUSH);
-				if (result == Z_OK || result == Z_STREAM_END) {
-					// nothing to do
-
-				}
-				else if (result == Z_BUF_ERROR) {
-					// transient error, fall through
-				}
-				else {
-					//  _error = result;
-					SG_LOG(SG_IO, SG_WARN, "Permanent ZLib error:" << zlibStream.msg);
-					state = TarExtractorPrivate::BAD_DATA;
-					return;
-				}
-
-				writtenSize = ZLIB_DECOMPRESS_BUFFER_SIZE - zlibStream.avail_out;
-				if (writtenSize > 0) {
-					processBytes((const char*) zlibOutput, writtenSize);
-				}
-
-				if (result == Z_STREAM_END) {
-					break;
-				}
-			} while ((zlibStream.avail_in > 0) || (writtenSize > 0));
-		} // of Zlib-compressed data
-	}
-
-	void flush() override
-	{
-		// no-op for tar files, we process everything greedily
-	}
-
-    void processHeader()
-    {
-        if (headerIsAllZeros()) {
-            if (state == PRE_END_OF_ARCHVE) {
-                setState(END_OF_ARCHIVE);
+            } else if (result == Z_BUF_ERROR) {
+                // transient error, fall through
             } else {
-                setState(PRE_END_OF_ARCHVE);
+                //  _error = result;
+                SG_LOG(SG_IO, SG_WARN, "Permanent ZLib error:" << zlibStream.msg);
+                state = TarExtractorPrivate::BAD_DATA;
+                return;
             }
-            return;
-        }
 
-        if (strncmp(header.magic, TMAGIC, TMAGLEN) != 0) {
-            SG_LOG(SG_IO, SG_WARN, "Untar: magic is wrong");
-            state = BAD_ARCHIVE;
-            return;
-        }
-
-        skipCurrentEntry = false;
-        std::string tarPath = std::string(header.prefix) + std::string(header.fileName);
-        if (!paxPathName.empty()) {
-            tarPath = paxPathName;
-            paxPathName.clear(); // clear for next file
-        }
-        
-        if (!isSafePath(tarPath)) {
-            SG_LOG(SG_IO, SG_WARN, "unsafe tar path, skipping::" << tarPath);
-            skipCurrentEntry = true;
-        }
-
-        auto result = filterPath(tarPath);
-        if (result == ArchiveExtractor::Stop) {
-            setState(FILTER_STOPPED);
-            return;
-        } else if (result == ArchiveExtractor::Skipped) {
-            skipCurrentEntry = true;
-        }
-        
-        SGPath p = extractRootPath() / tarPath;
-        if (header.typeflag == DIRTYPE) {
-            if (!skipCurrentEntry) {
-                Dir dir(p);
-                dir.create(0755);
+            writtenSize = ZLIB_DECOMPRESS_BUFFER_SIZE - zlibStream.avail_out;
+            if (writtenSize > 0) {
+                processBytes((const char*)zlibOutput, writtenSize);
             }
-            setState(READING_HEADER);
-        } else if ((header.typeflag == REGTYPE) || (header.typeflag == AREGTYPE)) {
-            currentFileSize = ::strtol(header.size, NULL, 8);
-            bytesRemaining = currentFileSize;
-            if (!skipCurrentEntry) {
-                currentFile.reset(new SGBinaryFile(p));
-                currentFile->open(SG_IO_OUT);
-            }
-            setState(READING_FILE);
-        } else if (header.typeflag == PAX_GLOBAL_HEADER) {
-            setState(READING_PAX_GLOBAL_ATTRIBUTES);
-            currentFileSize = ::strtol(header.size, NULL, 8);
-            bytesRemaining = currentFileSize;
-            paxAttributes.clear();
-        } else if (header.typeflag == PAX_FILE_ATTRIBUTES) {
-            setState(READING_PAX_FILE_ATTRIBUTES);
-            currentFileSize = ::strtol(header.size, NULL, 8);
-            bytesRemaining = currentFileSize;
-            paxAttributes.clear();
-        } else if ((header.typeflag == SYMTYPE) || (header.typeflag == LNKTYPE)) {
-            SG_LOG(SG_IO, SG_WARN, "Tarball contains a link or symlink, will be skipped:" << tarPath);
-            skipCurrentEntry = true;
-            setState(READING_HEADER);
-        } else {
-            SG_LOG(SG_IO, SG_WARN, "Unsupported tar file type:" << header.typeflag);
-            state = BAD_ARCHIVE;
-        }
-    }
 
-    void processBytes(const char* bytes, size_t count)
-    {
-        if ((state >= ERROR_STATE) || (state == END_OF_ARCHIVE)) {
-            return;
-        }
-
-        size_t curBytes = std::min(bytesRemaining, count);
-        if (state == READING_FILE) {
-            if (currentFile) {
-                currentFile->write(bytes, curBytes);
-            }
-            bytesRemaining -= curBytes;
-        } else if ((state == READING_HEADER) || (state == PRE_END_OF_ARCHVE) || (state == END_OF_ARCHIVE)) {
-            memcpy(headerPtr, bytes, curBytes);
-            bytesRemaining -= curBytes;
-            headerPtr += curBytes;
-        } else if (state == READING_PADDING) {
-            bytesRemaining -= curBytes;
-        } else if ((state == READING_PAX_FILE_ATTRIBUTES) || (state == READING_PAX_GLOBAL_ATTRIBUTES)) {
-            bytesRemaining -= curBytes;
-            paxAttributes.append(bytes, curBytes);
-        }
-
-        checkEndOfState();
-        if (count > curBytes) {
-            // recurse with unprocessed bytes
-            processBytes(bytes + curBytes, count - curBytes);
-        }
-    }
-
-    bool headerIsAllZeros() const
-    {
-        char* headerAsChar = (char*) &header;
-        for (size_t i=0; i < offsetof(UstarHeaderBlock, magic); ++i) {
-            if (*headerAsChar++ != 0) {
-                return false;
-            }
-        }
-
-        return true;
-    }
-
-    // https://www.ibm.com/support/knowledgecenter/en/SSLTBW_2.3.0/com.ibm.zos.v2r3.bpxa500/paxex.htm#paxex
-    void parsePAXAttributes(bool areGlobal)
-    {
-        auto lineStart = 0;
-        for (;;) {
-            auto firstSpace = paxAttributes.find(' ', lineStart);
-            auto firstEq = paxAttributes.find('=', lineStart);
-            if ((firstEq == std::string::npos) || (firstSpace == std::string::npos)) {
-                SG_LOG(SG_IO, SG_WARN, "Malfroemd PAX attributes in tarfile");
+            if (result == Z_STREAM_END) {
                 break;
             }
-            
-            uint32_t lengthBytes = std::stoul(paxAttributes.substr(lineStart, firstSpace));
-            uint32_t dataBytes = lengthBytes - (firstEq + 1);
-            std::string name = paxAttributes.substr(firstSpace+1, firstEq - (firstSpace + 1));
-            
-            // dataBytes - 1 here to trim off the trailing newline
-            std::string data = paxAttributes.substr(firstEq+1, dataBytes - 1);
-            
-            processPAXAttribute(areGlobal, name, data);
-            
-            lineStart += lengthBytes;
-        }
-    }
-    
-    void processPAXAttribute(bool isGlobalAttr, const std::string& attrName, const std::string& data)
-    {
-        if (!isGlobalAttr && (attrName == "path")) {
-            // data is UTF-8 encoded path name
-            paxPathName = data;
-        }
+        } while ((zlibStream.avail_in > 0) || (writtenSize > 0));
     }
+
+private:
+    z_stream zlibStream;
+    uint8_t* zlibOutput;
+    bool haveInitedZLib = false;
 };
 
 ///////////////////////////////////////////////////////////////////////////////
 
+#if 1 || defined(HAVE_XZ)
+
+#include <lzma.h>
+
+class XZTarExtractor : public TarExtractorPrivate
+{
+public:
+    XZTarExtractor(ArchiveExtractor* outer) : TarExtractorPrivate(outer)
+    {
+        _xzStream = LZMA_STREAM_INIT;
+        _outputBuffer = (uint8_t*)malloc(ZLIB_DECOMPRESS_BUFFER_SIZE);
+
+
+        auto ret = lzma_stream_decoder(&_xzStream, UINT64_MAX, LZMA_TELL_ANY_CHECK);
+        if (ret != LZMA_OK) {
+            setState(BAD_ARCHIVE);
+            return;
+        }
+    }
+
+    ~XZTarExtractor()
+    {
+        free(_outputBuffer);
+    }
+
+    void extractBytes(const uint8_t* bytes, size_t count) override
+    {
+        lzma_action action = LZMA_RUN;
+        _xzStream.next_in = bytes;
+        _xzStream.avail_in = count;
+
+        size_t writtenSize;
+        do {
+            _xzStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE;
+            _xzStream.next_out = _outputBuffer;
+
+            const auto ret = lzma_code(&_xzStream, action);
+
+            writtenSize = ZLIB_DECOMPRESS_BUFFER_SIZE - _xzStream.avail_out;
+            if (writtenSize > 0) {
+                processBytes((const char*)_outputBuffer, writtenSize);
+            }
+
+            if (ret == LZMA_GET_CHECK) {
+                //
+            } else if (ret == LZMA_STREAM_END) {
+                setState(END_OF_ARCHIVE);
+                break;
+            } else if (ret != LZMA_OK) {
+                setState(BAD_ARCHIVE);
+                break;
+            }
+        } while ((_xzStream.avail_in > 0) || (writtenSize > 0));
+    }
+
+    void flush() override
+    {
+        const auto ret = lzma_code(&_xzStream, LZMA_FINISH);
+        if (ret != LZMA_STREAM_END) {
+            setState(BAD_ARCHIVE);
+        }
+    }
+
+private:
+    lzma_stream _xzStream;
+    uint8_t* _outputBuffer = nullptr;
+};
+
+#endif
+
+///////////////////////////////////////////////////////////////////////////////
+
 extern "C" {
 	void fill_memory_filefunc(zlib_filefunc_def*);
 }
@@ -511,32 +533,34 @@ public:
 		const size_t BUFFER_SIZE = 1024 * 1024;
 		void* buf = malloc(BUFFER_SIZE);
 
-		try {
-			int result = unzGoToFirstFile(zip);
-			if (result != UNZ_OK) {
-				throw sg_exception("failed to go to first file in archive");
-			}
-
-			while (true) {
-				extractCurrentFile(zip, (char*)buf, BUFFER_SIZE);
-				if (state == FILTER_STOPPED) {
-					break;
-				}
-
-				result = unzGoToNextFile(zip);
-				if (result == UNZ_END_OF_LIST_OF_FILE) {
-					break;
-				}
-				else if (result != UNZ_OK) {
-					throw sg_io_exception("failed to go to next file in the archive");
-				}
-			}
-			state = END_OF_ARCHIVE;
-		}
-		catch (sg_exception&) {
+        int result = unzGoToFirstFile(zip);
+        if (result != UNZ_OK) {
+            SG_LOG(SG_IO, SG_ALERT, outer->rootPath() << "failed to go to first file in archive:" << result);
             state = BAD_ARCHIVE;
+            free(buf);
+            unzClose(zip);
+            return;
         }
-        
+
+        while (true) {
+            extractCurrentFile(zip, (char*)buf, BUFFER_SIZE);
+            if (state == FILTER_STOPPED) {
+                state = END_OF_ARCHIVE;
+                break;
+            }
+
+            result = unzGoToNextFile(zip);
+            if (result == UNZ_END_OF_LIST_OF_FILE) {
+                state = END_OF_ARCHIVE;
+                break;
+            } else if (result != UNZ_OK) {
+                SG_LOG(SG_IO, SG_ALERT, outer->rootPath() << "failed to go to next file in archive:" << result);
+                state = BAD_ARCHIVE;
+                break;
+            }
+        }
+
+
         free(buf);
         unzClose(zip);
     }
@@ -635,17 +659,19 @@ void ArchiveExtractor::extractBytes(const uint8_t* bytes, size_t count)
 
 		if (r == TarData) {
 			d.reset(new TarExtractorPrivate(this));
-		}
-		else if (r == ZipData) {
-			d.reset(new ZipExtractorPrivate(this));
-		}
-		else {
-			SG_LOG(SG_IO, SG_WARN, "Invalid archive type");
+        } else if (r == GZData) {
+            d.reset(new GZTarExtractor(this));
+        } else if (r == XZData) {
+            d.reset(new XZTarExtractor(this));
+        } else if (r == ZipData) {
+            d.reset(new ZipExtractorPrivate(this));
+        } else {
+            SG_LOG(SG_IO, SG_WARN, "Invalid archive type");
 			_invalidDataType = true;
 			return;
-		}
+        }
 
-		// if hit here, we created the extractor. Feed the prefbuffer
+        // if hit here, we created the extractor. Feed the prefbuffer
 		// bytes through it
 		d->extractBytes((uint8_t*) _prebuffer.data(), _prebuffer.size());
 		_prebuffer.clear();
@@ -697,9 +723,18 @@ ArchiveExtractor::DetermineResult ArchiveExtractor::determineType(const uint8_t*
 		return ZipData;
 	}
 
-	auto r = isTarData(bytes, count);
-	if ((r == TarData) || (r == InsufficientData))
-		return r;
+    if (count < 6) {
+        return InsufficientData;
+    }
+
+    const uint8_t XZ_HEADER[6] = {0xFD, '7', 'z', 'X', 'Z', 0x00};
+    if (memcmp(bytes, XZ_HEADER, 6) == 0) {
+        return XZData;
+    }
+
+    auto r = isTarData(bytes, count);
+    if ((r == TarData) || (r == InsufficientData) || (r == GZData))
+        return r;
 
 	return Invalid;
 }
@@ -712,6 +747,8 @@ ArchiveExtractor::DetermineResult ArchiveExtractor::isTarData(const uint8_t* byt
     }
 
     UstarHeaderBlock* header = 0;
+    DetermineResult result = InsufficientData;
+
     if ((bytes[0] == 0x1f) && (bytes[1] == 0x8b)) {
         // GZIP identification bytes
         z_stream z;
@@ -729,11 +766,11 @@ ArchiveExtractor::DetermineResult ArchiveExtractor::isTarData(const uint8_t* byt
             return Invalid;
         }
 
-        int result = inflate(&z, Z_SYNC_FLUSH);
-        if ((result == Z_OK) || (result == Z_STREAM_END)) {
+        int zResult = inflate(&z, Z_SYNC_FLUSH);
+        if ((zResult == Z_OK) || (zResult == Z_STREAM_END)) {
             // all good
         } else {
-            SG_LOG(SG_IO, SG_WARN, "isTarData: Zlib inflate failed:" << result);
+            SG_LOG(SG_IO, SG_WARN, "isTarData: Zlib inflate failed:" << zResult);
             inflateEnd(&z);
             return Invalid; // not tar data
         }
@@ -746,6 +783,7 @@ ArchiveExtractor::DetermineResult ArchiveExtractor::isTarData(const uint8_t* byt
 
         header = reinterpret_cast<UstarHeaderBlock*>(zlibOutput);
         inflateEnd(&z);
+        result = GZData;
     } else {
         // uncompressed tar
         if (count < TAR_HEADER_BLOCK_SIZE) {
@@ -753,13 +791,14 @@ ArchiveExtractor::DetermineResult ArchiveExtractor::isTarData(const uint8_t* byt
         }
 
         header = (UstarHeaderBlock*) bytes;
+        result = TarData;
     }
 
     if (strncmp(header->magic, TMAGIC, TMAGLEN) != 0) {
         return Invalid;
     }
 
-    return TarData;
+    return result;
 }
 
 void ArchiveExtractor::extractLocalFile(const SGPath& archiveFile)

simgear/io/untar.hxx

@@ -36,15 +36,16 @@ public:
 	ArchiveExtractor(const SGPath& rootPath);
 	virtual ~ArchiveExtractor();
 
-	enum DetermineResult
-	{
-		Invalid,
-		InsufficientData,
-		TarData,
-		ZipData
-	};
+    enum DetermineResult {
+        Invalid,
+        InsufficientData,
+        TarData,
+        ZipData,
+        GZData, // Gzipped-tar
+        XZData  // XZ (aka LZMA) tar
+    };
 
-	static DetermineResult determineType(const uint8_t* bytes, size_t count);
+    static DetermineResult determineType(const uint8_t* bytes, size_t count);
 
 	/**
 	 * @brief API to extract a local zip or tar.gz 
@@ -70,6 +71,11 @@ public:
 		Stop
 	};
 
+    SGPath rootPath() const
+    {
+        return _rootPath;
+    }
+
 protected:
 
 

simgear/math/SGGeod.hxx

@@ -29,6 +29,13 @@ public:
   /// Default constructor, initializes the instance to lat = lon = elev = 0
   SGGeod(void);
 
+  /**
+    return an SGGeod for which isValid() returns false.
+    This is necessaerby becuase for historical reasons, ther defaulrt constructor above initialsies to zero,zero,zero
+    which *is*
+   */
+  static SGGeod invalid();
+
   /// Factory from angular values in radians and elevation is 0
   static SGGeod fromRad(double lon, double lat);
   /// Factory from angular values in degrees and elevation is 0

simgear/math/SGGeodesy.cxx

@@ -663,3 +663,8 @@ SGGeodesy::radialIntersection(const SGGeod& a, double aRadial,
   result = SGGeod::fromGeoc(r);
   return true;
 }
+
+SGGeod SGGeod::invalid()
+{
+    return SGGeod::fromDeg(-999.9, -999.0);
+}

simgear/misc/sg_path.cxx

@@ -48,6 +48,7 @@
 #endif
 
 #include "sg_path.hxx"
+#include <simgear/misc/sg_dir.hxx>
 
 using std::string;
 using simgear::strutils::starts_with;
@@ -995,8 +996,9 @@ SGPath SGPath::realpath() const
               // (needed for fgValidatePath security)
     {
         if (path.empty()) {
-            return SGPath(".").realpath(); // current directory
+            return simgear::Dir::current().path();
         }
+
         std::string this_dir = dir();
         if (isAbsolute() && this_dir.empty()) { // top level
             this_dir = "/";

simgear/misc/simgear_optional.hxx

@@ -74,6 +74,22 @@ public:
         return _value;
     }
 
+    const T& value_or(const T& defaultValue) const
+    {
+        if (!_haveValue) {
+            return defaultValue;
+        }
+        return _value;
+    }
+
+    T& value_or(T& defaultValue) const
+    {
+        if (!_haveValue) {
+            return defaultValue;
+        }
+        return _value;
+    }
+
     T& value()
     {
         if (!_haveValue) {

simgear/nasal/code.c

@@ -248,7 +248,7 @@ void naFreeContext(naContext c)
     // than I have right now. So instead I'm clearing the stack tops here, so
     // a freed context looks the same as a new one returned by initContext.
 
-    c->fTop = c->opTop = c->markTop = 0;
+    c->fTop = c->opTop = c->markTop = c->ntemps = 0;
 
     c->nextFree = globals->freeContexts;
     globals->freeContexts = c;

simgear/nasal/cppbind/NasalEmesaryInterface.hxx

@@ -63,7 +63,7 @@ namespace nasal
 
     class NasalMainLoopRecipient : public simgear::Emesary::IReceiver {
     public:
-        NasalMainLoopRecipient() : receiveCount(0) {
+        NasalMainLoopRecipient() : receiveCount(0), Active(false), CanWait(false) {
             simgear::Emesary::GlobalTransmitter::instance()->Register(*this);
         }
         virtual ~NasalMainLoopRecipient() {

simgear/nasal/misc.c

@@ -100,7 +100,14 @@ naRef naNewHash(struct Context* c)
 
 naRef naNewCode(struct Context* c)
 {
-    return naNew(c, T_CODE);
+    naRef r = naNew(c, T_CODE);
+    // naNew can return a previously used naCode. naCodeGen will init
+    // all these members but a GC can occur inside naCodeGen, so we see
+    // partially initalized state here. To avoid this, clear out the values
+    // which mark() cares about.
+    PTR(r).code->srcFile = naNil();
+    PTR(r).code->nConstants = 0;
+    return r;
 }
 
 naRef naNewCCode(struct Context* c, naCFunction fptr)

simgear/package/Catalog.cxx

@@ -143,9 +143,15 @@ protected:
         Dir d(m_owner->installRoot());
         SGPath p = d.file("catalog.xml");
         sg_ofstream f(p, std::ios::out | std::ios::trunc);
-        f.write(m_buffer.data(), m_buffer.size());
+        const auto sz = m_buffer.size();
+        f.write(m_buffer.data(), sz);
         f.close();
 
+        if (f.fail()) {
+            m_owner->refreshComplete(Delegate::FAIL_FILESYSTEM);
+            return;
+        }
+
         time(&m_owner->m_retrievedTime);
         m_owner->writeTimestamp();
         m_owner->refreshComplete(Delegate::STATUS_REFRESHED);
@@ -619,6 +625,8 @@ void Catalog::setUserEnabled(bool b)
 
 void Catalog::processAlternate(SGPropertyNode_ptr alt)
 {
+    m_refreshRequest.reset();
+
     std::string altId;
     const auto idPtr = alt->getStringValue("id");
     if (idPtr) {
@@ -694,8 +702,8 @@ void Catalog::processAlternate(SGPropertyNode_ptr alt)
 
     SG_LOG(SG_GENERAL, SG_INFO, "Migrating catalog " << id() << " to new URL:" << altUrl);
     setUrl(altUrl);
-    Downloader* dl = new Downloader(this, altUrl);
-    root()->makeHTTPRequest(dl);
+    m_refreshRequest = new Downloader(this, altUrl);
+    root()->makeHTTPRequest(m_refreshRequest);
 }
 
 int Catalog::markPackagesForInstallation(const string_list &packageIds) {

simgear/package/CatalogTest.cxx

@@ -166,7 +166,8 @@ int parseTest()
 {
     SGPath rootPath = simgear::Dir::current().path();
     rootPath.append("testRoot");
-    pkg::Root* root = new pkg::Root(rootPath, "8.1.12");
+
+    pkg::RootRef root(new pkg::Root(rootPath, "8.1.2"));
     root->setLocale("de");
     pkg::CatalogRef cat = pkg::Catalog::createFromPath(root, SGPath(SRC_DIR "/catalogTest1"));
 
@@ -344,7 +345,6 @@ int parseTest()
     SG_CHECK_EQUAL(urls.size(), 3);
     SG_CHECK_EQUAL(urls.at(1), "http://localhost:2000/mirrorB/b737.tar.gz");
 
-    delete root;
     return EXIT_SUCCESS;
 }
 
@@ -953,7 +953,7 @@ int parseInvalidTest()
 {
     SGPath rootPath = simgear::Dir::current().path();
     rootPath.append("testRoot");
-    pkg::Root* root = new pkg::Root(rootPath, "8.1.12");
+    pkg::RootRef root(new pkg::Root(rootPath, "8.1.12"));
     pkg::CatalogRef cat = pkg::Catalog::createFromPath(root, SGPath(SRC_DIR "/catalogTestInvalid"));
     SG_VERIFY(cat.valid());
 
@@ -1333,25 +1333,25 @@ int main(int argc, char* argv[])
     testRefreshCatalog(&cl);
 
     testInstallTarPackage(&cl);
-    
+
     testInstallArchiveType(&cl);
-    
+
     testDisableDueToVersion(&cl);
-    
+
     testOfflineMode(&cl);
-    
+
     testVersionMigrate(&cl);
-    
+
     updateInvalidToValid(&cl);
     updateValidToInvalid(&cl);
     updateInvalidToInvalid(&cl);
-    
+
     removeInvalidCatalog(&cl);
-    
+
     testVersionMigrateToId(&cl);
 
     testInstallBadPackage(&cl);
-    
+
     testMirrorsFailure(&cl);
 
     testMigrateInstalled(&cl);

simgear/package/Root.cxx

@@ -133,7 +133,9 @@ public:
 
     void fireRefreshStatus(CatalogRef catalog, Delegate::StatusCode status)
     {
-        for (auto d : delegates) {
+        // take a copy of delegates since firing this can modify the data
+        const auto currentDelegates = delegates;
+        for (auto d : currentDelegates) {
             d->catalogRefreshed(catalog, status);
         }
     }
@@ -722,12 +724,6 @@ void Root::catalogRefreshStatus(CatalogRef aCat, Delegate::StatusCode aReason)
     auto catIt = d->catalogs.find(aCat->id());
     d->fireRefreshStatus(aCat, aReason);
 
-    if (aReason == Delegate::STATUS_IN_PROGRESS) {
-        d->refreshing.insert(aCat);
-    } else {
-        d->refreshing.erase(aCat);
-    }
-
     if (aCat->isUserEnabled() &&
         (aReason == Delegate::STATUS_REFRESHED) && 
         (catIt == d->catalogs.end())) 
@@ -761,6 +757,17 @@ void Root::catalogRefreshStatus(CatalogRef aCat, Delegate::StatusCode aReason)
         }
     } // of catalog is disabled
 
+    // remove from refreshing /after/ checking for enable / disabled, since for
+    // new catalogs, the reference in d->refreshing might be our /only/
+    // reference to the catalog. Once the refresh is done (either failed or
+    // succeeded) the Catalog will be in either d->catalogs or
+    // d->disabledCatalogs
+    if (aReason == Delegate::STATUS_IN_PROGRESS) {
+      d->refreshing.insert(aCat);
+    } else {
+      d->refreshing.erase(aCat);
+    }
+
     if (d->refreshing.empty()) {
         d->fireRefreshStatus(CatalogRef(), Delegate::STATUS_REFRESHED);
         d->firePackagesChanged();

simgear/props/condition.cxx

@@ -535,7 +535,8 @@ readComparison( SGPropertyNode *prop_root,
 {
   SGComparisonCondition * condition = new SGComparisonCondition(type, reverse);
   if (node->nChildren() < 2 || node->nChildren() > 3 ) {
-    throw sg_exception("condition: comparison without two or three children");
+    throw sg_exception("condition: comparison without two or three children",
+                       {}, {}, false);
   }
   
   const SGPropertyNode* left = node->getChild(0), 
@@ -551,7 +552,8 @@ readComparison( SGPropertyNode *prop_root,
       SGExpressiond* exp = SGReadDoubleExpression(prop_root, left->getChild(0));
       condition->setLeftDExpression(exp);
     } else {
-      throw sg_exception("Unknown condition comparison left child:" + leftName);
+      throw sg_exception("Unknown condition comparison left child:" + leftName,
+                         {}, {}, false);
     }
   }
     
@@ -565,7 +567,8 @@ readComparison( SGPropertyNode *prop_root,
       SGExpressiond* exp = SGReadDoubleExpression(prop_root, right->getChild(0));
       condition->setRightDExpression(exp);
     } else {
-      throw sg_exception("Unknown condition comparison right child:" + rightName);
+      throw sg_exception("Unknown condition comparison right child:" + rightName,
+                         {}, {}, false);
     }
   }
   
@@ -580,7 +583,8 @@ readComparison( SGPropertyNode *prop_root,
       SGExpressiond* exp = SGReadDoubleExpression(prop_root, n->getChild(0));
       condition->setPrecisionDExpression(exp);
     } else {
-      throw sg_exception("Unknown condition comparison precision child:" + name );
+      throw sg_exception("Unknown condition comparison precision child:" + name,
+                         {}, {}, false);
     }
   }
   

simgear/props/props.cxx

@@ -19,6 +19,10 @@
 #include <sstream>
 #include <iomanip>
 #include <iterator>
+#include <exception> // can't use sg_exception becuase of PROPS_STANDALONE
+#include <mutex>
+#include <thread>
+
 #include <stdio.h>
 #include <string.h>
 
@@ -51,11 +55,15 @@ using namespace simgear;
 
 struct SGPropertyNodeListeners
 {
+  /* Protect _num_iterators and _items. We use a recursive mutex to allow
+  nested access to work as normal. */
+  std::recursive_mutex  _rmutex;
+
   /* This keeps a count of the current number of nested invocations of
   forEachListener(). If non-zero, other code higher up the stack is iterating
   _items[] so for example code must not erase items in the vector. */
   int _num_iterators = 0;
-  
+
   std::vector<SGPropertyChangeListener *> _items;
 };
 
@@ -558,23 +566,15 @@ find_node (SGPropertyNode * current,
   }
 }
 #else
-template<typename Range>
-SGPropertyNode*
-find_node (SGPropertyNode * current,
-           const Range& path,
-           bool create,
-           int last_index = -1)
-{
+template <typename Range>
+SGPropertyNode *find_node(SGPropertyNode *current, const Range &path,
+                          bool create, int last_index = -1) {
   using namespace boost;
-  typedef split_iterator<typename range_result_iterator<Range>::type>
-    PathSplitIterator;
-
-  PathSplitIterator itr
-    = make_split_iterator(path, first_finder("/", is_equal()));
+  auto itr = make_split_iterator(path, first_finder("/", is_equal()));
   if (*path.begin() == '/')
     return find_node_aux(current->getRootNode(), itr, create, last_index);
-   else
-     return find_node_aux(current, itr, create, last_index);
+  else
+    return find_node_aux(current, itr, create, last_index);
 }
 #endif
 
@@ -2406,6 +2406,7 @@ SGPropertyNode::addChangeListener (SGPropertyChangeListener * listener,
   if (_listeners == 0)
     _listeners = new SGPropertyNodeListeners;
 
+  std::lock_guard<std::recursive_mutex> lock(_listeners->_rmutex);
   /* If there's a nullptr entry (a listener that was unregistered), we
   overwrite it. This ensures that listeners that routinely unregister+register
   themselves don't make _listeners->_items grow unnecessarily. Otherwise simply
@@ -2429,9 +2430,13 @@ SGPropertyNode::removeChangeListener (SGPropertyChangeListener * listener)
 {
   if (_listeners == 0)
     return;
+  /* We use a std::unique_lock rather than a std::lock_guard because we may
+  need to unlock early. */
+  std::unique_lock<std::recursive_mutex> lock(_listeners->_rmutex);
   vector<SGPropertyChangeListener*>::iterator it =
     find(_listeners->_items.begin(), _listeners->_items.end(), listener);
   if (it != _listeners->_items.end()) {
+    assert(_listeners->_num_iterators >= 0);
     if (_listeners->_num_iterators) {
       /* _listeners._items is currently being iterated further up the stack in
       this thread by one or more nested invocations of forEachListener(), so
@@ -2450,6 +2455,7 @@ SGPropertyNode::removeChangeListener (SGPropertyChangeListener * listener)
       _listeners->_items.erase(it);
       listener->unregister_property(this);
       if (_listeners->_items.empty()) {
+        lock.unlock();
         delete _listeners;
         _listeners = 0;
       }
@@ -2511,9 +2517,11 @@ static void forEachListener(
     )
 {
   if (!_listeners) return;
-  
+
+  std::lock_guard<std::recursive_mutex> lock(_listeners->_rmutex);
+  assert(_listeners->_num_iterators >= 0);
   _listeners->_num_iterators += 1;
-  
+
   /* We need to use an index here when iterating _listeners->_items, not an
   iterator. This is because a listener may add new listeners, causing the
   vector to be reallocated, which would invalidate any iterator. */
@@ -2528,10 +2536,12 @@ static void forEachListener(
       }
     }
   }
-  
+
   _listeners->_num_iterators -= 1;
-  
+  assert(_listeners->_num_iterators >= 0);
+
   if (_listeners->_num_iterators == 0) {
+
     /* Remove any items that have been set to nullptr. */
     _listeners->_items.erase(
         std::remove(_listeners->_items.begin(), _listeners->_items.end(), (SGPropertyChangeListener*) nullptr),
@@ -2547,6 +2557,7 @@ static void forEachListener(
 int SGPropertyNode::nListeners() const
 {
   if (!_listeners) return 0;
+  std::lock_guard<std::recursive_mutex> lock(_listeners->_rmutex);
   int   n = 0;
   for (auto listener: _listeners->_items) {
     if (listener)   n += 1;

simgear/props/props_io.cxx

@@ -161,7 +161,7 @@ setFlag( int& mode,
     string message = "Unrecognized flag value '";
     message += flag;
     message += '\'';
-    throw sg_io_exception(message, location, SG_ORIGIN);
+    throw sg_io_exception(message, location, SG_ORIGIN, false);
   }
 }
 
@@ -176,7 +176,7 @@ PropsVisitor::startElement (const char * name, const XMLAttributes &atts)
       string message = "Root element name is ";
       message += name;
       message += "; expected PropertyList";
-      throw sg_io_exception(message, location, SG_ORIGIN);
+      throw sg_io_exception(message, location, SG_ORIGIN, false);
     }
 
     // Check for an include.
@@ -188,7 +188,7 @@ PropsVisitor::startElement (const char * name, const XMLAttributes &atts)
           {
               string message ="Cannot open file ";
               message += attval;
-              throw sg_io_exception(message, location, SG_ORIGIN);
+              throw sg_io_exception(message, location, SG_ORIGIN, false);
           }
           readProperties(path, _root, 0, _extended);
       } catch (sg_io_exception &e) {
@@ -278,7 +278,7 @@ PropsVisitor::startElement (const char * name, const XMLAttributes &atts)
           {
             string message ="Cannot open file ";
             message += val;
-            throw sg_io_exception(message, location, SG_ORIGIN);
+            throw sg_io_exception(message, location, SG_ORIGIN, false);
           }
           readProperties(path, node, 0, _extended);
         }
@@ -353,7 +353,7 @@ PropsVisitor::endElement (const char * name)
       string message = "Unrecognized data type '";
       message += st.type;
       message += '\'';
-      throw sg_io_exception(message, location, SG_ORIGIN);
+      throw sg_io_exception(message, location, SG_ORIGIN, false);
     }
     if( !ret )
       SG_LOG
@@ -701,7 +701,7 @@ writeProperties (const SGPath &path, const SGPropertyNode * start_node,
   if (output.good()) {
     writeProperties(output, start_node, write_all, archive_flag);
   } else {
-    throw sg_io_exception("Cannot open file", sg_location(path.utf8Str()));
+    throw sg_io_exception("Cannot open file", sg_location(path.utf8Str()), "", false);
   }
 }
 
@@ -770,7 +770,7 @@ copyPropertyValue(const SGPropertyNode *in, SGPropertyNode *out)
                 break;
             string message = "Unknown internal SGPropertyNode type";
             message += in->getType();
-            throw sg_error(message, SG_ORIGIN);
+            throw sg_error(message, SG_ORIGIN, false);
     }
     
     return retval;

simgear/scene/material/Effect.cxx

@@ -67,16 +67,21 @@
 #include <osgDB/ReadFile>
 #include <osgDB/Registry>
 
-#include <simgear/scene/util/SGReaderWriterOptions.hxx>
+#include <simgear/debug/ErrorReportingCallback.hxx>
+#include <simgear/io/iostreams/sgstream.hxx>
+#include <simgear/props/props_io.hxx>
+#include <simgear/props/vectorPropTemplates.hxx>
 #include <simgear/scene/tgdb/userdata.hxx>
 #include <simgear/scene/util/OsgMath.hxx>
+#include <simgear/scene/util/SGProgram.hxx>
+#include <simgear/scene/util/SGReaderWriterOptions.hxx>
 #include <simgear/scene/util/SGSceneFeatures.hxx>
 #include <simgear/scene/util/StateAttributeFactory.hxx>
 #include <simgear/structure/OSGUtils.hxx>
 #include <simgear/structure/SGExpression.hxx>
 #include <simgear/props/props_io.hxx>
 #include <simgear/props/vectorPropTemplates.hxx>
-
+#include <simgear/debug/ErrorReportingCallback.hxx>
 #include <simgear/io/iostreams/sgstream.hxx>
 
 namespace simgear
@@ -103,6 +108,7 @@ bool loadShaderFromUTF8File(osg::Shader* shader, const std::string& fileName)
     if (!inStream.is_open())
         return false;
 
+    shader->setFileName(fileName);
     shader->setShaderSource(inStream.read_all());
     return true;
 }
@@ -221,8 +227,9 @@ Effect::Effect()
 }
 
 Effect::Effect(const Effect& rhs, const CopyOp& copyop)
-    : osg::Object(rhs,copyop), root(rhs.root), parametersProp(rhs.parametersProp), _cache(0),
-      _isRealized(rhs._isRealized)
+    : osg::Object(rhs, copyop), root(rhs.root), parametersProp(rhs.parametersProp), _cache(0),
+      _isRealized(rhs._isRealized),
+      _effectFilePath(rhs._effectFilePath)
 {
     typedef vector<ref_ptr<Technique> > TechniqueList;
     for (TechniqueList::const_iterator itr = rhs.techniques.begin(),
@@ -287,6 +294,8 @@ Effect::~Effect()
 void buildPass(Effect* effect, Technique* tniq, const SGPropertyNode* prop,
                const SGReaderWriterOptions* options)
 {
+    simgear::ErrorReportContext ec("effect-pass", prop->getPath());
+
     Pass* pass = new Pass;
     tniq->passes.push_back(pass);
     for (int i = 0; i < prop->nChildren(); ++i) {
@@ -837,10 +846,12 @@ void reload_shaders()
         string fileName = SGModelLib::findDataFile(sitr->first.first);
         if (!fileName.empty()) {
             loadShaderFromUTF8File(shader, fileName);
-        }
-        else
+        } else {
             SG_LOG(SG_INPUT, SG_ALERT, "Could not locate shader: " << fileName);
-
+            simgear::reportFailure(simgear::LoadFailure::NotFound,
+                                   simgear::ErrorCode::LoadEffectsShaders,
+                                   "Reload: couldn't find shader:" + sitr->first.first);
+        }
     }
 }
 
@@ -928,15 +939,16 @@ void ShaderProgramBuilder::buildAttribute(Effect* effect, Pass* pass,
         // FIXME orig: const string& shaderName = shaderKey.first;
         string shaderName = shaderKey.first;
         Shader::Type stype = (Shader::Type)shaderKey.second;
-        if (getPropertyRoot()->getBoolValue("/sim/version/compositor-support", false) &&
+        const bool compositorEnabled = getPropertyRoot()->getBoolValue("/sim/version/compositor-support", false);
+        if (compositorEnabled &&
             shaderName.substr(0, shaderName.find("/")) == "Shaders") {
             shaderName = "Compositor/" + shaderName;
         }
         string fileName = SGModelLib::findDataFile(shaderName, options);
         if (fileName.empty())
         {
-            SG_LOG(SG_INPUT, SG_ALERT, "Could not locate shader" << shaderName);
-
+            simgear::reportFailure(simgear::LoadFailure::NotFound, simgear::ErrorCode::LoadEffectsShaders,
+                                   "Couldn't locate shader:" + shaderName, sg_location{shaderName});
 
             throw BuilderException(string("couldn't find shader ") +
                 shaderName);
@@ -950,7 +962,11 @@ void ShaderProgramBuilder::buildAttribute(Effect* effect, Pass* pass,
         pass->setAttributeAndModes(program);
         return;
     }
-    program = new Program;
+
+    auto sgprogram = new SGProgram;
+    program = sgprogram;
+    sgprogram->setEffectFilePath(effect->filePath());
+
     for (const auto& skey : resolvedKey.shaders) {
         const string& fileName = skey.first;
         Shader::Type stype = (Shader::Type)skey.second;
@@ -961,11 +977,30 @@ void ShaderProgramBuilder::buildAttribute(Effect* effect, Pass* pass,
             ref_ptr<Shader> shader = new Shader(stype);
 			shader->setName(fileName);
             if (loadShaderFromUTF8File(shader, fileName)) {
-                program->addShader(shader.get());
+                if (!program->addShader(shader.get())) {
+                    simgear::reportFailure(simgear::LoadFailure::BadData,
+                                           simgear::ErrorCode::LoadEffectsShaders,
+                                           "Program::addShader failed",
+                                           SGPath::fromUtf8(fileName));
+                }
+
                 shaderMap.insert(ShaderMap::value_type(skey, shader));
+            } else {
+                simgear::reportFailure(simgear::LoadFailure::BadData,
+                                       simgear::ErrorCode::LoadEffectsShaders,
+                                       "Failed to read shader source code",
+                                       SGPath::fromUtf8(fileName));
             }
         }
     }
+
+    if (sgprogram->getNumShaders() == 0) {
+        simgear::reportFailure(simgear::LoadFailure::BadData,
+                               simgear::ErrorCode::LoadEffectsShaders,
+                               "No shader source code defined for effect",
+                               effect->filePath());
+    }
+
     for (const auto& key : prgKey.attributes) {
         program->addBindAttribLocation(key.first, key.second);
     }
@@ -1326,6 +1361,8 @@ InstallAttributeBuilder<DepthBuilder> installDepth("depth");
 void buildTechnique(Effect* effect, const SGPropertyNode* prop,
                     const SGReaderWriterOptions* options)
 {
+    simgear::ErrorReportContext ec("effect-technique", prop->getPath());
+
     Technique* tniq = new Technique;
     effect->techniques.push_back(tniq);
     tniq->setScheme(prop->getStringValue("scheme"));
@@ -1481,20 +1518,21 @@ void mergeSchemesFallbacks(Effect *effect, const SGReaderWriterOptions *options)
 
 // Walk the techniques property tree, building techniques and
 // passes.
-static std::mutex  realizeTechniques_lock;
 bool Effect::realizeTechniques(const SGReaderWriterOptions* options)
 {
-    std::lock_guard<std::mutex> g(realizeTechniques_lock);
+    simgear::ErrorReportContext ec{"effect", getName()};
+
     if (getPropertyRoot()->getBoolValue("/sim/version/compositor-support", false))
         mergeSchemesFallbacks(this, options);
 
     if (_isRealized)
         return true;
+
     PropertyList tniqList = root->getChildren("technique");
-    for (PropertyList::iterator itr = tniqList.begin(), e = tniqList.end();
-         itr != e;
-         ++itr)
-        buildTechnique(this, *itr, options);
+    for (const auto& tniq : tniqList) {
+        buildTechnique(this, tniq, options);
+    }
+
     _isRealized = true;
     return true;
 }
@@ -1630,3 +1668,15 @@ expression::ExpParserRegistrar propvalueRegistrar("float-property",
                                                  propertyExpressionParser<float>);
 
 }
+
+using namespace simgear;
+
+void Effect::setFilePath(const SGPath& path)
+{
+    _effectFilePath = path;
+}
+
+SGPath Effect::filePath() const
+{
+    return _effectFilePath;
+}

simgear/scene/material/Effect.hxx

@@ -29,10 +29,11 @@
 #include <osg/observer_ptr>
 #include <osgDB/ReaderWriter>
 
+#include <simgear/misc/sg_path.hxx>
 #include <simgear/props/props.hxx>
 #include <simgear/scene/util/UpdateOnceCallback.hxx>
-#include <simgear/threads/SGThread.hxx>
 #include <simgear/structure/Singleton.hxx>
+#include <simgear/threads/SGThread.hxx>
 
 namespace osg
 {
@@ -106,6 +107,10 @@ public:
 
     std::string getName(){return _name;}
     void setName(std::string name){_name = name;}
+
+    void setFilePath(const SGPath& path);
+    SGPath filePath() const;
+
 protected:
     ~Effect();
     // Support for a cache of effects that inherit from this one, so
@@ -142,21 +147,26 @@ protected:
     Cache* _cache;
     friend size_t hash_value(const Key& key);
     friend Effect* makeEffect(SGPropertyNode* prop, bool realizeTechniques,
-                              const SGReaderWriterOptions* options);
+                              const SGReaderWriterOptions* options,
+                              const SGPath& path);
     bool _isRealized;
     std::string _name;
+    SGPath _effectFilePath;
 };
+
 // Automatic support for boost hash function
 size_t hash_value(const Effect::Key&);
 
 
 Effect* makeEffect(const std::string& name,
                    bool realizeTechniques,
-                   const SGReaderWriterOptions* options);
+                   const SGReaderWriterOptions* options,
+                   const SGPath& modelPath = SGPath{});
 
 Effect* makeEffect(SGPropertyNode* prop,
                    bool realizeTechniques,
-                   const SGReaderWriterOptions* options);
+                   const SGReaderWriterOptions* options,
+                   const SGPath& path = SGPath{});
 
 bool makeParametersFromStateSet(SGPropertyNode* paramRoot,
                                 const osg::StateSet* ss);

simgear/scene/material/EffectBuilder.cxx

@@ -75,13 +75,13 @@ BuilderException::BuilderException()
 }
 
 BuilderException::BuilderException(const char* message, const char* origin)
-  : sg_exception(message, origin)
+    : sg_exception(message, origin, {}, false)
 {
 }
 
 BuilderException::BuilderException(const std::string& message,
                                    const std::string& origin)
-  : sg_exception(message, origin)
+    : sg_exception(message, origin, {}, false)
 {
 }
 

simgear/scene/material/TextureBuilder.cxx

@@ -39,12 +39,13 @@
 #include <OpenThreads/Mutex>
 #include <OpenThreads/ScopedLock>
 
+#include <simgear/debug/ErrorReportingCallback.hxx>
+#include <simgear/props/vectorPropTemplates.hxx>
 #include <simgear/scene/util/OsgMath.hxx>
 #include <simgear/scene/util/SGReaderWriterOptions.hxx>
 #include <simgear/scene/util/SGSceneFeatures.hxx>
 #include <simgear/scene/util/StateAttributeFactory.hxx>
 #include <simgear/structure/OSGUtils.hxx>
-#include <simgear/props/vectorPropTemplates.hxx>
 
 namespace simgear
 {
@@ -273,11 +274,20 @@ bool setAttrs(const TexTuple& attrs, Texture* tex,
         options->setLoadOriginHint(SGReaderWriterOptions::LoadOriginHint::ORIGIN_EFFECTS_NORMALIZED);
     else
         options->setLoadOriginHint(SGReaderWriterOptions::LoadOriginHint::ORIGIN_EFFECTS);
-#if OSG_VERSION_LESS_THAN(3,4,2)
-    result = osgDB::readImageFile(imageName, options);
-#else
-    result = osgDB::readRefImageFile(imageName, options);
-#endif
+
+    try {
+    #if OSG_VERSION_LESS_THAN(3,4,2)
+        result = osgDB::readImageFile(imageName, options);
+    #else
+        result = osgDB::readRefImageFile(imageName, options);
+    #endif
+    } catch (std::exception& e) {
+        simgear::reportFailure(simgear::LoadFailure::OutOfMemory, simgear::ErrorCode::LoadingTexture,
+                               string{"osgDB::readRefImageFile failed:"} + e.what(),
+                               SGPath::fromUtf8(imageName));
+        return false;
+    }
+
     options->setLoadOriginHint(origLOH);
     osg::ref_ptr<osg::Image> image;
     if (result.success())
@@ -296,6 +306,9 @@ bool setAttrs(const TexTuple& attrs, Texture* tex,
         tex->setMaxAnisotropy(SGSceneFeatures::instance()->getTextureFilter());
     } else {
         SG_LOG(SG_INPUT, SG_ALERT, "failed to load effect texture file " << imageName);
+        simgear::reportFailure(simgear::LoadFailure::BadData, simgear::ErrorCode::LoadingTexture,
+                               "osgDB::readRefImageFile failed:" + result.message(),
+                               SGPath::fromUtf8(imageName));
         return false;
     }
 
@@ -580,8 +593,8 @@ Texture* CubeMapBuilder::build(Effect* effect, Pass* pass, const SGPropertyNode*
     const SGPropertyNode* texturesProp = getEffectPropertyChild(effect, props, "images");
     const SGPropertyNode* crossProp = getEffectPropertyChild(effect, props, "image");
     if (!texturesProp && !crossProp) {
+        simgear::reportFailure(simgear::LoadFailure::NotFound, simgear::ErrorCode::LoadingTexture, "No images defined for cube map");
         throw BuilderException("no images defined for cube map");
-        return NULL; // This is redundant
     }
 
     // Using 6 separate images
@@ -762,6 +775,9 @@ Texture* CubeMapBuilder::build(Effect* effect, Pass* pass, const SGPropertyNode*
             return cubeTexture.release();
 
         } else {
+            simgear::reportFailure(simgear::LoadFailure::BadData, simgear::ErrorCode::LoadingTexture,
+                                   "Could not load cube-map image:" + result.message(),
+                                   sg_location{texname});
             throw BuilderException("Could not load cube cross");
         }
     }
@@ -842,6 +858,9 @@ Texture* Texture3DBuilder::build(Effect* effect, Pass* pass,
         tex->setImage(image3d.get());
     } else {
         SG_LOG(SG_INPUT, SG_ALERT, "failed to load effect texture file " << imageName);
+        simgear::reportFailure(simgear::LoadFailure::BadData, simgear::ErrorCode::LoadingTexture,
+                               "osgDB::readRefImageFile failed:" + result.message(),
+                               SGPath::fromUtf8(imageName));
         return NULL;
     }
 

simgear/scene/material/makeEffect.cxx

@@ -25,9 +25,11 @@
 #include <osgDB/ReadFile>
 #include <osgDB/Registry>
 
+#include <simgear/debug/ErrorReportingCallback.hxx>
 #include <simgear/debug/logstream.hxx>
-#include <simgear/scene/util/SGReaderWriterOptions.hxx>
+#include <simgear/misc/sg_path.hxx>
 #include <simgear/props/props_io.hxx>
+#include <simgear/scene/util/SGReaderWriterOptions.hxx>
 #include <simgear/scene/util/SGSceneFeatures.hxx>
 #include <simgear/scene/util/SplicingVisitor.hxx>
 #include <simgear/structure/SGExpression.hxx>
@@ -112,7 +114,8 @@ void mergePropertyTrees(SGPropertyNode* resultNode,
 
 Effect* makeEffect(const string& name,
                    bool realizeTechniques,
-                   const SGReaderWriterOptions* options)
+                   const SGReaderWriterOptions* options,
+                   const SGPath& modelPath)
 {
     {
         OpenThreads::ScopedLock<OpenThreads::ReentrantMutex> lock(effectMutex);
@@ -123,23 +126,22 @@ Effect* makeEffect(const string& name,
     }
     string effectFileName(name);
     effectFileName += ".eff";
-    string absFileName
-        = SGModelLib::findDataFile(effectFileName, options);
+    string absFileName = SGModelLib::findDataFile(effectFileName, options, modelPath);
     if (absFileName.empty()) {
-        SG_LOG(SG_INPUT, SG_ALERT, "can't find \"" << effectFileName << "\"");
-        return 0;
+        simgear::reportFailure(simgear::LoadFailure::NotFound, simgear::ErrorCode::LoadEffectsShaders, "Couldn't find Effect:" + effectFileName);
+        return nullptr;
     }
     SGPropertyNode_ptr effectProps = new SGPropertyNode();
     try {
         readProperties(absFileName, effectProps.ptr(), 0, true);
     }
     catch (sg_io_exception& e) {
-        SG_LOG(SG_INPUT, SG_ALERT, "error reading \"" << absFileName << "\": "
-               << e.getFormattedMessage());
-        return 0;
+        simgear::reportFailure(simgear::LoadFailure::BadData, simgear::ErrorCode::LoadEffectsShaders, e.getFormattedMessage(),
+                               e.getLocation());
+        return nullptr;
     }
     ref_ptr<Effect> result = makeEffect(effectProps.ptr(), realizeTechniques,
-                                        options);
+                                        options, SGPath::fromUtf8(absFileName));
     if (result.valid()) {
         OpenThreads::ScopedLock<OpenThreads::ReentrantMutex> lock(effectMutex);
         pair<EffectMap::iterator, bool> irslt
@@ -156,7 +158,8 @@ Effect* makeEffect(const string& name,
 
 Effect* makeEffect(SGPropertyNode* prop,
                    bool realizeTechniques,
-                   const SGReaderWriterOptions* options)
+                   const SGReaderWriterOptions* options,
+                   const SGPath& filePath)
 {
     // Give default names to techniques and passes
     vector<SGPropertyNode_ptr> techniques = prop->getChildren("technique");
@@ -194,7 +197,7 @@ Effect* makeEffect(SGPropertyNode* prop,
     Effect* parent = 0;
     if (inheritProp) {
         //prop->removeChild("inherits-from");
-        parent = makeEffect(inheritProp->getStringValue(), false, options);
+        parent = makeEffect(inheritProp->getStringValue(), false, options, filePath);
         if (parent) {
             Effect::Key key;
             key.unmerged = prop;
@@ -217,6 +220,7 @@ Effect* makeEffect(SGPropertyNode* prop,
             if (!effect.valid()) {
                 effect = new Effect;
                 effect->setName(nameProp->getStringValue());
+                effect->setFilePath(filePath.isNull() ? parent->filePath() : filePath);
                 effect->root = new SGPropertyNode;
                 mergePropertyTrees(effect->root, prop, parent->root);
                 effect->parametersProp = effect->root->getChild("parameters");
@@ -234,12 +238,13 @@ Effect* makeEffect(SGPropertyNode* prop,
                 effect->generator = parent->generator;  // Copy the generators
             }
         } else {
-            SG_LOG(SG_INPUT, SG_ALERT, "can't find base effect " <<
-                   inheritProp->getStringValue());
-            return 0;
+            simgear::reportFailure(simgear::LoadFailure::NotFound, simgear::ErrorCode::LoadEffectsShaders,
+                                   string{"couldn't find base effect to inherit from:"} + inheritProp->getStringValue(), filePath);
+            return nullptr;
         }
     } else {
         effect = new Effect;
+        effect->setFilePath(filePath);
         effect->setName(nameProp->getStringValue());
         effect->root = prop;
         effect->parametersProp = effect->root->getChild("parameters");
@@ -266,9 +271,9 @@ Effect* makeEffect(SGPropertyNode* prop,
             effect->realizeTechniques(options);
         }
         catch (BuilderException& e) {
-            SG_LOG(SG_INPUT, SG_ALERT, "Error building technique: "
-                   << e.getFormattedMessage());
-            return 0;
+            simgear::reportFailure(simgear::LoadFailure::Misconfigured, simgear::ErrorCode::LoadEffectsShaders,
+                                   "Failed to build technique:" + e.getFormattedMessage(), filePath);
+            return nullptr;
         }
     }
     return effect.release();

simgear/scene/material/mat.cxx

@@ -97,11 +97,13 @@ SGMaterial::SGMaterial( const SGReaderWriterOptions* options,
                         const SGPropertyNode *props,
                         SGPropertyNode *prop_root,
                         AreaList *a,
-						SGSharedPtr<const SGCondition> c)
+                        SGSharedPtr<const SGCondition> c,
+                        const std::string& n)
 {
     init();
     areas = a;
     condition = c;
+    region = n;
     read_properties( options, props, prop_root );
     buildEffectProperties(options);
 }
@@ -110,12 +112,14 @@ SGMaterial::SGMaterial( const osgDB::Options* options,
                         const SGPropertyNode *props,
                         SGPropertyNode *prop_root,
                         AreaList *a,
-                        SGSharedPtr<const SGCondition> c)
+                        SGSharedPtr<const SGCondition> c,
+                        const std::string& n)
 {
     osg::ref_ptr<SGReaderWriterOptions> opt;
     opt = SGReaderWriterOptions::copyOrCreate(options);
     areas = a;
     condition = c;
+    region = n;
     init();
     read_properties(opt.get(), props, prop_root);
     buildEffectProperties(opt.get());

simgear/scene/material/mat.hxx

@@ -97,14 +97,16 @@ public:
              const SGPropertyNode *props,
              SGPropertyNode *prop_root,
              AreaList *a,
-			 SGSharedPtr<const SGCondition> c);
+             SGSharedPtr<const SGCondition> c,
+             const std::string& n);
 
 
   SGMaterial(const simgear::SGReaderWriterOptions*,
              const SGPropertyNode *props,
              SGPropertyNode *prop_root,
              AreaList *a,
-			 SGSharedPtr<const SGCondition> c);
+             SGSharedPtr<const SGCondition> c,
+             const std::string& n);
 
   /**
    * Destructor.
@@ -123,6 +125,11 @@ public:
   simgear::Effect* get_one_effect(int texIndex);
   simgear::Effect* get_effect();
 
+  /**
+   * Get the region Name.
+   */
+  const std::string get_region_name() const { return region; }
+
   /**
    * Get the Effect Name.
    */
@@ -487,6 +494,9 @@ private:
   SGVec4f ambient, diffuse, specular, emission;
   double shininess;
 
+  // region of this material
+  std::string region;
+
   // effect for this material
   std::string effect;
 

simgear/scene/material/matlib.cxx

@@ -83,6 +83,8 @@ bool SGMaterialLib::load( const SGPath &fg_root, const SGPath& mpath,
     options->setObjectCacheHint(osgDB::Options::CACHE_ALL);
     options->setDatabasePath(fg_root.utf8Str());
 
+    std::lock_guard<std::mutex> g(d->mutex);
+
     simgear::PropertyList blocks = materialblocks.getChildren("region");
     simgear::PropertyList::const_iterator block_iter = blocks.begin();
 
@@ -128,12 +130,13 @@ bool SGMaterialLib::load( const SGPath &fg_root, const SGPath& mpath,
 
 		// Now build all the materials for this set of areas and conditions
 
+		const std::string region = node->getStringValue("name");
 		const simgear::PropertyList materials = node->getChildren("material");
 		simgear::PropertyList::const_iterator materials_iter = materials.begin();
 		for (; materials_iter != materials.end(); materials_iter++) {
 			const SGPropertyNode *node = materials_iter->get();
 			SGSharedPtr<SGMaterial> m =
-					new SGMaterial(options.get(), node, prop_root, arealist, condition);
+					new SGMaterial(options.get(), node, prop_root, arealist, condition, region);
 
 			std::vector<SGPropertyNode_ptr>names = node->getChildren("name");
 			for ( unsigned int j = 0; j < names.size(); j++ ) {
@@ -152,14 +155,20 @@ bool SGMaterialLib::load( const SGPath &fg_root, const SGPath& mpath,
 
 // find a material record by material name and tile center
 SGMaterial *SGMaterialLib::find( const string& material, const SGVec2f center ) const
+{
+    std::lock_guard<std::mutex> g(d->mutex);
+    return internalFind(material, center);
+}
+
+SGMaterial* SGMaterialLib::internalFind(const string& material, const SGVec2f center) const
 {
     SGMaterial *result = NULL;
     const_material_map_iterator it = matlib.find( material );
-    if ( it != end() ) {            
+    if (it != end()) {
         // We now have a list of materials that match this
         // name. Find the first one that matches.
-    	// We start at the end of the list, as the materials
-    	// list is ordered with the smallest regions at the end.
+        // We start at the end of the list, as the materials
+        // list is ordered with the smallest regions at the end.
         material_list::const_reverse_iterator iter = it->second.rbegin();
         while (iter != it->second.rend()) {
             result = *iter;
@@ -183,11 +192,12 @@ SGMaterial *SGMaterialLib::find( const string& material, const SGGeod& center )
 SGMaterialCache *SGMaterialLib::generateMatCache(SGVec2f center)
 {
 	SGMaterialCache* newCache = new SGMaterialCache();
+    std::lock_guard<std::mutex> g(d->mutex);
+
     material_map::const_reverse_iterator it = matlib.rbegin();
     for (; it != matlib.rend(); ++it) {
-        newCache->insert(it->first, find(it->first, center));
+        newCache->insert(it->first, internalFind(it->first, center));
     }
-    
     return newCache;
 }
 

simgear/scene/material/matlib.hxx

@@ -81,7 +81,10 @@ private:
     typedef material_map::const_iterator const_material_map_iterator;
 
     material_map matlib;
-    
+
+
+    SGMaterial* internalFind(const std::string& material, const SGVec2f center) const;
+
 public:
 
     // Constructor

simgear/scene/model/ModelRegistry.cxx

@@ -56,12 +56,15 @@
 #include <simgear/scene/util/SGReaderWriterOptions.hxx>
 #include <simgear/scene/util/NodeAndDrawableVisitor.hxx>
 
-#include <simgear/structure/exception.hxx>
-#include <simgear/props/props.hxx>
-#include <simgear/props/props_io.hxx>
-#include <simgear/props/condition.hxx>
+
+#include <simgear/debug/ErrorReportingCallback.hxx>
+
 #include <simgear/io/sg_file.hxx>
 #include <simgear/misc/lru_cache.hxx>
+#include <simgear/props/condition.hxx>
+#include <simgear/props/props.hxx>
+#include <simgear/props/props_io.hxx>
+#include <simgear/structure/exception.hxx>
 
 #include "BoundingVolumeBuildVisitor.hxx"
 #include "model.hxx"
@@ -289,6 +292,11 @@ ModelRegistry::readImage(const string& fileName,
 
     const SGReaderWriterOptions* sgoptC = dynamic_cast<const SGReaderWriterOptions*>(opt);
 
+    simgear::ErrorReportContext ec;
+    if (sgoptC && sgoptC->getModelData()) {
+        ec.addFromMap(sgoptC->getModelData()->getErrorContext());
+    }
+
     if (cache_active && (!sgoptC || sgoptC->getLoadOriginHint() != SGReaderWriterOptions::LoadOriginHint::ORIGIN_SPLASH_SCREEN)) {
         if (fileExtension != "dds" && fileExtension != "gz") {
 
@@ -393,6 +401,10 @@ ModelRegistry::readImage(const string& fileName,
                         isEffect = true;
 //                        can_compress = false;
                     }
+                    else if (sgoptC && !transparent && sgoptC->getLoadOriginHint() == SGReaderWriterOptions::LoadOriginHint::ORIGIN_CANVAS) {
+                        SG_LOG(SG_IO, SG_INFO, "From Canvas " + absFileName + " will generate mipmap only");
+                        can_compress = false;
+                    }
                     if (can_compress)
                     {
                         std::string pot_message;
@@ -545,7 +557,13 @@ ModelRegistry::readImage(const string& fileName,
         }
     }
 
-    res = registry->readImageImplementation(absFileName, opt);
+    try {
+        res = registry->readImageImplementation(absFileName, opt);
+    } catch (std::bad_alloc&) {
+        simgear::reportFailure(simgear::LoadFailure::OutOfMemory, simgear::ErrorCode::ThreeDModelLoad,
+                               "Out of memory loading texture", sg_location{absFileName});
+        return ReaderWriter::ReadResult::INSUFFICIENT_MEMORY_TO_LOAD;
+    }
 
     if (!res.success()) {
         SG_LOG(SG_IO, SG_DEV_WARN, "Image loading failed:" << res.message());
@@ -710,7 +728,17 @@ ReaderWriter::ReadResult
 ModelRegistry::readNode(const string& fileName,
                         const Options* opt)
 {
-    ReaderWriter::ReadResult res;
+    // propogate error context from the caller
+    simgear::ErrorReportContext ec;
+    auto sgopt = dynamic_cast<const SGReaderWriterOptions*>(opt);
+    if (sgopt) {
+        if (sgopt->getModelData()) {
+            ec.addFromMap(sgopt->getModelData()->getErrorContext());
+        }
+
+        ec.addFromMap(sgopt->getErrorContext());
+    }
+
     CallbackMap::iterator iter
         = nodeCallbackMap.find(getFileExtension(fileName));
     ReaderWriter::ReadResult result;
@@ -719,6 +747,11 @@ ModelRegistry::readNode(const string& fileName,
     else
         result = _defaultCallback->readNode(fileName, opt);
 
+    if (!result.validNode()) {
+        simgear::reportFailure(simgear::LoadFailure::BadData, simgear::ErrorCode::ThreeDModelLoad,
+                               "Failed to load 3D model:" + result.message(), sg_location{fileName});
+    }
+
     return result;
 }
 

simgear/scene/model/SGReaderWriterXML.cxx

@@ -35,12 +35,13 @@
 #include <osgDB/FileNameUtils>
 
 #include <simgear/compiler.h>
-#include <simgear/structure/exception.hxx>
+#include <simgear/debug/ErrorReportingCallback.hxx>
+#include <simgear/props/condition.hxx>
 #include <simgear/props/props.hxx>
 #include <simgear/props/props_io.hxx>
-#include <simgear/props/condition.hxx>
 #include <simgear/scene/util/SGNodeMasks.hxx>
 #include <simgear/scene/util/SGReaderWriterOptions.hxx>
+#include <simgear/structure/exception.hxx>
 
 #include "modellib.hxx"
 #include "SGReaderWriterXML.hxx"
@@ -81,6 +82,7 @@ SGReaderWriterXML::readNode(const std::string& name,
                             const osgDB::Options* options) const
 {
     std::string fileName = osgDB::findDataFile(name, options);
+    simgear::ErrorReportContext ec{"model-xml", fileName};
 
     osg::Node *result=0;
     try {
@@ -262,8 +264,10 @@ sgLoad3DModel_internal(const SGPath& path,
                        SGPropertyNode *overlay)
 {
     if (!path.exists()) {
-      SG_LOG(SG_IO, SG_DEV_ALERT, "Failed to load file: \"" << path << "\"");
-      return std::make_tuple(0, (osg::Node *) NULL);
+        simgear::reportFailure(simgear::LoadFailure::NotFound, simgear::ErrorCode::XMLModelLoad,
+                               "Failed to load model XML: not found", path);
+        SG_LOG(SG_IO, SG_DEV_ALERT, "Failed to load file: \"" << path << "\"");
+        return std::make_tuple(0, (osg::Node*)NULL);
     }
 
     osg::ref_ptr<SGReaderWriterOptions> options;
@@ -291,6 +295,8 @@ sgLoad3DModel_internal(const SGPath& path,
        try {
             readProperties(modelpath, props);
         } catch (const sg_exception &t) {
+            simgear::reportFailure(simgear::LoadFailure::BadData, simgear::ErrorCode::XMLModelLoad,
+                                   "Failed to load model XML:" + t.getFormattedMessage(), t.getLocation());
             SG_LOG(SG_IO, SG_DEV_ALERT, "Failed to load xml: "
                    << t.getFormattedMessage());
             throw;
@@ -305,18 +311,24 @@ sgLoad3DModel_internal(const SGPath& path,
         if (props->hasValue("/path")) {
             string modelPathStr = props->getStringValue("/path");
             modelpath = SGModelLib::findDataFile(modelPathStr, NULL, modelDir);
-            if (modelpath.isNull())
+            if (modelpath.isNull()) {
+                simgear::reportFailure(simgear::LoadFailure::NotFound, simgear::ErrorCode::ThreeDModelLoad,
+                                       "Model not found:" + modelPathStr, path);
                 throw sg_io_exception("Model file not found: '" + modelPathStr + "'",
-                        path);
+                                      path, {}, false);
+            }
 
             if (props->hasValue("/texture-path")) {
                 string texturePathStr = props->getStringValue("/texture-path");
                 if (!texturePathStr.empty())
                 {
                     texturepath = SGModelLib::findDataFile(texturePathStr, NULL, modelDir);
-                    if (texturepath.isNull())
+                    if (texturepath.isNull()) {
+                        simgear::reportFailure(simgear::LoadFailure::NotFound, simgear::ErrorCode::LoadingTexture,
+                                               "Texture file not found:" + texturePathStr, path);
                         throw sg_io_exception("Texture file not found: '" + texturePathStr + "'",
                                 path);
+                    }
                 }
             }
         } else {
@@ -343,9 +355,13 @@ sgLoad3DModel_internal(const SGPath& path,
 #else
         modelResult = osgDB::readRefNodeFile(modelpath.utf8Str(), options.get());
 #endif
-        if (!modelResult.validNode())
+        if (!modelResult.validNode()) {
+            simgear::reportFailure(simgear::LoadFailure::BadData, simgear::ErrorCode::XMLModelLoad,
+                                   "Failed to load 3D model:" + modelResult.message(), modelpath);
             throw sg_io_exception("Failed to load 3D model:" + modelResult.message(),
-                                  modelpath);
+                                  modelpath, {}, false);
+        }
+
         model = copyModel(modelResult.getNode());
         // Add an extra reference to the model stored in the database.
         // That is to avoid expiring the object from the cache even if
@@ -413,6 +429,9 @@ sgLoad3DModel_internal(const SGPath& path,
 
         if (submodelPath.isNull()) {
           SG_LOG(SG_IO, SG_DEV_ALERT, "Failed to load file: \"" << subPathStr << "\"");
+          simgear::reportFailure(simgear::LoadFailure::NotFound, simgear::ErrorCode::XMLModelLoad,
+                                 "Couldn't find file for submodel:" + subPathStr,
+                                 SGPath::fromUtf8(subPathStr));
           continue;
         }
 
@@ -425,6 +444,8 @@ sgLoad3DModel_internal(const SGPath& path,
             }
         }
 
+        simgear::ErrorReportContext("submodel", submodelPath.utf8Str());
+
         try {
             int num_anims;
             std::tie(num_anims, submodel) = sgLoad3DModel_internal(submodelPath, options.get(),
@@ -538,8 +559,8 @@ sgLoad3DModel_internal(const SGPath& path,
     // Some material animations (eventually all) are actually effects.
     makeEffectAnimations(animation_nodes, effect_nodes);
     {
-        ref_ptr<Node> modelWithEffects
-            = instantiateEffects(group.get(), effect_nodes, options.get());
+        ref_ptr<Node> modelWithEffects = instantiateEffects(group.get(), effect_nodes, options.get(),
+                                                            path.dirPath());
         group = static_cast<Group*>(modelWithEffects.get());
     }
 
@@ -553,13 +574,22 @@ sgLoad3DModel_internal(const SGPath& path,
             } // of object-names in the animation
             continue;
         }
-        /*
-         * Setup the model data for the node currently being animated.
-         */
-        modelData.LoadAnimationValuesForElement(animation_nodes[i], i);
+        
+        try {
+            /*
+             * Setup the model data for the node currently being animated.
+             */
+            modelData.LoadAnimationValuesForElement(animation_nodes[i], i);
 
-        /// OSGFIXME: duh, why not only model?????
-        SGAnimation::animate(modelData);
+            /// OSGFIXME: duh, why not only model?????
+            SGAnimation::animate(modelData);
+        } catch (sg_exception& e) {
+            simgear::reportFailure(simgear::LoadFailure::NotFound, simgear::ErrorCode::XMLModelLoad,
+                                   "Couldn't load animation " + animation_nodes[i]->getNameString()
+                                    + ":" + e.getFormattedMessage(),
+                                   modelpath);
+            throw;
+        }
     }
 
     animationcount += animation_nodes.size();

simgear/scene/model/SGText.cxx

@@ -16,23 +16,27 @@
 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 //
 
-#ifdef HAVE_CONFIG_H
-#  include <simgear_config.h>
-#endif
+#include <simgear_config.h>
 
 #include <cstdio>
 
 #include "SGText.hxx"
 
-#include <simgear/math/SGMath.hxx>
-#include <simgear/misc/sg_path.hxx>
-#include <simgear/misc/strutils.hxx>
 
 #include <osg/Geode>
 #include <osg/MatrixTransform>
 #include <osgText/Text>
 #include <osgText/Font>
 
+#include <simgear/misc/ResourceManager.hxx>
+#include <simgear/math/SGMath.hxx>
+#include <simgear/misc/sg_path.hxx>
+#include <simgear/misc/strutils.hxx>
+#include <simgear/scene/material/Effect.hxx>
+#include <simgear/scene/material/EffectGeode.hxx>
+#include <simgear/scene/util/SGReaderWriterOptions.hxx>
+#include <simgear/debug/ErrorReportingCallback.hxx>
+
 using std::string;
 
 class SGText::UpdateCallback : public osg::NodeCallback {
@@ -80,7 +84,7 @@ void SGText::UpdateCallback::operator()(osg::Node * node, osg::NodeVisitor *nv )
     // be lazy and set the text only if the property has changed.
     // update() computes the glyph representation which looks 
     // more expensive than a the above string compare.
-    text->setText( buf );
+    text->setText( buf, osgText::String::ENCODING_UTF8 );
     text->update();
   }
   traverse( node, nv );
@@ -95,9 +99,15 @@ osg::Node * SGText::appendText(const SGPropertyNode* configNode,
   osg::Geode * g = new osg::Geode;
   g->addDrawable( text );
 
-  SGPath path("Fonts" );
-  path.append( configNode->getStringValue( "font", "Helvetica" ));
-  text->setFont( path.utf8Str() );
+  const std::string requestedFont = configNode->getStringValue("font","Helvetica");
+  const SGPath fontPath = simgear::ResourceManager::instance()->findPath("Fonts/" + requestedFont);
+  if ( !fontPath.isNull() ) {
+    text->setFont( fontPath.utf8Str() );
+  } else {
+    simgear::reportFailure(simgear::LoadFailure::NotFound,
+                          simgear::ErrorCode::LoadingTexture,
+                                   "SGText: couldn;t find font:" + requestedFont);
+  }
 
   text->setCharacterSize(configNode->getDoubleValue("character-size", 1.0 ), 
                          configNode->getDoubleValue("character-aspect-ratio", 1.0 ));

simgear/scene/model/animation.cxx

@@ -39,9 +39,11 @@
 #include <simgear/bvh/BVHGroup.hxx>
 #include <simgear/bvh/BVHLineGeometry.hxx>
 
+#include <simgear/debug/ErrorReportingCallback.hxx>
 #include <simgear/math/interpolater.hxx>
 #include <simgear/props/condition.hxx>
 #include <simgear/props/props.hxx>
+
 #include <simgear/scene/material/EffectGeode.hxx>
 #include <simgear/scene/material/EffectCullVisitor.hxx>
 #include <simgear/scene/util/DeletionManager.hxx>
@@ -773,10 +775,16 @@ bool SGAnimation::setCenterAndAxisFromObject(osg::Node *rootNode, SGVec3d& cente
                     object_group->setNodeMask(0);
                 }
                 else {
+                    reportFailure(LoadFailure::Misconfigured, ErrorCode::XMLModelLoad,
+                                  "Could not find valid line segment for axis animation:" + axis_object_name,
+                                  SGPath::fromUtf8(_modelData.getPath()));
                     SG_LOG(SG_IO, SG_DEV_ALERT, "Could not find a valid line segment for animation:  " << axis_object_name << " in file: " << _modelData.getPath());
                 }
             }
             else if (can_warn) {
+                reportFailure(LoadFailure::Misconfigured, ErrorCode::XMLModelLoad,
+                              "Could not find object for axis animation:" + axis_object_name,
+                              SGPath::fromUtf8(_modelData.getPath()));
                 SG_LOG(SG_IO, SG_DEV_ALERT, "Could not find at least one of the following objects for axis animation: " << axis_object_name << " in file: " << _modelData.getPath());
             }
         }
@@ -1135,16 +1143,8 @@ void SpinAnimCallback::operator()(osg::Node* node, osg::NodeVisitor* nv)
         double intPart;
         double rot = modf(rotation, &intPart);
         double angle = rot * 2.0 * osg::PI;
-        const SGVec3d& sgcenter = transform->getCenter();
-        const SGVec3d& sgaxis = transform->getAxis();
-        Matrixd mat = Matrixd::translate(-sgcenter[0], -sgcenter[1], -sgcenter[2])
-            * Matrixd::rotate(angle, sgaxis[0], sgaxis[1], sgaxis[2])
-            * Matrixd::translate(sgcenter[0], sgcenter[1], sgcenter[2])
-            * *cv->getModelViewMatrix();
-        ref_ptr<RefMatrix> refmat = new RefMatrix(mat);
-        cv->pushModelViewMatrix(refmat.get(), transform->getReferenceFrame());
+        transform->setAngleRad(angle);
         traverse(transform, nv);
-        cv->popModelViewMatrix();
     } else {
         traverse(transform, nv);
     }

simgear/scene/model/model.cxx

@@ -24,11 +24,12 @@
 #include <simgear/scene/util/SplicingVisitor.hxx>
 #include <simgear/scene/util/SGReaderWriterOptions.hxx>
 
-#include <simgear/structure/exception.hxx>
-#include <simgear/structure/Singleton.hxx>
+#include <simgear/debug/ErrorReportingCallback.hxx>
+#include <simgear/props/condition.hxx>
 #include <simgear/props/props.hxx>
 #include <simgear/props/props_io.hxx>
-#include <simgear/props/condition.hxx>
+#include <simgear/structure/Singleton.hxx>
+#include <simgear/structure/exception.hxx>
 
 #include "model.hxx"
 
@@ -233,7 +234,7 @@ public:
     typedef std::map<string, SGPropertyNode_ptr> EffectMap;
     using SplicingVisitor::apply;
     MakeEffectVisitor(const SGReaderWriterOptions* options = 0)
-        : _options(options)
+        : _options(options), _modelPath(SGPath{})
     {
     }
     virtual void apply(osg::Group& node);
@@ -245,10 +246,17 @@ public:
         _currentEffectParent = effect;
     }
     SGPropertyNode* getDefaultEffect() { return _currentEffectParent; }
+
+    void setModelPath(const SGPath& p)
+    {
+        _modelPath = p;
+    }
+
 protected:
     EffectMap _effectMap;
     SGPropertyNode_ptr _currentEffectParent;
     osg::ref_ptr<const SGReaderWriterOptions> _options;
+    SGPath _modelPath;
 };
 
 void MakeEffectVisitor::apply(osg::Group& node)
@@ -286,7 +294,7 @@ void MakeEffectVisitor::apply(osg::Geode& geode)
     makeParametersFromStateSet(ssRoot, ss);
     SGPropertyNode_ptr effectRoot = new SGPropertyNode;
     effect::mergePropertyTrees(effectRoot, ssRoot, _currentEffectParent);
-    Effect* effect = makeEffect(effectRoot, true, _options.get());
+    Effect* effect = makeEffect(effectRoot, true, _options.get(), _modelPath);
     EffectGeode* eg = dynamic_cast<EffectGeode*>(&geode);
     if (eg) {
         eg->setEffect(effect);
@@ -331,7 +339,8 @@ protected:
 
 ref_ptr<Node> instantiateEffects(osg::Node* modelGroup,
                                  PropertyList& effectProps,
-                                 const SGReaderWriterOptions* options)
+                                 const SGReaderWriterOptions* options,
+                                 const SGPath& modelPath)
 {
     SGPropertyNode_ptr defaultEffectPropRoot;
     MakeEffectVisitor visitor(options);
@@ -356,13 +365,15 @@ ref_ptr<Node> instantiateEffects(osg::Node* modelGroup,
     if (!defaultEffectPropRoot)
         defaultEffectPropRoot = DefaultEffect::instance()->getEffect();
     visitor.setDefaultEffect(defaultEffectPropRoot.ptr());
+    visitor.setModelPath(modelPath);
     modelGroup->accept(visitor);
     osg::NodeList& result = visitor.getResults();
     return ref_ptr<Node>(result[0].get());
 }
 
 ref_ptr<Node> instantiateMaterialEffects(osg::Node* modelGroup,
-                                        const SGReaderWriterOptions* options)
+                                         const SGReaderWriterOptions* options,
+                                         const SGPath& modelPath)
 {
 
     SGPropertyNode_ptr effect;
@@ -380,6 +391,8 @@ ref_ptr<Node> instantiateMaterialEffects(osg::Node* modelGroup,
       } else {
         effect = DefaultEffect::instance()->getEffect();
         SG_LOG( SG_TERRAIN, SG_ALERT, "Unable to get effect for " << options->getMaterialName());
+        simgear::reportFailure(simgear::LoadFailure::NotFound, simgear::ErrorCode::LoadEffectsShaders,
+                               "Unable to get effect for material:" + options->getMaterialName());
       }
     } else {
       effect = DefaultEffect::instance()->getEffect();
@@ -387,7 +400,7 @@ ref_ptr<Node> instantiateMaterialEffects(osg::Node* modelGroup,
 
     effect->addChild("default")->setBoolValue(true);
     effectProps.push_back(effect);
-    return instantiateEffects(modelGroup, effectProps, options);
+    return instantiateEffects(modelGroup, effectProps, options, modelPath);
 }
 
 }

simgear/scene/model/model.hxx

@@ -102,7 +102,8 @@ public:
 osg::ref_ptr<osg::Node>
 instantiateEffects(osg::Node* model,
                    PropertyList& effectProps,
-                   const SGReaderWriterOptions* options);
+                   const SGReaderWriterOptions* options,
+                   const SGPath& currentDir = SGPath{});
 
 /**
  * Apply a set of material-defined effects to a model
@@ -112,9 +113,10 @@ instantiateEffects(osg::Node* model,
  *
  * returns a copy if any nodes are changed
  */
- osg::ref_ptr<osg::Node>
- instantiateMaterialEffects(osg::Node* model,
-                    const SGReaderWriterOptions* options);
+osg::ref_ptr<osg::Node>
+instantiateMaterialEffects(osg::Node* model,
+                           const SGReaderWriterOptions* options,
+                           const SGPath& modelPath = SGPath{});
 
 /**
  * Transform an OSG subgraph by substituting the Effects and
@@ -127,10 +129,11 @@ instantiateEffects(osg::Node* model,
 
 inline osg::ref_ptr<osg::Node>
 instantiateEffects(osg::Node* model,
-                   const SGReaderWriterOptions* options)
+                   const SGReaderWriterOptions* options,
+                   const SGPath& currentDir = SGPath{})
 {
     PropertyList effectProps;
-    return instantiateEffects(model, effectProps, options);
+    return instantiateEffects(model, effectProps, options, currentDir);
 }
 }
 #endif // __MODEL_HXX

simgear/scene/model/modellib.cxx

@@ -29,12 +29,13 @@
 #include <osgDB/Registry>
 
 #include <simgear/constants.h>
+#include <simgear/debug/ErrorReportingCallback.hxx>
+#include <simgear/misc/ResourceManager.hxx>
 #include <simgear/props/props.hxx>
 #include <simgear/props/props_io.hxx>
-#include <simgear/scene/model/model.hxx>
 #include <simgear/scene/model/ModelRegistry.hxx>
+#include <simgear/scene/model/model.hxx>
 #include <simgear/scene/util/SGReaderWriterOptions.hxx>
-#include <simgear/misc/ResourceManager.hxx>
 
 #include "SGReaderWriterXML.hxx"
 
@@ -159,7 +160,6 @@ SGModelLib::loadDeferredModel(const string &path, SGPropertyNode *prop_root,
     proxyNode->setLoadingExternalReferenceMode(osg::ProxyNode::DEFER_LOADING_TO_DATABASE_PAGER);
     proxyNode->setFileName(0, path);
 
-
     osg::ref_ptr<SGReaderWriterOptions> opt;
     opt = SGReaderWriterOptions::copyOrCreate(osgDB::Registry::instance()->getOptions());
     opt->getDatabasePathList().push_front( osgDB::getFilePath(path) );

simgear/scene/model/modellib.hxx

@@ -18,12 +18,9 @@
 #ifndef _SG_MODEL_LIB_HXX
 #define _SG_MODEL_LIB_HXX 1
 
-#ifndef __cplusplus
-# error This library requires C++
-#endif
-
 #include <simgear/compiler.h>	// for SG_USING_STD
 
+#include <map>
 #include <string>
 
 #include <osg/Node>
@@ -112,6 +109,10 @@ public:
     virtual void modelLoaded(const std::string& path, SGPropertyNode *prop,
                              osg::Node* branch) = 0;
     virtual SGModelData* clone() const = 0;
+
+    using ErrorContext = std::map<std::string, std::string>;
+
+    virtual ErrorContext getErrorContext() const = 0;
 };
 
 /*

simgear/scene/model/particles.cxx

@@ -36,10 +36,10 @@
 #include <osgParticle/MultiSegmentPlacer>
 #include <osgParticle/SectorPlacer>
 #include <osgParticle/ConstantRateCounter>
-#include <osgParticle/ParticleSystemUpdater>
 #include <osgParticle/ParticleSystem>
 #include <osgParticle/FluidProgram>
 
+#include <osgUtil/CullVisitor>
 #include <osg/Geode>
 #include <osg/Group>
 #include <osg/MatrixTransform>
@@ -48,67 +48,250 @@
 
 #include <simgear/scene/model/animation.hxx>
 
+using ParticleSystemRef = osg::ref_ptr<osgParticle::ParticleSystem>;
+
+
 namespace simgear
 {
 
 class ParticlesGlobalManager::ParticlesGlobalManagerPrivate : public osg::NodeCallback
 {
 public:
-    ParticlesGlobalManagerPrivate() : _updater(new osgParticle::ParticleSystemUpdater),
-                                      _commonGeode(new osg::Geode)
+    ParticlesGlobalManagerPrivate();
+
+    void operator()(osg::Node* node, osg::NodeVisitor* nv) override;
+
+    // only call this with the lock held!
+    osg::Group* internalGetCommonRoot();
+
+    void updateParticleSystemsFromCullCallback(int currentFrameNumber, osg::NodeVisitor* nv);
+
+    void addParticleSystem(osgParticle::ParticleSystem* ps, const osg::ref_ptr<osg::Group>& frame);
+
+    void registerNewLocalParticleSystem(osg::Node* node, ParticleSystemRef ps);
+    void registerNewWorldParticleSystem(osg::Node* node, ParticleSystemRef ps, osg::Group* frame);
+
+    std::mutex _lock;
+    bool _frozen = false;
+    double _simulationDt = 0.0;
+    osg::ref_ptr<osg::Group> _commonRoot;
+    osg::ref_ptr<osg::Geode> _commonGeode;
+    osg::Vec3 _wind;
+    bool _enabled = true;
+    osg::Vec3 _gravity;
+    //  osg::Vec3 _localWind;
+    SGGeod _currentPosition;
+
+    SGConstPropertyNode_ptr _enabledNode;
+
+    using ParticleSystemWeakRef = osg::observer_ptr<osgParticle::ParticleSystem>;
+    using ParticleSystemsWeakRefVec = std::vector<ParticleSystemWeakRef>;
+
+    using ParticleSystemsStrongRefVec = std::vector<ParticleSystemRef>;
+
+    ParticleSystemsWeakRefVec _systems;
+    osg::ref_ptr<ParticlesGlobalManager::UpdaterCallback> _cullCallback;
+
+    using GroupRefVec = std::vector<osg::ref_ptr<osg::Group>>;
+    GroupRefVec _newWorldParticles;
+};
+
+/**
+ @brief this class replaces the need to use osgParticle::ParticleSystemUpdater, which has some
+ thread-safety and ownership complications in our us case
+ */
+class ParticlesGlobalManager::UpdaterCallback : public osg::NodeCallback
+{
+public:
+    UpdaterCallback(ParticlesGlobalManagerPrivate* p) : _manager(p)
     {
     }
 
     void operator()(osg::Node* node, osg::NodeVisitor* nv) override
     {
-        std::lock_guard<std::mutex> g(_lock);
-        _enabled = !_enabledNode || _enabledNode->getBoolValue();
-
-        if (!_enabled)
-            return;
-
-        const auto q = SGQuatd::fromLonLatDeg(_longitudeNode->getFloatValue(), _latitudeNode->getFloatValue());
-        osg::Matrix om(toOsg(q));
-        osg::Vec3 v(0, 0, 9.81);
-
-        _gravity = om.preMult(v);
-
-        // NOTE: THIS WIND COMPUTATION DOESN'T SEEM TO AFFECT PARTICLES
-        // const osg::Vec3& zUpWind = _wind;
-        // osg::Vec3 w(zUpWind.y(), zUpWind.x(), -zUpWind.z());
-        // _localWind = om.preMult(w);
-    }
-
-    // only call this with the lock held!
-    osg::Group* internalGetCommonRoot()
-    {
-        if (!_commonRoot.valid()) {
-            SG_LOG(SG_PARTICLES, SG_DEBUG, "Particle common root called.");
-            _commonRoot = new osg::Group;
-            _commonRoot->setName("common particle system root");
-            _commonGeode->setName("common particle system geode");
-            _commonRoot->addChild(_commonGeode);
-            _commonRoot->addChild(_updater);
-            _commonRoot->setNodeMask(~simgear::MODELLIGHT_BIT);
+        osgUtil::CullVisitor* cv = dynamic_cast<osgUtil::CullVisitor*>(nv);
+        if (cv && nv->getFrameStamp()) {
+            if (_frameNumber < nv->getFrameStamp()->getFrameNumber()) {
+                _frameNumber = nv->getFrameStamp()->getFrameNumber();
+                _manager->updateParticleSystemsFromCullCallback(_frameNumber, nv);
+            }
         }
-        return _commonRoot.get();
+
+        // note, callback is responsible for scenegraph traversal so
+        // they must call traverse(node,nv) to ensure that the
+        // scene graph subtree (and associated callbacks) are traversed.
+        traverse(node, nv);
     }
 
-    std::mutex _lock;
-    bool _frozen = false;
-    osg::ref_ptr<osgParticle::ParticleSystemUpdater> _updater;
-    osg::ref_ptr<osg::Group> _commonRoot;
-    osg::ref_ptr<osg::Geode> _commonGeode;
-    osg::Vec3 _wind;
-    bool _globalCallbackRegistered = false;
-    bool _enabled = true;
-    osg::Vec3 _gravity;
-    //  osg::Vec3 _localWind;
-
-    SGConstPropertyNode_ptr _enabledNode;
-    SGConstPropertyNode_ptr _longitudeNode, _latitudeNode;
+    unsigned int _frameNumber = 0;
+    ParticlesGlobalManagerPrivate* _manager;
 };
 
+/**
+  single-shot node callback, used to register a particle system with the global manager. Once run, removes
+  itself. We used this to avoid updating a particle system until the load is complete and merged into the
+  main scene.
+ */
+class ParticlesGlobalManager::RegistrationCallback : public osg::NodeCallback
+{
+public:
+    void operator()(osg::Node* node, osg::NodeVisitor* nv) override
+    {
+        auto d = ParticlesGlobalManager::instance()->d;
+        d->addParticleSystem(_system, _frame);
+        node->removeUpdateCallback(this); // suicide
+    }
+
+    ParticleSystemRef _system;
+    osg::ref_ptr<osg::Group> _frame;
+};
+
+ParticlesGlobalManager::ParticlesGlobalManagerPrivate::ParticlesGlobalManagerPrivate() : _commonGeode(new osg::Geode),
+                                                                                         _cullCallback(new UpdaterCallback(this))
+{
+    // callbacks are registered in initFromMainThread : depending on timing,
+    // this constructor might be called from an osgDB thread
+}
+
+void ParticlesGlobalManager::ParticlesGlobalManagerPrivate::addParticleSystem(osgParticle::ParticleSystem* ps, const osg::ref_ptr<osg::Group>& frame)
+{
+    std::lock_guard<std::mutex> g(_lock);
+    _systems.push_back(ps);
+
+    // we're inside an update callback here, so better not modify the
+    // scene structure; defer it to later
+    if (frame.get()) {
+        _newWorldParticles.push_back(frame);
+    }
+}
+
+void ParticlesGlobalManager::ParticlesGlobalManagerPrivate::registerNewLocalParticleSystem(osg::Node* node, ParticleSystemRef ps)
+{
+    auto cb = new RegistrationCallback;
+    cb->_system = ps;
+    node->addUpdateCallback(cb);
+}
+
+void ParticlesGlobalManager::ParticlesGlobalManagerPrivate::registerNewWorldParticleSystem(osg::Node* node, ParticleSystemRef ps, osg::Group* frame)
+{
+    auto cb = new RegistrationCallback;
+    cb->_system = ps;
+    cb->_frame = frame;
+    node->addUpdateCallback(cb);
+}
+
+// this is called from the main thread during scenery init
+// after this, we beign updarting particle systems
+void ParticlesGlobalManager::initFromMainThread()
+{
+    std::lock_guard<std::mutex> g(d->_lock);
+    d->internalGetCommonRoot();
+    d->_commonRoot->addUpdateCallback(d.get());
+    d->_commonRoot->setCullingActive(false);
+    d->_commonRoot->addCullCallback(d->_cullCallback);
+}
+
+void ParticlesGlobalManager::update(double dt, const SGGeod& pos)
+{
+    std::lock_guard<std::mutex> g(d->_lock);
+    d->_simulationDt = dt;
+    d->_currentPosition = pos;
+
+    for (auto f : d->_newWorldParticles) {
+        d->internalGetCommonRoot()->addChild(f);
+    }
+
+    d->_newWorldParticles.clear();
+}
+
+// this is called from the main thread, since it's an update callback
+// lock any state used by updateParticleSystemsFromCullCallback, which
+// runs during culling, potentialy on a different thread
+void ParticlesGlobalManager::ParticlesGlobalManagerPrivate::operator()(osg::Node* node, osg::NodeVisitor* nv)
+{
+    std::lock_guard<std::mutex> g(_lock);
+    _enabled = !_enabledNode || _enabledNode->getBoolValue();
+
+    if (!_enabled)
+        return;
+
+    const auto q = SGQuatd::fromLonLat(_currentPosition);
+    osg::Matrix om(toOsg(q));
+    osg::Vec3 v(0, 0, 9.81);
+    _gravity = om.preMult(v);
+
+    // NOTE: THIS WIND COMPUTATION DOESN'T SEEM TO AFFECT PARTICLES
+    // const osg::Vec3& zUpWind = _wind;
+    // osg::Vec3 w(zUpWind.y(), zUpWind.x(), -zUpWind.z());
+    // _localWind = om.preMult(w);
+
+    // while we have the lock, remove all expired systems
+    auto firstInvalid = std::remove_if(_systems.begin(), _systems.end(), [](const ParticleSystemWeakRef& weak) {
+        return !weak.valid();
+    });
+    _systems.erase(firstInvalid, _systems.end());
+}
+
+// only call this with the lock held!
+osg::Group* ParticlesGlobalManager::ParticlesGlobalManagerPrivate::internalGetCommonRoot()
+{
+    if (!_commonRoot.valid()) {
+        _commonRoot = new osg::Group;
+        _commonRoot->setName("common particle system root");
+        _commonGeode->setName("common particle system geode");
+        _commonRoot->addChild(_commonGeode);
+        _commonRoot->setNodeMask(~simgear::MODELLIGHT_BIT);
+    }
+    return _commonRoot.get();
+}
+
+void ParticlesGlobalManager::ParticlesGlobalManagerPrivate::updateParticleSystemsFromCullCallback(int frameNumber, osg::NodeVisitor* nv)
+{
+    ParticleSystemsStrongRefVec activeSystems;
+    double dt = 0.0;
+
+    // begin locked section
+    {
+        std::lock_guard<std::mutex> g(_lock);
+        activeSystems.reserve(_systems.size());
+        for (const auto& psref : _systems) {
+            ParticleSystemRef owningRef;
+            if (!psref.lock(owningRef)) {
+                // pointed to system is gone, skip it
+                // we will clean these up in the update callback, don't
+                // worry about that here
+                continue;
+            }
+
+            // add to the list we will update now
+            activeSystems.push_back(owningRef);
+        }
+
+        dt = _simulationDt;
+    } // of locked section
+
+    // from here on, don't access class data; copy it all to local variables
+    // before this line. this is important so we're not holding _lock during
+    // culling, which might block osgDB threads for example.
+
+    if (dt <= 0.0) {
+        return;
+    }
+
+    for (const auto& ps : activeSystems) {
+        // code inside here is copied from osgParticle::ParticleSystemUpdate
+        osgParticle::ParticleSystem::ScopedWriteLock lock(*(ps->getReadWriteMutex()));
+
+        // We need to allow at least 2 frames difference, because the particle system's lastFrameNumber
+        // is updated in the draw thread which may not have completed yet.
+        if (!ps->isFrozen() &&
+            (!ps->getFreezeOnCull() || ((frameNumber - ps->getLastFrameNumber()) <= 2))) {
+            ps->update(dt, *nv);
+        }
+        // end of code copied from osgParticle::ParticleSystemUpdate
+    }
+}
+
 static std::mutex static_managerLock;
 static std::unique_ptr<ParticlesGlobalManager> static_instance;
 
@@ -132,14 +315,6 @@ ParticlesGlobalManager::ParticlesGlobalManager() : d(new ParticlesGlobalManagerP
 {
 }
 
-ParticlesGlobalManager::~ParticlesGlobalManager()
-{
-    if (d->_globalCallbackRegistered) {
-        // is this actually necessary? possibly not
-        d->_updater->setUpdateCallback(nullptr);
-    }
-}
-
 bool ParticlesGlobalManager::isEnabled() const
 {
     std::lock_guard<std::mutex> g(d->_lock);
@@ -337,6 +512,14 @@ void Particles::setupStaticColorComponent(float r1, float g1, float b1, float a1
     staticColorComponents[7] = a2;
 }
 
+ParticlesGlobalManager::~ParticlesGlobalManager()
+{
+    // break a ref-counting cycle
+    if (d->_commonRoot) {
+        d->_commonRoot->removeUpdateCallback(d.get());
+    }
+}
+
 void ParticlesGlobalManager::setWindVector(const osg::Vec3& wind)
 {
     std::lock_guard<std::mutex> g(d->_lock);
@@ -728,23 +911,10 @@ osg::ref_ptr<osg::Group> ParticlesGlobalManager::appendParticles(const SGPropert
         emitter->setUpdateCallback(callback.get());
     }
 
-    // touch shared data now (and not before)
-
-    {
-        std::lock_guard<std::mutex> g(d->_lock);
-        d->_updater->addParticleSystem(particleSys);
-
-        if (attach != "local") {
-            d->internalGetCommonRoot()->addChild(callback()->particleFrame);
-        }
-
-        if (!d->_globalCallbackRegistered) {
-            SG_LOG(SG_PARTICLES, SG_INFO, "Registering global particles callback");
-            d->_globalCallbackRegistered = true;
-            d->_longitudeNode = modelRoot->getNode("/position/longitude-deg", true);
-            d->_latitudeNode = modelRoot->getNode("/position/latitude-deg", true);
-            d->_updater->setUpdateCallback(d.get());
-        }
+    if (attach == "local") {
+        d->registerNewLocalParticleSystem(align, particleSys);
+    } else {
+        d->registerNewWorldParticleSystem(align, particleSys, callback()->particleFrame);
     }
 
     return align;

simgear/scene/model/particles.hxx

@@ -41,7 +41,6 @@ class NodeVisitor;
 namespace osgParticle
 {
 class ParticleSystem;
-class ParticleSystemUpdater;
 }
 
 #include <simgear/scene/util/SGNodeMasks.hxx>
@@ -161,14 +160,18 @@ public:
     osg::ref_ptr<osg::Group> appendParticles(const SGPropertyNode* configNode, SGPropertyNode* modelRoot, const osgDB::Options* options);
 
     osg::Group* getCommonRoot();
-
     osg::Geode* getCommonGeode();
 
-    osgParticle::ParticleSystemUpdater* getPSU();
+    void initFromMainThread();
 
     void setFrozen(bool e);
     bool isFrozen() const;
 
+    /**
+        @brief update function: call from the main thread , outside of OSG calls.
+     */
+    void update(double dt, const SGGeod& pos);
+
     void setSwitchNode(const SGPropertyNode* n);
 
     /**
@@ -185,6 +188,8 @@ private:
     ParticlesGlobalManager();
 
     class ParticlesGlobalManagerPrivate;
+    class UpdaterCallback;
+    class RegistrationCallback;
 
     // because Private inherits NodeCallback, we need to own it
     // via an osg::ref_ptr

simgear/scene/tgdb/ReaderWriterSPT.cxx

@@ -294,7 +294,7 @@ ReaderWriterSPT::createTree(const BucketBox& bucketBox, const LocalOptions& opti
 osg::ref_ptr<osg::Node>
 ReaderWriterSPT::createPagedLOD(const BucketBox& bucketBox, const LocalOptions& options) const
 {
-    osg::PagedLOD* pagedLOD = new osg::PagedLOD;
+    osg::ref_ptr<osg::PagedLOD> pagedLOD = new osg::PagedLOD;
 
     pagedLOD->setCenterMode(osg::PagedLOD::USER_DEFINED_CENTER);
     SGSpheref sphere = bucketBox.getBoundingSphere();

simgear/scene/tgdb/ReaderWriterSTG.cxx

@@ -38,10 +38,12 @@
 #include <osgDB/ReaderWriter>
 #include <osgDB/ReadFile>
 
-#include <simgear/math/sg_random.h>
-#include <simgear/math/SGGeometry.hxx>
 #include <simgear/bucket/newbucket.hxx>
+#include <simgear/debug/ErrorReportingCallback.hxx>
 #include <simgear/debug/logstream.hxx>
+#include <simgear/math/SGGeometry.hxx>
+#include <simgear/math/sg_random.h>
+
 #include <simgear/io/iostreams/sgstream.hxx>
 #include <simgear/scene/util/OptionsReadFileCallback.hxx>
 #include <simgear/scene/util/OsgMath.hxx>
@@ -154,7 +156,7 @@ struct ReaderWriterSTG::_ModelBin {
                 proxy->setName("proxyNode");
                 proxy->setLoadingExternalReferenceMode(osg::ProxyNode::DEFER_LOADING_TO_DATABASE_PAGER);
                 proxy->setFileName(0, o._name);
-                proxy->setDatabaseOptions(o._options.get());
+                proxy->setDatabaseOptions(o._options);
 
                 // Give the node some values so the Quadtree builder has
                 // a BoundingBox to work with prior to the model being loaded.
@@ -163,6 +165,7 @@ struct ReaderWriterSTG::_ModelBin {
                 proxy->setCenterMode(osg::ProxyNode::UNION_OF_BOUNDING_SPHERE_AND_USER_DEFINED);
                 node = proxy;
             } else {
+                ErrorReportContext ec("terrain-stg", o._errorLocation.utf8Str());
 #if OSG_VERSION_LESS_THAN(3,4,0)
                 node = osgDB::readNodeFile(o._name, o._options.get());
 #else
@@ -209,10 +212,13 @@ struct ReaderWriterSTG::_ModelBin {
         virtual osgDB::ReaderWriter::ReadResult
         readNode(const std::string&, const osgDB::Options*)
         {
+            ErrorReportContext ec("terrain-bucket", _bucket.gen_index_str());
+
             STGObjectsQuadtree quadtree((GetModelLODCoord()), (AddModelLOD()));
             quadtree.buildQuadTree(_objectStaticList.begin(), _objectStaticList.end());
             osg::ref_ptr<osg::Group> group = quadtree.getRoot();
-            group->setName("STG-group-A");
+            string group_name = string("STG-group-A ").append(_bucket.gen_index_str());
+            group->setName(group_name);
             group->setDataVariance(osg::Object::STATIC);
 
             simgear::AirportSignBuilder signBuilder(_options->getMaterialLib(), _bucket.get_center());
@@ -464,15 +470,15 @@ struct ReaderWriterSTG::_ModelBin {
 										else
 												opt->setInstantiateEffects(false);
 										_ObjectStatic obj;
-
-										obj._errorLocation = absoluteFileName;
-										obj._token = token;
-										obj._name = name;
-										obj._agl = (token == "OBJECT_STATIC_AGL");
-										obj._proxy = true;
-										in >> obj._lon >> obj._lat >> obj._elev >> obj._hdg >> obj._pitch >> obj._roll;
-										obj._range = range;
-										obj._options = opt;
+                                        opt->addErrorContext("terrain-stg", absoluteFileName.utf8Str());
+                                        obj._errorLocation = absoluteFileName;
+                                        obj._token = token;
+                                        obj._name = name;
+                                        obj._agl = (token == "OBJECT_STATIC_AGL");
+                                        obj._proxy = true;
+                                        in >> obj._lon >> obj._lat >> obj._elev >> obj._hdg >> obj._pitch >> obj._roll;
+                                        obj._range = range;
+                                        obj._options = opt;
                     checkInsideBucket(absoluteFileName, obj._lon, obj._lat);
                     _objectStaticList.push_back(obj);
                 } else if (token == "OBJECT_SHARED" || token == "OBJECT_SHARED_AGL") {
@@ -508,6 +514,8 @@ struct ReaderWriterSTG::_ModelBin {
                     _ObjectStatic obj;
 
                     opt->setInstantiateEffects(false);
+                    opt->addErrorContext("terrain-stg", absoluteFileName.utf8Str());
+
                     if (SGPath(name).lower_extension() == "ac") {
                       // Generate material/Effects lookups for raw models, i.e.
                       // those not wrapped in an XML while will include Effects
@@ -572,7 +580,9 @@ struct ReaderWriterSTG::_ModelBin {
                             }
                             callback(token,name, SGGeod::fromDegM(obj._lon, obj._lat, obj._elev), obj._hdg,restofline);
                         } else {
-                            SG_LOG( SG_TERRAIN, SG_ALERT, absoluteFileName << ": Unknown token '" << token << "'" );
+                            // SG_LOG( SG_TERRAIN, SG_ALERT, absoluteFileName << ": Unknown token '" << token << "'" );
+                            simgear::reportFailure(simgear::LoadFailure::Misconfigured, simgear::ErrorCode::BTGLoad,
+                                                   "Unknown STG token:" + token, absoluteFileName);
                         }
                     }
                 }
@@ -586,14 +596,20 @@ struct ReaderWriterSTG::_ModelBin {
     {
         osg::ref_ptr<SGReaderWriterOptions> options;
         options = SGReaderWriterOptions::copyOrCreate(opt);
+        float pagedLODExpiry = atoi(options->getPluginStringData("SimGear::PAGED_LOD_EXPIRY").c_str());
 
         osg::ref_ptr<osg::Group> terrainGroup = new osg::Group;
         terrainGroup->setDataVariance(osg::Object::STATIC);
-        terrainGroup->setName("terrain");
+        std::string terrain_name = string("terrain ").append(bucket.gen_index_str());
+        terrainGroup->setName(terrain_name);
+
+        simgear::ErrorReportContext ec{"terrain-bucket", bucket.gen_index_str()};
 
         if (_foundBase) {
             for (auto stgObject : _objectList) {
                 osg::ref_ptr<osg::Node> node;
+                simgear::ErrorReportContext ec("terrain-stg", stgObject._errorLocation.utf8Str());
+
 #if OSG_VERSION_LESS_THAN(3,4,0)
                 node = osgDB::readNodeFile(stgObject._name, stgObject._options.get());
 #else
@@ -618,7 +634,6 @@ struct ReaderWriterSTG::_ModelBin {
                         "Warning: failed to generate ocean tile!" );
             }
         }
-
         for (std::list<_ObjectStatic>::iterator i = _objectStaticList.begin(); i != _objectStaticList.end(); ++i) {
             if (!i->_agl)
                 continue;
@@ -637,11 +652,13 @@ struct ReaderWriterSTG::_ModelBin {
         } else {
             osg::PagedLOD* pagedLOD = new osg::PagedLOD;
             pagedLOD->setCenterMode(osg::PagedLOD::USE_BOUNDING_SPHERE_CENTER);
-            pagedLOD->setName("pagedObjectLOD");
+            std::string name = string("pagedObjectLOD ").append(bucket.gen_index_str());
+            pagedLOD->setName(name);
 
             // This should be visible in any case.
             // If this is replaced by some lower level of detail, the parent LOD node handles this.
             pagedLOD->addChild(terrainGroup, 0, std::numeric_limits<float>::max());
+            pagedLOD->setMinimumExpiryTime(0, pagedLODExpiry);
 
             // we just need to know about the read file callback that itself holds the data
             osg::ref_ptr<DelayLoadReadFileCallback> readFileCallback = new DelayLoadReadFileCallback;
@@ -658,10 +675,11 @@ struct ReaderWriterSTG::_ModelBin {
 
             // Objects may end up displayed up to 2x the object range.
             pagedLOD->setRange(pagedLOD->getNumChildren(), 0, 2.0 * _object_range_rough);
+            pagedLOD->setMinimumExpiryTime(pagedLOD->getNumChildren(), pagedLODExpiry);
             pagedLOD->setRadius(SG_TILE_RADIUS);
-            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile PagedLOD Center: " << pagedLOD->getCenter().x() << "," << pagedLOD->getCenter().y() << "," << pagedLOD->getCenter().z() );
-            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile PagedLOD Range: " << (2.0 * _object_range_rough));
-            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile PagedLOD Radius: " << SG_TILE_RADIUS);
+            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile " << bucket.gen_index_str() << " PagedLOD Center: " << pagedLOD->getCenter().x() << "," << pagedLOD->getCenter().y() << "," << pagedLOD->getCenter().z() );
+            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile " << bucket.gen_index_str() << " PagedLOD Range: " << (2.0 * _object_range_rough));
+            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile " << bucket.gen_index_str() << " PagedLOD Radius: " << SG_TILE_RADIUS);
             return pagedLOD;
         }
     }
@@ -695,6 +713,7 @@ ReaderWriterSTG::readNode(const std::string& fileName, const osgDB::Options* opt
 {
     _ModelBin modelBin;
     SGBucket bucket(bucketIndexFromFileName(fileName));
+    simgear::ErrorReportContext ec("terrain-bucket", bucket.gen_index_str());
 
     // We treat 123.stg different than ./123.stg.
     // The difference is that ./123.stg as well as any absolute path
@@ -702,6 +721,7 @@ ReaderWriterSTG::readNode(const std::string& fileName, const osgDB::Options* opt
     // In contrast 123.stg uses the search paths to load a set of stg
     // files spread across the scenery directories.
     if (osgDB::getSimpleFileName(fileName) != fileName) {
+        simgear::ErrorReportContext ec("terrain-stg", fileName);
         if (!modelBin.read(fileName, options))
             return ReadResult::FILE_NOT_FOUND;
     }
@@ -711,11 +731,10 @@ ReaderWriterSTG::readNode(const std::string& fileName, const osgDB::Options* opt
         return ReadResult::FILE_NOT_FOUND;
     }
 
-    osg::ref_ptr<SGReaderWriterOptions> sgOpts(SGReaderWriterOptions::copyOrCreate(options));
-    if (sgOpts->getSceneryPathSuffixes().empty()) {
+    const auto sgOpts = dynamic_cast<const SGReaderWriterOptions*>(options);
+    if (!sgOpts || sgOpts->getSceneryPathSuffixes().empty()) {
         SG_LOG(SG_TERRAIN, SG_ALERT, "Loading tile " << fileName << ", no scenery path suffixes were configured so giving up");
         return ReadResult::FILE_NOT_FOUND;
-
     }
 
     SG_LOG(SG_TERRAIN, SG_INFO, "Loading tile " << fileName);
@@ -740,7 +759,8 @@ ReaderWriterSTG::readNode(const std::string& fileName, const osgDB::Options* opt
         }
 
         for (auto suffix : sgOpts->getSceneryPathSuffixes()) {
-            SGPath p = base / suffix / basePath / fileName;
+            const auto p = base / suffix / basePath / fileName;
+            simgear::ErrorReportContext ec("terrain-stg", p.utf8Str());
             modelBin.read(p, options);
         }
     }

simgear/scene/tgdb/SGBuildingBin.cxx

@@ -441,6 +441,10 @@ typedef QuadTreeBuilder<LOD*, SGBuildingBin::BuildingInstance, MakeBuildingLeaf,
       if (hash_pos != std::string::npos)
           line.resize(hash_pos);
 
+      if (line.empty()) {
+          continue; // skip blank lines
+      }
+
       // and process further
       std::stringstream in(line);
 

simgear/scene/tgdb/SGNodeTriangles.hxx

@@ -267,12 +267,12 @@ public:
                             SGVec2f texCoord = a*t0 + b*t1 + c*t2;
                     
                             // Check this random point against the object mask
-                            // red channel.
+                            // blue channel.
                             osg::Image* img = object_mask->getImage();            
                             unsigned int x = (int) (img->s() * texCoord.x()) % img->s();
                             unsigned int y = (int) (img->t() * texCoord.y()) % img->t();
                     
-                            if (mt_rand(&seed) < img->getColor(x, y).r()) {                
+                            if (mt_rand(&seed) < img->getColor(x, y).b()) {
                                 points.push_back(randomPoint);        
                             }                    
                         } else {      

simgear/scene/tgdb/SGOceanTile.cxx

@@ -41,6 +41,7 @@
 #include <simgear/scene/material/EffectGeode.hxx>
 #include <simgear/scene/material/mat.hxx>
 #include <simgear/scene/material/matlib.hxx>
+#include <simgear/scene/model/BoundingVolumeBuildVisitor.hxx>
 #include <simgear/scene/util/OsgMath.hxx>
 #include <simgear/scene/util/VectorArrayAdapter.hxx>
 #include <simgear/scene/util/SGNodeMasks.hxx>
@@ -340,5 +341,10 @@ osg::Node* SGOceanTile(const SGBucket& b, SGMaterialLib *matlib, int latPoints,
     transform->addChild(geode);
     transform->setNodeMask( ~(simgear::CASTSHADOW_BIT | simgear::MODELLIGHT_BIT) );
 
+    // Create a BVH at this point.  This is normally provided by the file loader, but as we create the 
+    // geometry programmatically, no file loader is involved.
+    BoundingVolumeBuildVisitor bvhBuilder(false);
+    transform->accept(bvhBuilder);
+    
     return transform;
 }

simgear/scene/tgdb/SGReaderWriterBTG.cxx

@@ -22,6 +22,7 @@
 #include <osgDB/FileNameUtils>
 #include <osgDB/Registry>
 
+#include <simgear/debug/ErrorReportingCallback.hxx>
 #include <simgear/scene/model/ModelRegistry.hxx>
 #include <simgear/scene/util/SGReaderWriterOptions.hxx>
 #include <simgear/structure/exception.hxx>
@@ -62,16 +63,22 @@ SGReaderWriterBTG::readNode(const std::string& fileName,
     const SGReaderWriterOptions* sgOptions;
     sgOptions = dynamic_cast<const SGReaderWriterOptions*>(options);
     osg::Node* result = NULL;
+    simgear::ErrorReportContext ec{"btg", fileName};
     try {
         result = SGLoadBTG(fileName, sgOptions);
         if (!result)
             return ReadResult::FILE_NOT_HANDLED;
     } catch (sg_exception& e) {
-        SG_LOG(SG_IO, SG_WARN, "error reading:" << fileName << ":" <<
-            e.getFormattedMessage());
+        simgear::reportFailure(simgear::LoadFailure::BadData, simgear::ErrorCode::BTGLoad,
+                               "Failed to load BTG file:" + e.getFormattedMessage(),
+                               e.getLocation());
+        return ReadResult::ERROR_IN_READING_FILE;
+    } catch (std::bad_alloc&) {
+        simgear::reportFailure(simgear::LoadFailure::OutOfMemory, simgear::ErrorCode::BTGLoad,
+                               "Out of memory loading BTG:" + fileName, sg_location{fileName});
         return ReadResult::ERROR_IN_READING_FILE;
     }
-    
+
     return result;
 }
 

simgear/scene/tgdb/SGTexturedTriangleBin.hxx

@@ -192,12 +192,12 @@ public:
           SGVec2f texCoord = a*t0 + b*t1 + c*t2;
           
           // Check this random point against the object mask
-          // red channel.
+          // blue channel.
           osg::Image* img = object_mask->getImage();            
           unsigned int x = (int) (img->s() * texCoord.x()) % img->s();
           unsigned int y = (int) (img->t() * texCoord.y()) % img->t();
           
-          if (mt_rand(&seed) < img->getColor(x, y).r()) {                
+          if (mt_rand(&seed) < img->getColor(x, y).b()) {
             points.push_back(randomPoint);        
           }                    
         } else {      

simgear/scene/tgdb/SGTileDetailsCallback.hxx

@@ -37,6 +37,7 @@
 #include <simgear/scene/util/SGReaderWriterOptions.hxx>
 #include <simgear/scene/util/OptionsReadFileCallback.hxx>
 #include <simgear/scene/util/SGNodeMasks.hxx>
+#include <simgear/debug/ErrorReportingCallback.hxx>
 
 #include "SGNodeTriangles.hxx"
 #include "GroundLightManager.hxx"
@@ -55,9 +56,11 @@ typedef std::list<SGDirectionalLightBin> SGDirectionalLightListBin;
 #define SG_SIMPLIFIER_RATIO         (0.001)
 #define SG_SIMPLIFIER_MAX_LENGTH    (1000.0)
 #define SG_SIMPLIFIER_MAX_ERROR     (2000.0)
-#define SG_TILE_MIN_EXPIRY          (180.0)
+
 using namespace simgear;
 
+using ReadResult = osgDB::ReaderWriter::ReadResult;
+
 // QuadTreeBuilder is used by Random Objects Generator
 typedef std::pair<osg::Node*, int> ModelLOD;
 struct MakeQuadLeaf {
@@ -97,47 +100,59 @@ public:
         SG_LOG( SG_TERRAIN, SG_DEBUG, "SGTileDetailsCallback::~SGTileDetailsCallback() num cbs left " << num_tdcb  );
     }
     
-    virtual osgDB::ReaderWriter::ReadResult readNode(
-        const std::string&, const osgDB::Options*)
+    ReadResult readNode(const std::string&, const osgDB::Options*) override
     {
-        SGMaterialLibPtr matlib;
-        osg::ref_ptr<SGMaterialCache> matcache; 
-        
-        osg::ref_ptr<osg::Group> group = new osg::Group;
-        group->setDataVariance(osg::Object::STATIC);
+        osg::ref_ptr<osg::Group> group;
+        simgear::ErrorReportContext ec{"btg", _path};
 
-        // generate textured triangle list
-        std::vector<SGTriangleInfo> matTris;
-        GetNodeTriangles nodeTris(_gbs_center, &matTris);
-        _rootNode->accept( nodeTris );
+        try {
+            group = new osg::Group;
+            SGMaterialLibPtr matlib;
+            osg::ref_ptr<SGMaterialCache> matcache;             
+            group->setDataVariance(osg::Object::STATIC);
 
-        // build matcache
-        matlib = _options->getMaterialLib();
-        if (matlib) {
-            SGGeod geodPos = SGGeod::fromCart(_gbs_center);            
-            matcache = matlib->generateMatCache(geodPos);
-        }
-        
-#if 0
-        // TEST : See if we can regenerate landclass shapes from node
-        for ( unsigned int i=0; i<matTris.size(); i++ ) {
-            matTris[i].dumpBorder(_gbs_center);
-        }
-#endif
+            // generate textured triangle list
+            std::vector<SGTriangleInfo> matTris;
+            GetNodeTriangles nodeTris(_gbs_center, &matTris);
+            _rootNode->accept( nodeTris );
 
-        osg::Node* node = loadTerrain();
-        if (node) {
-            group->addChild(node);
-        }
+            // build matcache
+            matlib = _options->getMaterialLib();
+            if (matlib) {
+                SGGeod geodPos = SGGeod::fromCart(_gbs_center);            
+                matcache = matlib->generateMatCache(geodPos);
+            }
+            
+    #if 0
+            // TEST : See if we can regenerate landclass shapes from node
+            for ( unsigned int i=0; i<matTris.size(); i++ ) {
+                matTris[i].dumpBorder(_gbs_center);
+            }
+    #endif
 
-        osg::LOD* lightLOD = generateLightingTileObjects(matTris, matcache);
-        if (lightLOD) {
-            group->addChild(lightLOD);
-        }
+            osg::Node* node = loadTerrain();
+            if (node) {
+                group->addChild(node);
+            }
 
-        osg::LOD* objectLOD = generateRandomTileObjects(matTris, matcache);
-        if (objectLOD) {
-            group->addChild(objectLOD);
+            osg::LOD* lightLOD = generateLightingTileObjects(matTris, matcache);
+            if (lightLOD) {
+                group->addChild(lightLOD);
+            }
+
+            osg::LOD* objectLOD = generateRandomTileObjects(matTris, matcache);
+            if (objectLOD) {
+                group->addChild(objectLOD);
+            }
+        } catch (sg_exception& sge) {
+            simgear::reportFailure(simgear::LoadFailure::BadData, simgear::ErrorCode::BTGLoad,
+                                "Failed to load BTG file:" + sge.getFormattedMessage(),
+                                sge.getLocation());
+            return ReadResult::ERROR_IN_READING_FILE;
+        } catch (std::bad_alloc &e) {
+            simgear::reportFailure(simgear::LoadFailure::OutOfMemory, simgear::ErrorCode::BTGLoad,
+                                "Out of memory loading tile details", sg_location{_path});
+            return ReadResult::INSUFFICIENT_MEMORY_TO_LOAD;
         }
         
         return group.release();

simgear/scene/tsync/terrasync.cxx

@@ -109,13 +109,13 @@ bool hasWhitespace(string path)
 class SyncItem
 {
 public:
-    enum Type
-    {
-        Stop = 0,   ///< special item indicating to stop the SVNThread
+    enum Type {
+        Stop = 0, ///< special item indicating to stop the SVNThread
         Tile,
         AirportData,
         SharedModels,
-        AIData
+        AIData,
+        OSMTile ///< OSm2City per-Tile data
     };
 
     enum Status
@@ -165,13 +165,16 @@ public:
     SGTimeStamp stamp;
     bool busy = false; ///< is the slot working or idle
     unsigned int pendingKBytes = 0;
+    unsigned int pendingExtractKBytes = 0;
     unsigned int nextWarnTimeout = 0;
 };
 
 static const int SYNC_SLOT_TILES = 0; ///< Terrain and Objects sync
 static const int SYNC_SLOT_SHARED_DATA = 1; /// shared Models and Airport data
 static const int SYNC_SLOT_AI_DATA = 2; /// AI traffic and models
-static const unsigned int NUM_SYNC_SLOTS = 3;
+static const int SYNC_SLOT_OSM_TILE_DATA = 3;
+
+static const unsigned int NUM_SYNC_SLOTS = 4;
 
 /**
  * @brief translate a sync item type into one of the available slots.
@@ -186,7 +189,8 @@ static unsigned int syncSlotForType(SyncItem::Type ty)
         return SYNC_SLOT_SHARED_DATA;
     case SyncItem::AIData:
         return SYNC_SLOT_AI_DATA;
-
+    case SyncItem::OSMTile:
+        return SYNC_SLOT_OSM_TILE_DATA;
     default:
         return SYNC_SLOT_SHARED_DATA;
     }
@@ -194,18 +198,18 @@ static unsigned int syncSlotForType(SyncItem::Type ty)
 
 struct TerrasyncThreadState
 {
-    TerrasyncThreadState() :
-        _busy(false),
-        _stalled(false),
-        _hasServer(false),
-        _fail_count(0),
-        _updated_tile_count(0),
-        _success_count(0),
-        _consecutive_errors(0),
-        _cache_hits(0),
-        _transfer_rate(0),
-        _total_kb_downloaded(0),
-        _totalKbPending(0)
+    TerrasyncThreadState() : _busy(false),
+                             _stalled(false),
+                             _hasServer(false),
+                             _fail_count(0),
+                             _updated_tile_count(0),
+                             _success_count(0),
+                             _consecutive_errors(0),
+                             _cache_hits(0),
+                             _transfer_rate(0),
+                             _total_kb_downloaded(0),
+                             _totalKbPending(0),
+                             _extractTotalKbPending(0)
     {}
 
     bool _busy;
@@ -220,6 +224,7 @@ struct TerrasyncThreadState
     // kbytes, not bytes, because bytes might overflow 2^31
     int _total_kb_downloaded;
     unsigned int _totalKbPending;
+    unsigned int _extractTotalKbPending;
 };
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -275,9 +280,10 @@ public:
 
    SyncItem getNewTile() { return _freshTiles.pop_front();}
 
-   void   setHTTPServer(const std::string& server)
+   void setHTTPServer(const std::string& server, const std::string& osmServer)
    {
       _httpServer = stripPath(server);
+      _osmCityServer = stripPath(osmServer);
       _isAutomaticServer = (server == "automatic");
    }
 
@@ -296,6 +302,11 @@ public:
      _sceneryVersion = simgear::strutils::strip(sceneryVersion);
    }
 
+   void setOSMCityVersion(const std::string& osmCityVersion)
+   {
+       _osmCityService = osmCityVersion;
+   }
+
    void   setLocalDir(string dir)           { _local_dir    = stripPath(dir);}
    string getLocalDir()                     { return _local_dir;}
 
@@ -322,10 +333,12 @@ public:
     void setCachePath(const SGPath &p) { _persistentCachePath = p; }
 
   private:
-    void incrementCacheHits()
-    {
-        std::lock_guard<std::mutex> g(_stateLock);
-        _state._cache_hits++;
+      std::string dnsSelectServerForService(const std::string& service);
+
+      void incrementCacheHits()
+      {
+          std::lock_guard<std::mutex> g(_stateLock);
+          _state._cache_hits++;
     }
 
    virtual void run();
@@ -334,9 +347,9 @@ public:
     void runInternal();
     void updateSyncSlot(SyncSlot& slot);
 
-    void beginSyncAirports(SyncSlot& slot);
-    void beginSyncTile(SyncSlot& slot);
-    void beginNormalSync(SyncSlot& slot);
+    bool beginSyncAirports(SyncSlot& slot);
+    bool beginSyncTile(SyncSlot& slot);
+    bool beginNormalSync(SyncSlot& slot);
 
     void drainWaitingTiles();
 
@@ -363,6 +376,9 @@ public:
     string _local_dir;
     SGPath _persistentCachePath;
     string _httpServer;
+    string _osmCityServer;
+    string _osmCityService = "o2c";
+
     bool _isAutomaticServer;
     SGPath _installRoot;
     string _sceneryVersion;
@@ -474,8 +490,19 @@ bool SGTerraSync::WorkerThread::findServer()
 {
     if ( false == _isAutomaticServer ) return true;
 
+    _httpServer = dnsSelectServerForService(MakeQService(_protocol, _sceneryVersion));
+
+    if (!_osmCityService.empty()) {
+        _osmCityServer = dnsSelectServerForService(_osmCityService);
+    }
+
+    return !_httpServer.empty();
+}
+
+std::string SGTerraSync::WorkerThread::dnsSelectServerForService(const std::string& service)
+{
     DNS::NAPTRRequest * naptrRequest = new DNS::NAPTRRequest(_dnsdn);
-    naptrRequest->qservice = MakeQService(_protocol, _sceneryVersion);
+    naptrRequest->qservice = service;
 
     naptrRequest->qflags = "U";
     DNS::Request_ptr r(naptrRequest);
@@ -483,14 +510,15 @@ bool SGTerraSync::WorkerThread::findServer()
     DNS::Client dnsClient;
     dnsClient.makeRequest(r);
     SG_LOG(SG_TERRASYNC,SG_DEBUG,"DNS NAPTR query for '" << _dnsdn << "' '" << naptrRequest->qservice << "'" );
-    while( !r->isComplete() && !r->isTimeout() )
-      dnsClient.update(0);
+    while (!r->isComplete() && !r->isTimeout()) {
+        dnsClient.update(0);
+    }
 
     if( naptrRequest->entries.empty() ) {
         SG_LOG(SG_TERRASYNC, SG_ALERT, "Warning: no DNS entry found for '" << _dnsdn << "' '" << naptrRequest->qservice << "'" );
-        _httpServer = "";
-        return false;
+        return {};
     }
+
     // walk through responses, they are ordered by 1. order and 2. preference
     // For now, only take entries with lowest order
     // TODO: try all available servers in the order given by preferenc and order
@@ -498,38 +526,33 @@ bool SGTerraSync::WorkerThread::findServer()
 
     // get all servers with this order and the same (for now only lowest preference)
     DNS::NAPTRRequest::NAPTR_list availableServers;
-    for( DNS::NAPTRRequest::NAPTR_list::const_iterator it = naptrRequest->entries.begin();
-         it != naptrRequest->entries.end();
-         ++it ) {
-
-        if( (*it)->order != order )
+    for (const auto& entry : naptrRequest->entries) {
+        if (entry->order != order)
             continue;
 
-        string regex = (*it)->regexp;
-        if( false == simgear::strutils::starts_with( (*it)->regexp, "!^.*$!" ) ) {
-            SG_LOG(SG_TERRASYNC,SG_WARN, "ignoring unsupported regexp: " << (*it)->regexp );
+        const string regex = entry->regexp;
+        if (false == simgear::strutils::starts_with(regex, "!^.*$!")) {
+            SG_LOG(SG_TERRASYNC, SG_WARN, "ignoring unsupported regexp: " << regex);
             continue;
         }
 
-        if( false == simgear::strutils::ends_with( (*it)->regexp, "!" ) ) {
-            SG_LOG(SG_TERRASYNC,SG_WARN, "ignoring unsupported regexp: " << (*it)->regexp );
+        if (false == simgear::strutils::ends_with(regex, "!")) {
+            SG_LOG(SG_TERRASYNC, SG_WARN, "ignoring unsupported regexp: " << regex);
             continue;
         }
 
         // always use first entry
-        if( availableServers.empty() || (*it)->preference == availableServers[0]->preference) {
-            SG_LOG(SG_TERRASYNC,SG_DEBUG, "available server regexp: " << (*it)->regexp );
-            availableServers.push_back( *it );
+        if (availableServers.empty() || entry->preference == availableServers[0]->preference) {
+            SG_LOG(SG_TERRASYNC, SG_DEBUG, "available server regexp: " << regex);
+            availableServers.push_back(entry);
         }
     }
 
     // now pick a random entry from the available servers
-    DNS::NAPTRRequest::NAPTR_list::size_type idx = sg_random() * availableServers.size();
-    _httpServer = availableServers[idx]->regexp;
-    _httpServer = _httpServer.substr( 6, _httpServer.length()-7 ); // strip search pattern and separators
-
-    SG_LOG(SG_TERRASYNC,SG_INFO, "picking entry # " << idx << ", server is " << _httpServer );
-    return true;
+    auto idx = static_cast<int>(sg_random() * availableServers.size());
+    const auto server = availableServers.at(idx)->regexp;
+    SG_LOG(SG_TERRASYNC, SG_INFO, "picking entry # " << idx << ", server is " << server.substr(6, server.length() - 7););
+    return server.substr(6, server.length() - 7);
 }
 
 void SGTerraSync::WorkerThread::run()
@@ -562,6 +585,7 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
 #endif
             // convert bytes to kbytes here
             slot.pendingKBytes = (slot.repository->bytesToDownload() >> 10);
+            slot.pendingExtractKBytes = (slot.repository->bytesToExtract() >> 10);
             return; // easy, still working
         }
 
@@ -569,9 +593,21 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
         HTTPRepository::ResultCode res = slot.repository->failure();
 
         if (res == HTTPRepository::REPO_ERROR_NOT_FOUND) {
+            // not founds should never happen any more (unless the server-
+            // side data is incorrect), since we now check top-down that
+            // a 1x1 dir exists or not.
             notFound(slot.currentItem);
         } else if (res != HTTPRepository::REPO_NO_ERROR) {
             fail(slot.currentItem);
+
+            // in case the Airports_archive download fails, create the
+            // directory, so that next sync, we do a manual sync
+            if ((slot.currentItem._type == SyncItem::AirportData) && slot.isNewDirectory) {
+                SG_LOG(SG_TERRASYNC, SG_ALERT, "Failed to download Airports_archive, will download discrete files next time");
+                simgear::Dir d(_local_dir + "/Airports");
+                d.create(0755);
+                _completedTiles.erase(slot.currentItem._dir);
+            }
         } else {
             updated(slot.currentItem, slot.isNewDirectory);
             SG_LOG(SG_TERRASYNC, SG_DEBUG, "sync of " << slot.repository->baseUrl() << " finished ("
@@ -582,6 +618,7 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
         slot.busy = false;
         slot.repository.reset();
         slot.pendingKBytes = 0;
+        slot.pendingExtractKBytes = 0;
         slot.currentItem = {};
     }
 
@@ -595,12 +632,23 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
         slot.isNewDirectory = !path.exists();
         const auto type = slot.currentItem._type;
 
+        bool ok = false;
         if (type == SyncItem::AirportData) {
-            beginSyncAirports(slot);
+            ok = beginSyncAirports(slot);
+        } else if (type == SyncItem::OSMTile) {
+            ok = beginSyncTile(slot);
         } else if (type == SyncItem::Tile) {
-            beginSyncTile(slot);
+            ok = beginSyncTile(slot);
         } else {
-            beginNormalSync(slot);
+            ok = beginNormalSync(slot);
+        }
+
+        if (!ok) {
+            SG_LOG(SG_TERRASYNC, SG_INFO, "sync of " << slot.currentItem._dir << " failed to start");
+            fail(slot.currentItem);
+            slot.busy = false;
+            slot.repository.reset();
+            return;
         }
 
         try {
@@ -617,17 +665,17 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
         slot.nextWarnTimeout = 30 * 1000;
         slot.stamp.stamp();
         slot.busy = true;
-        slot.pendingKBytes = slot.repository->bytesToDownload();
+        slot.pendingKBytes = slot.repository->bytesToDownload() >> 10;
+        slot.pendingExtractKBytes = slot.repository->bytesToExtract() >> 10;
 
-        SG_LOG(SG_TERRASYNC, SG_INFO, "sync of " << slot.repository->baseUrl() << " started, queue size is " << slot.queue.size());
+        SG_LOG(SG_TERRASYNC, SG_INFO, "sync of " << slot.repository->baseUrl() << ":" << slot.currentItem._dir << " started, queue size is " << slot.queue.size());
     }
 }
 
-void SGTerraSync::WorkerThread::beginSyncAirports(SyncSlot& slot)
+bool SGTerraSync::WorkerThread::beginSyncAirports(SyncSlot& slot)
 {
     if (!slot.isNewDirectory) {
-        beginNormalSync(slot);
-        return;
+        return beginNormalSync(slot);
     }
 
     SG_LOG(SG_TERRASYNC, SG_INFO, "doing Airports download via tarball");
@@ -647,9 +695,10 @@ void SGTerraSync::WorkerThread::beginSyncAirports(SyncSlot& slot)
     };
 
     slot.repository->setFilter(f);
+    return true;
 }
 
-void SGTerraSync::WorkerThread::beginSyncTile(SyncSlot& slot)
+bool SGTerraSync::WorkerThread::beginSyncTile(SyncSlot& slot)
 {
     // avoid 404 requests by doing a sync which excludes all paths
     // except our tile path. In the case of a missing 1x1 tile, we will
@@ -658,8 +707,7 @@ void SGTerraSync::WorkerThread::beginSyncTile(SyncSlot& slot)
     auto comps = strutils::split(slot.currentItem._dir, "/");
     if (comps.size() != 3) {
         SG_LOG(SG_TERRASYNC, SG_ALERT, "Bad tile path:" << slot.currentItem._dir);
-        beginNormalSync(slot);
-        return;
+        return false;
     }
 
     const auto tileCategory = comps.front();
@@ -668,7 +716,17 @@ void SGTerraSync::WorkerThread::beginSyncTile(SyncSlot& slot)
 
     const auto path = SGPath::fromUtf8(_local_dir) / tileCategory;
     slot.repository.reset(new HTTPRepository(path, &_http));
-    slot.repository->setBaseUrl(_httpServer + "/" + tileCategory);
+
+    if (slot.currentItem._type == SyncItem::OSMTile) {
+        if (_osmCityServer.empty()) {
+            SG_LOG(SG_TERRASYNC, SG_WARN, "No OSM2City server defined for:" << slot.currentItem._dir);
+            return false;
+        }
+
+        slot.repository->setBaseUrl(_osmCityServer + "/" + tileCategory);
+    } else {
+        slot.repository->setBaseUrl(_httpServer + "/" + tileCategory);
+    }
 
     if (_installRoot.exists()) {
       SGPath p = _installRoot / tileCategory;
@@ -676,6 +734,7 @@ void SGTerraSync::WorkerThread::beginSyncTile(SyncSlot& slot)
     }
 
     const auto dirPrefix = tenByTenDir + "/" + oneByOneDir;
+    using simgear::strutils::starts_with;
 
     // filter callback to *only* sync the 1x1 dir we want, if it exists
     // if doesn't, we'll simply stop, which is what we want
@@ -684,7 +743,10 @@ void SGTerraSync::WorkerThread::beginSyncTile(SyncSlot& slot)
         if (item.directory.empty()) {
             return item.filename == tenByTenDir;
         } else if (item.directory == tenByTenDir) {
-            return item.filename == oneByOneDir;
+            // allow 10x10 dir to contain 1x1_dir.tgz/.zip and still be accepted
+            // this does mean we'd also download oneByOneDir_foobar but that
+            // doesn't seem unreasonable either
+            return starts_with(item.filename, oneByOneDir);
         }
 
         // allow arbitrary children below dirPrefix, including sub-dirs
@@ -697,9 +759,10 @@ void SGTerraSync::WorkerThread::beginSyncTile(SyncSlot& slot)
     };
 
     slot.repository->setFilter(f);
+    return true;
 }
 
-void SGTerraSync::WorkerThread::beginNormalSync(SyncSlot& slot)
+bool SGTerraSync::WorkerThread::beginNormalSync(SyncSlot& slot)
 {
     SGPath path(_local_dir);
     path.append(slot.currentItem._dir);
@@ -711,6 +774,8 @@ void SGTerraSync::WorkerThread::beginNormalSync(SyncSlot& slot)
       p.append(slot.currentItem._dir);
       slot.repository->setInstalledCopyPath(p);
     }
+
+    return true;
 }
 
 void SGTerraSync::WorkerThread::runInternal()
@@ -753,20 +818,24 @@ void SGTerraSync::WorkerThread::runInternal()
 
         bool anySlotBusy = false;
         unsigned int newPendingCount = 0;
+        unsigned int newExtractCount = 0; // how much is left to extract
 
         // update each sync slot in turn
         for (unsigned int slot=0; slot < NUM_SYNC_SLOTS; ++slot) {
             updateSyncSlot(_syncSlots[slot]);
             newPendingCount += _syncSlots[slot].pendingKBytes;
+            newExtractCount += _syncSlots[slot].pendingExtractKBytes;
             anySlotBusy |= _syncSlots[slot].busy;
         }
 
         {
             std::lock_guard<std::mutex> g(_stateLock);
             _state._totalKbPending = newPendingCount; // approximately atomic update
+            _state._extractTotalKbPending = newExtractCount;
             _state._busy = anySlotBusy;
         }
 
+
         if (!anySlotBusy) {
             // wait on the blocking deque here, otherwise we spin
             // the loop very fast, since _http::update with no connections
@@ -802,12 +871,19 @@ void SGTerraSync::WorkerThread::fail(SyncItem failedItem)
 {
     std::lock_guard<std::mutex> g(_stateLock);
     time_t now = time(0);
-    _state._consecutive_errors++;
-    _state._fail_count++;
+
+    if (_osmCityServer.empty() && (failedItem._type == SyncItem::OSMTile)) {
+        // don't count these as errors, otherwise normla sync will keep
+        // being abandoned
+    } else {
+        _state._consecutive_errors++;
+        _state._fail_count++;
+    }
+
     failedItem._status = SyncItem::Failed;
     _freshTiles.push_back(failedItem);
     // not we also end up here for partial syncs
-    SG_LOG(SG_TERRASYNC,SG_INFO,
+    SG_LOG(SG_TERRASYNC, SG_WARN,
            "Failed to sync'" << failedItem._dir << "'");
     _completedTiles[ failedItem._dir ] = now + UpdateInterval::FailedAttempt;
 }
@@ -819,6 +895,8 @@ void SGTerraSync::WorkerThread::notFound(SyncItem item)
     // we don't spam the server with lookups for models that don't
     // exist
 
+    SG_LOG(SG_TERRASYNC, SG_WARN, "Not found for: '" << item._dir << "'");
+
     time_t now = time(0);
     item._status = SyncItem::NotFound;
     _freshTiles.push_back(item);
@@ -1031,8 +1109,11 @@ void SGTerraSync::reinit()
     if (enabled)
     {
         _availableNode->setBoolValue(true);
-        _workerThread->setHTTPServer( _terraRoot->getStringValue("http-server","automatic") );
+        _workerThread->setHTTPServer(
+            _terraRoot->getStringValue("http-server", "automatic"),
+            _terraRoot->getStringValue("osm2city-server", ""));
         _workerThread->setSceneryVersion( _terraRoot->getStringValue("scenery-version","ws20") );
+        _workerThread->setOSMCityVersion(_terraRoot->getStringValue("osm2city-version", "o2c"));
         _workerThread->setProtocol( _terraRoot->getStringValue("protocol","") );
 #if 1
         // leave it hardcoded for now, not sure about the security implications for now
@@ -1043,6 +1124,9 @@ void SGTerraSync::reinit()
 
         SGPath sceneryRoot{_terraRoot->getStringValue("scenery-dir", "")};
         _workerThread->setLocalDir(sceneryRoot.utf8Str());
+        if (sceneryRoot.exists()) {
+            writeWarningFile(sceneryRoot);
+        }
 
         SGPath installPath(_terraRoot->getStringValue("installation-dir"));
         _workerThread->setInstalledDir(installPath);
@@ -1091,6 +1175,7 @@ void SGTerraSync::bind()
     _transferRateBytesSecNode = _terraRoot->getNode("transfer-rate-bytes-sec", true);
     _pendingKbytesNode = _terraRoot->getNode("pending-kbytes", true);
     _downloadedKBtesNode = _terraRoot->getNode("downloaded-kbytes", true);
+    _extractPendingKbytesNode = _terraRoot->getNode("extract-pending-kbytes", true);
     _enabledNode = _terraRoot->getNode("enabled", true);
     _availableNode = _terraRoot->getNode("available", true);
     _maxErrorsNode = _terraRoot->getNode("max-errors", true);
@@ -1116,6 +1201,7 @@ void SGTerraSync::unbind()
     _transferRateBytesSecNode.clear();
     _pendingKbytesNode.clear();
     _downloadedKBtesNode.clear();
+    _extractPendingKbytesNode.clear();
     _enabledNode.clear();
     _availableNode.clear();
     _maxErrorsNode.clear();
@@ -1152,6 +1238,7 @@ void SGTerraSync::update(double)
     _transferRateBytesSecNode->setIntValue(copiedState._transfer_rate);
     _pendingKbytesNode->setIntValue(copiedState._totalKbPending);
     _downloadedKBtesNode->setIntValue(copiedState._total_kb_downloaded);
+    _extractPendingKbytesNode->setIntValue(copiedState._extractTotalKbPending);
 
     _stalledNode->setBoolValue(_workerThread->isStalled());
     _activeNode->setBoolValue(worker_running);
@@ -1201,6 +1288,10 @@ string_list SGTerraSync::getSceneryPathSuffixes() const
     return scenerySuffixes;
 }
 
+bool isOSMSuffix(const std::string& suffix)
+{
+    return (suffix == "Buildings") || (suffix == "Roads") || (suffix == "Pylons") || (suffix == "Details");
+}
 
 void SGTerraSync::syncAreaByPath(const std::string& aPath)
 {
@@ -1213,8 +1304,7 @@ void SGTerraSync::syncAreaByPath(const std::string& aPath)
         if (_workerThread->isDirActive(dir)) {
             continue;
         }
-
-        SyncItem w(dir, SyncItem::Tile);
+        SyncItem w(dir, isOSMSuffix(suffix) ? SyncItem::OSMTile : SyncItem::Tile);
         _workerThread->request( w );
     }
 }
@@ -1233,6 +1323,11 @@ bool SGTerraSync::isTileDirPending(const std::string& sceneryDir) const
     }
 
     for (const auto& suffix : getSceneryPathSuffixes()) {
+        // don't wait on OSM dirs, even if enabled
+        if (isOSMSuffix(suffix)) {
+            continue;
+        }
+
         const auto s = suffix + "/" + sceneryDir;
         if (_workerThread->isDirActive(s)) {
             return true;
@@ -1270,6 +1365,23 @@ void SGTerraSync::reposition()
     // stub, remove
 }
 
+void SGTerraSync::writeWarningFile(const SGPath& sceneryDir)
+{
+    SGPath p = sceneryDir / "TerraSync-WARNING.txt";
+    if (p.exists())
+        return;
+
+    sg_ofstream os(p, std::ios::out | std::ios::trunc);
+    os << "This folder is managed by FlightGear's download system.\n";
+    os << "Any changes you make here or in sub-folders will be overwritten when TerraSync\n";
+    os << "downloads updates.\n";
+    os << "\n";
+    os << "To use custom scenery or data with FlightGear, put it in a differnet location\n";
+    os << "on your computer, then add the location using either the launcher 'Add-ons' page, or by\n";
+    os << "passing '--fg-scenery=<location>' on the command line.";
+    os << endl;
+}
+
 
 // Register the subsystem.
 SGSubsystemMgr::Registrant<SGTerraSync> registrantSGTerraSync(

simgear/scene/tsync/terrasync.hxx

@@ -90,6 +90,9 @@ protected:
 
     class WorkerThread;
 
+private:
+    void writeWarningFile(const SGPath& sceneryDir);
+
 private:
     WorkerThread* _workerThread;
     SGPropertyNode_ptr _terraRoot;
@@ -107,6 +110,7 @@ private:
     SGPropertyNode_ptr _transferRateBytesSecNode;
     SGPropertyNode_ptr _pendingKbytesNode;
     SGPropertyNode_ptr _downloadedKBtesNode;
+    SGPropertyNode_ptr _extractPendingKbytesNode;
     SGPropertyNode_ptr _maxErrorsNode;
 
     // we manually bind+init TerraSync during early startup

simgear/scene/util/CMakeLists.txt

@@ -31,6 +31,7 @@ set(HEADERS
     UpdateOnceCallback.hxx
     VectorArrayAdapter.hxx
     project.hxx
+    SGProgram.hxx
     )
 
 set(SOURCES
@@ -55,6 +56,7 @@ set(SOURCES
     StateAttributeFactory.cxx
     UpdateOnceCallback.cxx
     project.cxx
+    SGProgram.cxx
     )
 
 simgear_scene_component(util scene/util "${SOURCES}" "${HEADERS}")

simgear/scene/util/SGProgram.cxx

@@ -0,0 +1,83 @@
+////
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Library General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+// Boston, MA  02110-1301, USA.
+
+
+#include "simgear_config.h"
+
+#include "SGProgram.hxx"
+
+#include <osg/State>
+#include <sstream>
+
+#include <simgear/debug/ErrorReportingCallback.hxx>
+#include <simgear/debug/logstream.hxx>
+
+SGProgram::SGProgram()
+{
+}
+
+SGProgram::SGProgram(const SGProgram& rhs, const osg::CopyOp& copyop) : osg::Program(rhs, copyop),
+                                                                        _effectFilePath(rhs._effectFilePath)
+{
+}
+
+void SGProgram::setEffectFilePath(const SGPath& p)
+{
+    _effectFilePath = p;
+}
+
+void SGProgram::apply(osg::State& state) const
+{
+    osg::Program::apply(state);
+
+    if (_checkState != NotApplied) {
+        return; // already done
+    }
+
+    auto pcp = state.getLastAppliedProgramObject();
+    if (!pcp) {
+        std::string infoLog;
+        _checkState = FailedToApply;
+        getPCP(state)->getInfoLog(infoLog);
+        if (!infoLog.empty()) {
+            // log all the shader source file names, to help in debugging link errors
+            std::ostringstream os;
+            for (int i = 0; i < getNumShaders(); ++i) {
+                const auto shader = getShader(i);
+                os << "\t" << shader->getFileName() << "\n";
+            }
+
+            simgear::reportFailure(simgear::LoadFailure::BadData, simgear::ErrorCode::LoadEffectsShaders,
+                                   "Shader program errors: " + infoLog +
+                                       "\n\nShader sources:\n" + os.str(),
+                                   _effectFilePath);
+        }
+
+        for (int i = 0; i < getNumShaders(); ++i) {
+            const auto shader = getShader(i);
+            auto pcs = shader->getPCS(state);
+
+            std::string shaderLog;
+            pcs->getInfoLog(shaderLog);
+            if (!shaderLog.empty()) {
+                simgear::reportFailure(simgear::LoadFailure::BadData, simgear::ErrorCode::LoadEffectsShaders,
+                                       "Shader source errors: " + shaderLog, sg_location(shader->getFileName()));
+            }
+        }
+    } else {
+        _checkState = AppliedOk;
+    }
+}

simgear/scene/util/SGProgram.hxx

@@ -0,0 +1,50 @@
+////
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Library General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+// Boston, MA  02110-1301, USA.
+
+#pragma once
+
+#include <osg/Program>
+
+#include <simgear/misc/sg_path.hxx>
+
+/**
+ * @brief wrapper around osg::Program to allow detecting shader/link
+ * errors in the GLSL code at runtime, and reporting of them
+ * 
+ */
+class SGProgram : public osg::Program
+{
+public:
+    SGProgram();
+
+    SGProgram(const SGProgram& rhs, const osg::CopyOp& copyop = osg::CopyOp::SHALLOW_COPY);
+
+    META_StateAttribute(simgear, SGProgram, PROGRAM);
+
+    void apply(osg::State& state) const override;
+
+    enum ErrorCheckState {
+        NotApplied,
+        AppliedOk,
+        FailedToApply
+    };
+
+    void setEffectFilePath(const SGPath& p);
+
+private:
+    SGPath _effectFilePath;
+    mutable ErrorCheckState _checkState = NotApplied;
+};

simgear/scene/util/SGReaderWriterOptions.cxx

@@ -17,9 +17,7 @@
 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 //
 
-#ifdef HAVE_CONFIG_H
-#  include <simgear_config.h>
-#endif
+#include <simgear_config.h>
 
 #include <simgear/scene/util/OsgMath.hxx>
 
@@ -54,4 +52,10 @@ SGReaderWriterOptions::fromPath(const SGPath& path)
     return options;
 }
 
+void SGReaderWriterOptions::addErrorContext(const std::string& key, const std::string& value)
+{
+    _errorContext[key] = value;
 }
+
+
+} // namespace simgear

simgear/scene/util/SGReaderWriterOptions.hxx

@@ -46,6 +46,7 @@ public:
         ORIGIN_EFFECTS,
         ORIGIN_EFFECTS_NORMALIZED,
         ORIGIN_SPLASH_SCREEN,
+        ORIGIN_CANVAS,
     };
 
     //SGReaderWriterOptions* cloneOptions(const osg::CopyOp& copyop = osg::CopyOp::SHALLOW_COPY) const { return static_cast<SGReaderWriterOptions*>(clone(copyop)); }
@@ -75,12 +76,11 @@ public:
         _LoadOriginHint(ORIGIN_MODEL)
     { }
     SGReaderWriterOptions(const SGReaderWriterOptions& options,
-                          const osg::CopyOp& copyop = osg::CopyOp::SHALLOW_COPY) :
-        osgDB::Options(options, copyop),
-        _propertyNode(options._propertyNode),
-        _materialLib(options._materialLib),
+                          const osg::CopyOp& copyop = osg::CopyOp::SHALLOW_COPY) : osgDB::Options(options, copyop),
+                                                                                   _propertyNode(options._propertyNode),
+                                                                                   _materialLib(options._materialLib),
 #ifdef ENABLE_GDAL
-        _dem(options._dem),
+                                                                                   _dem(options._dem),
 #endif
         _load_panel(options._load_panel),
         _model_data(options._model_data),
@@ -88,7 +88,8 @@ public:
         _instantiateMaterialEffects(options._instantiateMaterialEffects),
         _materialName(options._materialName),
         _sceneryPathSuffixes(options._sceneryPathSuffixes),
-        _LoadOriginHint(ORIGIN_MODEL)
+        _LoadOriginHint(ORIGIN_MODEL),
+        _errorContext(options._errorContext)
     { }
 
     META_Object(simgear, SGReaderWriterOptions);
@@ -157,7 +158,16 @@ public:
     // any texture that is used in a shader, as these often have special values
     // encoded into the channels that aren't suitable for conversion.
     void setLoadOriginHint(LoadOriginHint _v) const { _LoadOriginHint = _v; } 
-    LoadOriginHint getLoadOriginHint() const { return _LoadOriginHint; } 
+    LoadOriginHint getLoadOriginHint() const { return _LoadOriginHint; }
+
+    using ErrorContext = std::map<std::string, std::string>;
+
+    void addErrorContext(const std::string& key, const std::string& value);
+
+    ErrorContext getErrorContext() const
+    {
+        return _errorContext;
+    }
 
 protected:
     virtual ~SGReaderWriterOptions();
@@ -178,6 +188,7 @@ private:
     string_list _sceneryPathSuffixes;
     SGGeod _geod;
     mutable LoadOriginHint _LoadOriginHint;
+    ErrorContext _errorContext;
 };
 
 }

simgear/sound/readwav.cxx

@@ -32,6 +32,7 @@
 #include <simgear/debug/logstream.hxx>
 #include <simgear/misc/stdint.hxx>
 #include <simgear/structure/exception.hxx>
+#include <simgear/debug/ErrorReportingCallback.hxx>
 
 #include "sample.hxx"
 
@@ -56,7 +57,7 @@ namespace
     }
   };
   
-  unsigned int formatConstruct(ALint numChannels, ALint bitsPerSample, bool compressed)
+  unsigned int formatConstruct(ALint numChannels, ALint bitsPerSample, bool compressed, const SGPath& path)
   {
     unsigned int rv = 0;
     if (!compressed) {
@@ -67,7 +68,7 @@ namespace
       else {
         char msg[81];
         snprintf(msg, 80, "Unsupported audio format: tracks: %i, bits/sample: %i", numChannels, bitsPerSample);
-        throw sg_exception(msg);
+          throw sg_exception(msg, {}, path, false);
       }
     } else {
       if (numChannels == 1 && bitsPerSample == 4) rv = SG_SAMPLE_ADPCM; 
@@ -75,7 +76,7 @@ namespace
       else {
         char msg[81];
         snprintf(msg, 80, "Unsupported compressed audio format: tracks: %i, bits/sample: %i", numChannels, bitsPerSample);
-        throw sg_exception(msg);
+          throw sg_exception(msg, {}, path, false);
       }
     }
     return rv;
@@ -275,29 +276,29 @@ namespace
     Codec *codec = codecLinear;
 
     if (!wavReadBE(fd, magic))
-      throw sg_io_exception("corrupt or truncated WAV data", b->path);
+        throw sg_io_exception("corrupt or truncated WAV data", b->path, {}, false);
     
     if (magic != WAV_RIFF_4CC) {
-      throw sg_io_exception("not a .wav file", b->path);
+      throw sg_io_exception("not a .wav file", b->path, {}, false);
     }
 
     if (!wavReadLE(fd, chunkLength) || !wavReadBE(fd, magic))
-      throw sg_io_exception("corrupt or truncated WAV data", b->path);
+      throw sg_io_exception("corrupt or truncated WAV data", b->path, {}, false);
 
     if (magic != WAV_WAVE_4CC)      /* "WAVE" */
     {
-        throw sg_io_exception("unrecognized WAV magic", b->path);
+        throw sg_io_exception("unrecognized WAV magic", b->path, {}, false);
     }
 
     while (1) {
         if (!wavReadBE(fd, magic) || !wavReadLE(fd, chunkLength))
-            throw sg_io_exception("corrupt or truncated WAV data", b->path);
+            throw sg_io_exception("corrupt or truncated WAV data", b->path, {}, false);
 
         if (magic == WAV_FORMAT_4CC)  /* "fmt " */
         {
             found_header = true;
             if (chunkLength < 16) {
-              throw sg_io_exception("corrupt or truncated WAV data", b->path);
+              throw sg_io_exception("corrupt or truncated WAV data", b->path, {}, false);
             }
 
             if (!wavReadLE (fd, audioFormat) ||
@@ -307,11 +308,11 @@ namespace
                 !wavReadLE (fd, blockAlign) ||
                 !wavReadLE (fd, bitsPerSample))
             {
-                throw sg_io_exception("corrupt or truncated WAV data", b->path);
+                throw sg_io_exception("corrupt or truncated WAV data", b->path, {}, false);
             }
 
             if (!gzSkip(fd, chunkLength - 16))
-                throw sg_io_exception("corrupt or truncated WAV data", b->path);
+                throw sg_io_exception("corrupt or truncated WAV data", b->path, {}, false);
 
             switch (audioFormat)
               {
@@ -340,33 +341,33 @@ namespace
                 }
                 break;
               default:
-                throw sg_io_exception("unsupported WAV encoding", b->path);
+                throw sg_io_exception("unsupported WAV encoding:" + std::to_string(audioFormat), b->path, {}, false);
               }
               
               b->block_align = blockAlign;
               b->frequency = samplesPerSecond;
-              b->format = formatConstruct(numChannels, bitsPerSample, compressed);
+              b->format = formatConstruct(numChannels, bitsPerSample, compressed, b->path);
         } else if (magic == WAV_DATA_4CC) {
             if (!found_header) {
                 /* ToDo: A bit wrong to check here, fmt chunk could come later... */
-                throw sg_io_exception("corrupt or truncated WAV data", b->path);
+                throw sg_io_exception("corrupt or truncated WAV data", b->path, {}, false);
             }
             
             b->data = malloc(chunkLength);
             b->length = chunkLength;
             size_t read = gzread(fd, b->data, chunkLength);
             if (read != chunkLength) {
-                throw sg_io_exception("insufficent data reading WAV file", b->path);
+                throw sg_io_exception("insufficent data reading WAV file", b->path, {}, false);
             }
             
             break;
         } else {
             if (!gzSkip(fd, chunkLength))
-              throw sg_io_exception("corrupt or truncated WAV data", b->path);
+              throw sg_io_exception("corrupt or truncated WAV data", b->path, {}, false);
         }
 
         if ((chunkLength & 1) && !gzeof(fd) && !gzSkip(fd, 1))
-          throw sg_io_exception("corrupt or truncated WAV data", b->path);
+          throw sg_io_exception("corrupt or truncated WAV data", b->path, {}, false);
       } // of file chunk parser loop
       
       codec(b); // might throw if something really bad occurs
@@ -380,7 +381,7 @@ namespace simgear
 ALvoid* loadWAVFromFile(const SGPath& path, unsigned int& format, ALsizei& size, ALfloat& freqf, unsigned int& block_align)
 {
   if (!path.exists()) {
-    SG_LOG(SG_IO, SG_DEV_ALERT, "loadWAVFromFile: file not found:" << path);
+      simgear::reportFailure(simgear::LoadFailure::NotFound, simgear::ErrorCode::AudioFX, "loadWAVFromFile: not found", path);
     return nullptr;
   }
 
@@ -396,14 +397,15 @@ ALvoid* loadWAVFromFile(const SGPath& path, unsigned int& format, ALsizei& size,
   fd = gzopen(ps.c_str(), "rb");
 #endif
   if (!fd) {
-    SG_LOG(SG_IO, SG_DEV_ALERT, "loadWAVFromFile: unable to open file:" << path);
+      simgear::reportFailure(simgear::LoadFailure::IOError, simgear::ErrorCode::AudioFX, "loadWAVFromFile: unable to open file", path);
     return nullptr;
   }
 
   try {
       loadWavFile(fd, &b);
   } catch (sg_exception& e) {
-      SG_LOG(SG_IO, SG_DEV_ALERT, "loadWAVFromFile:" << e.getFormattedMessage() << "\nfor: " << path);
+      simgear::reportFailure(simgear::LoadFailure::IOError, simgear::ErrorCode::AudioFX, "loadWAVFromFile: unable to read file:"
+                             + e.getFormattedMessage(), e.getLocation());
       return nullptr;
   }
 

simgear/sound/sample.hxx

@@ -343,7 +343,7 @@ public:
      * Schedule this audio sample to stop (or start) playing.
      */
     inline void set_out_of_range(bool oor = true) {
-        _out_of_range = oor; _playing = (!oor && _loop); _changed = true;
+        _out_of_range = oor; _changed = true;
     }
 
     /**

simgear/sound/xmlsound.cxx

@@ -347,7 +347,9 @@ SGXmlSound::update (double dt)
    // else
    //   if a property is defined then test if it's value is FALSE
    //      or if the mode is IN_TRANSIT then
-   //            test whether the current value matches the previous value.
+   //            test whether the current value matches the previous value,
+   // else
+   //   check if out of range.
    if ( 							// Lisp, anyone?
        (_condition && !_condition->test()) ||
        (!_condition && _property &&
@@ -355,7 +357,8 @@ SGXmlSound::update (double dt)
          !curr_value ||
          ( (_mode == SGXmlSound::IN_TRANSIT) && (curr_value == _prev_value) )
          )
-        )
+        ) ||
+        _sample->test_out_of_range()
        )
    {
        if ((_mode != SGXmlSound::IN_TRANSIT) || (_stopping > MAX_TRANSIT_TIME))

simgear/structure/StateMachine.cxx

@@ -441,7 +441,7 @@ void StateMachine::initFromPlist(SGPropertyNode* desc, SGPropertyNode* root)
         std::string nm = stateDesc->getStringValue("name");
 
         if (nm.empty()) {
-          SG_LOG(SG_GENERAL, SG_ALERT, "No name found for state in branch " << path);
+          SG_LOG(SG_GENERAL, SG_DEV_ALERT, "No name found for state in branch " << path);
           throw sg_exception("No name element in state");
         }
 
@@ -464,8 +464,8 @@ void StateMachine::initFromPlist(SGPropertyNode* desc, SGPropertyNode* root)
         std::string target_id = tDesc->getStringValue("target");
 
         if (nm.empty()) {
-          SG_LOG(SG_GENERAL, SG_ALERT, "No name found for transition in branch " << path);
-          throw sg_exception("No name element in transition");
+          SG_LOG(SG_GENERAL, SG_DEV_WARN, "No name found for transition in branch " << path);
+          nm = "transition-to-" + target_id;
         }
 
         if (target_id.empty()) {

simgear/structure/exception.cxx

@@ -19,66 +19,65 @@ static ThrowCallback static_callback;
 // Implementation of sg_location class.
 ////////////////////////////////////////////////////////////////////////
 
-sg_location::sg_location ()
-  : _line(-1),
-    _column(-1),
-    _byte(-1)
+sg_location::sg_location() noexcept
+    : _line(-1),
+      _column(-1),
+      _byte(-1)
 {
     _path[0] = '\0';
 }
 
-sg_location::sg_location (const std::string& path, int line, int column)
-  : _line(line),
-    _column(column),
-    _byte(-1)
+sg_location::sg_location(const std::string& path, int line, int column) noexcept
+    : _line(line),
+      _column(column),
+      _byte(-1)
 {
   setPath(path.c_str());
 }
 
-sg_location::sg_location (const SGPath& path, int line, int column)
-: _line(line),
-_column(column),
-_byte(-1)
+sg_location::sg_location(const SGPath& path, int line, int column) noexcept
+    : _line(line),
+      _column(column),
+      _byte(-1)
 {
     setPath(path.utf8Str().c_str());
 }
 
-sg_location::sg_location (const char* path, int line, int column)
-  : _line(line),
-    _column(column),
-    _byte(-1)
+sg_location::sg_location(const char* path, int line, int column) noexcept
+    : _line(line),
+      _column(column),
+      _byte(-1)
 {
   setPath(path);
 }
 
-void sg_location::setPath(const char *path) {
-  if (path) {
-    strncpy(_path, path, max_path);
-    _path[max_path -1] = '\0';
-  } else {
-    _path[0] = '\0';
-  }
+void sg_location::setPath(const char* path) noexcept
+{
+    if (path) {
+        strncpy(_path, path, max_path);
+        _path[max_path - 1] = '\0';
+    } else {
+        _path[0] = '\0';
+    }
 }
 
-const char *sg_location::getPath() const { return _path; }
+const char* sg_location::getPath() const noexcept { return _path; }
 
-int sg_location::getLine() const { return _line; }
+int sg_location::getLine() const noexcept { return _line; }
 
-int
-sg_location::getColumn () const
+int sg_location::getColumn() const noexcept
 {
   return _column;
 }
 
 
-int
-sg_location::getByte () const
+int sg_location::getByte() const noexcept
 {
   return _byte;
 }
 
 std::string
-sg_location::asString () const
+sg_location::asString() const noexcept
 {
   std::ostringstream out;
   if (_path[0]) {
@@ -97,46 +96,48 @@ sg_location::asString () const
   return out.str();
 }
 
-bool sg_location::isValid() const { return strlen(_path) > 0; }
+bool sg_location::isValid() const noexcept { return strlen(_path) > 0; }
 
 ////////////////////////////////////////////////////////////////////////
 // Implementation of sg_throwable class.
 ////////////////////////////////////////////////////////////////////////
 
-sg_throwable::sg_throwable ()
+sg_throwable::sg_throwable() noexcept
 {
   _message[0] = '\0';
   _origin[0] = '\0';
 }
 
-sg_throwable::sg_throwable(const char *message, const char *origin,
-                           const sg_location &loc) {
-  setMessage(message);
-  setOrigin(origin);
-
-  if (static_callback) {
-    static_callback(_message, _origin, loc);
-  }
-}
-
-sg_throwable::~sg_throwable ()
+sg_throwable::sg_throwable(const char* message, const char* origin,
+                           const sg_location& loc, bool report) noexcept : _location(loc)
 {
+    setMessage(message);
+    setOrigin(origin);
+
+    if (static_callback && report) {
+        static_callback(_message, _origin, loc);
+    }
 }
 
 const char*
-sg_throwable::getMessage () const
+sg_throwable::getMessage() const noexcept
 {
   return _message;
 }
 
-const std::string
-sg_throwable::getFormattedMessage () const
+std::string
+sg_throwable::getFormattedMessage() const noexcept
 {
-  return std::string(getMessage());
+    std::string ret = getMessage();
+    const std::string loc = getLocation().asString();
+    if (loc.length()) {
+        ret += "\n at ";
+        ret += loc;
+    }
+    return ret;
 }
 
-void
-sg_throwable::setMessage (const char* message)
+void sg_throwable::setMessage(const char* message) noexcept
 {
   strncpy(_message, message, MAX_TEXT_LEN);
   _message[MAX_TEXT_LEN - 1] = '\0';
@@ -144,13 +145,12 @@ sg_throwable::setMessage (const char* message)
 }
 
 const char*
-sg_throwable::getOrigin () const
+sg_throwable::getOrigin() const noexcept
 {
   return _origin;
 }
 
-void
-sg_throwable::setOrigin (const char* origin)
+void sg_throwable::setOrigin(const char* origin) noexcept
 {
   if (origin) {
     strncpy(_origin, origin, MAX_TEXT_LEN);
@@ -170,27 +170,27 @@ const char* sg_throwable::what() const noexcept
   }
 }
 
-
+sg_location sg_throwable::getLocation() const noexcept
+{
+    return _location;
+}
+
+void sg_throwable::setLocation(const sg_location& location) noexcept
+{
+    _location = location;
+}
+
 ////////////////////////////////////////////////////////////////////////
 // Implementation of sg_error class.
 ////////////////////////////////////////////////////////////////////////
 
-sg_error::sg_error ()
-  : sg_throwable ()
+sg_error::sg_error(const char* message, const char* origin, bool report) noexcept
+    : sg_throwable(message, origin, {}, report)
 {
 }
 
-sg_error::sg_error (const char* message, const char *origin)
-  : sg_throwable(message, origin)
-{
-}
-
-sg_error::sg_error(const std::string& message, const std::string& origin)
-  : sg_throwable(message.c_str(), origin.c_str())
-{
-}
-
-sg_error::~sg_error ()
+sg_error::sg_error(const std::string& message, const std::string& origin, bool report) noexcept
+    : sg_throwable(message.c_str(), origin.c_str(), {}, report)
 {
 }
 
@@ -198,121 +198,81 @@ sg_error::~sg_error ()
 // Implementation of sg_exception class.
 ////////////////////////////////////////////////////////////////////////
 
-sg_exception::sg_exception ()
-  : sg_throwable ()
-{
-}
+sg_exception::sg_exception(const char* message, const char* origin,
+                           const sg_location& loc, bool report) noexcept
+    : sg_throwable(message, origin, loc, report) {}
 
-sg_exception::sg_exception(const char *message, const char *origin,
-                           const sg_location &loc)
-    : sg_throwable(message, origin, loc) {}
+sg_exception::sg_exception(const std::string& message,
+                           const std::string& origin, const sg_location& loc,
+                           bool report) noexcept
+    : sg_throwable(message.c_str(), origin.c_str(), loc, report) {}
 
-sg_exception::sg_exception(const std::string &message,
-                           const std::string &origin, const sg_location &loc)
-    : sg_throwable(message.c_str(), origin.c_str(), loc) {}
-
-sg_exception::~sg_exception ()
-{
-}
 
 ////////////////////////////////////////////////////////////////////////
 // Implementation of sg_io_exception.
 ////////////////////////////////////////////////////////////////////////
 
-sg_io_exception::sg_io_exception ()
-  : sg_exception()
+sg_io_exception::sg_io_exception(const char* message, const char* origin, bool report)
+    : sg_exception(message, origin, {}, report)
 {
 }
 
-sg_io_exception::sg_io_exception (const char* message, const char* origin)
-  : sg_exception(message, origin)
+sg_io_exception::sg_io_exception(const char* message,
+                                 const sg_location& location,
+                                 const char* origin,
+                                 bool report)
+    : sg_exception(message, origin, location, report) {}
+
+sg_io_exception::sg_io_exception(const std::string& message,
+                                 const std::string& origin,
+                                 bool report)
+    : sg_exception(message, origin, {}, report)
 {
 }
 
-sg_io_exception::sg_io_exception(const char *message,
-                                 const sg_location &location,
-                                 const char *origin)
-    : sg_exception(message, origin, location), _location(location) {}
-
-sg_io_exception::sg_io_exception( const std::string& message,
-                                  const std::string& origin )
-  : sg_exception(message, origin)
-{
-}
-
-sg_io_exception::sg_io_exception(const std::string &message,
-                                 const sg_location &location,
-                                 const std::string &origin)
-    : sg_exception(message, origin, location), _location(location) {}
-
-sg_io_exception::~sg_io_exception ()
-{
-}
-
-const std::string
-sg_io_exception::getFormattedMessage () const
-{
-  std::string ret = getMessage();
-  std::string loc = getLocation().asString();
-  if (loc.length()) {
-    ret += "\n at ";
-    ret += loc;
-  }
-  return ret;
-}
-
-const sg_location &
-sg_io_exception::getLocation () const
-{
-  return _location;
-}
-
-void
-sg_io_exception::setLocation (const sg_location &location)
-{
-  _location = location;
-}
+sg_io_exception::sg_io_exception(const std::string& message,
+                                 const sg_location& location,
+                                 const std::string& origin,
+                                 bool report)
+    : sg_exception(message, origin, location, report) {}
 
 
-
 ////////////////////////////////////////////////////////////////////////
 // Implementation of sg_format_exception.
 ////////////////////////////////////////////////////////////////////////
 
-sg_format_exception::sg_format_exception ()
-  : sg_exception()
+sg_format_exception::sg_format_exception() noexcept
+    : sg_exception()
 {
   _text[0] = '\0';
 }
 
-sg_format_exception::sg_format_exception (const char* message,
-					  const char* text,
-					  const char* origin)
-  : sg_exception(message, origin)
+sg_format_exception::sg_format_exception(const char* message,
+                                         const char* text,
+                                         const char* origin,
+                                         bool report)
+    : sg_exception(message, origin, {}, report)
 {
   setText(text);
 }
 
-sg_format_exception::sg_format_exception( const std::string& message,
-                                          const std::string& text,
-                                          const std::string& origin )
-  : sg_exception(message, origin)
+sg_format_exception::sg_format_exception(const std::string& message,
+                                         const std::string& text,
+                                         const std::string& origin,
+                                         bool report)
+    : sg_exception(message, origin, {}, report)
 {
   setText(text.c_str());
 }
 
-sg_format_exception::~sg_format_exception ()
-{
-}
 
 const char*
-sg_format_exception::getText () const
+sg_format_exception::getText() const noexcept
 {
   return _text;
 }
 
-void
-sg_format_exception::setText (const char* text)
+void sg_format_exception::setText(const char* text) noexcept
 {
   if (text) {
     strncpy(_text, text, MAX_TEXT_LEN);
@@ -328,26 +288,19 @@ sg_format_exception::setText (const char* text)
 // Implementation of sg_range_exception.
 ////////////////////////////////////////////////////////////////////////
 
-sg_range_exception::sg_range_exception ()
-  : sg_exception()
-{
-}
-
-sg_range_exception::sg_range_exception (const char* message,
-					const char* origin)
-  : sg_exception(message, origin)
+sg_range_exception::sg_range_exception(const char* message,
+                                       const char* origin, bool report)
+    : sg_exception(message, origin, {}, report)
 {
 }
 
 sg_range_exception::sg_range_exception(const std::string& message,
-                                       const std::string& origin)
-  : sg_exception(message, origin)
+                                       const std::string& origin,
+                                       bool report)
+    : sg_exception(message, origin, {}, report)
 {
 }
 
-sg_range_exception::~sg_range_exception ()
-{
-}
 
 ////////////////////////////////////////////////////////////////////////
 

simgear/structure/exception.hxx

@@ -28,23 +28,23 @@ class sg_location
 {
 public:
   enum {max_path = 1024};
-  sg_location ();
-  sg_location(const std::string& path, int line = -1, int column = -1);
-  sg_location(const SGPath& path, int line = -1, int column = -1);
-  explicit sg_location(const char* path, int line = -1, int column = -1);
+  sg_location() noexcept;
+  sg_location(const std::string& path, int line = -1, int column = -1) noexcept;
+  sg_location(const SGPath& path, int line = -1, int column = -1) noexcept;
+  explicit sg_location(const char* path, int line = -1, int column = -1) noexcept;
 
   ~sg_location() = default;
 
-  const char *getPath() const;
-  int getLine() const;
-  int getColumn() const;
-  int getByte() const;
+  const char* getPath() const noexcept;
+  int getLine() const noexcept;
+  int getColumn() const noexcept;
+  int getByte() const noexcept;
 
-  std::string asString() const;
-  bool isValid() const;
+  std::string asString() const noexcept;
+  bool isValid() const noexcept;
 
-private:
-  void setPath(const char *p);
+  private:
+  void setPath(const char* p) noexcept;
 
   char _path[max_path];
   int _line;
@@ -60,20 +60,26 @@ class sg_throwable : public std::exception
 {
 public:
   enum {MAX_TEXT_LEN = 1024};
-  sg_throwable ();
-  sg_throwable(const char *message, const char *origin = 0,
-               const sg_location &loc = {});
+  sg_throwable() noexcept;
+  sg_throwable(const char* message, const char* origin = 0,
+               const sg_location& loc = {}, bool report = true) noexcept;
 
-  virtual ~sg_throwable ();
-  virtual const char* getMessage () const;
-  virtual const std::string getFormattedMessage () const;
-  virtual void setMessage (const char* message);
-  virtual const char* getOrigin () const;
-  virtual void setOrigin (const char *origin);
+  virtual ~sg_throwable() noexcept = default;
+
+  virtual const char* getMessage() const noexcept;
+  std::string getFormattedMessage() const noexcept;
+  virtual void setMessage(const char* message) noexcept;
+  virtual const char* getOrigin() const noexcept;
+  virtual void setOrigin(const char* origin) noexcept;
   virtual const char* what() const noexcept;
-private:
+
+  sg_location getLocation() const noexcept;
+  void setLocation(const sg_location& location) noexcept;
+
+  private:
   char _message[MAX_TEXT_LEN];
   char _origin[MAX_TEXT_LEN];
+  sg_location _location;
 };
 
 
@@ -90,10 +96,10 @@ private:
 class sg_error : public sg_throwable
 {
 public:
-  sg_error ();
-  sg_error (const char* message, const char* origin = 0);
-  sg_error(const std::string &message, const std::string &origin = {});
-  virtual ~sg_error ();
+    sg_error() noexcept = default;
+    sg_error(const char* message, const char* origin = 0, bool report = true) noexcept;
+    sg_error(const std::string& message, const std::string& origin = {}, bool report = true) noexcept;
+    virtual ~sg_error() noexcept = default;
 };
 
 
@@ -114,12 +120,12 @@ public:
 class sg_exception : public sg_throwable
 {
 public:
-  sg_exception ();
-  sg_exception(const char *message, const char *origin = 0,
-               const sg_location &loc = {});
-  sg_exception(const std::string &message, const std::string & = {},
-               const sg_location &loc = {});
-  virtual ~sg_exception ();
+    sg_exception() noexcept = default;
+    sg_exception(const char* message, const char* origin = 0,
+                 const sg_location& loc = {}, bool report = true) noexcept;
+    sg_exception(const std::string& message, const std::string& = {},
+                 const sg_location& loc = {}, bool report = true) noexcept;
+    virtual ~sg_exception() noexcept = default;
 };
 
 
@@ -137,20 +143,15 @@ public:
 class sg_io_exception : public sg_exception
 {
 public:
-  sg_io_exception ();
-  sg_io_exception (const char* message, const char* origin = 0);
-  sg_io_exception (const char* message, const sg_location &location,
-		   const char* origin = 0);
-  sg_io_exception (const std::string &message, const std::string &origin = "");
-  sg_io_exception (const std::string &message, const sg_location &location, 
-    const std::string &origin = "");
+    sg_io_exception() noexcept = default;
+    sg_io_exception(const char* message, const char* origin = 0, bool report = true);
+    sg_io_exception(const char* message, const sg_location& location,
+                    const char* origin = 0, bool report = true);
+    sg_io_exception(const std::string& message, const std::string& origin = {}, bool report = true);
+    sg_io_exception(const std::string& message, const sg_location& location,
+                    const std::string& origin = {}, bool report = true);
 
-  virtual ~sg_io_exception ();
-  virtual const std::string getFormattedMessage () const;
-  virtual const sg_location &getLocation () const;
-  virtual void setLocation (const sg_location &location);
-private:
-  sg_location _location;
+    virtual ~sg_io_exception() noexcept = default;
 };
 
 
@@ -168,14 +169,15 @@ private:
 class sg_format_exception : public sg_exception
 {
 public:
-  sg_format_exception ();
-  sg_format_exception (const char* message, const char* text,
-		       const char* origin = 0);
-  sg_format_exception (const std::string& message, const std::string& text,
-		       const std::string& origin = "");
-  virtual ~sg_format_exception ();
-  virtual const char* getText () const;
-  virtual void setText (const char* text);
+    sg_format_exception() noexcept;
+    sg_format_exception(const char* message, const char* text,
+                        const char* origin = 0, bool report = true);
+    sg_format_exception(const std::string& message, const std::string& text,
+                        const std::string& origin = {}, bool report = true);
+
+    const char* getText() const noexcept;
+    void setText(const char* text) noexcept;
+
 private:
   char _text[MAX_TEXT_LEN];
 };
@@ -193,12 +195,14 @@ private:
 class sg_range_exception : public sg_exception
 {
 public:
-  sg_range_exception ();
-  sg_range_exception (const char* message,
-                      const char* origin = 0);
-  sg_range_exception (const std::string& message,
-                      const std::string& origin = "");
-  virtual ~sg_range_exception ();
+    sg_range_exception() noexcept = default;
+    sg_range_exception(const char* message,
+                       const char* origin = 0,
+                       bool report = true);
+    sg_range_exception(const std::string& message,
+                       const std::string& origin = {},
+                       bool report = true);
+    virtual ~sg_range_exception() noexcept = default;
 };
 
 using ThrowCallback = std::function<void(

simgear/structure/subsystem_mgr.cxx

@@ -30,9 +30,10 @@
 #include "subsystem_mgr.hxx"
 #include "commands.hxx"
 
-#include <simgear/props/props.hxx>
-#include <simgear/math/SGMath.hxx>
 #include "SGSmplstat.hxx"
+#include <simgear/debug/ErrorReportingCallback.hxx>
+#include <simgear/math/SGMath.hxx>
+#include <simgear/props/props.hxx>
 
 const int SG_MAX_SUBSYSTEM_EXCEPTIONS = 4;
 const char SUBSYSTEM_NAME_SEPARATOR = '.';
@@ -249,10 +250,15 @@ void
 SGSubsystemGroup::init ()
 {
     assert(_state == State::BIND);
-    forEach([this](SGSubsystem* s){
-        this->notifyWillChange(s, State::INIT);
-        s->init();
-        this->notifyDidChange(s, State::INIT);
+    forEach([this](SGSubsystem* s) {
+        try {
+            this->notifyWillChange(s, State::INIT);
+            s->init();
+            this->notifyDidChange(s, State::INIT);
+        } catch (std::exception& e) {
+            simgear::reportError("Caught exception init-ing subsystem " + s->subsystemId() + "\n\t" + e.what());
+            throw;
+        }
     });
     _state = State::INIT;
 }
@@ -282,7 +288,15 @@ SGSubsystemGroup::incrementalInit()
     const auto m = _members[_initPosition]; 
     SGTimeStamp st;
     st.stamp();
-    const InitStatus memberStatus = m->subsystem->incrementalInit();
+    InitStatus memberStatus;
+
+    try {
+        memberStatus = m->subsystem->incrementalInit();
+    } catch (std::exception& e) {
+        simgear::reportError("Caught exception init-ing subsystem " + m->subsystem->subsystemId() + "\n\t" + e.what());
+        throw;
+    }
+
     m->initTime += st.elapsedMSec();
   
     if (memberStatus == INIT_DONE) {
@@ -350,10 +364,15 @@ SGSubsystemGroup::shutdown ()
 void
 SGSubsystemGroup::bind ()
 {
-    forEach([this](SGSubsystem* s){
-        this->notifyWillChange(s, State::BIND);
-        s->bind();
-        this->notifyDidChange(s, State::BIND);
+    forEach([this](SGSubsystem* s) {
+        try {
+            this->notifyWillChange(s, State::BIND);
+            s->bind();
+            this->notifyDidChange(s, State::BIND);
+        } catch (std::exception& e) {
+            simgear::reportError("Caught exception binding subsystem " + s->subsystemId() + "\n\t" + e.what());
+            throw;
+        }
     });
     _state = State::BIND;
 }
@@ -828,8 +847,18 @@ SGSubsystemGroup::Member::update (double delta_time_sec)
       if (++exceptionCount > SG_MAX_SUBSYSTEM_EXCEPTIONS) {
         SG_LOG(SG_GENERAL, SG_ALERT, "(exceptionCount=" << exceptionCount <<
           ", suspending)");
+        simgear::reportError("suspending subsystem after too many errors:" + name);
         subsystem->suspend();
       }
+    } catch (std::bad_alloc& ba) {
+        // attempting to track down source of these on Sentry.io
+        SG_LOG(SG_GENERAL, SG_ALERT, "caught bad_alloc processing subsystem:" << name);
+        simgear::reportError("caught bad_alloc processing subsystem:" + name);
+
+        if (++exceptionCount > SG_MAX_SUBSYSTEM_EXCEPTIONS) {
+            SG_LOG(SG_GENERAL, SG_ALERT, "(exceptionCount=" << exceptionCount << ", suspending)");
+            subsystem->suspend();
+        }
     }
 }
 

simgear/xml/easyxml.cxx

@@ -244,40 +244,43 @@ readXML (istream &input, XMLVisitor &visitor, const string &path)
 
 				// FIXME: get proper error string from system
     if (!input.good()) {
-      sg_io_exception ex ("Problem reading file",
-			    sg_location(path,
-					XML_GetCurrentLineNumber(parser),
-					XML_GetCurrentColumnNumber(parser)),
-			    "SimGear XML Parser");
-      visitor.setParser(0);
-      XML_ParserFree(parser);
-      throw ex;
+        sg_io_exception ex("Problem reading file",
+                           sg_location(path,
+                                       XML_GetCurrentLineNumber(parser),
+                                       XML_GetCurrentColumnNumber(parser)),
+                           "SimGear XML Parser",
+                           false /* don't report */);
+        visitor.setParser(0);
+        XML_ParserFree(parser);
+        throw ex;
     }
 
     input.read(buf,16384);
     if (!XML_Parse(parser, buf, input.gcount(), false)) {
-      sg_io_exception ex (XML_ErrorString(XML_GetErrorCode(parser)),
-			    sg_location(path,
-					XML_GetCurrentLineNumber(parser),
-					XML_GetCurrentColumnNumber(parser)),
-			    "SimGear XML Parser");
-      visitor.setParser(0);
-      XML_ParserFree(parser);
-      throw ex;
+        sg_io_exception ex(XML_ErrorString(XML_GetErrorCode(parser)),
+                           sg_location(path,
+                                       XML_GetCurrentLineNumber(parser),
+                                       XML_GetCurrentColumnNumber(parser)),
+                           "SimGear XML Parser",
+                           false /* don't report */);
+        visitor.setParser(0);
+        XML_ParserFree(parser);
+        throw ex;
     }
 
   }
 
 				// Verify end of document.
   if (!XML_Parse(parser, buf, 0, true)) {
-    sg_io_exception ex (XML_ErrorString(XML_GetErrorCode(parser)),
-			  sg_location(path,
-				      XML_GetCurrentLineNumber(parser),
-				      XML_GetCurrentColumnNumber(parser)),
-			  "SimGear XML Parser");
-    visitor.setParser(0);
-    XML_ParserFree(parser);
-    throw ex;
+      sg_io_exception ex(XML_ErrorString(XML_GetErrorCode(parser)),
+                         sg_location(path,
+                                     XML_GetCurrentLineNumber(parser),
+                                     XML_GetCurrentColumnNumber(parser)),
+                         "SimGear XML Parser",
+                         false /* don't report */);
+      visitor.setParser(0);
+      XML_ParserFree(parser);
+      throw ex;
   }
 
   visitor.setParser(0);
@@ -300,8 +303,8 @@ readXML (const SGPath &path, XMLVisitor &visitor)
       throw;
     }
   } else {
-    throw sg_io_exception("Failed to open file", sg_location(path),
-			  "SimGear XML Parser");
+      throw sg_io_exception("Failed to open file", sg_location(path),
+                            "SimGear XML Parser", false /* don't report */);
   }
   input.close();
 }
@@ -318,11 +321,12 @@ readXML (const char *buf, const int size, XMLVisitor &visitor)
   visitor.startXML();
 
   if (!XML_Parse(parser, buf, size, false)) {
-      sg_io_exception ex (XML_ErrorString(XML_GetErrorCode(parser)),
-			    sg_location("In-memory XML buffer",
-					XML_GetCurrentLineNumber(parser),
-					XML_GetCurrentColumnNumber(parser)),
-			    "SimGear XML Parser");
+      sg_io_exception ex(XML_ErrorString(XML_GetErrorCode(parser)),
+                         sg_location("In-memory XML buffer",
+                                     XML_GetCurrentLineNumber(parser),
+                                     XML_GetCurrentColumnNumber(parser)),
+                         "SimGear XML Parser",
+                         false /* don't report */);
       XML_ParserFree(parser);
       throw ex;
   }