TerraSync: allow an explicit osm2city server

Fixes an error case where a manual TerraSync server is specified; we would attempt to use an empty string as the OSM2City server, with hilarious consequences. Sentry-Id: FLIGHTGEAR-NCZ
Remove stray include of std::filesystem
2021-03-16 20:14:34 +00:00 · 2021-03-07 10:53:36 +00:00 · 2021-03-04 22:02:11 +00:00 · 2021-03-01 12:11:33 +00:00 · 2021-03-01 09:56:48 +00:00 · 2021-03-01 09:56:48 +00:00
53 changed files with 1776 additions and 905 deletions
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -286,6 +286,7 @@ else()
    find_package(ZLIB 1.2.4 REQUIRED)
 endif()

+find_package(LibLZMA REQUIRED)
 find_package(CURL REQUIRED)

 if (SYSTEM_EXPAT)
--- a/CMakeModules/FindLibLZMA.cmake
+++ b/CMakeModules/FindLibLZMA.cmake
@@ -0,0 +1,124 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+#[=======================================================================[.rst:
+FindLibLZMA
+-----------
+
+Find LZMA compression algorithm headers and library.
+
+
+Imported Targets
+^^^^^^^^^^^^^^^^
+
+This module defines :prop_tgt:`IMPORTED` target ``LibLZMA::LibLZMA``, if
+liblzma has been found.
+
+Result variables
+^^^^^^^^^^^^^^^^
+
+This module will set the following variables in your project:
+
+``LIBLZMA_FOUND``
+  True if liblzma headers and library were found.
+``LIBLZMA_INCLUDE_DIRS``
+  Directory where liblzma headers are located.
+``LIBLZMA_LIBRARIES``
+  Lzma libraries to link against.
+``LIBLZMA_HAS_AUTO_DECODER``
+  True if lzma_auto_decoder() is found (required).
+``LIBLZMA_HAS_EASY_ENCODER``
+  True if lzma_easy_encoder() is found (required).
+``LIBLZMA_HAS_LZMA_PRESET``
+  True if lzma_lzma_preset() is found (required).
+``LIBLZMA_VERSION_MAJOR``
+  The major version of lzma
+``LIBLZMA_VERSION_MINOR``
+  The minor version of lzma
+``LIBLZMA_VERSION_PATCH``
+  The patch version of lzma
+``LIBLZMA_VERSION_STRING``
+  version number as a string (ex: "5.0.3")
+#]=======================================================================]
+
+find_path(LIBLZMA_INCLUDE_DIR lzma.h )
+if(NOT LIBLZMA_LIBRARY)
+  find_library(LIBLZMA_LIBRARY_RELEASE NAMES lzma liblzma NAMES_PER_DIR PATH_SUFFIXES lib)
+  find_library(LIBLZMA_LIBRARY_DEBUG NAMES lzmad liblzmad NAMES_PER_DIR PATH_SUFFIXES lib)
+  include(SelectLibraryConfigurations)
+  select_library_configurations(LIBLZMA)
+else()
+  file(TO_CMAKE_PATH "${LIBLZMA_LIBRARY}" LIBLZMA_LIBRARY)
+endif()
+
+if(LIBLZMA_INCLUDE_DIR AND EXISTS "${LIBLZMA_INCLUDE_DIR}/lzma/version.h")
+    file(STRINGS "${LIBLZMA_INCLUDE_DIR}/lzma/version.h" LIBLZMA_HEADER_CONTENTS REGEX "#define LZMA_VERSION_[A-Z]+ [0-9]+")
+
+    string(REGEX REPLACE ".*#define LZMA_VERSION_MAJOR ([0-9]+).*" "\\1" LIBLZMA_VERSION_MAJOR "${LIBLZMA_HEADER_CONTENTS}")
+    string(REGEX REPLACE ".*#define LZMA_VERSION_MINOR ([0-9]+).*" "\\1" LIBLZMA_VERSION_MINOR "${LIBLZMA_HEADER_CONTENTS}")
+    string(REGEX REPLACE ".*#define LZMA_VERSION_PATCH ([0-9]+).*" "\\1" LIBLZMA_VERSION_PATCH "${LIBLZMA_HEADER_CONTENTS}")
+
+    set(LIBLZMA_VERSION_STRING "${LIBLZMA_VERSION_MAJOR}.${LIBLZMA_VERSION_MINOR}.${LIBLZMA_VERSION_PATCH}")
+    unset(LIBLZMA_HEADER_CONTENTS)
+endif()
+
+# We're using new code known now as XZ, even library still been called LZMA
+# it can be found in http://tukaani.org/xz/
+# Avoid using old codebase
+if (LIBLZMA_LIBRARY)
+  include(CheckLibraryExists)
+  set(CMAKE_REQUIRED_QUIET_SAVE ${CMAKE_REQUIRED_QUIET})
+  set(CMAKE_REQUIRED_QUIET ${LibLZMA_FIND_QUIETLY})
+  if(NOT LIBLZMA_LIBRARY_RELEASE AND NOT LIBLZMA_LIBRARY_DEBUG)
+    set(LIBLZMA_LIBRARY_check ${LIBLZMA_LIBRARY})
+  elseif(LIBLZMA_LIBRARY_RELEASE)
+    set(LIBLZMA_LIBRARY_check ${LIBLZMA_LIBRARY_RELEASE})
+  elseif(LIBLZMA_LIBRARY_DEBUG)
+    set(LIBLZMA_LIBRARY_check ${LIBLZMA_LIBRARY_DEBUG})
+  endif()
+  CHECK_LIBRARY_EXISTS(${LIBLZMA_LIBRARY_check} lzma_auto_decoder "" LIBLZMA_HAS_AUTO_DECODER)
+  CHECK_LIBRARY_EXISTS(${LIBLZMA_LIBRARY_check} lzma_easy_encoder "" LIBLZMA_HAS_EASY_ENCODER)
+  CHECK_LIBRARY_EXISTS(${LIBLZMA_LIBRARY_check} lzma_lzma_preset "" LIBLZMA_HAS_LZMA_PRESET)
+  unset(LIBLZMA_LIBRARY_check)
+  set(CMAKE_REQUIRED_QUIET ${CMAKE_REQUIRED_QUIET_SAVE})
+endif ()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(LibLZMA  REQUIRED_VARS  LIBLZMA_LIBRARY
+                                                          LIBLZMA_INCLUDE_DIR
+                                                          LIBLZMA_HAS_AUTO_DECODER
+                                                          LIBLZMA_HAS_EASY_ENCODER
+                                                          LIBLZMA_HAS_LZMA_PRESET
+                                           VERSION_VAR    LIBLZMA_VERSION_STRING
+                                 )
+mark_as_advanced( LIBLZMA_INCLUDE_DIR LIBLZMA_LIBRARY )
+
+if (LIBLZMA_FOUND)
+    set(LIBLZMA_LIBRARIES ${LIBLZMA_LIBRARY})
+    set(LIBLZMA_INCLUDE_DIRS ${LIBLZMA_INCLUDE_DIR})
+    if(NOT TARGET LibLZMA::LibLZMA)
+        add_library(LibLZMA::LibLZMA UNKNOWN IMPORTED)
+        set_target_properties(LibLZMA::LibLZMA PROPERTIES
+                              INTERFACE_INCLUDE_DIRECTORIES ${LIBLZMA_INCLUDE_DIR}
+                              IMPORTED_LINK_INTERFACE_LANGUAGES C)
+
+        if(LIBLZMA_LIBRARY_RELEASE)
+            set_property(TARGET LibLZMA::LibLZMA APPEND PROPERTY
+                IMPORTED_CONFIGURATIONS RELEASE)
+            set_target_properties(LibLZMA::LibLZMA PROPERTIES
+                IMPORTED_LOCATION_RELEASE "${LIBLZMA_LIBRARY_RELEASE}")
+        endif()
+
+        if(LIBLZMA_LIBRARY_DEBUG)
+            set_property(TARGET LibLZMA::LibLZMA APPEND PROPERTY
+                IMPORTED_CONFIGURATIONS DEBUG)
+            set_target_properties(LibLZMA::LibLZMA PROPERTIES
+                IMPORTED_LOCATION_DEBUG "${LIBLZMA_LIBRARY_DEBUG}")
+        endif()
+
+        if(NOT LIBLZMA_LIBRARY_RELEASE AND NOT LIBLZMA_LIBRARY_DEBUG)
+            set_target_properties(LibLZMA::LibLZMA PROPERTIES
+                IMPORTED_LOCATION "${LIBLZMA_LIBRARY}")
+        endif()
+    endif()
+endif ()
--- a/SimGearConfig.cmake.in
+++ b/SimGearConfig.cmake.in
@@ -1,6 +1,7 @@
 include(CMakeFindDependencyMacro)

 find_dependency(ZLIB)
+find_dependency(LibLZMA)
 find_dependency(Threads)

 # OSG
--- a/2
+++ b/2
@@ -1 +1 @@
-2020.3.2
+2020.3.7
--- a/simgear/CMakeLists.txt
+++ b/simgear/CMakeLists.txt
@@ -168,7 +168,9 @@ target_link_libraries(SimGearCore PRIVATE
    ${COCOA_LIBRARY}
    ${CURL_LIBRARIES}
    ${WINSOCK_LIBRARY}
-    ${SHLWAPI_LIBRARY})
+    ${SHLWAPI_LIBRARY}
+    LibLZMA::LibLZMA
+)

 if(SYSTEM_EXPAT)
    target_link_libraries(SimGearCore PRIVATE ${EXPAT_LIBRARIES})
--- a/simgear/canvas/Canvas.cxx
+++ b/simgear/canvas/Canvas.cxx
@@ -691,6 +691,10 @@ namespace canvas
      {
        _sampling_dirty = true;
      }
+      else if( name == "anisotropy" )
+      {
+        _texture.setMaxAnisotropy( node->getFloatValue() );
+      }
      else if( name == "additive-blend" )
      {
        _texture.useAdditiveBlend( node->getBoolValue() );
--- a/simgear/canvas/ODGauge.cxx
+++ b/simgear/canvas/ODGauge.cxx
@@ -202,6 +202,12 @@ namespace canvas
    updateSampling();
  }

+  //----------------------------------------------------------------------------
+  void ODGauge::setMaxAnisotropy(float anis)
+  {
+    texture->setMaxAnisotropy(anis);
+  }
+
  //----------------------------------------------------------------------------
  void ODGauge::setRender(bool render)
  {
--- a/simgear/canvas/ODGauge.hxx
+++ b/simgear/canvas/ODGauge.hxx
@@ -109,6 +109,8 @@ namespace canvas
                        int coverage_samples = 0,
                        int color_samples = 0 );

+      void setMaxAnisotropy(float anis);
+
      /**
       * Enable/Disable updating the texture (If disabled the contents of the
       * texture remains with the outcome of the last rendering pass)
--- a/simgear/constants.h
+++ b/simgear/constants.h
@@ -218,6 +218,9 @@ const float SG_RADIANS_TO_DEGREES = 180.0f / SG_PI;
 #define SG_OBJECT_RANGE_ROUGH     9000.0
 #define SG_OBJECT_RANGE_DETAILED  1500.0

+/** Minimum expiry time of PagedLOD within the Tile.  Overridden by /sim/rendering/plod-minimum-expiry-time-secs **/
+#define SG_TILE_MIN_EXPIRY 180.0
+
 /** Radius of scenery tiles in m **/
 #define SG_TILE_RADIUS      14000.0

--- a/simgear/debug/CMakeLists.txt
+++ b/simgear/debug/CMakeLists.txt
@@ -3,8 +3,12 @@ include (SimGearComponent)

 set(HEADERS debug_types.h 
    logstream.hxx BufferedLogCallback.hxx OsgIoCapture.hxx
-    LogCallback.hxx LogEntry.hxx)
+    LogCallback.hxx LogEntry.hxx
+    ErrorReportingCallback.hxx)
+
 set(SOURCES logstream.cxx BufferedLogCallback.cxx
-    LogCallback.cxx LogEntry.cxx)
+    LogCallback.cxx LogEntry.cxx
+    ErrorReportingCallback.cxx
+    )

 simgear_component(debug debug "${SOURCES}" "${HEADERS}")
--- a/simgear/debug/ErrorReportingCallback.cxx
+++ b/simgear/debug/ErrorReportingCallback.cxx
@@ -0,0 +1,47 @@
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Library General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Library General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//
+
+#include <simgear_config.h>
+
+#include "ErrorReportingCallback.hxx"
+
+using std::string;
+
+namespace simgear {
+
+static ErrorReportCallback static_callback;
+
+void setErrorReportCallback(ErrorReportCallback cb)
+{
+    static_callback = cb;
+}
+
+
+void reportError(const std::string& msg, const std::string& more)
+{
+    if (!static_callback)
+        return;
+
+    static_callback(msg, more, false);
+}
+
+void reportFatalError(const std::string& msg, const std::string& more)
+{
+    if (!static_callback)
+        return;
+    static_callback(msg, more, true);
+}
+
+} // namespace simgear
--- a/simgear/debug/ErrorReportingCallback.hxx
+++ b/simgear/debug/ErrorReportingCallback.hxx
@@ -0,0 +1,31 @@
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Library General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Library General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//
+
+#pragma once
+
+#include <functional>
+#include <string>
+
+namespace simgear {
+
+void reportError(const std::string& msg, const std::string& more = {});
+
+void reportFatalError(const std::string& msg, const std::string& more = {});
+
+using ErrorReportCallback = std::function<void(const std::string& msg, const std::string& more, bool isFatal)>;
+
+void setErrorReportCallback(ErrorReportCallback cb);
+
+} // namespace simgear
--- a/simgear/emesary/Transmitter.hxx
+++ b/simgear/emesary/Transmitter.hxx
@@ -45,14 +45,6 @@ namespace simgear
            std::atomic<int> receiveDepth;
            std::atomic<int> sentMessageCount;

-            void UnlockList()
-            {
-                _lock.unlock();
-            }
-            void LockList()
-            {
-                _lock.lock();
-            }
        public:
            Transmitter() : receiveDepth(0), sentMessageCount(0)
            {
@@ -69,20 +61,17 @@ namespace simgear
            // most recently registered recipients should process the messages/events first.
            virtual void Register(IReceiver& r)
            {
-                LockList();
+                std::lock_guard<std::mutex> scopeLock(_lock);
                recipient_list.push_back(&r);
                r.OnRegisteredAtTransmitter(this);
                if (std::find(deleted_recipients.begin(), deleted_recipients.end(), &r) != deleted_recipients.end())
                    deleted_recipients.remove(&r);
-
-                UnlockList();
            }

            //  Removes an object from receving message from this transmitter
            virtual void DeRegister(IReceiver& R)
            {
-                LockList();
-                //printf("Remove %x\n", &R);
+                std::lock_guard<std::mutex> scopeLock(_lock);
                if (recipient_list.size())
                {
                    if (std::find(recipient_list.begin(), recipient_list.end(), &R) != recipient_list.end())
@@ -93,7 +82,6 @@ namespace simgear
                            deleted_recipients.push_back(&R);
                    }
                }
-                UnlockList();
            }

            // Notify all registered recipients. Stop when receipt status of abort or finished are received.
@@ -107,69 +95,68 @@ namespace simgear
                ReceiptStatus return_status = ReceiptStatusNotProcessed;

                sentMessageCount++;
-                try
+
+                std::vector<IReceiver*> temp;
                {
-                    LockList();
+                    std::lock_guard<std::mutex> scopeLock(_lock);
                    if (receiveDepth == 0)
                        deleted_recipients.clear();
                    receiveDepth++;
-                    std::vector<IReceiver*> temp(recipient_list.size());
+
                    int idx = 0;
                    for (RecipientList::iterator i = recipient_list.begin(); i != recipient_list.end(); i++)
                    {
-                        temp[idx++] = *i;
+                        temp.push_back(*i);
                    }
-                    UnlockList();
-                    int tempSize = temp.size();
-                    for (int index = 0; index < tempSize; index++)
+                }
+                int tempSize = temp.size();
+                for (int index = 0; index < tempSize; index++)
+                {
+                    IReceiver* R = temp[index];
                    {
-                        IReceiver* R = temp[index];
-                        LockList();
+                        std::lock_guard<std::mutex> scopeLock(_lock);
                        if (deleted_recipients.size())
                        {
                            if (std::find(deleted_recipients.begin(), deleted_recipients.end(), R) != deleted_recipients.end())
                            {
-                                UnlockList();
                                continue;
                            }
                        }
-                        UnlockList();
-                        if (R)
-                        {
-                            ReceiptStatus rstat = R->Receive(M);
-                            switch (rstat)
-                            {
-                            case ReceiptStatusFail:
-                                return_status = ReceiptStatusFail;
-                                break;
-                            case ReceiptStatusPending:
-                                return_status = ReceiptStatusPending;
-                                break;
-                            case ReceiptStatusPendingFinished:
-                                return rstat;
-
-                            case ReceiptStatusNotProcessed:
-                                break;
-                            case ReceiptStatusOK:
-                                if (return_status == ReceiptStatusNotProcessed)
-                                    return_status = rstat;
-                                break;
-
-                            case ReceiptStatusAbort:
-                                return ReceiptStatusAbort;
-
-                            case ReceiptStatusFinished:
-                                return ReceiptStatusOK;
-                            }
-                        }
-
                    }
+                    if (R)
+                    {
+                        ReceiptStatus rstat = R->Receive(M);
+                        switch (rstat)
+                        {
+                        case ReceiptStatusFail:
+                            return_status = ReceiptStatusFail;
+                            break;
+
+                        case ReceiptStatusPending:
+                            return_status = ReceiptStatusPending;
+                            break;
+
+                        case ReceiptStatusPendingFinished:
+                            return rstat;
+
+                        case ReceiptStatusNotProcessed:
+                            break;
+
+                        case ReceiptStatusOK:
+                            if (return_status == ReceiptStatusNotProcessed)
+                                return_status = rstat;
+                            break;
+
+                        case ReceiptStatusAbort:
+                            return ReceiptStatusAbort;
+
+                        case ReceiptStatusFinished:
+                            return ReceiptStatusOK;
+                        }
+                    }
+
                }
-                catch (...)
-                {
-                    throw;
-                    // return_status = ReceiptStatusAbort;
-                }
+
                receiveDepth--;
                return return_status;
            }
@@ -177,9 +164,8 @@ namespace simgear
            // number of currently registered recipients
            virtual int Count()
            {
-                LockList();
+                std::lock_guard<std::mutex> scopeLock(_lock);
                return recipient_list.size();
-                UnlockList();
            }

            // number of sent messages.
--- a/simgear/environment/metar.cxx
+++ b/simgear/environment/metar.cxx
@@ -46,6 +46,7 @@
 #include <sstream>

 #include <simgear/debug/logstream.hxx>
+#include <simgear/math/sg_random.h>
 #include <simgear/structure/exception.hxx>

 #include "metar.hxx"
@@ -646,16 +647,21 @@ bool SGMetar::scanWind()
 	double gust = NaN;
 	if (*m == 'G') {
 		m++;
-		if (!scanNumber(&m, &i, 2, 3))
+		if (!strncmp(m, "//", 2))	// speed not measurable
+			m += 2, i = -1;
+		else if (!scanNumber(&m, &i, 2, 3))
 			return false;
-		gust = i;
+
+		if (i != -1)
+			gust = i;
 	}
+
 	double factor;
 	if (!strncmp(m, "KT", 2))
 		m += 2, factor = SG_KT_TO_MPS;
-	else if (!strncmp(m, "KMH", 3))
+	else if (!strncmp(m, "KMH", 3))		// invalid Km/h
 		m += 3, factor = SG_KMH_TO_MPS;
-	else if (!strncmp(m, "KPH", 3))		// ??
+	else if (!strncmp(m, "KPH", 3))		// invalid Km/h
 		m += 3, factor = SG_KMH_TO_MPS;
 	else if (!strncmp(m, "MPS", 3))
 		m += 3, factor = 1.0;
@@ -680,18 +686,28 @@ bool SGMetar::scanVariability()
 {
 	char *m = _m;
 	int from, to;
-	if (!scanNumber(&m, &from, 3))
+
+	if (!strncmp(m, "///", 3))	// direction not measurable
+		m += 3, from = -1;
+	else if (!scanNumber(&m, &from, 3))
 		return false;
+
 	if (*m++ != 'V')
 		return false;
-	if (!scanNumber(&m, &to, 3))
+
+	if (!strncmp(m, "///", 3))	// direction not measurable
+		m += 3, to = -1;
+	else if (!scanNumber(&m, &to, 3))
 		return false;
+
 	if (!scanBoundary(&m))
 		return false;
+
 	_m = m;
 	_wind_range_from = from;
 	_wind_range_to = to;
 	_grpcount++;
+
 	return true;
 }

@@ -1053,9 +1069,12 @@ bool SGMetar::scanSkyCondition()
 		return false;
 	m += i;

-	if (!strncmp(m, "///", 3))	// vis not measurable (e.g. because of heavy snowing)
+	if (!strncmp(m, "///", 3))	{ // vis not measurable (e.g. because of heavy snowing)
 		m += 3, i = -1;
-	else if (scanBoundary(&m)) {
+		sg_srandom_time();
+		// randomize the base height to avoid the black sky issue
+		i = 50 + static_cast<int>(sg_random() * 250.0);		// range [5,000, 30,000]
+	} else if (scanBoundary(&m)) {
 		_m = m;
 		return true;				// ignore single OVC/BKN/...
 	} else if (!scanNumber(&m, &i, 3))
--- a/simgear/environment/test_metar.cxx
+++ b/simgear/environment/test_metar.cxx
@@ -81,6 +81,15 @@ void test_sensor_failure_wind()
    SGMetar m1("2020/10/23 16:55 LIVD 231655Z /////KT 9999 OVC025 10/08 Q1020 RMK OVC VIS MIN 9999 BLU");
    SG_CHECK_EQUAL(m1.getWindDir(), -1);
    SG_CHECK_EQUAL_EP2(m1.getWindSpeed_kt(), -1, TEST_EPSILON);
+
+    SGMetar m2("2020/10/21 16:55 LIVD 211655Z /////KT CAVOK 07/03 Q1023 RMK SKC VIS MIN 9999 BLU");
+    SG_CHECK_EQUAL(m2.getWindDir(), -1);
+    SG_CHECK_EQUAL_EP2(m2.getWindSpeed_kt(), -1, TEST_EPSILON);
+
+    SGMetar m3("2020/11/17 16:00 CYAZ 171600Z 14040G//KT 10SM -RA OVC012 12/11 A2895 RMK NS8 VIA CYXY SLP806 DENSITY ALT 900FT");
+    SG_CHECK_EQUAL(m3.getWindDir(), 140);
+    SG_CHECK_EQUAL_EP2(m3.getWindSpeed_kt(), 40, TEST_EPSILON);
+    SG_CHECK_EQUAL_EP2(m3.getGustSpeed_kt(), SGMetarNaN, TEST_EPSILON);
 }

 void test_wind_unit_not_specified()
--- a/simgear/io/ArchiveExtractor_private.hxx
+++ b/simgear/io/ArchiveExtractor_private.hxx
@@ -0,0 +1,90 @@
+// Copyright (C) 2021  James Turner - <james@flightgear.org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Library General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Library General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//
+
+
+#pragma once
+
+#include "untar.hxx"
+
+namespace simgear {
+
+class ArchiveExtractorPrivate
+{
+public:
+    ArchiveExtractorPrivate(ArchiveExtractor* o) : outer(o)
+    {
+        assert(outer);
+    }
+
+    virtual ~ArchiveExtractorPrivate() = default;
+
+    typedef enum {
+        INVALID = 0,
+        READING_HEADER,
+        READING_FILE,
+        READING_PADDING,
+        READING_PAX_GLOBAL_ATTRIBUTES,
+        READING_PAX_FILE_ATTRIBUTES,
+        PRE_END_OF_ARCHVE,
+        END_OF_ARCHIVE,
+        ERROR_STATE, ///< states above this are error conditions
+        BAD_ARCHIVE,
+        BAD_DATA,
+        FILTER_STOPPED
+    } State;
+
+    State state = INVALID;
+    ArchiveExtractor* outer = nullptr;
+
+    virtual void extractBytes(const uint8_t* bytes, size_t count) = 0;
+
+    virtual void flush() = 0;
+
+    SGPath extractRootPath()
+    {
+        return outer->_rootPath;
+    }
+
+    ArchiveExtractor::PathResult filterPath(std::string& pathToExtract)
+    {
+        return outer->filterPath(pathToExtract);
+    }
+
+
+    bool isSafePath(const std::string& p) const
+    {
+        if (p.empty()) {
+            return false;
+        }
+
+        // reject absolute paths
+        if (p.at(0) == '/') {
+            return false;
+        }
+
+        // reject paths containing '..'
+        size_t doubleDot = p.find("..");
+        if (doubleDot != std::string::npos) {
+            return false;
+        }
+
+        // on POSIX could use realpath to sanity check
+        return true;
+    }
+};
+
+} // namespace simgear
--- a/simgear/io/DNSClient.cxx
+++ b/simgear/io/DNSClient.cxx
@@ -61,6 +61,10 @@ public:

    struct dns_ctx * ctx;
    static size_t instanceCounter;
+
+    using RequestVec = std::vector<Request_ptr>;
+
+    RequestVec _activeRequests;
 };

 size_t Client::ClientPrivate::instanceCounter = 0;
@@ -78,6 +82,11 @@ Request::~Request()
 {
 }

+void Request::cancel()
+{
+    _cancelled = true;
+}
+
 bool Request::isTimeout() const
 {
    return (time(NULL) - _start) > _timeout_secs;
@@ -114,18 +123,20 @@ static void dnscbSRV(struct dns_ctx *ctx, struct dns_rr_srv *result, void *data)
 {
    SRVRequest * r = static_cast<SRVRequest*>(data);
    if (result) {
-        r->cname = result->dnssrv_cname;
-        r->qname = result->dnssrv_qname;
-        r->ttl = result->dnssrv_ttl;
-        for (int i = 0; i < result->dnssrv_nrr; i++) {
-            SRVRequest::SRV_ptr srv(new SRVRequest::SRV);
-            r->entries.push_back(srv);
-            srv->priority = result->dnssrv_srv[i].priority;
-            srv->weight = result->dnssrv_srv[i].weight;
-            srv->port = result->dnssrv_srv[i].port;
-            srv->target = result->dnssrv_srv[i].name;
+        if (!r->isCancelled()) {
+            r->cname = result->dnssrv_cname;
+            r->qname = result->dnssrv_qname;
+            r->ttl = result->dnssrv_ttl;
+            for (int i = 0; i < result->dnssrv_nrr; i++) {
+                SRVRequest::SRV_ptr srv(new SRVRequest::SRV);
+                r->entries.push_back(srv);
+                srv->priority = result->dnssrv_srv[i].priority;
+                srv->weight = result->dnssrv_srv[i].weight;
+                srv->port = result->dnssrv_srv[i].port;
+                srv->target = result->dnssrv_srv[i].name;
+            }
+            std::sort(r->entries.begin(), r->entries.end(), sortSRV);
        }
-        std::sort( r->entries.begin(), r->entries.end(), sortSRV );
        free(result);
    }
    r->setComplete();
@@ -134,11 +145,16 @@ static void dnscbSRV(struct dns_ctx *ctx, struct dns_rr_srv *result, void *data)
 void SRVRequest::submit( Client * client )
 {
    // if service is defined, pass service and protocol
-    if (!dns_submit_srv(client->d->ctx, getDn().c_str(), _service.empty() ? NULL : _service.c_str(), _service.empty() ? NULL : _protocol.c_str(), 0, dnscbSRV, this )) {
+    auto q = dns_submit_srv(client->d->ctx, getDn().c_str(), _service.empty() ? NULL : _service.c_str(),
+                            _service.empty() ? NULL : _protocol.c_str(),
+                            0, dnscbSRV, this);
+
+    if (!q) {
        SG_LOG(SG_IO, SG_ALERT, "Can't submit dns request for " << getDn());
        return;
    }
    _start = time(NULL);
+    _query = q;
 }

 TXTRequest::TXTRequest( const std::string & dn ) :
@@ -151,22 +167,24 @@ static void dnscbTXT(struct dns_ctx *ctx, struct dns_rr_txt *result, void *data)
 {
    TXTRequest * r = static_cast<TXTRequest*>(data);
    if (result) {
-        r->cname = result->dnstxt_cname;
-        r->qname = result->dnstxt_qname;
-        r->ttl = result->dnstxt_ttl;
-        for (int i = 0; i < result->dnstxt_nrr; i++) {
-          //TODO: interprete the .len field of dnstxt_txt?
-          auto rawTxt = reinterpret_cast<char*>(result->dnstxt_txt[i].txt);
-          if (!rawTxt) {
-              continue;
-          }
+        if (!r->isCancelled()) {
+            r->cname = result->dnstxt_cname;
+            r->qname = result->dnstxt_qname;
+            r->ttl = result->dnstxt_ttl;
+            for (int i = 0; i < result->dnstxt_nrr; i++) {
+                //TODO: interprete the .len field of dnstxt_txt?
+                auto rawTxt = reinterpret_cast<char*>(result->dnstxt_txt[i].txt);
+                if (!rawTxt) {
+                    continue;
+                }

-          const string txt{rawTxt};
-          r->entries.push_back(txt);
-          string_list tokens = simgear::strutils::split( txt, "=", 1 );
-          if( tokens.size() == 2 ) {
-            r->attributes[tokens[0]] = tokens[1];
-          }
+                const string txt{rawTxt};
+                r->entries.push_back(txt);
+                string_list tokens = simgear::strutils::split(txt, "=", 1);
+                if (tokens.size() == 2) {
+                    r->attributes[tokens[0]] = tokens[1];
+                }
+            }
        }
        free(result);
    }
@@ -176,11 +194,13 @@ static void dnscbTXT(struct dns_ctx *ctx, struct dns_rr_txt *result, void *data)
 void TXTRequest::submit( Client * client )
 {
    // protocol and service an already encoded in DN so pass in NULL for both
-    if (!dns_submit_txt(client->d->ctx, getDn().c_str(), DNS_C_IN, 0, dnscbTXT, this )) {
+    auto q = dns_submit_txt(client->d->ctx, getDn().c_str(), DNS_C_IN, 0, dnscbTXT, this);
+    if (!q) {
        SG_LOG(SG_IO, SG_ALERT, "Can't submit dns request for " << getDn());
        return;
    }
    _start = time(NULL);
+    _query = q;
 }


@@ -195,27 +215,29 @@ static void dnscbNAPTR(struct dns_ctx *ctx, struct dns_rr_naptr *result, void *d
 {
    NAPTRRequest * r = static_cast<NAPTRRequest*>(data);
    if (result) {
-        r->cname = result->dnsnaptr_cname;
-        r->qname = result->dnsnaptr_qname;
-        r->ttl = result->dnsnaptr_ttl;
-        for (int i = 0; i < result->dnsnaptr_nrr; i++) {
-            if( !r->qservice.empty() && r->qservice != result->dnsnaptr_naptr[i].service )
-                continue;
+        if (!r->isCancelled()) {
+            r->cname = result->dnsnaptr_cname;
+            r->qname = result->dnsnaptr_qname;
+            r->ttl = result->dnsnaptr_ttl;
+            for (int i = 0; i < result->dnsnaptr_nrr; i++) {
+                if (!r->qservice.empty() && r->qservice != result->dnsnaptr_naptr[i].service)
+                    continue;

-            //TODO: case ignore and result flags may have more than one flag
-            if( !r->qflags.empty() && r->qflags != result->dnsnaptr_naptr[i].flags )
-                continue;
+                //TODO: case ignore and result flags may have more than one flag
+                if (!r->qflags.empty() && r->qflags != result->dnsnaptr_naptr[i].flags)
+                    continue;

-            NAPTRRequest::NAPTR_ptr naptr(new NAPTRRequest::NAPTR);
-            r->entries.push_back(naptr);
-            naptr->order = result->dnsnaptr_naptr[i].order;
-            naptr->preference = result->dnsnaptr_naptr[i].preference;
-            naptr->flags = result->dnsnaptr_naptr[i].flags;
-            naptr->service = result->dnsnaptr_naptr[i].service;
-            naptr->regexp = result->dnsnaptr_naptr[i].regexp;
-            naptr->replacement = result->dnsnaptr_naptr[i].replacement;
+                NAPTRRequest::NAPTR_ptr naptr(new NAPTRRequest::NAPTR);
+                r->entries.push_back(naptr);
+                naptr->order = result->dnsnaptr_naptr[i].order;
+                naptr->preference = result->dnsnaptr_naptr[i].preference;
+                naptr->flags = result->dnsnaptr_naptr[i].flags;
+                naptr->service = result->dnsnaptr_naptr[i].service;
+                naptr->regexp = result->dnsnaptr_naptr[i].regexp;
+                naptr->replacement = result->dnsnaptr_naptr[i].replacement;
+            }
+            std::sort(r->entries.begin(), r->entries.end(), sortNAPTR);
        }
-        std::sort( r->entries.begin(), r->entries.end(), sortNAPTR );
        free(result);
    }
    r->setComplete();
@@ -223,11 +245,13 @@ static void dnscbNAPTR(struct dns_ctx *ctx, struct dns_rr_naptr *result, void *d

 void NAPTRRequest::submit( Client * client )
 {
-    if (!dns_submit_naptr(client->d->ctx, getDn().c_str(), 0, dnscbNAPTR, this )) {
+    auto q = dns_submit_naptr(client->d->ctx, getDn().c_str(), 0, dnscbNAPTR, this);
+    if (!q) {
        SG_LOG(SG_IO, SG_ALERT, "Can't submit dns request for " << getDn());
        return;
    }
    _start = time(NULL);
+    _query = q;
 }


@@ -242,6 +266,7 @@ Client::Client() :

 void Client::makeRequest(const Request_ptr& r)
 {
+    d->_activeRequests.push_back(r);
    r->submit(this);
 }

@@ -252,6 +277,19 @@ void Client::update(int waitTimeout)
        return;

    dns_ioevent(d->ctx, now);
+
+    // drop our owning ref to completed requests,
+    // and cancel any which timed out
+    auto it = std::remove_if(d->_activeRequests.begin(), d->_activeRequests.end(),
+                             [this](const Request_ptr& r) {
+                                 if (r->isTimeout()) {
+                                     dns_cancel(d->ctx, reinterpret_cast<struct dns_query*>(r->_query));
+                                     return true;
+                                 }
+
+                                 return r->isComplete();
+                             });
+    d->_activeRequests.erase(it, d->_activeRequests.end());
 }

 } // of namespace DNS
--- a/simgear/io/DNSClient.hxx
+++ b/simgear/io/DNSClient.hxx
@@ -40,28 +40,38 @@ namespace DNS
 {

 class Client;
+
+using UDNSQueryPtr = void*;
+
 class Request : public SGReferenced
 {
 public:
    Request( const std::string & dn );
    virtual ~Request();
-    std::string getDn() const { return _dn; }
+    const std::string& getDn() const { return _dn; }
    int getType() const { return _type; }
    bool isComplete() const { return _complete; }
    bool isTimeout() const;
    void setComplete( bool b = true ) { _complete = b; }
+    bool isCancelled() const { return _cancelled; }

    virtual void submit( Client * client) = 0;

+    void cancel();
+
    std::string cname;
    std::string qname;
    unsigned ttl;
 protected:
+    friend class Client;
+
+    UDNSQueryPtr _query = nullptr;
    std::string _dn;
    int _type;
    bool _complete;
    time_t _timeout_secs;
    time_t _start;
+    bool _cancelled = false;
 };
 typedef SGSharedPtr<Request> Request_ptr;

@@ -69,7 +79,7 @@ class NAPTRRequest : public Request
 {
 public:
    NAPTRRequest( const std::string & dn );
-    virtual void submit( Client * client );
+    void submit(Client* client) override;

    struct NAPTR : SGReferenced {
        int order;
@@ -92,7 +102,7 @@ class SRVRequest : public Request
 public:
    SRVRequest( const std::string & dn );
    SRVRequest( const std::string & dn, const string & service, const string & protocol );
-    virtual void submit( Client * client );
+    void submit(Client* client) override;

    struct SRV : SGReferenced {
      int priority;
@@ -112,7 +122,7 @@ class TXTRequest : public Request
 {
 public:
    TXTRequest( const std::string & dn );
-    virtual void submit( Client * client );
+    void submit(Client* client) override;

    typedef std::vector<string> TXT_list;
    typedef std::map<std::string,std::string> TXT_Attribute_map;
--- a/simgear/io/HTTPClient.cxx
+++ b/simgear/io/HTTPClient.cxx
@@ -81,21 +81,8 @@ void Client::ClientPrivate::createCurlMulti() {
 #endif
 }

-Client::Client() :
-    d(new ClientPrivate)
+Client::Client()
 {
-    d->proxyPort = 0;
-    d->maxConnections = 4;
-    d->maxHostConnections = 4;
-    d->bytesTransferred = 0;
-    d->lastTransferRate = 0;
-    d->timeTransferSample.stamp();
-    d->totalBytesDownloaded = 0;
-    d->maxPipelineDepth = 5;
-    setUserAgent("SimGear-" SG_STRINGIZE(SIMGEAR_VERSION));
-
-    d->tlsCertificatePath = SGPath::fromEnv("SIMGEAR_TLS_CERT_PATH");
-
    static bool didInitCurlGlobal = false;
    static std::mutex initMutex;

@@ -105,7 +92,7 @@ Client::Client() :
      didInitCurlGlobal = true;
    }

-    d->createCurlMulti();
+    reset();
 }

 Client::~Client()
@@ -139,8 +126,21 @@ void Client::setMaxPipelineDepth(unsigned int depth)

 void Client::reset()
 {
-    curl_multi_cleanup(d->curlMulti);
+    if (d.get()) {
+        curl_multi_cleanup(d->curlMulti);
+    }
+
    d.reset(new ClientPrivate);
+
+    d->proxyPort = 0;
+    d->maxConnections = 4;
+    d->maxHostConnections = 4;
+    d->bytesTransferred = 0;
+    d->lastTransferRate = 0;
+    d->timeTransferSample.stamp();
+    d->totalBytesDownloaded = 0;
+    d->maxPipelineDepth = 5;
+    setUserAgent("SimGear-" SG_STRINGIZE(SIMGEAR_VERSION));
    d->tlsCertificatePath = SGPath::fromEnv("SIMGEAR_TLS_CERT_PATH");
    d->createCurlMulti();
 }
--- a/simgear/io/HTTPRepository.cxx
+++ b/simgear/io/HTTPRepository.cxx
@@ -20,15 +20,15 @@

 #include "HTTPRepository.hxx"

-#include <iostream>
-#include <cassert>
 #include <algorithm>
-#include <sstream>
+#include <cassert>
+#include <cstdlib>
+#include <fstream>
+#include <iostream>
+#include <limits>
 #include <map>
 #include <set>
-#include <fstream>
-#include <limits>
-#include <cstdlib>
+#include <sstream>

 #include <fcntl.h>

@@ -143,8 +143,8 @@ class HTTPDirectory

        HTTPRepository::EntryType type;
        std::string name, hash;
-		size_t sizeInBytes = 0;
-		SGPath path; // absolute path on disk
+        size_t sizeInBytes = 0;
+        SGPath path; // absolute path on disk
    };

    typedef std::vector<ChildInfo> ChildInfoList;
@@ -218,53 +218,66 @@ public:
            return;
        }

-		char* buf = nullptr;
-		size_t bufSize = 0;
+        char* buf = nullptr;
+        size_t bufSize = 0;

-                for (auto &child : children) {
-                  if (child.type != HTTPRepository::FileType)
+        for (auto& child : children) {
+            if (child.type != HTTPRepository::FileType)
+                continue;
+
+            if (child.path.exists())
+                continue;
+
+            SGPath cp = _repository->installedCopyPath;
+            cp.append(relativePath());
+            cp.append(child.name);
+            if (!cp.exists()) {
+                continue;
+            }
+
+            SGBinaryFile src(cp);
+            SGBinaryFile dst(child.path);
+            src.open(SG_IO_IN);
+            dst.open(SG_IO_OUT);
+
+            if (bufSize < cp.sizeInBytes()) {
+                bufSize = cp.sizeInBytes();
+                free(buf);
+                buf = (char*)malloc(bufSize);
+                if (!buf) {
                    continue;
+                }
+            }

-                  if (child.path.exists())
-                    continue;
+            src.read(buf, cp.sizeInBytes());
+            dst.write(buf, cp.sizeInBytes());
+            src.close();
+            dst.close();

-                  SGPath cp = _repository->installedCopyPath;
-                  cp.append(relativePath());
-                  cp.append(child.name);
-                  if (!cp.exists()) {
-                    continue;
-                  }
+            // reset caching
+            child.path.set_cached(false);
+            child.path.set_cached(true);

-                  SGBinaryFile src(cp);
-                  SGBinaryFile dst(child.path);
-                  src.open(SG_IO_IN);
-                  dst.open(SG_IO_OUT);
-
-                  if (bufSize < cp.sizeInBytes()) {
-                    bufSize = cp.sizeInBytes();
-                    free(buf);
-                    buf = (char *)malloc(bufSize);
-                    if (!buf) {
-                      continue;
-                    }
-                  }
-
-                  src.read(buf, cp.sizeInBytes());
-                  dst.write(buf, cp.sizeInBytes());
-                  src.close();
-                  dst.close();
-
-                  // reset caching
-                  child.path.set_cached(false);
-                  child.path.set_cached(true);
-
-                  std::string hash = computeHashForPath(child.path);
-                  updatedFileContents(child.path, hash);
+            std::string hash = computeHashForPath(child.path);
+            updatedFileContents(child.path, hash);
                }

                free(buf);
    }

+    /// helper to check and erase 'fooBar' from paths, if passed fooBar.zip, fooBar.tgz, etc.
+    void removeExtractedDirectoryFromList(PathList& paths, const std::string& tarballName)
+    {
+        const auto directoryName = SGPath::fromUtf8(tarballName).file_base();
+        auto it = std::find_if(paths.begin(), paths.end(), [directoryName](const SGPath& p) {
+            return p.isDir() && (p.file() == directoryName);
+        });
+
+        if (it != paths.end()) {
+            paths.erase(it);
+        }
+    }
+
    void updateChildrenBasedOnHash()
    {
      using SAct = HTTPRepository::SyncAction;
@@ -298,6 +311,11 @@ public:
                        orphans.end());
        }

+        // ensure the extracted directory corresponding to a tarball, is *not* considered an orphan
+        if (c.type == HTTPRepository::TarballType) {
+            removeExtractedDirectoryFromList(orphans, c.name);
+        }
+
        if (_repository->syncPredicate) {
          const auto pathOnDisk = isNew ? absolutePath() / c.name : *p;
          // never handle deletes here, do them at the end
@@ -320,7 +338,6 @@ public:
          toBeUpdated.push_back(c);
        } else {
          // File/Directory exists and hash is valid.
-
          if (c.type == HTTPRepository::DirectoryType) {
            // If it's a directory,perform a recursive check.
            HTTPDirectory *childDir = childDirectory(c.name);
@@ -432,41 +449,54 @@ public:
          return;
        }

+        compressedBytes = p.sizeInBytes();
        buffer = (uint8_t *)malloc(bufferSize);
      }

      ArchiveExtractTask(const ArchiveExtractTask &) = delete;

-      HTTPRepoPrivate::ProcessResult run(HTTPRepoPrivate *repo) {
-        size_t rd = file.read((char *)buffer, bufferSize);
-        extractor.extractBytes(buffer, rd);
-
-        if (file.eof()) {
-          extractor.flush();
-          file.close();
-
-          if (!extractor.isAtEndOfArchive()) {
-            SG_LOG(SG_TERRASYNC, SG_ALERT, "Corrupt tarball " << relativePath);
-            repo->failedToUpdateChild(relativePath,
-                                      HTTPRepository::REPO_ERROR_IO);
-            return HTTPRepoPrivate::ProcessFailed;
+      HTTPRepoPrivate::ProcessResult run(HTTPRepoPrivate* repo)
+      {
+          if (!buffer) {
+              return HTTPRepoPrivate::ProcessFailed;
          }

-          if (extractor.hasError()) {
-            SG_LOG(SG_TERRASYNC, SG_ALERT, "Error extracting " << relativePath);
-            repo->failedToUpdateChild(relativePath,
-                                      HTTPRepository::REPO_ERROR_IO);
-            return HTTPRepoPrivate::ProcessFailed;
+          size_t rd = file.read((char*)buffer, bufferSize);
+          repo->bytesExtracted += rd;
+          extractor.extractBytes(buffer, rd);
+
+          if (file.eof()) {
+              extractor.flush();
+              file.close();
+
+              if (!extractor.isAtEndOfArchive()) {
+                  SG_LOG(SG_TERRASYNC, SG_ALERT, "Corrupt tarball " << relativePath);
+                  repo->failedToUpdateChild(relativePath,
+                                            HTTPRepository::REPO_ERROR_IO);
+                  return HTTPRepoPrivate::ProcessFailed;
+              }
+
+              if (extractor.hasError()) {
+                  SG_LOG(SG_TERRASYNC, SG_ALERT, "Error extracting " << relativePath);
+                  repo->failedToUpdateChild(relativePath,
+                                            HTTPRepository::REPO_ERROR_IO);
+                  return HTTPRepoPrivate::ProcessFailed;
+              }
+
+              return HTTPRepoPrivate::ProcessDone;
          }

-          return HTTPRepoPrivate::ProcessDone;
-        }
+          return HTTPRepoPrivate::ProcessContinue;
+      }

-        return HTTPRepoPrivate::ProcessContinue;
+      size_t archiveSizeBytes() const
+      {
+          return compressedBytes;
      }

      ~ArchiveExtractTask() { free(buffer); }

+
    private:
      // intentionally small so we extract incrementally on Windows
      // where Defender throttles many small files, sorry
@@ -478,6 +508,7 @@ public:
      uint8_t *buffer = nullptr;
      SGBinaryFile file;
      ArchiveExtractor extractor;
+      std::size_t compressedBytes;
    };

    using ArchiveExtractTaskPtr = std::shared_ptr<ArchiveExtractTask>;
@@ -504,42 +535,42 @@ public:
                _repository->totalDownloaded += sz;
                SGPath p = SGPath(absolutePath(), file);

-                if ((p.extension() == "tgz") || (p.extension() == "zip")) {
-                  // We require that any compressed files have the same filename as the file or directory
-                  // they expand to, so we can remove the old file/directory before extracting the new
-                  // data.
-                  SGPath removePath = SGPath(p.base());
-                  bool pathAvailable = true;
-                  if (removePath.exists()) {
-                    if (removePath.isDir()) {
-                      simgear::Dir pd(removePath);
-                    	pathAvailable = pd.removeChildren();
-                    } else {
-                      pathAvailable = removePath.remove();
+                if (it->type == HTTPRepository::TarballType) {
+                    // We require that any compressed files have the same filename as the file or directory
+                    // they expand to, so we can remove the old file/directory before extracting the new
+                    // data.
+                    SGPath removePath = SGPath(p.base());
+                    bool pathAvailable = true;
+                    if (removePath.exists()) {
+                        if (removePath.isDir()) {
+                            simgear::Dir pd(removePath);
+                            pathAvailable = pd.removeChildren();
+                        } else {
+                            pathAvailable = removePath.remove();
+                        }
                    }
-                  }

-                  if (pathAvailable) {
-                    // we use a Task helper to extract tarballs incrementally.
-                    // without this, archive extraction blocks here, which
-                    // prevents other repositories downloading / updating.
-                    // Unfortunately due Windows AV (Defender, etc) we cna block
-                    // here for many minutes.
+                    if (pathAvailable) {
+                        // we use a Task helper to extract tarballs incrementally.
+                        // without this, archive extraction blocks here, which
+                        // prevents other repositories downloading / updating.
+                        // Unfortunately due Windows AV (Defender, etc) we cna block
+                        // here for many minutes.

-                    // use a lambda to own this shared_ptr; this means when the
-                    // lambda is destroyed, the ArchiveExtraTask will get
-                    // cleaned up.
-                    ArchiveExtractTaskPtr t =
-                        std::make_shared<ArchiveExtractTask>(p, _relativePath);
-                    auto cb = [t](HTTPRepoPrivate *repo) {
-                      return t->run(repo);
-                    };
-
-                    _repository->addTask(cb);
-                  } else {
-                    SG_LOG(SG_TERRASYNC, SG_ALERT, "Unable to remove old file/directory " << removePath);
-                  } // of pathAvailable
-                } // of handling tgz files
+                        // use a lambda to own this shared_ptr; this means when the
+                        // lambda is destroyed, the ArchiveExtraTask will get
+                        // cleaned up.
+                        ArchiveExtractTaskPtr t =
+                            std::make_shared<ArchiveExtractTask>(p, _relativePath);
+                        auto cb = [t](HTTPRepoPrivate* repo) {
+                            return t->run(repo);
+                        };
+                        _repository->bytesToExtract += t->archiveSizeBytes();
+                        _repository->addTask(cb);
+                    } else {
+                        SG_LOG(SG_TERRASYNC, SG_ALERT, "Unable to remove old file/directory " << removePath);
+                    } // of pathAvailable
+                }     // of handling archive files
            } // of hash matches
        } // of found in child list
    }
@@ -731,11 +762,9 @@ private:
    std::string hashForChild(const ChildInfo& child) const
    {
      SGPath p(child.path);
-      if (child.type == HTTPRepository::DirectoryType)
-        p.append(".dirindex");
-      if (child.type == HTTPRepository::TarballType)
-        p.concat(
-            ".tgz"); // For tarballs the hash is against the tarball file itself
+      if (child.type == HTTPRepository::DirectoryType) {
+          p.append(".dirindex");
+      }
      return hashForPath(p);
    }

@@ -932,6 +961,11 @@ size_t HTTPRepository::bytesDownloaded() const
    return result;
 }

+size_t HTTPRepository::bytesToExtract() const
+{
+    return _d->bytesToExtract - _d->bytesExtracted;
+}
+
 void HTTPRepository::setInstalledCopyPath(const SGPath& copyPath)
 {
    _d->installedCopyPath = copyPath;
@@ -976,49 +1010,71 @@ HTTPRepository::failure() const
        }

    protected:
-      void gotBodyData(const char *s, int n) override {
-        if (!file.get()) {
-          file.reset(new SGBinaryFile(pathInRepo));
-          if (!file->open(SG_IO_OUT)) {
-            SG_LOG(SG_TERRASYNC, SG_WARN,
-                   "unable to create file " << pathInRepo);
-            _directory->repository()->http->cancelRequest(
-                this, "Unable to create output file:" + pathInRepo.utf8Str());
-          }
+        void gotBodyData(const char* s, int n) override
+        {
+            if (!file.get()) {
+                const bool ok = createOutputFile();
+                if (!ok) {
+                    _directory->repository()->http->cancelRequest(
+                        this, "Unable to create output file:" + pathInRepo.utf8Str());
+                }
+            }

-          sha1_init(&hashContext);
+            sha1_write(&hashContext, s, n);
+            file->write(s, n);
        }

-        sha1_write(&hashContext, s, n);
-        file->write(s, n);
-      }
+        bool createOutputFile()
+        {
+            file.reset(new SGBinaryFile(pathInRepo));
+            if (!file->open(SG_IO_OUT)) {
+                SG_LOG(SG_TERRASYNC, SG_WARN,
+                       "unable to create file " << pathInRepo);
+                return false;
+            }

-      void onDone() override {
-        file->close();
-        if (responseCode() == 200) {
-          std::string hash =
-              strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH);
-          _directory->didUpdateFile(fileName, hash, contentSize());
-        } else if (responseCode() == 404) {
-          SG_LOG(SG_TERRASYNC, SG_WARN,
-                 "terrasync file not found on server: "
-                     << fileName << " for " << _directory->absolutePath());
-          _directory->didFailToUpdateFile(
-              fileName, HTTPRepository::REPO_ERROR_FILE_NOT_FOUND);
-        } else {
-          SG_LOG(SG_TERRASYNC, SG_WARN,
-                 "terrasync file download error on server: "
-                     << fileName << " for " << _directory->absolutePath()
-                     << "\n\tserver responded: " << responseCode() << "/"
-                     << responseReason());
-          _directory->didFailToUpdateFile(fileName,
-                                          HTTPRepository::REPO_ERROR_HTTP);
-          // should we every retry here?
+            sha1_init(&hashContext);
+            return true;
        }

-        _directory->repository()->finishedRequest(
-            this, HTTPRepoPrivate::RequestFinish::Done);
-      }
+        void onDone() override
+        {
+            const bool is200Response = (responseCode() == 200);
+            if (!file && is200Response) {
+                // if the server defines a zero-byte file, we will never call
+                // gotBodyData, so create the file here
+                // this ensures all the logic below works as expected
+                createOutputFile();
+            }
+
+            if (file) {
+                file->close();
+            }
+
+            if (is200Response) {
+                std::string hash =
+                    strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH);
+                _directory->didUpdateFile(fileName, hash, contentSize());
+            } else if (responseCode() == 404) {
+                SG_LOG(SG_TERRASYNC, SG_WARN,
+                       "terrasync file not found on server: "
+                           << fileName << " for " << _directory->absolutePath());
+                _directory->didFailToUpdateFile(
+                    fileName, HTTPRepository::REPO_ERROR_FILE_NOT_FOUND);
+            } else {
+                SG_LOG(SG_TERRASYNC, SG_WARN,
+                       "terrasync file download error on server: "
+                           << fileName << " for " << _directory->absolutePath()
+                           << "\n\tserver responded: " << responseCode() << "/"
+                           << responseReason());
+                _directory->didFailToUpdateFile(fileName,
+                                                HTTPRepository::REPO_ERROR_HTTP);
+                // should we every retry here?
+            }
+
+            _directory->repository()->finishedRequest(
+                this, HTTPRepoPrivate::RequestFinish::Done);
+        }

      void onFail() override {
        HTTPRepository::ResultCode code = HTTPRepository::REPO_ERROR_SOCKET;
@@ -1266,7 +1322,7 @@ HTTPRepository::failure() const
        }

        Dir dir(absPath);
-		bool result = dir.remove(true);
+        bool result = dir.remove(true);
        return result;
    }

--- a/simgear/io/HTTPRepository.hxx
+++ b/simgear/io/HTTPRepository.hxx
@@ -73,6 +73,8 @@ public:

  virtual size_t bytesDownloaded() const;

+  virtual size_t bytesToExtract() const;
+
  /**
   * optionally provide the location of an installer copy of this
   * repository. When a file is missing it will be copied from this tree.
--- a/simgear/io/HTTPRepository_private.hxx
+++ b/simgear/io/HTTPRepository_private.hxx
@@ -60,22 +60,23 @@ public:
  HTTPRepository::FailureVec failures;
  int maxPermittedFailures = 16;

-  HTTPRepoPrivate(HTTPRepository *parent)
-      : p(parent), isUpdating(false), status(HTTPRepository::REPO_NO_ERROR),
-        totalDownloaded(0) {
-    ;
+  HTTPRepoPrivate(HTTPRepository* parent)
+      : p(parent)
+  {
  }

  ~HTTPRepoPrivate();

  HTTPRepository *p; // link back to outer
-  HTTP::Client *http;
+  HTTP::Client* http = nullptr;
  std::string baseUrl;
  SGPath basePath;
-  bool isUpdating;
-  HTTPRepository::ResultCode status;
+  bool isUpdating = false;
+  HTTPRepository::ResultCode status = HTTPRepository::REPO_NO_ERROR;
  HTTPDirectory_ptr rootDir;
-  size_t totalDownloaded;
+  size_t totalDownloaded = 0;
+  size_t bytesToExtract = 0;
+  size_t bytesExtracted = 0;
  HTTPRepository::SyncPredicate syncPredicate;

  HTTP::Request_ptr updateFile(HTTPDirectory *dir, const std::string &name,
--- a/simgear/io/sg_binobj.cxx
+++ b/simgear/io/sg_binobj.cxx
@@ -566,8 +566,10 @@ bool SGBinObject::read_bin( const SGPath& file ) {
    sgReadUInt( fp, &header );

    if (sgReadError()) {
+        int code = 0;
+        const char* gzErrorString = gzerror(fp, &code);
        gzclose(fp);
-        throw sg_io_exception("Unable to read BTG header: " + simgear::strutils::error_string(errno), sg_location(file));
+        throw sg_io_exception("Unable to read BTG header: " + string{gzErrorString} + ", code =" + std::to_string(code), sg_location(file));
    }

    if ( ((header & 0xFF000000) >> 24) == 'S' &&
--- a/simgear/io/sg_file.cxx
+++ b/simgear/io/sg_file.cxx
@@ -82,7 +82,9 @@ std::string SGFile::computeHash()
        [](char* p) { free(p); }};

    if (!buf) {
-        SG_LOG(SG_IO, SG_ALERT, "Failed to malloc buffer for SHA1 check");
+        // @TODO report out of memory error
+        SG_LOG(SG_IO, SG_ALERT, "Failed to malloc buffer for SHA1 check:" << file_name);
+        return {};
    }

    size_t readLen;
--- a/simgear/io/test.tar.xz
+++ b/simgear/io/test.tar.xz
--- a/simgear/io/test_repository.cxx
+++ b/simgear/io/test_repository.cxx
@@ -31,6 +31,10 @@ using TestApi = simgear::HTTP::TestApi;

 std::string dataForFile(const std::string& parentName, const std::string& name, int revision)
 {
+    if (name == "zeroByteFile") {
+        return {};
+    }
+
    std::ostringstream os;
    // random content but which definitely depends on our tree location
    // and revision.
@@ -446,6 +450,7 @@ void testBasicClone(HTTP::Client* cl)
    verifyFileState(p, "fileA");
    verifyFileState(p, "dirA/subdirA/fileAAA");
    verifyFileState(p, "dirC/subdirA/subsubA/fileCAAA");
+    verifyFileState(p, "dirA/subdirA/zeroByteFile");

    global_repo->findEntry("fileA")->revision++;
    global_repo->findEntry("dirB/subdirA/fileBAA")->revision++;
@@ -896,6 +901,27 @@ void testPersistentSocketFailure(HTTP::Client *cl) {
  verifyRequestCount("dirD/subdirDB/fileDBA", 1);
 }

+void testHashOnEmptyFile()
+{
+    std::unique_ptr<HTTPRepository> repo;
+    SGPath p(simgear::Dir::current().path());
+    p.append("sgfile_compute_hash");
+    simgear::Dir pd(p);
+    if (pd.exists()) {
+        pd.removeChildren();
+    } else {
+        pd.create(0700);
+    }
+
+    SGPath fPath = p / "test_empty_file";
+    
+    SGFile file(fPath);
+    file.open(SG_IO_OUT);
+    file.close();
+
+    const auto hash = file.computeHash();
+}
+
 int main(int argc, char* argv[])
 {
  sglog().setLogLevels( SG_ALL, SG_INFO );
@@ -911,6 +937,8 @@ int main(int argc, char* argv[])
    global_repo->defineFile("dirA/fileAC");
    global_repo->defineFile("dirA/subdirA/fileAAA");
    global_repo->defineFile("dirA/subdirA/fileAAB");
+    global_repo->defineFile("dirA/subdirA/zeroByteFile");
+
    global_repo->defineFile("dirB/subdirA/fileBAA");
    global_repo->defineFile("dirB/subdirA/fileBAB");
    global_repo->defineFile("dirB/subdirA/fileBAC");
@@ -944,6 +972,8 @@ int main(int argc, char* argv[])
    testRetryAfterSocketFailure(&cl);
    testPersistentSocketFailure(&cl);

+    testHashOnEmptyFile();
+
    std::cout << "all tests passed ok" << std::endl;
    return 0;
 }
--- a/simgear/io/test_untar.cxx
+++ b/simgear/io/test_untar.cxx
@@ -32,7 +32,7 @@ void testTarGz()
    uint8_t* buf = (uint8_t*) alloca(8192);
    size_t bufSize = f.read((char*) buf, 8192);

-    SG_VERIFY(ArchiveExtractor::determineType(buf, bufSize) == ArchiveExtractor::TarData);
+    SG_VERIFY(ArchiveExtractor::determineType(buf, bufSize) == ArchiveExtractor::GZData);

    f.close();
 }
@@ -174,6 +174,34 @@ void testPAXAttributes()

 }

+void testExtractXZ()
+{
+    SGPath p = SGPath(SRC_DIR);
+    p.append("test.tar.xz");
+
+    SGBinaryFile f(p);
+    f.open(SG_IO_IN);
+
+    SGPath extractDir = simgear::Dir::current().path() / "test_extract_xz";
+    simgear::Dir pd(extractDir);
+    pd.removeChildren();
+
+    ArchiveExtractor ex(extractDir);
+
+    uint8_t* buf = (uint8_t*)alloca(128);
+    while (!f.eof()) {
+        size_t bufSize = f.read((char*)buf, 128);
+        ex.extractBytes(buf, bufSize);
+    }
+
+    ex.flush();
+    SG_VERIFY(ex.isAtEndOfArchive());
+    SG_VERIFY(ex.hasError() == false);
+
+    SG_VERIFY((extractDir / "testDir/hello.c").exists());
+    SG_VERIFY((extractDir / "testDir/foo.txt").exists());
+}
+
 int main(int ac, char ** av)
 {
    testTarGz();
@@ -181,7 +209,8 @@ int main(int ac, char ** av)
    testFilterTar();
 	testExtractStreamed();
 	testExtractZip();
-   
+    testExtractXZ();
+
    // disabled to avoiding checking in large PAX archive
    // testPAXAttributes();
    
--- a/simgear/io/untar.cxx
+++ b/simgear/io/untar.cxx
@@ -38,82 +38,11 @@
 #include <simgear/package/unzip.h>
 #include <simgear/structure/exception.hxx>

+#include "ArchiveExtractor_private.hxx"
+
 namespace simgear
 {

-	class ArchiveExtractorPrivate
-	{
-	public:
-		ArchiveExtractorPrivate(ArchiveExtractor* o) :
-			outer(o)
-		{
-			assert(outer);
-		}
-
-        virtual ~ArchiveExtractorPrivate() = default;
-        
-		typedef enum {
-			INVALID = 0,
-			READING_HEADER,
-			READING_FILE,
-			READING_PADDING,
-            READING_PAX_GLOBAL_ATTRIBUTES,
-            READING_PAX_FILE_ATTRIBUTES,
-			PRE_END_OF_ARCHVE,
-			END_OF_ARCHIVE,
-			ERROR_STATE, ///< states above this are error conditions
-			BAD_ARCHIVE,
-			BAD_DATA,
-			FILTER_STOPPED
-		} State;
-
-		State state = INVALID;
-		ArchiveExtractor* outer = nullptr;
-
-		virtual void extractBytes(const uint8_t* bytes, size_t count) = 0;
-
-		virtual void flush() = 0;
-
-		SGPath extractRootPath()
-		{
-			return outer->_rootPath;
-		}
-
-		ArchiveExtractor::PathResult filterPath(std::string& pathToExtract)
-		{
-			return outer->filterPath(pathToExtract);
-		}
-
-
-		bool isSafePath(const std::string& p) const
-		{
-			if (p.empty()) {
-				return false;
-			}
-
-			// reject absolute paths
-			if (p.at(0) == '/') {
-				return false;
-			}
-
-			// reject paths containing '..'
-			size_t doubleDot = p.find("..");
-			if (doubleDot != std::string::npos) {
-				return false;
-			}
-
-			// on POSIX could use realpath to sanity check
-			return true;
-		}
-	};
-
-///////////////////////////////////////////////////////////////////////////////////////////////////
-
-const int ZLIB_DECOMPRESS_BUFFER_SIZE = 32 * 1024;
-const int ZLIB_INFLATE_WINDOW_BITS = MAX_WBITS;
-const int ZLIB_DECODE_GZIP_HEADER = 16;
-
-/* tar Header Block, from POSIX 1003.1-1990.  */

 typedef struct
 {
@@ -153,320 +82,410 @@ typedef struct
 #define FIFOTYPE '6'            /* FIFO special */
 #define CONTTYPE '7'            /* reserved */

-	const char PAX_GLOBAL_HEADER = 'g';
-	const char PAX_FILE_ATTRIBUTES = 'x';
+    const char PAX_GLOBAL_HEADER = 'g';
+    const char PAX_FILE_ATTRIBUTES = 'x';


-class TarExtractorPrivate : public ArchiveExtractorPrivate
+    ///////////////////////////////////////////////////////////////////////////////////////////////////
+
+    const int ZLIB_DECOMPRESS_BUFFER_SIZE = 32 * 1024;
+    const int ZLIB_INFLATE_WINDOW_BITS = MAX_WBITS;
+    const int ZLIB_DECODE_GZIP_HEADER = 16;
+
+    /* tar Header Block, from POSIX 1003.1-1990.  */
+
+
+    class TarExtractorPrivate : public ArchiveExtractorPrivate
+    {
+    public:
+        union {
+            UstarHeaderBlock header;
+            uint8_t headerBytes[TAR_HEADER_BLOCK_SIZE];
+        };
+
+        size_t bytesRemaining;
+        std::unique_ptr<SGFile> currentFile;
+        size_t currentFileSize;
+
+        uint8_t* headerPtr;
+        bool skipCurrentEntry = false;
+        std::string paxAttributes;
+        std::string paxPathName;
+
+        TarExtractorPrivate(ArchiveExtractor* o) : ArchiveExtractorPrivate(o)
+        {
+            setState(TarExtractorPrivate::READING_HEADER);
+        }
+
+        ~TarExtractorPrivate() = default;
+
+        void readPaddingIfRequired()
+        {
+            size_t pad = currentFileSize % TAR_HEADER_BLOCK_SIZE;
+            if (pad) {
+                bytesRemaining = TAR_HEADER_BLOCK_SIZE - pad;
+                setState(READING_PADDING);
+            } else {
+                setState(READING_HEADER);
+            }
+        }
+
+        void checkEndOfState()
+        {
+            if (bytesRemaining > 0) {
+                return;
+            }
+
+            if (state == READING_FILE) {
+                if (currentFile) {
+                    currentFile->close();
+                    currentFile.reset();
+                }
+                readPaddingIfRequired();
+            } else if (state == READING_HEADER) {
+                processHeader();
+            } else if (state == PRE_END_OF_ARCHVE) {
+                if (headerIsAllZeros()) {
+                    setState(END_OF_ARCHIVE);
+                } else {
+                    // what does the spec say here?
+                }
+            } else if (state == READING_PAX_GLOBAL_ATTRIBUTES) {
+                parsePAXAttributes(true);
+                readPaddingIfRequired();
+            } else if (state == READING_PAX_FILE_ATTRIBUTES) {
+                parsePAXAttributes(false);
+                readPaddingIfRequired();
+            } else if (state == READING_PADDING) {
+                setState(READING_HEADER);
+            }
+        }
+
+        void setState(State newState)
+        {
+            if ((newState == READING_HEADER) || (newState == PRE_END_OF_ARCHVE)) {
+                bytesRemaining = TAR_HEADER_BLOCK_SIZE;
+                headerPtr = headerBytes;
+            }
+
+            state = newState;
+        }
+
+        void extractBytes(const uint8_t* bytes, size_t count) override
+        {
+            // uncompressed, just pass through directly
+            processBytes((const char*)bytes, count);
+        }
+
+        void flush() override
+        {
+            // no-op for tar files, we process everything greedily
+        }
+
+        void processHeader()
+        {
+            if (headerIsAllZeros()) {
+                if (state == PRE_END_OF_ARCHVE) {
+                    setState(END_OF_ARCHIVE);
+                } else {
+                    setState(PRE_END_OF_ARCHVE);
+                }
+                return;
+            }
+
+            if (strncmp(header.magic, TMAGIC, TMAGLEN) != 0) {
+                SG_LOG(SG_IO, SG_WARN, "Untar: magic is wrong");
+                state = BAD_ARCHIVE;
+                return;
+            }
+
+            skipCurrentEntry = false;
+            std::string tarPath = std::string(header.prefix) + std::string(header.fileName);
+            if (!paxPathName.empty()) {
+                tarPath = paxPathName;
+                paxPathName.clear(); // clear for next file
+            }
+
+            if (!isSafePath(tarPath)) {
+                SG_LOG(SG_IO, SG_WARN, "unsafe tar path, skipping::" << tarPath);
+                skipCurrentEntry = true;
+            }
+
+            auto result = filterPath(tarPath);
+            if (result == ArchiveExtractor::Stop) {
+                setState(FILTER_STOPPED);
+                return;
+            } else if (result == ArchiveExtractor::Skipped) {
+                skipCurrentEntry = true;
+            }
+
+            SGPath p = extractRootPath() / tarPath;
+            if (header.typeflag == DIRTYPE) {
+                if (!skipCurrentEntry) {
+                    Dir dir(p);
+                    dir.create(0755);
+                }
+                setState(READING_HEADER);
+            } else if ((header.typeflag == REGTYPE) || (header.typeflag == AREGTYPE)) {
+                currentFileSize = ::strtol(header.size, NULL, 8);
+                bytesRemaining = currentFileSize;
+                if (!skipCurrentEntry) {
+                    currentFile.reset(new SGBinaryFile(p));
+                    currentFile->open(SG_IO_OUT);
+                }
+                setState(READING_FILE);
+            } else if (header.typeflag == PAX_GLOBAL_HEADER) {
+                setState(READING_PAX_GLOBAL_ATTRIBUTES);
+                currentFileSize = ::strtol(header.size, NULL, 8);
+                bytesRemaining = currentFileSize;
+                paxAttributes.clear();
+            } else if (header.typeflag == PAX_FILE_ATTRIBUTES) {
+                setState(READING_PAX_FILE_ATTRIBUTES);
+                currentFileSize = ::strtol(header.size, NULL, 8);
+                bytesRemaining = currentFileSize;
+                paxAttributes.clear();
+            } else if ((header.typeflag == SYMTYPE) || (header.typeflag == LNKTYPE)) {
+                SG_LOG(SG_IO, SG_WARN, "Tarball contains a link or symlink, will be skipped:" << tarPath);
+                skipCurrentEntry = true;
+                setState(READING_HEADER);
+            } else {
+                SG_LOG(SG_IO, SG_WARN, "Unsupported tar file type:" << header.typeflag);
+                state = BAD_ARCHIVE;
+            }
+        }
+
+        void processBytes(const char* bytes, size_t count)
+        {
+            if ((state >= ERROR_STATE) || (state == END_OF_ARCHIVE)) {
+                return;
+            }
+
+            size_t curBytes = std::min(bytesRemaining, count);
+            if (state == READING_FILE) {
+                if (currentFile) {
+                    currentFile->write(bytes, curBytes);
+                }
+                bytesRemaining -= curBytes;
+            } else if ((state == READING_HEADER) || (state == PRE_END_OF_ARCHVE) || (state == END_OF_ARCHIVE)) {
+                memcpy(headerPtr, bytes, curBytes);
+                bytesRemaining -= curBytes;
+                headerPtr += curBytes;
+            } else if (state == READING_PADDING) {
+                bytesRemaining -= curBytes;
+            } else if ((state == READING_PAX_FILE_ATTRIBUTES) || (state == READING_PAX_GLOBAL_ATTRIBUTES)) {
+                bytesRemaining -= curBytes;
+                paxAttributes.append(bytes, curBytes);
+            }
+
+            checkEndOfState();
+            if (count > curBytes) {
+                // recurse with unprocessed bytes
+                processBytes(bytes + curBytes, count - curBytes);
+            }
+        }
+
+        bool headerIsAllZeros() const
+        {
+            char* headerAsChar = (char*)&header;
+            for (size_t i = 0; i < offsetof(UstarHeaderBlock, magic); ++i) {
+                if (*headerAsChar++ != 0) {
+                    return false;
+                }
+            }
+
+            return true;
+        }
+
+        // https://www.ibm.com/support/knowledgecenter/en/SSLTBW_2.3.0/com.ibm.zos.v2r3.bpxa500/paxex.htm#paxex
+        void parsePAXAttributes(bool areGlobal)
+        {
+            auto lineStart = 0;
+            for (;;) {
+                auto firstSpace = paxAttributes.find(' ', lineStart);
+                auto firstEq = paxAttributes.find('=', lineStart);
+                if ((firstEq == std::string::npos) || (firstSpace == std::string::npos)) {
+                    SG_LOG(SG_IO, SG_WARN, "Malfroemd PAX attributes in tarfile");
+                    break;
+                }
+
+                uint32_t lengthBytes = std::stoul(paxAttributes.substr(lineStart, firstSpace));
+                uint32_t dataBytes = lengthBytes - (firstEq + 1);
+                std::string name = paxAttributes.substr(firstSpace + 1, firstEq - (firstSpace + 1));
+
+                // dataBytes - 1 here to trim off the trailing newline
+                std::string data = paxAttributes.substr(firstEq + 1, dataBytes - 1);
+
+                processPAXAttribute(areGlobal, name, data);
+
+                lineStart += lengthBytes;
+            }
+        }
+
+        void processPAXAttribute(bool isGlobalAttr, const std::string& attrName, const std::string& data)
+        {
+            if (!isGlobalAttr && (attrName == "path")) {
+                // data is UTF-8 encoded path name
+                paxPathName = data;
+            }
+        }
+};
+
+///////////////////////////////////////////////////////////////////////////////
+
+
+class GZTarExtractor : public TarExtractorPrivate
 {
 public:
-    
-    union {
-        UstarHeaderBlock header;
-        uint8_t headerBytes[TAR_HEADER_BLOCK_SIZE];
-    };
-
-    size_t bytesRemaining;
-    std::unique_ptr<SGFile> currentFile;
-    size_t currentFileSize;
-    z_stream zlibStream;
-    uint8_t* zlibOutput;
-    bool haveInitedZLib = false;
-    bool uncompressedData = false; // set if reading a plain .tar (not tar.gz)
-    uint8_t* headerPtr;
-    bool skipCurrentEntry = false;
-    std::string paxAttributes;
-    std::string paxPathName;
-    
-    TarExtractorPrivate(ArchiveExtractor* o) :
-		ArchiveExtractorPrivate(o)
+    GZTarExtractor(ArchiveExtractor* outer) : TarExtractorPrivate(outer)
    {
-		memset(&zlibStream, 0, sizeof(z_stream));
-		zlibOutput = (unsigned char*)malloc(ZLIB_DECOMPRESS_BUFFER_SIZE);
-		zlibStream.zalloc = Z_NULL;
-		zlibStream.zfree = Z_NULL;
-		zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE;
-		zlibStream.next_out = zlibOutput;
+        memset(&zlibStream, 0, sizeof(z_stream));
+        zlibOutput = (unsigned char*)malloc(ZLIB_DECOMPRESS_BUFFER_SIZE);
+        zlibStream.zalloc = Z_NULL;
+        zlibStream.zfree = Z_NULL;
+        zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE;
+        zlibStream.next_out = zlibOutput;
    }

-    ~TarExtractorPrivate()
+    ~GZTarExtractor()
    {
        free(zlibOutput);
    }

-    void readPaddingIfRequired()
+    void extractBytes(const uint8_t* bytes, size_t count) override
    {
-        size_t pad = currentFileSize % TAR_HEADER_BLOCK_SIZE;
-        if (pad) {
-            bytesRemaining = TAR_HEADER_BLOCK_SIZE - pad;
-            setState(READING_PADDING);
-        } else {
-            setState(READING_HEADER);
-        }
-    }
-    
-    void checkEndOfState()
-    {
-        if (bytesRemaining > 0) {
-            return;
-        }
+        zlibStream.next_in = (uint8_t*)bytes;
+        zlibStream.avail_in = count;

-        if (state == READING_FILE) {
-            if (currentFile) {
-                currentFile->close();
-                currentFile.reset();
-            }
-            readPaddingIfRequired();
-        } else if (state == READING_HEADER) {
-            processHeader();
-        } else if (state == PRE_END_OF_ARCHVE) {
-            if (headerIsAllZeros()) {
-                setState(END_OF_ARCHIVE);
+        if (!haveInitedZLib) {
+            // now we have data, see if we're dealing with GZ-compressed data or not
+            if ((bytes[0] == 0x1f) && (bytes[1] == 0x8b)) {
+                // GZIP identification bytes
+                if (inflateInit2(&zlibStream, ZLIB_INFLATE_WINDOW_BITS | ZLIB_DECODE_GZIP_HEADER) != Z_OK) {
+                    SG_LOG(SG_IO, SG_WARN, "inflateInit2 failed");
+                    state = TarExtractorPrivate::BAD_DATA;
+                    return;
+                }
            } else {
-                // what does the spec say here?
+                // set error state
+                state = TarExtractorPrivate::BAD_DATA;
+                return;
            }
-        } else if (state == READING_PAX_GLOBAL_ATTRIBUTES) {
-            parsePAXAttributes(true);
-            readPaddingIfRequired();
-        } else if (state == READING_PAX_FILE_ATTRIBUTES) {
-            parsePAXAttributes(false);
-            readPaddingIfRequired();
-        } else if (state == READING_PADDING) {
-            setState(READING_HEADER);
-        }
-    }

-    void setState(State newState)
-    {
-        if ((newState == READING_HEADER) || (newState == PRE_END_OF_ARCHVE)) {
-            bytesRemaining = TAR_HEADER_BLOCK_SIZE;
-            headerPtr = headerBytes;
-        }
+            haveInitedZLib = true;
+            setState(TarExtractorPrivate::READING_HEADER);
+        } // of init on first-bytes case

-        state = newState;
-    }
+        size_t writtenSize;
+        // loop, running zlib() inflate and sending output bytes to
+        // our request body handler. Keep calling inflate until no bytes are
+        // written, and ZLIB has consumed all available input
+        do {
+            zlibStream.next_out = zlibOutput;
+            zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE;
+            int result = inflate(&zlibStream, Z_NO_FLUSH);
+            if (result == Z_OK || result == Z_STREAM_END) {
+                // nothing to do

-	void extractBytes(const uint8_t* bytes, size_t count) override
-	{
-		zlibStream.next_in = (uint8_t*) bytes;
-		zlibStream.avail_in = count;
-
-		if (!haveInitedZLib) {
-			// now we have data, see if we're dealing with GZ-compressed data or not
-			if ((bytes[0] == 0x1f) && (bytes[1] == 0x8b)) {
-				// GZIP identification bytes
-				if (inflateInit2(&zlibStream, ZLIB_INFLATE_WINDOW_BITS | ZLIB_DECODE_GZIP_HEADER) != Z_OK) {
-					SG_LOG(SG_IO, SG_WARN, "inflateInit2 failed");
-					state = TarExtractorPrivate::BAD_DATA;
-					return;
-				}
-			} else {
-				UstarHeaderBlock* header = (UstarHeaderBlock*)bytes;
-				if (strncmp(header->magic, TMAGIC, TMAGLEN) != 0) {
-					SG_LOG(SG_IO, SG_WARN, "didn't find tar magic in header");
-					state = TarExtractorPrivate::BAD_DATA;
-					return;
-				}
-
-				uncompressedData = true;
-			}
-
-			haveInitedZLib = true;
-			setState(TarExtractorPrivate::READING_HEADER);
-		} // of init on first-bytes case
-
-		if (uncompressedData) {
-			processBytes((const char*) bytes, count);
-		} else {
-			size_t writtenSize;
-			// loop, running zlib() inflate and sending output bytes to
-			// our request body handler. Keep calling inflate until no bytes are
-			// written, and ZLIB has consumed all available input
-			do {
-				zlibStream.next_out = zlibOutput;
-				zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE;
-				int result = inflate(&zlibStream, Z_NO_FLUSH);
-				if (result == Z_OK || result == Z_STREAM_END) {
-					// nothing to do
-
-				}
-				else if (result == Z_BUF_ERROR) {
-					// transient error, fall through
-				}
-				else {
-					//  _error = result;
-					SG_LOG(SG_IO, SG_WARN, "Permanent ZLib error:" << zlibStream.msg);
-					state = TarExtractorPrivate::BAD_DATA;
-					return;
-				}
-
-				writtenSize = ZLIB_DECOMPRESS_BUFFER_SIZE - zlibStream.avail_out;
-				if (writtenSize > 0) {
-					processBytes((const char*) zlibOutput, writtenSize);
-				}
-
-				if (result == Z_STREAM_END) {
-					break;
-				}
-			} while ((zlibStream.avail_in > 0) || (writtenSize > 0));
-		} // of Zlib-compressed data
-	}
-
-	void flush() override
-	{
-		// no-op for tar files, we process everything greedily
-	}
-
-    void processHeader()
-    {
-        if (headerIsAllZeros()) {
-            if (state == PRE_END_OF_ARCHVE) {
-                setState(END_OF_ARCHIVE);
+            } else if (result == Z_BUF_ERROR) {
+                // transient error, fall through
            } else {
-                setState(PRE_END_OF_ARCHVE);
+                //  _error = result;
+                SG_LOG(SG_IO, SG_WARN, "Permanent ZLib error:" << zlibStream.msg);
+                state = TarExtractorPrivate::BAD_DATA;
+                return;
            }
-            return;
-        }

-        if (strncmp(header.magic, TMAGIC, TMAGLEN) != 0) {
-            SG_LOG(SG_IO, SG_WARN, "Untar: magic is wrong");
-            state = BAD_ARCHIVE;
-            return;
-        }
-
-        skipCurrentEntry = false;
-        std::string tarPath = std::string(header.prefix) + std::string(header.fileName);
-        if (!paxPathName.empty()) {
-            tarPath = paxPathName;
-            paxPathName.clear(); // clear for next file
-        }
-        
-        if (!isSafePath(tarPath)) {
-            SG_LOG(SG_IO, SG_WARN, "unsafe tar path, skipping::" << tarPath);
-            skipCurrentEntry = true;
-        }
-
-        auto result = filterPath(tarPath);
-        if (result == ArchiveExtractor::Stop) {
-            setState(FILTER_STOPPED);
-            return;
-        } else if (result == ArchiveExtractor::Skipped) {
-            skipCurrentEntry = true;
-        }
-        
-        SGPath p = extractRootPath() / tarPath;
-        if (header.typeflag == DIRTYPE) {
-            if (!skipCurrentEntry) {
-                Dir dir(p);
-                dir.create(0755);
+            writtenSize = ZLIB_DECOMPRESS_BUFFER_SIZE - zlibStream.avail_out;
+            if (writtenSize > 0) {
+                processBytes((const char*)zlibOutput, writtenSize);
            }
-            setState(READING_HEADER);
-        } else if ((header.typeflag == REGTYPE) || (header.typeflag == AREGTYPE)) {
-            currentFileSize = ::strtol(header.size, NULL, 8);
-            bytesRemaining = currentFileSize;
-            if (!skipCurrentEntry) {
-                currentFile.reset(new SGBinaryFile(p));
-                currentFile->open(SG_IO_OUT);
-            }
-            setState(READING_FILE);
-        } else if (header.typeflag == PAX_GLOBAL_HEADER) {
-            setState(READING_PAX_GLOBAL_ATTRIBUTES);
-            currentFileSize = ::strtol(header.size, NULL, 8);
-            bytesRemaining = currentFileSize;
-            paxAttributes.clear();
-        } else if (header.typeflag == PAX_FILE_ATTRIBUTES) {
-            setState(READING_PAX_FILE_ATTRIBUTES);
-            currentFileSize = ::strtol(header.size, NULL, 8);
-            bytesRemaining = currentFileSize;
-            paxAttributes.clear();
-        } else if ((header.typeflag == SYMTYPE) || (header.typeflag == LNKTYPE)) {
-            SG_LOG(SG_IO, SG_WARN, "Tarball contains a link or symlink, will be skipped:" << tarPath);
-            skipCurrentEntry = true;
-            setState(READING_HEADER);
-        } else {
-            SG_LOG(SG_IO, SG_WARN, "Unsupported tar file type:" << header.typeflag);
-            state = BAD_ARCHIVE;
-        }
-    }

-    void processBytes(const char* bytes, size_t count)
-    {
-        if ((state >= ERROR_STATE) || (state == END_OF_ARCHIVE)) {
-            return;
-        }
-
-        size_t curBytes = std::min(bytesRemaining, count);
-        if (state == READING_FILE) {
-            if (currentFile) {
-                currentFile->write(bytes, curBytes);
-            }
-            bytesRemaining -= curBytes;
-        } else if ((state == READING_HEADER) || (state == PRE_END_OF_ARCHVE) || (state == END_OF_ARCHIVE)) {
-            memcpy(headerPtr, bytes, curBytes);
-            bytesRemaining -= curBytes;
-            headerPtr += curBytes;
-        } else if (state == READING_PADDING) {
-            bytesRemaining -= curBytes;
-        } else if ((state == READING_PAX_FILE_ATTRIBUTES) || (state == READING_PAX_GLOBAL_ATTRIBUTES)) {
-            bytesRemaining -= curBytes;
-            paxAttributes.append(bytes, curBytes);
-        }
-
-        checkEndOfState();
-        if (count > curBytes) {
-            // recurse with unprocessed bytes
-            processBytes(bytes + curBytes, count - curBytes);
-        }
-    }
-
-    bool headerIsAllZeros() const
-    {
-        char* headerAsChar = (char*) &header;
-        for (size_t i=0; i < offsetof(UstarHeaderBlock, magic); ++i) {
-            if (*headerAsChar++ != 0) {
-                return false;
-            }
-        }
-
-        return true;
-    }
-
-    // https://www.ibm.com/support/knowledgecenter/en/SSLTBW_2.3.0/com.ibm.zos.v2r3.bpxa500/paxex.htm#paxex
-    void parsePAXAttributes(bool areGlobal)
-    {
-        auto lineStart = 0;
-        for (;;) {
-            auto firstSpace = paxAttributes.find(' ', lineStart);
-            auto firstEq = paxAttributes.find('=', lineStart);
-            if ((firstEq == std::string::npos) || (firstSpace == std::string::npos)) {
-                SG_LOG(SG_IO, SG_WARN, "Malfroemd PAX attributes in tarfile");
+            if (result == Z_STREAM_END) {
                break;
            }
-            
-            uint32_t lengthBytes = std::stoul(paxAttributes.substr(lineStart, firstSpace));
-            uint32_t dataBytes = lengthBytes - (firstEq + 1);
-            std::string name = paxAttributes.substr(firstSpace+1, firstEq - (firstSpace + 1));
-            
-            // dataBytes - 1 here to trim off the trailing newline
-            std::string data = paxAttributes.substr(firstEq+1, dataBytes - 1);
-            
-            processPAXAttribute(areGlobal, name, data);
-            
-            lineStart += lengthBytes;
-        }
-    }
-    
-    void processPAXAttribute(bool isGlobalAttr, const std::string& attrName, const std::string& data)
-    {
-        if (!isGlobalAttr && (attrName == "path")) {
-            // data is UTF-8 encoded path name
-            paxPathName = data;
-        }
+        } while ((zlibStream.avail_in > 0) || (writtenSize > 0));
    }
+
+private:
+    z_stream zlibStream;
+    uint8_t* zlibOutput;
+    bool haveInitedZLib = false;
 };

 ///////////////////////////////////////////////////////////////////////////////

+#if 1 || defined(HAVE_XZ)
+
+#include <lzma.h>
+
+class XZTarExtractor : public TarExtractorPrivate
+{
+public:
+    XZTarExtractor(ArchiveExtractor* outer) : TarExtractorPrivate(outer)
+    {
+        _xzStream = LZMA_STREAM_INIT;
+        _outputBuffer = (uint8_t*)malloc(ZLIB_DECOMPRESS_BUFFER_SIZE);
+
+
+        auto ret = lzma_stream_decoder(&_xzStream, UINT64_MAX, LZMA_TELL_ANY_CHECK);
+        if (ret != LZMA_OK) {
+            setState(BAD_ARCHIVE);
+            return;
+        }
+    }
+
+    ~XZTarExtractor()
+    {
+        free(_outputBuffer);
+    }
+
+    void extractBytes(const uint8_t* bytes, size_t count) override
+    {
+        lzma_action action = LZMA_RUN;
+        _xzStream.next_in = bytes;
+        _xzStream.avail_in = count;
+
+        size_t writtenSize;
+        do {
+            _xzStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE;
+            _xzStream.next_out = _outputBuffer;
+
+            const auto ret = lzma_code(&_xzStream, action);
+
+            writtenSize = ZLIB_DECOMPRESS_BUFFER_SIZE - _xzStream.avail_out;
+            if (writtenSize > 0) {
+                processBytes((const char*)_outputBuffer, writtenSize);
+            }
+
+            if (ret == LZMA_GET_CHECK) {
+                //
+            } else if (ret == LZMA_STREAM_END) {
+                setState(END_OF_ARCHIVE);
+                break;
+            } else if (ret != LZMA_OK) {
+                setState(BAD_ARCHIVE);
+                break;
+            }
+        } while ((_xzStream.avail_in > 0) || (writtenSize > 0));
+    }
+
+    void flush() override
+    {
+        const auto ret = lzma_code(&_xzStream, LZMA_FINISH);
+        if (ret != LZMA_STREAM_END) {
+            setState(BAD_ARCHIVE);
+        }
+    }
+
+private:
+    lzma_stream _xzStream;
+    uint8_t* _outputBuffer = nullptr;
+};
+
+#endif
+
+///////////////////////////////////////////////////////////////////////////////
+
 extern "C" {
 	void fill_memory_filefunc(zlib_filefunc_def*);
 }
@@ -511,32 +530,34 @@ public:
 		const size_t BUFFER_SIZE = 1024 * 1024;
 		void* buf = malloc(BUFFER_SIZE);

-		try {
-			int result = unzGoToFirstFile(zip);
-			if (result != UNZ_OK) {
-				throw sg_exception("failed to go to first file in archive");
-			}
-
-			while (true) {
-				extractCurrentFile(zip, (char*)buf, BUFFER_SIZE);
-				if (state == FILTER_STOPPED) {
-					break;
-				}
-
-				result = unzGoToNextFile(zip);
-				if (result == UNZ_END_OF_LIST_OF_FILE) {
-					break;
-				}
-				else if (result != UNZ_OK) {
-					throw sg_io_exception("failed to go to next file in the archive");
-				}
-			}
-			state = END_OF_ARCHIVE;
-		}
-		catch (sg_exception&) {
+        int result = unzGoToFirstFile(zip);
+        if (result != UNZ_OK) {
+            SG_LOG(SG_IO, SG_ALERT, outer->rootPath() << "failed to go to first file in archive:" << result);
            state = BAD_ARCHIVE;
+            free(buf);
+            unzClose(zip);
+            return;
        }
-        
+
+        while (true) {
+            extractCurrentFile(zip, (char*)buf, BUFFER_SIZE);
+            if (state == FILTER_STOPPED) {
+                state = END_OF_ARCHIVE;
+                break;
+            }
+
+            result = unzGoToNextFile(zip);
+            if (result == UNZ_END_OF_LIST_OF_FILE) {
+                state = END_OF_ARCHIVE;
+                break;
+            } else if (result != UNZ_OK) {
+                SG_LOG(SG_IO, SG_ALERT, outer->rootPath() << "failed to go to next file in archive:" << result);
+                state = BAD_ARCHIVE;
+                break;
+            }
+        }
+
+
        free(buf);
        unzClose(zip);
    }
@@ -635,17 +656,19 @@ void ArchiveExtractor::extractBytes(const uint8_t* bytes, size_t count)

 		if (r == TarData) {
 			d.reset(new TarExtractorPrivate(this));
-		}
-		else if (r == ZipData) {
-			d.reset(new ZipExtractorPrivate(this));
-		}
-		else {
-			SG_LOG(SG_IO, SG_WARN, "Invalid archive type");
+        } else if (r == GZData) {
+            d.reset(new GZTarExtractor(this));
+        } else if (r == XZData) {
+            d.reset(new XZTarExtractor(this));
+        } else if (r == ZipData) {
+            d.reset(new ZipExtractorPrivate(this));
+        } else {
+            SG_LOG(SG_IO, SG_WARN, "Invalid archive type");
 			_invalidDataType = true;
 			return;
-		}
+        }

-		// if hit here, we created the extractor. Feed the prefbuffer
+        // if hit here, we created the extractor. Feed the prefbuffer
 		// bytes through it
 		d->extractBytes((uint8_t*) _prebuffer.data(), _prebuffer.size());
 		_prebuffer.clear();
@@ -697,9 +720,18 @@ ArchiveExtractor::DetermineResult ArchiveExtractor::determineType(const uint8_t*
 		return ZipData;
 	}

-	auto r = isTarData(bytes, count);
-	if ((r == TarData) || (r == InsufficientData))
-		return r;
+    if (count < 6) {
+        return InsufficientData;
+    }
+
+    const uint8_t XZ_HEADER[6] = {0xFD, '7', 'z', 'X', 'Z', 0x00};
+    if (memcmp(bytes, XZ_HEADER, 6) == 0) {
+        return XZData;
+    }
+
+    auto r = isTarData(bytes, count);
+    if ((r == TarData) || (r == InsufficientData) || (r == GZData))
+        return r;

 	return Invalid;
 }
@@ -712,6 +744,8 @@ ArchiveExtractor::DetermineResult ArchiveExtractor::isTarData(const uint8_t* byt
    }

    UstarHeaderBlock* header = 0;
+    DetermineResult result = InsufficientData;
+
    if ((bytes[0] == 0x1f) && (bytes[1] == 0x8b)) {
        // GZIP identification bytes
        z_stream z;
@@ -729,11 +763,11 @@ ArchiveExtractor::DetermineResult ArchiveExtractor::isTarData(const uint8_t* byt
            return Invalid;
        }

-        int result = inflate(&z, Z_SYNC_FLUSH);
-        if ((result == Z_OK) || (result == Z_STREAM_END)) {
+        int zResult = inflate(&z, Z_SYNC_FLUSH);
+        if ((zResult == Z_OK) || (zResult == Z_STREAM_END)) {
            // all good
        } else {
-            SG_LOG(SG_IO, SG_WARN, "isTarData: Zlib inflate failed:" << result);
+            SG_LOG(SG_IO, SG_WARN, "isTarData: Zlib inflate failed:" << zResult);
            inflateEnd(&z);
            return Invalid; // not tar data
        }
@@ -746,6 +780,7 @@ ArchiveExtractor::DetermineResult ArchiveExtractor::isTarData(const uint8_t* byt

        header = reinterpret_cast<UstarHeaderBlock*>(zlibOutput);
        inflateEnd(&z);
+        result = GZData;
    } else {
        // uncompressed tar
        if (count < TAR_HEADER_BLOCK_SIZE) {
@@ -753,13 +788,14 @@ ArchiveExtractor::DetermineResult ArchiveExtractor::isTarData(const uint8_t* byt
        }

        header = (UstarHeaderBlock*) bytes;
+        result = TarData;
    }

    if (strncmp(header->magic, TMAGIC, TMAGLEN) != 0) {
        return Invalid;
    }

-    return TarData;
+    return result;
 }

 void ArchiveExtractor::extractLocalFile(const SGPath& archiveFile)
--- a/simgear/io/untar.hxx
+++ b/simgear/io/untar.hxx
@@ -36,15 +36,16 @@ public:
 	ArchiveExtractor(const SGPath& rootPath);
 	virtual ~ArchiveExtractor();

-	enum DetermineResult
-	{
-		Invalid,
-		InsufficientData,
-		TarData,
-		ZipData
-	};
+    enum DetermineResult {
+        Invalid,
+        InsufficientData,
+        TarData,
+        ZipData,
+        GZData, // Gzipped-tar
+        XZData  // XZ (aka LZMA) tar
+    };

-	static DetermineResult determineType(const uint8_t* bytes, size_t count);
+    static DetermineResult determineType(const uint8_t* bytes, size_t count);

 	/**
 	 * @brief API to extract a local zip or tar.gz 
@@ -70,6 +71,11 @@ public:
 		Stop
 	};

+    SGPath rootPath() const
+    {
+        return _rootPath;
+    }
+
 protected:


--- a/simgear/math/SGGeod.hxx
+++ b/simgear/math/SGGeod.hxx
@@ -29,6 +29,13 @@ public:
  /// Default constructor, initializes the instance to lat = lon = elev = 0
  SGGeod(void);

+  /**
+    return an SGGeod for which isValid() returns false.
+    This is necessaerby becuase for historical reasons, ther defaulrt constructor above initialsies to zero,zero,zero
+    which *is*
+   */
+  static SGGeod invalid();
+
  /// Factory from angular values in radians and elevation is 0
  static SGGeod fromRad(double lon, double lat);
  /// Factory from angular values in degrees and elevation is 0
--- a/simgear/math/SGGeodesy.cxx
+++ b/simgear/math/SGGeodesy.cxx
@@ -663,3 +663,8 @@ SGGeodesy::radialIntersection(const SGGeod& a, double aRadial,
  result = SGGeod::fromGeoc(r);
  return true;
 }
+
+SGGeod SGGeod::invalid()
+{
+    return SGGeod::fromDeg(-999.9, -999.0);
+}
--- a/simgear/nasal/code.c
+++ b/simgear/nasal/code.c
@@ -248,7 +248,7 @@ void naFreeContext(naContext c)
    // than I have right now. So instead I'm clearing the stack tops here, so
    // a freed context looks the same as a new one returned by initContext.

-    c->fTop = c->opTop = c->markTop = 0;
+    c->fTop = c->opTop = c->markTop = c->ntemps = 0;

    c->nextFree = globals->freeContexts;
    globals->freeContexts = c;
--- a/simgear/nasal/cppbind/NasalEmesaryInterface.hxx
+++ b/simgear/nasal/cppbind/NasalEmesaryInterface.hxx
@@ -63,7 +63,7 @@ namespace nasal

    class NasalMainLoopRecipient : public simgear::Emesary::IReceiver {
    public:
-        NasalMainLoopRecipient() : receiveCount(0) {
+        NasalMainLoopRecipient() : receiveCount(0), Active(false), CanWait(false) {
            simgear::Emesary::GlobalTransmitter::instance()->Register(*this);
        }
        virtual ~NasalMainLoopRecipient() {
--- a/simgear/nasal/misc.c
+++ b/simgear/nasal/misc.c
@@ -100,7 +100,14 @@ naRef naNewHash(struct Context* c)

 naRef naNewCode(struct Context* c)
 {
-    return naNew(c, T_CODE);
+    naRef r = naNew(c, T_CODE);
+    // naNew can return a previously used naCode. naCodeGen will init
+    // all these members but a GC can occur inside naCodeGen, so we see
+    // partially initalized state here. To avoid this, clear out the values
+    // which mark() cares about.
+    PTR(r).code->srcFile = naNil();
+    PTR(r).code->nConstants = 0;
+    return r;
 }

 naRef naNewCCode(struct Context* c, naCFunction fptr)
--- a/simgear/package/Root.cxx
+++ b/simgear/package/Root.cxx
@@ -133,7 +133,9 @@ public:

    void fireRefreshStatus(CatalogRef catalog, Delegate::StatusCode status)
    {
-        for (auto d : delegates) {
+        // take a copy of delegates since firing this can modify the data
+        const auto currentDelegates = delegates;
+        for (auto d : currentDelegates) {
            d->catalogRefreshed(catalog, status);
        }
    }
@@ -722,12 +724,6 @@ void Root::catalogRefreshStatus(CatalogRef aCat, Delegate::StatusCode aReason)
    auto catIt = d->catalogs.find(aCat->id());
    d->fireRefreshStatus(aCat, aReason);

-    if (aReason == Delegate::STATUS_IN_PROGRESS) {
-        d->refreshing.insert(aCat);
-    } else {
-        d->refreshing.erase(aCat);
-    }
-
    if (aCat->isUserEnabled() &&
        (aReason == Delegate::STATUS_REFRESHED) && 
        (catIt == d->catalogs.end())) 
@@ -761,6 +757,17 @@ void Root::catalogRefreshStatus(CatalogRef aCat, Delegate::StatusCode aReason)
        }
    } // of catalog is disabled

+    // remove from refreshing /after/ checking for enable / disabled, since for
+    // new catalogs, the reference in d->refreshing might be our /only/
+    // reference to the catalog. Once the refresh is done (either failed or
+    // succeeded) the Catalog will be in either d->catalogs or
+    // d->disabledCatalogs
+    if (aReason == Delegate::STATUS_IN_PROGRESS) {
+      d->refreshing.insert(aCat);
+    } else {
+      d->refreshing.erase(aCat);
+    }
+
    if (d->refreshing.empty()) {
        d->fireRefreshStatus(CatalogRef(), Delegate::STATUS_REFRESHED);
        d->firePackagesChanged();
--- a/simgear/props/props.cxx
+++ b/simgear/props/props.cxx
@@ -19,6 +19,10 @@
 #include <sstream>
 #include <iomanip>
 #include <iterator>
+#include <exception> // can't use sg_exception becuase of PROPS_STANDALONE
+#include <mutex>
+#include <thread>
+
 #include <stdio.h>
 #include <string.h>

@@ -51,11 +55,15 @@ using namespace simgear;

 struct SGPropertyNodeListeners
 {
+  /* Protect _num_iterators and _items. We use a recursive mutex to allow
+  nested access to work as normal. */
+  std::recursive_mutex  _rmutex;
+
  /* This keeps a count of the current number of nested invocations of
  forEachListener(). If non-zero, other code higher up the stack is iterating
  _items[] so for example code must not erase items in the vector. */
  int _num_iterators = 0;
-  
+
  std::vector<SGPropertyChangeListener *> _items;
 };

@@ -558,23 +566,15 @@ find_node (SGPropertyNode * current,
  }
 }
 #else
-template<typename Range>
-SGPropertyNode*
-find_node (SGPropertyNode * current,
-           const Range& path,
-           bool create,
-           int last_index = -1)
-{
+template <typename Range>
+SGPropertyNode *find_node(SGPropertyNode *current, const Range &path,
+                          bool create, int last_index = -1) {
  using namespace boost;
-  typedef split_iterator<typename range_result_iterator<Range>::type>
-    PathSplitIterator;
-
-  PathSplitIterator itr
-    = make_split_iterator(path, first_finder("/", is_equal()));
+  auto itr = make_split_iterator(path, first_finder("/", is_equal()));
  if (*path.begin() == '/')
    return find_node_aux(current->getRootNode(), itr, create, last_index);
-   else
-     return find_node_aux(current, itr, create, last_index);
+  else
+    return find_node_aux(current, itr, create, last_index);
 }
 #endif

@@ -2406,6 +2406,7 @@ SGPropertyNode::addChangeListener (SGPropertyChangeListener * listener,
  if (_listeners == 0)
    _listeners = new SGPropertyNodeListeners;

+  std::lock_guard<std::recursive_mutex> lock(_listeners->_rmutex);
  /* If there's a nullptr entry (a listener that was unregistered), we
  overwrite it. This ensures that listeners that routinely unregister+register
  themselves don't make _listeners->_items grow unnecessarily. Otherwise simply
@@ -2429,9 +2430,13 @@ SGPropertyNode::removeChangeListener (SGPropertyChangeListener * listener)
 {
  if (_listeners == 0)
    return;
+  /* We use a std::unique_lock rather than a std::lock_guard because we may
+  need to unlock early. */
+  std::unique_lock<std::recursive_mutex> lock(_listeners->_rmutex);
  vector<SGPropertyChangeListener*>::iterator it =
    find(_listeners->_items.begin(), _listeners->_items.end(), listener);
  if (it != _listeners->_items.end()) {
+    assert(_listeners->_num_iterators >= 0);
    if (_listeners->_num_iterators) {
      /* _listeners._items is currently being iterated further up the stack in
      this thread by one or more nested invocations of forEachListener(), so
@@ -2450,6 +2455,7 @@ SGPropertyNode::removeChangeListener (SGPropertyChangeListener * listener)
      _listeners->_items.erase(it);
      listener->unregister_property(this);
      if (_listeners->_items.empty()) {
+        lock.unlock();
        delete _listeners;
        _listeners = 0;
      }
@@ -2511,9 +2517,11 @@ static void forEachListener(
    )
 {
  if (!_listeners) return;
-  
+
+  std::lock_guard<std::recursive_mutex> lock(_listeners->_rmutex);
+  assert(_listeners->_num_iterators >= 0);
  _listeners->_num_iterators += 1;
-  
+
  /* We need to use an index here when iterating _listeners->_items, not an
  iterator. This is because a listener may add new listeners, causing the
  vector to be reallocated, which would invalidate any iterator. */
@@ -2528,10 +2536,12 @@ static void forEachListener(
      }
    }
  }
-  
+
  _listeners->_num_iterators -= 1;
-  
+  assert(_listeners->_num_iterators >= 0);
+
  if (_listeners->_num_iterators == 0) {
+
    /* Remove any items that have been set to nullptr. */
    _listeners->_items.erase(
        std::remove(_listeners->_items.begin(), _listeners->_items.end(), (SGPropertyChangeListener*) nullptr),
@@ -2547,6 +2557,7 @@ static void forEachListener(
 int SGPropertyNode::nListeners() const
 {
  if (!_listeners) return 0;
+  std::lock_guard<std::recursive_mutex> lock(_listeners->_rmutex);
  int   n = 0;
  for (auto listener: _listeners->_items) {
    if (listener)   n += 1;
--- a/simgear/scene/material/Effect.cxx
+++ b/simgear/scene/material/Effect.cxx
@@ -76,7 +76,7 @@
 #include <simgear/structure/SGExpression.hxx>
 #include <simgear/props/props_io.hxx>
 #include <simgear/props/vectorPropTemplates.hxx>
-
+#include <simgear/debug/ErrorReportingCallback.hxx>
 #include <simgear/io/iostreams/sgstream.hxx>

 namespace simgear
@@ -928,7 +928,8 @@ void ShaderProgramBuilder::buildAttribute(Effect* effect, Pass* pass,
        // FIXME orig: const string& shaderName = shaderKey.first;
        string shaderName = shaderKey.first;
        Shader::Type stype = (Shader::Type)shaderKey.second;
-        if (getPropertyRoot()->getBoolValue("/sim/version/compositor-support", false) &&
+        const bool compositorEnabled = getPropertyRoot()->getBoolValue("/sim/version/compositor-support", false);
+        if (compositorEnabled &&
            shaderName.substr(0, shaderName.find("/")) == "Shaders") {
            shaderName = "Compositor/" + shaderName;
        }
@@ -936,7 +937,9 @@ void ShaderProgramBuilder::buildAttribute(Effect* effect, Pass* pass,
        if (fileName.empty())
        {
            SG_LOG(SG_INPUT, SG_ALERT, "Could not locate shader" << shaderName);
-
+            if (!compositorEnabled) {
+                reportError("Missing shader", shaderName);
+            }

            throw BuilderException(string("couldn't find shader ") +
                shaderName);
@@ -1481,10 +1484,8 @@ void mergeSchemesFallbacks(Effect *effect, const SGReaderWriterOptions *options)

 // Walk the techniques property tree, building techniques and
 // passes.
-static std::mutex  realizeTechniques_lock;
 bool Effect::realizeTechniques(const SGReaderWriterOptions* options)
 {
-    std::lock_guard<std::mutex> g(realizeTechniques_lock);
    if (getPropertyRoot()->getBoolValue("/sim/version/compositor-support", false))
        mergeSchemesFallbacks(this, options);

--- a/simgear/scene/material/TextureBuilder.cxx
+++ b/simgear/scene/material/TextureBuilder.cxx
@@ -273,11 +273,19 @@ bool setAttrs(const TexTuple& attrs, Texture* tex,
        options->setLoadOriginHint(SGReaderWriterOptions::LoadOriginHint::ORIGIN_EFFECTS_NORMALIZED);
    else
        options->setLoadOriginHint(SGReaderWriterOptions::LoadOriginHint::ORIGIN_EFFECTS);
-#if OSG_VERSION_LESS_THAN(3,4,2)
-    result = osgDB::readImageFile(imageName, options);
-#else
-    result = osgDB::readRefImageFile(imageName, options);
-#endif
+
+    try {
+    #if OSG_VERSION_LESS_THAN(3,4,2)
+        result = osgDB::readImageFile(imageName, options);
+    #else
+        result = osgDB::readRefImageFile(imageName, options);
+    #endif
+    } catch (std::bad_alloc& ba) {
+        SG_LOG(SG_GL, SG_ALERT, "Bad allocation loading:" << imageName);
+        // todo: report low memory warning
+        return false;
+    }
+
    options->setLoadOriginHint(origLOH);
    osg::ref_ptr<osg::Image> image;
    if (result.success())
--- a/simgear/scene/model/ModelRegistry.cxx
+++ b/simgear/scene/model/ModelRegistry.cxx
@@ -393,6 +393,10 @@ ModelRegistry::readImage(const string& fileName,
                        isEffect = true;
 //                        can_compress = false;
                    }
+                    else if (sgoptC && !transparent && sgoptC->getLoadOriginHint() == SGReaderWriterOptions::LoadOriginHint::ORIGIN_CANVAS) {
+                        SG_LOG(SG_IO, SG_INFO, "From Canvas " + absFileName + " will generate mipmap only");
+                        can_compress = false;
+                    }
                    if (can_compress)
                    {
                        std::string pot_message;
--- a/simgear/scene/model/SGText.cxx
+++ b/simgear/scene/model/SGText.cxx
@@ -80,7 +80,7 @@ void SGText::UpdateCallback::operator()(osg::Node * node, osg::NodeVisitor *nv )
    // be lazy and set the text only if the property has changed.
    // update() computes the glyph representation which looks 
    // more expensive than a the above string compare.
-    text->setText( buf );
+    text->setText( buf, osgText::String::ENCODING_UTF8 );
    text->update();
  }
  traverse( node, nv );
--- a/simgear/scene/model/animation.cxx
+++ b/simgear/scene/model/animation.cxx
@@ -1135,16 +1135,8 @@ void SpinAnimCallback::operator()(osg::Node* node, osg::NodeVisitor* nv)
        double intPart;
        double rot = modf(rotation, &intPart);
        double angle = rot * 2.0 * osg::PI;
-        const SGVec3d& sgcenter = transform->getCenter();
-        const SGVec3d& sgaxis = transform->getAxis();
-        Matrixd mat = Matrixd::translate(-sgcenter[0], -sgcenter[1], -sgcenter[2])
-            * Matrixd::rotate(angle, sgaxis[0], sgaxis[1], sgaxis[2])
-            * Matrixd::translate(sgcenter[0], sgcenter[1], sgcenter[2])
-            * *cv->getModelViewMatrix();
-        ref_ptr<RefMatrix> refmat = new RefMatrix(mat);
-        cv->pushModelViewMatrix(refmat.get(), transform->getReferenceFrame());
+        transform->setAngleRad(angle);
        traverse(transform, nv);
-        cv->popModelViewMatrix();
    } else {
        traverse(transform, nv);
    }
--- a/simgear/scene/model/particles.cxx
+++ b/simgear/scene/model/particles.cxx
@@ -36,10 +36,10 @@
 #include <osgParticle/MultiSegmentPlacer>
 #include <osgParticle/SectorPlacer>
 #include <osgParticle/ConstantRateCounter>
-#include <osgParticle/ParticleSystemUpdater>
 #include <osgParticle/ParticleSystem>
 #include <osgParticle/FluidProgram>

+#include <osgUtil/CullVisitor>
 #include <osg/Geode>
 #include <osg/Group>
 #include <osg/MatrixTransform>
@@ -48,67 +48,249 @@

 #include <simgear/scene/model/animation.hxx>

+using ParticleSystemRef = osg::ref_ptr<osgParticle::ParticleSystem>;
+
+
 namespace simgear
 {

 class ParticlesGlobalManager::ParticlesGlobalManagerPrivate : public osg::NodeCallback
 {
 public:
-    ParticlesGlobalManagerPrivate() : _updater(new osgParticle::ParticleSystemUpdater),
-                                      _commonGeode(new osg::Geode)
+    ParticlesGlobalManagerPrivate();
+
+    void operator()(osg::Node* node, osg::NodeVisitor* nv) override;
+
+    // only call this with the lock held!
+    osg::Group* internalGetCommonRoot();
+
+    void updateParticleSystemsFromCullCallback(int currentFrameNumber, osg::NodeVisitor* nv);
+
+    void addParticleSystem(osgParticle::ParticleSystem* ps, const osg::ref_ptr<osg::Group>& frame);
+
+    void registerNewLocalParticleSystem(osg::Node* node, ParticleSystemRef ps);
+    void registerNewWorldParticleSystem(osg::Node* node, ParticleSystemRef ps, osg::Group* frame);
+
+    std::mutex _lock;
+    bool _frozen = false;
+    double _simulationDt = 0.0;
+    osg::ref_ptr<osg::Group> _commonRoot;
+    osg::ref_ptr<osg::Geode> _commonGeode;
+    osg::Vec3 _wind;
+    bool _enabled = true;
+    osg::Vec3 _gravity;
+    //  osg::Vec3 _localWind;
+    SGGeod _currentPosition;
+
+    SGConstPropertyNode_ptr _enabledNode;
+
+    using ParticleSystemWeakRef = osg::observer_ptr<osgParticle::ParticleSystem>;
+    using ParticleSystemsWeakRefVec = std::vector<ParticleSystemWeakRef>;
+
+    using ParticleSystemsStrongRefVec = std::vector<ParticleSystemRef>;
+
+    ParticleSystemsWeakRefVec _systems;
+    osg::ref_ptr<ParticlesGlobalManager::UpdaterCallback> _cullCallback;
+
+    using GroupRefVec = std::vector<osg::ref_ptr<osg::Group>>;
+    GroupRefVec _newWorldParticles;
+};
+
+/**
+ @brief this class replaces the need to use osgParticle::ParticleSystemUpdater, which has some
+ thread-safety and ownership complications in our us case
+ */
+class ParticlesGlobalManager::UpdaterCallback : public osg::NodeCallback
+{
+public:
+    UpdaterCallback(ParticlesGlobalManagerPrivate* p) : _manager(p)
    {
    }

    void operator()(osg::Node* node, osg::NodeVisitor* nv) override
    {
-        std::lock_guard<std::mutex> g(_lock);
-        _enabled = !_enabledNode || _enabledNode->getBoolValue();
-
-        if (!_enabled)
-            return;
-
-        const auto q = SGQuatd::fromLonLatDeg(_longitudeNode->getFloatValue(), _latitudeNode->getFloatValue());
-        osg::Matrix om(toOsg(q));
-        osg::Vec3 v(0, 0, 9.81);
-
-        _gravity = om.preMult(v);
-
-        // NOTE: THIS WIND COMPUTATION DOESN'T SEEM TO AFFECT PARTICLES
-        // const osg::Vec3& zUpWind = _wind;
-        // osg::Vec3 w(zUpWind.y(), zUpWind.x(), -zUpWind.z());
-        // _localWind = om.preMult(w);
-    }
-
-    // only call this with the lock held!
-    osg::Group* internalGetCommonRoot()
-    {
-        if (!_commonRoot.valid()) {
-            SG_LOG(SG_PARTICLES, SG_DEBUG, "Particle common root called.");
-            _commonRoot = new osg::Group;
-            _commonRoot->setName("common particle system root");
-            _commonGeode->setName("common particle system geode");
-            _commonRoot->addChild(_commonGeode);
-            _commonRoot->addChild(_updater);
-            _commonRoot->setNodeMask(~simgear::MODELLIGHT_BIT);
+        osgUtil::CullVisitor* cv = dynamic_cast<osgUtil::CullVisitor*>(nv);
+        if (cv && nv->getFrameStamp()) {
+            if (_frameNumber < nv->getFrameStamp()->getFrameNumber()) {
+                _frameNumber = nv->getFrameStamp()->getFrameNumber();
+                _manager->updateParticleSystemsFromCullCallback(_frameNumber, nv);
+            }
        }
-        return _commonRoot.get();
+
+        // note, callback is responsible for scenegraph traversal so
+        // they must call traverse(node,nv) to ensure that the
+        // scene graph subtree (and associated callbacks) are traversed.
+        traverse(node, nv);
    }

-    std::mutex _lock;
-    bool _frozen = false;
-    osg::ref_ptr<osgParticle::ParticleSystemUpdater> _updater;
-    osg::ref_ptr<osg::Group> _commonRoot;
-    osg::ref_ptr<osg::Geode> _commonGeode;
-    osg::Vec3 _wind;
-    bool _globalCallbackRegistered = false;
-    bool _enabled = true;
-    osg::Vec3 _gravity;
-    //  osg::Vec3 _localWind;
-
-    SGConstPropertyNode_ptr _enabledNode;
-    SGConstPropertyNode_ptr _longitudeNode, _latitudeNode;
+    unsigned int _frameNumber = 0;
+    ParticlesGlobalManagerPrivate* _manager;
 };

+/**
+  single-shot node callback, used to register a particle system with the global manager. Once run, removes
+  itself. We used this to avoid updating a particle system until the load is complete and merged into the
+  main scene.
+ */
+class ParticlesGlobalManager::RegistrationCallback : public osg::NodeCallback
+{
+public:
+    void operator()(osg::Node* node, osg::NodeVisitor* nv) override
+    {
+        auto d = ParticlesGlobalManager::instance()->d;
+        d->addParticleSystem(_system, _frame);
+        node->removeUpdateCallback(this); // suicide
+    }
+
+    ParticleSystemRef _system;
+    osg::ref_ptr<osg::Group> _frame;
+};
+
+ParticlesGlobalManager::ParticlesGlobalManagerPrivate::ParticlesGlobalManagerPrivate() : _commonGeode(new osg::Geode),
+                                                                                         _cullCallback(new UpdaterCallback(this))
+{
+    // callbacks are registered in initFromMainThread : depending on timing,
+    // this constructor might be called from an osgDB thread
+}
+
+void ParticlesGlobalManager::ParticlesGlobalManagerPrivate::addParticleSystem(osgParticle::ParticleSystem* ps, const osg::ref_ptr<osg::Group>& frame)
+{
+    std::lock_guard<std::mutex> g(_lock);
+    _systems.push_back(ps);
+
+    // we're inside an update callback here, so better not modify the
+    // scene structure; defer it to later
+    if (frame.get()) {
+        _newWorldParticles.push_back(frame);
+    }
+}
+
+void ParticlesGlobalManager::ParticlesGlobalManagerPrivate::registerNewLocalParticleSystem(osg::Node* node, ParticleSystemRef ps)
+{
+    auto cb = new RegistrationCallback;
+    cb->_system = ps;
+    node->addUpdateCallback(cb);
+}
+
+void ParticlesGlobalManager::ParticlesGlobalManagerPrivate::registerNewWorldParticleSystem(osg::Node* node, ParticleSystemRef ps, osg::Group* frame)
+{
+    auto cb = new RegistrationCallback;
+    cb->_system = ps;
+    cb->_frame = frame;
+    node->addUpdateCallback(cb);
+}
+
+// this is called from the main thread during scenery init
+// after this, we beign updarting particle systems
+void ParticlesGlobalManager::initFromMainThread()
+{
+    std::lock_guard<std::mutex> g(d->_lock);
+    d->internalGetCommonRoot();
+    d->_commonRoot->addUpdateCallback(d.get());
+    d->_commonRoot->addCullCallback(d->_cullCallback);
+}
+
+void ParticlesGlobalManager::update(double dt, const SGGeod& pos)
+{
+    std::lock_guard<std::mutex> g(d->_lock);
+    d->_simulationDt = dt;
+    d->_currentPosition = pos;
+
+    for (auto f : d->_newWorldParticles) {
+        d->internalGetCommonRoot()->addChild(f);
+    }
+
+    d->_newWorldParticles.clear();
+}
+
+// this is called from the main thread, since it's an update callback
+// lock any state used by updateParticleSystemsFromCullCallback, which
+// runs during culling, potentialy on a different thread
+void ParticlesGlobalManager::ParticlesGlobalManagerPrivate::operator()(osg::Node* node, osg::NodeVisitor* nv)
+{
+    std::lock_guard<std::mutex> g(_lock);
+    _enabled = !_enabledNode || _enabledNode->getBoolValue();
+
+    if (!_enabled)
+        return;
+
+    const auto q = SGQuatd::fromLonLat(_currentPosition);
+    osg::Matrix om(toOsg(q));
+    osg::Vec3 v(0, 0, 9.81);
+    _gravity = om.preMult(v);
+
+    // NOTE: THIS WIND COMPUTATION DOESN'T SEEM TO AFFECT PARTICLES
+    // const osg::Vec3& zUpWind = _wind;
+    // osg::Vec3 w(zUpWind.y(), zUpWind.x(), -zUpWind.z());
+    // _localWind = om.preMult(w);
+
+    // while we have the lock, remove all expired systems
+    auto firstInvalid = std::remove_if(_systems.begin(), _systems.end(), [](const ParticleSystemWeakRef& weak) {
+        return !weak.valid();
+    });
+    _systems.erase(firstInvalid, _systems.end());
+}
+
+// only call this with the lock held!
+osg::Group* ParticlesGlobalManager::ParticlesGlobalManagerPrivate::internalGetCommonRoot()
+{
+    if (!_commonRoot.valid()) {
+        _commonRoot = new osg::Group;
+        _commonRoot->setName("common particle system root");
+        _commonGeode->setName("common particle system geode");
+        _commonRoot->addChild(_commonGeode);
+        _commonRoot->setNodeMask(~simgear::MODELLIGHT_BIT);
+    }
+    return _commonRoot.get();
+}
+
+void ParticlesGlobalManager::ParticlesGlobalManagerPrivate::updateParticleSystemsFromCullCallback(int frameNumber, osg::NodeVisitor* nv)
+{
+    ParticleSystemsStrongRefVec activeSystems;
+    double dt = 0.0;
+
+    // begin locked section
+    {
+        std::lock_guard<std::mutex> g(_lock);
+        activeSystems.reserve(_systems.size());
+        for (const auto& psref : _systems) {
+            ParticleSystemRef owningRef;
+            if (!psref.lock(owningRef)) {
+                // pointed to system is gone, skip it
+                // we will clean these up in the update callback, don't
+                // worry about that here
+                continue;
+            }
+
+            // add to the list we will update now
+            activeSystems.push_back(owningRef);
+        }
+
+        dt = _simulationDt;
+    } // of locked section
+
+    // from here on, don't access class data; copy it all to local variables
+    // before this line. this is important so we're not holding _lock during
+    // culling, which might block osgDB threads for example.
+
+    if (dt <= 0.0) {
+        return;
+    }
+
+    for (const auto& ps : activeSystems) {
+        // code inside here is copied from osgParticle::ParticleSystemUpdate
+        osgParticle::ParticleSystem::ScopedWriteLock lock(*(ps->getReadWriteMutex()));
+
+        // We need to allow at least 2 frames difference, because the particle system's lastFrameNumber
+        // is updated in the draw thread which may not have completed yet.
+        if (!ps->isFrozen() &&
+            (!ps->getFreezeOnCull() || ((frameNumber - ps->getLastFrameNumber()) <= 2))) {
+            ps->update(dt, *nv);
+        }
+        // end of code copied from osgParticle::ParticleSystemUpdate
+    }
+}
+
 static std::mutex static_managerLock;
 static std::unique_ptr<ParticlesGlobalManager> static_instance;

@@ -132,14 +314,6 @@ ParticlesGlobalManager::ParticlesGlobalManager() : d(new ParticlesGlobalManagerP
 {
 }

-ParticlesGlobalManager::~ParticlesGlobalManager()
-{
-    if (d->_globalCallbackRegistered) {
-        // is this actually necessary? possibly not
-        d->_updater->setUpdateCallback(nullptr);
-    }
-}
-
 bool ParticlesGlobalManager::isEnabled() const
 {
    std::lock_guard<std::mutex> g(d->_lock);
@@ -728,23 +902,10 @@ osg::ref_ptr<osg::Group> ParticlesGlobalManager::appendParticles(const SGPropert
        emitter->setUpdateCallback(callback.get());
    }

-    // touch shared data now (and not before)
-
-    {
-        std::lock_guard<std::mutex> g(d->_lock);
-        d->_updater->addParticleSystem(particleSys);
-
-        if (attach != "local") {
-            d->internalGetCommonRoot()->addChild(callback()->particleFrame);
-        }
-
-        if (!d->_globalCallbackRegistered) {
-            SG_LOG(SG_PARTICLES, SG_INFO, "Registering global particles callback");
-            d->_globalCallbackRegistered = true;
-            d->_longitudeNode = modelRoot->getNode("/position/longitude-deg", true);
-            d->_latitudeNode = modelRoot->getNode("/position/latitude-deg", true);
-            d->_updater->setUpdateCallback(d.get());
-        }
+    if (attach == "local") {
+        d->registerNewLocalParticleSystem(align, particleSys);
+    } else {
+        d->registerNewWorldParticleSystem(align, particleSys, callback()->particleFrame);
    }

    return align;
--- a/simgear/scene/model/particles.hxx
+++ b/simgear/scene/model/particles.hxx
@@ -41,7 +41,6 @@ class NodeVisitor;
 namespace osgParticle
 {
 class ParticleSystem;
-class ParticleSystemUpdater;
 }

 #include <simgear/scene/util/SGNodeMasks.hxx>
@@ -151,7 +150,7 @@ protected:
 class ParticlesGlobalManager
 {
 public:
-    ~ParticlesGlobalManager();
+    ~ParticlesGlobalManager() = default;

    static ParticlesGlobalManager* instance();
    static void clear();
@@ -161,14 +160,18 @@ public:
    osg::ref_ptr<osg::Group> appendParticles(const SGPropertyNode* configNode, SGPropertyNode* modelRoot, const osgDB::Options* options);

    osg::Group* getCommonRoot();
-
    osg::Geode* getCommonGeode();

-    osgParticle::ParticleSystemUpdater* getPSU();
+    void initFromMainThread();

    void setFrozen(bool e);
    bool isFrozen() const;

+    /**
+        @brief update function: call from the main thread , outside of OSG calls.
+     */
+    void update(double dt, const SGGeod& pos);
+
    void setSwitchNode(const SGPropertyNode* n);

    /**
@@ -185,6 +188,8 @@ private:
    ParticlesGlobalManager();

    class ParticlesGlobalManagerPrivate;
+    class UpdaterCallback;
+    class RegistrationCallback;

    // because Private inherits NodeCallback, we need to own it
    // via an osg::ref_ptr
--- a/simgear/scene/tgdb/ReaderWriterSPT.cxx
+++ b/simgear/scene/tgdb/ReaderWriterSPT.cxx
@@ -294,7 +294,7 @@ ReaderWriterSPT::createTree(const BucketBox& bucketBox, const LocalOptions& opti
 osg::ref_ptr<osg::Node>
 ReaderWriterSPT::createPagedLOD(const BucketBox& bucketBox, const LocalOptions& options) const
 {
-    osg::PagedLOD* pagedLOD = new osg::PagedLOD;
+    osg::ref_ptr<osg::PagedLOD> pagedLOD = new osg::PagedLOD;

    pagedLOD->setCenterMode(osg::PagedLOD::USER_DEFINED_CENTER);
    SGSpheref sphere = bucketBox.getBoundingSphere();
--- a/simgear/scene/tgdb/ReaderWriterSTG.cxx
+++ b/simgear/scene/tgdb/ReaderWriterSTG.cxx
@@ -212,7 +212,8 @@ struct ReaderWriterSTG::_ModelBin {
            STGObjectsQuadtree quadtree((GetModelLODCoord()), (AddModelLOD()));
            quadtree.buildQuadTree(_objectStaticList.begin(), _objectStaticList.end());
            osg::ref_ptr<osg::Group> group = quadtree.getRoot();
-            group->setName("STG-group-A");
+            string group_name = string("STG-group-A ").append(_bucket.gen_index_str());
+            group->setName(group_name);
            group->setDataVariance(osg::Object::STATIC);

            simgear::AirportSignBuilder signBuilder(_options->getMaterialLib(), _bucket.get_center());
@@ -586,10 +587,12 @@ struct ReaderWriterSTG::_ModelBin {
    {
        osg::ref_ptr<SGReaderWriterOptions> options;
        options = SGReaderWriterOptions::copyOrCreate(opt);
+        float pagedLODExpiry = atoi(options->getPluginStringData("SimGear::PAGED_LOD_EXPIRY").c_str());

        osg::ref_ptr<osg::Group> terrainGroup = new osg::Group;
        terrainGroup->setDataVariance(osg::Object::STATIC);
-        terrainGroup->setName("terrain");
+        std::string terrain_name = string("terrain ").append(bucket.gen_index_str());
+        terrainGroup->setName(terrain_name);

        if (_foundBase) {
            for (auto stgObject : _objectList) {
@@ -637,11 +640,13 @@ struct ReaderWriterSTG::_ModelBin {
        } else {
            osg::PagedLOD* pagedLOD = new osg::PagedLOD;
            pagedLOD->setCenterMode(osg::PagedLOD::USE_BOUNDING_SPHERE_CENTER);
-            pagedLOD->setName("pagedObjectLOD");
+            std::string name = string("pagedObjectLOD ").append(bucket.gen_index_str());
+            pagedLOD->setName(name);

            // This should be visible in any case.
            // If this is replaced by some lower level of detail, the parent LOD node handles this.
            pagedLOD->addChild(terrainGroup, 0, std::numeric_limits<float>::max());
+            pagedLOD->setMinimumExpiryTime(0, pagedLODExpiry);

            // we just need to know about the read file callback that itself holds the data
            osg::ref_ptr<DelayLoadReadFileCallback> readFileCallback = new DelayLoadReadFileCallback;
@@ -658,10 +663,11 @@ struct ReaderWriterSTG::_ModelBin {

            // Objects may end up displayed up to 2x the object range.
            pagedLOD->setRange(pagedLOD->getNumChildren(), 0, 2.0 * _object_range_rough);
+            pagedLOD->setMinimumExpiryTime(pagedLOD->getNumChildren(), pagedLODExpiry);
            pagedLOD->setRadius(SG_TILE_RADIUS);
-            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile PagedLOD Center: " << pagedLOD->getCenter().x() << "," << pagedLOD->getCenter().y() << "," << pagedLOD->getCenter().z() );
-            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile PagedLOD Range: " << (2.0 * _object_range_rough));
-            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile PagedLOD Radius: " << SG_TILE_RADIUS);
+            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile " << bucket.gen_index_str() << " PagedLOD Center: " << pagedLOD->getCenter().x() << "," << pagedLOD->getCenter().y() << "," << pagedLOD->getCenter().z() );
+            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile " << bucket.gen_index_str() << " PagedLOD Range: " << (2.0 * _object_range_rough));
+            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile " << bucket.gen_index_str() << " PagedLOD Radius: " << SG_TILE_RADIUS);
            return pagedLOD;
        }
    }
--- a/simgear/scene/tgdb/SGBuildingBin.cxx
+++ b/simgear/scene/tgdb/SGBuildingBin.cxx
@@ -441,6 +441,10 @@ typedef QuadTreeBuilder<LOD*, SGBuildingBin::BuildingInstance, MakeBuildingLeaf,
      if (hash_pos != std::string::npos)
          line.resize(hash_pos);

+      if (line.empty()) {
+          continue; // skip blank lines
+      }
+
      // and process further
      std::stringstream in(line);

--- a/simgear/scene/tgdb/SGOceanTile.cxx
+++ b/simgear/scene/tgdb/SGOceanTile.cxx
@@ -41,6 +41,7 @@
 #include <simgear/scene/material/EffectGeode.hxx>
 #include <simgear/scene/material/mat.hxx>
 #include <simgear/scene/material/matlib.hxx>
+#include <simgear/scene/model/BoundingVolumeBuildVisitor.hxx>
 #include <simgear/scene/util/OsgMath.hxx>
 #include <simgear/scene/util/VectorArrayAdapter.hxx>
 #include <simgear/scene/util/SGNodeMasks.hxx>
@@ -340,5 +341,10 @@ osg::Node* SGOceanTile(const SGBucket& b, SGMaterialLib *matlib, int latPoints,
    transform->addChild(geode);
    transform->setNodeMask( ~(simgear::CASTSHADOW_BIT | simgear::MODELLIGHT_BIT) );

+    // Create a BVH at this point.  This is normally provided by the file loader, but as we create the 
+    // geometry programmatically, no file loader is involved.
+    BoundingVolumeBuildVisitor bvhBuilder(false);
+    transform->accept(bvhBuilder);
+    
    return transform;
 }
--- a/simgear/scene/tsync/terrasync.cxx
+++ b/simgear/scene/tsync/terrasync.cxx
@@ -109,13 +109,13 @@ bool hasWhitespace(string path)
 class SyncItem
 {
 public:
-    enum Type
-    {
-        Stop = 0,   ///< special item indicating to stop the SVNThread
+    enum Type {
+        Stop = 0, ///< special item indicating to stop the SVNThread
        Tile,
        AirportData,
        SharedModels,
-        AIData
+        AIData,
+        OSMTile ///< OSm2City per-Tile data
    };

    enum Status
@@ -165,13 +165,16 @@ public:
    SGTimeStamp stamp;
    bool busy = false; ///< is the slot working or idle
    unsigned int pendingKBytes = 0;
+    unsigned int pendingExtractKBytes = 0;
    unsigned int nextWarnTimeout = 0;
 };

 static const int SYNC_SLOT_TILES = 0; ///< Terrain and Objects sync
 static const int SYNC_SLOT_SHARED_DATA = 1; /// shared Models and Airport data
 static const int SYNC_SLOT_AI_DATA = 2; /// AI traffic and models
-static const unsigned int NUM_SYNC_SLOTS = 3;
+static const int SYNC_SLOT_OSM_TILE_DATA = 3;
+
+static const unsigned int NUM_SYNC_SLOTS = 4;

 /**
 * @brief translate a sync item type into one of the available slots.
@@ -186,7 +189,8 @@ static unsigned int syncSlotForType(SyncItem::Type ty)
        return SYNC_SLOT_SHARED_DATA;
    case SyncItem::AIData:
        return SYNC_SLOT_AI_DATA;
-
+    case SyncItem::OSMTile:
+        return SYNC_SLOT_OSM_TILE_DATA;
    default:
        return SYNC_SLOT_SHARED_DATA;
    }
@@ -194,18 +198,18 @@ static unsigned int syncSlotForType(SyncItem::Type ty)

 struct TerrasyncThreadState
 {
-    TerrasyncThreadState() :
-        _busy(false),
-        _stalled(false),
-        _hasServer(false),
-        _fail_count(0),
-        _updated_tile_count(0),
-        _success_count(0),
-        _consecutive_errors(0),
-        _cache_hits(0),
-        _transfer_rate(0),
-        _total_kb_downloaded(0),
-        _totalKbPending(0)
+    TerrasyncThreadState() : _busy(false),
+                             _stalled(false),
+                             _hasServer(false),
+                             _fail_count(0),
+                             _updated_tile_count(0),
+                             _success_count(0),
+                             _consecutive_errors(0),
+                             _cache_hits(0),
+                             _transfer_rate(0),
+                             _total_kb_downloaded(0),
+                             _totalKbPending(0),
+                             _extractTotalKbPending(0)
    {}

    bool _busy;
@@ -220,6 +224,7 @@ struct TerrasyncThreadState
    // kbytes, not bytes, because bytes might overflow 2^31
    int _total_kb_downloaded;
    unsigned int _totalKbPending;
+    unsigned int _extractTotalKbPending;
 };

 ///////////////////////////////////////////////////////////////////////////////
@@ -275,9 +280,10 @@ public:

   SyncItem getNewTile() { return _freshTiles.pop_front();}

-   void   setHTTPServer(const std::string& server)
+   void setHTTPServer(const std::string& server, const std::string& osmServer)
   {
      _httpServer = stripPath(server);
+      _osmCityServer = stripPath(osmServer);
      _isAutomaticServer = (server == "automatic");
   }

@@ -296,6 +302,11 @@ public:
     _sceneryVersion = simgear::strutils::strip(sceneryVersion);
   }

+   void setOSMCityVersion(const std::string& osmCityVersion)
+   {
+       _osmCityService = osmCityVersion;
+   }
+
   void   setLocalDir(string dir)           { _local_dir    = stripPath(dir);}
   string getLocalDir()                     { return _local_dir;}

@@ -322,10 +333,12 @@ public:
    void setCachePath(const SGPath &p) { _persistentCachePath = p; }

  private:
-    void incrementCacheHits()
-    {
-        std::lock_guard<std::mutex> g(_stateLock);
-        _state._cache_hits++;
+      std::string dnsSelectServerForService(const std::string& service);
+
+      void incrementCacheHits()
+      {
+          std::lock_guard<std::mutex> g(_stateLock);
+          _state._cache_hits++;
    }

   virtual void run();
@@ -334,9 +347,9 @@ public:
    void runInternal();
    void updateSyncSlot(SyncSlot& slot);

-    void beginSyncAirports(SyncSlot& slot);
-    void beginSyncTile(SyncSlot& slot);
-    void beginNormalSync(SyncSlot& slot);
+    bool beginSyncAirports(SyncSlot& slot);
+    bool beginSyncTile(SyncSlot& slot);
+    bool beginNormalSync(SyncSlot& slot);

    void drainWaitingTiles();

@@ -363,6 +376,9 @@ public:
    string _local_dir;
    SGPath _persistentCachePath;
    string _httpServer;
+    string _osmCityServer;
+    string _osmCityService = "o2c";
+
    bool _isAutomaticServer;
    SGPath _installRoot;
    string _sceneryVersion;
@@ -474,8 +490,19 @@ bool SGTerraSync::WorkerThread::findServer()
 {
    if ( false == _isAutomaticServer ) return true;

+    _httpServer = dnsSelectServerForService(MakeQService(_protocol, _sceneryVersion));
+
+    if (!_osmCityService.empty()) {
+        _osmCityServer = dnsSelectServerForService(_osmCityService);
+    }
+
+    return !_httpServer.empty();
+}
+
+std::string SGTerraSync::WorkerThread::dnsSelectServerForService(const std::string& service)
+{
    DNS::NAPTRRequest * naptrRequest = new DNS::NAPTRRequest(_dnsdn);
-    naptrRequest->qservice = MakeQService(_protocol, _sceneryVersion);
+    naptrRequest->qservice = service;

    naptrRequest->qflags = "U";
    DNS::Request_ptr r(naptrRequest);
@@ -483,14 +510,15 @@ bool SGTerraSync::WorkerThread::findServer()
    DNS::Client dnsClient;
    dnsClient.makeRequest(r);
    SG_LOG(SG_TERRASYNC,SG_DEBUG,"DNS NAPTR query for '" << _dnsdn << "' '" << naptrRequest->qservice << "'" );
-    while( !r->isComplete() && !r->isTimeout() )
-      dnsClient.update(0);
+    while (!r->isComplete() && !r->isTimeout()) {
+        dnsClient.update(0);
+    }

    if( naptrRequest->entries.empty() ) {
        SG_LOG(SG_TERRASYNC, SG_ALERT, "Warning: no DNS entry found for '" << _dnsdn << "' '" << naptrRequest->qservice << "'" );
-        _httpServer = "";
-        return false;
+        return {};
    }
+
    // walk through responses, they are ordered by 1. order and 2. preference
    // For now, only take entries with lowest order
    // TODO: try all available servers in the order given by preferenc and order
@@ -498,38 +526,33 @@ bool SGTerraSync::WorkerThread::findServer()

    // get all servers with this order and the same (for now only lowest preference)
    DNS::NAPTRRequest::NAPTR_list availableServers;
-    for( DNS::NAPTRRequest::NAPTR_list::const_iterator it = naptrRequest->entries.begin();
-         it != naptrRequest->entries.end();
-         ++it ) {
-
-        if( (*it)->order != order )
+    for (const auto& entry : naptrRequest->entries) {
+        if (entry->order != order)
            continue;

-        string regex = (*it)->regexp;
-        if( false == simgear::strutils::starts_with( (*it)->regexp, "!^.*$!" ) ) {
-            SG_LOG(SG_TERRASYNC,SG_WARN, "ignoring unsupported regexp: " << (*it)->regexp );
+        const string regex = entry->regexp;
+        if (false == simgear::strutils::starts_with(regex, "!^.*$!")) {
+            SG_LOG(SG_TERRASYNC, SG_WARN, "ignoring unsupported regexp: " << regex);
            continue;
        }

-        if( false == simgear::strutils::ends_with( (*it)->regexp, "!" ) ) {
-            SG_LOG(SG_TERRASYNC,SG_WARN, "ignoring unsupported regexp: " << (*it)->regexp );
+        if (false == simgear::strutils::ends_with(regex, "!")) {
+            SG_LOG(SG_TERRASYNC, SG_WARN, "ignoring unsupported regexp: " << regex);
            continue;
        }

        // always use first entry
-        if( availableServers.empty() || (*it)->preference == availableServers[0]->preference) {
-            SG_LOG(SG_TERRASYNC,SG_DEBUG, "available server regexp: " << (*it)->regexp );
-            availableServers.push_back( *it );
+        if (availableServers.empty() || entry->preference == availableServers[0]->preference) {
+            SG_LOG(SG_TERRASYNC, SG_DEBUG, "available server regexp: " << regex);
+            availableServers.push_back(entry);
        }
    }

    // now pick a random entry from the available servers
-    DNS::NAPTRRequest::NAPTR_list::size_type idx = sg_random() * availableServers.size();
-    _httpServer = availableServers[idx]->regexp;
-    _httpServer = _httpServer.substr( 6, _httpServer.length()-7 ); // strip search pattern and separators
-
-    SG_LOG(SG_TERRASYNC,SG_INFO, "picking entry # " << idx << ", server is " << _httpServer );
-    return true;
+    auto idx = static_cast<int>(sg_random() * availableServers.size());
+    const auto server = availableServers.at(idx)->regexp;
+    SG_LOG(SG_TERRASYNC, SG_INFO, "picking entry # " << idx << ", server is " << server.substr(6, server.length() - 7););
+    return server.substr(6, server.length() - 7);
 }

 void SGTerraSync::WorkerThread::run()
@@ -562,6 +585,7 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
 #endif
            // convert bytes to kbytes here
            slot.pendingKBytes = (slot.repository->bytesToDownload() >> 10);
+            slot.pendingExtractKBytes = (slot.repository->bytesToExtract() >> 10);
            return; // easy, still working
        }

@@ -569,9 +593,21 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
        HTTPRepository::ResultCode res = slot.repository->failure();

        if (res == HTTPRepository::REPO_ERROR_NOT_FOUND) {
+            // not founds should never happen any more (unless the server-
+            // side data is incorrect), since we now check top-down that
+            // a 1x1 dir exists or not.
            notFound(slot.currentItem);
        } else if (res != HTTPRepository::REPO_NO_ERROR) {
            fail(slot.currentItem);
+
+            // in case the Airports_archive download fails, create the
+            // directory, so that next sync, we do a manual sync
+            if ((slot.currentItem._type == SyncItem::AirportData) && slot.isNewDirectory) {
+                SG_LOG(SG_TERRASYNC, SG_ALERT, "Failed to download Airports_archive, will download discrete files next time");
+                simgear::Dir d(_local_dir + "/Airports");
+                d.create(0755);
+                _completedTiles.erase(slot.currentItem._dir);
+            }
        } else {
            updated(slot.currentItem, slot.isNewDirectory);
            SG_LOG(SG_TERRASYNC, SG_DEBUG, "sync of " << slot.repository->baseUrl() << " finished ("
@@ -582,6 +618,7 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
        slot.busy = false;
        slot.repository.reset();
        slot.pendingKBytes = 0;
+        slot.pendingExtractKBytes = 0;
        slot.currentItem = {};
    }

@@ -595,12 +632,23 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
        slot.isNewDirectory = !path.exists();
        const auto type = slot.currentItem._type;

+        bool ok = false;
        if (type == SyncItem::AirportData) {
-            beginSyncAirports(slot);
+            ok = beginSyncAirports(slot);
+        } else if (type == SyncItem::OSMTile) {
+            ok = beginSyncTile(slot);
        } else if (type == SyncItem::Tile) {
-            beginSyncTile(slot);
+            ok = beginSyncTile(slot);
        } else {
-            beginNormalSync(slot);
+            ok = beginNormalSync(slot);
+        }
+
+        if (!ok) {
+            SG_LOG(SG_TERRASYNC, SG_INFO, "sync of " << slot.currentItem._dir << " failed to start");
+            fail(slot.currentItem);
+            slot.busy = false;
+            slot.repository.reset();
+            return;
        }

        try {
@@ -617,17 +665,17 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
        slot.nextWarnTimeout = 30 * 1000;
        slot.stamp.stamp();
        slot.busy = true;
-        slot.pendingKBytes = slot.repository->bytesToDownload();
+        slot.pendingKBytes = slot.repository->bytesToDownload() >> 10;
+        slot.pendingExtractKBytes = slot.repository->bytesToExtract() >> 10;

-        SG_LOG(SG_TERRASYNC, SG_INFO, "sync of " << slot.repository->baseUrl() << " started, queue size is " << slot.queue.size());
+        SG_LOG(SG_TERRASYNC, SG_INFO, "sync of " << slot.repository->baseUrl() << ":" << slot.currentItem._dir << " started, queue size is " << slot.queue.size());
    }
 }

-void SGTerraSync::WorkerThread::beginSyncAirports(SyncSlot& slot)
+bool SGTerraSync::WorkerThread::beginSyncAirports(SyncSlot& slot)
 {
    if (!slot.isNewDirectory) {
-        beginNormalSync(slot);
-        return;
+        return beginNormalSync(slot);
    }

    SG_LOG(SG_TERRASYNC, SG_INFO, "doing Airports download via tarball");
@@ -647,9 +695,10 @@ void SGTerraSync::WorkerThread::beginSyncAirports(SyncSlot& slot)
    };

    slot.repository->setFilter(f);
+    return true;
 }

-void SGTerraSync::WorkerThread::beginSyncTile(SyncSlot& slot)
+bool SGTerraSync::WorkerThread::beginSyncTile(SyncSlot& slot)
 {
    // avoid 404 requests by doing a sync which excludes all paths
    // except our tile path. In the case of a missing 1x1 tile, we will
@@ -658,8 +707,7 @@ void SGTerraSync::WorkerThread::beginSyncTile(SyncSlot& slot)
    auto comps = strutils::split(slot.currentItem._dir, "/");
    if (comps.size() != 3) {
        SG_LOG(SG_TERRASYNC, SG_ALERT, "Bad tile path:" << slot.currentItem._dir);
-        beginNormalSync(slot);
-        return;
+        return false;
    }

    const auto tileCategory = comps.front();
@@ -668,7 +716,17 @@ void SGTerraSync::WorkerThread::beginSyncTile(SyncSlot& slot)

    const auto path = SGPath::fromUtf8(_local_dir) / tileCategory;
    slot.repository.reset(new HTTPRepository(path, &_http));
-    slot.repository->setBaseUrl(_httpServer + "/" + tileCategory);
+
+    if (slot.currentItem._type == SyncItem::OSMTile) {
+        if (_osmCityServer.empty()) {
+            SG_LOG(SG_TERRASYNC, SG_WARN, "No OSM2City server defined for:" << slot.currentItem._dir);
+            return false;
+        }
+
+        slot.repository->setBaseUrl(_osmCityServer + "/" + tileCategory);
+    } else {
+        slot.repository->setBaseUrl(_httpServer + "/" + tileCategory);
+    }

    if (_installRoot.exists()) {
      SGPath p = _installRoot / tileCategory;
@@ -676,6 +734,7 @@ void SGTerraSync::WorkerThread::beginSyncTile(SyncSlot& slot)
    }

    const auto dirPrefix = tenByTenDir + "/" + oneByOneDir;
+    using simgear::strutils::starts_with;

    // filter callback to *only* sync the 1x1 dir we want, if it exists
    // if doesn't, we'll simply stop, which is what we want
@@ -684,7 +743,10 @@ void SGTerraSync::WorkerThread::beginSyncTile(SyncSlot& slot)
        if (item.directory.empty()) {
            return item.filename == tenByTenDir;
        } else if (item.directory == tenByTenDir) {
-            return item.filename == oneByOneDir;
+            // allow 10x10 dir to contain 1x1_dir.tgz/.zip and still be accepted
+            // this does mean we'd also download oneByOneDir_foobar but that
+            // doesn't seem unreasonable either
+            return starts_with(item.filename, oneByOneDir);
        }

        // allow arbitrary children below dirPrefix, including sub-dirs
@@ -697,9 +759,10 @@ void SGTerraSync::WorkerThread::beginSyncTile(SyncSlot& slot)
    };

    slot.repository->setFilter(f);
+    return true;
 }

-void SGTerraSync::WorkerThread::beginNormalSync(SyncSlot& slot)
+bool SGTerraSync::WorkerThread::beginNormalSync(SyncSlot& slot)
 {
    SGPath path(_local_dir);
    path.append(slot.currentItem._dir);
@@ -711,6 +774,8 @@ void SGTerraSync::WorkerThread::beginNormalSync(SyncSlot& slot)
      p.append(slot.currentItem._dir);
      slot.repository->setInstalledCopyPath(p);
    }
+
+    return true;
 }

 void SGTerraSync::WorkerThread::runInternal()
@@ -753,20 +818,24 @@ void SGTerraSync::WorkerThread::runInternal()

        bool anySlotBusy = false;
        unsigned int newPendingCount = 0;
+        unsigned int newExtractCount = 0; // how much is left to extract

        // update each sync slot in turn
        for (unsigned int slot=0; slot < NUM_SYNC_SLOTS; ++slot) {
            updateSyncSlot(_syncSlots[slot]);
            newPendingCount += _syncSlots[slot].pendingKBytes;
+            newExtractCount += _syncSlots[slot].pendingExtractKBytes;
            anySlotBusy |= _syncSlots[slot].busy;
        }

        {
            std::lock_guard<std::mutex> g(_stateLock);
            _state._totalKbPending = newPendingCount; // approximately atomic update
+            _state._extractTotalKbPending = newExtractCount;
            _state._busy = anySlotBusy;
        }

+
        if (!anySlotBusy) {
            // wait on the blocking deque here, otherwise we spin
            // the loop very fast, since _http::update with no connections
@@ -802,12 +871,19 @@ void SGTerraSync::WorkerThread::fail(SyncItem failedItem)
 {
    std::lock_guard<std::mutex> g(_stateLock);
    time_t now = time(0);
-    _state._consecutive_errors++;
-    _state._fail_count++;
+
+    if (_osmCityServer.empty() && (failedItem._type == SyncItem::OSMTile)) {
+        // don't count these as errors, otherwise normla sync will keep
+        // being abandoned
+    } else {
+        _state._consecutive_errors++;
+        _state._fail_count++;
+    }
+
    failedItem._status = SyncItem::Failed;
    _freshTiles.push_back(failedItem);
    // not we also end up here for partial syncs
-    SG_LOG(SG_TERRASYNC,SG_INFO,
+    SG_LOG(SG_TERRASYNC, SG_WARN,
           "Failed to sync'" << failedItem._dir << "'");
    _completedTiles[ failedItem._dir ] = now + UpdateInterval::FailedAttempt;
 }
@@ -819,6 +895,8 @@ void SGTerraSync::WorkerThread::notFound(SyncItem item)
    // we don't spam the server with lookups for models that don't
    // exist

+    SG_LOG(SG_TERRASYNC, SG_WARN, "Not found for: '" << item._dir << "'");
+
    time_t now = time(0);
    item._status = SyncItem::NotFound;
    _freshTiles.push_back(item);
@@ -1031,8 +1109,11 @@ void SGTerraSync::reinit()
    if (enabled)
    {
        _availableNode->setBoolValue(true);
-        _workerThread->setHTTPServer( _terraRoot->getStringValue("http-server","automatic") );
+        _workerThread->setHTTPServer(
+            _terraRoot->getStringValue("http-server", "automatic"),
+            _terraRoot->getStringValue("osm2city-server", ""));
        _workerThread->setSceneryVersion( _terraRoot->getStringValue("scenery-version","ws20") );
+        _workerThread->setOSMCityVersion(_terraRoot->getStringValue("osm2city-version", "o2c"));
        _workerThread->setProtocol( _terraRoot->getStringValue("protocol","") );
 #if 1
        // leave it hardcoded for now, not sure about the security implications for now
@@ -1091,6 +1172,7 @@ void SGTerraSync::bind()
    _transferRateBytesSecNode = _terraRoot->getNode("transfer-rate-bytes-sec", true);
    _pendingKbytesNode = _terraRoot->getNode("pending-kbytes", true);
    _downloadedKBtesNode = _terraRoot->getNode("downloaded-kbytes", true);
+    _extractPendingKbytesNode = _terraRoot->getNode("extract-pending-kbytes", true);
    _enabledNode = _terraRoot->getNode("enabled", true);
    _availableNode = _terraRoot->getNode("available", true);
    _maxErrorsNode = _terraRoot->getNode("max-errors", true);
@@ -1116,6 +1198,7 @@ void SGTerraSync::unbind()
    _transferRateBytesSecNode.clear();
    _pendingKbytesNode.clear();
    _downloadedKBtesNode.clear();
+    _extractPendingKbytesNode.clear();
    _enabledNode.clear();
    _availableNode.clear();
    _maxErrorsNode.clear();
@@ -1152,6 +1235,7 @@ void SGTerraSync::update(double)
    _transferRateBytesSecNode->setIntValue(copiedState._transfer_rate);
    _pendingKbytesNode->setIntValue(copiedState._totalKbPending);
    _downloadedKBtesNode->setIntValue(copiedState._total_kb_downloaded);
+    _extractPendingKbytesNode->setIntValue(copiedState._extractTotalKbPending);

    _stalledNode->setBoolValue(_workerThread->isStalled());
    _activeNode->setBoolValue(worker_running);
@@ -1201,6 +1285,10 @@ string_list SGTerraSync::getSceneryPathSuffixes() const
    return scenerySuffixes;
 }

+bool isOSMSuffix(const std::string& suffix)
+{
+    return (suffix == "Buildings") || (suffix == "Roads") || (suffix == "Pylons") || (suffix == "Details");
+}

 void SGTerraSync::syncAreaByPath(const std::string& aPath)
 {
@@ -1213,8 +1301,7 @@ void SGTerraSync::syncAreaByPath(const std::string& aPath)
        if (_workerThread->isDirActive(dir)) {
            continue;
        }
-
-        SyncItem w(dir, SyncItem::Tile);
+        SyncItem w(dir, isOSMSuffix(suffix) ? SyncItem::OSMTile : SyncItem::Tile);
        _workerThread->request( w );
    }
 }
@@ -1233,6 +1320,11 @@ bool SGTerraSync::isTileDirPending(const std::string& sceneryDir) const
    }

    for (const auto& suffix : getSceneryPathSuffixes()) {
+        // don't wait on OSM dirs, even if enabled
+        if (isOSMSuffix(suffix)) {
+            continue;
+        }
+
        const auto s = suffix + "/" + sceneryDir;
        if (_workerThread->isDirActive(s)) {
            return true;
--- a/simgear/scene/tsync/terrasync.hxx
+++ b/simgear/scene/tsync/terrasync.hxx
@@ -107,6 +107,7 @@ private:
    SGPropertyNode_ptr _transferRateBytesSecNode;
    SGPropertyNode_ptr _pendingKbytesNode;
    SGPropertyNode_ptr _downloadedKBtesNode;
+    SGPropertyNode_ptr _extractPendingKbytesNode;
    SGPropertyNode_ptr _maxErrorsNode;

    // we manually bind+init TerraSync during early startup
--- a/simgear/scene/util/SGReaderWriterOptions.hxx
+++ b/simgear/scene/util/SGReaderWriterOptions.hxx
@@ -46,6 +46,7 @@ public:
        ORIGIN_EFFECTS,
        ORIGIN_EFFECTS_NORMALIZED,
        ORIGIN_SPLASH_SCREEN,
+        ORIGIN_CANVAS,
    };

    //SGReaderWriterOptions* cloneOptions(const osg::CopyOp& copyop = osg::CopyOp::SHALLOW_COPY) const { return static_cast<SGReaderWriterOptions*>(clone(copyop)); }
--- a/simgear/structure/StateMachine.cxx
+++ b/simgear/structure/StateMachine.cxx
@@ -441,7 +441,7 @@ void StateMachine::initFromPlist(SGPropertyNode* desc, SGPropertyNode* root)
        std::string nm = stateDesc->getStringValue("name");

        if (nm.empty()) {
-          SG_LOG(SG_GENERAL, SG_ALERT, "No name found for state in branch " << path);
+          SG_LOG(SG_GENERAL, SG_DEV_ALERT, "No name found for state in branch " << path);
          throw sg_exception("No name element in state");
        }

@@ -464,8 +464,8 @@ void StateMachine::initFromPlist(SGPropertyNode* desc, SGPropertyNode* root)
        std::string target_id = tDesc->getStringValue("target");

        if (nm.empty()) {
-          SG_LOG(SG_GENERAL, SG_ALERT, "No name found for transition in branch " << path);
-          throw sg_exception("No name element in transition");
+          SG_LOG(SG_GENERAL, SG_DEV_WARN, "No name found for transition in branch " << path);
+          nm = "transition-to-" + target_id;
        }

        if (target_id.empty()) {
--- a/simgear/structure/subsystem_mgr.cxx
+++ b/simgear/structure/subsystem_mgr.cxx
@@ -30,9 +30,10 @@
 #include "subsystem_mgr.hxx"
 #include "commands.hxx"

-#include <simgear/props/props.hxx>
-#include <simgear/math/SGMath.hxx>
 #include "SGSmplstat.hxx"
+#include <simgear/debug/ErrorReportingCallback.hxx>
+#include <simgear/math/SGMath.hxx>
+#include <simgear/props/props.hxx>

 const int SG_MAX_SUBSYSTEM_EXCEPTIONS = 4;
 const char SUBSYSTEM_NAME_SEPARATOR = '.';
@@ -828,8 +829,18 @@ SGSubsystemGroup::Member::update (double delta_time_sec)
      if (++exceptionCount > SG_MAX_SUBSYSTEM_EXCEPTIONS) {
        SG_LOG(SG_GENERAL, SG_ALERT, "(exceptionCount=" << exceptionCount <<
          ", suspending)");
+        simgear::reportError("suspending subsystem after too many errors:" + name);
        subsystem->suspend();
      }
+    } catch (std::bad_alloc& ba) {
+        // attempting to track down source of these on Sentry.io
+        SG_LOG(SG_GENERAL, SG_ALERT, "caught bad_alloc processing subsystem:" << name);
+        simgear::reportError("caught bad_alloc processing subsystem:" + name);
+
+        if (++exceptionCount > SG_MAX_SUBSYSTEM_EXCEPTIONS) {
+            SG_LOG(SG_GENERAL, SG_ALERT, "(exceptionCount=" << exceptionCount << ", suspending)");
+            subsystem->suspend();
+        }
    }
 }

--- a/1
+++ b/1
@@ -0,0 +1 @@
+2020.3.7
Author	SHA1	Message	Date
James Turner	c9d83fab6c	TerraSync: allow an explicit osm2city server Fixes an error case where a manual TerraSync server is specified; we would attempt to use an empty string as the OSM2City server, with hilarious consequences. Sentry-Id: FLIGHTGEAR-NCZ	2021-03-16 20:14:34 +00:00
James Turner	edcce32f24	Remove stray include of std::filesystem	2021-03-07 10:53:36 +00:00
Automatic Release Builder	68d265f0e7	Set correct version files	2021-03-04 22:02:11 +00:00
Automatic Release Builder	b985bb5757	new version: 2020.3.7	2021-03-01 12:11:33 +00:00
legoboyvdlp R	f030816385	TerraSync: counter to fetch number of bytes which have been extracted from a tarball file	2021-03-01 09:56:48 +00:00
James Turner	6f9f694eff	HTTPRepository: improving handling of archives Avoid hard-coding the archive extension, and ensure the extracted archive directory is not orphaned on update. Finally, use the literal filename in the .dirindex when computing the hash, rather than adding a .tgz extension.	2021-03-01 09:56:48 +00:00
James Turner	a0d7f0e172	TerraSync: allow separate OSM2City server Lookup OSM2City using a separate service profile, and use this server for requests for OSM2City suffix dirs.	2021-03-01 09:51:24 +00:00
James Turner	9f98e438cb	Fix HTTPClient reset() behaviour Ensure all data members are correctly re-initialzied when doing a reset. This shoed up as negative ‘bytes downloaded’ counts after a TerraSync abandon and retry.	2021-02-22 11:29:05 +00:00
James Turner	f029ca7b64	Particles: replace use of ParticleSystemUpdater Extend our own particle manager to replace the OSG particle system updater. This fixes thread-safety and also timing (better match to simulation dt values). We also use weak pointers (observer_ptr in OSG terminology) to ensure particle systems are released once their frame is gone.	2021-02-22 11:29:05 +00:00
James Turner	ca6c6dd6d3	Text-animation: fix missing encoding specification Ensure we can pass full UTF-8 strings into text animations. Will consider for back-port after discussion on the devel list. Ticket-Id: https://sourceforge.net/p/flightgear/codetickets/2512/	2021-02-22 11:10:37 +00:00
Automatic Release Builder	4ba4ea5602	Adding code to see if it improves crash on computeHash. Also add a test case to verify behaviour on empty files is correct. Sentry-Id: FLIGHTGEAR-AY2	2021-02-04 10:28:08 +00:00
Automatic Release Builder	ca5f66da9f	new version: 2020.3.6	2021-01-20 13:01:25 +00:00
James Turner	92d053d850	Tolerate blank lines in buildings lists	2021-01-20 11:35:33 +00:00
Colin Geniet	001ae80723	Animations: Fix spin for Compositor lights The 'spin' animation has a strange behaviour: it pushes a temporary rotation matrix on the transformation stack during cull traversal, and removes it thereafter. Thus, the rotation matrix is missing outside of cull traversal, e.g. when the position of Compositor lights is computed. The same issue was fixed for the 'rotate' animation by commit `e202d4e4a5` (this mentions broken 'picking' animations as a different manifestation of the same issue). Fix this by setting the angle of the persistent SGRotateTransform, instead of creating a temporary rotation matrix.	2021-01-15 15:45:26 +00:00
James Turner	58d9a6d0b5	Sentry: adjust reporting of missing shaders Reduce noise on the backend when a shader is missing, by special- casing how we report it.	2021-01-05 12:58:46 +00:00
Scott Giese	261316cb45	Support Boost v1.75 Resolved a breaking change.	2021-01-02 21:57:54 -06:00
Stuart Buchanan	2ad4d2e672	Fix hang on "Loading Scenery" over the sea Previously attempting to start on an ocean tile with static models present cause FG to spin on "Loading Scenery". This was caused by not creating a BVH for an ocean tile in the specific case where there was STG models, due to a PagedLOD node being inserted in the scene graph. So FG never thought the scenery was loaded sufficient to place the aircraft. By explicitly creating a BVH for ocean tiles the problem is fixed. Candidate for 2020.X	2021-01-01 22:41:47 +00:00
James Turner	1d9fa929fa	Launhcer: fix crashes adding a catalog Traversing a container which is modified causes crashes, take a copy during traversal for firePackageStatus. Sentry-Id: FLIGHTGEAR-CJF Sentry-Id: FLIGHTGEAR-CJ5	2021-01-01 22:41:42 +00:00
James Turner	26bb6236a0	Effect builder: catch allocation failures Fail on bad_alloc of reading texture images, instead of crashing.	2020-12-28 15:08:49 +00:00
Scott Giese	b2acf5a623	fix: sky and cockpit to turn black Metar sky condition without base layer height (e.g. FEW///) is the cause of this issue. Randomize the base layer height when the actual height is unknown.	2020-12-26 10:53:49 -06:00
James Turner	bea88bb3a9	Fix BTG error reporting Use gzerror, not strerror, for these codes.	2020-12-21 15:31:36 +00:00
Automatic Release Builder	def2af2bdd	new version: 2020.3.5	2020-12-17 13:48:00 +00:00
Fernando García Liñán	6f6d705f22	Add support for anisotropic filtering in Canvas	2020-12-17 13:00:01 +00:00
Richard Harrison	e9c33104d3	DDS-TC add option to exclude all Canvas orignated images.	2020-12-17 12:59:26 +00:00
Richard Harrison	985897f8ba	Emesary : change to use scoped lock - scoped locks using lock_guard are better - because of this the exception handler can also be removed; although this was originally intended to manage the locks it never did work properly with C++ exceptions.	2020-12-17 12:57:53 +00:00
Richard Harrison	4251b28e88	Emesary: add missing property initialisation to constructor	2020-12-17 12:56:49 +00:00
James Turner	76540a211c	SGGeod: add static constructor of an invalid Geod Use this to allow explicitly initializing a value which isValid will return false for.	2020-12-17 12:55:30 +00:00
James Turner	904fc5a7dd	Fix Nasal GC errors on tests/reset Ensure the Context temps are cleared, and when recycling an naCode, ensure old values are cleared explicily. Sentry-Id: FLIGHTGEAR-Y	2020-12-17 12:55:25 +00:00
Stuart Buchanan	4aebc159d5	Use ref_ptr for ReaderWriterSPT	2020-12-07 17:18:32 +00:00
Richard Harrison	f89227fc1a	Remove mutex lock on realizeTechniques This was causing a deadlock - and I can't quite remember what problem it was intended to solve so it is best to remove it.	2020-12-07 17:17:44 +00:00
James Turner	f50f383cc0	Reporting of std::bad_alloc in Subsystem::update Trying to trace down our bad-alloc exception, starting with the simplest place for now.	2020-12-07 17:16:58 +00:00
James Turner	706ab387de	Add reporting callback option to SimGear Allows us to trigger an error logging callback explicitly, which can be used to drive Sentry.io on the FlightGear side.	2020-12-07 17:16:47 +00:00
James Turner	cddfdb7d1d	TerraSync: stronger fix for handling 0-byte files Change logic so we create an empty file for such cases, i.e exactly matching the repository. This simplifies logic in downstream code, compared with not creating a local file. Add a test-case to cover this Modify TerraSync to detect a failure of Airports_archive downloading, and fall back to file-by-file updating.	2020-12-07 17:15:51 +00:00
James Turner	3c64578848	HTTPRepository: don’t crash on empty files Fix some additional crash cases around 0-length files	2020-12-07 17:15:38 +00:00
James Turner	e1fe9b45e0	Unzip: adjust error reporting mechanism Don’t use local exception throw+catch to report failures in extracting a zip archive, since this generates noise in Sentry.	2020-12-07 17:15:30 +00:00
James Turner	8fdc1d306f	State-machines: don’t require name for transitions Allow anonymous transitions, since the name is purely informational (unlike for states). Sentry-Id: FLIGHTGEAR-9H	2020-12-07 17:15:17 +00:00
Automatic Release Builder	6e0c39bb68	new version: 2020.3.4	2020-11-30 11:17:52 +00:00
Julian Smith	f20b416cfe	simgear/props/props.cxx: use rmutex to protect SGPropertyNodeListeners. Also added asserts to check _num_iterators always >= 0.	2020-11-29 20:16:46 +00:00
Scott Giese	47e06b5216	METAR: mitigate wind sensor failures	2020-11-29 16:25:01 +00:00
James Turner	bfcdf22705	TerraSync: fix crashes with null file return Not sure how this is happening, but, check for a null file object in FileGetRequest::onDone.	2020-11-29 16:23:44 +00:00
Stuart Buchanan	d0db407faa	Set minimum expiry time on STG nodes.	2020-11-29 16:23:40 +00:00
Automatic Release Builder	d95b1c0441	new version: 2020.3.3	2020-11-12 11:14:23 +00:00
James Turner	837ba86d57	DNSClient: own requests, and cancel them on timeout Fixes crashes where a request times-out, but then is completed by UDN sometime afterwards, with a free-d object. Have the DNS::Client own requests, and be able to retrieve the udns_query to cancel them, in the timeout case. Fixes a couple of Sentry reports.	2020-11-12 09:39:58 +00:00
Automatic Release Builder	0cb1b463e1	Catalogs: fix ownership of new Catalogs When doing the initial download of a Catalog, ensure we still keep an owning ref to it.	2020-11-11 21:18:57 +00:00
@@ -1 +1 @@
 .3.2
 .3.7