Backport the code to be able to display the region name of the file that holds the selected material.

Move sg_location member up to the base clase, since it’s potentially
useful in more types.

Allow suppression of the report callback when the exception is thrown

CMakeLists.txt

@@ -286,6 +286,7 @@ else()
     find_package(ZLIB 1.2.4 REQUIRED)
 endif()
 
+find_package(LibLZMA REQUIRED)
 find_package(CURL REQUIRED)
 
 if (SYSTEM_EXPAT)

CMakeModules/FindLibLZMA.cmake

@@ -0,0 +1,124 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+#[=======================================================================[.rst:
+FindLibLZMA
+-----------
+
+Find LZMA compression algorithm headers and library.
+
+
+Imported Targets
+^^^^^^^^^^^^^^^^
+
+This module defines :prop_tgt:`IMPORTED` target ``LibLZMA::LibLZMA``, if
+liblzma has been found.
+
+Result variables
+^^^^^^^^^^^^^^^^
+
+This module will set the following variables in your project:
+
+``LIBLZMA_FOUND``
+  True if liblzma headers and library were found.
+``LIBLZMA_INCLUDE_DIRS``
+  Directory where liblzma headers are located.
+``LIBLZMA_LIBRARIES``
+  Lzma libraries to link against.
+``LIBLZMA_HAS_AUTO_DECODER``
+  True if lzma_auto_decoder() is found (required).
+``LIBLZMA_HAS_EASY_ENCODER``
+  True if lzma_easy_encoder() is found (required).
+``LIBLZMA_HAS_LZMA_PRESET``
+  True if lzma_lzma_preset() is found (required).
+``LIBLZMA_VERSION_MAJOR``
+  The major version of lzma
+``LIBLZMA_VERSION_MINOR``
+  The minor version of lzma
+``LIBLZMA_VERSION_PATCH``
+  The patch version of lzma
+``LIBLZMA_VERSION_STRING``
+  version number as a string (ex: "5.0.3")
+#]=======================================================================]
+
+find_path(LIBLZMA_INCLUDE_DIR lzma.h )
+if(NOT LIBLZMA_LIBRARY)
+  find_library(LIBLZMA_LIBRARY_RELEASE NAMES lzma liblzma NAMES_PER_DIR PATH_SUFFIXES lib)
+  find_library(LIBLZMA_LIBRARY_DEBUG NAMES lzmad liblzmad NAMES_PER_DIR PATH_SUFFIXES lib)
+  include(SelectLibraryConfigurations)
+  select_library_configurations(LIBLZMA)
+else()
+  file(TO_CMAKE_PATH "${LIBLZMA_LIBRARY}" LIBLZMA_LIBRARY)
+endif()
+
+if(LIBLZMA_INCLUDE_DIR AND EXISTS "${LIBLZMA_INCLUDE_DIR}/lzma/version.h")
+    file(STRINGS "${LIBLZMA_INCLUDE_DIR}/lzma/version.h" LIBLZMA_HEADER_CONTENTS REGEX "#define LZMA_VERSION_[A-Z]+ [0-9]+")
+
+    string(REGEX REPLACE ".*#define LZMA_VERSION_MAJOR ([0-9]+).*" "\\1" LIBLZMA_VERSION_MAJOR "${LIBLZMA_HEADER_CONTENTS}")
+    string(REGEX REPLACE ".*#define LZMA_VERSION_MINOR ([0-9]+).*" "\\1" LIBLZMA_VERSION_MINOR "${LIBLZMA_HEADER_CONTENTS}")
+    string(REGEX REPLACE ".*#define LZMA_VERSION_PATCH ([0-9]+).*" "\\1" LIBLZMA_VERSION_PATCH "${LIBLZMA_HEADER_CONTENTS}")
+
+    set(LIBLZMA_VERSION_STRING "${LIBLZMA_VERSION_MAJOR}.${LIBLZMA_VERSION_MINOR}.${LIBLZMA_VERSION_PATCH}")
+    unset(LIBLZMA_HEADER_CONTENTS)
+endif()
+
+# We're using new code known now as XZ, even library still been called LZMA
+# it can be found in http://tukaani.org/xz/
+# Avoid using old codebase
+if (LIBLZMA_LIBRARY)
+  include(CheckLibraryExists)
+  set(CMAKE_REQUIRED_QUIET_SAVE ${CMAKE_REQUIRED_QUIET})
+  set(CMAKE_REQUIRED_QUIET ${LibLZMA_FIND_QUIETLY})
+  if(NOT LIBLZMA_LIBRARY_RELEASE AND NOT LIBLZMA_LIBRARY_DEBUG)
+    set(LIBLZMA_LIBRARY_check ${LIBLZMA_LIBRARY})
+  elseif(LIBLZMA_LIBRARY_RELEASE)
+    set(LIBLZMA_LIBRARY_check ${LIBLZMA_LIBRARY_RELEASE})
+  elseif(LIBLZMA_LIBRARY_DEBUG)
+    set(LIBLZMA_LIBRARY_check ${LIBLZMA_LIBRARY_DEBUG})
+  endif()
+  CHECK_LIBRARY_EXISTS(${LIBLZMA_LIBRARY_check} lzma_auto_decoder "" LIBLZMA_HAS_AUTO_DECODER)
+  CHECK_LIBRARY_EXISTS(${LIBLZMA_LIBRARY_check} lzma_easy_encoder "" LIBLZMA_HAS_EASY_ENCODER)
+  CHECK_LIBRARY_EXISTS(${LIBLZMA_LIBRARY_check} lzma_lzma_preset "" LIBLZMA_HAS_LZMA_PRESET)
+  unset(LIBLZMA_LIBRARY_check)
+  set(CMAKE_REQUIRED_QUIET ${CMAKE_REQUIRED_QUIET_SAVE})
+endif ()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(LibLZMA  REQUIRED_VARS  LIBLZMA_LIBRARY
+                                                          LIBLZMA_INCLUDE_DIR
+                                                          LIBLZMA_HAS_AUTO_DECODER
+                                                          LIBLZMA_HAS_EASY_ENCODER
+                                                          LIBLZMA_HAS_LZMA_PRESET
+                                           VERSION_VAR    LIBLZMA_VERSION_STRING
+                                 )
+mark_as_advanced( LIBLZMA_INCLUDE_DIR LIBLZMA_LIBRARY )
+
+if (LIBLZMA_FOUND)
+    set(LIBLZMA_LIBRARIES ${LIBLZMA_LIBRARY})
+    set(LIBLZMA_INCLUDE_DIRS ${LIBLZMA_INCLUDE_DIR})
+    if(NOT TARGET LibLZMA::LibLZMA)
+        add_library(LibLZMA::LibLZMA UNKNOWN IMPORTED)
+        set_target_properties(LibLZMA::LibLZMA PROPERTIES
+                              INTERFACE_INCLUDE_DIRECTORIES ${LIBLZMA_INCLUDE_DIR}
+                              IMPORTED_LINK_INTERFACE_LANGUAGES C)
+
+        if(LIBLZMA_LIBRARY_RELEASE)
+            set_property(TARGET LibLZMA::LibLZMA APPEND PROPERTY
+                IMPORTED_CONFIGURATIONS RELEASE)
+            set_target_properties(LibLZMA::LibLZMA PROPERTIES
+                IMPORTED_LOCATION_RELEASE "${LIBLZMA_LIBRARY_RELEASE}")
+        endif()
+
+        if(LIBLZMA_LIBRARY_DEBUG)
+            set_property(TARGET LibLZMA::LibLZMA APPEND PROPERTY
+                IMPORTED_CONFIGURATIONS DEBUG)
+            set_target_properties(LibLZMA::LibLZMA PROPERTIES
+                IMPORTED_LOCATION_DEBUG "${LIBLZMA_LIBRARY_DEBUG}")
+        endif()
+
+        if(NOT LIBLZMA_LIBRARY_RELEASE AND NOT LIBLZMA_LIBRARY_DEBUG)
+            set_target_properties(LibLZMA::LibLZMA PROPERTIES
+                IMPORTED_LOCATION "${LIBLZMA_LIBRARY}")
+        endif()
+    endif()
+endif ()

SimGearConfig.cmake.in

@@ -1,6 +1,7 @@
 include(CMakeFindDependencyMacro)
 
 find_dependency(ZLIB)
+find_dependency(LibLZMA)
 find_dependency(Threads)
 
 # OSG

simgear-version

@@ -1 +1 @@
-2020.3.6
+2020.3.9

simgear/CMakeLists.txt

@@ -168,7 +168,9 @@ target_link_libraries(SimGearCore PRIVATE
     ${COCOA_LIBRARY}
     ${CURL_LIBRARIES}
     ${WINSOCK_LIBRARY}
-    ${SHLWAPI_LIBRARY})
+    ${SHLWAPI_LIBRARY}
+    LibLZMA::LibLZMA
+)
 
 if(SYSTEM_EXPAT)
     target_link_libraries(SimGearCore PRIVATE ${EXPAT_LIBRARIES})

simgear/io/ArchiveExtractor_private.hxx

@@ -0,0 +1,90 @@
+// Copyright (C) 2021  James Turner - <james@flightgear.org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Library General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Library General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//
+
+
+#pragma once
+
+#include "untar.hxx"
+
+namespace simgear {
+
+class ArchiveExtractorPrivate
+{
+public:
+    ArchiveExtractorPrivate(ArchiveExtractor* o) : outer(o)
+    {
+        assert(outer);
+    }
+
+    virtual ~ArchiveExtractorPrivate() = default;
+
+    typedef enum {
+        INVALID = 0,
+        READING_HEADER,
+        READING_FILE,
+        READING_PADDING,
+        READING_PAX_GLOBAL_ATTRIBUTES,
+        READING_PAX_FILE_ATTRIBUTES,
+        PRE_END_OF_ARCHVE,
+        END_OF_ARCHIVE,
+        ERROR_STATE, ///< states above this are error conditions
+        BAD_ARCHIVE,
+        BAD_DATA,
+        FILTER_STOPPED
+    } State;
+
+    State state = INVALID;
+    ArchiveExtractor* outer = nullptr;
+
+    virtual void extractBytes(const uint8_t* bytes, size_t count) = 0;
+
+    virtual void flush() = 0;
+
+    SGPath extractRootPath()
+    {
+        return outer->_rootPath;
+    }
+
+    ArchiveExtractor::PathResult filterPath(std::string& pathToExtract)
+    {
+        return outer->filterPath(pathToExtract);
+    }
+
+
+    bool isSafePath(const std::string& p) const
+    {
+        if (p.empty()) {
+            return false;
+        }
+
+        // reject absolute paths
+        if (p.at(0) == '/') {
+            return false;
+        }
+
+        // reject paths containing '..'
+        size_t doubleDot = p.find("..");
+        if (doubleDot != std::string::npos) {
+            return false;
+        }
+
+        // on POSIX could use realpath to sanity check
+        return true;
+    }
+};
+
+} // namespace simgear

simgear/io/HTTPClient.cxx

@@ -81,21 +81,8 @@ void Client::ClientPrivate::createCurlMulti() {
 #endif
 }
 
-Client::Client() :
+Client::Client()
-    d(new ClientPrivate)
 {
-    d->proxyPort = 0;
-    d->maxConnections = 4;
-    d->maxHostConnections = 4;
-    d->bytesTransferred = 0;
-    d->lastTransferRate = 0;
-    d->timeTransferSample.stamp();
-    d->totalBytesDownloaded = 0;
-    d->maxPipelineDepth = 5;
-    setUserAgent("SimGear-" SG_STRINGIZE(SIMGEAR_VERSION));
-
-    d->tlsCertificatePath = SGPath::fromEnv("SIMGEAR_TLS_CERT_PATH");
-
     static bool didInitCurlGlobal = false;
     static std::mutex initMutex;
 
@@ -105,7 +92,7 @@ Client::Client() :
       didInitCurlGlobal = true;
     }
 
-    d->createCurlMulti();
+    reset();
 }
 
 Client::~Client()
@@ -139,8 +126,21 @@ void Client::setMaxPipelineDepth(unsigned int depth)
 
 void Client::reset()
 {
+    if (d.get()) {
         curl_multi_cleanup(d->curlMulti);
+    }
+
     d.reset(new ClientPrivate);
+
+    d->proxyPort = 0;
+    d->maxConnections = 4;
+    d->maxHostConnections = 4;
+    d->bytesTransferred = 0;
+    d->lastTransferRate = 0;
+    d->timeTransferSample.stamp();
+    d->totalBytesDownloaded = 0;
+    d->maxPipelineDepth = 5;
+    setUserAgent("SimGear-" SG_STRINGIZE(SIMGEAR_VERSION));
     d->tlsCertificatePath = SGPath::fromEnv("SIMGEAR_TLS_CERT_PATH");
     d->createCurlMulti();
 }

simgear/io/HTTPRepository.cxx

@@ -20,15 +20,15 @@
 
 #include "HTTPRepository.hxx"
 
-#include <iostream>
-#include <cassert>
 #include <algorithm>
-#include <sstream>
+#include <cassert>
+#include <cstdlib>
+#include <fstream>
+#include <iostream>
+#include <limits>
 #include <map>
 #include <set>
-#include <fstream>
+#include <sstream>
-#include <limits>
-#include <cstdlib>
 
 #include <fcntl.h>
 
@@ -265,6 +265,19 @@ public:
                 free(buf);
     }
 
+    /// helper to check and erase 'fooBar' from paths, if passed fooBar.zip, fooBar.tgz, etc.
+    void removeExtractedDirectoryFromList(PathList& paths, const std::string& tarballName)
+    {
+        const auto directoryName = SGPath::fromUtf8(tarballName).file_base();
+        auto it = std::find_if(paths.begin(), paths.end(), [directoryName](const SGPath& p) {
+            return p.isDir() && (p.file() == directoryName);
+        });
+
+        if (it != paths.end()) {
+            paths.erase(it);
+        }
+    }
+
     void updateChildrenBasedOnHash()
     {
       using SAct = HTTPRepository::SyncAction;
@@ -298,6 +311,11 @@ public:
                         orphans.end());
         }
 
+        // ensure the extracted directory corresponding to a tarball, is *not* considered an orphan
+        if (c.type == HTTPRepository::TarballType) {
+            removeExtractedDirectoryFromList(orphans, c.name);
+        }
+
         if (_repository->syncPredicate) {
           const auto pathOnDisk = isNew ? absolutePath() / c.name : *p;
           // never handle deletes here, do them at the end
@@ -320,7 +338,6 @@ public:
           toBeUpdated.push_back(c);
         } else {
           // File/Directory exists and hash is valid.
-
           if (c.type == HTTPRepository::DirectoryType) {
             // If it's a directory,perform a recursive check.
             HTTPDirectory *childDir = childDirectory(c.name);
@@ -360,7 +377,12 @@ public:
 
       // We now have a list of entries that need to be updated, and a list
       // of orphan files that should be removed.
+      try {
           removeOrphans(orphans);
+      } catch (sg_exception& e) {
+          _repository->failedToUpdateChild(_relativePath, HTTPRepository::ResultCode::REPO_ERROR_IO);
+      }
+
       scheduleUpdates(toBeUpdated);
     }
 
@@ -372,11 +394,10 @@ public:
 
     void removeOrphans(const PathList orphans)
     {
-        PathList::const_iterator it;
+        for (const auto& o : orphans) {
-        for (it = orphans.begin(); it != orphans.end(); ++it) {
+            if (o.file() == ".dirindex") continue;
-            if (it->file() == ".dirindex") continue;
+            if (o.file() == ".hash") continue;
-            if (it->file() == ".hash") continue;
+            removeChild(o);
-            removeChild(*it);
         }
     }
 
@@ -432,6 +453,7 @@ public:
           return;
         }
 
+        compressedBytes = p.sizeInBytes();
         buffer = (uint8_t *)malloc(bufferSize);
       }
 
@@ -444,6 +466,7 @@ public:
           }
 
           size_t rd = file.read((char*)buffer, bufferSize);
+          repo->bytesExtracted += rd;
           extractor.extractBytes(buffer, rd);
 
           if (file.eof()) {
@@ -470,8 +493,14 @@ public:
           return HTTPRepoPrivate::ProcessContinue;
       }
 
+      size_t archiveSizeBytes() const
+      {
+          return compressedBytes;
+      }
+
       ~ArchiveExtractTask() { free(buffer); }
 
+
     private:
       // intentionally small so we extract incrementally on Windows
       // where Defender throttles many small files, sorry
@@ -483,6 +512,7 @@ public:
       uint8_t *buffer = nullptr;
       SGBinaryFile file;
       ArchiveExtractor extractor;
+      std::size_t compressedBytes;
     };
 
     using ArchiveExtractTaskPtr = std::shared_ptr<ArchiveExtractTask>;
@@ -509,7 +539,7 @@ public:
                 _repository->totalDownloaded += sz;
                 SGPath p = SGPath(absolutePath(), file);
 
-                if ((p.extension() == "tgz") || (p.extension() == "zip")) {
+                if (it->type == HTTPRepository::TarballType) {
                     // We require that any compressed files have the same filename as the file or directory
                     // they expand to, so we can remove the old file/directory before extracting the new
                     // data.
@@ -539,12 +569,12 @@ public:
                         auto cb = [t](HTTPRepoPrivate* repo) {
                             return t->run(repo);
                         };
-
+                        _repository->bytesToExtract += t->archiveSizeBytes();
                         _repository->addTask(cb);
                     } else {
                         SG_LOG(SG_TERRASYNC, SG_ALERT, "Unable to remove old file/directory " << removePath);
                     } // of pathAvailable
-                } // of handling tgz files
+                }     // of handling archive files
             } // of hash matches
         } // of found in child list
     }
@@ -736,11 +766,9 @@ private:
     std::string hashForChild(const ChildInfo& child) const
     {
       SGPath p(child.path);
-      if (child.type == HTTPRepository::DirectoryType)
+      if (child.type == HTTPRepository::DirectoryType) {
           p.append(".dirindex");
-      if (child.type == HTTPRepository::TarballType)
+      }
-        p.concat(
-            ".tgz"); // For tarballs the hash is against the tarball file itself
       return hashForPath(p);
     }
 
@@ -937,6 +965,11 @@ size_t HTTPRepository::bytesDownloaded() const
     return result;
 }
 
+size_t HTTPRepository::bytesToExtract() const
+{
+    return _d->bytesToExtract - _d->bytesExtracted;
+}
+
 void HTTPRepository::setInstalledCopyPath(const SGPath& copyPath)
 {
     _d->installedCopyPath = copyPath;

simgear/io/HTTPRepository.hxx

@@ -73,6 +73,8 @@ public:
 
   virtual size_t bytesDownloaded() const;
 
+  virtual size_t bytesToExtract() const;
+
   /**
    * optionally provide the location of an installer copy of this
    * repository. When a file is missing it will be copied from this tree.

simgear/io/HTTPRepository_private.hxx

@@ -61,21 +61,22 @@ public:
   int maxPermittedFailures = 16;
 
   HTTPRepoPrivate(HTTPRepository* parent)
-      : p(parent), isUpdating(false), status(HTTPRepository::REPO_NO_ERROR),
+      : p(parent)
-        totalDownloaded(0) {
+  {
-    ;
   }
 
   ~HTTPRepoPrivate();
 
   HTTPRepository *p; // link back to outer
-  HTTP::Client *http;
+  HTTP::Client* http = nullptr;
   std::string baseUrl;
   SGPath basePath;
-  bool isUpdating;
+  bool isUpdating = false;
-  HTTPRepository::ResultCode status;
+  HTTPRepository::ResultCode status = HTTPRepository::REPO_NO_ERROR;
   HTTPDirectory_ptr rootDir;
-  size_t totalDownloaded;
+  size_t totalDownloaded = 0;
+  size_t bytesToExtract = 0;
+  size_t bytesExtracted = 0;
   HTTPRepository::SyncPredicate syncPredicate;
 
   HTTP::Request_ptr updateFile(HTTPDirectory *dir, const std::string &name,

simgear/io/lowlevel.cxx

@@ -23,28 +23,41 @@
 // $Id$
 
 
-#ifdef HAVE_CONFIG_H
 #include <simgear_config.h>
-#endif
 
 #include <string.h>		// for memcpy()
+#include <errno.h>
+
+#include <simgear/structure/exception.hxx>
+#include <simgear/misc/strutils.hxx>
+#include <simgear/misc/sg_path.hxx>
 
 #include "lowlevel.hxx"
 
+thread_local SGPath thread_gzPath;
 
-static int  read_error = false ;
+void setThreadLocalSimgearReadPath(const SGPath& path)
-static int write_error = false ;
+{
+    thread_gzPath = path;
+}
 
-void sgClearReadError() { read_error = false; }
+static std::string gzErrorMessage(gzFile fd)
-void sgClearWriteError() { write_error = false; }
+{
-int sgReadError() { return  read_error ; }
+    int errNum = 0;
-int sgWriteError() { return write_error ; }
+    const char *gzMsg = gzerror(fd, &errNum);
 
+    if (errNum == Z_ERRNO) {
+        return simgear::strutils::error_string(errno);
+    } else {
+        return {gzMsg};
+    }
+}
 
 void sgReadChar ( gzFile fd, char *var )
 {
     if ( gzread ( fd, var, sizeof(char) ) != sizeof(char) ) {
-        read_error = true ;
+        throw sg_io_exception("sgReadChar: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
 }
 
@@ -52,7 +65,7 @@ void sgReadChar ( gzFile fd, char *var )
 void sgWriteChar ( gzFile fd, const char var )
 {
     if ( gzwrite ( fd, (void *)(&var), sizeof(char) ) != sizeof(char) ) {
-        write_error = true ;
+        throw sg_io_exception("sgWriteChar: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -61,7 +74,8 @@ void sgReadFloat ( gzFile fd, float *var )
 {
     union { float v; uint32_t u; } buf;
     if ( gzread ( fd, &buf.u, sizeof(float) ) != sizeof(float) ) {
-        read_error = true ;
+        throw sg_io_exception("sgReadFloat: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         sgEndianSwap( &buf.u );
@@ -78,7 +92,7 @@ void sgWriteFloat ( gzFile fd, const float var )
         sgEndianSwap( &buf.u );
     }
     if ( gzwrite ( fd, (void *)(&buf.u), sizeof(float) ) != sizeof(float) ) {
-        write_error = true ;
+        throw sg_io_exception("sgWriteFloat: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -87,7 +101,8 @@ void sgReadDouble ( gzFile fd, double *var )
 {
     union { double v; uint64_t u; } buf;
     if ( gzread ( fd, &buf.u, sizeof(double) ) != sizeof(double) ) {
-        read_error = true ;
+        throw sg_io_exception("sgReadDouble: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         sgEndianSwap( &buf.u );
@@ -104,7 +119,7 @@ void sgWriteDouble ( gzFile fd, const double var )
         sgEndianSwap( &buf.u );
     }
     if ( gzwrite ( fd, (void *)(&buf.u), sizeof(double) ) != sizeof(double) ) {
-        write_error = true ;
+        throw sg_io_exception("sgWriteDouble: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -112,7 +127,8 @@ void sgWriteDouble ( gzFile fd, const double var )
 void sgReadUInt ( gzFile fd, unsigned int *var )
 {
     if ( gzread ( fd, var, sizeof(unsigned int) ) != sizeof(unsigned int) ) {
-        read_error = true ;
+        throw sg_io_exception("sgReadUInt: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         sgEndianSwap( (uint32_t *)var);
@@ -128,7 +144,7 @@ void sgWriteUInt ( gzFile fd, const unsigned int var )
     if ( gzwrite ( fd, (void *)(&var), sizeof(unsigned int) )
 	 != sizeof(unsigned int) )
     {
-        write_error = true ;
+        throw sg_io_exception("sgWriteUInt: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -136,7 +152,8 @@ void sgWriteUInt ( gzFile fd, const unsigned int var )
 void sgReadInt ( gzFile fd, int *var )
 {
     if ( gzread ( fd, var, sizeof(int) ) != sizeof(int) ) {
-        read_error = true ;
+        throw sg_io_exception("sgReadInt: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         sgEndianSwap( (uint32_t *)var);
@@ -150,7 +167,7 @@ void sgWriteInt ( gzFile fd, const int var )
         sgEndianSwap( (uint32_t *)&var);
     }
     if ( gzwrite ( fd, (void *)(&var), sizeof(int) ) != sizeof(int) ) {
-        write_error = true ;
+        throw sg_io_exception("sgWriteInt: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -158,7 +175,8 @@ void sgWriteInt ( gzFile fd, const int var )
 void sgReadLong ( gzFile fd, int32_t *var )
 {
     if ( gzread ( fd, var, sizeof(int32_t) ) != sizeof(int32_t) ) {
-        read_error = true ;
+        throw sg_io_exception("sgReadLong: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         sgEndianSwap( (uint32_t *)var);
@@ -174,7 +192,7 @@ void sgWriteLong ( gzFile fd, const int32_t var )
     if ( gzwrite ( fd, (void *)(&var), sizeof(int32_t) )
          != sizeof(int32_t) )
     {
-        write_error = true ;
+        throw sg_io_exception("sgWriteLong: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -182,7 +200,8 @@ void sgWriteLong ( gzFile fd, const int32_t var )
 void sgReadLongLong ( gzFile fd, int64_t *var )
 {
     if ( gzread ( fd, var, sizeof(int64_t) ) != sizeof(int64_t) ) {
-        read_error = true ;
+        throw sg_io_exception("sgReadLongLong: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         sgEndianSwap( (uint64_t *)var);
@@ -198,7 +217,7 @@ void sgWriteLongLong ( gzFile fd, const int64_t var )
     if ( gzwrite ( fd, (void *)(&var), sizeof(int64_t) )
          != sizeof(int64_t) )
     {
-        write_error = true ;
+        throw sg_io_exception("sgWriteLongLong: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -206,7 +225,8 @@ void sgWriteLongLong ( gzFile fd, const int64_t var )
 void sgReadUShort ( gzFile fd, unsigned short *var )
 {
     if ( gzread ( fd, var, sizeof(unsigned short) ) != sizeof(unsigned short) ){
-        read_error = true ;
+        throw sg_io_exception("sgReadUShort: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         sgEndianSwap( (uint16_t *)var);
@@ -222,7 +242,7 @@ void sgWriteUShort ( gzFile fd, const unsigned short var )
     if ( gzwrite ( fd, (void *)(&var), sizeof(unsigned short) )
 	 != sizeof(unsigned short) )
     {
-        write_error = true ;
+        throw sg_io_exception("sgWriteUShort: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -230,7 +250,8 @@ void sgWriteUShort ( gzFile fd, const unsigned short var )
 void sgReadShort ( gzFile fd, short *var )
 {
     if ( gzread ( fd, var, sizeof(short) ) != sizeof(short) ) {
-        read_error = true ;
+        throw sg_io_exception("sgReadShort: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         sgEndianSwap( (uint16_t *)var);
@@ -244,7 +265,7 @@ void sgWriteShort ( gzFile fd, const short var )
         sgEndianSwap( (uint16_t *)&var);
     }
     if ( gzwrite ( fd, (void *)(&var), sizeof(short) ) != sizeof(short) ) {
-        write_error = true ;
+        throw sg_io_exception("sgWriteShort: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -252,7 +273,8 @@ void sgWriteShort ( gzFile fd, const short var )
 void sgReadFloat ( gzFile fd, const unsigned int n, float *var )
 {
     if ( gzread ( fd, var, sizeof(float) * n ) != (int)(sizeof(float) * n) ) {
-        read_error = true ;
+        throw sg_io_exception("sgReadFloat array: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         for ( unsigned int i = 0; i < n; ++i ) {
@@ -276,14 +298,15 @@ void sgWriteFloat ( gzFile fd, const unsigned int n, const float *var )
     if ( gzwrite ( fd, (void *)var, sizeof(float) * n )
 	 != (int)(sizeof(float) * n) )
     {
-        write_error = true ;
+        throw sg_io_exception("sgWriteFloat array: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
 void sgReadDouble ( gzFile fd, const unsigned int n, double *var )
 {
     if ( gzread ( fd, var, sizeof(double) * n ) != (int)(sizeof(double) * n) ) {
-        read_error = true ;
+        throw sg_io_exception("sgReadDouble array: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         for ( unsigned int i = 0; i < n; ++i ) {
@@ -307,7 +330,7 @@ void sgWriteDouble ( gzFile fd, const unsigned int n, const double *var )
     if ( gzwrite ( fd, (void *)var, sizeof(double) * n )
 	 != (int)(sizeof(double) * n) )
     {
-        write_error = true ;
+        throw sg_io_exception("sgWriteDouble array: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -315,7 +338,8 @@ void sgReadBytes ( gzFile fd, const unsigned int n, void *var )
 {
     if ( n == 0) return;
     if ( gzread ( fd, var, n ) != (int)n ) {
-        read_error = true ;
+        throw sg_io_exception("sgReadBytes: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
 }
 
@@ -323,7 +347,7 @@ void sgWriteBytes ( gzFile fd, const unsigned int n, const void *var )
 {
     if ( n == 0) return;
     if ( gzwrite ( fd, (void *)var, n ) != (int)n ) {
-        write_error = true ;
+        throw sg_io_exception("sgWriteBytes: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -333,7 +357,8 @@ void sgReadUShort ( gzFile fd, const unsigned int n, unsigned short *var )
     if ( gzread ( fd, var, sizeof(unsigned short) * n )
 	 != (int)(sizeof(unsigned short) * n) )
     {
-        read_error = true ;
+        throw sg_io_exception("sgReadUShort array: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         for ( unsigned int i = 0; i < n; ++i ) {
@@ -357,7 +382,7 @@ void sgWriteUShort ( gzFile fd, const unsigned int n, const unsigned short *var
     if ( gzwrite ( fd, (void *)var, sizeof(unsigned short) * n )
 	 != (int)(sizeof(unsigned short) * n) )
     {
-        write_error = true ;
+        throw sg_io_exception("sgWriteUShort array: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -368,7 +393,8 @@ void sgReadShort ( gzFile fd, const unsigned int n, short *var )
     if ( gzread ( fd, var, sizeof(short) * n )
 	 != (int)(sizeof(short) * n) )
     {
-        read_error = true ;
+        throw sg_io_exception("sgReadShort array: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         for ( unsigned int i = 0; i < n; ++i ) {
@@ -392,7 +418,7 @@ void sgWriteShort ( gzFile fd, const unsigned int n, const short *var )
     if ( gzwrite ( fd, (void *)var, sizeof(short) * n )
 	 != (int)(sizeof(short) * n) )
     {
-        write_error = true ;
+        throw sg_io_exception("sgWriteShort array: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -402,7 +428,8 @@ void sgReadUInt ( gzFile fd, const unsigned int n, unsigned int *var )
     if ( gzread ( fd, var, sizeof(unsigned int) * n )
 	 != (int)(sizeof(unsigned int) * n) )
     {
-        read_error = true ;
+        throw sg_io_exception("sgReadUInt array: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         for ( unsigned int i = 0; i < n; ++i ) {
@@ -426,7 +453,7 @@ void sgWriteUInt ( gzFile fd, const unsigned int n, const unsigned int *var )
     if ( gzwrite ( fd, (void *)var, sizeof(unsigned int) * n )
 	 != (int)(sizeof(unsigned int) * n) )
     {
-        write_error = true ;
+        throw sg_io_exception("sgWriteUInt array: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 
@@ -437,7 +464,8 @@ void sgReadInt ( gzFile fd, const unsigned int n, int *var )
     if ( gzread ( fd, var, sizeof(int) * n )
 	 != (int)(sizeof(int) * n) )
     {
-        read_error = true ;
+        throw sg_io_exception("sgReadInt array: GZRead failed:" + gzErrorMessage(fd),
+                              sg_location{thread_gzPath}, nullptr, false);
     }
     if ( sgIsBigEndian() ) {
         for ( unsigned int i = 0; i < n; ++i ) {
@@ -461,7 +489,7 @@ void sgWriteInt ( gzFile fd, const unsigned int n, const int *var )
     if ( gzwrite ( fd, (void *)var, sizeof(int) * n )
 	 != (int)(sizeof(int) * n) )
     {
-        write_error = true ;
+        throw sg_io_exception("sgWriteInt array: gzwrite failed:" + gzErrorMessage(fd), {} /* origin */, false);
     }
 }
 

simgear/io/lowlevel.hxx

@@ -35,6 +35,9 @@
 
 #include <simgear/math/SGMath.hxx>
 
+// forward decls
+class SGPath;
+
 // Note that output is written in little endian form (and converted as
 // necessary for big endian machines)
 
@@ -121,9 +124,10 @@ inline void sgWriteGeod ( gzFile fd, const SGGeod& var ) {
     sgWriteDouble( fd, var.getElevationM() );
 }
 
-void sgClearReadError();
+/**
-void sgClearWriteError();
+    @ error aid: allow calling code to specify which file path we're reading from, so that erros we
-int sgReadError();
+ throw from sgReadXXXX can have a valid location set.
-int sgWriteError();
+ */
+void setThreadLocalSimgearReadPath(const SGPath& path);
 
 #endif // _SG_LOWLEVEL_HXX

simgear/io/sg_binobj.cxx

@@ -454,10 +454,6 @@ void SGBinObject::read_object( gzFile fp,
         }
     }
 
-    if ( sgReadError() ) {
-        throw sg_exception("Error reading object properties");
-    }
-
     size_t indexCount = std::bitset<32>((int)idx_mask).count();
     if (indexCount == 0) {
         throw sg_exception("object index mask has no bits set");
@@ -465,18 +461,10 @@ void SGBinObject::read_object( gzFile fp,
 
     for ( j = 0; j < nelements; ++j ) {
         sgReadUInt( fp, &nbytes );
-        if ( sgReadError() ) {
-            throw sg_exception("Error reading element size");
-        }
-
         buf.resize( nbytes );
         char *ptr = buf.get_ptr();
         sgReadBytes( fp, nbytes, ptr );
 
-        if ( sgReadError() ) {
-            throw sg_exception("Error reading element bytes");
-        }
-
         int_list vs;
         int_list ns;
         int_list cs;
@@ -503,7 +491,11 @@ void SGBinObject::read_object( gzFile fp,
 
 
 // read a binary file and populate the provided structures.
-bool SGBinObject::read_bin( const SGPath& file ) {
+bool SGBinObject::read_bin( const SGPath& file )
+{
+    gzFile fp = NULL;
+    
+    try {
         SGVec3d p;
         int i, k;
         size_t j;
@@ -552,25 +544,16 @@ bool SGBinObject::read_bin( const SGPath& file ) {
             withGZ.concat(".gz");
             fp = gzFileFromSGPath(withGZ, "rb");
             if (fp == nullptr) {
-            SG_LOG( SG_EVENT, SG_ALERT,
+                throw sg_io_exception("Error opening for reading (and .gz)", sg_location(file), {}, false);
-               "ERROR: opening " << file << " or " << withGZ << " for reading!");
-
-            throw sg_io_exception("Error opening for reading (and .gz)", sg_location(file));
             }
         }
         
-    sgClearReadError();
+        setThreadLocalSimgearReadPath(file);
 
         // read headers
         unsigned int header;
         sgReadUInt( fp, &header );
 
-    if (sgReadError()) {
-        int code = 0;
-        const char* gzErrorString = gzerror(fp, &code);
-        gzclose(fp);
-        throw sg_io_exception("Unable to read BTG header: " + string{gzErrorString} + ", code =" + std::to_string(code), sg_location(file));
-    }
 
         if ( ((header & 0xFF000000) >> 24) == 'S' &&
              ((header & 0x00FF0000) >> 16) == 'G' ) {
@@ -578,9 +561,7 @@ bool SGBinObject::read_bin( const SGPath& file ) {
             // read file version
             version = (header & 0x0000FFFF);
         } else {
-        // close the file before we return
+            throw sg_io_exception("Bad BTG magic/version", sg_location(file), {}, false);
-        gzclose(fp);
-        throw sg_io_exception("Bad BTG magic/version", sg_location(file));
         }
 
         // read creation time
@@ -616,9 +597,6 @@ bool SGBinObject::read_bin( const SGPath& file ) {
 
         SG_LOG(SG_IO, SG_DEBUG, "SGBinObject::read_bin Total objects to read = " << nobjects);
 
-    if ( sgReadError() ) {
-        throw sg_io_exception("Error reading BTG file header", sg_location(file));
-    }
 
         // read in objects
         for ( i = 0; i < nobjects; ++i ) {
@@ -803,14 +781,19 @@ bool SGBinObject::read_bin( const SGPath& file ) {
                     sgReadBytes( fp, nbytes, ptr );
                 }
             }
-
-        if ( sgReadError() ) {
-            throw sg_io_exception("Error while reading object", sg_location(file, i));
-        }
         }
 
+        gzclose(fp);
+        fp = NULL;
+    } catch (std::exception&) {
+        if (fp) {
             // close the file
             gzclose(fp);
+        }
+        
+        throw; // re-throw
+    }
+
 
     return true;
 }
@@ -984,8 +967,7 @@ bool SGBinObject::write_bin_file(const SGPath& file)
         return false;
     }
 
-    sgClearWriteError();
+    try {
-
         SG_LOG(SG_IO, SG_DEBUG, "points size = " << pts_v.size()
              << "  pt_materials = " << pt_materials.size() );
         SG_LOG(SG_IO, SG_DEBUG, "triangles size = " << tris_v.size()
@@ -1084,12 +1066,13 @@ bool SGBinObject::write_bin_file(const SGPath& file)
 
         // close the file
         gzclose(fp);
-
+        fp = NULL;
-    if ( sgWriteError() ) {
+    } catch (std::exception&) {
-        cout << "Error while writing file " << file << endl;
+        if (fp) {
+            gzclose(fp);
+        }
         return false;
     }
-
     return true;
 }
 

simgear/io/sg_file.cxx

@@ -82,7 +82,9 @@ std::string SGFile::computeHash()
         [](char* p) { free(p); }};
 
     if (!buf) {
-        SG_LOG(SG_IO, SG_ALERT, "Failed to malloc buffer for SHA1 check");
+        // @TODO report out of memory error
+        SG_LOG(SG_IO, SG_ALERT, "Failed to malloc buffer for SHA1 check:" << file_name);
+        return {};
     }
 
     size_t readLen;

simgear/io/test_repository.cxx

@@ -901,6 +901,27 @@ void testPersistentSocketFailure(HTTP::Client *cl) {
   verifyRequestCount("dirD/subdirDB/fileDBA", 1);
 }
 
+void testHashOnEmptyFile()
+{
+    std::unique_ptr<HTTPRepository> repo;
+    SGPath p(simgear::Dir::current().path());
+    p.append("sgfile_compute_hash");
+    simgear::Dir pd(p);
+    if (pd.exists()) {
+        pd.removeChildren();
+    } else {
+        pd.create(0700);
+    }
+
+    SGPath fPath = p / "test_empty_file";
+    
+    SGFile file(fPath);
+    file.open(SG_IO_OUT);
+    file.close();
+
+    const auto hash = file.computeHash();
+}
+
 int main(int argc, char* argv[])
 {
   sglog().setLogLevels( SG_ALL, SG_INFO );
@@ -951,6 +972,8 @@ int main(int argc, char* argv[])
     testRetryAfterSocketFailure(&cl);
     testPersistentSocketFailure(&cl);
 
+    testHashOnEmptyFile();
+
     std::cout << "all tests passed ok" << std::endl;
     return 0;
 }

simgear/io/test_untar.cxx

@@ -32,7 +32,7 @@ void testTarGz()
     uint8_t* buf = (uint8_t*) alloca(8192);
     size_t bufSize = f.read((char*) buf, 8192);
 
-    SG_VERIFY(ArchiveExtractor::determineType(buf, bufSize) == ArchiveExtractor::TarData);
+    SG_VERIFY(ArchiveExtractor::determineType(buf, bufSize) == ArchiveExtractor::GZData);
 
     f.close();
 }
@@ -174,6 +174,34 @@ void testPAXAttributes()
 
 }
 
+void testExtractXZ()
+{
+    SGPath p = SGPath(SRC_DIR);
+    p.append("test.tar.xz");
+
+    SGBinaryFile f(p);
+    f.open(SG_IO_IN);
+
+    SGPath extractDir = simgear::Dir::current().path() / "test_extract_xz";
+    simgear::Dir pd(extractDir);
+    pd.removeChildren();
+
+    ArchiveExtractor ex(extractDir);
+
+    uint8_t* buf = (uint8_t*)alloca(128);
+    while (!f.eof()) {
+        size_t bufSize = f.read((char*)buf, 128);
+        ex.extractBytes(buf, bufSize);
+    }
+
+    ex.flush();
+    SG_VERIFY(ex.isAtEndOfArchive());
+    SG_VERIFY(ex.hasError() == false);
+
+    SG_VERIFY((extractDir / "testDir/hello.c").exists());
+    SG_VERIFY((extractDir / "testDir/foo.txt").exists());
+}
+
 int main(int ac, char ** av)
 {
     testTarGz();
@@ -181,6 +209,7 @@ int main(int ac, char ** av)
     testFilterTar();
 	testExtractStreamed();
 	testExtractZip();
+    testExtractXZ();
 
     // disabled to avoiding checking in large PAX archive
     // testPAXAttributes();

simgear/io/untar.cxx

@@ -38,82 +38,11 @@
 #include <simgear/package/unzip.h>
 #include <simgear/structure/exception.hxx>
 
+#include "ArchiveExtractor_private.hxx"
+
 namespace simgear
 {
 
-	class ArchiveExtractorPrivate
-	{
-	public:
-		ArchiveExtractorPrivate(ArchiveExtractor* o) :
-			outer(o)
-		{
-			assert(outer);
-		}
-
-        virtual ~ArchiveExtractorPrivate() = default;
-        
-		typedef enum {
-			INVALID = 0,
-			READING_HEADER,
-			READING_FILE,
-			READING_PADDING,
-            READING_PAX_GLOBAL_ATTRIBUTES,
-            READING_PAX_FILE_ATTRIBUTES,
-			PRE_END_OF_ARCHVE,
-			END_OF_ARCHIVE,
-			ERROR_STATE, ///< states above this are error conditions
-			BAD_ARCHIVE,
-			BAD_DATA,
-			FILTER_STOPPED
-		} State;
-
-		State state = INVALID;
-		ArchiveExtractor* outer = nullptr;
-
-		virtual void extractBytes(const uint8_t* bytes, size_t count) = 0;
-
-		virtual void flush() = 0;
-
-		SGPath extractRootPath()
-		{
-			return outer->_rootPath;
-		}
-
-		ArchiveExtractor::PathResult filterPath(std::string& pathToExtract)
-		{
-			return outer->filterPath(pathToExtract);
-		}
-
-
-		bool isSafePath(const std::string& p) const
-		{
-			if (p.empty()) {
-				return false;
-			}
-
-			// reject absolute paths
-			if (p.at(0) == '/') {
-				return false;
-			}
-
-			// reject paths containing '..'
-			size_t doubleDot = p.find("..");
-			if (doubleDot != std::string::npos) {
-				return false;
-			}
-
-			// on POSIX could use realpath to sanity check
-			return true;
-		}
-	};
-
-///////////////////////////////////////////////////////////////////////////////////////////////////
-
-const int ZLIB_DECOMPRESS_BUFFER_SIZE = 32 * 1024;
-const int ZLIB_INFLATE_WINDOW_BITS = MAX_WBITS;
-const int ZLIB_DECODE_GZIP_HEADER = 16;
-
-/* tar Header Block, from POSIX 1003.1-1990.  */
 
 typedef struct
 {
@@ -157,10 +86,18 @@ typedef struct
     const char PAX_FILE_ATTRIBUTES = 'x';
 
 
+    ///////////////////////////////////////////////////////////////////////////////////////////////////
+
+    const int ZLIB_DECOMPRESS_BUFFER_SIZE = 32 * 1024;
+    const int ZLIB_INFLATE_WINDOW_BITS = MAX_WBITS;
+    const int ZLIB_DECODE_GZIP_HEADER = 16;
+
+    /* tar Header Block, from POSIX 1003.1-1990.  */
+
+
     class TarExtractorPrivate : public ArchiveExtractorPrivate
     {
     public:
-    
         union {
             UstarHeaderBlock header;
             uint8_t headerBytes[TAR_HEADER_BLOCK_SIZE];
@@ -169,30 +106,18 @@ public:
         size_t bytesRemaining;
         std::unique_ptr<SGFile> currentFile;
         size_t currentFileSize;
-    z_stream zlibStream;
+
-    uint8_t* zlibOutput;
-    bool haveInitedZLib = false;
-    bool uncompressedData = false; // set if reading a plain .tar (not tar.gz)
         uint8_t* headerPtr;
         bool skipCurrentEntry = false;
         std::string paxAttributes;
         std::string paxPathName;
 
-    TarExtractorPrivate(ArchiveExtractor* o) :
+        TarExtractorPrivate(ArchiveExtractor* o) : ArchiveExtractorPrivate(o)
-		ArchiveExtractorPrivate(o)
         {
-		memset(&zlibStream, 0, sizeof(z_stream));
+            setState(TarExtractorPrivate::READING_HEADER);
-		zlibOutput = (unsigned char*)malloc(ZLIB_DECOMPRESS_BUFFER_SIZE);
-		zlibStream.zalloc = Z_NULL;
-		zlibStream.zfree = Z_NULL;
-		zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE;
-		zlibStream.next_out = zlibOutput;
         }
 
-    ~TarExtractorPrivate()
+        ~TarExtractorPrivate() = default;
-    {
-        free(zlibOutput);
-    }
 
         void readPaddingIfRequired()
         {
@@ -248,68 +173,8 @@ public:
 
         void extractBytes(const uint8_t* bytes, size_t count) override
         {
-		zlibStream.next_in = (uint8_t*) bytes;
+            // uncompressed, just pass through directly
-		zlibStream.avail_in = count;
-
-		if (!haveInitedZLib) {
-			// now we have data, see if we're dealing with GZ-compressed data or not
-			if ((bytes[0] == 0x1f) && (bytes[1] == 0x8b)) {
-				// GZIP identification bytes
-				if (inflateInit2(&zlibStream, ZLIB_INFLATE_WINDOW_BITS | ZLIB_DECODE_GZIP_HEADER) != Z_OK) {
-					SG_LOG(SG_IO, SG_WARN, "inflateInit2 failed");
-					state = TarExtractorPrivate::BAD_DATA;
-					return;
-				}
-			} else {
-				UstarHeaderBlock* header = (UstarHeaderBlock*)bytes;
-				if (strncmp(header->magic, TMAGIC, TMAGLEN) != 0) {
-					SG_LOG(SG_IO, SG_WARN, "didn't find tar magic in header");
-					state = TarExtractorPrivate::BAD_DATA;
-					return;
-				}
-
-				uncompressedData = true;
-			}
-
-			haveInitedZLib = true;
-			setState(TarExtractorPrivate::READING_HEADER);
-		} // of init on first-bytes case
-
-		if (uncompressedData) {
             processBytes((const char*)bytes, count);
-		} else {
-			size_t writtenSize;
-			// loop, running zlib() inflate and sending output bytes to
-			// our request body handler. Keep calling inflate until no bytes are
-			// written, and ZLIB has consumed all available input
-			do {
-				zlibStream.next_out = zlibOutput;
-				zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE;
-				int result = inflate(&zlibStream, Z_NO_FLUSH);
-				if (result == Z_OK || result == Z_STREAM_END) {
-					// nothing to do
-
-				}
-				else if (result == Z_BUF_ERROR) {
-					// transient error, fall through
-				}
-				else {
-					//  _error = result;
-					SG_LOG(SG_IO, SG_WARN, "Permanent ZLib error:" << zlibStream.msg);
-					state = TarExtractorPrivate::BAD_DATA;
-					return;
-				}
-
-				writtenSize = ZLIB_DECOMPRESS_BUFFER_SIZE - zlibStream.avail_out;
-				if (writtenSize > 0) {
-					processBytes((const char*) zlibOutput, writtenSize);
-				}
-
-				if (result == Z_STREAM_END) {
-					break;
-				}
-			} while ((zlibStream.avail_in > 0) || (writtenSize > 0));
-		} // of Zlib-compressed data
         }
 
         void flush() override
@@ -467,6 +332,163 @@ public:
 
 ///////////////////////////////////////////////////////////////////////////////
 
+
+class GZTarExtractor : public TarExtractorPrivate
+{
+public:
+    GZTarExtractor(ArchiveExtractor* outer) : TarExtractorPrivate(outer)
+    {
+        memset(&zlibStream, 0, sizeof(z_stream));
+        zlibOutput = (unsigned char*)malloc(ZLIB_DECOMPRESS_BUFFER_SIZE);
+        zlibStream.zalloc = Z_NULL;
+        zlibStream.zfree = Z_NULL;
+        zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE;
+        zlibStream.next_out = zlibOutput;
+    }
+
+    ~GZTarExtractor()
+    {
+        if (haveInitedZLib) {
+            inflateEnd(&zlibStream);
+        }
+        free(zlibOutput);
+    }
+
+    void extractBytes(const uint8_t* bytes, size_t count) override
+    {
+        zlibStream.next_in = (uint8_t*)bytes;
+        zlibStream.avail_in = count;
+
+        if (!haveInitedZLib) {
+            // now we have data, see if we're dealing with GZ-compressed data or not
+            if ((bytes[0] == 0x1f) && (bytes[1] == 0x8b)) {
+                // GZIP identification bytes
+                if (inflateInit2(&zlibStream, ZLIB_INFLATE_WINDOW_BITS | ZLIB_DECODE_GZIP_HEADER) != Z_OK) {
+                    SG_LOG(SG_IO, SG_WARN, "inflateInit2 failed");
+                    state = TarExtractorPrivate::BAD_DATA;
+                    return;
+                }
+            } else {
+                // set error state
+                state = TarExtractorPrivate::BAD_DATA;
+                return;
+            }
+
+            haveInitedZLib = true;
+            setState(TarExtractorPrivate::READING_HEADER);
+        } // of init on first-bytes case
+
+        size_t writtenSize;
+        // loop, running zlib() inflate and sending output bytes to
+        // our request body handler. Keep calling inflate until no bytes are
+        // written, and ZLIB has consumed all available input
+        do {
+            zlibStream.next_out = zlibOutput;
+            zlibStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE;
+            int result = inflate(&zlibStream, Z_NO_FLUSH);
+            if (result == Z_OK || result == Z_STREAM_END) {
+                // nothing to do
+
+            } else if (result == Z_BUF_ERROR) {
+                // transient error, fall through
+            } else {
+                //  _error = result;
+                SG_LOG(SG_IO, SG_WARN, "Permanent ZLib error:" << zlibStream.msg);
+                state = TarExtractorPrivate::BAD_DATA;
+                return;
+            }
+
+            writtenSize = ZLIB_DECOMPRESS_BUFFER_SIZE - zlibStream.avail_out;
+            if (writtenSize > 0) {
+                processBytes((const char*)zlibOutput, writtenSize);
+            }
+
+            if (result == Z_STREAM_END) {
+                break;
+            }
+        } while ((zlibStream.avail_in > 0) || (writtenSize > 0));
+    }
+
+private:
+    z_stream zlibStream;
+    uint8_t* zlibOutput;
+    bool haveInitedZLib = false;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+
+#if 1 || defined(HAVE_XZ)
+
+#include <lzma.h>
+
+class XZTarExtractor : public TarExtractorPrivate
+{
+public:
+    XZTarExtractor(ArchiveExtractor* outer) : TarExtractorPrivate(outer)
+    {
+        _xzStream = LZMA_STREAM_INIT;
+        _outputBuffer = (uint8_t*)malloc(ZLIB_DECOMPRESS_BUFFER_SIZE);
+
+
+        auto ret = lzma_stream_decoder(&_xzStream, UINT64_MAX, LZMA_TELL_ANY_CHECK);
+        if (ret != LZMA_OK) {
+            setState(BAD_ARCHIVE);
+            return;
+        }
+    }
+
+    ~XZTarExtractor()
+    {
+        free(_outputBuffer);
+    }
+
+    void extractBytes(const uint8_t* bytes, size_t count) override
+    {
+        lzma_action action = LZMA_RUN;
+        _xzStream.next_in = bytes;
+        _xzStream.avail_in = count;
+
+        size_t writtenSize;
+        do {
+            _xzStream.avail_out = ZLIB_DECOMPRESS_BUFFER_SIZE;
+            _xzStream.next_out = _outputBuffer;
+
+            const auto ret = lzma_code(&_xzStream, action);
+
+            writtenSize = ZLIB_DECOMPRESS_BUFFER_SIZE - _xzStream.avail_out;
+            if (writtenSize > 0) {
+                processBytes((const char*)_outputBuffer, writtenSize);
+            }
+
+            if (ret == LZMA_GET_CHECK) {
+                //
+            } else if (ret == LZMA_STREAM_END) {
+                setState(END_OF_ARCHIVE);
+                break;
+            } else if (ret != LZMA_OK) {
+                setState(BAD_ARCHIVE);
+                break;
+            }
+        } while ((_xzStream.avail_in > 0) || (writtenSize > 0));
+    }
+
+    void flush() override
+    {
+        const auto ret = lzma_code(&_xzStream, LZMA_FINISH);
+        if (ret != LZMA_STREAM_END) {
+            setState(BAD_ARCHIVE);
+        }
+    }
+
+private:
+    lzma_stream _xzStream;
+    uint8_t* _outputBuffer = nullptr;
+};
+
+#endif
+
+///////////////////////////////////////////////////////////////////////////////
+
 extern "C" {
 	void fill_memory_filefunc(zlib_filefunc_def*);
 }
@@ -637,11 +659,13 @@ void ArchiveExtractor::extractBytes(const uint8_t* bytes, size_t count)
 
 		if (r == TarData) {
 			d.reset(new TarExtractorPrivate(this));
-		}
+        } else if (r == GZData) {
-		else if (r == ZipData) {
+            d.reset(new GZTarExtractor(this));
+        } else if (r == XZData) {
+            d.reset(new XZTarExtractor(this));
+        } else if (r == ZipData) {
             d.reset(new ZipExtractorPrivate(this));
-		}
+        } else {
-		else {
             SG_LOG(SG_IO, SG_WARN, "Invalid archive type");
 			_invalidDataType = true;
 			return;
@@ -699,8 +723,17 @@ ArchiveExtractor::DetermineResult ArchiveExtractor::determineType(const uint8_t*
 		return ZipData;
 	}
 
+    if (count < 6) {
+        return InsufficientData;
+    }
+
+    const uint8_t XZ_HEADER[6] = {0xFD, '7', 'z', 'X', 'Z', 0x00};
+    if (memcmp(bytes, XZ_HEADER, 6) == 0) {
+        return XZData;
+    }
+
     auto r = isTarData(bytes, count);
-	if ((r == TarData) || (r == InsufficientData))
+    if ((r == TarData) || (r == InsufficientData) || (r == GZData))
         return r;
 
 	return Invalid;
@@ -714,6 +747,8 @@ ArchiveExtractor::DetermineResult ArchiveExtractor::isTarData(const uint8_t* byt
     }
 
     UstarHeaderBlock* header = 0;
+    DetermineResult result = InsufficientData;
+
     if ((bytes[0] == 0x1f) && (bytes[1] == 0x8b)) {
         // GZIP identification bytes
         z_stream z;
@@ -731,11 +766,11 @@ ArchiveExtractor::DetermineResult ArchiveExtractor::isTarData(const uint8_t* byt
             return Invalid;
         }
 
-        int result = inflate(&z, Z_SYNC_FLUSH);
+        int zResult = inflate(&z, Z_SYNC_FLUSH);
-        if ((result == Z_OK) || (result == Z_STREAM_END)) {
+        if ((zResult == Z_OK) || (zResult == Z_STREAM_END)) {
             // all good
         } else {
-            SG_LOG(SG_IO, SG_WARN, "isTarData: Zlib inflate failed:" << result);
+            SG_LOG(SG_IO, SG_WARN, "isTarData: Zlib inflate failed:" << zResult);
             inflateEnd(&z);
             return Invalid; // not tar data
         }
@@ -748,6 +783,7 @@ ArchiveExtractor::DetermineResult ArchiveExtractor::isTarData(const uint8_t* byt
 
         header = reinterpret_cast<UstarHeaderBlock*>(zlibOutput);
         inflateEnd(&z);
+        result = GZData;
     } else {
         // uncompressed tar
         if (count < TAR_HEADER_BLOCK_SIZE) {
@@ -755,13 +791,14 @@ ArchiveExtractor::DetermineResult ArchiveExtractor::isTarData(const uint8_t* byt
         }
 
         header = (UstarHeaderBlock*) bytes;
+        result = TarData;
     }
 
     if (strncmp(header->magic, TMAGIC, TMAGLEN) != 0) {
         return Invalid;
     }
 
-    return TarData;
+    return result;
 }
 
 void ArchiveExtractor::extractLocalFile(const SGPath& archiveFile)

simgear/io/untar.hxx

@@ -36,12 +36,13 @@ public:
 	ArchiveExtractor(const SGPath& rootPath);
 	virtual ~ArchiveExtractor();
 
-	enum DetermineResult
+    enum DetermineResult {
-	{
         Invalid,
         InsufficientData,
         TarData,
-		ZipData
+        ZipData,
+        GZData, // Gzipped-tar
+        XZData  // XZ (aka LZMA) tar
     };
 
     static DetermineResult determineType(const uint8_t* bytes, size_t count);

simgear/package/Catalog.cxx

@@ -619,6 +619,8 @@ void Catalog::setUserEnabled(bool b)
 
 void Catalog::processAlternate(SGPropertyNode_ptr alt)
 {
+    m_refreshRequest.reset();
+
     std::string altId;
     const auto idPtr = alt->getStringValue("id");
     if (idPtr) {
@@ -694,8 +696,8 @@ void Catalog::processAlternate(SGPropertyNode_ptr alt)
 
     SG_LOG(SG_GENERAL, SG_INFO, "Migrating catalog " << id() << " to new URL:" << altUrl);
     setUrl(altUrl);
-    Downloader* dl = new Downloader(this, altUrl);
+    m_refreshRequest = new Downloader(this, altUrl);
-    root()->makeHTTPRequest(dl);
+    root()->makeHTTPRequest(m_refreshRequest);
 }
 
 int Catalog::markPackagesForInstallation(const string_list &packageIds) {

simgear/package/CatalogTest.cxx

@@ -166,7 +166,8 @@ int parseTest()
 {
     SGPath rootPath = simgear::Dir::current().path();
     rootPath.append("testRoot");
-    pkg::Root* root = new pkg::Root(rootPath, "8.1.12");
+
+    pkg::RootRef root(new pkg::Root(rootPath, "8.1.2"));
     root->setLocale("de");
     pkg::CatalogRef cat = pkg::Catalog::createFromPath(root, SGPath(SRC_DIR "/catalogTest1"));
 
@@ -344,7 +345,6 @@ int parseTest()
     SG_CHECK_EQUAL(urls.size(), 3);
     SG_CHECK_EQUAL(urls.at(1), "http://localhost:2000/mirrorB/b737.tar.gz");
 
-    delete root;
     return EXIT_SUCCESS;
 }
 
@@ -953,7 +953,7 @@ int parseInvalidTest()
 {
     SGPath rootPath = simgear::Dir::current().path();
     rootPath.append("testRoot");
-    pkg::Root* root = new pkg::Root(rootPath, "8.1.12");
+    pkg::RootRef root(new pkg::Root(rootPath, "8.1.12"));
     pkg::CatalogRef cat = pkg::Catalog::createFromPath(root, SGPath(SRC_DIR "/catalogTestInvalid"));
     SG_VERIFY(cat.valid());
 

simgear/scene/material/mat.cxx

@@ -97,11 +97,13 @@ SGMaterial::SGMaterial( const SGReaderWriterOptions* options,
                         const SGPropertyNode *props,
                         SGPropertyNode *prop_root,
                         AreaList *a,
-						SGSharedPtr<const SGCondition> c)
+                        SGSharedPtr<const SGCondition> c,
+                        const std::string& n)
 {
     init();
     areas = a;
     condition = c;
+    region = n;
     read_properties( options, props, prop_root );
     buildEffectProperties(options);
 }
@@ -110,12 +112,14 @@ SGMaterial::SGMaterial( const osgDB::Options* options,
                         const SGPropertyNode *props,
                         SGPropertyNode *prop_root,
                         AreaList *a,
-                        SGSharedPtr<const SGCondition> c)
+                        SGSharedPtr<const SGCondition> c,
+                        const std::string& n)
 {
     osg::ref_ptr<SGReaderWriterOptions> opt;
     opt = SGReaderWriterOptions::copyOrCreate(options);
     areas = a;
     condition = c;
+    region = n;
     init();
     read_properties(opt.get(), props, prop_root);
     buildEffectProperties(opt.get());

simgear/scene/material/mat.hxx

@@ -97,14 +97,16 @@ public:
              const SGPropertyNode *props,
              SGPropertyNode *prop_root,
              AreaList *a,
-			 SGSharedPtr<const SGCondition> c);
+             SGSharedPtr<const SGCondition> c,
+             const std::string& n);
 
 
   SGMaterial(const simgear::SGReaderWriterOptions*,
              const SGPropertyNode *props,
              SGPropertyNode *prop_root,
              AreaList *a,
-			 SGSharedPtr<const SGCondition> c);
+             SGSharedPtr<const SGCondition> c,
+             const std::string& n);
 
   /**
    * Destructor.
@@ -123,6 +125,11 @@ public:
   simgear::Effect* get_one_effect(int texIndex);
   simgear::Effect* get_effect();
 
+  /**
+   * Get the region Name.
+   */
+  const std::string get_region_name() const { return region; }
+
   /**
    * Get the Effect Name.
    */
@@ -487,6 +494,9 @@ private:
   SGVec4f ambient, diffuse, specular, emission;
   double shininess;
 
+  // region of this material
+  std::string region;
+
   // effect for this material
   std::string effect;
 

simgear/scene/material/matlib.cxx

@@ -128,12 +128,13 @@ bool SGMaterialLib::load( const SGPath &fg_root, const SGPath& mpath,
 
 		// Now build all the materials for this set of areas and conditions
 
+		const std::string region = node->getStringValue("name");
 		const simgear::PropertyList materials = node->getChildren("material");
 		simgear::PropertyList::const_iterator materials_iter = materials.begin();
 		for (; materials_iter != materials.end(); materials_iter++) {
 			const SGPropertyNode *node = materials_iter->get();
 			SGSharedPtr<SGMaterial> m =
-					new SGMaterial(options.get(), node, prop_root, arealist, condition);
+					new SGMaterial(options.get(), node, prop_root, arealist, condition, region);
 
 			std::vector<SGPropertyNode_ptr>names = node->getChildren("name");
 			for ( unsigned int j = 0; j < names.size(); j++ ) {

simgear/scene/model/SGText.cxx

@@ -80,7 +80,7 @@ void SGText::UpdateCallback::operator()(osg::Node * node, osg::NodeVisitor *nv )
     // be lazy and set the text only if the property has changed.
     // update() computes the glyph representation which looks 
     // more expensive than a the above string compare.
-    text->setText( buf );
+    text->setText( buf, osgText::String::ENCODING_UTF8 );
     text->update();
   }
   traverse( node, nv );

simgear/scene/model/particles.cxx

@@ -36,10 +36,10 @@
 #include <osgParticle/MultiSegmentPlacer>
 #include <osgParticle/SectorPlacer>
 #include <osgParticle/ConstantRateCounter>
-#include <osgParticle/ParticleSystemUpdater>
 #include <osgParticle/ParticleSystem>
 #include <osgParticle/FluidProgram>
 
+#include <osgUtil/CullVisitor>
 #include <osg/Geode>
 #include <osg/Group>
 #include <osg/MatrixTransform>
@@ -48,18 +48,166 @@
 
 #include <simgear/scene/model/animation.hxx>
 
+using ParticleSystemRef = osg::ref_ptr<osgParticle::ParticleSystem>;
+
+
 namespace simgear
 {
 
 class ParticlesGlobalManager::ParticlesGlobalManagerPrivate : public osg::NodeCallback
 {
 public:
-    ParticlesGlobalManagerPrivate() : _updater(new osgParticle::ParticleSystemUpdater),
+    ParticlesGlobalManagerPrivate();
-                                      _commonGeode(new osg::Geode)
+
+    void operator()(osg::Node* node, osg::NodeVisitor* nv) override;
+
+    // only call this with the lock held!
+    osg::Group* internalGetCommonRoot();
+
+    void updateParticleSystemsFromCullCallback(int currentFrameNumber, osg::NodeVisitor* nv);
+
+    void addParticleSystem(osgParticle::ParticleSystem* ps, const osg::ref_ptr<osg::Group>& frame);
+
+    void registerNewLocalParticleSystem(osg::Node* node, ParticleSystemRef ps);
+    void registerNewWorldParticleSystem(osg::Node* node, ParticleSystemRef ps, osg::Group* frame);
+
+    std::mutex _lock;
+    bool _frozen = false;
+    double _simulationDt = 0.0;
+    osg::ref_ptr<osg::Group> _commonRoot;
+    osg::ref_ptr<osg::Geode> _commonGeode;
+    osg::Vec3 _wind;
+    bool _enabled = true;
+    osg::Vec3 _gravity;
+    //  osg::Vec3 _localWind;
+    SGGeod _currentPosition;
+
+    SGConstPropertyNode_ptr _enabledNode;
+
+    using ParticleSystemWeakRef = osg::observer_ptr<osgParticle::ParticleSystem>;
+    using ParticleSystemsWeakRefVec = std::vector<ParticleSystemWeakRef>;
+
+    using ParticleSystemsStrongRefVec = std::vector<ParticleSystemRef>;
+
+    ParticleSystemsWeakRefVec _systems;
+    osg::ref_ptr<ParticlesGlobalManager::UpdaterCallback> _cullCallback;
+
+    using GroupRefVec = std::vector<osg::ref_ptr<osg::Group>>;
+    GroupRefVec _newWorldParticles;
+};
+
+/**
+ @brief this class replaces the need to use osgParticle::ParticleSystemUpdater, which has some
+ thread-safety and ownership complications in our us case
+ */
+class ParticlesGlobalManager::UpdaterCallback : public osg::NodeCallback
+{
+public:
+    UpdaterCallback(ParticlesGlobalManagerPrivate* p) : _manager(p)
     {
     }
 
     void operator()(osg::Node* node, osg::NodeVisitor* nv) override
+    {
+        osgUtil::CullVisitor* cv = dynamic_cast<osgUtil::CullVisitor*>(nv);
+        if (cv && nv->getFrameStamp()) {
+            if (_frameNumber < nv->getFrameStamp()->getFrameNumber()) {
+                _frameNumber = nv->getFrameStamp()->getFrameNumber();
+                _manager->updateParticleSystemsFromCullCallback(_frameNumber, nv);
+            }
+        }
+
+        // note, callback is responsible for scenegraph traversal so
+        // they must call traverse(node,nv) to ensure that the
+        // scene graph subtree (and associated callbacks) are traversed.
+        traverse(node, nv);
+    }
+
+    unsigned int _frameNumber = 0;
+    ParticlesGlobalManagerPrivate* _manager;
+};
+
+/**
+  single-shot node callback, used to register a particle system with the global manager. Once run, removes
+  itself. We used this to avoid updating a particle system until the load is complete and merged into the
+  main scene.
+ */
+class ParticlesGlobalManager::RegistrationCallback : public osg::NodeCallback
+{
+public:
+    void operator()(osg::Node* node, osg::NodeVisitor* nv) override
+    {
+        auto d = ParticlesGlobalManager::instance()->d;
+        d->addParticleSystem(_system, _frame);
+        node->removeUpdateCallback(this); // suicide
+    }
+
+    ParticleSystemRef _system;
+    osg::ref_ptr<osg::Group> _frame;
+};
+
+ParticlesGlobalManager::ParticlesGlobalManagerPrivate::ParticlesGlobalManagerPrivate() : _commonGeode(new osg::Geode),
+                                                                                         _cullCallback(new UpdaterCallback(this))
+{
+    // callbacks are registered in initFromMainThread : depending on timing,
+    // this constructor might be called from an osgDB thread
+}
+
+void ParticlesGlobalManager::ParticlesGlobalManagerPrivate::addParticleSystem(osgParticle::ParticleSystem* ps, const osg::ref_ptr<osg::Group>& frame)
+{
+    std::lock_guard<std::mutex> g(_lock);
+    _systems.push_back(ps);
+
+    // we're inside an update callback here, so better not modify the
+    // scene structure; defer it to later
+    if (frame.get()) {
+        _newWorldParticles.push_back(frame);
+    }
+}
+
+void ParticlesGlobalManager::ParticlesGlobalManagerPrivate::registerNewLocalParticleSystem(osg::Node* node, ParticleSystemRef ps)
+{
+    auto cb = new RegistrationCallback;
+    cb->_system = ps;
+    node->addUpdateCallback(cb);
+}
+
+void ParticlesGlobalManager::ParticlesGlobalManagerPrivate::registerNewWorldParticleSystem(osg::Node* node, ParticleSystemRef ps, osg::Group* frame)
+{
+    auto cb = new RegistrationCallback;
+    cb->_system = ps;
+    cb->_frame = frame;
+    node->addUpdateCallback(cb);
+}
+
+// this is called from the main thread during scenery init
+// after this, we beign updarting particle systems
+void ParticlesGlobalManager::initFromMainThread()
+{
+    std::lock_guard<std::mutex> g(d->_lock);
+    d->internalGetCommonRoot();
+    d->_commonRoot->addUpdateCallback(d.get());
+    d->_commonRoot->setCullingActive(false);
+    d->_commonRoot->addCullCallback(d->_cullCallback);
+}
+
+void ParticlesGlobalManager::update(double dt, const SGGeod& pos)
+{
+    std::lock_guard<std::mutex> g(d->_lock);
+    d->_simulationDt = dt;
+    d->_currentPosition = pos;
+
+    for (auto f : d->_newWorldParticles) {
+        d->internalGetCommonRoot()->addChild(f);
+    }
+
+    d->_newWorldParticles.clear();
+}
+
+// this is called from the main thread, since it's an update callback
+// lock any state used by updateParticleSystemsFromCullCallback, which
+// runs during culling, potentialy on a different thread
+void ParticlesGlobalManager::ParticlesGlobalManagerPrivate::operator()(osg::Node* node, osg::NodeVisitor* nv)
 {
     std::lock_guard<std::mutex> g(_lock);
     _enabled = !_enabledNode || _enabledNode->getBoolValue();
@@ -67,47 +215,82 @@ public:
     if (!_enabled)
         return;
 
-        const auto q = SGQuatd::fromLonLatDeg(_longitudeNode->getFloatValue(), _latitudeNode->getFloatValue());
+    const auto q = SGQuatd::fromLonLat(_currentPosition);
     osg::Matrix om(toOsg(q));
     osg::Vec3 v(0, 0, 9.81);
-
     _gravity = om.preMult(v);
 
     // NOTE: THIS WIND COMPUTATION DOESN'T SEEM TO AFFECT PARTICLES
     // const osg::Vec3& zUpWind = _wind;
     // osg::Vec3 w(zUpWind.y(), zUpWind.x(), -zUpWind.z());
     // _localWind = om.preMult(w);
+
+    // while we have the lock, remove all expired systems
+    auto firstInvalid = std::remove_if(_systems.begin(), _systems.end(), [](const ParticleSystemWeakRef& weak) {
+        return !weak.valid();
+    });
+    _systems.erase(firstInvalid, _systems.end());
 }
 
 // only call this with the lock held!
-    osg::Group* internalGetCommonRoot()
+osg::Group* ParticlesGlobalManager::ParticlesGlobalManagerPrivate::internalGetCommonRoot()
 {
     if (!_commonRoot.valid()) {
-            SG_LOG(SG_PARTICLES, SG_DEBUG, "Particle common root called.");
         _commonRoot = new osg::Group;
         _commonRoot->setName("common particle system root");
         _commonGeode->setName("common particle system geode");
         _commonRoot->addChild(_commonGeode);
-            _commonRoot->addChild(_updater);
         _commonRoot->setNodeMask(~simgear::MODELLIGHT_BIT);
     }
     return _commonRoot.get();
 }
 
-    std::mutex _lock;
+void ParticlesGlobalManager::ParticlesGlobalManagerPrivate::updateParticleSystemsFromCullCallback(int frameNumber, osg::NodeVisitor* nv)
-    bool _frozen = false;
+{
-    osg::ref_ptr<osgParticle::ParticleSystemUpdater> _updater;
+    ParticleSystemsStrongRefVec activeSystems;
-    osg::ref_ptr<osg::Group> _commonRoot;
+    double dt = 0.0;
-    osg::ref_ptr<osg::Geode> _commonGeode;
-    osg::Vec3 _wind;
-    bool _globalCallbackRegistered = false;
-    bool _enabled = true;
-    osg::Vec3 _gravity;
-    //  osg::Vec3 _localWind;
 
-    SGConstPropertyNode_ptr _enabledNode;
+    // begin locked section
-    SGConstPropertyNode_ptr _longitudeNode, _latitudeNode;
+    {
-};
+        std::lock_guard<std::mutex> g(_lock);
+        activeSystems.reserve(_systems.size());
+        for (const auto& psref : _systems) {
+            ParticleSystemRef owningRef;
+            if (!psref.lock(owningRef)) {
+                // pointed to system is gone, skip it
+                // we will clean these up in the update callback, don't
+                // worry about that here
+                continue;
+            }
+
+            // add to the list we will update now
+            activeSystems.push_back(owningRef);
+        }
+
+        dt = _simulationDt;
+    } // of locked section
+
+    // from here on, don't access class data; copy it all to local variables
+    // before this line. this is important so we're not holding _lock during
+    // culling, which might block osgDB threads for example.
+
+    if (dt <= 0.0) {
+        return;
+    }
+
+    for (const auto& ps : activeSystems) {
+        // code inside here is copied from osgParticle::ParticleSystemUpdate
+        osgParticle::ParticleSystem::ScopedWriteLock lock(*(ps->getReadWriteMutex()));
+
+        // We need to allow at least 2 frames difference, because the particle system's lastFrameNumber
+        // is updated in the draw thread which may not have completed yet.
+        if (!ps->isFrozen() &&
+            (!ps->getFreezeOnCull() || ((frameNumber - ps->getLastFrameNumber()) <= 2))) {
+            ps->update(dt, *nv);
+        }
+        // end of code copied from osgParticle::ParticleSystemUpdate
+    }
+}
 
 static std::mutex static_managerLock;
 static std::unique_ptr<ParticlesGlobalManager> static_instance;
@@ -132,14 +315,6 @@ ParticlesGlobalManager::ParticlesGlobalManager() : d(new ParticlesGlobalManagerP
 {
 }
 
-ParticlesGlobalManager::~ParticlesGlobalManager()
-{
-    if (d->_globalCallbackRegistered) {
-        // is this actually necessary? possibly not
-        d->_updater->setUpdateCallback(nullptr);
-    }
-}
-
 bool ParticlesGlobalManager::isEnabled() const
 {
     std::lock_guard<std::mutex> g(d->_lock);
@@ -728,23 +903,10 @@ osg::ref_ptr<osg::Group> ParticlesGlobalManager::appendParticles(const SGPropert
         emitter->setUpdateCallback(callback.get());
     }
 
-    // touch shared data now (and not before)
+    if (attach == "local") {
-
+        d->registerNewLocalParticleSystem(align, particleSys);
-    {
+    } else {
-        std::lock_guard<std::mutex> g(d->_lock);
+        d->registerNewWorldParticleSystem(align, particleSys, callback()->particleFrame);
-        d->_updater->addParticleSystem(particleSys);
-
-        if (attach != "local") {
-            d->internalGetCommonRoot()->addChild(callback()->particleFrame);
-        }
-
-        if (!d->_globalCallbackRegistered) {
-            SG_LOG(SG_PARTICLES, SG_INFO, "Registering global particles callback");
-            d->_globalCallbackRegistered = true;
-            d->_longitudeNode = modelRoot->getNode("/position/longitude-deg", true);
-            d->_latitudeNode = modelRoot->getNode("/position/latitude-deg", true);
-            d->_updater->setUpdateCallback(d.get());
-        }
     }
 
     return align;

simgear/scene/model/particles.hxx

@@ -41,7 +41,6 @@ class NodeVisitor;
 namespace osgParticle
 {
 class ParticleSystem;
-class ParticleSystemUpdater;
 }
 
 #include <simgear/scene/util/SGNodeMasks.hxx>
@@ -151,7 +150,7 @@ protected:
 class ParticlesGlobalManager
 {
 public:
-    ~ParticlesGlobalManager();
+    ~ParticlesGlobalManager() = default;
 
     static ParticlesGlobalManager* instance();
     static void clear();
@@ -161,14 +160,18 @@ public:
     osg::ref_ptr<osg::Group> appendParticles(const SGPropertyNode* configNode, SGPropertyNode* modelRoot, const osgDB::Options* options);
 
     osg::Group* getCommonRoot();
-
     osg::Geode* getCommonGeode();
 
-    osgParticle::ParticleSystemUpdater* getPSU();
+    void initFromMainThread();
 
     void setFrozen(bool e);
     bool isFrozen() const;
 
+    /**
+        @brief update function: call from the main thread , outside of OSG calls.
+     */
+    void update(double dt, const SGGeod& pos);
+
     void setSwitchNode(const SGPropertyNode* n);
 
     /**
@@ -185,6 +188,8 @@ private:
     ParticlesGlobalManager();
 
     class ParticlesGlobalManagerPrivate;
+    class UpdaterCallback;
+    class RegistrationCallback;
 
     // because Private inherits NodeCallback, we need to own it
     // via an osg::ref_ptr

simgear/scene/tsync/terrasync.cxx

@@ -109,13 +109,13 @@ bool hasWhitespace(string path)
 class SyncItem
 {
 public:
-    enum Type
+    enum Type {
-    {
         Stop = 0, ///< special item indicating to stop the SVNThread
         Tile,
         AirportData,
         SharedModels,
-        AIData
+        AIData,
+        OSMTile ///< OSm2City per-Tile data
     };
 
     enum Status
@@ -165,13 +165,16 @@ public:
     SGTimeStamp stamp;
     bool busy = false; ///< is the slot working or idle
     unsigned int pendingKBytes = 0;
+    unsigned int pendingExtractKBytes = 0;
     unsigned int nextWarnTimeout = 0;
 };
 
 static const int SYNC_SLOT_TILES = 0; ///< Terrain and Objects sync
 static const int SYNC_SLOT_SHARED_DATA = 1; /// shared Models and Airport data
 static const int SYNC_SLOT_AI_DATA = 2; /// AI traffic and models
-static const unsigned int NUM_SYNC_SLOTS = 3;
+static const int SYNC_SLOT_OSM_TILE_DATA = 3;
+
+static const unsigned int NUM_SYNC_SLOTS = 4;
 
 /**
  * @brief translate a sync item type into one of the available slots.
@@ -186,7 +189,8 @@ static unsigned int syncSlotForType(SyncItem::Type ty)
         return SYNC_SLOT_SHARED_DATA;
     case SyncItem::AIData:
         return SYNC_SLOT_AI_DATA;
-
+    case SyncItem::OSMTile:
+        return SYNC_SLOT_OSM_TILE_DATA;
     default:
         return SYNC_SLOT_SHARED_DATA;
     }
@@ -194,8 +198,7 @@ static unsigned int syncSlotForType(SyncItem::Type ty)
 
 struct TerrasyncThreadState
 {
-    TerrasyncThreadState() :
+    TerrasyncThreadState() : _busy(false),
-        _busy(false),
                              _stalled(false),
                              _hasServer(false),
                              _fail_count(0),
@@ -205,7 +208,8 @@ struct TerrasyncThreadState
                              _cache_hits(0),
                              _transfer_rate(0),
                              _total_kb_downloaded(0),
-        _totalKbPending(0)
+                             _totalKbPending(0),
+                             _extractTotalKbPending(0)
     {}
 
     bool _busy;
@@ -220,6 +224,7 @@ struct TerrasyncThreadState
     // kbytes, not bytes, because bytes might overflow 2^31
     int _total_kb_downloaded;
     unsigned int _totalKbPending;
+    unsigned int _extractTotalKbPending;
 };
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -275,9 +280,10 @@ public:
 
    SyncItem getNewTile() { return _freshTiles.pop_front();}
 
-   void   setHTTPServer(const std::string& server)
+   void setHTTPServer(const std::string& server, const std::string& osmServer)
    {
       _httpServer = stripPath(server);
+      _osmCityServer = stripPath(osmServer);
       _isAutomaticServer = (server == "automatic");
    }
 
@@ -296,6 +302,11 @@ public:
      _sceneryVersion = simgear::strutils::strip(sceneryVersion);
    }
 
+   void setOSMCityVersion(const std::string& osmCityVersion)
+   {
+       _osmCityService = osmCityVersion;
+   }
+
    void   setLocalDir(string dir)           { _local_dir    = stripPath(dir);}
    string getLocalDir()                     { return _local_dir;}
 
@@ -322,6 +333,8 @@ public:
     void setCachePath(const SGPath &p) { _persistentCachePath = p; }
 
   private:
+      std::string dnsSelectServerForService(const std::string& service);
+
       void incrementCacheHits()
       {
           std::lock_guard<std::mutex> g(_stateLock);
@@ -334,9 +347,9 @@ public:
     void runInternal();
     void updateSyncSlot(SyncSlot& slot);
 
-    void beginSyncAirports(SyncSlot& slot);
+    bool beginSyncAirports(SyncSlot& slot);
-    void beginSyncTile(SyncSlot& slot);
+    bool beginSyncTile(SyncSlot& slot);
-    void beginNormalSync(SyncSlot& slot);
+    bool beginNormalSync(SyncSlot& slot);
 
     void drainWaitingTiles();
 
@@ -363,6 +376,9 @@ public:
     string _local_dir;
     SGPath _persistentCachePath;
     string _httpServer;
+    string _osmCityServer;
+    string _osmCityService = "o2c";
+
     bool _isAutomaticServer;
     SGPath _installRoot;
     string _sceneryVersion;
@@ -474,8 +490,19 @@ bool SGTerraSync::WorkerThread::findServer()
 {
     if ( false == _isAutomaticServer ) return true;
 
+    _httpServer = dnsSelectServerForService(MakeQService(_protocol, _sceneryVersion));
+
+    if (!_osmCityService.empty()) {
+        _osmCityServer = dnsSelectServerForService(_osmCityService);
+    }
+
+    return !_httpServer.empty();
+}
+
+std::string SGTerraSync::WorkerThread::dnsSelectServerForService(const std::string& service)
+{
     DNS::NAPTRRequest * naptrRequest = new DNS::NAPTRRequest(_dnsdn);
-    naptrRequest->qservice = MakeQService(_protocol, _sceneryVersion);
+    naptrRequest->qservice = service;
 
     naptrRequest->qflags = "U";
     DNS::Request_ptr r(naptrRequest);
@@ -483,14 +510,15 @@ bool SGTerraSync::WorkerThread::findServer()
     DNS::Client dnsClient;
     dnsClient.makeRequest(r);
     SG_LOG(SG_TERRASYNC,SG_DEBUG,"DNS NAPTR query for '" << _dnsdn << "' '" << naptrRequest->qservice << "'" );
-    while( !r->isComplete() && !r->isTimeout() )
+    while (!r->isComplete() && !r->isTimeout()) {
         dnsClient.update(0);
+    }
 
     if( naptrRequest->entries.empty() ) {
         SG_LOG(SG_TERRASYNC, SG_ALERT, "Warning: no DNS entry found for '" << _dnsdn << "' '" << naptrRequest->qservice << "'" );
-        _httpServer = "";
+        return {};
-        return false;
     }
+
     // walk through responses, they are ordered by 1. order and 2. preference
     // For now, only take entries with lowest order
     // TODO: try all available servers in the order given by preferenc and order
@@ -498,38 +526,33 @@ bool SGTerraSync::WorkerThread::findServer()
 
     // get all servers with this order and the same (for now only lowest preference)
     DNS::NAPTRRequest::NAPTR_list availableServers;
-    for( DNS::NAPTRRequest::NAPTR_list::const_iterator it = naptrRequest->entries.begin();
+    for (const auto& entry : naptrRequest->entries) {
-         it != naptrRequest->entries.end();
+        if (entry->order != order)
-         ++it ) {
-
-        if( (*it)->order != order )
             continue;
 
-        string regex = (*it)->regexp;
+        const string regex = entry->regexp;
-        if( false == simgear::strutils::starts_with( (*it)->regexp, "!^.*$!" ) ) {
+        if (false == simgear::strutils::starts_with(regex, "!^.*$!")) {
-            SG_LOG(SG_TERRASYNC,SG_WARN, "ignoring unsupported regexp: " << (*it)->regexp );
+            SG_LOG(SG_TERRASYNC, SG_WARN, "ignoring unsupported regexp: " << regex);
             continue;
         }
 
-        if( false == simgear::strutils::ends_with( (*it)->regexp, "!" ) ) {
+        if (false == simgear::strutils::ends_with(regex, "!")) {
-            SG_LOG(SG_TERRASYNC,SG_WARN, "ignoring unsupported regexp: " << (*it)->regexp );
+            SG_LOG(SG_TERRASYNC, SG_WARN, "ignoring unsupported regexp: " << regex);
             continue;
         }
 
         // always use first entry
-        if( availableServers.empty() || (*it)->preference == availableServers[0]->preference) {
+        if (availableServers.empty() || entry->preference == availableServers[0]->preference) {
-            SG_LOG(SG_TERRASYNC,SG_DEBUG, "available server regexp: " << (*it)->regexp );
+            SG_LOG(SG_TERRASYNC, SG_DEBUG, "available server regexp: " << regex);
-            availableServers.push_back( *it );
+            availableServers.push_back(entry);
         }
     }
 
     // now pick a random entry from the available servers
-    DNS::NAPTRRequest::NAPTR_list::size_type idx = sg_random() * availableServers.size();
+    auto idx = static_cast<int>(sg_random() * availableServers.size());
-    _httpServer = availableServers[idx]->regexp;
+    const auto server = availableServers.at(idx)->regexp;
-    _httpServer = _httpServer.substr( 6, _httpServer.length()-7 ); // strip search pattern and separators
+    SG_LOG(SG_TERRASYNC, SG_INFO, "picking entry # " << idx << ", server is " << server.substr(6, server.length() - 7););
-
+    return server.substr(6, server.length() - 7);
-    SG_LOG(SG_TERRASYNC,SG_INFO, "picking entry # " << idx << ", server is " << _httpServer );
-    return true;
 }
 
 void SGTerraSync::WorkerThread::run()
@@ -562,6 +585,7 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
 #endif
             // convert bytes to kbytes here
             slot.pendingKBytes = (slot.repository->bytesToDownload() >> 10);
+            slot.pendingExtractKBytes = (slot.repository->bytesToExtract() >> 10);
             return; // easy, still working
         }
 
@@ -569,6 +593,9 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
         HTTPRepository::ResultCode res = slot.repository->failure();
 
         if (res == HTTPRepository::REPO_ERROR_NOT_FOUND) {
+            // not founds should never happen any more (unless the server-
+            // side data is incorrect), since we now check top-down that
+            // a 1x1 dir exists or not.
             notFound(slot.currentItem);
         } else if (res != HTTPRepository::REPO_NO_ERROR) {
             fail(slot.currentItem);
@@ -591,6 +618,7 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
         slot.busy = false;
         slot.repository.reset();
         slot.pendingKBytes = 0;
+        slot.pendingExtractKBytes = 0;
         slot.currentItem = {};
     }
 
@@ -604,12 +632,23 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
         slot.isNewDirectory = !path.exists();
         const auto type = slot.currentItem._type;
 
+        bool ok = false;
         if (type == SyncItem::AirportData) {
-            beginSyncAirports(slot);
+            ok = beginSyncAirports(slot);
+        } else if (type == SyncItem::OSMTile) {
+            ok = beginSyncTile(slot);
         } else if (type == SyncItem::Tile) {
-            beginSyncTile(slot);
+            ok = beginSyncTile(slot);
         } else {
-            beginNormalSync(slot);
+            ok = beginNormalSync(slot);
+        }
+
+        if (!ok) {
+            SG_LOG(SG_TERRASYNC, SG_INFO, "sync of " << slot.currentItem._dir << " failed to start");
+            fail(slot.currentItem);
+            slot.busy = false;
+            slot.repository.reset();
+            return;
         }
 
         try {
@@ -626,17 +665,17 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
         slot.nextWarnTimeout = 30 * 1000;
         slot.stamp.stamp();
         slot.busy = true;
-        slot.pendingKBytes = slot.repository->bytesToDownload();
+        slot.pendingKBytes = slot.repository->bytesToDownload() >> 10;
+        slot.pendingExtractKBytes = slot.repository->bytesToExtract() >> 10;
 
-        SG_LOG(SG_TERRASYNC, SG_INFO, "sync of " << slot.repository->baseUrl() << " started, queue size is " << slot.queue.size());
+        SG_LOG(SG_TERRASYNC, SG_INFO, "sync of " << slot.repository->baseUrl() << ":" << slot.currentItem._dir << " started, queue size is " << slot.queue.size());
     }
 }
 
-void SGTerraSync::WorkerThread::beginSyncAirports(SyncSlot& slot)
+bool SGTerraSync::WorkerThread::beginSyncAirports(SyncSlot& slot)
 {
     if (!slot.isNewDirectory) {
-        beginNormalSync(slot);
+        return beginNormalSync(slot);
-        return;
     }
 
     SG_LOG(SG_TERRASYNC, SG_INFO, "doing Airports download via tarball");
@@ -656,9 +695,10 @@ void SGTerraSync::WorkerThread::beginSyncAirports(SyncSlot& slot)
     };
 
     slot.repository->setFilter(f);
+    return true;
 }
 
-void SGTerraSync::WorkerThread::beginSyncTile(SyncSlot& slot)
+bool SGTerraSync::WorkerThread::beginSyncTile(SyncSlot& slot)
 {
     // avoid 404 requests by doing a sync which excludes all paths
     // except our tile path. In the case of a missing 1x1 tile, we will
@@ -667,8 +707,7 @@ void SGTerraSync::WorkerThread::beginSyncTile(SyncSlot& slot)
     auto comps = strutils::split(slot.currentItem._dir, "/");
     if (comps.size() != 3) {
         SG_LOG(SG_TERRASYNC, SG_ALERT, "Bad tile path:" << slot.currentItem._dir);
-        beginNormalSync(slot);
+        return false;
-        return;
     }
 
     const auto tileCategory = comps.front();
@@ -677,7 +716,17 @@ void SGTerraSync::WorkerThread::beginSyncTile(SyncSlot& slot)
 
     const auto path = SGPath::fromUtf8(_local_dir) / tileCategory;
     slot.repository.reset(new HTTPRepository(path, &_http));
+
+    if (slot.currentItem._type == SyncItem::OSMTile) {
+        if (_osmCityServer.empty()) {
+            SG_LOG(SG_TERRASYNC, SG_WARN, "No OSM2City server defined for:" << slot.currentItem._dir);
+            return false;
+        }
+
+        slot.repository->setBaseUrl(_osmCityServer + "/" + tileCategory);
+    } else {
         slot.repository->setBaseUrl(_httpServer + "/" + tileCategory);
+    }
 
     if (_installRoot.exists()) {
       SGPath p = _installRoot / tileCategory;
@@ -685,6 +734,7 @@ void SGTerraSync::WorkerThread::beginSyncTile(SyncSlot& slot)
     }
 
     const auto dirPrefix = tenByTenDir + "/" + oneByOneDir;
+    using simgear::strutils::starts_with;
 
     // filter callback to *only* sync the 1x1 dir we want, if it exists
     // if doesn't, we'll simply stop, which is what we want
@@ -693,7 +743,10 @@ void SGTerraSync::WorkerThread::beginSyncTile(SyncSlot& slot)
         if (item.directory.empty()) {
             return item.filename == tenByTenDir;
         } else if (item.directory == tenByTenDir) {
-            return item.filename == oneByOneDir;
+            // allow 10x10 dir to contain 1x1_dir.tgz/.zip and still be accepted
+            // this does mean we'd also download oneByOneDir_foobar but that
+            // doesn't seem unreasonable either
+            return starts_with(item.filename, oneByOneDir);
         }
 
         // allow arbitrary children below dirPrefix, including sub-dirs
@@ -706,9 +759,10 @@ void SGTerraSync::WorkerThread::beginSyncTile(SyncSlot& slot)
     };
 
     slot.repository->setFilter(f);
+    return true;
 }
 
-void SGTerraSync::WorkerThread::beginNormalSync(SyncSlot& slot)
+bool SGTerraSync::WorkerThread::beginNormalSync(SyncSlot& slot)
 {
     SGPath path(_local_dir);
     path.append(slot.currentItem._dir);
@@ -720,6 +774,8 @@ void SGTerraSync::WorkerThread::beginNormalSync(SyncSlot& slot)
       p.append(slot.currentItem._dir);
       slot.repository->setInstalledCopyPath(p);
     }
+
+    return true;
 }
 
 void SGTerraSync::WorkerThread::runInternal()
@@ -762,20 +818,24 @@ void SGTerraSync::WorkerThread::runInternal()
 
         bool anySlotBusy = false;
         unsigned int newPendingCount = 0;
+        unsigned int newExtractCount = 0; // how much is left to extract
 
         // update each sync slot in turn
         for (unsigned int slot=0; slot < NUM_SYNC_SLOTS; ++slot) {
             updateSyncSlot(_syncSlots[slot]);
             newPendingCount += _syncSlots[slot].pendingKBytes;
+            newExtractCount += _syncSlots[slot].pendingExtractKBytes;
             anySlotBusy |= _syncSlots[slot].busy;
         }
 
         {
             std::lock_guard<std::mutex> g(_stateLock);
             _state._totalKbPending = newPendingCount; // approximately atomic update
+            _state._extractTotalKbPending = newExtractCount;
             _state._busy = anySlotBusy;
         }
 
+
         if (!anySlotBusy) {
             // wait on the blocking deque here, otherwise we spin
             // the loop very fast, since _http::update with no connections
@@ -811,12 +871,19 @@ void SGTerraSync::WorkerThread::fail(SyncItem failedItem)
 {
     std::lock_guard<std::mutex> g(_stateLock);
     time_t now = time(0);
+
+    if (_osmCityServer.empty() && (failedItem._type == SyncItem::OSMTile)) {
+        // don't count these as errors, otherwise normla sync will keep
+        // being abandoned
+    } else {
         _state._consecutive_errors++;
         _state._fail_count++;
+    }
+
     failedItem._status = SyncItem::Failed;
     _freshTiles.push_back(failedItem);
     // not we also end up here for partial syncs
-    SG_LOG(SG_TERRASYNC,SG_INFO,
+    SG_LOG(SG_TERRASYNC, SG_WARN,
            "Failed to sync'" << failedItem._dir << "'");
     _completedTiles[ failedItem._dir ] = now + UpdateInterval::FailedAttempt;
 }
@@ -828,6 +895,8 @@ void SGTerraSync::WorkerThread::notFound(SyncItem item)
     // we don't spam the server with lookups for models that don't
     // exist
 
+    SG_LOG(SG_TERRASYNC, SG_WARN, "Not found for: '" << item._dir << "'");
+
     time_t now = time(0);
     item._status = SyncItem::NotFound;
     _freshTiles.push_back(item);
@@ -1040,8 +1109,11 @@ void SGTerraSync::reinit()
     if (enabled)
     {
         _availableNode->setBoolValue(true);
-        _workerThread->setHTTPServer( _terraRoot->getStringValue("http-server","automatic") );
+        _workerThread->setHTTPServer(
+            _terraRoot->getStringValue("http-server", "automatic"),
+            _terraRoot->getStringValue("osm2city-server", ""));
         _workerThread->setSceneryVersion( _terraRoot->getStringValue("scenery-version","ws20") );
+        _workerThread->setOSMCityVersion(_terraRoot->getStringValue("osm2city-version", "o2c"));
         _workerThread->setProtocol( _terraRoot->getStringValue("protocol","") );
 #if 1
         // leave it hardcoded for now, not sure about the security implications for now
@@ -1052,6 +1124,9 @@ void SGTerraSync::reinit()
 
         SGPath sceneryRoot{_terraRoot->getStringValue("scenery-dir", "")};
         _workerThread->setLocalDir(sceneryRoot.utf8Str());
+        if (sceneryRoot.exists()) {
+            writeWarningFile(sceneryRoot);
+        }
 
         SGPath installPath(_terraRoot->getStringValue("installation-dir"));
         _workerThread->setInstalledDir(installPath);
@@ -1100,6 +1175,7 @@ void SGTerraSync::bind()
     _transferRateBytesSecNode = _terraRoot->getNode("transfer-rate-bytes-sec", true);
     _pendingKbytesNode = _terraRoot->getNode("pending-kbytes", true);
     _downloadedKBtesNode = _terraRoot->getNode("downloaded-kbytes", true);
+    _extractPendingKbytesNode = _terraRoot->getNode("extract-pending-kbytes", true);
     _enabledNode = _terraRoot->getNode("enabled", true);
     _availableNode = _terraRoot->getNode("available", true);
     _maxErrorsNode = _terraRoot->getNode("max-errors", true);
@@ -1125,6 +1201,7 @@ void SGTerraSync::unbind()
     _transferRateBytesSecNode.clear();
     _pendingKbytesNode.clear();
     _downloadedKBtesNode.clear();
+    _extractPendingKbytesNode.clear();
     _enabledNode.clear();
     _availableNode.clear();
     _maxErrorsNode.clear();
@@ -1161,6 +1238,7 @@ void SGTerraSync::update(double)
     _transferRateBytesSecNode->setIntValue(copiedState._transfer_rate);
     _pendingKbytesNode->setIntValue(copiedState._totalKbPending);
     _downloadedKBtesNode->setIntValue(copiedState._total_kb_downloaded);
+    _extractPendingKbytesNode->setIntValue(copiedState._extractTotalKbPending);
 
     _stalledNode->setBoolValue(_workerThread->isStalled());
     _activeNode->setBoolValue(worker_running);
@@ -1210,6 +1288,10 @@ string_list SGTerraSync::getSceneryPathSuffixes() const
     return scenerySuffixes;
 }
 
+bool isOSMSuffix(const std::string& suffix)
+{
+    return (suffix == "Buildings") || (suffix == "Roads") || (suffix == "Pylons") || (suffix == "Details");
+}
 
 void SGTerraSync::syncAreaByPath(const std::string& aPath)
 {
@@ -1222,8 +1304,7 @@ void SGTerraSync::syncAreaByPath(const std::string& aPath)
         if (_workerThread->isDirActive(dir)) {
             continue;
         }
-
+        SyncItem w(dir, isOSMSuffix(suffix) ? SyncItem::OSMTile : SyncItem::Tile);
-        SyncItem w(dir, SyncItem::Tile);
         _workerThread->request( w );
     }
 }
@@ -1242,6 +1323,11 @@ bool SGTerraSync::isTileDirPending(const std::string& sceneryDir) const
     }
 
     for (const auto& suffix : getSceneryPathSuffixes()) {
+        // don't wait on OSM dirs, even if enabled
+        if (isOSMSuffix(suffix)) {
+            continue;
+        }
+
         const auto s = suffix + "/" + sceneryDir;
         if (_workerThread->isDirActive(s)) {
             return true;
@@ -1279,6 +1365,23 @@ void SGTerraSync::reposition()
     // stub, remove
 }
 
+void SGTerraSync::writeWarningFile(const SGPath& sceneryDir)
+{
+    SGPath p = sceneryDir / "TerraSync-WARNING.txt";
+    if (p.exists())
+        return;
+
+    sg_ofstream os(p, std::ios::out | std::ios::trunc);
+    os << "This folder is managed by FlightGear's download system.\n";
+    os << "Any changes you make here or in sub-folders will be overwritten when TerraSync\n";
+    os << "downloads updates.\n";
+    os << "\n";
+    os << "To use custom scenery or data with FlightGear, put it in a differnet location\n";
+    os << "on your computer, then add the location using either the launcher 'Add-ons' page, or by\n";
+    os << "passing '--fg-scenery=<location>' on the command line.";
+    os << endl;
+}
+
 
 // Register the subsystem.
 SGSubsystemMgr::Registrant<SGTerraSync> registrantSGTerraSync(

simgear/scene/tsync/terrasync.hxx

@@ -90,6 +90,9 @@ protected:
 
     class WorkerThread;
 
+private:
+    void writeWarningFile(const SGPath& sceneryDir);
+
 private:
     WorkerThread* _workerThread;
     SGPropertyNode_ptr _terraRoot;
@@ -107,6 +110,7 @@ private:
     SGPropertyNode_ptr _transferRateBytesSecNode;
     SGPropertyNode_ptr _pendingKbytesNode;
     SGPropertyNode_ptr _downloadedKBtesNode;
+    SGPropertyNode_ptr _extractPendingKbytesNode;
     SGPropertyNode_ptr _maxErrorsNode;
 
     // we manually bind+init TerraSync during early startup

simgear/structure/exception.cxx

@@ -19,7 +19,7 @@ static ThrowCallback static_callback;
 // Implementation of sg_location class.
 ////////////////////////////////////////////////////////////////////////
 
-sg_location::sg_location ()
+sg_location::sg_location() noexcept
     : _line(-1),
       _column(-1),
       _byte(-1)
@@ -27,7 +27,7 @@ sg_location::sg_location ()
     _path[0] = '\0';
 }
 
-sg_location::sg_location (const std::string& path, int line, int column)
+sg_location::sg_location(const std::string& path, int line, int column) noexcept
     : _line(line),
       _column(column),
       _byte(-1)
@@ -35,7 +35,7 @@ sg_location::sg_location (const std::string& path, int line, int column)
   setPath(path.c_str());
 }
 
-sg_location::sg_location (const SGPath& path, int line, int column)
+sg_location::sg_location(const SGPath& path, int line, int column) noexcept
     : _line(line),
       _column(column),
       _byte(-1)
@@ -43,7 +43,7 @@ _byte(-1)
     setPath(path.utf8Str().c_str());
 }
 
-sg_location::sg_location (const char* path, int line, int column)
+sg_location::sg_location(const char* path, int line, int column) noexcept
     : _line(line),
       _column(column),
       _byte(-1)
@@ -51,7 +51,8 @@ sg_location::sg_location (const char* path, int line, int column)
   setPath(path);
 }
 
-void sg_location::setPath(const char *path) {
+void sg_location::setPath(const char* path) noexcept
+{
     if (path) {
         strncpy(_path, path, max_path);
         _path[max_path - 1] = '\0';
@@ -60,25 +61,23 @@ void sg_location::setPath(const char *path) {
     }
 }
 
-const char *sg_location::getPath() const { return _path; }
+const char* sg_location::getPath() const noexcept { return _path; }
 
-int sg_location::getLine() const { return _line; }
+int sg_location::getLine() const noexcept { return _line; }
 
-int
+int sg_location::getColumn() const noexcept
-sg_location::getColumn () const
 {
   return _column;
 }
 
 
-int
+int sg_location::getByte() const noexcept
-sg_location::getByte () const
 {
   return _byte;
 }
 
 std::string
-sg_location::asString () const
+sg_location::asString() const noexcept
 {
   std::ostringstream out;
   if (_path[0]) {
@@ -97,46 +96,48 @@ sg_location::asString () const
   return out.str();
 }
 
-bool sg_location::isValid() const { return strlen(_path) > 0; }
+bool sg_location::isValid() const noexcept { return strlen(_path) > 0; }
 
 ////////////////////////////////////////////////////////////////////////
 // Implementation of sg_throwable class.
 ////////////////////////////////////////////////////////////////////////
 
-sg_throwable::sg_throwable ()
+sg_throwable::sg_throwable() noexcept
 {
   _message[0] = '\0';
   _origin[0] = '\0';
 }
 
 sg_throwable::sg_throwable(const char* message, const char* origin,
-                           const sg_location &loc) {
+                           const sg_location& loc, bool report) noexcept : _location(loc)
+{
     setMessage(message);
     setOrigin(origin);
 
-  if (static_callback) {
+    if (static_callback && report) {
         static_callback(_message, _origin, loc);
     }
 }
 
-sg_throwable::~sg_throwable ()
-{
-}
-
 const char*
-sg_throwable::getMessage () const
+sg_throwable::getMessage() const noexcept
 {
   return _message;
 }
 
-const std::string
+std::string
-sg_throwable::getFormattedMessage () const
+sg_throwable::getFormattedMessage() const noexcept
 {
-  return std::string(getMessage());
+    std::string ret = getMessage();
+    const std::string loc = getLocation().asString();
+    if (loc.length()) {
+        ret += "\n at ";
+        ret += loc;
+    }
+    return ret;
 }
 
-void
+void sg_throwable::setMessage(const char* message) noexcept
-sg_throwable::setMessage (const char* message)
 {
   strncpy(_message, message, MAX_TEXT_LEN);
   _message[MAX_TEXT_LEN - 1] = '\0';
@@ -144,13 +145,12 @@ sg_throwable::setMessage (const char* message)
 }
 
 const char*
-sg_throwable::getOrigin () const
+sg_throwable::getOrigin() const noexcept
 {
   return _origin;
 }
 
-void
+void sg_throwable::setOrigin(const char* origin) noexcept
-sg_throwable::setOrigin (const char* origin)
 {
   if (origin) {
     strncpy(_origin, origin, MAX_TEXT_LEN);
@@ -170,27 +170,27 @@ const char* sg_throwable::what() const noexcept
   }
 }
 
-
+sg_location sg_throwable::getLocation() const noexcept
+{
+    return _location;
+}
+
+void sg_throwable::setLocation(const sg_location& location) noexcept
+{
+    _location = location;
+}
+
 ////////////////////////////////////////////////////////////////////////
 // Implementation of sg_error class.
 ////////////////////////////////////////////////////////////////////////
 
-sg_error::sg_error ()
+sg_error::sg_error(const char* message, const char* origin, bool report) noexcept
-  : sg_throwable ()
+    : sg_throwable(message, origin, {}, report)
 {
 }
 
-sg_error::sg_error (const char* message, const char *origin)
+sg_error::sg_error(const std::string& message, const std::string& origin, bool report) noexcept
-  : sg_throwable(message, origin)
+    : sg_throwable(message.c_str(), origin.c_str(), {}, report)
-{
-}
-
-sg_error::sg_error(const std::string& message, const std::string& origin)
-  : sg_throwable(message.c_str(), origin.c_str())
-{
-}
-
-sg_error::~sg_error ()
 {
 }
 
@@ -198,88 +198,50 @@ sg_error::~sg_error ()
 // Implementation of sg_exception class.
 ////////////////////////////////////////////////////////////////////////
 
-sg_exception::sg_exception ()
-  : sg_throwable ()
-{
-}
-
 sg_exception::sg_exception(const char* message, const char* origin,
-                           const sg_location &loc)
+                           const sg_location& loc, bool report) noexcept
-    : sg_throwable(message, origin, loc) {}
+    : sg_throwable(message, origin, loc, report) {}
 
 sg_exception::sg_exception(const std::string& message,
-                           const std::string &origin, const sg_location &loc)
+                           const std::string& origin, const sg_location& loc,
-    : sg_throwable(message.c_str(), origin.c_str(), loc) {}
+                           bool report) noexcept
+    : sg_throwable(message.c_str(), origin.c_str(), loc, report) {}
 
-sg_exception::~sg_exception ()
-{
-}
 
 ////////////////////////////////////////////////////////////////////////
 // Implementation of sg_io_exception.
 ////////////////////////////////////////////////////////////////////////
 
-sg_io_exception::sg_io_exception ()
+sg_io_exception::sg_io_exception(const char* message, const char* origin, bool report)
-  : sg_exception()
+    : sg_exception(message, origin, {}, report)
-{
-}
-
-sg_io_exception::sg_io_exception (const char* message, const char* origin)
-  : sg_exception(message, origin)
 {
 }
 
 sg_io_exception::sg_io_exception(const char* message,
                                  const sg_location& location,
-                                 const char *origin)
+                                 const char* origin,
-    : sg_exception(message, origin, location), _location(location) {}
+                                 bool report)
+    : sg_exception(message, origin, location, report) {}
 
 sg_io_exception::sg_io_exception(const std::string& message,
-                                  const std::string& origin )
+                                 const std::string& origin,
-  : sg_exception(message, origin)
+                                 bool report)
+    : sg_exception(message, origin, {}, report)
 {
 }
 
 sg_io_exception::sg_io_exception(const std::string& message,
                                  const sg_location& location,
-                                 const std::string &origin)
+                                 const std::string& origin,
-    : sg_exception(message, origin, location), _location(location) {}
+                                 bool report)
-
+    : sg_exception(message, origin, location, report) {}
-sg_io_exception::~sg_io_exception ()
-{
-}
-
-const std::string
-sg_io_exception::getFormattedMessage () const
-{
-  std::string ret = getMessage();
-  std::string loc = getLocation().asString();
-  if (loc.length()) {
-    ret += "\n at ";
-    ret += loc;
-  }
-  return ret;
-}
-
-const sg_location &
-sg_io_exception::getLocation () const
-{
-  return _location;
-}
-
-void
-sg_io_exception::setLocation (const sg_location &location)
-{
-  _location = location;
-}
 
 
-
 ////////////////////////////////////////////////////////////////////////
 // Implementation of sg_format_exception.
 ////////////////////////////////////////////////////////////////////////
 
-sg_format_exception::sg_format_exception ()
+sg_format_exception::sg_format_exception() noexcept
     : sg_exception()
 {
   _text[0] = '\0';
@@ -287,32 +249,30 @@ sg_format_exception::sg_format_exception ()
 
 sg_format_exception::sg_format_exception(const char* message,
                                          const char* text,
-					  const char* origin)
+                                         const char* origin,
-  : sg_exception(message, origin)
+                                         bool report)
+    : sg_exception(message, origin, {}, report)
 {
   setText(text);
 }
 
 sg_format_exception::sg_format_exception(const std::string& message,
                                          const std::string& text,
-                                          const std::string& origin )
+                                         const std::string& origin,
-  : sg_exception(message, origin)
+                                         bool report)
+    : sg_exception(message, origin, {}, report)
 {
   setText(text.c_str());
 }
 
-sg_format_exception::~sg_format_exception ()
-{
-}
 
 const char*
-sg_format_exception::getText () const
+sg_format_exception::getText() const noexcept
 {
   return _text;
 }
 
-void
+void sg_format_exception::setText(const char* text) noexcept
-sg_format_exception::setText (const char* text)
 {
   if (text) {
     strncpy(_text, text, MAX_TEXT_LEN);
@@ -328,26 +288,19 @@ sg_format_exception::setText (const char* text)
 // Implementation of sg_range_exception.
 ////////////////////////////////////////////////////////////////////////
 
-sg_range_exception::sg_range_exception ()
-  : sg_exception()
-{
-}
-
 sg_range_exception::sg_range_exception(const char* message,
-					const char* origin)
+                                       const char* origin, bool report)
-  : sg_exception(message, origin)
+    : sg_exception(message, origin, {}, report)
 {
 }
 
 sg_range_exception::sg_range_exception(const std::string& message,
-                                       const std::string& origin)
+                                       const std::string& origin,
-  : sg_exception(message, origin)
+                                       bool report)
+    : sg_exception(message, origin, {}, report)
 {
 }
 
-sg_range_exception::~sg_range_exception ()
-{
-}
 
 ////////////////////////////////////////////////////////////////////////
 

simgear/structure/exception.hxx

@@ -28,23 +28,23 @@ class sg_location
 {
 public:
   enum {max_path = 1024};
-  sg_location ();
+  sg_location() noexcept;
-  sg_location(const std::string& path, int line = -1, int column = -1);
+  sg_location(const std::string& path, int line = -1, int column = -1) noexcept;
-  sg_location(const SGPath& path, int line = -1, int column = -1);
+  sg_location(const SGPath& path, int line = -1, int column = -1) noexcept;
-  explicit sg_location(const char* path, int line = -1, int column = -1);
+  explicit sg_location(const char* path, int line = -1, int column = -1) noexcept;
 
   ~sg_location() = default;
 
-  const char *getPath() const;
+  const char* getPath() const noexcept;
-  int getLine() const;
+  int getLine() const noexcept;
-  int getColumn() const;
+  int getColumn() const noexcept;
-  int getByte() const;
+  int getByte() const noexcept;
 
-  std::string asString() const;
+  std::string asString() const noexcept;
-  bool isValid() const;
+  bool isValid() const noexcept;
 
   private:
-  void setPath(const char *p);
+  void setPath(const char* p) noexcept;
 
   char _path[max_path];
   int _line;
@@ -60,20 +60,26 @@ class sg_throwable : public std::exception
 {
 public:
   enum {MAX_TEXT_LEN = 1024};
-  sg_throwable ();
+  sg_throwable() noexcept;
   sg_throwable(const char* message, const char* origin = 0,
-               const sg_location &loc = {});
+               const sg_location& loc = {}, bool report = true) noexcept;
 
-  virtual ~sg_throwable ();
+  virtual ~sg_throwable() noexcept = default;
-  virtual const char* getMessage () const;
+
-  virtual const std::string getFormattedMessage () const;
+  virtual const char* getMessage() const noexcept;
-  virtual void setMessage (const char* message);
+  std::string getFormattedMessage() const noexcept;
-  virtual const char* getOrigin () const;
+  virtual void setMessage(const char* message) noexcept;
-  virtual void setOrigin (const char *origin);
+  virtual const char* getOrigin() const noexcept;
+  virtual void setOrigin(const char* origin) noexcept;
   virtual const char* what() const noexcept;
+
+  sg_location getLocation() const noexcept;
+  void setLocation(const sg_location& location) noexcept;
+
   private:
   char _message[MAX_TEXT_LEN];
   char _origin[MAX_TEXT_LEN];
+  sg_location _location;
 };
 
 
@@ -90,10 +96,10 @@ private:
 class sg_error : public sg_throwable
 {
 public:
-  sg_error ();
+    sg_error() noexcept = default;
-  sg_error (const char* message, const char* origin = 0);
+    sg_error(const char* message, const char* origin = 0, bool report = true) noexcept;
-  sg_error(const std::string &message, const std::string &origin = {});
+    sg_error(const std::string& message, const std::string& origin = {}, bool report = true) noexcept;
-  virtual ~sg_error ();
+    virtual ~sg_error() noexcept = default;
 };
 
 
@@ -114,12 +120,12 @@ public:
 class sg_exception : public sg_throwable
 {
 public:
-  sg_exception ();
+    sg_exception() noexcept = default;
     sg_exception(const char* message, const char* origin = 0,
-               const sg_location &loc = {});
+                 const sg_location& loc = {}, bool report = true) noexcept;
     sg_exception(const std::string& message, const std::string& = {},
-               const sg_location &loc = {});
+                 const sg_location& loc = {}, bool report = true) noexcept;
-  virtual ~sg_exception ();
+    virtual ~sg_exception() noexcept = default;
 };
 
 
@@ -137,20 +143,15 @@ public:
 class sg_io_exception : public sg_exception
 {
 public:
-  sg_io_exception ();
+    sg_io_exception() noexcept = default;
-  sg_io_exception (const char* message, const char* origin = 0);
+    sg_io_exception(const char* message, const char* origin = 0, bool report = true);
     sg_io_exception(const char* message, const sg_location& location,
-		   const char* origin = 0);
+                    const char* origin = 0, bool report = true);
-  sg_io_exception (const std::string &message, const std::string &origin = "");
+    sg_io_exception(const std::string& message, const std::string& origin = {}, bool report = true);
     sg_io_exception(const std::string& message, const sg_location& location,
-    const std::string &origin = "");
+                    const std::string& origin = {}, bool report = true);
 
-  virtual ~sg_io_exception ();
+    virtual ~sg_io_exception() noexcept = default;
-  virtual const std::string getFormattedMessage () const;
-  virtual const sg_location &getLocation () const;
-  virtual void setLocation (const sg_location &location);
-private:
-  sg_location _location;
 };
 
 
@@ -168,14 +169,15 @@ private:
 class sg_format_exception : public sg_exception
 {
 public:
-  sg_format_exception ();
+    sg_format_exception() noexcept;
     sg_format_exception(const char* message, const char* text,
-		       const char* origin = 0);
+                        const char* origin = 0, bool report = true);
     sg_format_exception(const std::string& message, const std::string& text,
-		       const std::string& origin = "");
+                        const std::string& origin = {}, bool report = true);
-  virtual ~sg_format_exception ();
+
-  virtual const char* getText () const;
+    const char* getText() const noexcept;
-  virtual void setText (const char* text);
+    void setText(const char* text) noexcept;
+
 private:
   char _text[MAX_TEXT_LEN];
 };
@@ -193,12 +195,14 @@ private:
 class sg_range_exception : public sg_exception
 {
 public:
-  sg_range_exception ();
+    sg_range_exception() noexcept = default;
     sg_range_exception(const char* message,
-                      const char* origin = 0);
+                       const char* origin = 0,
+                       bool report = true);
     sg_range_exception(const std::string& message,
-                      const std::string& origin = "");
+                       const std::string& origin = {},
-  virtual ~sg_range_exception ();
+                       bool report = true);
+    virtual ~sg_range_exception() noexcept = default;
 };
 
 using ThrowCallback = std::function<void(

version

@@ -0,0 +1 @@
+2020.3.8