new version: 2020.3.1

FLIGHTGEAR-F6 resolved

simgear-version

@@ -1 +1 @@
-2020.2.1
+2020.3.1

simgear/CMakeLists.txt

@@ -167,7 +167,8 @@ target_link_libraries(SimGearCore PRIVATE
     ${CMAKE_THREAD_LIBS_INIT}
     ${COCOA_LIBRARY}
     ${CURL_LIBRARIES}
-    ${WINSOCK_LIBRARY})
+    ${WINSOCK_LIBRARY}
+    ${SHLWAPI_LIBRARY})
 
 if(SYSTEM_EXPAT)
     target_link_libraries(SimGearCore PRIVATE ${EXPAT_LIBRARIES})

simgear/environment/metar.cxx

@@ -631,11 +631,15 @@ bool SGMetar::scanWind()
 	int dir;
 	if (!strncmp(m, "VRB", 3))
 		m += 3, dir = -1;
+	else if (!strncmp(m, "///", 3))	// direction not measurable
+		m += 3, dir = -1;
 	else if (!scanNumber(&m, &dir, 3))
 		return false;
 
 	int i;
-	if (!scanNumber(&m, &i, 2, 3))
+	if (!strncmp(m, "//", 2))	// speed not measurable
+		m += 2, i = -1;
+	else if (!scanNumber(&m, &i, 2, 3))
 		return false;
 	double speed = i;
 
@@ -655,6 +659,8 @@ bool SGMetar::scanWind()
 		m += 3, factor = SG_KMH_TO_MPS;
 	else if (!strncmp(m, "MPS", 3))
 		m += 3, factor = 1.0;
+	else if (!strncmp(m, " ", 1))		// default to Knots
+		factor = SG_KT_TO_MPS;
 	else
 		return false;
 	if (!scanBoundary(&m))

simgear/environment/test_metar.cxx

@@ -76,12 +76,29 @@ void test_sensor_failure_cloud()
     SG_CHECK_EQUAL_EP2(m1.getPressure_hPa(), 1025, TEST_EPSILON);
 }
 
+void test_sensor_failure_wind()
+{
+    SGMetar m1("2020/10/23 16:55 LIVD 231655Z /////KT 9999 OVC025 10/08 Q1020 RMK OVC VIS MIN 9999 BLU");
+    SG_CHECK_EQUAL(m1.getWindDir(), -1);
+    SG_CHECK_EQUAL_EP2(m1.getWindSpeed_kt(), -1, TEST_EPSILON);
+}
+
+void test_wind_unit_not_specified()
+{
+    SGMetar m1("2020/10/23 11:58 KLSV 231158Z 05010G14 10SM CLR 16/M04 A2992 RMK SLPNO WND DATA ESTMD ALSTG/SLP ESTMD 10320 20124 5//// $");
+    SG_CHECK_EQUAL(m1.getWindDir(), 50);
+    SG_CHECK_EQUAL_EP2(m1.getWindSpeed_kt(), 10.0, TEST_EPSILON);
+    SG_CHECK_EQUAL_EP2(m1.getGustSpeed_kt(), 14.0, TEST_EPSILON);
+}
+
 int main(int argc, char* argv[])
 {
     try {
         test_basic();
         test_sensor_failure_weather();
         test_sensor_failure_cloud();
+        test_sensor_failure_wind();
+        test_wind_unit_not_specified();
     } catch (sg_exception& e) {
         cerr << "got exception:" << e.getMessage() << endl;
         return -1;

simgear/io/CMakeLists.txt

@@ -35,10 +35,12 @@ set(SOURCES
     sg_socket.cxx
     sg_socket_udp.cxx
     HTTPClient.cxx
+    HTTPTestApi_private.hxx
     HTTPFileRequest.cxx
     HTTPMemoryRequest.cxx
     HTTPRequest.cxx
     HTTPRepository.cxx
+    HTTPRepository_private.hxx
     untar.cxx
     )
 
@@ -81,6 +83,7 @@ add_test(binobj ${EXECUTABLE_OUTPUT_PATH}/test_binobj)
 
 add_executable(test_repository test_repository.cxx)
 target_link_libraries(test_repository ${TEST_LIBS})
+target_compile_definitions(test_repository PUBLIC BUILDING_TESTSUITE)
 add_test(http_repository ${EXECUTABLE_OUTPUT_PATH}/test_repository)
 
 add_executable(test_untar test_untar.cxx)

simgear/io/DNSClient.cxx

@@ -156,8 +156,13 @@ static void dnscbTXT(struct dns_ctx *ctx, struct dns_rr_txt *result, void *data)
         r->ttl = result->dnstxt_ttl;
         for (int i = 0; i < result->dnstxt_nrr; i++) {
           //TODO: interprete the .len field of dnstxt_txt?
-          string txt = string((char*)result->dnstxt_txt[i].txt);
-          r->entries.push_back( txt );
+          auto rawTxt = reinterpret_cast<char*>(result->dnstxt_txt[i].txt);
+          if (!rawTxt) {
+              continue;
+          }
+
+          const string txt{rawTxt};
+          r->entries.push_back(txt);
           string_list tokens = simgear::strutils::split( txt, "=", 1 );
           if( tokens.size() == 2 ) {
             r->attributes[tokens[0]] = tokens[1];

simgear/io/HTTPClient.cxx

@@ -37,7 +37,6 @@
 
 #include <simgear/simgear_config.h>
 
-#include <curl/multi.h>
 
 #include <simgear/io/sg_netChat.hxx>
 
@@ -47,6 +46,9 @@
 #include <simgear/timing/timestamp.hxx>
 #include <simgear/structure/exception.hxx>
 
+#include "HTTPClient_private.hxx"
+#include "HTTPTestApi_private.hxx"
+
 #if defined( HAVE_VERSION_H ) && HAVE_VERSION_H
 #include "version.h"
 #else
@@ -64,50 +66,20 @@ namespace HTTP
 extern const int DEFAULT_HTTP_PORT = 80;
 const char* CONTENT_TYPE_URL_ENCODED = "application/x-www-form-urlencoded";
 
-class Connection;
-typedef std::multimap<std::string, Connection*> ConnectionDict;
-typedef std::list<Request_ptr> RequestList;
-
-class Client::ClientPrivate
-{
-public:
-    CURLM* curlMulti;
-
-    void createCurlMulti()
-    {
-        curlMulti = curl_multi_init();
-        // see https://curl.haxx.se/libcurl/c/CURLMOPT_PIPELINING.html
-        // we request HTTP 1.1 pipelining
-        curl_multi_setopt(curlMulti, CURLMOPT_PIPELINING, 1 /* aka CURLPIPE_HTTP1 */);
+void Client::ClientPrivate::createCurlMulti() {
+  curlMulti = curl_multi_init();
+  // see https://curl.haxx.se/libcurl/c/CURLMOPT_PIPELINING.html
+  // we request HTTP 1.1 pipelining
+  curl_multi_setopt(curlMulti, CURLMOPT_PIPELINING, 1 /* aka CURLPIPE_HTTP1 */);
 #if (LIBCURL_VERSION_MINOR >= 30)
-        curl_multi_setopt(curlMulti, CURLMOPT_MAX_TOTAL_CONNECTIONS, (long) maxConnections);
-        curl_multi_setopt(curlMulti, CURLMOPT_MAX_PIPELINE_LENGTH,
-                          (long) maxPipelineDepth);
-        curl_multi_setopt(curlMulti, CURLMOPT_MAX_HOST_CONNECTIONS,
-                          (long) maxHostConnections);
+  curl_multi_setopt(curlMulti, CURLMOPT_MAX_TOTAL_CONNECTIONS,
+                    (long)maxConnections);
+  curl_multi_setopt(curlMulti, CURLMOPT_MAX_PIPELINE_LENGTH,
+                    (long)maxPipelineDepth);
+  curl_multi_setopt(curlMulti, CURLMOPT_MAX_HOST_CONNECTIONS,
+                    (long)maxHostConnections);
 #endif
-    }
-
-    typedef std::map<Request_ptr, CURL*> RequestCurlMap;
-    RequestCurlMap requests;
-
-    std::string userAgent;
-    std::string proxy;
-    int proxyPort;
-    std::string proxyAuth;
-    unsigned int maxConnections;
-    unsigned int maxHostConnections;
-    unsigned int maxPipelineDepth;
-
-    RequestList pendingRequests;
-
-    SGTimeStamp timeTransferSample;
-    unsigned int bytesTransferred;
-    unsigned int lastTransferRate;
-    uint64_t totalBytesDownloaded;
-
-    SGPath tlsCertificatePath;
-};
+}
 
 Client::Client() :
     d(new ClientPrivate)
@@ -223,12 +195,23 @@ void Client::update(int waitTimeout)
           assert(it->second == e);
           d->requests.erase(it);
 
-        if (msg->data.result == 0) {
-          req->responseComplete();
-        } else {
-          SG_LOG(SG_IO, SG_WARN, "CURL Result:" << msg->data.result << " " << curl_easy_strerror(msg->data.result));
-          req->setFailure(msg->data.result, curl_easy_strerror(msg->data.result));
-        }
+          bool doProcess = true;
+          if (d->testsuiteResponseDoneCallback) {
+            doProcess =
+                !d->testsuiteResponseDoneCallback(msg->data.result, req);
+          }
+
+          if (doProcess) {
+            if (msg->data.result == 0) {
+              req->responseComplete();
+            } else {
+              SG_LOG(SG_IO, SG_WARN,
+                     "CURL Result:" << msg->data.result << " "
+                                    << curl_easy_strerror(msg->data.result));
+              req->setFailure(msg->data.result,
+                              curl_easy_strerror(msg->data.result));
+            }
+          }
 
         curl_multi_remove_handle(d->curlMulti, e);
         curl_easy_cleanup(e);
@@ -559,6 +542,17 @@ void Client::clearAllConnections()
     d->createCurlMulti();
 }
 
+/////////////////////////////////////////////////////////////////////
+
+void TestApi::setResponseDoneCallback(Client *cl, ResponseDoneCallback cb) {
+  cl->d->testsuiteResponseDoneCallback = cb;
+}
+
+void TestApi::markRequestAsFailed(Request_ptr req, int curlCode,
+                                  const std::string &message) {
+  req->setFailure(curlCode, message);
+}
+
 } // of namespace HTTP
 
 } // of namespace simgear

simgear/io/HTTPClient.hxx

@@ -24,7 +24,8 @@
 #ifndef SG_HTTP_CLIENT_HXX
 #define SG_HTTP_CLIENT_HXX
 
-#include <memory> // for std::unique_ptr
+#include <functional>
+#include <memory>   // for std::unique_ptr
 #include <stdint.h> // for uint_64t
 
 #include <simgear/io/HTTPFileRequest.hxx>
@@ -125,6 +126,7 @@ private:
 
     friend class Connection;
     friend class Request;
+    friend class TestApi;
 
     class ClientPrivate;
     std::unique_ptr<ClientPrivate> d;

simgear/io/HTTPClient_private.hxx

@@ -0,0 +1,68 @@
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Library General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301, USA
+
+#pragma once
+
+#include <list>
+#include <map>
+
+#include "HTTPClient.hxx"
+#include "HTTPRequest.hxx"
+
+#include <simgear/timing/timestamp.hxx>
+
+#include <curl/multi.h>
+
+namespace simgear {
+namespace HTTP {
+
+typedef std::list<Request_ptr> RequestList;
+
+using ResponseDoneCallback =
+    std::function<bool(int curlResult, Request_ptr req)>;
+
+class Client::ClientPrivate {
+public:
+  CURLM *curlMulti;
+
+  void createCurlMulti();
+
+  typedef std::map<Request_ptr, CURL *> RequestCurlMap;
+  RequestCurlMap requests;
+
+  std::string userAgent;
+  std::string proxy;
+  int proxyPort;
+  std::string proxyAuth;
+  unsigned int maxConnections;
+  unsigned int maxHostConnections;
+  unsigned int maxPipelineDepth;
+
+  RequestList pendingRequests;
+
+  SGTimeStamp timeTransferSample;
+  unsigned int bytesTransferred;
+  unsigned int lastTransferRate;
+  uint64_t totalBytesDownloaded;
+
+  SGPath tlsCertificatePath;
+
+  // only used by unit-tests / test-api, but
+  // only costs us a pointe here to declare it.
+  ResponseDoneCallback testsuiteResponseDoneCallback;
+};
+
+} // namespace HTTP
+
+} // namespace simgear

simgear/io/HTTPRepository.cxx

@@ -45,156 +45,56 @@
 
 #include <simgear/misc/sg_hash.hxx>
 
+#include "HTTPRepository_private.hxx"
+
 namespace simgear
 {
 
-    class HTTPDirectory;
-    using HTTPDirectory_ptr = std::unique_ptr<HTTPDirectory>;
+namespace {
 
-    class HTTPRepoGetRequest : public HTTP::Request
-    {
-    public:
-        HTTPRepoGetRequest(HTTPDirectory* d, const std::string& u) :
-            HTTP::Request(u),
-            _directory(d)
-        {
-        }
+std::string innerResultCodeAsString(HTTPRepository::ResultCode code) {
+  switch (code) {
+  case HTTPRepository::REPO_NO_ERROR:
+    return "no error";
+  case HTTPRepository::REPO_ERROR_NOT_FOUND:
+    return "not found";
+  case HTTPRepository::REPO_ERROR_SOCKET:
+    return "socket error";
+  case HTTPRepository::SVN_ERROR_XML:
+    return "malformed XML";
+  case HTTPRepository::SVN_ERROR_TXDELTA:
+    return "malformed XML";
+  case HTTPRepository::REPO_ERROR_IO:
+    return "I/O error";
+  case HTTPRepository::REPO_ERROR_CHECKSUM:
+    return "checksum verification error";
+  case HTTPRepository::REPO_ERROR_FILE_NOT_FOUND:
+    return "file not found";
+  case HTTPRepository::REPO_ERROR_HTTP:
+    return "HTTP-level error";
+  case HTTPRepository::REPO_ERROR_CANCELLED:
+    return "cancelled";
+  case HTTPRepository::REPO_PARTIAL_UPDATE:
+    return "partial update (incomplete)";
+  }
 
-        virtual void cancel();
+  return "Unknown response code";
+}
 
-        size_t contentSize() const
-        {
-            return _contentSize;
-        }
-
-        void setContentSize(size_t sz)
-        {
-            _contentSize = sz;
-        }
-    protected:
-        HTTPDirectory* _directory;
-        size_t _contentSize = 0;
-    };
-
-    typedef SGSharedPtr<HTTPRepoGetRequest> RepoRequestPtr;
-
-    std::string innerResultCodeAsString(HTTPRepository::ResultCode code)
-    {
-        switch (code) {
-        case HTTPRepository::REPO_NO_ERROR:     return "no error";
-        case HTTPRepository::REPO_ERROR_NOT_FOUND:  return "not found";
-        case HTTPRepository::REPO_ERROR_SOCKET:     return "socket error";
-        case HTTPRepository::SVN_ERROR_XML:        return "malformed XML";
-        case HTTPRepository::SVN_ERROR_TXDELTA:        return "malformed XML";
-        case HTTPRepository::REPO_ERROR_IO:         return "I/O error";
-        case HTTPRepository::REPO_ERROR_CHECKSUM:         return "checksum verification error";
-        case HTTPRepository::REPO_ERROR_FILE_NOT_FOUND: return "file not found";
-        case HTTPRepository::REPO_ERROR_HTTP:   return "HTTP-level error";
-        case HTTPRepository::REPO_ERROR_CANCELLED:   return "cancelled";
-        case HTTPRepository::REPO_PARTIAL_UPDATE:   return "partial update (incomplete)";
-        }
-
-        return "Unknown response code";
-    }
-
-class HTTPRepoPrivate
-{
-public:
-    struct HashCacheEntry
-    {
-        std::string filePath;
-        time_t modTime;
-        size_t lengthBytes;
-        std::string hashHex;
-
-    };
-
-    typedef std::vector<HashCacheEntry> HashCache;
-    HashCache hashes;
-    bool hashCacheDirty;
-
-    struct Failure
-    {
-        SGPath path;
-        HTTPRepository::ResultCode error;
-    };
-
-    typedef std::vector<Failure> FailureList;
-    FailureList failures;
-
-    HTTPRepoPrivate(HTTPRepository* parent) :
-        hashCacheDirty(false),
-        p(parent),
-        isUpdating(false),
-        status(HTTPRepository::REPO_NO_ERROR),
-        totalDownloaded(0)
-    { ; }
-
-    ~HTTPRepoPrivate();
-
-    HTTPRepository* p; // link back to outer
-    HTTP::Client* http;
-    std::string baseUrl;
-    SGPath basePath;
-    bool isUpdating;
-    HTTPRepository::ResultCode status;
-    HTTPDirectory_ptr rootDir;
-    size_t totalDownloaded;
-
-    HTTP::Request_ptr updateFile(HTTPDirectory* dir, const std::string& name,
-                                 size_t sz);
-    HTTP::Request_ptr updateDir(HTTPDirectory* dir, const std::string& hash,
-                                size_t sz);
-
-    std::string hashForPath(const SGPath& p);
-    void updatedFileContents(const SGPath& p, const std::string& newHash);
-    void parseHashCache();
-    std::string computeHashForPath(const SGPath& p);
-    void writeHashCache();
-
-    void failedToGetRootIndex(HTTPRepository::ResultCode st);
-    void failedToUpdateChild(const SGPath& relativePath,
-                             HTTPRepository::ResultCode fileStatus);
-
-    typedef std::vector<RepoRequestPtr> RequestVector;
-    RequestVector queuedRequests,
-        activeRequests;
-
-    void makeRequest(RepoRequestPtr req);
-    void finishedRequest(const RepoRequestPtr& req);
-
-    HTTPDirectory* getOrCreateDirectory(const std::string& path);
-    bool deleteDirectory(const std::string& relPath, const SGPath& absPath);
-
-    typedef std::vector<HTTPDirectory_ptr> DirectoryVector;
-    DirectoryVector directories;
-
-    SGPath installedCopyPath;
-};
+} // namespace
 
 class HTTPDirectory
 {
     struct ChildInfo
     {
-        enum Type
-        {
-            FileType,
-            DirectoryType,
-            TarballType
-        };
+      ChildInfo(HTTPRepository::EntryType ty, const std::string &nameData,
+                const std::string &hashData)
+          : type(ty), name(nameData), hash(hashData) {}
 
-        ChildInfo(Type ty, const std::string & nameData, const std::string & hashData) :
-            type(ty),
-            name(nameData),
-            hash(hashData)
-        {
-        }
+      ChildInfo(const ChildInfo &other) = default;
 
-		    ChildInfo(const ChildInfo& other) = default;
-
-        void setSize(const std::string & sizeData)
-        {
-            sizeInBytes = ::strtol(sizeData.c_str(), NULL, 10);
+      void setSize(const std::string &sizeData) {
+        sizeInBytes = ::strtol(sizeData.c_str(), NULL, 10);
         }
 
         bool operator<(const ChildInfo& other) const
@@ -202,7 +102,7 @@ class HTTPDirectory
             return name < other.name;
         }
 
-        Type type;
+        HTTPRepository::EntryType type;
         std::string name, hash;
 		size_t sizeInBytes = 0;
 		SGPath path; // absolute path on disk
@@ -238,9 +138,9 @@ public:
 
     std::string url() const
     {
-        if (_relativePath.empty()) {
-            return _repository->baseUrl;
-        }
+      if (_relativePath.empty()) { // root directory of the repo
+        return _repository->baseUrl;
+      }
 
         return _repository->baseUrl + "/" + _relativePath;
     }
@@ -254,6 +154,8 @@ public:
         children.clear();
         parseDirIndex(children);
         std::sort(children.begin(), children.end());
+
+        _repository->updatedChildSuccessfully(_relativePath);
     }
 
     void failedToUpdate(HTTPRepository::ResultCode status)
@@ -276,17 +178,17 @@ public:
 		size_t bufSize = 0;
 
 		for (const auto& child : children) {
-			if (child.type != ChildInfo::FileType)
-				continue;
+                  if (child.type != HTTPRepository::FileType)
+                    continue;
 
-			if (child.path.exists())
-				continue;
+                  if (child.path.exists())
+                    continue;
 
-			SGPath cp = _repository->installedCopyPath;
-			cp.append(relativePath());
-			cp.append(child.name);
-			if (!cp.exists()) {
-				continue;
+                  SGPath cp = _repository->installedCopyPath;
+                  cp.append(relativePath());
+                  cp.append(child.name);
+                  if (!cp.exists()) {
+                    continue;
 			}
 
 			SGBinaryFile src(cp);
@@ -315,35 +217,95 @@ public:
 
     void updateChildrenBasedOnHash()
     {
+      using SAct = HTTPRepository::SyncAction;
+
       copyInstalledChildren();
 
       ChildInfoList toBeUpdated;
 
       simgear::Dir d(absolutePath());
       PathList fsChildren = d.children(0);
-      PathList orphans = d.children(0);
+      PathList orphans = fsChildren;
 
-      ChildInfoList::const_iterator it;
-      for (it=children.begin(); it != children.end(); ++it) {
+      // on Windows, children() will return our .hashes and .dirIndex
+      // entries; skip them.
+      orphans.erase(std::remove_if(orphans.begin(), orphans.end(),
+                                   [](const SGPath &p) {
+                                     return p.file().front() == '.';
+                                   }),
+                    orphans.end());
+
+      for (const auto &c : children) {
         // Check if the file exists
-        PathList::const_iterator p = std::find_if(fsChildren.begin(), fsChildren.end(), LocalFileMatcher(*it));
-        if (p == fsChildren.end()) {
-          // File or directory does not exist on local disk, so needs to be updated.
-          toBeUpdated.push_back(ChildInfo(*it));
-        } else if (hashForChild(*it) != it->hash) {
-          // File/directory exists, but hash doesn't match.
-          toBeUpdated.push_back(ChildInfo(*it));
-          orphans.erase(std::remove(orphans.begin(), orphans.end(), *p), orphans.end());
-        } else {
-          // File/Directory exists and hash is valid.
-          orphans.erase(std::remove(orphans.begin(), orphans.end(), *p), orphans.end());
+        auto p = std::find_if(fsChildren.begin(), fsChildren.end(),
+                              LocalFileMatcher(c));
 
-          if (it->type == ChildInfo::DirectoryType) {
-              // If it's a directory,perform a recursive check.
-              HTTPDirectory* childDir = childDirectory(it->name);
-              childDir->updateChildrenBasedOnHash();
+        const bool isNew = (p == fsChildren.end());
+        const bool upToDate = hashForChild(c) == c.hash;
+
+        if (!isNew) {
+          orphans.erase(std::remove(orphans.begin(), orphans.end(), *p),
+                        orphans.end());
+        }
+
+        if (_repository->syncPredicate) {
+          const auto pathOnDisk = isNew ? absolutePath() / c.name : *p;
+          // never handle deletes here, do them at the end
+          const auto action =
+              isNew ? SAct::Add : (upToDate ? SAct::UpToDate : SAct::Update);
+          const HTTPRepository::SyncItem item = {relativePath(), c.type, c.name,
+                                                 action, pathOnDisk};
+
+          const bool doSync = _repository->syncPredicate(item);
+          if (!doSync) {
+            continue; // skip it, predicate filtered it out
           }
         }
+
+        if (isNew) {
+          // File or directory does not exist on local disk, so needs to be updated.
+          toBeUpdated.push_back(c);
+        } else if (!upToDate) {
+          // File/directory exists, but hash doesn't match.
+          toBeUpdated.push_back(c);
+        } else {
+          // File/Directory exists and hash is valid.
+
+          if (c.type == HTTPRepository::DirectoryType) {
+            // If it's a directory,perform a recursive check.
+            HTTPDirectory *childDir = childDirectory(c.name);
+            childDir->updateChildrenBasedOnHash();
+          }
+        }
+      } // of repository-defined (well, .dirIndex) children iteration
+
+      // allow the filtering of orphans; this is important so that a filter
+      // can be used to preserve non-repo files in a directory,
+      // i.e somewhat like a .gitignore
+      if (!orphans.empty() && _repository->syncPredicate) {
+        const auto ourPath = relativePath();
+        const auto pred = _repository->syncPredicate;
+
+        auto l = [ourPath, pred](const SGPath &o) {
+          // this doesn't special-case for tarballs (they will be reported as a
+          // file) I think that's okay, since a filter can see the full path
+          const auto type = o.isDir() ? HTTPRepository::DirectoryType
+                                      : HTTPRepository::FileType;
+
+          const HTTPRepository::SyncItem item = {ourPath, type, o.file(),
+                                                 SAct::Delete, o};
+
+          const bool r = pred(item);
+          // clarification: the predicate returns true if the file should be
+          // handled as normal, false if it should be skipped. But since we're
+          // inside a remove_if, we want to remove *skipped* files from orphans,
+          // so they don't get deleted. So we want to return true here, if the
+          // file should be skipped.
+          return (r == false);
+        };
+
+        auto it = std::remove_if(orphans.begin(), orphans.end(), l);
+        orphans.erase(it, orphans.end());
       }
 
       // We now have a list of entries that need to be updated, and a list
@@ -383,17 +345,18 @@ public:
     {
         ChildInfoList::const_iterator it;
         for (it = names.begin(); it != names.end(); ++it) {
-            if (it->type == ChildInfo::FileType) {
-                _repository->updateFile(this, it->name, it->sizeInBytes);
-            } else if (it->type == ChildInfo::DirectoryType){
-                HTTPDirectory* childDir = childDirectory(it->name);
-                _repository->updateDir(childDir, it->hash, it->sizeInBytes);
-            } else if (it->type == ChildInfo::TarballType) {
-                // Download a tarball just as a file.
-                _repository->updateFile(this, it->name, it->sizeInBytes);
-            } else {
-                SG_LOG(SG_TERRASYNC, SG_ALERT, "Coding error!  Unknown Child type to schedule update");
-            }
+          if (it->type == HTTPRepository::FileType) {
+            _repository->updateFile(this, it->name, it->sizeInBytes);
+          } else if (it->type == HTTPRepository::DirectoryType) {
+            HTTPDirectory *childDir = childDirectory(it->name);
+            _repository->updateDir(childDir, it->hash, it->sizeInBytes);
+          } else if (it->type == HTTPRepository::TarballType) {
+            // Download a tarball just as a file.
+            _repository->updateFile(this, it->name, it->sizeInBytes);
+          } else {
+            SG_LOG(SG_TERRASYNC, SG_ALERT,
+                   "Coding error!  Unknown Child type to schedule update");
+          }
         }
     }
 
@@ -420,9 +383,14 @@ public:
                 SG_LOG(SG_TERRASYNC, SG_WARN, "Checksum error for " << absolutePath() << "/" << file << " " << it->hash << " " << hash);
                 // we don't erase the file on a hash mismatch, because if we're syncing during the
                 // middle of a server-side update, the downloaded file may actually become valid.
-                _repository->failedToUpdateChild(_relativePath, HTTPRepository::REPO_ERROR_CHECKSUM);
+                _repository->failedToUpdateChild(
+                    _relativePath + "/" + file,
+                    HTTPRepository::REPO_ERROR_CHECKSUM);
             } else {
                 _repository->updatedFileContents(it->path, hash);
+                _repository->updatedChildSuccessfully(_relativePath + "/" +
+                                                      file);
+
                 _repository->totalDownloaded += sz;
                 SGPath p = SGPath(absolutePath(), file);
 
@@ -450,19 +418,23 @@ public:
                     SGPath extractDir = p.dir();
                     ArchiveExtractor ex(extractDir);
 
-                    uint8_t* buf = (uint8_t*) alloca(128);
+                    uint8_t *buf = (uint8_t *)alloca(2048);
                     while (!f.eof()) {
-                      size_t bufSize = f.read((char*) buf, 128);
+                      size_t bufSize = f.read((char *)buf, 2048);
                       ex.extractBytes(buf, bufSize);
                     }
 
                     ex.flush();
                     if (! ex.isAtEndOfArchive()) {
                       SG_LOG(SG_TERRASYNC, SG_ALERT, "Corrupt tarball " << p);
+                      _repository->failedToUpdateChild(
+                          _relativePath, HTTPRepository::REPO_ERROR_IO);
                     }
 
                     if (ex.hasError()) {
                       SG_LOG(SG_TERRASYNC, SG_ALERT, "Error extracting " << p);
+                      _repository->failedToUpdateChild(
+                          _relativePath, HTTPRepository::REPO_ERROR_IO);
                     }
 
                   } else {
@@ -576,9 +548,12 @@ private:
                 continue;
             }
 
-            ChildInfo ci = ChildInfo(ChildInfo::FileType, tokens[1], tokens[2]);
-            if (typeData == "d") ci.type = ChildInfo::DirectoryType;
-            if (typeData == "t") ci.type = ChildInfo::TarballType;
+            ChildInfo ci =
+                ChildInfo(HTTPRepository::FileType, tokens[1], tokens[2]);
+            if (typeData == "d")
+              ci.type = HTTPRepository::DirectoryType;
+            if (typeData == "t")
+              ci.type = HTTPRepository::TarballType;
 
             children.emplace_back(ci);
 			      children.back().path = absolutePath() / tokens[1];
@@ -612,10 +587,13 @@ private:
 
     std::string hashForChild(const ChildInfo& child) const
     {
-		    SGPath p(child.path);
-        if (child.type == ChildInfo::DirectoryType) p.append(".dirindex");
-        if (child.type == ChildInfo::TarballType) p.concat(".tgz");  // For tarballs the hash is against the tarball file itself
-        return _repository->hashForPath(p);
+      SGPath p(child.path);
+      if (child.type == HTTPRepository::DirectoryType)
+        p.append(".dirindex");
+      if (child.type == HTTPRepository::TarballType)
+        p.concat(
+            ".tgz"); // For tarballs the hash is against the tarball file itself
+      return _repository->hashForPath(p);
     }
 
     HTTPRepoPrivate* _repository;
@@ -719,6 +697,12 @@ std::string HTTPRepository::resultCodeAsString(ResultCode code)
     return innerResultCodeAsString(code);
 }
 
+HTTPRepository::FailureVec HTTPRepository::failures() const {
+  return _d->failures;
+}
+
+void HTTPRepository::setFilter(SyncPredicate sp) { _d->syncPredicate = sp; }
+
 HTTPRepository::ResultCode
 HTTPRepository::failure() const
 {
@@ -747,61 +731,80 @@ HTTPRepository::failure() const
         }
 
     protected:
-        virtual void gotBodyData(const char* s, int n)
-        {
-            if (!file.get()) {
-                file.reset(new SGBinaryFile(pathInRepo));
-                if (!file->open(SG_IO_OUT)) {
-                  SG_LOG(SG_TERRASYNC, SG_WARN, "unable to create file " << pathInRepo);
-                    _directory->repository()->http->cancelRequest(this, "Unable to create output file:" + pathInRepo.utf8Str());
-                }
+      void gotBodyData(const char *s, int n) override {
+        if (!file.get()) {
+          file.reset(new SGBinaryFile(pathInRepo));
+          if (!file->open(SG_IO_OUT)) {
+            SG_LOG(SG_TERRASYNC, SG_WARN,
+                   "unable to create file " << pathInRepo);
+            _directory->repository()->http->cancelRequest(
+                this, "Unable to create output file:" + pathInRepo.utf8Str());
+          }
 
-                sha1_init(&hashContext);
-            }
-
-            sha1_write(&hashContext, s, n);
-            file->write(s, n);
+          sha1_init(&hashContext);
         }
 
-        virtual void onDone()
-        {
-            file->close();
-            if (responseCode() == 200) {
-                std::string hash = strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH);
-                _directory->didUpdateFile(fileName, hash, contentSize());
-            } else if (responseCode() == 404) {
-                SG_LOG(SG_TERRASYNC, SG_WARN, "terrasync file not found on server: " << fileName << " for " << _directory->absolutePath());
-                _directory->didFailToUpdateFile(fileName, HTTPRepository::REPO_ERROR_FILE_NOT_FOUND);
-            } else {
-                SG_LOG(SG_TERRASYNC, SG_WARN, "terrasync file download error on server: " << fileName << " for " << _directory->absolutePath() <<
-                       "\n\tserver responded: " << responseCode() << "/" << responseReason());
-                _directory->didFailToUpdateFile(fileName, HTTPRepository::REPO_ERROR_HTTP);
-            }
+        sha1_write(&hashContext, s, n);
+        file->write(s, n);
+      }
 
-            _directory->repository()->finishedRequest(this);
+      void onDone() override {
+        file->close();
+        if (responseCode() == 200) {
+          std::string hash =
+              strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH);
+          _directory->didUpdateFile(fileName, hash, contentSize());
+        } else if (responseCode() == 404) {
+          SG_LOG(SG_TERRASYNC, SG_WARN,
+                 "terrasync file not found on server: "
+                     << fileName << " for " << _directory->absolutePath());
+          _directory->didFailToUpdateFile(
+              fileName, HTTPRepository::REPO_ERROR_FILE_NOT_FOUND);
+        } else {
+          SG_LOG(SG_TERRASYNC, SG_WARN,
+                 "terrasync file download error on server: "
+                     << fileName << " for " << _directory->absolutePath()
+                     << "\n\tserver responded: " << responseCode() << "/"
+                     << responseReason());
+          _directory->didFailToUpdateFile(fileName,
+                                          HTTPRepository::REPO_ERROR_HTTP);
+          // should we every retry here?
         }
 
-        virtual void onFail()
-        {
-            HTTPRepository::ResultCode code = HTTPRepository::REPO_ERROR_SOCKET;
-            if (responseCode() == -1) {
-                code = HTTPRepository::REPO_ERROR_CANCELLED;
-            }
+        _directory->repository()->finishedRequest(
+            this, HTTPRepoPrivate::RequestFinish::Done);
+      }
 
-			if (file) {
-				file->close();
-			}
-
-            file.reset();
-            if (pathInRepo.exists()) {
-                pathInRepo.remove();
-            }
-
-            if (_directory) {
-                _directory->didFailToUpdateFile(fileName, code);
-                _directory->repository()->finishedRequest(this);
-            }
+      void onFail() override {
+        HTTPRepository::ResultCode code = HTTPRepository::REPO_ERROR_SOCKET;
+        if (responseCode() == -1) {
+          code = HTTPRepository::REPO_ERROR_CANCELLED;
         }
+
+        if (file) {
+          file->close();
+        }
+
+        file.reset();
+        if (pathInRepo.exists()) {
+          pathInRepo.remove();
+        }
+
+        if (_directory) {
+          _directory->didFailToUpdateFile(fileName, code);
+
+          const auto doRetry = code == HTTPRepository::REPO_ERROR_SOCKET
+                                   ? HTTPRepoPrivate::RequestFinish::Retry
+                                   : HTTPRepoPrivate::RequestFinish::Done;
+          _directory->repository()->finishedRequest(this, doRetry);
+        }
+      }
+
+      void prepareForRetry() override {
+        HTTP::Request::prepareForRetry();
+        file.reset();
+      }
+
     private:
         static std::string makeUrl(HTTPDirectory* d, const std::string& file)
         {
@@ -835,74 +838,106 @@ HTTPRepository::failure() const
             return _isRootDir;
         }
 
-    protected:
-        virtual void gotBodyData(const char* s, int n)
-        {
-            body += std::string(s, n);
-            sha1_write(&hashContext, s, n);
+        void prepareForRetry() override {
+          body.clear();
+          sha1_init(&hashContext);
+          HTTP::Request::prepareForRetry();
         }
 
-        virtual void onDone()
-        {
-            if (responseCode() == 200) {
-                std::string hash = strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH);
-                if (!_targetHash.empty() && (hash != _targetHash)) {
-                    _directory->failedToUpdate(HTTPRepository::REPO_ERROR_CHECKSUM);
-                    _directory->repository()->finishedRequest(this);
-                    return;
-                }
+      protected:
+        void gotBodyData(const char *s, int n) override {
+          body += std::string(s, n);
+          sha1_write(&hashContext, s, n);
+        }
 
-                std::string curHash = _directory->repository()->hashForPath(path());
-                if (hash != curHash) {
-                    simgear::Dir d(_directory->absolutePath());
-                    if (!d.exists()) {
-                        if (!d.create(0700)) {
-                            throw sg_io_exception("Unable to create directory", d.path());
-                        }
-                    }
+        void onDone() override {
+          if (responseCode() == 200) {
+            std::string hash =
+                strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH);
+            if (!_targetHash.empty() && (hash != _targetHash)) {
+              SG_LOG(SG_TERRASYNC, SG_WARN,
+                     "Checksum error getting dirIndex for:"
+                         << _directory->relativePath() << "; expected "
+                         << _targetHash << " but received " << hash);
 
-                    // dir index data has changed, so write to disk and update
-                    // the hash accordingly
-                    sg_ofstream of(pathInRepo(), std::ios::trunc | std::ios::out | std::ios::binary);
-                    if (!of.is_open()) {
-                        throw sg_io_exception("Failed to open directory index file for writing", pathInRepo());
-                    }
+              _directory->failedToUpdate(HTTPRepository::REPO_ERROR_CHECKSUM);
 
-                    of.write(body.data(), body.size());
-                    of.close();
-                    _directory->dirIndexUpdated(hash);
-
-                    //SG_LOG(SG_TERRASYNC, SG_INFO, "updated dir index " << _directory->absolutePath());
-                }
-
-                _directory->repository()->totalDownloaded += contentSize();
-
-                try {
-                    // either way we've confirmed the index is valid so update
-                    // children now
-                    SGTimeStamp st;
-                    st.stamp();
-                    _directory->updateChildrenBasedOnHash();
-                    SG_LOG(SG_TERRASYNC, SG_DEBUG, "after update of:" << _directory->absolutePath() << " child update took:" << st.elapsedMSec());
-                } catch (sg_exception& ) {
-                    _directory->failedToUpdate(HTTPRepository::REPO_ERROR_IO);
-                }
-            } else if (responseCode() == 404) {
-                _directory->failedToUpdate(HTTPRepository::REPO_ERROR_FILE_NOT_FOUND);
-            } else {
-                _directory->failedToUpdate(HTTPRepository::REPO_ERROR_HTTP);
+              // don't retry checkums failures
+              _directory->repository()->finishedRequest(
+                  this, HTTPRepoPrivate::RequestFinish::Done);
+              return;
             }
 
-            _directory->repository()->finishedRequest(this);
+            std::string curHash = _directory->repository()->hashForPath(path());
+            if (hash != curHash) {
+              simgear::Dir d(_directory->absolutePath());
+              if (!d.exists()) {
+                if (!d.create(0700)) {
+                  throw sg_io_exception("Unable to create directory", d.path());
+                }
+              }
+
+              // dir index data has changed, so write to disk and update
+              // the hash accordingly
+              sg_ofstream of(pathInRepo(), std::ios::trunc | std::ios::out |
+                                               std::ios::binary);
+              if (!of.is_open()) {
+                throw sg_io_exception(
+                    "Failed to open directory index file for writing",
+                    pathInRepo());
+              }
+
+              of.write(body.data(), body.size());
+              of.close();
+              _directory->dirIndexUpdated(hash);
+
+              // SG_LOG(SG_TERRASYNC, SG_INFO, "updated dir index " <<
+              // _directory->absolutePath());
+            }
+
+            _directory->repository()->totalDownloaded += contentSize();
+
+            try {
+              // either way we've confirmed the index is valid so update
+              // children now
+              SGTimeStamp st;
+              st.stamp();
+              _directory->updateChildrenBasedOnHash();
+              SG_LOG(SG_TERRASYNC, SG_DEBUG,
+                     "after update of:" << _directory->absolutePath()
+                                        << " child update took:"
+                                        << st.elapsedMSec());
+            } catch (sg_exception &) {
+              _directory->failedToUpdate(HTTPRepository::REPO_ERROR_IO);
+            }
+          } else if (responseCode() == 404) {
+            _directory->failedToUpdate(
+                HTTPRepository::REPO_ERROR_FILE_NOT_FOUND);
+          } else {
+            _directory->failedToUpdate(HTTPRepository::REPO_ERROR_HTTP);
+          }
+
+          _directory->repository()->finishedRequest(
+              this, HTTPRepoPrivate::RequestFinish::Done);
         }
 
-        virtual void onFail()
-        {
-            if (_directory) {
-                _directory->failedToUpdate(HTTPRepository::REPO_ERROR_SOCKET);
-                _directory->repository()->finishedRequest(this);
-            }
+        void onFail() override {
+          HTTPRepository::ResultCode code = HTTPRepository::REPO_ERROR_SOCKET;
+          if (responseCode() == -1) {
+            code = HTTPRepository::REPO_ERROR_CANCELLED;
+          }
+
+          SG_LOG(SG_TERRASYNC, SG_WARN,
+                 "Socket failure getting directory: " << url());
+          if (_directory) {
+            _directory->failedToUpdate(code);
+            const auto doRetry = code == HTTPRepository::REPO_ERROR_SOCKET
+                                     ? HTTPRepoPrivate::RequestFinish::Retry
+                                     : HTTPRepoPrivate::RequestFinish::Done;
+            _directory->repository()->finishedRequest(this, doRetry);
+          }
         }
+
     private:
         static std::string makeUrl(HTTPDirectory* d)
         {
@@ -1014,10 +1049,10 @@ HTTPRepository::failure() const
     void HTTPRepoPrivate::updatedFileContents(const SGPath& p, const std::string& newHash)
     {
         // remove the existing entry
-        HashCache::iterator it = std::find_if(hashes.begin(), hashes.end(), HashEntryWithPath(p));
+        auto it = std::find_if(hashes.begin(), hashes.end(), HashEntryWithPath(p));
         if (it != hashes.end()) {
             hashes.erase(it);
-            hashCacheDirty = true;
+            ++hashCacheDirty;
         }
 
         if (newHash.empty()) {
@@ -1036,12 +1071,12 @@ HTTPRepository::failure() const
         entry.lengthBytes = p2.sizeInBytes();
         hashes.push_back(entry);
 
-        hashCacheDirty = true;
+        ++hashCacheDirty ;
     }
 
     void HTTPRepoPrivate::writeHashCache()
     {
-        if (!hashCacheDirty) {
+        if (hashCacheDirty == 0) {
             return;
         }
 
@@ -1054,7 +1089,7 @@ HTTPRepository::failure() const
             << it->lengthBytes << "*" << it->hashHex << "\n";
         }
         stream.close();
-        hashCacheDirty = false;
+        hashCacheDirty = 0;
     }
 
     void HTTPRepoPrivate::parseHashCache()
@@ -1126,10 +1161,9 @@ HTTPRepository::failure() const
         DirectoryWithPath p(relPath);
         auto it = std::find_if(directories.begin(), directories.end(), p);
         if (it != directories.end()) {
-            HTTPDirectory* d = it->get();
-			assert(d->absolutePath() == absPath);
-            directories.erase(it);
-		} else {
+          assert((*it)->absolutePath() == absPath);
+          directories.erase(it);
+                } else {
 			// we encounter this code path when deleting an orphaned directory
 		}
 
@@ -1145,34 +1179,46 @@ HTTPRepository::failure() const
     void HTTPRepoPrivate::makeRequest(RepoRequestPtr req)
     {
         if (activeRequests.size() > 4) {
-            queuedRequests.push_back(req);
+          queuedRequests.push_back(req);
         } else {
             activeRequests.push_back(req);
             http->makeRequest(req);
         }
     }
 
-    void HTTPRepoPrivate::finishedRequest(const RepoRequestPtr& req)
-    {
-        RequestVector::iterator it = std::find(activeRequests.begin(), activeRequests.end(), req);
-        // in some cases, for example a checksum failure, we clear the active
-        // and queued request vectors, so the ::find above can fail
-        if (it != activeRequests.end()) {
-            activeRequests.erase(it);
-        }
+    void HTTPRepoPrivate::finishedRequest(const RepoRequestPtr &req,
+                                          RequestFinish retryRequest) {
+      auto it = std::find(activeRequests.begin(), activeRequests.end(), req);
+      // in some cases, we clear the active
+      // and queued request vectors, so the ::find above can fail
+      if (it != activeRequests.end()) {
+        activeRequests.erase(it);
+      }
 
-        if (!queuedRequests.empty()) {
-            RepoRequestPtr rr = queuedRequests.front();
-            queuedRequests.erase(queuedRequests.begin());
-            activeRequests.push_back(rr);
-            http->makeRequest(rr);
-        }
+      if (retryRequest == HTTPRepoPrivate::RequestFinish::Retry) {
+        SG_LOG(SG_TERRASYNC, SG_INFO, "Retrying request for:" << req->url());
+        req->prepareForRetry();
+        queuedRequests.push_back(req);
+      }
 
+      if (!queuedRequests.empty()) {
+        RepoRequestPtr rr = queuedRequests.front();
+        queuedRequests.erase(queuedRequests.begin());
+        activeRequests.push_back(rr);
+        http->makeRequest(rr);
+      }
+
+      // rate limit how often we write this, since otherwise
+      // it dominates the time on Windows. 256 seems about right,
+      // causes a write a few times a minute.
+      if (hashCacheDirty > 256) {
         writeHashCache();
+      }
 
-        if (activeRequests.empty() && queuedRequests.empty()) {
-            isUpdating = false;
-        }
+      if (activeRequests.empty() && queuedRequests.empty()) {
+        isUpdating = false;
+        writeHashCache();
+      }
     }
 
     void HTTPRepoPrivate::failedToGetRootIndex(HTTPRepository::ResultCode st)
@@ -1188,36 +1234,49 @@ HTTPRepository::failure() const
     void HTTPRepoPrivate::failedToUpdateChild(const SGPath& relativePath,
                                               HTTPRepository::ResultCode fileStatus)
     {
-        if (fileStatus == HTTPRepository::REPO_ERROR_CHECKSUM) {
-            // stop updating, and mark repository as failed, becuase this
-            // usually indicates we need to start a fresh update from the
-            // root.
-            // (we could issue a retry here, but we leave that to higher layers)
-            status = fileStatus;
+      if (fileStatus == HTTPRepository::REPO_ERROR_CANCELLED) {
+        // if we were cancelled, don't report or log
+        return;
+      } else {
+        SG_LOG(SG_TERRASYNC, SG_WARN,
+               "failed to update entry:" << relativePath << " status/code: "
+                                         << innerResultCodeAsString(fileStatus)
+                                         << "/" << fileStatus);
+      }
 
-            queuedRequests.clear();
+      HTTPRepository::Failure f;
+      f.path = relativePath;
+      f.error = fileStatus;
+      failures.push_back(f);
 
-            RequestVector copyOfActive(activeRequests);
-            RequestVector::iterator rq;
-            for (rq = copyOfActive.begin(); rq != copyOfActive.end(); ++rq) {
-                http->cancelRequest(*rq, "Repository updated failed due to checksum error");
-            }
+      if (failures.size() >= maxPermittedFailures) {
+        SG_LOG(SG_TERRASYNC, SG_WARN,
+               "Repo:" << baseUrl << " exceeded failure count ("
+                       << failures.size() << "), abandoning");
 
-            SG_LOG(SG_TERRASYNC, SG_WARN, "failed to update repository:" << baseUrl
-                   << "\n\tchecksum failure for: " << relativePath
-                   << "\n\tthis typically indicates the remote repository is corrupt or was being updated during the sync");
-        } else if (fileStatus == HTTPRepository::REPO_ERROR_CANCELLED) {
-            // if we were cancelled, don't report or log
-            return;
-        } else {
-            SG_LOG(SG_TERRASYNC, SG_WARN, "failed to update entry:" << relativePath << " status/code: "
-               << innerResultCodeAsString(fileStatus) << "/" << fileStatus);
+        status = HTTPRepository::REPO_PARTIAL_UPDATE;
+
+        queuedRequests.clear();
+        auto copyOfActiveRequests = activeRequests;
+        for (auto rq : copyOfActiveRequests) {
+          http->cancelRequest(rq,
+                              "Abandoning repo sync due to multiple failures");
         }
+      }
+    }
 
-        Failure f;
-        f.path = relativePath;
-        f.error = fileStatus;
-        failures.push_back(f);
+    void HTTPRepoPrivate::updatedChildSuccessfully(const SGPath &relativePath) {
+      if (failures.empty()) {
+        return;
+      }
+
+      // find and remove any existing failures for that path
+      failures.erase(
+          std::remove_if(failures.begin(), failures.end(),
+                         [relativePath](const HTTPRepository::Failure &f) {
+                           return f.path == relativePath;
+                         }),
+          failures.end());
     }
 
 } // of namespace simgear

simgear/io/HTTPRepository.hxx

@@ -20,6 +20,7 @@
 #ifndef SG_IO_HTTP_REPOSITORY_HXX
 #define SG_IO_HTTP_REPOSITORY_HXX
 
+#include <functional>
 #include <memory>
 
 #include <simgear/misc/sg_path.hxx>
@@ -32,49 +33,78 @@ class HTTPRepoPrivate;
 class HTTPRepository
 {
 public:
-    enum ResultCode {
-        REPO_NO_ERROR = 0,
-        REPO_ERROR_NOT_FOUND,
-        REPO_ERROR_SOCKET,
-        SVN_ERROR_XML,
-        SVN_ERROR_TXDELTA,
-        REPO_ERROR_IO,
-        REPO_ERROR_CHECKSUM,
-        REPO_ERROR_FILE_NOT_FOUND,
-        REPO_ERROR_HTTP,
-        REPO_ERROR_CANCELLED,
-        REPO_PARTIAL_UPDATE
+  enum ResultCode {
+    REPO_NO_ERROR = 0,
+    REPO_ERROR_NOT_FOUND,
+    REPO_ERROR_SOCKET,
+    SVN_ERROR_XML,
+    SVN_ERROR_TXDELTA,
+    REPO_ERROR_IO,
+    REPO_ERROR_CHECKSUM,
+    REPO_ERROR_FILE_NOT_FOUND,
+    REPO_ERROR_HTTP,
+    REPO_ERROR_CANCELLED,
+    REPO_PARTIAL_UPDATE ///< repository is working, but file-level failures
+                        ///< occurred
+  };
+
+  HTTPRepository(const SGPath &root, HTTP::Client *cl);
+  virtual ~HTTPRepository();
+
+  virtual SGPath fsBase() const;
+
+  virtual void setBaseUrl(const std::string &url);
+  virtual std::string baseUrl() const;
+
+  virtual HTTP::Client *http() const;
+
+  virtual void update();
+
+  virtual bool isDoingSync() const;
+
+  virtual ResultCode failure() const;
+
+  virtual size_t bytesToDownload() const;
+
+  virtual size_t bytesDownloaded() const;
+
+  /**
+   * optionally provide the location of an installer copy of this
+   * repository. When a file is missing it will be copied from this tree.
+   */
+  void setInstalledCopyPath(const SGPath &copyPath);
+
+  static std::string resultCodeAsString(ResultCode code);
+
+  enum class SyncAction { Add, Update, Delete, UpToDate };
+
+  enum EntryType { FileType, DirectoryType, TarballType };
+
+  struct SyncItem {
+    const std::string directory; // relative path in the repository
+    const EntryType type;
+    const std::string filename;
+    const SyncAction action;
+    const SGPath pathOnDisk; // path the entry does / will have
+  };
+
+    using SyncPredicate = std::function<bool(const SyncItem &item)>;
+
+    void setFilter(SyncPredicate sp);
+
+    struct Failure {
+      SGPath path;
+      ResultCode error;
     };
 
-    HTTPRepository(const SGPath& root, HTTP::Client* cl);
-    virtual ~HTTPRepository();
-
-    virtual SGPath fsBase() const;
-
-    virtual void setBaseUrl(const std::string& url);
-    virtual std::string baseUrl() const;
-
-    virtual HTTP::Client* http() const;
-
-    virtual void update();
-
-    virtual bool isDoingSync() const;
-
-    virtual ResultCode failure() const;
-
-    virtual size_t bytesToDownload() const;
-
-    virtual size_t bytesDownloaded() const;
+    using FailureVec = std::vector<Failure>;
 
     /**
-     * optionally provide the location of an installer copy of this
-     * repository. When a file is missing it will be copied from this tree.
+     * @brief return file-level failures
      */
-    void setInstalledCopyPath(const SGPath& copyPath);
-    
-    static std::string resultCodeAsString(ResultCode code);
+    FailureVec failures() const;
 
-private:
+  private:
     bool isBare() const;
 
     std::unique_ptr<HTTPRepoPrivate> _d;

simgear/io/HTTPRepository_private.hxx

@@ -0,0 +1,122 @@
+// HTTPRepository.cxx -- plain HTTP TerraSync remote client
+//
+// Copyright (C) 20126  James Turner <zakalawe@mac.com>
+//
+// This program is free software; you can redistribute it and/or
+// modify it under the terms of the GNU General Public License as
+// published by the Free Software Foundation; either version 2 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
+// USA.
+
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include <simgear/io/HTTPClient.hxx>
+#include <simgear/misc/sg_path.hxx>
+
+#include "HTTPRepository.hxx"
+
+namespace simgear {
+
+class HTTPDirectory;
+using HTTPDirectory_ptr = std::unique_ptr<HTTPDirectory>;
+
+class HTTPRepoGetRequest : public HTTP::Request {
+public:
+  HTTPRepoGetRequest(HTTPDirectory *d, const std::string &u)
+      : HTTP::Request(u), _directory(d) {}
+
+  virtual void cancel();
+
+  size_t contentSize() const { return _contentSize; }
+
+  void setContentSize(size_t sz) { _contentSize = sz; }
+
+protected:
+  HTTPDirectory *_directory;
+  size_t _contentSize = 0;
+};
+
+typedef SGSharedPtr<HTTPRepoGetRequest> RepoRequestPtr;
+
+class HTTPRepoPrivate {
+public:
+  struct HashCacheEntry {
+    std::string filePath;
+    time_t modTime;
+    size_t lengthBytes;
+    std::string hashHex;
+  };
+
+  typedef std::vector<HashCacheEntry> HashCache;
+  HashCache hashes;
+  int hashCacheDirty = 0;
+
+  HTTPRepository::FailureVec failures;
+  int maxPermittedFailures = 16;
+
+  HTTPRepoPrivate(HTTPRepository *parent)
+      : p(parent), isUpdating(false), status(HTTPRepository::REPO_NO_ERROR),
+        totalDownloaded(0) {
+    ;
+  }
+
+  ~HTTPRepoPrivate();
+
+  HTTPRepository *p; // link back to outer
+  HTTP::Client *http;
+  std::string baseUrl;
+  SGPath basePath;
+  bool isUpdating;
+  HTTPRepository::ResultCode status;
+  HTTPDirectory_ptr rootDir;
+  size_t totalDownloaded;
+  HTTPRepository::SyncPredicate syncPredicate;
+
+  HTTP::Request_ptr updateFile(HTTPDirectory *dir, const std::string &name,
+                               size_t sz);
+  HTTP::Request_ptr updateDir(HTTPDirectory *dir, const std::string &hash,
+                              size_t sz);
+
+  std::string hashForPath(const SGPath &p);
+  void updatedFileContents(const SGPath &p, const std::string &newHash);
+  void parseHashCache();
+  std::string computeHashForPath(const SGPath &p);
+  void writeHashCache();
+
+  void failedToGetRootIndex(HTTPRepository::ResultCode st);
+  void failedToUpdateChild(const SGPath &relativePath,
+                           HTTPRepository::ResultCode fileStatus);
+
+  void updatedChildSuccessfully(const SGPath &relativePath);
+
+  typedef std::vector<RepoRequestPtr> RequestVector;
+  RequestVector queuedRequests, activeRequests;
+
+  void makeRequest(RepoRequestPtr req);
+
+  enum class RequestFinish { Done, Retry };
+
+  void finishedRequest(const RepoRequestPtr &req, RequestFinish retryRequest);
+
+  HTTPDirectory *getOrCreateDirectory(const std::string &path);
+  bool deleteDirectory(const std::string &relPath, const SGPath &absPath);
+
+  typedef std::vector<HTTPDirectory_ptr> DirectoryVector;
+  DirectoryVector directories;
+
+  SGPath installedCopyPath;
+};
+
+} // namespace simgear

simgear/io/HTTPRequest.cxx

@@ -56,6 +56,15 @@ Request::~Request()
 
 }
 
+void Request::prepareForRetry() {
+  setReadyState(UNSENT);
+  _willClose = false;
+  _connectionCloseHeader = false;
+  _responseStatus = 0;
+  _responseLength = 0;
+  _receivedBodyBytes = 0;
+}
+
 //------------------------------------------------------------------------------
 Request* Request::done(const Callback& cb)
 {

simgear/io/HTTPRequest.hxx

@@ -208,7 +208,9 @@ public:
      */
     bool serverSupportsPipelining() const;
 
-protected:
+    virtual void prepareForRetry();
+
+  protected:
     Request(const std::string& url, const std::string method = "GET");
 
     virtual void requestStart();
@@ -222,12 +224,14 @@ protected:
     virtual void onFail();
     virtual void onAlways();
 
-    void setFailure(int code, const std::string& reason);
     void setSuccess(int code);
-private:
+    void setFailure(int code, const std::string &reason);
+
+  private:
     friend class Client;
     friend class Connection;
     friend class ContentDecoder;
+    friend class TestApi;
 
     Request(const Request&); // = delete;
     Request& operator=(const Request&); // = delete;

simgear/io/HTTPTestApi_private.hxx

@@ -0,0 +1,45 @@
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Library General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301, USA
+
+#pragma once
+
+#include <functional>
+
+#include "HTTPRequest.hxx"
+
+namespace simgear {
+namespace HTTP {
+
+class Client;
+
+using ResponseDoneCallback =
+    std::function<bool(int curlResult, Request_ptr req)>;
+
+/**
+ * @brief this API is for unit-testing HTTP code.
+ * Don't use it for anything else. It's for unit-testing.
+ */
+class TestApi {
+public:
+  // alow test suite to manipulate requests to simulate network errors;
+  // without this, it's hard to provoke certain failures in a loop-back
+  // network sitation.
+  static void setResponseDoneCallback(Client *cl, ResponseDoneCallback cb);
+
+  static void markRequestAsFailed(Request_ptr req, int curlCode,
+                                  const std::string &message);
+};
+
+} // namespace HTTP
+} // namespace simgear

simgear/io/sg_file.cxx

@@ -71,21 +71,31 @@ SGFile::~SGFile() {
 std::string SGFile::computeHash()
 {
     if (!file_name.exists())
-        return std::string();
+        return {};
+        
     simgear::sha1nfo info;
     sha1_init(&info);
-    char* buf = static_cast<char*>(malloc(1024 * 1024));
+    
+    // unique_ptr with custom deleter for exception safety
+    const int bufSize = 1024 * 1024;
+    std::unique_ptr<char, std::function<void(char*)>> buf{static_cast<char*>(malloc(bufSize)), 
+        [](char* p) { free(p); }};
+
+    if (!buf) {
+        SG_LOG(SG_IO, SG_ALERT, "Failed to malloc buffer for SHA1 check");
+    }
+
     size_t readLen;
     SGBinaryFile f(file_name);
     if (!f.open(SG_IO_IN)) {
-        throw sg_io_exception("Couldn't open file for compute hash", file_name);
+        SG_LOG(SG_IO, SG_ALERT, "SGFile::computeHash: Failed to open " << file_name);
+        return {};
     }
-    while ((readLen = f.read(buf, 1024 * 1024)) > 0) {
-        sha1_write(&info, buf, readLen);
+    while ((readLen = f.read(buf.get(), bufSize)) > 0) {
+        sha1_write(&info, buf.get(), readLen);
     }
 
     f.close();
-    free(buf);
     std::string hashBytes((char*)sha1_result(&info), HASH_LENGTH);
     return simgear::strutils::encodeHex(hashBytes);
 }

simgear/io/test_repository.cxx

@@ -1,16 +1,18 @@
 #include <cassert>
 #include <cstdlib>
+#include <errno.h>
+#include <fcntl.h>
+#include <functional>
 #include <iostream>
 #include <map>
 #include <sstream>
-#include <errno.h>
-#include <fcntl.h>
 
 #include <simgear/simgear_config.h>
 
-#include "test_HTTP.hxx"
-#include "HTTPRepository.hxx"
 #include "HTTPClient.hxx"
+#include "HTTPRepository.hxx"
+#include "HTTPTestApi_private.hxx"
+#include "test_HTTP.hxx"
 
 #include <simgear/misc/strutils.hxx>
 #include <simgear/misc/sg_hash.hxx>
@@ -25,6 +27,8 @@
 
 using namespace simgear;
 
+using TestApi = simgear::HTTP::TestApi;
+
 std::string dataForFile(const std::string& parentName, const std::string& name, int revision)
 {
     std::ostringstream os;
@@ -45,6 +49,9 @@ std::string hashForData(const std::string& d)
     return strutils::encodeHex(sha1_result(&info), HASH_LENGTH);
 }
 
+class TestRepoEntry;
+using AccessCallback = std::function<void(TestRepoEntry &entry)>;
+
 class TestRepoEntry
 {
 public:
@@ -70,7 +77,8 @@ public:
     int requestCount;
     bool getWillFail;
     bool returnCorruptData;
-    std::unique_ptr<SGCallback> accessCallback;
+
+    AccessCallback accessCallback;
 
     void clearRequestCounts();
 
@@ -270,8 +278,8 @@ public:
                 return;
             }
 
-            if (entry->accessCallback.get()) {
-                (*entry->accessCallback)();
+            if (entry->accessCallback) {
+              entry->accessCallback(*entry);
             }
 
             if (entry->getWillFail) {
@@ -282,20 +290,29 @@ public:
             entry->requestCount++;
 
             std::string content;
+            bool closeSocket = false;
+            size_t contentSize = 0;
+
             if (entry->returnCorruptData) {
                 content = dataForFile("!$£$!" + entry->parent->name,
                                       "corrupt_" + entry->name,
                                       entry->revision);
+                contentSize = content.size();
             } else {
-                content = entry->data();
+              content = entry->data();
+              contentSize = content.size();
             }
 
             std::stringstream d;
             d << "HTTP/1.1 " << 200 << " " << reasonForCode(200) << "\r\n";
-            d << "Content-Length:" << content.size() << "\r\n";
+            d << "Content-Length:" << contentSize << "\r\n";
             d << "\r\n"; // final CRLF to terminate the headers
             d << content;
             push(d.str().c_str());
+
+            if (closeSocket) {
+              closeWhenDone();
+            }
         } else {
             sendErrorResponse(404, false, "");
         }
@@ -401,6 +418,15 @@ void waitForUpdateComplete(HTTP::Client* cl, HTTPRepository* repo)
     std::cerr << "timed out" << std::endl;
 }
 
+void runForTime(HTTP::Client *cl, HTTPRepository *repo, int msec = 15) {
+  SGTimeStamp start(SGTimeStamp::now());
+  while (start.elapsedMSec() < msec) {
+    cl->update();
+    testServer.poll();
+    SGTimeStamp::sleepForMSec(1);
+  }
+}
+
 void testBasicClone(HTTP::Client* cl)
 {
     std::unique_ptr<HTTPRepository> repo;
@@ -618,9 +644,19 @@ void testAbandonCorruptFiles(HTTP::Client* cl)
     repo->setBaseUrl("http://localhost:2000/repo");
     repo->update();
     waitForUpdateComplete(cl, repo.get());
-    if (repo->failure() != HTTPRepository::REPO_ERROR_CHECKSUM) {
-        std::cerr << "Got failure state:" << repo->failure() << std::endl;
-        throw sg_exception("Bad result from corrupt files test");
+    if (repo->failure() != HTTPRepository::REPO_PARTIAL_UPDATE) {
+      std::cerr << "Got failure state:" << repo->failure() << std::endl;
+      throw sg_exception("Bad result from corrupt files test");
+    }
+
+    auto failedFiles = repo->failures();
+    if (failedFiles.size() != 1) {
+      throw sg_exception("Bad result from corrupt files test");
+    }
+
+    if (failedFiles.front().path.utf8Str() != "dirB/subdirG/fileBGA") {
+      throw sg_exception("Bad path from corrupt files test:" +
+                         failedFiles.front().path.utf8Str());
     }
 
     repo.reset();
@@ -657,15 +693,21 @@ void testServerModifyDuringSync(HTTP::Client* cl)
     repo.reset(new HTTPRepository(p, cl));
     repo->setBaseUrl("http://localhost:2000/repo");
 
-    global_repo->findEntry("dirA/fileAA")->accessCallback.reset(make_callback(&modifyBTree));
+    global_repo->findEntry("dirA/fileAA")->accessCallback =
+        [](const TestRepoEntry &r) {
+          std::cout << "Modifying sub-tree" << std::endl;
+          global_repo->findEntry("dirB/subdirA/fileBAC")->revision++;
+          global_repo->defineFile("dirB/subdirZ/fileBZA");
+          global_repo->findEntry("dirB/subdirB/fileBBB")->revision++;
+        };
 
     repo->update();
     waitForUpdateComplete(cl, repo.get());
 
-    global_repo->findEntry("dirA/fileAA")->accessCallback.reset();
+    global_repo->findEntry("dirA/fileAA")->accessCallback = AccessCallback{};
 
-    if (repo->failure() != HTTPRepository::REPO_ERROR_CHECKSUM) {
-        throw sg_exception("Bad result from modify during sync test");
+    if (repo->failure() != HTTPRepository::REPO_PARTIAL_UPDATE) {
+      throw sg_exception("Bad result from modify during sync test");
     }
 
     std::cout << "Passed test modify server during sync" << std::endl;
@@ -755,6 +797,103 @@ void testCopyInstalledChildren(HTTP::Client* cl)
     std::cout << "passed Copy installed children" << std::endl;
 }
 
+void testRetryAfterSocketFailure(HTTP::Client *cl) {
+  global_repo->clearRequestCounts();
+  global_repo->clearFailFlags();
+
+  std::unique_ptr<HTTPRepository> repo;
+  SGPath p(simgear::Dir::current().path());
+  p.append("http_repo_retry_after_socket_fail");
+  simgear::Dir pd(p);
+  if (pd.exists()) {
+    pd.removeChildren();
+  }
+
+  repo.reset(new HTTPRepository(p, cl));
+  repo->setBaseUrl("http://localhost:2000/repo");
+
+  int aaFailsRemaining = 2;
+  int subdirBAFailsRemaining = 2;
+  TestApi::setResponseDoneCallback(
+      cl, [&aaFailsRemaining, &subdirBAFailsRemaining](int curlResult,
+                                                       HTTP::Request_ptr req) {
+        if (req->url() == "http://localhost:2000/repo/dirA/fileAA") {
+          if (aaFailsRemaining == 0)
+            return false;
+
+          --aaFailsRemaining;
+          TestApi::markRequestAsFailed(req, 56, "Simulated socket failure");
+          return true;
+        } else if (req->url() ==
+                   "http://localhost:2000/repo/dirB/subdirA/.dirindex") {
+          if (subdirBAFailsRemaining == 0)
+            return false;
+
+          --subdirBAFailsRemaining;
+          TestApi::markRequestAsFailed(req, 56, "Simulated socket failure");
+          return true;
+        } else {
+          return false;
+        }
+      });
+
+  repo->update();
+  waitForUpdateComplete(cl, repo.get());
+
+  if (repo->failure() != HTTPRepository::REPO_NO_ERROR) {
+    throw sg_exception("Bad result from retry socket failure test");
+  }
+
+  verifyFileState(p, "dirA/fileAA");
+  verifyFileState(p, "dirB/subdirA/fileBAA");
+  verifyFileState(p, "dirB/subdirA/fileBAC");
+
+  verifyRequestCount("dirA/fileAA", 3);
+  verifyRequestCount("dirB/subdirA", 3);
+  verifyRequestCount("dirB/subdirA/fileBAC", 1);
+}
+
+void testPersistentSocketFailure(HTTP::Client *cl) {
+  global_repo->clearRequestCounts();
+  global_repo->clearFailFlags();
+
+  std::unique_ptr<HTTPRepository> repo;
+  SGPath p(simgear::Dir::current().path());
+  p.append("http_repo_persistent_socket_fail");
+  simgear::Dir pd(p);
+  if (pd.exists()) {
+    pd.removeChildren();
+  }
+
+  repo.reset(new HTTPRepository(p, cl));
+  repo->setBaseUrl("http://localhost:2000/repo");
+
+  TestApi::setResponseDoneCallback(
+      cl, [](int curlResult, HTTP::Request_ptr req) {
+        const auto url = req->url();
+        if (url.find("http://localhost:2000/repo/dirB") == 0) {
+          TestApi::markRequestAsFailed(req, 56, "Simulated socket failure");
+          return true;
+        }
+
+        return false;
+      });
+
+  repo->update();
+  waitForUpdateComplete(cl, repo.get());
+
+  if (repo->failure() != HTTPRepository::REPO_PARTIAL_UPDATE) {
+    throw sg_exception("Bad result from retry socket failure test");
+  }
+
+  verifyFileState(p, "dirA/fileAA");
+  verifyRequestCount("dirA/fileAA", 1);
+
+  verifyRequestCount("dirD/fileDA", 1);
+  verifyRequestCount("dirD/subdirDA/fileDAA", 1);
+  verifyRequestCount("dirD/subdirDB/fileDBA", 1);
+}
+
 int main(int argc, char* argv[])
 {
   sglog().setLogLevels( SG_ALL, SG_INFO );
@@ -800,6 +939,8 @@ int main(int argc, char* argv[])
     cl.clearAllConnections();
 
     testCopyInstalledChildren(&cl);
+    testRetryAfterSocketFailure(&cl);
+    testPersistentSocketFailure(&cl);
 
     std::cout << "all tests passed ok" << std::endl;
     return 0;

simgear/misc/ResourceManager.cxx

@@ -49,6 +49,11 @@ ResourceManager* ResourceManager::instance()
     return static_manager;
 }    
 
+bool ResourceManager::haveInstance()
+{
+    return static_manager != nullptr;
+}
+
 ResourceManager::~ResourceManager()
 {
     assert(this == static_manager);

simgear/misc/ResourceManager.hxx

@@ -47,6 +47,8 @@ public:
 
     static ResourceManager* instance();
 
+    static bool haveInstance();
+
     static void reset();
 
     /**

simgear/misc/sg_path.cxx

@@ -44,6 +44,7 @@
 #if defined(SG_WINDOWS)
 #  include <direct.h>
 #  include <sys/utime.h>
+#  include <Shlwapi.h>
 #endif
 
 #include "sg_path.hxx"
@@ -194,7 +195,8 @@ SGPath::SGPath(PermissionChecker validator)
     _permission_checker(validator),
     _cached(false),
     _rwCached(false),
-    _cacheEnabled(true)
+    _cacheEnabled(true),
+    _existsCached(false)
 {
 }
 
@@ -205,7 +207,8 @@ SGPath::SGPath( const std::string& p, PermissionChecker validator )
     _permission_checker(validator),
     _cached(false),
     _rwCached(false),
-    _cacheEnabled(true)
+    _cacheEnabled(true),
+    _existsCached(false)
 {
     fix();
 }
@@ -230,7 +233,8 @@ SGPath::SGPath( const SGPath& p,
     _permission_checker(validator),
     _cached(false),
     _rwCached(false),
-    _cacheEnabled(p._cacheEnabled)
+    _cacheEnabled(p._cacheEnabled),
+    _existsCached(false)
 {
     append(r);
     fix();
@@ -510,6 +514,19 @@ void SGPath::checkAccess() const
 
 bool SGPath::exists() const
 {
+#if defined(SG_WINDOWS)
+  // optimisation: _wstat is slow, eg for TerraSync
+  if (!_cached && !_existsCached) {
+      std::wstring w(wstr());
+    if ((path.length() > 1) && (path.back() == '/')) {
+	    w.pop_back();
+    }
+
+    _existsCached = true;
+    _exists = PathFileExistsW(w.c_str());
+    return _exists;
+  }
+#endif
   validate();
   return _exists;
 }

simgear/misc/sg_path.hxx

@@ -356,6 +356,7 @@ private:
     mutable bool _exists : 1;
     mutable bool _isDir : 1;
     mutable bool _isFile : 1;
+    mutable bool _existsCached : 1; ///< only used on Windows
     mutable time_t _modTime;
     mutable size_t _size;
 };

simgear/package/Catalog.cxx

@@ -643,16 +643,53 @@ void Catalog::processAlternate(SGPropertyNode_ptr alt)
         return;
     }
 
-    // we have an alternate ID, and it's differnt from our ID, so let's
+    // we have an alternate ID, and it's different from our ID, so let's
     // define a new catalog
     if (!altId.empty()) {
-        SG_LOG(SG_GENERAL, SG_INFO, "Adding new catalog:" << altId << " as version alternate for " << id());
-        // new catalog being added
-        createFromUrl(root(), altUrl);
-
-        // and we can go idle now
+      // don't auto-re-add Catalogs the user has explicilty rmeoved, that would
+      // suck
+      const auto removedByUser = root()->explicitlyRemovedCatalogs();
+      auto it = std::find(removedByUser.begin(), removedByUser.end(), altId);
+      if (it != removedByUser.end()) {
         changeStatus(Delegate::FAIL_VERSION);
         return;
+      }
+
+      SG_LOG(SG_GENERAL, SG_WARN,
+             "Adding new catalog:" << altId << " as version alternate for "
+                                   << id());
+      // new catalog being added
+      auto newCat = createFromUrl(root(), altUrl);
+
+      bool didRun = false;
+      newCat->m_migratedFrom = this;
+
+      auto migratePackagesCb = [didRun](Catalog *c) mutable {
+        // removing callbacks is awkward, so use this
+        // flag to only run once. (and hence, we need to be mutable)
+        if (didRun)
+          return;
+
+        if (c->status() == Delegate::STATUS_REFRESHED) {
+          didRun = true;
+
+          string_list existing;
+          for (const auto &pack : c->migratedFrom()->installedPackages()) {
+            existing.push_back(pack->id());
+          }
+
+          const int count = c->markPackagesForInstallation(existing);
+          SG_LOG(
+              SG_GENERAL, SG_INFO,
+              "Marked " << count
+                        << " packages from previous catalog for installation");
+        }
+      };
+
+      newCat->addStatusCallback(migratePackagesCb);
+      // and we can go idle now
+      changeStatus(Delegate::FAIL_VERSION);
+      return;
     }
 
     SG_LOG(SG_GENERAL, SG_INFO, "Migrating catalog " << id() << " to new URL:" << altUrl);
@@ -661,6 +698,26 @@ void Catalog::processAlternate(SGPropertyNode_ptr alt)
     root()->makeHTTPRequest(dl);
 }
 
+int Catalog::markPackagesForInstallation(const string_list &packageIds) {
+  int result = 0;
+
+  for (const auto &id : packageIds) {
+    auto ourPkg = getPackageById(id);
+    if (!ourPkg)
+      continue;
+
+    auto existing = ourPkg->existingInstall();
+    if (!existing) {
+      ourPkg->markForInstall();
+      ++result;
+    }
+  } // of outer package ID candidates iteration
+
+  return result;
+}
+
+CatalogRef Catalog::migratedFrom() const { return m_migratedFrom; }
+
 } // of namespace pkg
 
 } // of namespace simgear

simgear/package/Catalog.hxx

@@ -23,6 +23,7 @@
 #include <map>
 
 #include <simgear/misc/sg_path.hxx>
+#include <simgear/misc/strutils.hxx>
 #include <simgear/props/props.hxx>
 
 #include <simgear/structure/SGReferenced.hxx>
@@ -93,7 +94,7 @@ public:
 
     /**
      * retrieve all the packages in the catalog which are installed
-     * and have a pendig update
+     * and have a pending update
      */
     PackageList packagesNeedingUpdate() const;
 
@@ -151,7 +152,32 @@ public:
     
     bool isUserEnabled() const;
     void setUserEnabled(bool b);
-private:
+
+    /**
+     * Given a list of package IDs, mark all which exist in this package,
+     * for installation. ANy packahe IDs not present in this catalog,
+     * will be ignored.
+     *
+     * @result The number for packages newly marked for installation.
+     */
+    int markPackagesForInstallation(const string_list &packageIds);
+
+    /**
+     * When a catalog is added due to migration, this will contain the
+     * Catalog which triggered the add. Usually this will be a catalog
+     * corresponding to an earlier version.
+     *
+     * Note it's only valid at the time, the migration actually took place;
+     * when the new catalog is loaded from disk, this value will return
+     * null.
+     *
+     * This is intended to allow Uis to show a 'catalog was migrated'
+     * feedback, when they see a catalog refresh, which has a non-null
+     * value of this method.
+     */
+    CatalogRef migratedFrom() const;
+
+  private:
     Catalog(Root* aRoot);
 
     class Downloader;
@@ -197,6 +223,8 @@ private:
     PackageWeakMap m_variantDict;
 
     function_list<Callback> m_statusCallbacks;
+
+    CatalogRef m_migratedFrom;
 };
 
 } // of namespace pkg

simgear/package/CatalogTest.cxx

@@ -827,7 +827,10 @@ void testVersionMigrateToId(HTTP::Client* cl)
         
         it = std::find(enabledCats.begin(), enabledCats.end(), altCat);
         SG_VERIFY(it != enabledCats.end());
-        
+
+        SG_CHECK_EQUAL(altCat->packagesNeedingUpdate().size(),
+                       1); // should be the 737
+
         // install a parallel package from the new catalog
         pkg::PackageRef p2 = root->getPackageById("org.flightgear.test.catalog-alt.b737-NG");
         SG_CHECK_EQUAL(p2->id(), "b737-NG");
@@ -840,8 +843,8 @@ void testVersionMigrateToId(HTTP::Client* cl)
         pkg::PackageRef p3 = root->getPackageById("b737-NG");
         SG_CHECK_EQUAL(p2, p3);
     }
-    
-    // test that re-init-ing doesn't mirgate again
+
+    // test that re-init-ing doesn't migrate again
     {
         pkg::RootRef root(new pkg::Root(rootPath, "7.5"));
         root->setHTTPClient(cl);
@@ -1184,6 +1187,128 @@ void testMirrorsFailure(HTTP::Client* cl)
     
 }
 
+void testMigrateInstalled(HTTP::Client *cl) {
+  SGPath rootPath(simgear::Dir::current().path());
+  rootPath.append("pkg_migrate_installed");
+  simgear::Dir pd(rootPath);
+  pd.removeChildren();
+
+  pkg::RootRef root(new pkg::Root(rootPath, "8.1.2"));
+  root->setHTTPClient(cl);
+
+  pkg::CatalogRef oldCatalog, newCatalog;
+
+  {
+    oldCatalog = pkg::Catalog::createFromUrl(
+        root.ptr(), "http://localhost:2000/catalogTest1/catalog.xml");
+    waitForUpdateComplete(cl, root);
+
+    pkg::PackageRef p1 =
+        root->getPackageById("org.flightgear.test.catalog1.b747-400");
+    p1->install();
+    auto p2 = root->getPackageById("org.flightgear.test.catalog1.c172p");
+    p2->install();
+    auto p3 = root->getPackageById("org.flightgear.test.catalog1.b737-NG");
+    p3->install();
+    waitForUpdateComplete(cl, root);
+  }
+
+  {
+    newCatalog = pkg::Catalog::createFromUrl(
+        root.ptr(), "http://localhost:2000/catalogTest2/catalog.xml");
+    waitForUpdateComplete(cl, root);
+
+    string_list existing;
+    for (const auto &pack : oldCatalog->installedPackages()) {
+      existing.push_back(pack->id());
+    }
+
+    SG_CHECK_EQUAL(4, existing.size());
+
+    int result = newCatalog->markPackagesForInstallation(existing);
+    SG_CHECK_EQUAL(2, result);
+    SG_CHECK_EQUAL(2, newCatalog->packagesNeedingUpdate().size());
+
+    auto p1 = root->getPackageById("org.flightgear.test.catalog2.b737-NG");
+    auto ins = p1->existingInstall();
+    SG_CHECK_EQUAL(0, ins->revsion());
+  }
+
+  {
+    root->scheduleAllUpdates();
+    waitForUpdateComplete(cl, root);
+
+    SG_CHECK_EQUAL(0, newCatalog->packagesNeedingUpdate().size());
+
+    auto p1 = root->getPackageById("org.flightgear.test.catalog2.b737-NG");
+    auto ins = p1->existingInstall();
+    SG_CHECK_EQUAL(ins->revsion(), p1->revision());
+  }
+}
+
+void testDontMigrateRemoved(HTTP::Client *cl) {
+  global_catalogVersion = 2; // version which has migration info
+  SGPath rootPath(simgear::Dir::current().path());
+  rootPath.append("cat_dont_migrate_id");
+  simgear::Dir pd(rootPath);
+  pd.removeChildren();
+
+  // install and mnaully remove the alt catalog
+
+  {
+    pkg::RootRef root(new pkg::Root(rootPath, "8.1.2"));
+    root->setHTTPClient(cl);
+
+    pkg::CatalogRef c = pkg::Catalog::createFromUrl(
+        root.ptr(), "http://localhost:2000/catalogTest1/catalog-alt.xml");
+    waitForUpdateComplete(cl, root);
+
+    root->removeCatalogById("org.flightgear.test.catalog-alt");
+  }
+
+  // install the migration catalog
+  {
+    pkg::RootRef root(new pkg::Root(rootPath, "8.1.2"));
+    root->setHTTPClient(cl);
+
+    pkg::CatalogRef c = pkg::Catalog::createFromUrl(
+        root.ptr(), "http://localhost:2000/catalogTest1/catalog.xml");
+    waitForUpdateComplete(cl, root);
+    SG_VERIFY(c->isEnabled());
+  }
+
+  // change version to an alternate one
+  {
+    pkg::RootRef root(new pkg::Root(rootPath, "7.5"));
+
+    auto removed = root->explicitlyRemovedCatalogs();
+    auto j = std::find(removed.begin(), removed.end(),
+                       "org.flightgear.test.catalog-alt");
+    SG_VERIFY(j != removed.end());
+
+    root->setHTTPClient(cl);
+
+    // this would tirgger migration, but we blocked it
+    root->refresh(true);
+    waitForUpdateComplete(cl, root);
+
+    pkg::CatalogRef cat = root->getCatalogById("org.flightgear.test.catalog1");
+    SG_VERIFY(!cat->isEnabled());
+    SG_CHECK_EQUAL(cat->status(), pkg::Delegate::FAIL_VERSION);
+    SG_CHECK_EQUAL(cat->id(), "org.flightgear.test.catalog1");
+    SG_CHECK_EQUAL(cat->url(),
+                   "http://localhost:2000/catalogTest1/catalog.xml");
+
+    auto enabledCats = root->catalogs();
+    auto it = std::find(enabledCats.begin(), enabledCats.end(), cat);
+    SG_VERIFY(it == enabledCats.end());
+
+    // check the new catalog
+    auto altCat = root->getCatalogById("org.flightgear.test.catalog-alt");
+    SG_VERIFY(altCat.get() == nullptr);
+  }
+}
+
 int main(int argc, char* argv[])
 {
     sglog().setLogLevels( SG_ALL, SG_WARN );
@@ -1228,7 +1353,11 @@ int main(int argc, char* argv[])
     testInstallBadPackage(&cl);
     
     testMirrorsFailure(&cl);
-    
+
+    testMigrateInstalled(&cl);
+
+    testDontMigrateRemoved(&cl);
+
     cerr << "Successfully passed all tests!" << endl;
     return EXIT_SUCCESS;
 }

simgear/package/Package.cxx

@@ -198,6 +198,11 @@ InstallRef Package::install()
 {
     InstallRef ins = existingInstall();
     if (ins) {
+      // if there's updates, treat this as a 'start update' request
+      if (ins->hasUpdate()) {
+        m_catalog->root()->scheduleToUpdate(ins);
+      }
+
         return ins;
     }
 
@@ -210,13 +215,39 @@ InstallRef Package::install()
     return ins;
 }
 
+InstallRef Package::markForInstall() {
+  InstallRef ins = existingInstall();
+  if (ins) {
+    return ins;
+  }
+
+  const auto pd = pathOnDisk();
+
+  Dir dir(pd);
+  if (!dir.create(0700)) {
+    SG_LOG(SG_IO, SG_ALERT,
+           "Package::markForInstall: couldn't create directory at:" << pd);
+    return {};
+  }
+
+  ins = new Install{this, pd};
+  _install_cb(this, ins); // not sure if we should trigger the callback for this
+
+  // repeat for dependencies to be kind
+  for (auto dep : dependencies()) {
+    dep->markForInstall();
+  }
+
+  return ins;
+}
+
 InstallRef Package::existingInstall(const InstallCallback& cb) const
 {
     InstallRef install;
     try {
         install = m_catalog->root()->existingInstallForPackage(const_cast<Package*>(this));
     } catch (std::exception& ) {
-        return InstallRef();
+      return {};
     }
 
   if( cb )

simgear/package/Package.hxx

@@ -61,6 +61,13 @@ public:
     InstallRef
     existingInstall(const InstallCallback& cb = InstallCallback()) const;
 
+    /**
+     * Mark this package for installation, but don't actually start the
+     * download process. This creates the on-disk placeholder, so
+     * the package will appear an eededing to be updated.
+     */
+    InstallRef markForInstall();
+
     bool isInstalled() const;
 
     /**

simgear/package/Root.cxx

@@ -283,6 +283,32 @@ public:
         fireDataForThumbnail(url, reinterpret_cast<const uint8_t*>(bytes.data()), bytes.size());
     }
 
+    void writeRemovedCatalogsFile() const {
+      SGPath p = path / "RemovedCatalogs";
+      sg_ofstream stream(p, std::ios::out | std::ios::trunc | std::ios::binary);
+      for (const auto &cid : manuallyRemovedCatalogs) {
+        stream << cid << "\n";
+      }
+      stream.close();
+    }
+
+    void loadRemovedCatalogsFile() {
+      manuallyRemovedCatalogs.clear();
+      SGPath p = path / "RemovedCatalogs";
+      if (!p.exists())
+        return;
+
+      sg_ifstream stream(p, std::ios::in);
+      while (!stream.eof()) {
+        std::string line;
+        std::getline(stream, line);
+        const auto trimmed = strutils::strip(line);
+        if (!trimmed.empty()) {
+          manuallyRemovedCatalogs.push_back(trimmed);
+        }
+      } // of lines iteration
+    }
+
     DelegateVec delegates;
 
     SGPath path;
@@ -312,6 +338,9 @@ public:
 
     typedef std::map<PackageRef, InstallRef> InstallCache;
     InstallCache m_installs;
+
+    /// persistent list of catalogs the user has manually removed
+    string_list manuallyRemovedCatalogs;
 };
 
 
@@ -400,6 +429,8 @@ Root::Root(const SGPath& aPath, const std::string& aVersion) :
         thumbsCacheDir.create(0755);
     }
 
+    d->loadRemovedCatalogsFile();
+
     for (SGPath c : dir.children(Dir::TYPE_DIR | Dir::NO_DOT_OR_DOTDOT)) {
         // note this will set the catalog status, which will insert into
         // disabled catalogs automatically if necesary
@@ -621,6 +652,13 @@ void Root::scheduleToUpdate(InstallRef aInstall)
     }
 }
 
+void Root::scheduleAllUpdates() {
+  auto toBeUpdated = packagesNeedingUpdate(); // make a copy
+  for (const auto &u : toBeUpdated) {
+    scheduleToUpdate(u->existingInstall());
+  }
+}
+
 bool Root::isInstallQueued(InstallRef aInstall) const
 {
     auto it = std::find(d->updateDeque.begin(), d->updateDeque.end(), aInstall);
@@ -783,6 +821,9 @@ bool Root::removeCatalogById(const std::string& aId)
             << "failed to remove directory");
     }
 
+    d->manuallyRemovedCatalogs.push_back(aId);
+    d->writeRemovedCatalogsFile();
+
     // notify that a catalog is being removed
     d->firePackagesChanged();
 
@@ -854,6 +895,10 @@ void Root::unregisterInstall(InstallRef ins)
     d->fireFinishUninstall(ins->package());
 }
 
+string_list Root::explicitlyRemovedCatalogs() const {
+  return d->manuallyRemovedCatalogs;
+}
+
 } // of namespace pkg
 
 } // of namespace simgear

simgear/package/Root.hxx

@@ -155,7 +155,22 @@ public:
     void requestThumbnailData(const std::string& aUrl);
     
     bool isInstallQueued(InstallRef aInstall) const;
-private:
+
+    /**
+     * Mark all 'to be updated' packages for update now
+     */
+    void scheduleAllUpdates();
+
+    /**
+     * @brief list of catalog IDs, the user has explicitly removed via
+     * removeCatalogById(). This is important to allow the user to opt-out
+     * of migrated packages.
+     *
+     * This information is stored in a helper file, in the root directory
+     */
+    string_list explicitlyRemovedCatalogs() const;
+
+  private:
     friend class Install;
     friend class Catalog;    
     friend class Package;

simgear/package/catalogTest2/catalog.xml

@@ -0,0 +1,86 @@
+<?xml version="1.0"?>
+
+<PropertyList>
+    <id>org.flightgear.test.catalog2</id>
+    <description>Second test catalog</description>
+    <url>http://localhost:2000/catalogTest2/catalog.xml</url>
+    <catalog-version>4</catalog-version>
+
+    <version>8.1.*</version>
+    <version>8.0.0</version>
+    <version>8.2.0</version>
+
+    <package>
+        <id>alpha</id>
+        <name>Alpha package</name>
+        <revision type="int">8</revision>
+        <file-size-bytes type="int">593</file-size-bytes>
+
+        <md5>a469c4b837f0521db48616cfe65ac1ea</md5>
+        <url>http://localhost:2000/catalogTest1/alpha.zip</url>
+
+        <dir>alpha</dir>
+
+    </package>
+
+
+    <package>
+        <id>b737-NG</id>
+        <name>Boeing 737 NG</name>
+        <dir>b737NG</dir>
+        <description>A popular twin-engined narrow body jet</description>
+        <revision type="int">111</revision>
+        <file-size-bytes type="int">860</file-size-bytes>
+
+        <tag>boeing</tag>
+        <tag>jet</tag>
+        <tag>ifr</tag>
+
+      <!-- not within a localized element -->
+          <de>
+            <description>German description of B737NG XYZ</description>
+          </de>
+          <fr>
+            <description>French description of B737NG</description>
+          </fr>
+
+        <rating>
+          <FDM type="int">5</FDM>
+          <systems type="int">5</systems>
+          <model type="int">4</model>
+          <cockpit type="int">4</cockpit>
+        </rating>
+
+        <md5>a94ca5704f305b90767f40617d194ed6</md5>
+        <url>http://localhost:2000/mirrorA/b737.tar.gz</url>
+        <url>http://localhost:2000/mirrorB/b737.tar.gz</url>
+        <url>http://localhost:2000/mirrorC/b737.tar.gz</url>
+      </package>
+
+    <package>
+        <id>b747-400</id>
+        <name>Boeing 747-400</name>
+        <dir>b744</dir>
+        <description>A popular four-engined wide-body jet</description>
+        <revision type="int">111</revision>
+        <file-size-bytes type="int">860</file-size-bytes>
+
+        <tag>boeing</tag>
+        <tag>jet</tag>
+        <tag>ifr</tag>
+
+  
+
+        <rating>
+          <FDM type="int">5</FDM>
+          <systems type="int">5</systems>
+          <model type="int">4</model>
+          <cockpit type="int">4</cockpit>
+        </rating>
+
+        <md5>4d3f7417d74f811aa20ccc4f35673d20</md5>
+        <!-- this URL will sometimes fail, on purpose -->
+        <url>http://localhost:2000/catalogTest1/b747.tar.gz</url>
+      </package>
+
+</PropertyList>

simgear/scene/tsync/terrasync.cxx

@@ -598,8 +598,32 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
             }
         } // of creating directory step
 
-        slot.repository.reset(new HTTPRepository(path, &_http));
-        slot.repository->setBaseUrl(_httpServer + "/" + slot.currentItem._dir);
+        // optimise initial Airport download
+        if (slot.isNewDirectory &&
+            (slot.currentItem._type == SyncItem::AirportData)) {
+          SG_LOG(SG_TERRASYNC, SG_INFO, "doing Airports download via tarball");
+
+          // we want to sync the 'root' TerraSync dir, but not all of it, just
+          // the Airports_archive.tar.gz file so we use our TerraSync local root
+          // as the path (since the archive will add Airports/)
+          slot.repository.reset(new HTTPRepository(_local_dir, &_http));
+          slot.repository->setBaseUrl(_httpServer + "/");
+
+          // filter callback to *only* sync the Airport_archive tarball,
+          // and ensure no other contents are touched
+          auto f = [](const HTTPRepository::SyncItem &item) {
+            if (!item.directory.empty())
+              return false;
+            return (item.filename.find("Airports_archive.") == 0);
+          };
+
+          slot.repository->setFilter(f);
+
+        } else {
+          slot.repository.reset(new HTTPRepository(path, &_http));
+          slot.repository->setBaseUrl(_httpServer + "/" +
+                                      slot.currentItem._dir);
+        }
 
         if (_installRoot.exists()) {
             SGPath p = _installRoot;
@@ -720,6 +744,7 @@ void SGTerraSync::WorkerThread::fail(SyncItem failedItem)
     _state._fail_count++;
     failedItem._status = SyncItem::Failed;
     _freshTiles.push_back(failedItem);
+    // not we also end up here for partial syncs
     SG_LOG(SG_TERRASYNC,SG_INFO,
            "Failed to sync'" << failedItem._dir << "'");
     _completedTiles[ failedItem._dir ] = now + UpdateInterval::FailedAttempt;
@@ -1090,21 +1115,10 @@ bool SGTerraSync::isIdle() {return _workerThread->isIdle();}
 
 void SGTerraSync::syncAirportsModels()
 {
-    static const char* bounds = "MZAJKL"; // airport sync order: K-L, A-J, M-Z
-    // note "request" method uses LIFO order, i.e. processes most recent request first
-    for( unsigned i = 0; i < strlen(bounds)/2; i++ )
-    {
-        for ( char synced_other = bounds[2*i]; synced_other <= bounds[2*i+1]; synced_other++ )
-        {
-            ostringstream dir;
-            dir << "Airports/" << synced_other;
-            SyncItem w(dir.str(), SyncItem::AirportData);
-            _workerThread->request( w );
-        }
-    }
-
-    SyncItem w("Models", SyncItem::SharedModels);
-    _workerThread->request( w );
+  SyncItem w("Airports", SyncItem::AirportData);
+  SyncItem a("Models", SyncItem::SharedModels);
+  _workerThread->request(w);
+  _workerThread->request(a);
 }
 
 string_list SGTerraSync::getSceneryPathSuffixes() const

simgear/sound/readwav.cxx

@@ -380,7 +380,8 @@ namespace simgear
 ALvoid* loadWAVFromFile(const SGPath& path, unsigned int& format, ALsizei& size, ALfloat& freqf, unsigned int& block_align)
 {
   if (!path.exists()) {
-    throw sg_io_exception("loadWAVFromFile: file not found", path);
+    SG_LOG(SG_IO, SG_DEV_ALERT, "loadWAVFromFile: file not found:" << path);
+    return nullptr;
   }
 
   Buffer b;
@@ -395,13 +396,15 @@ ALvoid* loadWAVFromFile(const SGPath& path, unsigned int& format, ALsizei& size,
   fd = gzopen(ps.c_str(), "rb");
 #endif
   if (!fd) {
-    throw sg_io_exception("loadWAVFromFile: unable to open file", path);
+    SG_LOG(SG_IO, SG_DEV_ALERT, "loadWAVFromFile: unable to open file:" << path);
+    return nullptr;
   }
 
   try {
       loadWavFile(fd, &b);
   } catch (sg_exception& e) {
-      throw sg_io_exception(e.getFormattedMessage() + "\nfor: " + path.str());
+      SG_LOG(SG_IO, SG_DEV_ALERT, "loadWAVFromFile:" << e.getFormattedMessage() << "\nfor: " << path);
+      return nullptr;
   }
 
   ALvoid* data = b.data;

simgear/sound/soundmgr_openal.cxx

@@ -813,13 +813,14 @@ bool SGSoundMgr::load( const std::string &samplepath,
 
     auto data = simgear::loadWAVFromFile(samplepath, format, size, freqf, blocksz);
     freq = (ALsizei)freqf;
-    if (data == nullptr) {
-        throw sg_io_exception("Failed to load wav file", sg_location(samplepath));
+    if (!data) {
+        return false;
     }
 
     if (format == AL_FORMAT_STEREO8 || format == AL_FORMAT_STEREO16) {
          free(data);
-        throw sg_io_exception("Warning: STEREO files are not supported for 3D audio effects: " + samplepath);
+        SG_LOG(SG_IO, SG_DEV_ALERT, "Warning: STEREO files are not supported for 3D audio effects: " << samplepath);
+        return false;
     }
 
     *dbuf = (void *)data;

simgear/sound/xmlsound.cxx

@@ -84,7 +84,7 @@ SGXmlSound::~SGXmlSound()
     _pitch.clear();
 }
 
-void
+bool
 SGXmlSound::init( SGPropertyNode *root,
                   SGPropertyNode *node,
                   SGSampleGroup *sgrp,
@@ -315,8 +315,8 @@ SGXmlSound::init( SGPropertyNode *root,
    string soundFileStr = node->getStringValue("path", "");
    _sample = new SGSoundSample(soundFileStr.c_str(), path);
    if (!_sample->file_path().exists()) {
-      throw sg_io_exception("XML sound: couldn't find file: '" + soundFileStr + "'");
-      return;
+      SG_LOG(SG_SOUND, SG_WARN, "XML sound: couldn't find file: '" + soundFileStr + "'");
+      return false;
    }
 
    _sample->set_relative_position( offset_pos );
@@ -328,6 +328,8 @@ SGXmlSound::init( SGPropertyNode *root,
    _sample->set_volume( v );
    _sample->set_pitch( p );
    _sgrp->add( _sample, _name );
+
+   return true;
 }
 
 void

simgear/sound/xmlsound.hxx

@@ -107,7 +107,7 @@ public:
    * @param avionics    A pointer to the pre-initialized avionics sample group.
    * @param path        The path where the audio files remain.
    */
-  virtual void init( SGPropertyNode *root,
+  virtual bool init( SGPropertyNode *root,
                      SGPropertyNode *child,
                      SGSampleGroup *sgrp,
                      SGSampleGroup *avionics,