new version: 2020.3.6

Tolerate blank lines in buildings lists
Animations: Fix spin for Compositor lights
2021-01-20 13:01:25 +00:00 · 2021-01-20 11:35:33 +00:00 · 2021-01-15 15:45:26 +00:00 · 2021-01-05 12:58:46 +00:00 · 2021-01-02 21:57:54 -06:00 · 2021-01-01 22:41:47 +00:00
68 changed files with 2535 additions and 902 deletions
@@ -278,7 +278,14 @@ else()
    endif()
 endif(SIMGEAR_HEADLESS)

-find_package(ZLIB 1.2.4 REQUIRED)
+if(${CMAKE_SYSTEM_NAME} MATCHES "OpenBSD")
+    # As of 2020-08-01, OpenBSD's system zlib is slightly old, but it's usable
+    # with a workaround in simgear/io/iostreams/gzfstream.cxx.
+    find_package(ZLIB 1.2.3 REQUIRED)
+else()
+    find_package(ZLIB 1.2.4 REQUIRED)
+endif()
+
 find_package(CURL REQUIRED)

 if (SYSTEM_EXPAT)
@@ -1 +1 @@
-2020.2.1
+2020.3.6
@@ -167,7 +167,8 @@ target_link_libraries(SimGearCore PRIVATE
    ${CMAKE_THREAD_LIBS_INIT}
    ${COCOA_LIBRARY}
    ${CURL_LIBRARIES}
-    ${WINSOCK_LIBRARY})
+    ${WINSOCK_LIBRARY}
+    ${SHLWAPI_LIBRARY})

 if(SYSTEM_EXPAT)
    target_link_libraries(SimGearCore PRIVATE ${EXPAT_LIBRARIES})
@@ -691,6 +691,10 @@ namespace canvas
      {
        _sampling_dirty = true;
      }
+      else if( name == "anisotropy" )
+      {
+        _texture.setMaxAnisotropy( node->getFloatValue() );
+      }
      else if( name == "additive-blend" )
      {
        _texture.useAdditiveBlend( node->getBoolValue() );
@@ -202,6 +202,12 @@ namespace canvas
    updateSampling();
  }

+  //----------------------------------------------------------------------------
+  void ODGauge::setMaxAnisotropy(float anis)
+  {
+    texture->setMaxAnisotropy(anis);
+  }
+
  //----------------------------------------------------------------------------
  void ODGauge::setRender(bool render)
  {
@@ -109,6 +109,8 @@ namespace canvas
                        int coverage_samples = 0,
                        int color_samples = 0 );

+      void setMaxAnisotropy(float anis);
+
      /**
       * Enable/Disable updating the texture (If disabled the contents of the
       * texture remains with the outcome of the last rendering pass)
@@ -218,6 +218,9 @@ const float SG_RADIANS_TO_DEGREES = 180.0f / SG_PI;
 #define SG_OBJECT_RANGE_ROUGH     9000.0
 #define SG_OBJECT_RANGE_DETAILED  1500.0

+/** Minimum expiry time of PagedLOD within the Tile.  Overridden by /sim/rendering/plod-minimum-expiry-time-secs **/
+#define SG_TILE_MIN_EXPIRY 180.0
+
 /** Radius of scenery tiles in m **/
 #define SG_TILE_RADIUS      14000.0

@@ -3,8 +3,12 @@ include (SimGearComponent)

 set(HEADERS debug_types.h 
    logstream.hxx BufferedLogCallback.hxx OsgIoCapture.hxx
-    LogCallback.hxx LogEntry.hxx)
+    LogCallback.hxx LogEntry.hxx
+    ErrorReportingCallback.hxx)
+
 set(SOURCES logstream.cxx BufferedLogCallback.cxx
-    LogCallback.cxx LogEntry.cxx)
+    LogCallback.cxx LogEntry.cxx
+    ErrorReportingCallback.cxx
+    )

 simgear_component(debug debug "${SOURCES}" "${HEADERS}")
@@ -0,0 +1,47 @@
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Library General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Library General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//
+
+#include <simgear_config.h>
+
+#include "ErrorReportingCallback.hxx"
+
+using std::string;
+
+namespace simgear {
+
+static ErrorReportCallback static_callback;
+
+void setErrorReportCallback(ErrorReportCallback cb)
+{
+    static_callback = cb;
+}
+
+
+void reportError(const std::string& msg, const std::string& more)
+{
+    if (!static_callback)
+        return;
+
+    static_callback(msg, more, false);
+}
+
+void reportFatalError(const std::string& msg, const std::string& more)
+{
+    if (!static_callback)
+        return;
+    static_callback(msg, more, true);
+}
+
+} // namespace simgear
@@ -0,0 +1,31 @@
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Library General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Library General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//
+
+#pragma once
+
+#include <functional>
+#include <string>
+
+namespace simgear {
+
+void reportError(const std::string& msg, const std::string& more = {});
+
+void reportFatalError(const std::string& msg, const std::string& more = {});
+
+using ErrorReportCallback = std::function<void(const std::string& msg, const std::string& more, bool isFatal)>;
+
+void setErrorReportCallback(ErrorReportCallback cb);
+
+} // namespace simgear
@@ -45,14 +45,6 @@ namespace simgear
            std::atomic<int> receiveDepth;
            std::atomic<int> sentMessageCount;

-            void UnlockList()
-            {
-                _lock.unlock();
-            }
-            void LockList()
-            {
-                _lock.lock();
-            }
        public:
            Transmitter() : receiveDepth(0), sentMessageCount(0)
            {
@@ -69,20 +61,17 @@ namespace simgear
            // most recently registered recipients should process the messages/events first.
            virtual void Register(IReceiver& r)
            {
-                LockList();
+                std::lock_guard<std::mutex> scopeLock(_lock);
                recipient_list.push_back(&r);
                r.OnRegisteredAtTransmitter(this);
                if (std::find(deleted_recipients.begin(), deleted_recipients.end(), &r) != deleted_recipients.end())
                    deleted_recipients.remove(&r);
-
-                UnlockList();
            }

            //  Removes an object from receving message from this transmitter
            virtual void DeRegister(IReceiver& R)
            {
-                LockList();
-                //printf("Remove %x\n", &R);
+                std::lock_guard<std::mutex> scopeLock(_lock);
                if (recipient_list.size())
                {
                    if (std::find(recipient_list.begin(), recipient_list.end(), &R) != recipient_list.end())
@@ -93,7 +82,6 @@ namespace simgear
                            deleted_recipients.push_back(&R);
                    }
                }
-                UnlockList();
            }

            // Notify all registered recipients. Stop when receipt status of abort or finished are received.
@@ -107,69 +95,68 @@ namespace simgear
                ReceiptStatus return_status = ReceiptStatusNotProcessed;

                sentMessageCount++;
-                try
+
+                std::vector<IReceiver*> temp;
                {
-                    LockList();
+                    std::lock_guard<std::mutex> scopeLock(_lock);
                    if (receiveDepth == 0)
                        deleted_recipients.clear();
                    receiveDepth++;
-                    std::vector<IReceiver*> temp(recipient_list.size());
+
                    int idx = 0;
                    for (RecipientList::iterator i = recipient_list.begin(); i != recipient_list.end(); i++)
                    {
-                        temp[idx++] = *i;
+                        temp.push_back(*i);
                    }
-                    UnlockList();
-                    int tempSize = temp.size();
-                    for (int index = 0; index < tempSize; index++)
+                }
+                int tempSize = temp.size();
+                for (int index = 0; index < tempSize; index++)
+                {
+                    IReceiver* R = temp[index];
                    {
-                        IReceiver* R = temp[index];
-                        LockList();
+                        std::lock_guard<std::mutex> scopeLock(_lock);
                        if (deleted_recipients.size())
                        {
                            if (std::find(deleted_recipients.begin(), deleted_recipients.end(), R) != deleted_recipients.end())
                            {
-                                UnlockList();
                                continue;
                            }
                        }
-                        UnlockList();
-                        if (R)
-                        {
-                            ReceiptStatus rstat = R->Receive(M);
-                            switch (rstat)
-                            {
-                            case ReceiptStatusFail:
-                                return_status = ReceiptStatusFail;
-                                break;
-                            case ReceiptStatusPending:
-                                return_status = ReceiptStatusPending;
-                                break;
-                            case ReceiptStatusPendingFinished:
-                                return rstat;
-
-                            case ReceiptStatusNotProcessed:
-                                break;
-                            case ReceiptStatusOK:
-                                if (return_status == ReceiptStatusNotProcessed)
-                                    return_status = rstat;
-                                break;
-
-                            case ReceiptStatusAbort:
-                                return ReceiptStatusAbort;
-
-                            case ReceiptStatusFinished:
-                                return ReceiptStatusOK;
-                            }
-                        }
-
                    }
+                    if (R)
+                    {
+                        ReceiptStatus rstat = R->Receive(M);
+                        switch (rstat)
+                        {
+                        case ReceiptStatusFail:
+                            return_status = ReceiptStatusFail;
+                            break;
+
+                        case ReceiptStatusPending:
+                            return_status = ReceiptStatusPending;
+                            break;
+
+                        case ReceiptStatusPendingFinished:
+                            return rstat;
+
+                        case ReceiptStatusNotProcessed:
+                            break;
+
+                        case ReceiptStatusOK:
+                            if (return_status == ReceiptStatusNotProcessed)
+                                return_status = rstat;
+                            break;
+
+                        case ReceiptStatusAbort:
+                            return ReceiptStatusAbort;
+
+                        case ReceiptStatusFinished:
+                            return ReceiptStatusOK;
+                        }
+                    }
+
                }
-                catch (...)
-                {
-                    throw;
-                    // return_status = ReceiptStatusAbort;
-                }
+
                receiveDepth--;
                return return_status;
            }
@@ -177,9 +164,8 @@ namespace simgear
            // number of currently registered recipients
            virtual int Count()
            {
-                LockList();
+                std::lock_guard<std::mutex> scopeLock(_lock);
                return recipient_list.size();
-                UnlockList();
            }

            // number of sent messages.
@@ -46,6 +46,7 @@
 #include <sstream>

 #include <simgear/debug/logstream.hxx>
+#include <simgear/math/sg_random.h>
 #include <simgear/structure/exception.hxx>

 #include "metar.hxx"
@@ -631,30 +632,41 @@ bool SGMetar::scanWind()
 	int dir;
 	if (!strncmp(m, "VRB", 3))
 		m += 3, dir = -1;
+	else if (!strncmp(m, "///", 3))	// direction not measurable
+		m += 3, dir = -1;
 	else if (!scanNumber(&m, &dir, 3))
 		return false;

 	int i;
-	if (!scanNumber(&m, &i, 2, 3))
+	if (!strncmp(m, "//", 2))	// speed not measurable
+		m += 2, i = -1;
+	else if (!scanNumber(&m, &i, 2, 3))
 		return false;
 	double speed = i;

 	double gust = NaN;
 	if (*m == 'G') {
 		m++;
-		if (!scanNumber(&m, &i, 2, 3))
+		if (!strncmp(m, "//", 2))	// speed not measurable
+			m += 2, i = -1;
+		else if (!scanNumber(&m, &i, 2, 3))
 			return false;
-		gust = i;
+
+		if (i != -1)
+			gust = i;
 	}
+
 	double factor;
 	if (!strncmp(m, "KT", 2))
 		m += 2, factor = SG_KT_TO_MPS;
-	else if (!strncmp(m, "KMH", 3))
+	else if (!strncmp(m, "KMH", 3))		// invalid Km/h
 		m += 3, factor = SG_KMH_TO_MPS;
-	else if (!strncmp(m, "KPH", 3))		// ??
+	else if (!strncmp(m, "KPH", 3))		// invalid Km/h
 		m += 3, factor = SG_KMH_TO_MPS;
 	else if (!strncmp(m, "MPS", 3))
 		m += 3, factor = 1.0;
+	else if (!strncmp(m, " ", 1))		// default to Knots
+		factor = SG_KT_TO_MPS;
 	else
 		return false;
 	if (!scanBoundary(&m))
@@ -674,18 +686,28 @@ bool SGMetar::scanVariability()
 {
 	char *m = _m;
 	int from, to;
-	if (!scanNumber(&m, &from, 3))
+
+	if (!strncmp(m, "///", 3))	// direction not measurable
+		m += 3, from = -1;
+	else if (!scanNumber(&m, &from, 3))
 		return false;
+
 	if (*m++ != 'V')
 		return false;
-	if (!scanNumber(&m, &to, 3))
+
+	if (!strncmp(m, "///", 3))	// direction not measurable
+		m += 3, to = -1;
+	else if (!scanNumber(&m, &to, 3))
 		return false;
+
 	if (!scanBoundary(&m))
 		return false;
+
 	_m = m;
 	_wind_range_from = from;
 	_wind_range_to = to;
 	_grpcount++;
+
 	return true;
 }

@@ -1047,9 +1069,12 @@ bool SGMetar::scanSkyCondition()
 		return false;
 	m += i;

-	if (!strncmp(m, "///", 3))	// vis not measurable (e.g. because of heavy snowing)
+	if (!strncmp(m, "///", 3))	{ // vis not measurable (e.g. because of heavy snowing)
 		m += 3, i = -1;
-	else if (scanBoundary(&m)) {
+		sg_srandom_time();
+		// randomize the base height to avoid the black sky issue
+		i = 50 + static_cast<int>(sg_random() * 250.0);		// range [5,000, 30,000]
+	} else if (scanBoundary(&m)) {
 		_m = m;
 		return true;				// ignore single OVC/BKN/...
 	} else if (!scanNumber(&m, &i, 3))
@@ -76,12 +76,38 @@ void test_sensor_failure_cloud()
    SG_CHECK_EQUAL_EP2(m1.getPressure_hPa(), 1025, TEST_EPSILON);
 }

+void test_sensor_failure_wind()
+{
+    SGMetar m1("2020/10/23 16:55 LIVD 231655Z /////KT 9999 OVC025 10/08 Q1020 RMK OVC VIS MIN 9999 BLU");
+    SG_CHECK_EQUAL(m1.getWindDir(), -1);
+    SG_CHECK_EQUAL_EP2(m1.getWindSpeed_kt(), -1, TEST_EPSILON);
+
+    SGMetar m2("2020/10/21 16:55 LIVD 211655Z /////KT CAVOK 07/03 Q1023 RMK SKC VIS MIN 9999 BLU");
+    SG_CHECK_EQUAL(m2.getWindDir(), -1);
+    SG_CHECK_EQUAL_EP2(m2.getWindSpeed_kt(), -1, TEST_EPSILON);
+
+    SGMetar m3("2020/11/17 16:00 CYAZ 171600Z 14040G//KT 10SM -RA OVC012 12/11 A2895 RMK NS8 VIA CYXY SLP806 DENSITY ALT 900FT");
+    SG_CHECK_EQUAL(m3.getWindDir(), 140);
+    SG_CHECK_EQUAL_EP2(m3.getWindSpeed_kt(), 40, TEST_EPSILON);
+    SG_CHECK_EQUAL_EP2(m3.getGustSpeed_kt(), SGMetarNaN, TEST_EPSILON);
+}
+
+void test_wind_unit_not_specified()
+{
+    SGMetar m1("2020/10/23 11:58 KLSV 231158Z 05010G14 10SM CLR 16/M04 A2992 RMK SLPNO WND DATA ESTMD ALSTG/SLP ESTMD 10320 20124 5//// $");
+    SG_CHECK_EQUAL(m1.getWindDir(), 50);
+    SG_CHECK_EQUAL_EP2(m1.getWindSpeed_kt(), 10.0, TEST_EPSILON);
+    SG_CHECK_EQUAL_EP2(m1.getGustSpeed_kt(), 14.0, TEST_EPSILON);
+}
+
 int main(int argc, char* argv[])
 {
    try {
        test_basic();
        test_sensor_failure_weather();
        test_sensor_failure_cloud();
+        test_sensor_failure_wind();
+        test_wind_unit_not_specified();
    } catch (sg_exception& e) {
        cerr << "got exception:" << e.getMessage() << endl;
        return -1;
@@ -35,10 +35,12 @@ set(SOURCES
    sg_socket.cxx
    sg_socket_udp.cxx
    HTTPClient.cxx
+    HTTPTestApi_private.hxx
    HTTPFileRequest.cxx
    HTTPMemoryRequest.cxx
    HTTPRequest.cxx
    HTTPRepository.cxx
+    HTTPRepository_private.hxx
    untar.cxx
    )

@@ -81,6 +83,7 @@ add_test(binobj ${EXECUTABLE_OUTPUT_PATH}/test_binobj)

 add_executable(test_repository test_repository.cxx)
 target_link_libraries(test_repository ${TEST_LIBS})
+target_compile_definitions(test_repository PUBLIC BUILDING_TESTSUITE)
 add_test(http_repository ${EXECUTABLE_OUTPUT_PATH}/test_repository)

 add_executable(test_untar test_untar.cxx)
@@ -61,6 +61,10 @@ public:

    struct dns_ctx * ctx;
    static size_t instanceCounter;
+
+    using RequestVec = std::vector<Request_ptr>;
+
+    RequestVec _activeRequests;
 };

 size_t Client::ClientPrivate::instanceCounter = 0;
@@ -78,6 +82,11 @@ Request::~Request()
 {
 }

+void Request::cancel()
+{
+    _cancelled = true;
+}
+
 bool Request::isTimeout() const
 {
    return (time(NULL) - _start) > _timeout_secs;
@@ -114,18 +123,20 @@ static void dnscbSRV(struct dns_ctx *ctx, struct dns_rr_srv *result, void *data)
 {
    SRVRequest * r = static_cast<SRVRequest*>(data);
    if (result) {
-        r->cname = result->dnssrv_cname;
-        r->qname = result->dnssrv_qname;
-        r->ttl = result->dnssrv_ttl;
-        for (int i = 0; i < result->dnssrv_nrr; i++) {
-            SRVRequest::SRV_ptr srv(new SRVRequest::SRV);
-            r->entries.push_back(srv);
-            srv->priority = result->dnssrv_srv[i].priority;
-            srv->weight = result->dnssrv_srv[i].weight;
-            srv->port = result->dnssrv_srv[i].port;
-            srv->target = result->dnssrv_srv[i].name;
+        if (!r->isCancelled()) {
+            r->cname = result->dnssrv_cname;
+            r->qname = result->dnssrv_qname;
+            r->ttl = result->dnssrv_ttl;
+            for (int i = 0; i < result->dnssrv_nrr; i++) {
+                SRVRequest::SRV_ptr srv(new SRVRequest::SRV);
+                r->entries.push_back(srv);
+                srv->priority = result->dnssrv_srv[i].priority;
+                srv->weight = result->dnssrv_srv[i].weight;
+                srv->port = result->dnssrv_srv[i].port;
+                srv->target = result->dnssrv_srv[i].name;
+            }
+            std::sort(r->entries.begin(), r->entries.end(), sortSRV);
        }
-        std::sort( r->entries.begin(), r->entries.end(), sortSRV );
        free(result);
    }
    r->setComplete();
@@ -134,11 +145,16 @@ static void dnscbSRV(struct dns_ctx *ctx, struct dns_rr_srv *result, void *data)
 void SRVRequest::submit( Client * client )
 {
    // if service is defined, pass service and protocol
-    if (!dns_submit_srv(client->d->ctx, getDn().c_str(), _service.empty() ? NULL : _service.c_str(), _service.empty() ? NULL : _protocol.c_str(), 0, dnscbSRV, this )) {
+    auto q = dns_submit_srv(client->d->ctx, getDn().c_str(), _service.empty() ? NULL : _service.c_str(),
+                            _service.empty() ? NULL : _protocol.c_str(),
+                            0, dnscbSRV, this);
+
+    if (!q) {
        SG_LOG(SG_IO, SG_ALERT, "Can't submit dns request for " << getDn());
        return;
    }
    _start = time(NULL);
+    _query = q;
 }

 TXTRequest::TXTRequest( const std::string & dn ) :
@@ -151,17 +167,24 @@ static void dnscbTXT(struct dns_ctx *ctx, struct dns_rr_txt *result, void *data)
 {
    TXTRequest * r = static_cast<TXTRequest*>(data);
    if (result) {
-        r->cname = result->dnstxt_cname;
-        r->qname = result->dnstxt_qname;
-        r->ttl = result->dnstxt_ttl;
-        for (int i = 0; i < result->dnstxt_nrr; i++) {
-          //TODO: interprete the .len field of dnstxt_txt?
-          string txt = string((char*)result->dnstxt_txt[i].txt);
-          r->entries.push_back( txt );
-          string_list tokens = simgear::strutils::split( txt, "=", 1 );
-          if( tokens.size() == 2 ) {
-            r->attributes[tokens[0]] = tokens[1];
-          }
+        if (!r->isCancelled()) {
+            r->cname = result->dnstxt_cname;
+            r->qname = result->dnstxt_qname;
+            r->ttl = result->dnstxt_ttl;
+            for (int i = 0; i < result->dnstxt_nrr; i++) {
+                //TODO: interprete the .len field of dnstxt_txt?
+                auto rawTxt = reinterpret_cast<char*>(result->dnstxt_txt[i].txt);
+                if (!rawTxt) {
+                    continue;
+                }
+
+                const string txt{rawTxt};
+                r->entries.push_back(txt);
+                string_list tokens = simgear::strutils::split(txt, "=", 1);
+                if (tokens.size() == 2) {
+                    r->attributes[tokens[0]] = tokens[1];
+                }
+            }
        }
        free(result);
    }
@@ -171,11 +194,13 @@ static void dnscbTXT(struct dns_ctx *ctx, struct dns_rr_txt *result, void *data)
 void TXTRequest::submit( Client * client )
 {
    // protocol and service an already encoded in DN so pass in NULL for both
-    if (!dns_submit_txt(client->d->ctx, getDn().c_str(), DNS_C_IN, 0, dnscbTXT, this )) {
+    auto q = dns_submit_txt(client->d->ctx, getDn().c_str(), DNS_C_IN, 0, dnscbTXT, this);
+    if (!q) {
        SG_LOG(SG_IO, SG_ALERT, "Can't submit dns request for " << getDn());
        return;
    }
    _start = time(NULL);
+    _query = q;
 }


@@ -190,27 +215,29 @@ static void dnscbNAPTR(struct dns_ctx *ctx, struct dns_rr_naptr *result, void *d
 {
    NAPTRRequest * r = static_cast<NAPTRRequest*>(data);
    if (result) {
-        r->cname = result->dnsnaptr_cname;
-        r->qname = result->dnsnaptr_qname;
-        r->ttl = result->dnsnaptr_ttl;
-        for (int i = 0; i < result->dnsnaptr_nrr; i++) {
-            if( !r->qservice.empty() && r->qservice != result->dnsnaptr_naptr[i].service )
-                continue;
+        if (!r->isCancelled()) {
+            r->cname = result->dnsnaptr_cname;
+            r->qname = result->dnsnaptr_qname;
+            r->ttl = result->dnsnaptr_ttl;
+            for (int i = 0; i < result->dnsnaptr_nrr; i++) {
+                if (!r->qservice.empty() && r->qservice != result->dnsnaptr_naptr[i].service)
+                    continue;

-            //TODO: case ignore and result flags may have more than one flag
-            if( !r->qflags.empty() && r->qflags != result->dnsnaptr_naptr[i].flags )
-                continue;
+                //TODO: case ignore and result flags may have more than one flag
+                if (!r->qflags.empty() && r->qflags != result->dnsnaptr_naptr[i].flags)
+                    continue;

-            NAPTRRequest::NAPTR_ptr naptr(new NAPTRRequest::NAPTR);
-            r->entries.push_back(naptr);
-            naptr->order = result->dnsnaptr_naptr[i].order;
-            naptr->preference = result->dnsnaptr_naptr[i].preference;
-            naptr->flags = result->dnsnaptr_naptr[i].flags;
-            naptr->service = result->dnsnaptr_naptr[i].service;
-            naptr->regexp = result->dnsnaptr_naptr[i].regexp;
-            naptr->replacement = result->dnsnaptr_naptr[i].replacement;
+                NAPTRRequest::NAPTR_ptr naptr(new NAPTRRequest::NAPTR);
+                r->entries.push_back(naptr);
+                naptr->order = result->dnsnaptr_naptr[i].order;
+                naptr->preference = result->dnsnaptr_naptr[i].preference;
+                naptr->flags = result->dnsnaptr_naptr[i].flags;
+                naptr->service = result->dnsnaptr_naptr[i].service;
+                naptr->regexp = result->dnsnaptr_naptr[i].regexp;
+                naptr->replacement = result->dnsnaptr_naptr[i].replacement;
+            }
+            std::sort(r->entries.begin(), r->entries.end(), sortNAPTR);
        }
-        std::sort( r->entries.begin(), r->entries.end(), sortNAPTR );
        free(result);
    }
    r->setComplete();
@@ -218,11 +245,13 @@ static void dnscbNAPTR(struct dns_ctx *ctx, struct dns_rr_naptr *result, void *d

 void NAPTRRequest::submit( Client * client )
 {
-    if (!dns_submit_naptr(client->d->ctx, getDn().c_str(), 0, dnscbNAPTR, this )) {
+    auto q = dns_submit_naptr(client->d->ctx, getDn().c_str(), 0, dnscbNAPTR, this);
+    if (!q) {
        SG_LOG(SG_IO, SG_ALERT, "Can't submit dns request for " << getDn());
        return;
    }
    _start = time(NULL);
+    _query = q;
 }


@@ -237,6 +266,7 @@ Client::Client() :

 void Client::makeRequest(const Request_ptr& r)
 {
+    d->_activeRequests.push_back(r);
    r->submit(this);
 }

@@ -247,6 +277,19 @@ void Client::update(int waitTimeout)
        return;

    dns_ioevent(d->ctx, now);
+
+    // drop our owning ref to completed requests,
+    // and cancel any which timed out
+    auto it = std::remove_if(d->_activeRequests.begin(), d->_activeRequests.end(),
+                             [this](const Request_ptr& r) {
+                                 if (r->isTimeout()) {
+                                     dns_cancel(d->ctx, reinterpret_cast<struct dns_query*>(r->_query));
+                                     return true;
+                                 }
+
+                                 return r->isComplete();
+                             });
+    d->_activeRequests.erase(it, d->_activeRequests.end());
 }

 } // of namespace DNS
@@ -40,28 +40,38 @@ namespace DNS
 {

 class Client;
+
+using UDNSQueryPtr = void*;
+
 class Request : public SGReferenced
 {
 public:
    Request( const std::string & dn );
    virtual ~Request();
-    std::string getDn() const { return _dn; }
+    const std::string& getDn() const { return _dn; }
    int getType() const { return _type; }
    bool isComplete() const { return _complete; }
    bool isTimeout() const;
    void setComplete( bool b = true ) { _complete = b; }
+    bool isCancelled() const { return _cancelled; }

    virtual void submit( Client * client) = 0;

+    void cancel();
+
    std::string cname;
    std::string qname;
    unsigned ttl;
 protected:
+    friend class Client;
+
+    UDNSQueryPtr _query = nullptr;
    std::string _dn;
    int _type;
    bool _complete;
    time_t _timeout_secs;
    time_t _start;
+    bool _cancelled = false;
 };
 typedef SGSharedPtr<Request> Request_ptr;

@@ -69,7 +79,7 @@ class NAPTRRequest : public Request
 {
 public:
    NAPTRRequest( const std::string & dn );
-    virtual void submit( Client * client );
+    void submit(Client* client) override;

    struct NAPTR : SGReferenced {
        int order;
@@ -92,7 +102,7 @@ class SRVRequest : public Request
 public:
    SRVRequest( const std::string & dn );
    SRVRequest( const std::string & dn, const string & service, const string & protocol );
-    virtual void submit( Client * client );
+    void submit(Client* client) override;

    struct SRV : SGReferenced {
      int priority;
@@ -112,7 +122,7 @@ class TXTRequest : public Request
 {
 public:
    TXTRequest( const std::string & dn );
-    virtual void submit( Client * client );
+    void submit(Client* client) override;

    typedef std::vector<string> TXT_list;
    typedef std::map<std::string,std::string> TXT_Attribute_map;
@@ -37,7 +37,6 @@

 #include <simgear/simgear_config.h>

-#include <curl/multi.h>

 #include <simgear/io/sg_netChat.hxx>

@@ -47,6 +46,9 @@
 #include <simgear/timing/timestamp.hxx>
 #include <simgear/structure/exception.hxx>

+#include "HTTPClient_private.hxx"
+#include "HTTPTestApi_private.hxx"
+
 #if defined( HAVE_VERSION_H ) && HAVE_VERSION_H
 #include "version.h"
 #else
@@ -64,50 +66,20 @@ namespace HTTP
 extern const int DEFAULT_HTTP_PORT = 80;
 const char* CONTENT_TYPE_URL_ENCODED = "application/x-www-form-urlencoded";

-class Connection;
-typedef std::multimap<std::string, Connection*> ConnectionDict;
-typedef std::list<Request_ptr> RequestList;
-
-class Client::ClientPrivate
-{
-public:
-    CURLM* curlMulti;
-
-    void createCurlMulti()
-    {
-        curlMulti = curl_multi_init();
-        // see https://curl.haxx.se/libcurl/c/CURLMOPT_PIPELINING.html
-        // we request HTTP 1.1 pipelining
-        curl_multi_setopt(curlMulti, CURLMOPT_PIPELINING, 1 /* aka CURLPIPE_HTTP1 */);
+void Client::ClientPrivate::createCurlMulti() {
+  curlMulti = curl_multi_init();
+  // see https://curl.haxx.se/libcurl/c/CURLMOPT_PIPELINING.html
+  // we request HTTP 1.1 pipelining
+  curl_multi_setopt(curlMulti, CURLMOPT_PIPELINING, 1 /* aka CURLPIPE_HTTP1 */);
 #if (LIBCURL_VERSION_MINOR >= 30)
-        curl_multi_setopt(curlMulti, CURLMOPT_MAX_TOTAL_CONNECTIONS, (long) maxConnections);
-        curl_multi_setopt(curlMulti, CURLMOPT_MAX_PIPELINE_LENGTH,
-                          (long) maxPipelineDepth);
-        curl_multi_setopt(curlMulti, CURLMOPT_MAX_HOST_CONNECTIONS,
-                          (long) maxHostConnections);
+  curl_multi_setopt(curlMulti, CURLMOPT_MAX_TOTAL_CONNECTIONS,
+                    (long)maxConnections);
+  curl_multi_setopt(curlMulti, CURLMOPT_MAX_PIPELINE_LENGTH,
+                    (long)maxPipelineDepth);
+  curl_multi_setopt(curlMulti, CURLMOPT_MAX_HOST_CONNECTIONS,
+                    (long)maxHostConnections);
 #endif
-    }
-
-    typedef std::map<Request_ptr, CURL*> RequestCurlMap;
-    RequestCurlMap requests;
-
-    std::string userAgent;
-    std::string proxy;
-    int proxyPort;
-    std::string proxyAuth;
-    unsigned int maxConnections;
-    unsigned int maxHostConnections;
-    unsigned int maxPipelineDepth;
-
-    RequestList pendingRequests;
-
-    SGTimeStamp timeTransferSample;
-    unsigned int bytesTransferred;
-    unsigned int lastTransferRate;
-    uint64_t totalBytesDownloaded;
-
-    SGPath tlsCertificatePath;
-};
+}

 Client::Client() :
    d(new ClientPrivate)
@@ -223,12 +195,23 @@ void Client::update(int waitTimeout)
          assert(it->second == e);
          d->requests.erase(it);

-        if (msg->data.result == 0) {
-          req->responseComplete();
-        } else {
-          SG_LOG(SG_IO, SG_WARN, "CURL Result:" << msg->data.result << " " << curl_easy_strerror(msg->data.result));
-          req->setFailure(msg->data.result, curl_easy_strerror(msg->data.result));
-        }
+          bool doProcess = true;
+          if (d->testsuiteResponseDoneCallback) {
+            doProcess =
+                !d->testsuiteResponseDoneCallback(msg->data.result, req);
+          }
+
+          if (doProcess) {
+            if (msg->data.result == 0) {
+              req->responseComplete();
+            } else {
+              SG_LOG(SG_IO, SG_WARN,
+                     "CURL Result:" << msg->data.result << " "
+                                    << curl_easy_strerror(msg->data.result));
+              req->setFailure(msg->data.result,
+                              curl_easy_strerror(msg->data.result));
+            }
+          }

        curl_multi_remove_handle(d->curlMulti, e);
        curl_easy_cleanup(e);
@@ -559,6 +542,17 @@ void Client::clearAllConnections()
    d->createCurlMulti();
 }

+/////////////////////////////////////////////////////////////////////
+
+void TestApi::setResponseDoneCallback(Client *cl, ResponseDoneCallback cb) {
+  cl->d->testsuiteResponseDoneCallback = cb;
+}
+
+void TestApi::markRequestAsFailed(Request_ptr req, int curlCode,
+                                  const std::string &message) {
+  req->setFailure(curlCode, message);
+}
+
 } // of namespace HTTP

 } // of namespace simgear
@@ -24,7 +24,8 @@
 #ifndef SG_HTTP_CLIENT_HXX
 #define SG_HTTP_CLIENT_HXX

-#include <memory> // for std::unique_ptr
+#include <functional>
+#include <memory>   // for std::unique_ptr
 #include <stdint.h> // for uint_64t

 #include <simgear/io/HTTPFileRequest.hxx>
@@ -125,6 +126,7 @@ private:

    friend class Connection;
    friend class Request;
+    friend class TestApi;

    class ClientPrivate;
    std::unique_ptr<ClientPrivate> d;
@@ -0,0 +1,68 @@
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Library General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301, USA
+
+#pragma once
+
+#include <list>
+#include <map>
+
+#include "HTTPClient.hxx"
+#include "HTTPRequest.hxx"
+
+#include <simgear/timing/timestamp.hxx>
+
+#include <curl/multi.h>
+
+namespace simgear {
+namespace HTTP {
+
+typedef std::list<Request_ptr> RequestList;
+
+using ResponseDoneCallback =
+    std::function<bool(int curlResult, Request_ptr req)>;
+
+class Client::ClientPrivate {
+public:
+  CURLM *curlMulti;
+
+  void createCurlMulti();
+
+  typedef std::map<Request_ptr, CURL *> RequestCurlMap;
+  RequestCurlMap requests;
+
+  std::string userAgent;
+  std::string proxy;
+  int proxyPort;
+  std::string proxyAuth;
+  unsigned int maxConnections;
+  unsigned int maxHostConnections;
+  unsigned int maxPipelineDepth;
+
+  RequestList pendingRequests;
+
+  SGTimeStamp timeTransferSample;
+  unsigned int bytesTransferred;
+  unsigned int lastTransferRate;
+  uint64_t totalBytesDownloaded;
+
+  SGPath tlsCertificatePath;
+
+  // only used by unit-tests / test-api, but
+  // only costs us a pointe here to declare it.
+  ResponseDoneCallback testsuiteResponseDoneCallback;
+};
+
+} // namespace HTTP
+
+} // namespace simgear
@@ -20,6 +20,7 @@
 #ifndef SG_IO_HTTP_REPOSITORY_HXX
 #define SG_IO_HTTP_REPOSITORY_HXX

+#include <functional>
 #include <memory>

 #include <simgear/misc/sg_path.hxx>
@@ -32,49 +33,83 @@ class HTTPRepoPrivate;
 class HTTPRepository
 {
 public:
-    enum ResultCode {
-        REPO_NO_ERROR = 0,
-        REPO_ERROR_NOT_FOUND,
-        REPO_ERROR_SOCKET,
-        SVN_ERROR_XML,
-        SVN_ERROR_TXDELTA,
-        REPO_ERROR_IO,
-        REPO_ERROR_CHECKSUM,
-        REPO_ERROR_FILE_NOT_FOUND,
-        REPO_ERROR_HTTP,
-        REPO_ERROR_CANCELLED,
-        REPO_PARTIAL_UPDATE
+  enum ResultCode {
+    REPO_NO_ERROR = 0,
+    REPO_ERROR_NOT_FOUND,
+    REPO_ERROR_SOCKET,
+    SVN_ERROR_XML,
+    SVN_ERROR_TXDELTA,
+    REPO_ERROR_IO,
+    REPO_ERROR_CHECKSUM,
+    REPO_ERROR_FILE_NOT_FOUND,
+    REPO_ERROR_HTTP,
+    REPO_ERROR_CANCELLED,
+    REPO_PARTIAL_UPDATE ///< repository is working, but file-level failures
+                        ///< occurred
+  };
+
+  HTTPRepository(const SGPath &root, HTTP::Client *cl);
+  virtual ~HTTPRepository();
+
+  virtual SGPath fsBase() const;
+
+  virtual void setBaseUrl(const std::string &url);
+  virtual std::string baseUrl() const;
+
+  virtual HTTP::Client *http() const;
+
+  virtual void update();
+
+  virtual bool isDoingSync() const;
+
+  /**
+        @brief call this periodically to progress non-network tasks
+     */
+  void process();
+
+  virtual ResultCode failure() const;
+
+  virtual size_t bytesToDownload() const;
+
+  virtual size_t bytesDownloaded() const;
+
+  /**
+   * optionally provide the location of an installer copy of this
+   * repository. When a file is missing it will be copied from this tree.
+   */
+  void setInstalledCopyPath(const SGPath &copyPath);
+
+  static std::string resultCodeAsString(ResultCode code);
+
+  enum class SyncAction { Add, Update, Delete, UpToDate };
+
+  enum EntryType { FileType, DirectoryType, TarballType };
+
+  struct SyncItem {
+    const std::string directory; // relative path in the repository
+    const EntryType type;
+    const std::string filename;
+    const SyncAction action;
+    const SGPath pathOnDisk; // path the entry does / will have
+  };
+
+    using SyncPredicate = std::function<bool(const SyncItem &item)>;
+
+    void setFilter(SyncPredicate sp);
+
+    struct Failure {
+      SGPath path;
+      ResultCode error;
    };

-    HTTPRepository(const SGPath& root, HTTP::Client* cl);
-    virtual ~HTTPRepository();
-
-    virtual SGPath fsBase() const;
-
-    virtual void setBaseUrl(const std::string& url);
-    virtual std::string baseUrl() const;
-
-    virtual HTTP::Client* http() const;
-
-    virtual void update();
-
-    virtual bool isDoingSync() const;
-
-    virtual ResultCode failure() const;
-
-    virtual size_t bytesToDownload() const;
-
-    virtual size_t bytesDownloaded() const;
+    using FailureVec = std::vector<Failure>;

    /**
-     * optionally provide the location of an installer copy of this
-     * repository. When a file is missing it will be copied from this tree.
+     * @brief return file-level failures
     */
-    void setInstalledCopyPath(const SGPath& copyPath);
-    
-    static std::string resultCodeAsString(ResultCode code);
+    FailureVec failures() const;

-private:
+  private:
    bool isBare() const;

    std::unique_ptr<HTTPRepoPrivate> _d;
@@ -0,0 +1,126 @@
+// HTTPRepository.cxx -- plain HTTP TerraSync remote client
+//
+// Copyright (C) 20126  James Turner <zakalawe@mac.com>
+//
+// This program is free software; you can redistribute it and/or
+// modify it under the terms of the GNU General Public License as
+// published by the Free Software Foundation; either version 2 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
+// USA.
+
+#pragma once
+
+#include <deque>
+#include <functional>
+#include <memory>
+#include <string>
+#include <unordered_map>
+
+#include <simgear/io/HTTPClient.hxx>
+#include <simgear/misc/sg_path.hxx>
+
+#include "HTTPRepository.hxx"
+
+namespace simgear {
+
+class HTTPDirectory;
+using HTTPDirectory_ptr = std::unique_ptr<HTTPDirectory>;
+
+class HTTPRepoGetRequest : public HTTP::Request {
+public:
+  HTTPRepoGetRequest(HTTPDirectory *d, const std::string &u)
+      : HTTP::Request(u), _directory(d) {}
+
+  virtual void cancel();
+
+  size_t contentSize() const { return _contentSize; }
+
+  void setContentSize(size_t sz) { _contentSize = sz; }
+
+protected:
+  HTTPDirectory *_directory;
+  size_t _contentSize = 0;
+};
+
+using RepoRequestPtr = SGSharedPtr<HTTPRepoGetRequest>;
+
+class HTTPRepoPrivate {
+public:
+
+
+  HTTPRepository::FailureVec failures;
+  int maxPermittedFailures = 16;
+
+  HTTPRepoPrivate(HTTPRepository *parent)
+      : p(parent), isUpdating(false), status(HTTPRepository::REPO_NO_ERROR),
+        totalDownloaded(0) {
+    ;
+  }
+
+  ~HTTPRepoPrivate();
+
+  HTTPRepository *p; // link back to outer
+  HTTP::Client *http;
+  std::string baseUrl;
+  SGPath basePath;
+  bool isUpdating;
+  HTTPRepository::ResultCode status;
+  HTTPDirectory_ptr rootDir;
+  size_t totalDownloaded;
+  HTTPRepository::SyncPredicate syncPredicate;
+
+  HTTP::Request_ptr updateFile(HTTPDirectory *dir, const std::string &name,
+                               size_t sz);
+  HTTP::Request_ptr updateDir(HTTPDirectory *dir, const std::string &hash,
+                              size_t sz);
+
+  void failedToGetRootIndex(HTTPRepository::ResultCode st);
+  void failedToUpdateChild(const SGPath &relativePath,
+                           HTTPRepository::ResultCode fileStatus);
+
+  void updatedChildSuccessfully(const SGPath &relativePath);
+
+  void checkForComplete();
+
+  typedef std::vector<RepoRequestPtr> RequestVector;
+  RequestVector queuedRequests, activeRequests;
+
+  void makeRequest(RepoRequestPtr req);
+
+  enum class RequestFinish { Done, Retry };
+
+  void finishedRequest(const RepoRequestPtr &req, RequestFinish retryRequest);
+
+  HTTPDirectory *getOrCreateDirectory(const std::string &path);
+  bool deleteDirectory(const std::string &relPath, const SGPath &absPath);
+
+
+  typedef std::vector<HTTPDirectory_ptr> DirectoryVector;
+  DirectoryVector directories;
+
+  void scheduleUpdateOfChildren(HTTPDirectory *dir);
+
+  SGPath installedCopyPath;
+
+  int countDirtyHashCaches() const;
+  void flushHashCaches();
+
+  enum ProcessResult { ProcessContinue, ProcessDone, ProcessFailed };
+
+  using RepoProcessTask = std::function<ProcessResult(HTTPRepoPrivate *repo)>;
+
+  void addTask(RepoProcessTask task);
+
+  std::deque<RepoProcessTask> pendingTasks;
+};
+
+} // namespace simgear
@@ -56,6 +56,15 @@ Request::~Request()

 }

+void Request::prepareForRetry() {
+  setReadyState(UNSENT);
+  _willClose = false;
+  _connectionCloseHeader = false;
+  _responseStatus = 0;
+  _responseLength = 0;
+  _receivedBodyBytes = 0;
+}
+
 //------------------------------------------------------------------------------
 Request* Request::done(const Callback& cb)
 {
@@ -208,7 +208,9 @@ public:
     */
    bool serverSupportsPipelining() const;

-protected:
+    virtual void prepareForRetry();
+
+  protected:
    Request(const std::string& url, const std::string method = "GET");

    virtual void requestStart();
@@ -222,12 +224,14 @@ protected:
    virtual void onFail();
    virtual void onAlways();

-    void setFailure(int code, const std::string& reason);
    void setSuccess(int code);
-private:
+    void setFailure(int code, const std::string &reason);
+
+  private:
    friend class Client;
    friend class Connection;
    friend class ContentDecoder;
+    friend class TestApi;

    Request(const Request&); // = delete;
    Request& operator=(const Request&); // = delete;
@@ -0,0 +1,45 @@
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Library General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Library General Public License for more details.
+//
+// You should have received a copy of the GNU Library General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301, USA
+
+#pragma once
+
+#include <functional>
+
+#include "HTTPRequest.hxx"
+
+namespace simgear {
+namespace HTTP {
+
+class Client;
+
+using ResponseDoneCallback =
+    std::function<bool(int curlResult, Request_ptr req)>;
+
+/**
+ * @brief this API is for unit-testing HTTP code.
+ * Don't use it for anything else. It's for unit-testing.
+ */
+class TestApi {
+public:
+  // alow test suite to manipulate requests to simulate network errors;
+  // without this, it's hard to provoke certain failures in a loop-back
+  // network sitation.
+  static void setResponseDoneCallback(Client *cl, ResponseDoneCallback cb);
+
+  static void markRequestAsFailed(Request_ptr req, int curlCode,
+                                  const std::string &message);
+};
+
+} // namespace HTTP
+} // namespace simgear
@@ -185,6 +185,9 @@ gzfilebuf::setcompressionstrategy( int comp_strategy )

 z_off_t
 gzfilebuf::approxOffset() {
+    #ifdef __OpenBSD__
+    z_off_t res = 0;
+    #else
    z_off_t res = gzoffset(file);

    if (res == -1) {
@@ -201,7 +204,7 @@ gzfilebuf::approxOffset() {
        SG_LOG( SG_GENERAL, SG_ALERT, errMsg );
        throw sg_io_exception(errMsg);
    }
-
+    #endif
    return res;
 }

@@ -42,6 +42,7 @@

 #include <simgear/bucket/newbucket.hxx>
 #include <simgear/misc/sg_path.hxx>
+#include <simgear/misc/strutils.hxx>
 #include <simgear/math/SGGeometry.hxx>
 #include <simgear/structure/exception.hxx>

@@ -563,6 +564,14 @@ bool SGBinObject::read_bin( const SGPath& file ) {
    // read headers
    unsigned int header;
    sgReadUInt( fp, &header );
+
+    if (sgReadError()) {
+        int code = 0;
+        const char* gzErrorString = gzerror(fp, &code);
+        gzclose(fp);
+        throw sg_io_exception("Unable to read BTG header: " + string{gzErrorString} + ", code =" + std::to_string(code), sg_location(file));
+    }
+
    if ( ((header & 0xFF000000) >> 24) == 'S' &&
         ((header & 0x00FF0000) >> 16) == 'G' ) {

@@ -71,21 +71,31 @@ SGFile::~SGFile() {
 std::string SGFile::computeHash()
 {
    if (!file_name.exists())
-        return std::string();
+        return {};
+        
    simgear::sha1nfo info;
    sha1_init(&info);
-    char* buf = static_cast<char*>(malloc(1024 * 1024));
+    
+    // unique_ptr with custom deleter for exception safety
+    const int bufSize = 1024 * 1024;
+    std::unique_ptr<char, std::function<void(char*)>> buf{static_cast<char*>(malloc(bufSize)), 
+        [](char* p) { free(p); }};
+
+    if (!buf) {
+        SG_LOG(SG_IO, SG_ALERT, "Failed to malloc buffer for SHA1 check");
+    }
+
    size_t readLen;
    SGBinaryFile f(file_name);
    if (!f.open(SG_IO_IN)) {
-        throw sg_io_exception("Couldn't open file for compute hash", file_name);
+        SG_LOG(SG_IO, SG_ALERT, "SGFile::computeHash: Failed to open " << file_name);
+        return {};
    }
-    while ((readLen = f.read(buf, 1024 * 1024)) > 0) {
-        sha1_write(&info, buf, readLen);
+    while ((readLen = f.read(buf.get(), bufSize)) > 0) {
+        sha1_write(&info, buf.get(), readLen);
    }

    f.close();
-    free(buf);
    std::string hashBytes((char*)sha1_result(&info), HASH_LENGTH);
    return simgear::strutils::encodeHex(hashBytes);
 }
@@ -1,16 +1,18 @@
 #include <cassert>
 #include <cstdlib>
+#include <errno.h>
+#include <fcntl.h>
+#include <functional>
 #include <iostream>
 #include <map>
 #include <sstream>
-#include <errno.h>
-#include <fcntl.h>

 #include <simgear/simgear_config.h>

-#include "test_HTTP.hxx"
-#include "HTTPRepository.hxx"
 #include "HTTPClient.hxx"
+#include "HTTPRepository.hxx"
+#include "HTTPTestApi_private.hxx"
+#include "test_HTTP.hxx"

 #include <simgear/misc/strutils.hxx>
 #include <simgear/misc/sg_hash.hxx>
@@ -25,8 +27,14 @@

 using namespace simgear;

+using TestApi = simgear::HTTP::TestApi;
+
 std::string dataForFile(const std::string& parentName, const std::string& name, int revision)
 {
+    if (name == "zeroByteFile") {
+        return {};
+    }
+
    std::ostringstream os;
    // random content but which definitely depends on our tree location
    // and revision.
@@ -45,6 +53,9 @@ std::string hashForData(const std::string& d)
    return strutils::encodeHex(sha1_result(&info), HASH_LENGTH);
 }

+class TestRepoEntry;
+using AccessCallback = std::function<void(TestRepoEntry &entry)>;
+
 class TestRepoEntry
 {
 public:
@@ -70,7 +81,8 @@ public:
    int requestCount;
    bool getWillFail;
    bool returnCorruptData;
-    std::unique_ptr<SGCallback> accessCallback;
+
+    AccessCallback accessCallback;

    void clearRequestCounts();

@@ -270,8 +282,8 @@ public:
                return;
            }

-            if (entry->accessCallback.get()) {
-                (*entry->accessCallback)();
+            if (entry->accessCallback) {
+              entry->accessCallback(*entry);
            }

            if (entry->getWillFail) {
@@ -282,20 +294,29 @@ public:
            entry->requestCount++;

            std::string content;
+            bool closeSocket = false;
+            size_t contentSize = 0;
+
            if (entry->returnCorruptData) {
                content = dataForFile("!$£$!" + entry->parent->name,
                                      "corrupt_" + entry->name,
                                      entry->revision);
+                contentSize = content.size();
            } else {
-                content = entry->data();
+              content = entry->data();
+              contentSize = content.size();
            }

            std::stringstream d;
            d << "HTTP/1.1 " << 200 << " " << reasonForCode(200) << "\r\n";
-            d << "Content-Length:" << content.size() << "\r\n";
+            d << "Content-Length:" << contentSize << "\r\n";
            d << "\r\n"; // final CRLF to terminate the headers
            d << content;
            push(d.str().c_str());
+
+            if (closeSocket) {
+              closeWhenDone();
+            }
        } else {
            sendErrorResponse(404, false, "");
        }
@@ -392,6 +413,7 @@ void waitForUpdateComplete(HTTP::Client* cl, HTTPRepository* repo)
        cl->update();
        testServer.poll();

+        repo->process();
        if (!repo->isDoingSync()) {
            return;
        }
@@ -401,6 +423,16 @@ void waitForUpdateComplete(HTTP::Client* cl, HTTPRepository* repo)
    std::cerr << "timed out" << std::endl;
 }

+void runForTime(HTTP::Client *cl, HTTPRepository *repo, int msec = 15) {
+  SGTimeStamp start(SGTimeStamp::now());
+  while (start.elapsedMSec() < msec) {
+    cl->update();
+    testServer.poll();
+    repo->process();
+    SGTimeStamp::sleepForMSec(1);
+  }
+}
+
 void testBasicClone(HTTP::Client* cl)
 {
    std::unique_ptr<HTTPRepository> repo;
@@ -418,6 +450,7 @@ void testBasicClone(HTTP::Client* cl)
    verifyFileState(p, "fileA");
    verifyFileState(p, "dirA/subdirA/fileAAA");
    verifyFileState(p, "dirC/subdirA/subsubA/fileCAAA");
+    verifyFileState(p, "dirA/subdirA/zeroByteFile");

    global_repo->findEntry("fileA")->revision++;
    global_repo->findEntry("dirB/subdirA/fileBAA")->revision++;
@@ -618,9 +651,19 @@ void testAbandonCorruptFiles(HTTP::Client* cl)
    repo->setBaseUrl("http://localhost:2000/repo");
    repo->update();
    waitForUpdateComplete(cl, repo.get());
-    if (repo->failure() != HTTPRepository::REPO_ERROR_CHECKSUM) {
-        std::cerr << "Got failure state:" << repo->failure() << std::endl;
-        throw sg_exception("Bad result from corrupt files test");
+    if (repo->failure() != HTTPRepository::REPO_PARTIAL_UPDATE) {
+      std::cerr << "Got failure state:" << repo->failure() << std::endl;
+      throw sg_exception("Bad result from corrupt files test");
+    }
+
+    auto failedFiles = repo->failures();
+    if (failedFiles.size() != 1) {
+      throw sg_exception("Bad result from corrupt files test");
+    }
+
+    if (failedFiles.front().path.utf8Str() != "dirB/subdirG/fileBGA") {
+      throw sg_exception("Bad path from corrupt files test:" +
+                         failedFiles.front().path.utf8Str());
    }

    repo.reset();
@@ -657,15 +700,21 @@ void testServerModifyDuringSync(HTTP::Client* cl)
    repo.reset(new HTTPRepository(p, cl));
    repo->setBaseUrl("http://localhost:2000/repo");

-    global_repo->findEntry("dirA/fileAA")->accessCallback.reset(make_callback(&modifyBTree));
+    global_repo->findEntry("dirA/fileAA")->accessCallback =
+        [](const TestRepoEntry &r) {
+          std::cout << "Modifying sub-tree" << std::endl;
+          global_repo->findEntry("dirB/subdirA/fileBAC")->revision++;
+          global_repo->defineFile("dirB/subdirZ/fileBZA");
+          global_repo->findEntry("dirB/subdirB/fileBBB")->revision++;
+        };

    repo->update();
    waitForUpdateComplete(cl, repo.get());

-    global_repo->findEntry("dirA/fileAA")->accessCallback.reset();
+    global_repo->findEntry("dirA/fileAA")->accessCallback = AccessCallback{};

-    if (repo->failure() != HTTPRepository::REPO_ERROR_CHECKSUM) {
-        throw sg_exception("Bad result from modify during sync test");
+    if (repo->failure() != HTTPRepository::REPO_PARTIAL_UPDATE) {
+      throw sg_exception("Bad result from modify during sync test");
    }

    std::cout << "Passed test modify server during sync" << std::endl;
@@ -755,6 +804,103 @@ void testCopyInstalledChildren(HTTP::Client* cl)
    std::cout << "passed Copy installed children" << std::endl;
 }

+void testRetryAfterSocketFailure(HTTP::Client *cl) {
+  global_repo->clearRequestCounts();
+  global_repo->clearFailFlags();
+
+  std::unique_ptr<HTTPRepository> repo;
+  SGPath p(simgear::Dir::current().path());
+  p.append("http_repo_retry_after_socket_fail");
+  simgear::Dir pd(p);
+  if (pd.exists()) {
+    pd.removeChildren();
+  }
+
+  repo.reset(new HTTPRepository(p, cl));
+  repo->setBaseUrl("http://localhost:2000/repo");
+
+  int aaFailsRemaining = 2;
+  int subdirBAFailsRemaining = 2;
+  TestApi::setResponseDoneCallback(
+      cl, [&aaFailsRemaining, &subdirBAFailsRemaining](int curlResult,
+                                                       HTTP::Request_ptr req) {
+        if (req->url() == "http://localhost:2000/repo/dirA/fileAA") {
+          if (aaFailsRemaining == 0)
+            return false;
+
+          --aaFailsRemaining;
+          TestApi::markRequestAsFailed(req, 56, "Simulated socket failure");
+          return true;
+        } else if (req->url() ==
+                   "http://localhost:2000/repo/dirB/subdirA/.dirindex") {
+          if (subdirBAFailsRemaining == 0)
+            return false;
+
+          --subdirBAFailsRemaining;
+          TestApi::markRequestAsFailed(req, 56, "Simulated socket failure");
+          return true;
+        } else {
+          return false;
+        }
+      });
+
+  repo->update();
+  waitForUpdateComplete(cl, repo.get());
+
+  if (repo->failure() != HTTPRepository::REPO_NO_ERROR) {
+    throw sg_exception("Bad result from retry socket failure test");
+  }
+
+  verifyFileState(p, "dirA/fileAA");
+  verifyFileState(p, "dirB/subdirA/fileBAA");
+  verifyFileState(p, "dirB/subdirA/fileBAC");
+
+  verifyRequestCount("dirA/fileAA", 3);
+  verifyRequestCount("dirB/subdirA", 3);
+  verifyRequestCount("dirB/subdirA/fileBAC", 1);
+}
+
+void testPersistentSocketFailure(HTTP::Client *cl) {
+  global_repo->clearRequestCounts();
+  global_repo->clearFailFlags();
+
+  std::unique_ptr<HTTPRepository> repo;
+  SGPath p(simgear::Dir::current().path());
+  p.append("http_repo_persistent_socket_fail");
+  simgear::Dir pd(p);
+  if (pd.exists()) {
+    pd.removeChildren();
+  }
+
+  repo.reset(new HTTPRepository(p, cl));
+  repo->setBaseUrl("http://localhost:2000/repo");
+
+  TestApi::setResponseDoneCallback(
+      cl, [](int curlResult, HTTP::Request_ptr req) {
+        const auto url = req->url();
+        if (url.find("http://localhost:2000/repo/dirB") == 0) {
+          TestApi::markRequestAsFailed(req, 56, "Simulated socket failure");
+          return true;
+        }
+
+        return false;
+      });
+
+  repo->update();
+  waitForUpdateComplete(cl, repo.get());
+
+  if (repo->failure() != HTTPRepository::REPO_PARTIAL_UPDATE) {
+    throw sg_exception("Bad result from retry socket failure test");
+  }
+
+  verifyFileState(p, "dirA/fileAA");
+  verifyRequestCount("dirA/fileAA", 1);
+
+  verifyRequestCount("dirD/fileDA", 1);
+  verifyRequestCount("dirD/subdirDA/fileDAA", 1);
+  verifyRequestCount("dirD/subdirDB/fileDBA", 1);
+}
+
 int main(int argc, char* argv[])
 {
  sglog().setLogLevels( SG_ALL, SG_INFO );
@@ -770,6 +916,8 @@ int main(int argc, char* argv[])
    global_repo->defineFile("dirA/fileAC");
    global_repo->defineFile("dirA/subdirA/fileAAA");
    global_repo->defineFile("dirA/subdirA/fileAAB");
+    global_repo->defineFile("dirA/subdirA/zeroByteFile");
+
    global_repo->defineFile("dirB/subdirA/fileBAA");
    global_repo->defineFile("dirB/subdirA/fileBAB");
    global_repo->defineFile("dirB/subdirA/fileBAC");
@@ -800,6 +948,8 @@ int main(int argc, char* argv[])
    cl.clearAllConnections();

    testCopyInstalledChildren(&cl);
+    testRetryAfterSocketFailure(&cl);
+    testPersistentSocketFailure(&cl);

    std::cout << "all tests passed ok" << std::endl;
    return 0;
@@ -31,6 +31,8 @@
 #include <simgear/sg_inlines.h>
 #include <simgear/io/sg_file.hxx>
 #include <simgear/misc/sg_dir.hxx>
+#include <simgear/misc/strutils.hxx>
+
 #include <simgear/io/iostreams/sgstream.hxx>
 #include <simgear/debug/logstream.hxx>
 #include <simgear/package/unzip.h>
@@ -509,32 +511,34 @@ public:
 		const size_t BUFFER_SIZE = 1024 * 1024;
 		void* buf = malloc(BUFFER_SIZE);

-		try {
-			int result = unzGoToFirstFile(zip);
-			if (result != UNZ_OK) {
-				throw sg_exception("failed to go to first file in archive");
-			}
-
-			while (true) {
-				extractCurrentFile(zip, (char*)buf, BUFFER_SIZE);
-				if (state == FILTER_STOPPED) {
-					break;
-				}
-
-				result = unzGoToNextFile(zip);
-				if (result == UNZ_END_OF_LIST_OF_FILE) {
-					break;
-				}
-				else if (result != UNZ_OK) {
-					throw sg_io_exception("failed to go to next file in the archive");
-				}
-			}
-			state = END_OF_ARCHIVE;
-		}
-		catch (sg_exception&) {
+        int result = unzGoToFirstFile(zip);
+        if (result != UNZ_OK) {
+            SG_LOG(SG_IO, SG_ALERT, outer->rootPath() << "failed to go to first file in archive:" << result);
            state = BAD_ARCHIVE;
+            free(buf);
+            unzClose(zip);
+            return;
        }
-        
+
+        while (true) {
+            extractCurrentFile(zip, (char*)buf, BUFFER_SIZE);
+            if (state == FILTER_STOPPED) {
+                state = END_OF_ARCHIVE;
+                break;
+            }
+
+            result = unzGoToNextFile(zip);
+            if (result == UNZ_END_OF_LIST_OF_FILE) {
+                state = END_OF_ARCHIVE;
+                break;
+            } else if (result != UNZ_OK) {
+                SG_LOG(SG_IO, SG_ALERT, outer->rootPath() << "failed to go to next file in archive:" << result);
+                state = BAD_ARCHIVE;
+                break;
+            }
+        }
+
+
        free(buf);
        unzClose(zip);
    }
@@ -592,7 +596,7 @@ public:

 		outFile.open(path, std::ios::binary | std::ios::trunc | std::ios::out);
 		if (outFile.fail()) {
-			throw sg_io_exception("failed to open output file for writing", path);
+			throw sg_io_exception("failed to open output file for writing:" + strutils::error_string(errno), path);
 		}

 		while (!eof) {
@@ -70,6 +70,11 @@ public:
 		Stop
 	};

+    SGPath rootPath() const
+    {
+        return _rootPath;
+    }
+
 protected:


@@ -29,6 +29,13 @@ public:
  /// Default constructor, initializes the instance to lat = lon = elev = 0
  SGGeod(void);

+  /**
+    return an SGGeod for which isValid() returns false.
+    This is necessaerby becuase for historical reasons, ther defaulrt constructor above initialsies to zero,zero,zero
+    which *is*
+   */
+  static SGGeod invalid();
+
  /// Factory from angular values in radians and elevation is 0
  static SGGeod fromRad(double lon, double lat);
  /// Factory from angular values in degrees and elevation is 0
@@ -663,3 +663,8 @@ SGGeodesy::radialIntersection(const SGGeod& a, double aRadial,
  result = SGGeod::fromGeoc(r);
  return true;
 }
+
+SGGeod SGGeod::invalid()
+{
+    return SGGeod::fromDeg(-999.9, -999.0);
+}
@@ -49,6 +49,11 @@ ResourceManager* ResourceManager::instance()
    return static_manager;
 }    

+bool ResourceManager::haveInstance()
+{
+    return static_manager != nullptr;
+}
+
 ResourceManager::~ResourceManager()
 {
    assert(this == static_manager);
@@ -47,6 +47,8 @@ public:

    static ResourceManager* instance();

+    static bool haveInstance();
+
    static void reset();

    /**
@@ -44,6 +44,7 @@
 #if defined(SG_WINDOWS)
 #  include <direct.h>
 #  include <sys/utime.h>
+#  include <Shlwapi.h>
 #endif

 #include "sg_path.hxx"
@@ -194,7 +195,8 @@ SGPath::SGPath(PermissionChecker validator)
    _permission_checker(validator),
    _cached(false),
    _rwCached(false),
-    _cacheEnabled(true)
+    _cacheEnabled(true),
+    _existsCached(false)
 {
 }

@@ -205,7 +207,8 @@ SGPath::SGPath( const std::string& p, PermissionChecker validator )
    _permission_checker(validator),
    _cached(false),
    _rwCached(false),
-    _cacheEnabled(true)
+    _cacheEnabled(true),
+    _existsCached(false)
 {
    fix();
 }
@@ -230,7 +233,8 @@ SGPath::SGPath( const SGPath& p,
    _permission_checker(validator),
    _cached(false),
    _rwCached(false),
-    _cacheEnabled(p._cacheEnabled)
+    _cacheEnabled(p._cacheEnabled),
+    _existsCached(false)
 {
    append(r);
    fix();
@@ -510,6 +514,19 @@ void SGPath::checkAccess() const

 bool SGPath::exists() const
 {
+#if defined(SG_WINDOWS)
+  // optimisation: _wstat is slow, eg for TerraSync
+  if (!_cached && !_existsCached) {
+      std::wstring w(wstr());
+    if ((path.length() > 1) && (path.back() == '/')) {
+	    w.pop_back();
+    }
+
+    _existsCached = true;
+    _exists = PathFileExistsW(w.c_str());
+    return _exists;
+  }
+#endif
  validate();
  return _exists;
 }
@@ -356,6 +356,7 @@ private:
    mutable bool _exists : 1;
    mutable bool _isDir : 1;
    mutable bool _isFile : 1;
+    mutable bool _existsCached : 1; ///< only used on Windows
    mutable time_t _modTime;
    mutable size_t _size;
 };
@@ -248,7 +248,7 @@ void naFreeContext(naContext c)
    // than I have right now. So instead I'm clearing the stack tops here, so
    // a freed context looks the same as a new one returned by initContext.

-    c->fTop = c->opTop = c->markTop = 0;
+    c->fTop = c->opTop = c->markTop = c->ntemps = 0;

    c->nextFree = globals->freeContexts;
    globals->freeContexts = c;
@@ -63,7 +63,7 @@ namespace nasal

    class NasalMainLoopRecipient : public simgear::Emesary::IReceiver {
    public:
-        NasalMainLoopRecipient() : receiveCount(0) {
+        NasalMainLoopRecipient() : receiveCount(0), Active(false), CanWait(false) {
            simgear::Emesary::GlobalTransmitter::instance()->Register(*this);
        }
        virtual ~NasalMainLoopRecipient() {
@@ -100,7 +100,14 @@ naRef naNewHash(struct Context* c)

 naRef naNewCode(struct Context* c)
 {
-    return naNew(c, T_CODE);
+    naRef r = naNew(c, T_CODE);
+    // naNew can return a previously used naCode. naCodeGen will init
+    // all these members but a GC can occur inside naCodeGen, so we see
+    // partially initalized state here. To avoid this, clear out the values
+    // which mark() cares about.
+    PTR(r).code->srcFile = naNil();
+    PTR(r).code->nConstants = 0;
+    return r;
 }

 naRef naNewCCode(struct Context* c, naCFunction fptr)
@@ -643,16 +643,53 @@ void Catalog::processAlternate(SGPropertyNode_ptr alt)
        return;
    }

-    // we have an alternate ID, and it's differnt from our ID, so let's
+    // we have an alternate ID, and it's different from our ID, so let's
    // define a new catalog
    if (!altId.empty()) {
-        SG_LOG(SG_GENERAL, SG_INFO, "Adding new catalog:" << altId << " as version alternate for " << id());
-        // new catalog being added
-        createFromUrl(root(), altUrl);
-
-        // and we can go idle now
+      // don't auto-re-add Catalogs the user has explicilty rmeoved, that would
+      // suck
+      const auto removedByUser = root()->explicitlyRemovedCatalogs();
+      auto it = std::find(removedByUser.begin(), removedByUser.end(), altId);
+      if (it != removedByUser.end()) {
        changeStatus(Delegate::FAIL_VERSION);
        return;
+      }
+
+      SG_LOG(SG_GENERAL, SG_WARN,
+             "Adding new catalog:" << altId << " as version alternate for "
+                                   << id());
+      // new catalog being added
+      auto newCat = createFromUrl(root(), altUrl);
+
+      bool didRun = false;
+      newCat->m_migratedFrom = this;
+
+      auto migratePackagesCb = [didRun](Catalog *c) mutable {
+        // removing callbacks is awkward, so use this
+        // flag to only run once. (and hence, we need to be mutable)
+        if (didRun)
+          return;
+
+        if (c->status() == Delegate::STATUS_REFRESHED) {
+          didRun = true;
+
+          string_list existing;
+          for (const auto &pack : c->migratedFrom()->installedPackages()) {
+            existing.push_back(pack->id());
+          }
+
+          const int count = c->markPackagesForInstallation(existing);
+          SG_LOG(
+              SG_GENERAL, SG_INFO,
+              "Marked " << count
+                        << " packages from previous catalog for installation");
+        }
+      };
+
+      newCat->addStatusCallback(migratePackagesCb);
+      // and we can go idle now
+      changeStatus(Delegate::FAIL_VERSION);
+      return;
    }

    SG_LOG(SG_GENERAL, SG_INFO, "Migrating catalog " << id() << " to new URL:" << altUrl);
@@ -661,6 +698,26 @@ void Catalog::processAlternate(SGPropertyNode_ptr alt)
    root()->makeHTTPRequest(dl);
 }

+int Catalog::markPackagesForInstallation(const string_list &packageIds) {
+  int result = 0;
+
+  for (const auto &id : packageIds) {
+    auto ourPkg = getPackageById(id);
+    if (!ourPkg)
+      continue;
+
+    auto existing = ourPkg->existingInstall();
+    if (!existing) {
+      ourPkg->markForInstall();
+      ++result;
+    }
+  } // of outer package ID candidates iteration
+
+  return result;
+}
+
+CatalogRef Catalog::migratedFrom() const { return m_migratedFrom; }
+
 } // of namespace pkg

 } // of namespace simgear
@@ -23,6 +23,7 @@
 #include <map>

 #include <simgear/misc/sg_path.hxx>
+#include <simgear/misc/strutils.hxx>
 #include <simgear/props/props.hxx>

 #include <simgear/structure/SGReferenced.hxx>
@@ -93,7 +94,7 @@ public:

    /**
     * retrieve all the packages in the catalog which are installed
-     * and have a pendig update
+     * and have a pending update
     */
    PackageList packagesNeedingUpdate() const;

@@ -151,7 +152,32 @@ public:
    
    bool isUserEnabled() const;
    void setUserEnabled(bool b);
-private:
+
+    /**
+     * Given a list of package IDs, mark all which exist in this package,
+     * for installation. ANy packahe IDs not present in this catalog,
+     * will be ignored.
+     *
+     * @result The number for packages newly marked for installation.
+     */
+    int markPackagesForInstallation(const string_list &packageIds);
+
+    /**
+     * When a catalog is added due to migration, this will contain the
+     * Catalog which triggered the add. Usually this will be a catalog
+     * corresponding to an earlier version.
+     *
+     * Note it's only valid at the time, the migration actually took place;
+     * when the new catalog is loaded from disk, this value will return
+     * null.
+     *
+     * This is intended to allow Uis to show a 'catalog was migrated'
+     * feedback, when they see a catalog refresh, which has a non-null
+     * value of this method.
+     */
+    CatalogRef migratedFrom() const;
+
+  private:
    Catalog(Root* aRoot);

    class Downloader;
@@ -197,6 +223,8 @@ private:
    PackageWeakMap m_variantDict;

    function_list<Callback> m_statusCallbacks;
+
+    CatalogRef m_migratedFrom;
 };

 } // of namespace pkg
@@ -827,7 +827,10 @@ void testVersionMigrateToId(HTTP::Client* cl)
        
        it = std::find(enabledCats.begin(), enabledCats.end(), altCat);
        SG_VERIFY(it != enabledCats.end());
-        
+
+        SG_CHECK_EQUAL(altCat->packagesNeedingUpdate().size(),
+                       1); // should be the 737
+
        // install a parallel package from the new catalog
        pkg::PackageRef p2 = root->getPackageById("org.flightgear.test.catalog-alt.b737-NG");
        SG_CHECK_EQUAL(p2->id(), "b737-NG");
@@ -840,8 +843,8 @@ void testVersionMigrateToId(HTTP::Client* cl)
        pkg::PackageRef p3 = root->getPackageById("b737-NG");
        SG_CHECK_EQUAL(p2, p3);
    }
-    
-    // test that re-init-ing doesn't mirgate again
+
+    // test that re-init-ing doesn't migrate again
    {
        pkg::RootRef root(new pkg::Root(rootPath, "7.5"));
        root->setHTTPClient(cl);
@@ -1184,6 +1187,128 @@ void testMirrorsFailure(HTTP::Client* cl)
    
 }

+void testMigrateInstalled(HTTP::Client *cl) {
+  SGPath rootPath(simgear::Dir::current().path());
+  rootPath.append("pkg_migrate_installed");
+  simgear::Dir pd(rootPath);
+  pd.removeChildren();
+
+  pkg::RootRef root(new pkg::Root(rootPath, "8.1.2"));
+  root->setHTTPClient(cl);
+
+  pkg::CatalogRef oldCatalog, newCatalog;
+
+  {
+    oldCatalog = pkg::Catalog::createFromUrl(
+        root.ptr(), "http://localhost:2000/catalogTest1/catalog.xml");
+    waitForUpdateComplete(cl, root);
+
+    pkg::PackageRef p1 =
+        root->getPackageById("org.flightgear.test.catalog1.b747-400");
+    p1->install();
+    auto p2 = root->getPackageById("org.flightgear.test.catalog1.c172p");
+    p2->install();
+    auto p3 = root->getPackageById("org.flightgear.test.catalog1.b737-NG");
+    p3->install();
+    waitForUpdateComplete(cl, root);
+  }
+
+  {
+    newCatalog = pkg::Catalog::createFromUrl(
+        root.ptr(), "http://localhost:2000/catalogTest2/catalog.xml");
+    waitForUpdateComplete(cl, root);
+
+    string_list existing;
+    for (const auto &pack : oldCatalog->installedPackages()) {
+      existing.push_back(pack->id());
+    }
+
+    SG_CHECK_EQUAL(4, existing.size());
+
+    int result = newCatalog->markPackagesForInstallation(existing);
+    SG_CHECK_EQUAL(2, result);
+    SG_CHECK_EQUAL(2, newCatalog->packagesNeedingUpdate().size());
+
+    auto p1 = root->getPackageById("org.flightgear.test.catalog2.b737-NG");
+    auto ins = p1->existingInstall();
+    SG_CHECK_EQUAL(0, ins->revsion());
+  }
+
+  {
+    root->scheduleAllUpdates();
+    waitForUpdateComplete(cl, root);
+
+    SG_CHECK_EQUAL(0, newCatalog->packagesNeedingUpdate().size());
+
+    auto p1 = root->getPackageById("org.flightgear.test.catalog2.b737-NG");
+    auto ins = p1->existingInstall();
+    SG_CHECK_EQUAL(ins->revsion(), p1->revision());
+  }
+}
+
+void testDontMigrateRemoved(HTTP::Client *cl) {
+  global_catalogVersion = 2; // version which has migration info
+  SGPath rootPath(simgear::Dir::current().path());
+  rootPath.append("cat_dont_migrate_id");
+  simgear::Dir pd(rootPath);
+  pd.removeChildren();
+
+  // install and mnaully remove the alt catalog
+
+  {
+    pkg::RootRef root(new pkg::Root(rootPath, "8.1.2"));
+    root->setHTTPClient(cl);
+
+    pkg::CatalogRef c = pkg::Catalog::createFromUrl(
+        root.ptr(), "http://localhost:2000/catalogTest1/catalog-alt.xml");
+    waitForUpdateComplete(cl, root);
+
+    root->removeCatalogById("org.flightgear.test.catalog-alt");
+  }
+
+  // install the migration catalog
+  {
+    pkg::RootRef root(new pkg::Root(rootPath, "8.1.2"));
+    root->setHTTPClient(cl);
+
+    pkg::CatalogRef c = pkg::Catalog::createFromUrl(
+        root.ptr(), "http://localhost:2000/catalogTest1/catalog.xml");
+    waitForUpdateComplete(cl, root);
+    SG_VERIFY(c->isEnabled());
+  }
+
+  // change version to an alternate one
+  {
+    pkg::RootRef root(new pkg::Root(rootPath, "7.5"));
+
+    auto removed = root->explicitlyRemovedCatalogs();
+    auto j = std::find(removed.begin(), removed.end(),
+                       "org.flightgear.test.catalog-alt");
+    SG_VERIFY(j != removed.end());
+
+    root->setHTTPClient(cl);
+
+    // this would tirgger migration, but we blocked it
+    root->refresh(true);
+    waitForUpdateComplete(cl, root);
+
+    pkg::CatalogRef cat = root->getCatalogById("org.flightgear.test.catalog1");
+    SG_VERIFY(!cat->isEnabled());
+    SG_CHECK_EQUAL(cat->status(), pkg::Delegate::FAIL_VERSION);
+    SG_CHECK_EQUAL(cat->id(), "org.flightgear.test.catalog1");
+    SG_CHECK_EQUAL(cat->url(),
+                   "http://localhost:2000/catalogTest1/catalog.xml");
+
+    auto enabledCats = root->catalogs();
+    auto it = std::find(enabledCats.begin(), enabledCats.end(), cat);
+    SG_VERIFY(it == enabledCats.end());
+
+    // check the new catalog
+    auto altCat = root->getCatalogById("org.flightgear.test.catalog-alt");
+    SG_VERIFY(altCat.get() == nullptr);
+  }
+}
+
 int main(int argc, char* argv[])
 {
    sglog().setLogLevels( SG_ALL, SG_WARN );
@@ -1228,7 +1353,11 @@ int main(int argc, char* argv[])
    testInstallBadPackage(&cl);
    
    testMirrorsFailure(&cl);
-    
+
+    testMigrateInstalled(&cl);
+
+    testDontMigrateRemoved(&cl);
+
    cerr << "Successfully passed all tests!" << endl;
    return EXIT_SUCCESS;
 }
@@ -198,6 +198,11 @@ InstallRef Package::install()
 {
    InstallRef ins = existingInstall();
    if (ins) {
+      // if there's updates, treat this as a 'start update' request
+      if (ins->hasUpdate()) {
+        m_catalog->root()->scheduleToUpdate(ins);
+      }
+
        return ins;
    }

@@ -210,13 +215,39 @@ InstallRef Package::install()
    return ins;
 }

+InstallRef Package::markForInstall() {
+  InstallRef ins = existingInstall();
+  if (ins) {
+    return ins;
+  }
+
+  const auto pd = pathOnDisk();
+
+  Dir dir(pd);
+  if (!dir.create(0700)) {
+    SG_LOG(SG_IO, SG_ALERT,
+           "Package::markForInstall: couldn't create directory at:" << pd);
+    return {};
+  }
+
+  ins = new Install{this, pd};
+  _install_cb(this, ins); // not sure if we should trigger the callback for this
+
+  // repeat for dependencies to be kind
+  for (auto dep : dependencies()) {
+    dep->markForInstall();
+  }
+
+  return ins;
+}
+
 InstallRef Package::existingInstall(const InstallCallback& cb) const
 {
    InstallRef install;
    try {
        install = m_catalog->root()->existingInstallForPackage(const_cast<Package*>(this));
    } catch (std::exception& ) {
-        return InstallRef();
+      return {};
    }

  if( cb )
@@ -61,6 +61,13 @@ public:
    InstallRef
    existingInstall(const InstallCallback& cb = InstallCallback()) const;

+    /**
+     * Mark this package for installation, but don't actually start the
+     * download process. This creates the on-disk placeholder, so
+     * the package will appear an eededing to be updated.
+     */
+    InstallRef markForInstall();
+
    bool isInstalled() const;

    /**
@@ -133,7 +133,9 @@ public:

    void fireRefreshStatus(CatalogRef catalog, Delegate::StatusCode status)
    {
-        for (auto d : delegates) {
+        // take a copy of delegates since firing this can modify the data
+        const auto currentDelegates = delegates;
+        for (auto d : currentDelegates) {
            d->catalogRefreshed(catalog, status);
        }
    }
@@ -283,6 +285,32 @@ public:
        fireDataForThumbnail(url, reinterpret_cast<const uint8_t*>(bytes.data()), bytes.size());
    }

+    void writeRemovedCatalogsFile() const {
+      SGPath p = path / "RemovedCatalogs";
+      sg_ofstream stream(p, std::ios::out | std::ios::trunc | std::ios::binary);
+      for (const auto &cid : manuallyRemovedCatalogs) {
+        stream << cid << "\n";
+      }
+      stream.close();
+    }
+
+    void loadRemovedCatalogsFile() {
+      manuallyRemovedCatalogs.clear();
+      SGPath p = path / "RemovedCatalogs";
+      if (!p.exists())
+        return;
+
+      sg_ifstream stream(p, std::ios::in);
+      while (!stream.eof()) {
+        std::string line;
+        std::getline(stream, line);
+        const auto trimmed = strutils::strip(line);
+        if (!trimmed.empty()) {
+          manuallyRemovedCatalogs.push_back(trimmed);
+        }
+      } // of lines iteration
+    }
+
    DelegateVec delegates;

    SGPath path;
@@ -312,6 +340,9 @@ public:

    typedef std::map<PackageRef, InstallRef> InstallCache;
    InstallCache m_installs;
+
+    /// persistent list of catalogs the user has manually removed
+    string_list manuallyRemovedCatalogs;
 };


@@ -400,6 +431,8 @@ Root::Root(const SGPath& aPath, const std::string& aVersion) :
        thumbsCacheDir.create(0755);
    }

+    d->loadRemovedCatalogsFile();
+
    for (SGPath c : dir.children(Dir::TYPE_DIR | Dir::NO_DOT_OR_DOTDOT)) {
        // note this will set the catalog status, which will insert into
        // disabled catalogs automatically if necesary
@@ -621,6 +654,13 @@ void Root::scheduleToUpdate(InstallRef aInstall)
    }
 }

+void Root::scheduleAllUpdates() {
+  auto toBeUpdated = packagesNeedingUpdate(); // make a copy
+  for (const auto &u : toBeUpdated) {
+    scheduleToUpdate(u->existingInstall());
+  }
+}
+
 bool Root::isInstallQueued(InstallRef aInstall) const
 {
    auto it = std::find(d->updateDeque.begin(), d->updateDeque.end(), aInstall);
@@ -684,12 +724,6 @@ void Root::catalogRefreshStatus(CatalogRef aCat, Delegate::StatusCode aReason)
    auto catIt = d->catalogs.find(aCat->id());
    d->fireRefreshStatus(aCat, aReason);

-    if (aReason == Delegate::STATUS_IN_PROGRESS) {
-        d->refreshing.insert(aCat);
-    } else {
-        d->refreshing.erase(aCat);
-    }
-
    if (aCat->isUserEnabled() &&
        (aReason == Delegate::STATUS_REFRESHED) && 
        (catIt == d->catalogs.end())) 
@@ -723,6 +757,17 @@ void Root::catalogRefreshStatus(CatalogRef aCat, Delegate::StatusCode aReason)
        }
    } // of catalog is disabled

+    // remove from refreshing /after/ checking for enable / disabled, since for
+    // new catalogs, the reference in d->refreshing might be our /only/
+    // reference to the catalog. Once the refresh is done (either failed or
+    // succeeded) the Catalog will be in either d->catalogs or
+    // d->disabledCatalogs
+    if (aReason == Delegate::STATUS_IN_PROGRESS) {
+      d->refreshing.insert(aCat);
+    } else {
+      d->refreshing.erase(aCat);
+    }
+
    if (d->refreshing.empty()) {
        d->fireRefreshStatus(CatalogRef(), Delegate::STATUS_REFRESHED);
        d->firePackagesChanged();
@@ -783,6 +828,9 @@ bool Root::removeCatalogById(const std::string& aId)
            << "failed to remove directory");
    }

+    d->manuallyRemovedCatalogs.push_back(aId);
+    d->writeRemovedCatalogsFile();
+
    // notify that a catalog is being removed
    d->firePackagesChanged();

@@ -854,6 +902,10 @@ void Root::unregisterInstall(InstallRef ins)
    d->fireFinishUninstall(ins->package());
 }

+string_list Root::explicitlyRemovedCatalogs() const {
+  return d->manuallyRemovedCatalogs;
+}
+
 } // of namespace pkg

 } // of namespace simgear
@@ -155,7 +155,22 @@ public:
    void requestThumbnailData(const std::string& aUrl);
    
    bool isInstallQueued(InstallRef aInstall) const;
-private:
+
+    /**
+     * Mark all 'to be updated' packages for update now
+     */
+    void scheduleAllUpdates();
+
+    /**
+     * @brief list of catalog IDs, the user has explicitly removed via
+     * removeCatalogById(). This is important to allow the user to opt-out
+     * of migrated packages.
+     *
+     * This information is stored in a helper file, in the root directory
+     */
+    string_list explicitlyRemovedCatalogs() const;
+
+  private:
    friend class Install;
    friend class Catalog;    
    friend class Package;
@@ -0,0 +1,86 @@
+<?xml version="1.0"?>
+
+<PropertyList>
+    <id>org.flightgear.test.catalog2</id>
+    <description>Second test catalog</description>
+    <url>http://localhost:2000/catalogTest2/catalog.xml</url>
+    <catalog-version>4</catalog-version>
+
+    <version>8.1.*</version>
+    <version>8.0.0</version>
+    <version>8.2.0</version>
+
+    <package>
+        <id>alpha</id>
+        <name>Alpha package</name>
+        <revision type="int">8</revision>
+        <file-size-bytes type="int">593</file-size-bytes>
+
+        <md5>a469c4b837f0521db48616cfe65ac1ea</md5>
+        <url>http://localhost:2000/catalogTest1/alpha.zip</url>
+
+        <dir>alpha</dir>
+
+    </package>
+
+
+    <package>
+        <id>b737-NG</id>
+        <name>Boeing 737 NG</name>
+        <dir>b737NG</dir>
+        <description>A popular twin-engined narrow body jet</description>
+        <revision type="int">111</revision>
+        <file-size-bytes type="int">860</file-size-bytes>
+
+        <tag>boeing</tag>
+        <tag>jet</tag>
+        <tag>ifr</tag>
+
+      <!-- not within a localized element -->
+          <de>
+            <description>German description of B737NG XYZ</description>
+          </de>
+          <fr>
+            <description>French description of B737NG</description>
+          </fr>
+
+        <rating>
+          <FDM type="int">5</FDM>
+          <systems type="int">5</systems>
+          <model type="int">4</model>
+          <cockpit type="int">4</cockpit>
+        </rating>
+
+        <md5>a94ca5704f305b90767f40617d194ed6</md5>
+        <url>http://localhost:2000/mirrorA/b737.tar.gz</url>
+        <url>http://localhost:2000/mirrorB/b737.tar.gz</url>
+        <url>http://localhost:2000/mirrorC/b737.tar.gz</url>
+      </package>
+
+    <package>
+        <id>b747-400</id>
+        <name>Boeing 747-400</name>
+        <dir>b744</dir>
+        <description>A popular four-engined wide-body jet</description>
+        <revision type="int">111</revision>
+        <file-size-bytes type="int">860</file-size-bytes>
+
+        <tag>boeing</tag>
+        <tag>jet</tag>
+        <tag>ifr</tag>
+
+  
+
+        <rating>
+          <FDM type="int">5</FDM>
+          <systems type="int">5</systems>
+          <model type="int">4</model>
+          <cockpit type="int">4</cockpit>
+        </rating>
+
+        <md5>4d3f7417d74f811aa20ccc4f35673d20</md5>
+        <!-- this URL will sometimes fail, on purpose -->
+        <url>http://localhost:2000/catalogTest1/b747.tar.gz</url>
+      </package>
+
+</PropertyList>
@@ -59,6 +59,12 @@ void AtomicChangeListener::fireChangeListeners()
    listeners.clear();
 }

+void AtomicChangeListener::clearPendingChanges()
+{
+    auto& listeners = ListenerListSingleton::instance()->listeners;
+    listeners.clear();
+}
+
 void AtomicChangeListener::valueChangedImplementation()
 {
    if (!_dirty) {
@@ -52,7 +52,17 @@ public:
    bool isDirty() { return _dirty; }
    bool isValid() { return _valid; }
    virtual void unregister_property(SGPropertyNode* node) override;
+
    static void fireChangeListeners();
+
+    /**
+     * @brief Ensure we've deleted any pending changes. 
+     * 
+     * This is important in shutdown and reset, to avoid holding
+     * property listeners around after the property tree is destroyed
+     */
+    static void clearPendingChanges();
+
 private:
    virtual void valueChangedImplementation() override;
    virtual void valuesChanged();
@@ -19,6 +19,10 @@
 #include <sstream>
 #include <iomanip>
 #include <iterator>
+#include <exception> // can't use sg_exception becuase of PROPS_STANDALONE
+#include <mutex>
+#include <thread>
+
 #include <stdio.h>
 #include <string.h>

@@ -51,11 +55,15 @@ using namespace simgear;

 struct SGPropertyNodeListeners
 {
+  /* Protect _num_iterators and _items. We use a recursive mutex to allow
+  nested access to work as normal. */
+  std::recursive_mutex  _rmutex;
+
  /* This keeps a count of the current number of nested invocations of
  forEachListener(). If non-zero, other code higher up the stack is iterating
  _items[] so for example code must not erase items in the vector. */
  int _num_iterators = 0;
-  
+
  std::vector<SGPropertyChangeListener *> _items;
 };

@@ -558,23 +566,15 @@ find_node (SGPropertyNode * current,
  }
 }
 #else
-template<typename Range>
-SGPropertyNode*
-find_node (SGPropertyNode * current,
-           const Range& path,
-           bool create,
-           int last_index = -1)
-{
+template <typename Range>
+SGPropertyNode *find_node(SGPropertyNode *current, const Range &path,
+                          bool create, int last_index = -1) {
  using namespace boost;
-  typedef split_iterator<typename range_result_iterator<Range>::type>
-    PathSplitIterator;
-
-  PathSplitIterator itr
-    = make_split_iterator(path, first_finder("/", is_equal()));
+  auto itr = make_split_iterator(path, first_finder("/", is_equal()));
  if (*path.begin() == '/')
    return find_node_aux(current->getRootNode(), itr, create, last_index);
-   else
-     return find_node_aux(current, itr, create, last_index);
+  else
+    return find_node_aux(current, itr, create, last_index);
 }
 #endif

@@ -2406,6 +2406,7 @@ SGPropertyNode::addChangeListener (SGPropertyChangeListener * listener,
  if (_listeners == 0)
    _listeners = new SGPropertyNodeListeners;

+  std::lock_guard<std::recursive_mutex> lock(_listeners->_rmutex);
  /* If there's a nullptr entry (a listener that was unregistered), we
  overwrite it. This ensures that listeners that routinely unregister+register
  themselves don't make _listeners->_items grow unnecessarily. Otherwise simply
@@ -2429,9 +2430,13 @@ SGPropertyNode::removeChangeListener (SGPropertyChangeListener * listener)
 {
  if (_listeners == 0)
    return;
+  /* We use a std::unique_lock rather than a std::lock_guard because we may
+  need to unlock early. */
+  std::unique_lock<std::recursive_mutex> lock(_listeners->_rmutex);
  vector<SGPropertyChangeListener*>::iterator it =
    find(_listeners->_items.begin(), _listeners->_items.end(), listener);
  if (it != _listeners->_items.end()) {
+    assert(_listeners->_num_iterators >= 0);
    if (_listeners->_num_iterators) {
      /* _listeners._items is currently being iterated further up the stack in
      this thread by one or more nested invocations of forEachListener(), so
@@ -2450,6 +2455,7 @@ SGPropertyNode::removeChangeListener (SGPropertyChangeListener * listener)
      _listeners->_items.erase(it);
      listener->unregister_property(this);
      if (_listeners->_items.empty()) {
+        lock.unlock();
        delete _listeners;
        _listeners = 0;
      }
@@ -2511,9 +2517,11 @@ static void forEachListener(
    )
 {
  if (!_listeners) return;
-  
+
+  std::lock_guard<std::recursive_mutex> lock(_listeners->_rmutex);
+  assert(_listeners->_num_iterators >= 0);
  _listeners->_num_iterators += 1;
-  
+
  /* We need to use an index here when iterating _listeners->_items, not an
  iterator. This is because a listener may add new listeners, causing the
  vector to be reallocated, which would invalidate any iterator. */
@@ -2528,10 +2536,12 @@ static void forEachListener(
      }
    }
  }
-  
+
  _listeners->_num_iterators -= 1;
-  
+  assert(_listeners->_num_iterators >= 0);
+
  if (_listeners->_num_iterators == 0) {
+
    /* Remove any items that have been set to nullptr. */
    _listeners->_items.erase(
        std::remove(_listeners->_items.begin(), _listeners->_items.end(), (SGPropertyChangeListener*) nullptr),
@@ -2547,6 +2557,7 @@ static void forEachListener(
 int SGPropertyNode::nListeners() const
 {
  if (!_listeners) return 0;
+  std::lock_guard<std::recursive_mutex> lock(_listeners->_rmutex);
  int   n = 0;
  for (auto listener: _listeners->_items) {
    if (listener)   n += 1;
@@ -76,7 +76,7 @@
 #include <simgear/structure/SGExpression.hxx>
 #include <simgear/props/props_io.hxx>
 #include <simgear/props/vectorPropTemplates.hxx>
-
+#include <simgear/debug/ErrorReportingCallback.hxx>
 #include <simgear/io/iostreams/sgstream.hxx>

 namespace simgear
@@ -928,7 +928,8 @@ void ShaderProgramBuilder::buildAttribute(Effect* effect, Pass* pass,
        // FIXME orig: const string& shaderName = shaderKey.first;
        string shaderName = shaderKey.first;
        Shader::Type stype = (Shader::Type)shaderKey.second;
-        if (getPropertyRoot()->getBoolValue("/sim/version/compositor-support", false) &&
+        const bool compositorEnabled = getPropertyRoot()->getBoolValue("/sim/version/compositor-support", false);
+        if (compositorEnabled &&
            shaderName.substr(0, shaderName.find("/")) == "Shaders") {
            shaderName = "Compositor/" + shaderName;
        }
@@ -936,7 +937,9 @@ void ShaderProgramBuilder::buildAttribute(Effect* effect, Pass* pass,
        if (fileName.empty())
        {
            SG_LOG(SG_INPUT, SG_ALERT, "Could not locate shader" << shaderName);
-
+            if (!compositorEnabled) {
+                reportError("Missing shader", shaderName);
+            }

            throw BuilderException(string("couldn't find shader ") +
                shaderName);
@@ -1481,10 +1484,8 @@ void mergeSchemesFallbacks(Effect *effect, const SGReaderWriterOptions *options)

 // Walk the techniques property tree, building techniques and
 // passes.
-static std::mutex  realizeTechniques_lock;
 bool Effect::realizeTechniques(const SGReaderWriterOptions* options)
 {
-    std::lock_guard<std::mutex> g(realizeTechniques_lock);
    if (getPropertyRoot()->getBoolValue("/sim/version/compositor-support", false))
        mergeSchemesFallbacks(this, options);

@@ -273,11 +273,19 @@ bool setAttrs(const TexTuple& attrs, Texture* tex,
        options->setLoadOriginHint(SGReaderWriterOptions::LoadOriginHint::ORIGIN_EFFECTS_NORMALIZED);
    else
        options->setLoadOriginHint(SGReaderWriterOptions::LoadOriginHint::ORIGIN_EFFECTS);
-#if OSG_VERSION_LESS_THAN(3,4,2)
-    result = osgDB::readImageFile(imageName, options);
-#else
-    result = osgDB::readRefImageFile(imageName, options);
-#endif
+
+    try {
+    #if OSG_VERSION_LESS_THAN(3,4,2)
+        result = osgDB::readImageFile(imageName, options);
+    #else
+        result = osgDB::readRefImageFile(imageName, options);
+    #endif
+    } catch (std::bad_alloc& ba) {
+        SG_LOG(SG_GL, SG_ALERT, "Bad allocation loading:" << imageName);
+        // todo: report low memory warning
+        return false;
+    }
+
    options->setLoadOriginHint(origLOH);
    osg::ref_ptr<osg::Image> image;
    if (result.success())
@@ -393,6 +393,10 @@ ModelRegistry::readImage(const string& fileName,
                        isEffect = true;
 //                        can_compress = false;
                    }
+                    else if (sgoptC && !transparent && sgoptC->getLoadOriginHint() == SGReaderWriterOptions::LoadOriginHint::ORIGIN_CANVAS) {
+                        SG_LOG(SG_IO, SG_INFO, "From Canvas " + absFileName + " will generate mipmap only");
+                        can_compress = false;
+                    }
                    if (can_compress)
                    {
                        std::string pot_message;
@@ -1135,16 +1135,8 @@ void SpinAnimCallback::operator()(osg::Node* node, osg::NodeVisitor* nv)
        double intPart;
        double rot = modf(rotation, &intPart);
        double angle = rot * 2.0 * osg::PI;
-        const SGVec3d& sgcenter = transform->getCenter();
-        const SGVec3d& sgaxis = transform->getAxis();
-        Matrixd mat = Matrixd::translate(-sgcenter[0], -sgcenter[1], -sgcenter[2])
-            * Matrixd::rotate(angle, sgaxis[0], sgaxis[1], sgaxis[2])
-            * Matrixd::translate(sgcenter[0], sgcenter[1], sgcenter[2])
-            * *cv->getModelViewMatrix();
-        ref_ptr<RefMatrix> refmat = new RefMatrix(mat);
-        cv->pushModelViewMatrix(refmat.get(), transform->getReferenceFrame());
+        transform->setAngleRad(angle);
        traverse(transform, nv);
-        cv->popModelViewMatrix();
    } else {
        traverse(transform, nv);
    }
@@ -294,7 +294,7 @@ ReaderWriterSPT::createTree(const BucketBox& bucketBox, const LocalOptions& opti
 osg::ref_ptr<osg::Node>
 ReaderWriterSPT::createPagedLOD(const BucketBox& bucketBox, const LocalOptions& options) const
 {
-    osg::PagedLOD* pagedLOD = new osg::PagedLOD;
+    osg::ref_ptr<osg::PagedLOD> pagedLOD = new osg::PagedLOD;

    pagedLOD->setCenterMode(osg::PagedLOD::USER_DEFINED_CENTER);
    SGSpheref sphere = bucketBox.getBoundingSphere();
@@ -212,7 +212,8 @@ struct ReaderWriterSTG::_ModelBin {
            STGObjectsQuadtree quadtree((GetModelLODCoord()), (AddModelLOD()));
            quadtree.buildQuadTree(_objectStaticList.begin(), _objectStaticList.end());
            osg::ref_ptr<osg::Group> group = quadtree.getRoot();
-            group->setName("STG-group-A");
+            string group_name = string("STG-group-A ").append(_bucket.gen_index_str());
+            group->setName(group_name);
            group->setDataVariance(osg::Object::STATIC);

            simgear::AirportSignBuilder signBuilder(_options->getMaterialLib(), _bucket.get_center());
@@ -586,10 +587,12 @@ struct ReaderWriterSTG::_ModelBin {
    {
        osg::ref_ptr<SGReaderWriterOptions> options;
        options = SGReaderWriterOptions::copyOrCreate(opt);
+        float pagedLODExpiry = atoi(options->getPluginStringData("SimGear::PAGED_LOD_EXPIRY").c_str());

        osg::ref_ptr<osg::Group> terrainGroup = new osg::Group;
        terrainGroup->setDataVariance(osg::Object::STATIC);
-        terrainGroup->setName("terrain");
+        std::string terrain_name = string("terrain ").append(bucket.gen_index_str());
+        terrainGroup->setName(terrain_name);

        if (_foundBase) {
            for (auto stgObject : _objectList) {
@@ -637,11 +640,13 @@ struct ReaderWriterSTG::_ModelBin {
        } else {
            osg::PagedLOD* pagedLOD = new osg::PagedLOD;
            pagedLOD->setCenterMode(osg::PagedLOD::USE_BOUNDING_SPHERE_CENTER);
-            pagedLOD->setName("pagedObjectLOD");
+            std::string name = string("pagedObjectLOD ").append(bucket.gen_index_str());
+            pagedLOD->setName(name);

            // This should be visible in any case.
            // If this is replaced by some lower level of detail, the parent LOD node handles this.
            pagedLOD->addChild(terrainGroup, 0, std::numeric_limits<float>::max());
+            pagedLOD->setMinimumExpiryTime(0, pagedLODExpiry);

            // we just need to know about the read file callback that itself holds the data
            osg::ref_ptr<DelayLoadReadFileCallback> readFileCallback = new DelayLoadReadFileCallback;
@@ -658,10 +663,11 @@ struct ReaderWriterSTG::_ModelBin {

            // Objects may end up displayed up to 2x the object range.
            pagedLOD->setRange(pagedLOD->getNumChildren(), 0, 2.0 * _object_range_rough);
+            pagedLOD->setMinimumExpiryTime(pagedLOD->getNumChildren(), pagedLODExpiry);
            pagedLOD->setRadius(SG_TILE_RADIUS);
-            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile PagedLOD Center: " << pagedLOD->getCenter().x() << "," << pagedLOD->getCenter().y() << "," << pagedLOD->getCenter().z() );
-            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile PagedLOD Range: " << (2.0 * _object_range_rough));
-            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile PagedLOD Radius: " << SG_TILE_RADIUS);
+            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile " << bucket.gen_index_str() << " PagedLOD Center: " << pagedLOD->getCenter().x() << "," << pagedLOD->getCenter().y() << "," << pagedLOD->getCenter().z() );
+            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile " << bucket.gen_index_str() << " PagedLOD Range: " << (2.0 * _object_range_rough));
+            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile " << bucket.gen_index_str() << " PagedLOD Radius: " << SG_TILE_RADIUS);
            return pagedLOD;
        }
    }
@@ -441,6 +441,10 @@ typedef QuadTreeBuilder<LOD*, SGBuildingBin::BuildingInstance, MakeBuildingLeaf,
      if (hash_pos != std::string::npos)
          line.resize(hash_pos);

+      if (line.empty()) {
+          continue; // skip blank lines
+      }
+
      // and process further
      std::stringstream in(line);

@@ -41,6 +41,7 @@
 #include <simgear/scene/material/EffectGeode.hxx>
 #include <simgear/scene/material/mat.hxx>
 #include <simgear/scene/material/matlib.hxx>
+#include <simgear/scene/model/BoundingVolumeBuildVisitor.hxx>
 #include <simgear/scene/util/OsgMath.hxx>
 #include <simgear/scene/util/VectorArrayAdapter.hxx>
 #include <simgear/scene/util/SGNodeMasks.hxx>
@@ -340,5 +341,10 @@ osg::Node* SGOceanTile(const SGBucket& b, SGMaterialLib *matlib, int latPoints,
    transform->addChild(geode);
    transform->setNodeMask( ~(simgear::CASTSHADOW_BIT | simgear::MODELLIGHT_BIT) );

+    // Create a BVH at this point.  This is normally provided by the file loader, but as we create the 
+    // geometry programmatically, no file loader is involved.
+    BoundingVolumeBuildVisitor bvhBuilder(false);
+    transform->accept(bvhBuilder);
+    
    return transform;
 }
@@ -319,7 +319,9 @@ public:

    bool isDirActive(const std::string& path) const;

-private:
+    void setCachePath(const SGPath &p) { _persistentCachePath = p; }
+
+  private:
    void incrementCacheHits()
    {
        std::lock_guard<std::mutex> g(_stateLock);
@@ -332,6 +334,10 @@ private:
    void runInternal();
    void updateSyncSlot(SyncSlot& slot);

+    void beginSyncAirports(SyncSlot& slot);
+    void beginSyncTile(SyncSlot& slot);
+    void beginNormalSync(SyncSlot& slot);
+
    void drainWaitingTiles();

    // commond helpers between both internal and external models
@@ -341,6 +347,9 @@ private:
    void fail(SyncItem failedItem);
    void notFound(SyncItem notFoundItem);

+    void initCompletedTilesPersistentCache();
+    void writeCompletedTilesPersistentCache() const;
+
    HTTP::Client _http;
    SyncSlot _syncSlots[NUM_SYNC_SLOTS];

@@ -530,6 +539,7 @@ void SGTerraSync::WorkerThread::run()
        _running = true;
    }

+    initCompletedTilesPersistentCache();
    runInternal();

    {
@@ -541,6 +551,7 @@ void SGTerraSync::WorkerThread::run()
 void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
 {
    if (slot.repository.get()) {
+        slot.repository->process();
        if (slot.repository->isDoingSync()) {
 #if 1
            if (slot.stamp.elapsedMSec() > (int)slot.nextWarnTimeout) {
@@ -561,6 +572,15 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
            notFound(slot.currentItem);
        } else if (res != HTTPRepository::REPO_NO_ERROR) {
            fail(slot.currentItem);
+
+            // in case the Airports_archive download fails, create the
+            // directory, so that next sync, we do a manual sync
+            if ((slot.currentItem._type == SyncItem::AirportData) && slot.isNewDirectory) {
+                SG_LOG(SG_TERRASYNC, SG_ALERT, "Failed to download Airports_archive, will download discrete files next time");
+                simgear::Dir d(_local_dir + "/Airports");
+                d.create(0755);
+                _completedTiles.erase(slot.currentItem._dir);
+            }
        } else {
            updated(slot.currentItem, slot.isNewDirectory);
            SG_LOG(SG_TERRASYNC, SG_DEBUG, "sync of " << slot.repository->baseUrl() << " finished ("
@@ -582,23 +602,14 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
        SGPath path(_local_dir);
        path.append(slot.currentItem._dir);
        slot.isNewDirectory = !path.exists();
-        if (slot.isNewDirectory) {
-            int rc = path.create_dir( 0755 );
-            if (rc) {
-                SG_LOG(SG_TERRASYNC,SG_ALERT,
-                       "Cannot create directory '" << path << "', return code = " << rc );
-                fail(slot.currentItem);
-                return;
-            }
-        } // of creating directory step
+        const auto type = slot.currentItem._type;

-        slot.repository.reset(new HTTPRepository(path, &_http));
-        slot.repository->setBaseUrl(_httpServer + "/" + slot.currentItem._dir);
-
-        if (_installRoot.exists()) {
-            SGPath p = _installRoot;
-            p.append(slot.currentItem._dir);
-            slot.repository->setInstalledCopyPath(p);
+        if (type == SyncItem::AirportData) {
+            beginSyncAirports(slot);
+        } else if (type == SyncItem::Tile) {
+            beginSyncTile(slot);
+        } else {
+            beginNormalSync(slot);
        }

        try {
@@ -621,6 +632,96 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
    }
 }

+void SGTerraSync::WorkerThread::beginSyncAirports(SyncSlot& slot)
+{
+    if (!slot.isNewDirectory) {
+        beginNormalSync(slot);
+        return;
+    }
+
+    SG_LOG(SG_TERRASYNC, SG_INFO, "doing Airports download via tarball");
+
+    // we want to sync the 'root' TerraSync dir, but not all of it, just
+    // the Airports_archive.tar.gz file so we use our TerraSync local root
+    // as the path (since the archive will add Airports/)
+    slot.repository.reset(new HTTPRepository(_local_dir, &_http));
+    slot.repository->setBaseUrl(_httpServer);
+
+    // filter callback to *only* sync the Airport_archive tarball,
+    // and ensure no other contents are touched
+    auto f = [](const HTTPRepository::SyncItem& item) {
+        if (!item.directory.empty())
+            return false;
+        return (item.filename.find("Airports_archive.") == 0);
+    };
+
+    slot.repository->setFilter(f);
+}
+
+void SGTerraSync::WorkerThread::beginSyncTile(SyncSlot& slot)
+{
+    // avoid 404 requests by doing a sync which excludes all paths
+    // except our tile path. In the case of a missing 1x1 tile, we will
+    // stop becuase all directories are filtered out, which is what we want
+
+    auto comps = strutils::split(slot.currentItem._dir, "/");
+    if (comps.size() != 3) {
+        SG_LOG(SG_TERRASYNC, SG_ALERT, "Bad tile path:" << slot.currentItem._dir);
+        beginNormalSync(slot);
+        return;
+    }
+
+    const auto tileCategory = comps.front();
+    const auto tenByTenDir = comps.at(1);
+    const auto oneByOneDir = comps.at(2);
+
+    const auto path = SGPath::fromUtf8(_local_dir) / tileCategory;
+    slot.repository.reset(new HTTPRepository(path, &_http));
+    slot.repository->setBaseUrl(_httpServer + "/" + tileCategory);
+
+    if (_installRoot.exists()) {
+      SGPath p = _installRoot / tileCategory;
+      slot.repository->setInstalledCopyPath(p);
+    }
+
+    const auto dirPrefix = tenByTenDir + "/" + oneByOneDir;
+
+    // filter callback to *only* sync the 1x1 dir we want, if it exists
+    // if doesn't, we'll simply stop, which is what we want
+    auto f = [tenByTenDir, oneByOneDir, dirPrefix](const HTTPRepository::SyncItem& item) {
+        // only allow the specific 10x10 and 1x1 dirs we want
+        if (item.directory.empty()) {
+            return item.filename == tenByTenDir;
+        } else if (item.directory == tenByTenDir) {
+            return item.filename == oneByOneDir;
+        }
+
+        // allow arbitrary children below dirPrefix, including sub-dirs
+        if (item.directory.find(dirPrefix) == 0) {
+            return true;
+        }
+
+        SG_LOG(SG_TERRASYNC, SG_ALERT, "Tile sync: saw weird path:" << item.directory << " file " << item.filename);
+        return false;
+    };
+
+    slot.repository->setFilter(f);
+}
+
+void SGTerraSync::WorkerThread::beginNormalSync(SyncSlot& slot)
+{
+    SGPath path(_local_dir);
+    path.append(slot.currentItem._dir);
+    slot.repository.reset(new HTTPRepository(path, &_http));
+    slot.repository->setBaseUrl(_httpServer + "/" + slot.currentItem._dir);
+
+    if (_installRoot.exists()) {
+      SGPath p = _installRoot;
+      p.append(slot.currentItem._dir);
+      slot.repository->setInstalledCopyPath(p);
+    }
+}
+
 void SGTerraSync::WorkerThread::runInternal()
 {
    while (!_stop) {
@@ -714,6 +815,7 @@ void SGTerraSync::WorkerThread::fail(SyncItem failedItem)
    _state._fail_count++;
    failedItem._status = SyncItem::Failed;
    _freshTiles.push_back(failedItem);
+    // not we also end up here for partial syncs
    SG_LOG(SG_TERRASYNC,SG_INFO,
           "Failed to sync'" << failedItem._dir << "'");
    _completedTiles[ failedItem._dir ] = now + UpdateInterval::FailedAttempt;
@@ -730,6 +832,7 @@ void SGTerraSync::WorkerThread::notFound(SyncItem item)
    item._status = SyncItem::NotFound;
    _freshTiles.push_back(item);
    _notFoundItems[ item._dir ] = now + UpdateInterval::SuccessfulAttempt;
+    writeCompletedTilesPersistentCache();
 }

 void SGTerraSync::WorkerThread::updated(SyncItem item, bool isNewDirectory)
@@ -750,6 +853,8 @@ void SGTerraSync::WorkerThread::updated(SyncItem item, bool isNewDirectory)
        _freshTiles.push_back(item);
        _completedTiles[ item._dir ] = now + UpdateInterval::SuccessfulAttempt;
    }
+
+    writeCompletedTilesPersistentCache();
 }

 void SGTerraSync::WorkerThread::drainWaitingTiles()
@@ -803,6 +908,68 @@ bool SGTerraSync::WorkerThread::isDirActive(const std::string& path) const
    return false;
 }

+void SGTerraSync::WorkerThread::initCompletedTilesPersistentCache() {
+  if (!_persistentCachePath.exists()) {
+    return;
+  }
+
+  SGPropertyNode_ptr cacheRoot(new SGPropertyNode);
+  time_t now = time(0);
+
+  try {
+    readProperties(_persistentCachePath, cacheRoot);
+  } catch (sg_exception &e) {
+    SG_LOG(SG_TERRASYNC, SG_INFO, "corrupted persistent cache, discarding");
+    return;
+  }
+
+  for (int i = 0; i < cacheRoot->nChildren(); ++i) {
+    SGPropertyNode *entry = cacheRoot->getChild(i);
+    bool isNotFound = (strcmp(entry->getName(), "not-found") == 0);
+    string tileName = entry->getStringValue("path");
+    time_t stamp = entry->getIntValue("stamp");
+    if (stamp < now) {
+      continue;
+    }
+
+    if (isNotFound) {
+      _notFoundItems[tileName] = stamp;
+    } else {
+      _completedTiles[tileName] = stamp;
+    }
+  }
+}
+
+void SGTerraSync::WorkerThread::writeCompletedTilesPersistentCache() const {
+  // cache is disabled
+  if (_persistentCachePath.isNull()) {
+    return;
+  }
+
+  sg_ofstream f(_persistentCachePath, std::ios::trunc);
+  if (!f.is_open()) {
+    return;
+  }
+
+  SGPropertyNode_ptr cacheRoot(new SGPropertyNode);
+  TileAgeCache::const_iterator it = _completedTiles.begin();
+  for (; it != _completedTiles.end(); ++it) {
+    SGPropertyNode *entry = cacheRoot->addChild("entry");
+    entry->setStringValue("path", it->first);
+    entry->setIntValue("stamp", it->second);
+  }
+
+  it = _notFoundItems.begin();
+  for (; it != _notFoundItems.end(); ++it) {
+    SGPropertyNode *entry = cacheRoot->addChild("not-found");
+    entry->setStringValue("path", it->first);
+    entry->setIntValue("stamp", it->second);
+  }
+
+  writeProperties(f, cacheRoot, true /* write_all */);
+  f.close();
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 // SGTerraSync ////////////////////////////////////////////////////////////////
 ///////////////////////////////////////////////////////////////////////////////
@@ -882,10 +1049,17 @@ void SGTerraSync::reinit()
 #else
        _workerThread->setDNSDN( _terraRoot->getStringValue("dnsdn","terrasync.flightgear.org") );
 #endif
-        _workerThread->setLocalDir(_terraRoot->getStringValue("scenery-dir",""));
+
+        SGPath sceneryRoot{_terraRoot->getStringValue("scenery-dir", "")};
+        _workerThread->setLocalDir(sceneryRoot.utf8Str());

        SGPath installPath(_terraRoot->getStringValue("installation-dir"));
        _workerThread->setInstalledDir(installPath);
+
+        if (_terraRoot->getBoolValue("enable-persistent-cache", true)) {
+          _workerThread->setCachePath(sceneryRoot / "RecheckCache");
+        }
+
        _workerThread->setCacheHits(_terraRoot->getIntValue("cache-hit", 0));

        if (_workerThread->start())
@@ -1012,21 +1186,10 @@ bool SGTerraSync::isIdle() {return _workerThread->isIdle();}

 void SGTerraSync::syncAirportsModels()
 {
-    static const char* bounds = "MZAJKL"; // airport sync order: K-L, A-J, M-Z
-    // note "request" method uses LIFO order, i.e. processes most recent request first
-    for( unsigned i = 0; i < strlen(bounds)/2; i++ )
-    {
-        for ( char synced_other = bounds[2*i]; synced_other <= bounds[2*i+1]; synced_other++ )
-        {
-            ostringstream dir;
-            dir << "Airports/" << synced_other;
-            SyncItem w(dir.str(), SyncItem::AirportData);
-            _workerThread->request( w );
-        }
-    }
-
-    SyncItem w("Models", SyncItem::SharedModels);
-    _workerThread->request( w );
+  SyncItem w("Airports", SyncItem::AirportData);
+  SyncItem a("Models", SyncItem::SharedModels);
+  _workerThread->request(w);
+  _workerThread->request(a);
 }

 string_list SGTerraSync::getSceneryPathSuffixes() const
@@ -46,6 +46,7 @@ public:
        ORIGIN_EFFECTS,
        ORIGIN_EFFECTS_NORMALIZED,
        ORIGIN_SPLASH_SCREEN,
+        ORIGIN_CANVAS,
    };

    //SGReaderWriterOptions* cloneOptions(const osg::CopyOp& copyop = osg::CopyOp::SHALLOW_COPY) const { return static_cast<SGReaderWriterOptions*>(clone(copyop)); }
@@ -380,7 +380,8 @@ namespace simgear
 ALvoid* loadWAVFromFile(const SGPath& path, unsigned int& format, ALsizei& size, ALfloat& freqf, unsigned int& block_align)
 {
  if (!path.exists()) {
-    throw sg_io_exception("loadWAVFromFile: file not found", path);
+    SG_LOG(SG_IO, SG_DEV_ALERT, "loadWAVFromFile: file not found:" << path);
+    return nullptr;
  }

  Buffer b;
@@ -395,13 +396,15 @@ ALvoid* loadWAVFromFile(const SGPath& path, unsigned int& format, ALsizei& size,
  fd = gzopen(ps.c_str(), "rb");
 #endif
  if (!fd) {
-    throw sg_io_exception("loadWAVFromFile: unable to open file", path);
+    SG_LOG(SG_IO, SG_DEV_ALERT, "loadWAVFromFile: unable to open file:" << path);
+    return nullptr;
  }

  try {
      loadWavFile(fd, &b);
  } catch (sg_exception& e) {
-      throw sg_io_exception(e.getFormattedMessage() + "\nfor: " + path.str());
+      SG_LOG(SG_IO, SG_DEV_ALERT, "loadWAVFromFile:" << e.getFormattedMessage() << "\nfor: " << path);
+      return nullptr;
  }

  ALvoid* data = b.data;
@@ -813,13 +813,14 @@ bool SGSoundMgr::load( const std::string &samplepath,

    auto data = simgear::loadWAVFromFile(samplepath, format, size, freqf, blocksz);
    freq = (ALsizei)freqf;
-    if (data == nullptr) {
-        throw sg_io_exception("Failed to load wav file", sg_location(samplepath));
+    if (!data) {
+        return false;
    }

    if (format == AL_FORMAT_STEREO8 || format == AL_FORMAT_STEREO16) {
         free(data);
-        throw sg_io_exception("Warning: STEREO files are not supported for 3D audio effects: " + samplepath);
+        SG_LOG(SG_IO, SG_DEV_ALERT, "Warning: STEREO files are not supported for 3D audio effects: " << samplepath);
+        return false;
    }

    *dbuf = (void *)data;
@@ -84,7 +84,7 @@ SGXmlSound::~SGXmlSound()
    _pitch.clear();
 }

-void
+bool
 SGXmlSound::init( SGPropertyNode *root,
                  SGPropertyNode *node,
                  SGSampleGroup *sgrp,
@@ -315,8 +315,8 @@ SGXmlSound::init( SGPropertyNode *root,
   string soundFileStr = node->getStringValue("path", "");
   _sample = new SGSoundSample(soundFileStr.c_str(), path);
   if (!_sample->file_path().exists()) {
-      throw sg_io_exception("XML sound: couldn't find file: '" + soundFileStr + "'");
-      return;
+      SG_LOG(SG_SOUND, SG_WARN, "XML sound: couldn't find file: '" + soundFileStr + "'");
+      return false;
   }

   _sample->set_relative_position( offset_pos );
@@ -328,6 +328,8 @@ SGXmlSound::init( SGPropertyNode *root,
   _sample->set_volume( v );
   _sample->set_pitch( p );
   _sgrp->add( _sample, _name );
+
+   return true;
 }

 void
@@ -107,7 +107,7 @@ public:
   * @param avionics    A pointer to the pre-initialized avionics sample group.
   * @param path        The path where the audio files remain.
   */
-  virtual void init( SGPropertyNode *root,
+  virtual bool init( SGPropertyNode *root,
                     SGPropertyNode *child,
                     SGSampleGroup *sgrp,
                     SGSampleGroup *avionics,
@@ -0,0 +1,86 @@
+
+
+#if 0
+
+set bindings for action seperate from defintion ?
+    - in XML, especially aircraft XML
+
+define actions from command / Nasal
+add behaviours from Nasal 
+
+define keymapping
+    - manager of keybindings defined against actions?
+    - for a toggle or enum, define behaviour
+        - each key repeat cycles
+        - alternate key to go the other way (G/shift-G)
+
+release bindings for momentary actions:
+    button up / key-up
+
+send activate, release
+
+    send deactivate, release for 'alternate' action
+
+#endif
+
+
+void SGAction::setValueExpression()
+{
+    // watch all the properties
+}
+
+void SGAction::setValueCondition()
+{
+    // 
+}
+
+void SGAction::updateProperties()
+{
+    //
+    _node->setBoolValue("enabled", isEnabled());
+    switch (_type) {
+    case Momentary:
+    case Toggle:
+        _node->setBoolValue("value", getValue());
+        break;
+
+    case Enumerated: 
+        if (!_valueEnumeration.empty()) {
+            // map to the string value
+            _node->setStringValue("value", _valueEnumeration.at(getValue()));
+        } else {
+            // set as an integer
+            _node->setIntValue("value", getValue());
+        }
+    }
+
+    // set description
+}
+
+bool SGAction::isEnabled()
+{
+    if (_enableCondition) {
+
+    } else {
+        return _enabled;
+    }
+
+    updateProperties();
+}
+
+int SGAction::getValue()
+{
+    if (type == Enumerated) {
+        if (_valueExpression) {
+            // invoke it
+        }
+    } else {
+        if (_valueCondition) {
+            return _valueCondition.test();
+        }
+    }
+
+    return _value;
+}
+
+// commands enable-action, disable-action
@@ -441,7 +441,7 @@ void StateMachine::initFromPlist(SGPropertyNode* desc, SGPropertyNode* root)
        std::string nm = stateDesc->getStringValue("name");

        if (nm.empty()) {
-          SG_LOG(SG_GENERAL, SG_ALERT, "No name found for state in branch " << path);
+          SG_LOG(SG_GENERAL, SG_DEV_ALERT, "No name found for state in branch " << path);
          throw sg_exception("No name element in state");
        }

@@ -464,8 +464,8 @@ void StateMachine::initFromPlist(SGPropertyNode* desc, SGPropertyNode* root)
        std::string target_id = tDesc->getStringValue("target");

        if (nm.empty()) {
-          SG_LOG(SG_GENERAL, SG_ALERT, "No name found for transition in branch " << path);
-          throw sg_exception("No name element in transition");
+          SG_LOG(SG_GENERAL, SG_DEV_WARN, "No name found for transition in branch " << path);
+          nm = "transition-to-" + target_id;
        }

        if (target_id.empty()) {
@@ -30,9 +30,10 @@
 #include "subsystem_mgr.hxx"
 #include "commands.hxx"

-#include <simgear/props/props.hxx>
-#include <simgear/math/SGMath.hxx>
 #include "SGSmplstat.hxx"
+#include <simgear/debug/ErrorReportingCallback.hxx>
+#include <simgear/math/SGMath.hxx>
+#include <simgear/props/props.hxx>

 const int SG_MAX_SUBSYSTEM_EXCEPTIONS = 4;
 const char SUBSYSTEM_NAME_SEPARATOR = '.';
@@ -828,8 +829,18 @@ SGSubsystemGroup::Member::update (double delta_time_sec)
      if (++exceptionCount > SG_MAX_SUBSYSTEM_EXCEPTIONS) {
        SG_LOG(SG_GENERAL, SG_ALERT, "(exceptionCount=" << exceptionCount <<
          ", suspending)");
+        simgear::reportError("suspending subsystem after too many errors:" + name);
        subsystem->suspend();
      }
+    } catch (std::bad_alloc& ba) {
+        // attempting to track down source of these on Sentry.io
+        SG_LOG(SG_GENERAL, SG_ALERT, "caught bad_alloc processing subsystem:" << name);
+        simgear::reportError("caught bad_alloc processing subsystem:" + name);
+
+        if (++exceptionCount > SG_MAX_SUBSYSTEM_EXCEPTIONS) {
+            SG_LOG(SG_GENERAL, SG_ALERT, "(exceptionCount=" << exceptionCount << ", suspending)");
+            subsystem->suspend();
+        }
    }
 }
Author	SHA1	Message	Date
Automatic Release Builder	ca5f66da9f	new version: 2020.3.6	2021-01-20 13:01:25 +00:00
James Turner	92d053d850	Tolerate blank lines in buildings lists	2021-01-20 11:35:33 +00:00
Colin Geniet	001ae80723	Animations: Fix spin for Compositor lights The 'spin' animation has a strange behaviour: it pushes a temporary rotation matrix on the transformation stack during cull traversal, and removes it thereafter. Thus, the rotation matrix is missing outside of cull traversal, e.g. when the position of Compositor lights is computed. The same issue was fixed for the 'rotate' animation by commit `e202d4e4a5` (this mentions broken 'picking' animations as a different manifestation of the same issue). Fix this by setting the angle of the persistent SGRotateTransform, instead of creating a temporary rotation matrix.	2021-01-15 15:45:26 +00:00
James Turner	58d9a6d0b5	Sentry: adjust reporting of missing shaders Reduce noise on the backend when a shader is missing, by special- casing how we report it.	2021-01-05 12:58:46 +00:00
Scott Giese	261316cb45	Support Boost v1.75 Resolved a breaking change.	2021-01-02 21:57:54 -06:00
Stuart Buchanan	2ad4d2e672	Fix hang on "Loading Scenery" over the sea Previously attempting to start on an ocean tile with static models present cause FG to spin on "Loading Scenery". This was caused by not creating a BVH for an ocean tile in the specific case where there was STG models, due to a PagedLOD node being inserted in the scene graph. So FG never thought the scenery was loaded sufficient to place the aircraft. By explicitly creating a BVH for ocean tiles the problem is fixed. Candidate for 2020.X	2021-01-01 22:41:47 +00:00
James Turner	1d9fa929fa	Launhcer: fix crashes adding a catalog Traversing a container which is modified causes crashes, take a copy during traversal for firePackageStatus. Sentry-Id: FLIGHTGEAR-CJF Sentry-Id: FLIGHTGEAR-CJ5	2021-01-01 22:41:42 +00:00
James Turner	26bb6236a0	Effect builder: catch allocation failures Fail on bad_alloc of reading texture images, instead of crashing.	2020-12-28 15:08:49 +00:00
Scott Giese	b2acf5a623	fix: sky and cockpit to turn black Metar sky condition without base layer height (e.g. FEW///) is the cause of this issue. Randomize the base layer height when the actual height is unknown.	2020-12-26 10:53:49 -06:00
James Turner	bea88bb3a9	Fix BTG error reporting Use gzerror, not strerror, for these codes.	2020-12-21 15:31:36 +00:00
Automatic Release Builder	def2af2bdd	new version: 2020.3.5	2020-12-17 13:48:00 +00:00
Fernando García Liñán	6f6d705f22	Add support for anisotropic filtering in Canvas	2020-12-17 13:00:01 +00:00
Richard Harrison	e9c33104d3	DDS-TC add option to exclude all Canvas orignated images.	2020-12-17 12:59:26 +00:00
Richard Harrison	985897f8ba	Emesary : change to use scoped lock - scoped locks using lock_guard are better - because of this the exception handler can also be removed; although this was originally intended to manage the locks it never did work properly with C++ exceptions.	2020-12-17 12:57:53 +00:00
Richard Harrison	4251b28e88	Emesary: add missing property initialisation to constructor	2020-12-17 12:56:49 +00:00
James Turner	76540a211c	SGGeod: add static constructor of an invalid Geod Use this to allow explicitly initializing a value which isValid will return false for.	2020-12-17 12:55:30 +00:00
James Turner	904fc5a7dd	Fix Nasal GC errors on tests/reset Ensure the Context temps are cleared, and when recycling an naCode, ensure old values are cleared explicily. Sentry-Id: FLIGHTGEAR-Y	2020-12-17 12:55:25 +00:00
Stuart Buchanan	4aebc159d5	Use ref_ptr for ReaderWriterSPT	2020-12-07 17:18:32 +00:00
Richard Harrison	f89227fc1a	Remove mutex lock on realizeTechniques This was causing a deadlock - and I can't quite remember what problem it was intended to solve so it is best to remove it.	2020-12-07 17:17:44 +00:00
James Turner	f50f383cc0	Reporting of std::bad_alloc in Subsystem::update Trying to trace down our bad-alloc exception, starting with the simplest place for now.	2020-12-07 17:16:58 +00:00
James Turner	706ab387de	Add reporting callback option to SimGear Allows us to trigger an error logging callback explicitly, which can be used to drive Sentry.io on the FlightGear side.	2020-12-07 17:16:47 +00:00
James Turner	cddfdb7d1d	TerraSync: stronger fix for handling 0-byte files Change logic so we create an empty file for such cases, i.e exactly matching the repository. This simplifies logic in downstream code, compared with not creating a local file. Add a test-case to cover this Modify TerraSync to detect a failure of Airports_archive downloading, and fall back to file-by-file updating.	2020-12-07 17:15:51 +00:00
James Turner	3c64578848	HTTPRepository: don’t crash on empty files Fix some additional crash cases around 0-length files	2020-12-07 17:15:38 +00:00
James Turner	e1fe9b45e0	Unzip: adjust error reporting mechanism Don’t use local exception throw+catch to report failures in extracting a zip archive, since this generates noise in Sentry.	2020-12-07 17:15:30 +00:00
James Turner	8fdc1d306f	State-machines: don’t require name for transitions Allow anonymous transitions, since the name is purely informational (unlike for states). Sentry-Id: FLIGHTGEAR-9H	2020-12-07 17:15:17 +00:00
Automatic Release Builder	6e0c39bb68	new version: 2020.3.4	2020-11-30 11:17:52 +00:00
Julian Smith	f20b416cfe	simgear/props/props.cxx: use rmutex to protect SGPropertyNodeListeners. Also added asserts to check _num_iterators always >= 0.	2020-11-29 20:16:46 +00:00
Scott Giese	47e06b5216	METAR: mitigate wind sensor failures	2020-11-29 16:25:01 +00:00
James Turner	bfcdf22705	TerraSync: fix crashes with null file return Not sure how this is happening, but, check for a null file object in FileGetRequest::onDone.	2020-11-29 16:23:44 +00:00
Stuart Buchanan	d0db407faa	Set minimum expiry time on STG nodes.	2020-11-29 16:23:40 +00:00
Automatic Release Builder	d95b1c0441	new version: 2020.3.3	2020-11-12 11:14:23 +00:00
James Turner	837ba86d57	DNSClient: own requests, and cancel them on timeout Fixes crashes where a request times-out, but then is completed by UDN sometime afterwards, with a free-d object. Have the DNS::Client own requests, and be able to retrieve the udns_query to cancel them, in the timeout case. Fixes a couple of Sentry reports.	2020-11-12 09:39:58 +00:00
Automatic Release Builder	0cb1b463e1	Catalogs: fix ownership of new Catalogs When doing the initial download of a Catalog, ensure we still keep an owning ref to it.	2020-11-11 21:18:57 +00:00
Automatic Release Builder	8a772c8edd	new version: 2020.3.2	2020-11-04 22:08:28 +00:00
James Turner	03bdad0a10	Try to capture cause of BTG load failures	2020-11-04 11:52:58 +00:00
James Turner	537776e1f8	TerraSync: fix local-file copying Avoid downloading data corresponding to files shipped in FGData.	2020-11-04 11:22:48 +00:00
James Turner	f34a4a304e	Untar: log error details when output create fails Trying to understand why writing to the output files fails for some users. Sentry-Id: FLIGHTGEAR-SS	2020-11-03 21:26:03 +00:00
James Turner	a59c4e2c8b	TerraSync: make tarball extraction asynchronous	2020-11-03 17:49:19 +00:00
Julian Smith	46f4967f6e	Allow use of old zlib-1.2.3 on OpenBSD. As of 2020-08-01, OpenBSD's system zlib is 1.2.3 which doesn't have gzoffset(). However we can get away with this by making gzfilebuf::approxOffset() always return zero.	2020-11-01 11:12:00 +00:00
James Turner	9305417207	Terrasync: tarball extraction, use larger buffer Use a 1MB buffer, 2kByte is very 1979 :)	2020-11-01 11:11:31 +00:00
James Turner	11da8b33f9	TerraSync: switch to per-directory hash caching Avoid a central hash cache becoming enormous, now we use the selective download scheme for the tile dirs. Hash name is changed to co-exist with older versions.	2020-11-01 11:11:27 +00:00
James Turner	c7b320eb55	Fix Airports/ initial sync	2020-11-01 11:11:23 +00:00
James Turner	72b2eb0ebf	TerraSync: avoid 404s to probe missing tiles Use the top-level dirIndex files to determine if a 1x1 tile dir exists, instead of proving the server via a 404. This reduces the number of requests we make considerably, which is … important.	2020-11-01 11:11:19 +00:00
James Turner	ec3829addb	TerraSync: validate local dirs incrementally Add a process() method to HTTPRepository, and use this to incrementally validate subdirs after the .dirIndex is received. This avoids large pauses of the TerraSync thread, when all of Airports/ is validated at once.	2020-11-01 11:11:15 +00:00
James Turner	96bafef3f3	TerraSync: use an unordered_map for the hash cache Linear-scan is a bit slow in debug builds, for the large Airports/ tree; switch to an unordered_map. Will back-port to the LTS once tested a bit more.	2020-11-01 11:11:12 +00:00
James Turner	3ff3bd0a6c	Props: allow flushing the atomic change listener Trying to narrow down causes of the ‘unregister listeners crashes on shutdown’ reports.	2020-11-01 11:11:07 +00:00
Automatic Release Builder	78d073a0f0	new version: 2020.3.1	2020-10-26 09:09:22 +00:00
Scott Giese	d62796c19d	METAR: When Wind unit not specified, default to knots. FLIGHTGEAR-F6 resolved	2020-10-26 08:58:06 +00:00
Scott Giese	7ea7ff43fc	METAR Wind Sensor Failure. FLIGHTGEAR-F9 resolved.	2020-10-26 08:57:24 +00:00
James Turner	dafd185595	Fix for crash reported by Michael Danilov In case uDNS returns a NULL txt pointer, don’t try to create a std::string from it, since this will crash, See: https://sourceforge.net/p/flightgear/codetickets/2398/	2020-10-26 08:54:18 +00:00
Automatic Release Builder	1568ed8b97	Catalog migration: migrate packages too When doing a catalog migration to a new ID (eg, 2018 -> 2020), also mark the installed packages for installation, on the new catalog. Related to this, when manually removing a catalog, record this fact, so we don’t re-add it automatically due to migration. Add unit-tests covering both of these cases.	2020-10-22 21:27:47 +01:00
Automatic Release Builder	444e2ffb2d	Sound: readWAV: avoid common exceptions. Avoid exceptions for the common ‘file not found’ case, and instead return false / nullptr. Erik says it’s fine.	2020-10-15 17:00:30 +01:00
Automatic Release Builder	32ccdaec6f	Update version file	2020-10-13 22:27:38 +01:00
Automatic Release Builder	bd9f04d980	TerraSync: retry after socket failures Assume socket failures are intermittent, up to some maximum count (currently configured as 16). Add a test case to cover this.	2020-10-13 11:38:35 +01:00
Automatic Release Builder	9c530d6978	TerraSync: Rate-limit hash-cache writes This helps with IO-limited performance on Windows	2020-10-07 12:23:33 +01:00
Automatic Release Builder	05094510be	SGPath: optimise exists() on Windows For a very common existence only check, use a dedicated win32 API function, to save some time.	2020-10-07 12:21:49 +01:00
Automatic Release Builder	41e43eeba0	TerraSync: fix Windows behaviour SGDir on Windows returns dot-files by default, which includes our hash-cache, causing it to be orphaned.	2020-10-06 23:51:06 +01:00
Automatic Release Builder	f95cbd703a	Add a hasInstance to ResourceManager This is needed to allow order-independent shutdown, due to the dumb ownership behaviour for providers.	2020-10-06 11:34:03 +01:00
Automatic Release Builder	e39036a635	SGFile::computeHash: check malloc result Trying to fix a reported crash, in this code. malloc() returning null seems unlikely but worth checking for. ALso use a unique_ptr with a custom deleter to avoid a leak if we throw. Sentry-Id: FLIGHTGEAR-3Y	2020-10-06 11:33:17 +01:00
Automatic Release Builder	1171d57b72	XMLSound: Avoid using exceptions for missing file Reducing how noisy our exception reporting is, by using simple return value from SGXMLSound::init.	2020-10-05 14:21:37 +01:00
Automatic Release Builder	852058150b	TerraSync: fast start Airports/ sync If TerraSync/Airports is missing, download Airport_archive.tgz instead. In all other cases update as normal. This has the effect, that fresh installs make many fewer requests to TerraSync servers.	2020-10-05 11:58:03 +01:00
Automatic Release Builder	9be955262e	Terrasync: re-add persistent update cache Re-add (with some tweaks) the persistent tile-cache code, so that TerraSync checks the server at most once per 24 hour period, for a given repository path. (Disbale the cache by setting /sim/terrasync/enable-persistent-cache=0)	2020-10-01 10:00:22 +01:00
@@ -1 +1 @@
 .2.1
 .3.6