new version: 2020.3.6

Tolerate blank lines in buildings lists
Animations: Fix spin for Compositor lights
2021-01-20 13:01:25 +00:00 · 2021-01-20 11:35:33 +00:00 · 2021-01-15 15:45:26 +00:00 · 2021-01-05 12:58:46 +00:00 · 2021-01-02 21:57:54 -06:00 · 2021-01-01 22:41:47 +00:00
53 changed files with 1563 additions and 537 deletions
@@ -278,7 +278,14 @@ else()
    endif()
 endif(SIMGEAR_HEADLESS)

-find_package(ZLIB 1.2.4 REQUIRED)
+if(${CMAKE_SYSTEM_NAME} MATCHES "OpenBSD")
+    # As of 2020-08-01, OpenBSD's system zlib is slightly old, but it's usable
+    # with a workaround in simgear/io/iostreams/gzfstream.cxx.
+    find_package(ZLIB 1.2.3 REQUIRED)
+else()
+    find_package(ZLIB 1.2.4 REQUIRED)
+endif()
+
 find_package(CURL REQUIRED)

 if (SYSTEM_EXPAT)
@@ -1 +1 @@
-2020.3.0
+2020.3.6
@@ -691,6 +691,10 @@ namespace canvas
      {
        _sampling_dirty = true;
      }
+      else if( name == "anisotropy" )
+      {
+        _texture.setMaxAnisotropy( node->getFloatValue() );
+      }
      else if( name == "additive-blend" )
      {
        _texture.useAdditiveBlend( node->getBoolValue() );
@@ -202,6 +202,12 @@ namespace canvas
    updateSampling();
  }

+  //----------------------------------------------------------------------------
+  void ODGauge::setMaxAnisotropy(float anis)
+  {
+    texture->setMaxAnisotropy(anis);
+  }
+
  //----------------------------------------------------------------------------
  void ODGauge::setRender(bool render)
  {
@@ -109,6 +109,8 @@ namespace canvas
                        int coverage_samples = 0,
                        int color_samples = 0 );

+      void setMaxAnisotropy(float anis);
+
      /**
       * Enable/Disable updating the texture (If disabled the contents of the
       * texture remains with the outcome of the last rendering pass)
@@ -218,6 +218,9 @@ const float SG_RADIANS_TO_DEGREES = 180.0f / SG_PI;
 #define SG_OBJECT_RANGE_ROUGH     9000.0
 #define SG_OBJECT_RANGE_DETAILED  1500.0

+/** Minimum expiry time of PagedLOD within the Tile.  Overridden by /sim/rendering/plod-minimum-expiry-time-secs **/
+#define SG_TILE_MIN_EXPIRY 180.0
+
 /** Radius of scenery tiles in m **/
 #define SG_TILE_RADIUS      14000.0

@@ -3,8 +3,12 @@ include (SimGearComponent)

 set(HEADERS debug_types.h 
    logstream.hxx BufferedLogCallback.hxx OsgIoCapture.hxx
-    LogCallback.hxx LogEntry.hxx)
+    LogCallback.hxx LogEntry.hxx
+    ErrorReportingCallback.hxx)
+
 set(SOURCES logstream.cxx BufferedLogCallback.cxx
-    LogCallback.cxx LogEntry.cxx)
+    LogCallback.cxx LogEntry.cxx
+    ErrorReportingCallback.cxx
+    )

 simgear_component(debug debug "${SOURCES}" "${HEADERS}")
@@ -0,0 +1,47 @@
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Library General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Library General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//
+
+#include <simgear_config.h>
+
+#include "ErrorReportingCallback.hxx"
+
+using std::string;
+
+namespace simgear {
+
+static ErrorReportCallback static_callback;
+
+void setErrorReportCallback(ErrorReportCallback cb)
+{
+    static_callback = cb;
+}
+
+
+void reportError(const std::string& msg, const std::string& more)
+{
+    if (!static_callback)
+        return;
+
+    static_callback(msg, more, false);
+}
+
+void reportFatalError(const std::string& msg, const std::string& more)
+{
+    if (!static_callback)
+        return;
+    static_callback(msg, more, true);
+}
+
+} // namespace simgear
@@ -0,0 +1,31 @@
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Library General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Library General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//
+
+#pragma once
+
+#include <functional>
+#include <string>
+
+namespace simgear {
+
+void reportError(const std::string& msg, const std::string& more = {});
+
+void reportFatalError(const std::string& msg, const std::string& more = {});
+
+using ErrorReportCallback = std::function<void(const std::string& msg, const std::string& more, bool isFatal)>;
+
+void setErrorReportCallback(ErrorReportCallback cb);
+
+} // namespace simgear
@@ -45,14 +45,6 @@ namespace simgear
            std::atomic<int> receiveDepth;
            std::atomic<int> sentMessageCount;

-            void UnlockList()
-            {
-                _lock.unlock();
-            }
-            void LockList()
-            {
-                _lock.lock();
-            }
        public:
            Transmitter() : receiveDepth(0), sentMessageCount(0)
            {
@@ -69,20 +61,17 @@ namespace simgear
            // most recently registered recipients should process the messages/events first.
            virtual void Register(IReceiver& r)
            {
-                LockList();
+                std::lock_guard<std::mutex> scopeLock(_lock);
                recipient_list.push_back(&r);
                r.OnRegisteredAtTransmitter(this);
                if (std::find(deleted_recipients.begin(), deleted_recipients.end(), &r) != deleted_recipients.end())
                    deleted_recipients.remove(&r);
-
-                UnlockList();
            }

            //  Removes an object from receving message from this transmitter
            virtual void DeRegister(IReceiver& R)
            {
-                LockList();
-                //printf("Remove %x\n", &R);
+                std::lock_guard<std::mutex> scopeLock(_lock);
                if (recipient_list.size())
                {
                    if (std::find(recipient_list.begin(), recipient_list.end(), &R) != recipient_list.end())
@@ -93,7 +82,6 @@ namespace simgear
                            deleted_recipients.push_back(&R);
                    }
                }
-                UnlockList();
            }

            // Notify all registered recipients. Stop when receipt status of abort or finished are received.
@@ -107,69 +95,68 @@ namespace simgear
                ReceiptStatus return_status = ReceiptStatusNotProcessed;

                sentMessageCount++;
-                try
+
+                std::vector<IReceiver*> temp;
                {
-                    LockList();
+                    std::lock_guard<std::mutex> scopeLock(_lock);
                    if (receiveDepth == 0)
                        deleted_recipients.clear();
                    receiveDepth++;
-                    std::vector<IReceiver*> temp(recipient_list.size());
+
                    int idx = 0;
                    for (RecipientList::iterator i = recipient_list.begin(); i != recipient_list.end(); i++)
                    {
-                        temp[idx++] = *i;
+                        temp.push_back(*i);
                    }
-                    UnlockList();
-                    int tempSize = temp.size();
-                    for (int index = 0; index < tempSize; index++)
+                }
+                int tempSize = temp.size();
+                for (int index = 0; index < tempSize; index++)
+                {
+                    IReceiver* R = temp[index];
                    {
-                        IReceiver* R = temp[index];
-                        LockList();
+                        std::lock_guard<std::mutex> scopeLock(_lock);
                        if (deleted_recipients.size())
                        {
                            if (std::find(deleted_recipients.begin(), deleted_recipients.end(), R) != deleted_recipients.end())
                            {
-                                UnlockList();
                                continue;
                            }
                        }
-                        UnlockList();
-                        if (R)
-                        {
-                            ReceiptStatus rstat = R->Receive(M);
-                            switch (rstat)
-                            {
-                            case ReceiptStatusFail:
-                                return_status = ReceiptStatusFail;
-                                break;
-                            case ReceiptStatusPending:
-                                return_status = ReceiptStatusPending;
-                                break;
-                            case ReceiptStatusPendingFinished:
-                                return rstat;
-
-                            case ReceiptStatusNotProcessed:
-                                break;
-                            case ReceiptStatusOK:
-                                if (return_status == ReceiptStatusNotProcessed)
-                                    return_status = rstat;
-                                break;
-
-                            case ReceiptStatusAbort:
-                                return ReceiptStatusAbort;
-
-                            case ReceiptStatusFinished:
-                                return ReceiptStatusOK;
-                            }
-                        }
-
                    }
+                    if (R)
+                    {
+                        ReceiptStatus rstat = R->Receive(M);
+                        switch (rstat)
+                        {
+                        case ReceiptStatusFail:
+                            return_status = ReceiptStatusFail;
+                            break;
+
+                        case ReceiptStatusPending:
+                            return_status = ReceiptStatusPending;
+                            break;
+
+                        case ReceiptStatusPendingFinished:
+                            return rstat;
+
+                        case ReceiptStatusNotProcessed:
+                            break;
+
+                        case ReceiptStatusOK:
+                            if (return_status == ReceiptStatusNotProcessed)
+                                return_status = rstat;
+                            break;
+
+                        case ReceiptStatusAbort:
+                            return ReceiptStatusAbort;
+
+                        case ReceiptStatusFinished:
+                            return ReceiptStatusOK;
+                        }
+                    }
+
                }
-                catch (...)
-                {
-                    throw;
-                    // return_status = ReceiptStatusAbort;
-                }
+
                receiveDepth--;
                return return_status;
            }
@@ -177,9 +164,8 @@ namespace simgear
            // number of currently registered recipients
            virtual int Count()
            {
-                LockList();
+                std::lock_guard<std::mutex> scopeLock(_lock);
                return recipient_list.size();
-                UnlockList();
            }

            // number of sent messages.
@@ -46,6 +46,7 @@
 #include <sstream>

 #include <simgear/debug/logstream.hxx>
+#include <simgear/math/sg_random.h>
 #include <simgear/structure/exception.hxx>

 #include "metar.hxx"
@@ -631,30 +632,41 @@ bool SGMetar::scanWind()
 	int dir;
 	if (!strncmp(m, "VRB", 3))
 		m += 3, dir = -1;
+	else if (!strncmp(m, "///", 3))	// direction not measurable
+		m += 3, dir = -1;
 	else if (!scanNumber(&m, &dir, 3))
 		return false;

 	int i;
-	if (!scanNumber(&m, &i, 2, 3))
+	if (!strncmp(m, "//", 2))	// speed not measurable
+		m += 2, i = -1;
+	else if (!scanNumber(&m, &i, 2, 3))
 		return false;
 	double speed = i;

 	double gust = NaN;
 	if (*m == 'G') {
 		m++;
-		if (!scanNumber(&m, &i, 2, 3))
+		if (!strncmp(m, "//", 2))	// speed not measurable
+			m += 2, i = -1;
+		else if (!scanNumber(&m, &i, 2, 3))
 			return false;
-		gust = i;
+
+		if (i != -1)
+			gust = i;
 	}
+
 	double factor;
 	if (!strncmp(m, "KT", 2))
 		m += 2, factor = SG_KT_TO_MPS;
-	else if (!strncmp(m, "KMH", 3))
+	else if (!strncmp(m, "KMH", 3))		// invalid Km/h
 		m += 3, factor = SG_KMH_TO_MPS;
-	else if (!strncmp(m, "KPH", 3))		// ??
+	else if (!strncmp(m, "KPH", 3))		// invalid Km/h
 		m += 3, factor = SG_KMH_TO_MPS;
 	else if (!strncmp(m, "MPS", 3))
 		m += 3, factor = 1.0;
+	else if (!strncmp(m, " ", 1))		// default to Knots
+		factor = SG_KT_TO_MPS;
 	else
 		return false;
 	if (!scanBoundary(&m))
@@ -674,18 +686,28 @@ bool SGMetar::scanVariability()
 {
 	char *m = _m;
 	int from, to;
-	if (!scanNumber(&m, &from, 3))
+
+	if (!strncmp(m, "///", 3))	// direction not measurable
+		m += 3, from = -1;
+	else if (!scanNumber(&m, &from, 3))
 		return false;
+
 	if (*m++ != 'V')
 		return false;
-	if (!scanNumber(&m, &to, 3))
+
+	if (!strncmp(m, "///", 3))	// direction not measurable
+		m += 3, to = -1;
+	else if (!scanNumber(&m, &to, 3))
 		return false;
+
 	if (!scanBoundary(&m))
 		return false;
+
 	_m = m;
 	_wind_range_from = from;
 	_wind_range_to = to;
 	_grpcount++;
+
 	return true;
 }

@@ -1047,9 +1069,12 @@ bool SGMetar::scanSkyCondition()
 		return false;
 	m += i;

-	if (!strncmp(m, "///", 3))	// vis not measurable (e.g. because of heavy snowing)
+	if (!strncmp(m, "///", 3))	{ // vis not measurable (e.g. because of heavy snowing)
 		m += 3, i = -1;
-	else if (scanBoundary(&m)) {
+		sg_srandom_time();
+		// randomize the base height to avoid the black sky issue
+		i = 50 + static_cast<int>(sg_random() * 250.0);		// range [5,000, 30,000]
+	} else if (scanBoundary(&m)) {
 		_m = m;
 		return true;				// ignore single OVC/BKN/...
 	} else if (!scanNumber(&m, &i, 3))
@@ -76,12 +76,38 @@ void test_sensor_failure_cloud()
    SG_CHECK_EQUAL_EP2(m1.getPressure_hPa(), 1025, TEST_EPSILON);
 }

+void test_sensor_failure_wind()
+{
+    SGMetar m1("2020/10/23 16:55 LIVD 231655Z /////KT 9999 OVC025 10/08 Q1020 RMK OVC VIS MIN 9999 BLU");
+    SG_CHECK_EQUAL(m1.getWindDir(), -1);
+    SG_CHECK_EQUAL_EP2(m1.getWindSpeed_kt(), -1, TEST_EPSILON);
+
+    SGMetar m2("2020/10/21 16:55 LIVD 211655Z /////KT CAVOK 07/03 Q1023 RMK SKC VIS MIN 9999 BLU");
+    SG_CHECK_EQUAL(m2.getWindDir(), -1);
+    SG_CHECK_EQUAL_EP2(m2.getWindSpeed_kt(), -1, TEST_EPSILON);
+
+    SGMetar m3("2020/11/17 16:00 CYAZ 171600Z 14040G//KT 10SM -RA OVC012 12/11 A2895 RMK NS8 VIA CYXY SLP806 DENSITY ALT 900FT");
+    SG_CHECK_EQUAL(m3.getWindDir(), 140);
+    SG_CHECK_EQUAL_EP2(m3.getWindSpeed_kt(), 40, TEST_EPSILON);
+    SG_CHECK_EQUAL_EP2(m3.getGustSpeed_kt(), SGMetarNaN, TEST_EPSILON);
+}
+
+void test_wind_unit_not_specified()
+{
+    SGMetar m1("2020/10/23 11:58 KLSV 231158Z 05010G14 10SM CLR 16/M04 A2992 RMK SLPNO WND DATA ESTMD ALSTG/SLP ESTMD 10320 20124 5//// $");
+    SG_CHECK_EQUAL(m1.getWindDir(), 50);
+    SG_CHECK_EQUAL_EP2(m1.getWindSpeed_kt(), 10.0, TEST_EPSILON);
+    SG_CHECK_EQUAL_EP2(m1.getGustSpeed_kt(), 14.0, TEST_EPSILON);
+}
+
 int main(int argc, char* argv[])
 {
    try {
        test_basic();
        test_sensor_failure_weather();
        test_sensor_failure_cloud();
+        test_sensor_failure_wind();
+        test_wind_unit_not_specified();
    } catch (sg_exception& e) {
        cerr << "got exception:" << e.getMessage() << endl;
        return -1;
@@ -61,6 +61,10 @@ public:

    struct dns_ctx * ctx;
    static size_t instanceCounter;
+
+    using RequestVec = std::vector<Request_ptr>;
+
+    RequestVec _activeRequests;
 };

 size_t Client::ClientPrivate::instanceCounter = 0;
@@ -78,6 +82,11 @@ Request::~Request()
 {
 }

+void Request::cancel()
+{
+    _cancelled = true;
+}
+
 bool Request::isTimeout() const
 {
    return (time(NULL) - _start) > _timeout_secs;
@@ -114,18 +123,20 @@ static void dnscbSRV(struct dns_ctx *ctx, struct dns_rr_srv *result, void *data)
 {
    SRVRequest * r = static_cast<SRVRequest*>(data);
    if (result) {
-        r->cname = result->dnssrv_cname;
-        r->qname = result->dnssrv_qname;
-        r->ttl = result->dnssrv_ttl;
-        for (int i = 0; i < result->dnssrv_nrr; i++) {
-            SRVRequest::SRV_ptr srv(new SRVRequest::SRV);
-            r->entries.push_back(srv);
-            srv->priority = result->dnssrv_srv[i].priority;
-            srv->weight = result->dnssrv_srv[i].weight;
-            srv->port = result->dnssrv_srv[i].port;
-            srv->target = result->dnssrv_srv[i].name;
+        if (!r->isCancelled()) {
+            r->cname = result->dnssrv_cname;
+            r->qname = result->dnssrv_qname;
+            r->ttl = result->dnssrv_ttl;
+            for (int i = 0; i < result->dnssrv_nrr; i++) {
+                SRVRequest::SRV_ptr srv(new SRVRequest::SRV);
+                r->entries.push_back(srv);
+                srv->priority = result->dnssrv_srv[i].priority;
+                srv->weight = result->dnssrv_srv[i].weight;
+                srv->port = result->dnssrv_srv[i].port;
+                srv->target = result->dnssrv_srv[i].name;
+            }
+            std::sort(r->entries.begin(), r->entries.end(), sortSRV);
        }
-        std::sort( r->entries.begin(), r->entries.end(), sortSRV );
        free(result);
    }
    r->setComplete();
@@ -134,11 +145,16 @@ static void dnscbSRV(struct dns_ctx *ctx, struct dns_rr_srv *result, void *data)
 void SRVRequest::submit( Client * client )
 {
    // if service is defined, pass service and protocol
-    if (!dns_submit_srv(client->d->ctx, getDn().c_str(), _service.empty() ? NULL : _service.c_str(), _service.empty() ? NULL : _protocol.c_str(), 0, dnscbSRV, this )) {
+    auto q = dns_submit_srv(client->d->ctx, getDn().c_str(), _service.empty() ? NULL : _service.c_str(),
+                            _service.empty() ? NULL : _protocol.c_str(),
+                            0, dnscbSRV, this);
+
+    if (!q) {
        SG_LOG(SG_IO, SG_ALERT, "Can't submit dns request for " << getDn());
        return;
    }
    _start = time(NULL);
+    _query = q;
 }

 TXTRequest::TXTRequest( const std::string & dn ) :
@@ -151,17 +167,24 @@ static void dnscbTXT(struct dns_ctx *ctx, struct dns_rr_txt *result, void *data)
 {
    TXTRequest * r = static_cast<TXTRequest*>(data);
    if (result) {
-        r->cname = result->dnstxt_cname;
-        r->qname = result->dnstxt_qname;
-        r->ttl = result->dnstxt_ttl;
-        for (int i = 0; i < result->dnstxt_nrr; i++) {
-          //TODO: interprete the .len field of dnstxt_txt?
-          string txt = string((char*)result->dnstxt_txt[i].txt);
-          r->entries.push_back( txt );
-          string_list tokens = simgear::strutils::split( txt, "=", 1 );
-          if( tokens.size() == 2 ) {
-            r->attributes[tokens[0]] = tokens[1];
-          }
+        if (!r->isCancelled()) {
+            r->cname = result->dnstxt_cname;
+            r->qname = result->dnstxt_qname;
+            r->ttl = result->dnstxt_ttl;
+            for (int i = 0; i < result->dnstxt_nrr; i++) {
+                //TODO: interprete the .len field of dnstxt_txt?
+                auto rawTxt = reinterpret_cast<char*>(result->dnstxt_txt[i].txt);
+                if (!rawTxt) {
+                    continue;
+                }
+
+                const string txt{rawTxt};
+                r->entries.push_back(txt);
+                string_list tokens = simgear::strutils::split(txt, "=", 1);
+                if (tokens.size() == 2) {
+                    r->attributes[tokens[0]] = tokens[1];
+                }
+            }
        }
        free(result);
    }
@@ -171,11 +194,13 @@ static void dnscbTXT(struct dns_ctx *ctx, struct dns_rr_txt *result, void *data)
 void TXTRequest::submit( Client * client )
 {
    // protocol and service an already encoded in DN so pass in NULL for both
-    if (!dns_submit_txt(client->d->ctx, getDn().c_str(), DNS_C_IN, 0, dnscbTXT, this )) {
+    auto q = dns_submit_txt(client->d->ctx, getDn().c_str(), DNS_C_IN, 0, dnscbTXT, this);
+    if (!q) {
        SG_LOG(SG_IO, SG_ALERT, "Can't submit dns request for " << getDn());
        return;
    }
    _start = time(NULL);
+    _query = q;
 }


@@ -190,27 +215,29 @@ static void dnscbNAPTR(struct dns_ctx *ctx, struct dns_rr_naptr *result, void *d
 {
    NAPTRRequest * r = static_cast<NAPTRRequest*>(data);
    if (result) {
-        r->cname = result->dnsnaptr_cname;
-        r->qname = result->dnsnaptr_qname;
-        r->ttl = result->dnsnaptr_ttl;
-        for (int i = 0; i < result->dnsnaptr_nrr; i++) {
-            if( !r->qservice.empty() && r->qservice != result->dnsnaptr_naptr[i].service )
-                continue;
+        if (!r->isCancelled()) {
+            r->cname = result->dnsnaptr_cname;
+            r->qname = result->dnsnaptr_qname;
+            r->ttl = result->dnsnaptr_ttl;
+            for (int i = 0; i < result->dnsnaptr_nrr; i++) {
+                if (!r->qservice.empty() && r->qservice != result->dnsnaptr_naptr[i].service)
+                    continue;

-            //TODO: case ignore and result flags may have more than one flag
-            if( !r->qflags.empty() && r->qflags != result->dnsnaptr_naptr[i].flags )
-                continue;
+                //TODO: case ignore and result flags may have more than one flag
+                if (!r->qflags.empty() && r->qflags != result->dnsnaptr_naptr[i].flags)
+                    continue;

-            NAPTRRequest::NAPTR_ptr naptr(new NAPTRRequest::NAPTR);
-            r->entries.push_back(naptr);
-            naptr->order = result->dnsnaptr_naptr[i].order;
-            naptr->preference = result->dnsnaptr_naptr[i].preference;
-            naptr->flags = result->dnsnaptr_naptr[i].flags;
-            naptr->service = result->dnsnaptr_naptr[i].service;
-            naptr->regexp = result->dnsnaptr_naptr[i].regexp;
-            naptr->replacement = result->dnsnaptr_naptr[i].replacement;
+                NAPTRRequest::NAPTR_ptr naptr(new NAPTRRequest::NAPTR);
+                r->entries.push_back(naptr);
+                naptr->order = result->dnsnaptr_naptr[i].order;
+                naptr->preference = result->dnsnaptr_naptr[i].preference;
+                naptr->flags = result->dnsnaptr_naptr[i].flags;
+                naptr->service = result->dnsnaptr_naptr[i].service;
+                naptr->regexp = result->dnsnaptr_naptr[i].regexp;
+                naptr->replacement = result->dnsnaptr_naptr[i].replacement;
+            }
+            std::sort(r->entries.begin(), r->entries.end(), sortNAPTR);
        }
-        std::sort( r->entries.begin(), r->entries.end(), sortNAPTR );
        free(result);
    }
    r->setComplete();
@@ -218,11 +245,13 @@ static void dnscbNAPTR(struct dns_ctx *ctx, struct dns_rr_naptr *result, void *d

 void NAPTRRequest::submit( Client * client )
 {
-    if (!dns_submit_naptr(client->d->ctx, getDn().c_str(), 0, dnscbNAPTR, this )) {
+    auto q = dns_submit_naptr(client->d->ctx, getDn().c_str(), 0, dnscbNAPTR, this);
+    if (!q) {
        SG_LOG(SG_IO, SG_ALERT, "Can't submit dns request for " << getDn());
        return;
    }
    _start = time(NULL);
+    _query = q;
 }


@@ -237,6 +266,7 @@ Client::Client() :

 void Client::makeRequest(const Request_ptr& r)
 {
+    d->_activeRequests.push_back(r);
    r->submit(this);
 }

@@ -247,6 +277,19 @@ void Client::update(int waitTimeout)
        return;

    dns_ioevent(d->ctx, now);
+
+    // drop our owning ref to completed requests,
+    // and cancel any which timed out
+    auto it = std::remove_if(d->_activeRequests.begin(), d->_activeRequests.end(),
+                             [this](const Request_ptr& r) {
+                                 if (r->isTimeout()) {
+                                     dns_cancel(d->ctx, reinterpret_cast<struct dns_query*>(r->_query));
+                                     return true;
+                                 }
+
+                                 return r->isComplete();
+                             });
+    d->_activeRequests.erase(it, d->_activeRequests.end());
 }

 } // of namespace DNS
@@ -40,28 +40,38 @@ namespace DNS
 {

 class Client;
+
+using UDNSQueryPtr = void*;
+
 class Request : public SGReferenced
 {
 public:
    Request( const std::string & dn );
    virtual ~Request();
-    std::string getDn() const { return _dn; }
+    const std::string& getDn() const { return _dn; }
    int getType() const { return _type; }
    bool isComplete() const { return _complete; }
    bool isTimeout() const;
    void setComplete( bool b = true ) { _complete = b; }
+    bool isCancelled() const { return _cancelled; }

    virtual void submit( Client * client) = 0;

+    void cancel();
+
    std::string cname;
    std::string qname;
    unsigned ttl;
 protected:
+    friend class Client;
+
+    UDNSQueryPtr _query = nullptr;
    std::string _dn;
    int _type;
    bool _complete;
    time_t _timeout_secs;
    time_t _start;
+    bool _cancelled = false;
 };
 typedef SGSharedPtr<Request> Request_ptr;

@@ -69,7 +79,7 @@ class NAPTRRequest : public Request
 {
 public:
    NAPTRRequest( const std::string & dn );
-    virtual void submit( Client * client );
+    void submit(Client* client) override;

    struct NAPTR : SGReferenced {
        int order;
@@ -92,7 +102,7 @@ class SRVRequest : public Request
 public:
    SRVRequest( const std::string & dn );
    SRVRequest( const std::string & dn, const string & service, const string & protocol );
-    virtual void submit( Client * client );
+    void submit(Client* client) override;

    struct SRV : SGReferenced {
      int priority;
@@ -112,7 +122,7 @@ class TXTRequest : public Request
 {
 public:
    TXTRequest( const std::string & dn );
-    virtual void submit( Client * client );
+    void submit(Client* client) override;

    typedef std::vector<string> TXT_list;
    typedef std::map<std::string,std::string> TXT_Attribute_map;
@@ -81,6 +81,45 @@ std::string innerResultCodeAsString(HTTPRepository::ResultCode code) {
  return "Unknown response code";
 }

+struct HashCacheEntry {
+    std::string filePath;
+    time_t modTime;
+    size_t lengthBytes;
+    std::string hashHex;
+};
+
+using HashCache = std::unordered_map<std::string, HashCacheEntry>;
+
+std::string computeHashForPath(const SGPath& p)
+{
+    if (!p.exists())
+        return {};
+
+    sha1nfo info;
+    sha1_init(&info);
+
+    const int bufSize = 1024 * 1024;
+    char* buf = static_cast<char*>(malloc(bufSize));
+    if (!buf) {
+        sg_io_exception("Couldn't allocate SHA1 computation buffer");
+    }
+
+    size_t readLen;
+    SGBinaryFile f(p);
+    if (!f.open(SG_IO_IN)) {
+        free(buf);
+        throw sg_io_exception("Couldn't open file for compute hash", p);
+    }
+    while ((readLen = f.read(buf, bufSize)) > 0) {
+        sha1_write(&info, buf, readLen);
+    }
+
+    f.close();
+    free(buf);
+    std::string hashBytes((char*)sha1_result(&info), HASH_LENGTH);
+    return strutils::encodeHex(hashBytes);
+}
+
 } // namespace

 class HTTPDirectory
@@ -111,6 +150,9 @@ class HTTPDirectory
    typedef std::vector<ChildInfo> ChildInfoList;
    ChildInfoList children;

+    mutable HashCache hashes;
+    mutable bool hashCacheDirty = false;
+
 public:
    HTTPDirectory(HTTPRepoPrivate* repo, const std::string& path) :
        _repository(repo),
@@ -124,6 +166,8 @@ public:
              // already exists on disk
              parseDirIndex(children);
              std::sort(children.begin(), children.end());
+
+              parseHashCache();
          } catch (sg_exception& ) {
              // parsing cache failed
              children.clear();
@@ -149,7 +193,7 @@ public:
    {
        SGPath fpath(absolutePath());
        fpath.append(".dirindex");
-        _repository->updatedFileContents(fpath, hash);
+        updatedFileContents(fpath, hash);

        children.clear();
        parseDirIndex(children);
@@ -177,7 +221,7 @@ public:
 		char* buf = nullptr;
 		size_t bufSize = 0;

-		for (const auto& child : children) {
+                for (auto &child : children) {
                  if (child.type != HTTPRepository::FileType)
                    continue;

@@ -189,30 +233,36 @@ public:
                  cp.append(child.name);
                  if (!cp.exists()) {
                    continue;
-			}
+                  }

-			SGBinaryFile src(cp);
-			SGBinaryFile dst(child.path);
-			src.open(SG_IO_IN);
-			dst.open(SG_IO_OUT);
+                  SGBinaryFile src(cp);
+                  SGBinaryFile dst(child.path);
+                  src.open(SG_IO_IN);
+                  dst.open(SG_IO_OUT);

-			if (bufSize < cp.sizeInBytes()) {
-				bufSize = cp.sizeInBytes();
-				free(buf);
-				buf = (char*) malloc(bufSize);
-				if (!buf) {
-					continue;
-				}
-			}
+                  if (bufSize < cp.sizeInBytes()) {
+                    bufSize = cp.sizeInBytes();
+                    free(buf);
+                    buf = (char *)malloc(bufSize);
+                    if (!buf) {
+                      continue;
+                    }
+                  }

-			src.read(buf, cp.sizeInBytes());
-			dst.write(buf, cp.sizeInBytes());
-			src.close();
-			dst.close();
+                  src.read(buf, cp.sizeInBytes());
+                  dst.write(buf, cp.sizeInBytes());
+                  src.close();
+                  dst.close();

-		}
+                  // reset caching
+                  child.path.set_cached(false);
+                  child.path.set_cached(true);

-		free(buf);
+                  std::string hash = computeHashForPath(child.path);
+                  updatedFileContents(child.path, hash);
+                }
+
+                free(buf);
    }

    void updateChildrenBasedOnHash()
@@ -274,7 +324,7 @@ public:
          if (c.type == HTTPRepository::DirectoryType) {
            // If it's a directory,perform a recursive check.
            HTTPDirectory *childDir = childDirectory(c.name);
-            childDir->updateChildrenBasedOnHash();
+            _repository->scheduleUpdateOfChildren(childDir);
          }
        }
      } // of repository-defined (well, .dirIndex) children iteration
@@ -372,6 +422,71 @@ public:
        return _relativePath;
    }

+    class ArchiveExtractTask {
+    public:
+      ArchiveExtractTask(SGPath p, const std::string &relPath)
+          : relativePath(relPath), file(p), extractor(p.dir()) {
+        if (!file.open(SG_IO_IN)) {
+          SG_LOG(SG_TERRASYNC, SG_ALERT,
+                 "Unable to open " << p << " to extract");
+          return;
+        }
+
+        buffer = (uint8_t *)malloc(bufferSize);
+      }
+
+      ArchiveExtractTask(const ArchiveExtractTask &) = delete;
+
+      HTTPRepoPrivate::ProcessResult run(HTTPRepoPrivate* repo)
+      {
+          if (!buffer) {
+              return HTTPRepoPrivate::ProcessFailed;
+          }
+
+          size_t rd = file.read((char*)buffer, bufferSize);
+          extractor.extractBytes(buffer, rd);
+
+          if (file.eof()) {
+              extractor.flush();
+              file.close();
+
+              if (!extractor.isAtEndOfArchive()) {
+                  SG_LOG(SG_TERRASYNC, SG_ALERT, "Corrupt tarball " << relativePath);
+                  repo->failedToUpdateChild(relativePath,
+                                            HTTPRepository::REPO_ERROR_IO);
+                  return HTTPRepoPrivate::ProcessFailed;
+              }
+
+              if (extractor.hasError()) {
+                  SG_LOG(SG_TERRASYNC, SG_ALERT, "Error extracting " << relativePath);
+                  repo->failedToUpdateChild(relativePath,
+                                            HTTPRepository::REPO_ERROR_IO);
+                  return HTTPRepoPrivate::ProcessFailed;
+              }
+
+              return HTTPRepoPrivate::ProcessDone;
+          }
+
+          return HTTPRepoPrivate::ProcessContinue;
+      }
+
+      ~ArchiveExtractTask() { free(buffer); }
+
+    private:
+      // intentionally small so we extract incrementally on Windows
+      // where Defender throttles many small files, sorry
+      // if you make this bigger we will be more efficient but stall for
+      // longer when extracting the Airports_archive
+      const int bufferSize = 1024 * 64;
+
+      std::string relativePath;
+      uint8_t *buffer = nullptr;
+      SGBinaryFile file;
+      ArchiveExtractor extractor;
+    };
+
+    using ArchiveExtractTaskPtr = std::shared_ptr<ArchiveExtractTask>;
+
    void didUpdateFile(const std::string& file, const std::string& hash, size_t sz)
    {
        // check hash matches what we expected
@@ -387,7 +502,7 @@ public:
                    _relativePath + "/" + file,
                    HTTPRepository::REPO_ERROR_CHECKSUM);
            } else {
-                _repository->updatedFileContents(it->path, hash);
+                updatedFileContents(it->path, hash);
                _repository->updatedChildSuccessfully(_relativePath + "/" +
                                                      file);

@@ -410,33 +525,22 @@ public:
                  }

                  if (pathAvailable) {
-                    // If this is a tarball, then extract it.
-                    SGBinaryFile f(p);
-                    if (! f.open(SG_IO_IN)) SG_LOG(SG_TERRASYNC, SG_ALERT, "Unable to open " << p << " to extract");
+                    // we use a Task helper to extract tarballs incrementally.
+                    // without this, archive extraction blocks here, which
+                    // prevents other repositories downloading / updating.
+                    // Unfortunately due Windows AV (Defender, etc) we cna block
+                    // here for many minutes.

-                    SG_LOG(SG_TERRASYNC, SG_INFO, "Extracting " << absolutePath() << "/" << file << " to " << p.dir());
-                    SGPath extractDir = p.dir();
-                    ArchiveExtractor ex(extractDir);
-
-                    uint8_t *buf = (uint8_t *)alloca(2048);
-                    while (!f.eof()) {
-                      size_t bufSize = f.read((char *)buf, 2048);
-                      ex.extractBytes(buf, bufSize);
-                    }
-
-                    ex.flush();
-                    if (! ex.isAtEndOfArchive()) {
-                      SG_LOG(SG_TERRASYNC, SG_ALERT, "Corrupt tarball " << p);
-                      _repository->failedToUpdateChild(
-                          _relativePath, HTTPRepository::REPO_ERROR_IO);
-                    }
-
-                    if (ex.hasError()) {
-                      SG_LOG(SG_TERRASYNC, SG_ALERT, "Error extracting " << p);
-                      _repository->failedToUpdateChild(
-                          _relativePath, HTTPRepository::REPO_ERROR_IO);
-                    }
+                    // use a lambda to own this shared_ptr; this means when the
+                    // lambda is destroyed, the ArchiveExtraTask will get
+                    // cleaned up.
+                    ArchiveExtractTaskPtr t =
+                        std::make_shared<ArchiveExtractTask>(p, _relativePath);
+                    auto cb = [t](HTTPRepoPrivate *repo) {
+                      return t->run(repo);
+                    };

+                    _repository->addTask(cb);
                  } else {
                    SG_LOG(SG_TERRASYNC, SG_ALERT, "Unable to remove old file/directory " << removePath);
                  } // of pathAvailable
@@ -452,6 +556,50 @@ public:
        fpath.append(file);
        _repository->failedToUpdateChild(fpath, status);
    }
+
+    std::string hashForPath(const SGPath& p) const
+    {
+        const auto ps = p.utf8Str();
+        auto it = hashes.find(ps);
+        if (it != hashes.end()) {
+            const auto& entry = it->second;
+            // ensure data on disk hasn't changed.
+            // we could also use the file type here if we were paranoid
+            if ((p.sizeInBytes() == entry.lengthBytes) && (p.modTime() == entry.modTime)) {
+                return entry.hashHex;
+            }
+
+            // entry in the cache, but it's stale so remove and fall through
+            hashes.erase(it);
+        }
+
+        std::string hash = computeHashForPath(p);
+        updatedFileContents(p, hash);
+        return hash;
+    }
+
+    bool isHashCacheDirty() const
+    {
+        return hashCacheDirty;
+    }
+
+    void writeHashCache() const
+    {
+        if (!hashCacheDirty)
+            return;
+
+        hashCacheDirty = false;
+
+        SGPath cachePath = absolutePath() / ".dirhash";
+        sg_ofstream stream(cachePath, std::ios::out | std::ios::trunc | std::ios::binary);
+        for (const auto& e : hashes) {
+            const auto& entry = e.second;
+            stream << entry.filePath << "*" << entry.modTime << "*"
+                   << entry.lengthBytes << "*" << entry.hashHex << "\n";
+        }
+        stream.close();
+    }
+
 private:

    struct ChildWithName
@@ -575,7 +723,7 @@ private:
            ok = _repository->deleteDirectory(fpath, path);
        } else {
            // remove the hash cache entry
-            _repository->updatedFileContents(path, std::string());
+            updatedFileContents(path, std::string());
            ok = path.remove();
        }

@@ -593,9 +741,80 @@ private:
      if (child.type == HTTPRepository::TarballType)
        p.concat(
            ".tgz"); // For tarballs the hash is against the tarball file itself
-      return _repository->hashForPath(p);
+      return hashForPath(p);
    }

+    void parseHashCache()
+    {
+        hashes.clear();
+        SGPath cachePath = absolutePath() / ".dirhash";
+        if (!cachePath.exists()) {
+            return;
+        }
+
+        sg_ifstream stream(cachePath, std::ios::in);
+
+        while (!stream.eof()) {
+            std::string line;
+            std::getline(stream, line);
+            line = simgear::strutils::strip(line);
+            if (line.empty() || line[0] == '#')
+                continue;
+
+            string_list tokens = simgear::strutils::split(line, "*");
+            if (tokens.size() < 4) {
+                SG_LOG(SG_TERRASYNC, SG_WARN, "invalid entry in '" << cachePath << "': '" << line << "' (ignoring line)");
+                continue;
+            }
+            const std::string nameData = simgear::strutils::strip(tokens[0]);
+            const std::string timeData = simgear::strutils::strip(tokens[1]);
+            const std::string sizeData = simgear::strutils::strip(tokens[2]);
+            const std::string hashData = simgear::strutils::strip(tokens[3]);
+
+            if (nameData.empty() || timeData.empty() || sizeData.empty() || hashData.empty()) {
+                SG_LOG(SG_TERRASYNC, SG_WARN, "invalid entry in '" << cachePath << "': '" << line << "' (ignoring line)");
+                continue;
+            }
+
+            HashCacheEntry entry;
+            entry.filePath = nameData;
+            entry.hashHex = hashData;
+            entry.modTime = strtol(timeData.c_str(), NULL, 10);
+            entry.lengthBytes = strtol(sizeData.c_str(), NULL, 10);
+            hashes.insert(std::make_pair(entry.filePath, entry));
+        }
+    }
+
+    void updatedFileContents(const SGPath& p, const std::string& newHash) const
+    {
+        // remove the existing entry
+        const auto ps = p.utf8Str();
+        auto it = hashes.find(ps);
+        if (it != hashes.end()) {
+            hashes.erase(it);
+            hashCacheDirty = true;
+        }
+
+        if (newHash.empty()) {
+            return; // we're done
+        }
+
+        // use a cloned SGPath and reset its caching to force one stat() call
+        SGPath p2(p);
+        p2.set_cached(false);
+        p2.set_cached(true);
+
+        HashCacheEntry entry;
+        entry.filePath = ps;
+        entry.hashHex = newHash;
+        entry.modTime = p2.modTime();
+        entry.lengthBytes = p2.sizeInBytes();
+        hashes.insert(std::make_pair(ps, entry));
+
+        hashCacheDirty = true;
+    }
+
+
    HTTPRepoPrivate* _repository;
    std::string _relativePath; // in URL and file-system space
 };
@@ -606,7 +825,6 @@ HTTPRepository::HTTPRepository(const SGPath& base, HTTP::Client *cl) :
    _d->http = cl;
    _d->basePath = base;
    _d->rootDir.reset(new HTTPDirectory(_d.get(), ""));
-    _d->parseHashCache();
 }

 HTTPRepository::~HTTPRepository()
@@ -654,6 +872,38 @@ bool HTTPRepository::isDoingSync() const
    return _d->isUpdating;
 }

+void HTTPRepository::process()
+{
+    int processedCount = 0;
+    const int maxToProcess = 16;
+
+    while (processedCount < maxToProcess) {
+      if (_d->pendingTasks.empty()) {
+        break;
+      }
+
+      auto task = _d->pendingTasks.front();
+      auto result = task(_d.get());
+      if (result == HTTPRepoPrivate::ProcessContinue) {
+        // assume we're not complete
+        return;
+      }
+
+      _d->pendingTasks.pop_front();
+      ++processedCount;
+    }
+
+    _d->checkForComplete();
+}
+
+void HTTPRepoPrivate::checkForComplete()
+{
+  if (pendingTasks.empty() && activeRequests.empty() &&
+      queuedRequests.empty()) {
+    isUpdating = false;
+  }
+}
+
 size_t HTTPRepository::bytesToDownload() const
 {
    size_t result = 0;
@@ -731,49 +981,71 @@ HTTPRepository::failure() const
        }

    protected:
-      void gotBodyData(const char *s, int n) override {
-        if (!file.get()) {
-          file.reset(new SGBinaryFile(pathInRepo));
-          if (!file->open(SG_IO_OUT)) {
-            SG_LOG(SG_TERRASYNC, SG_WARN,
-                   "unable to create file " << pathInRepo);
-            _directory->repository()->http->cancelRequest(
-                this, "Unable to create output file:" + pathInRepo.utf8Str());
-          }
+        void gotBodyData(const char* s, int n) override
+        {
+            if (!file.get()) {
+                const bool ok = createOutputFile();
+                if (!ok) {
+                    _directory->repository()->http->cancelRequest(
+                        this, "Unable to create output file:" + pathInRepo.utf8Str());
+                }
+            }

-          sha1_init(&hashContext);
+            sha1_write(&hashContext, s, n);
+            file->write(s, n);
        }

-        sha1_write(&hashContext, s, n);
-        file->write(s, n);
-      }
+        bool createOutputFile()
+        {
+            file.reset(new SGBinaryFile(pathInRepo));
+            if (!file->open(SG_IO_OUT)) {
+                SG_LOG(SG_TERRASYNC, SG_WARN,
+                       "unable to create file " << pathInRepo);
+                return false;
+            }

-      void onDone() override {
-        file->close();
-        if (responseCode() == 200) {
-          std::string hash =
-              strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH);
-          _directory->didUpdateFile(fileName, hash, contentSize());
-        } else if (responseCode() == 404) {
-          SG_LOG(SG_TERRASYNC, SG_WARN,
-                 "terrasync file not found on server: "
-                     << fileName << " for " << _directory->absolutePath());
-          _directory->didFailToUpdateFile(
-              fileName, HTTPRepository::REPO_ERROR_FILE_NOT_FOUND);
-        } else {
-          SG_LOG(SG_TERRASYNC, SG_WARN,
-                 "terrasync file download error on server: "
-                     << fileName << " for " << _directory->absolutePath()
-                     << "\n\tserver responded: " << responseCode() << "/"
-                     << responseReason());
-          _directory->didFailToUpdateFile(fileName,
-                                          HTTPRepository::REPO_ERROR_HTTP);
-          // should we every retry here?
+            sha1_init(&hashContext);
+            return true;
        }

-        _directory->repository()->finishedRequest(
-            this, HTTPRepoPrivate::RequestFinish::Done);
-      }
+        void onDone() override
+        {
+            const bool is200Response = (responseCode() == 200);
+            if (!file && is200Response) {
+                // if the server defines a zero-byte file, we will never call
+                // gotBodyData, so create the file here
+                // this ensures all the logic below works as expected
+                createOutputFile();
+            }
+
+            if (file) {
+                file->close();
+            }
+
+            if (is200Response) {
+                std::string hash =
+                    strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH);
+                _directory->didUpdateFile(fileName, hash, contentSize());
+            } else if (responseCode() == 404) {
+                SG_LOG(SG_TERRASYNC, SG_WARN,
+                       "terrasync file not found on server: "
+                           << fileName << " for " << _directory->absolutePath());
+                _directory->didFailToUpdateFile(
+                    fileName, HTTPRepository::REPO_ERROR_FILE_NOT_FOUND);
+            } else {
+                SG_LOG(SG_TERRASYNC, SG_WARN,
+                       "terrasync file download error on server: "
+                           << fileName << " for " << _directory->absolutePath()
+                           << "\n\tserver responded: " << responseCode() << "/"
+                           << responseReason());
+                _directory->didFailToUpdateFile(fileName,
+                                                HTTPRepository::REPO_ERROR_HTTP);
+                // should we every retry here?
+            }
+
+            _directory->repository()->finishedRequest(
+                this, HTTPRepoPrivate::RequestFinish::Done);
+        }

      void onFail() override {
        HTTPRepository::ResultCode code = HTTPRepository::REPO_ERROR_SOCKET;
@@ -868,7 +1140,7 @@ HTTPRepository::failure() const
              return;
            }

-            std::string curHash = _directory->repository()->hashForPath(path());
+            std::string curHash = _directory->hashForPath(path());
            if (hash != curHash) {
              simgear::Dir d(_directory->absolutePath());
              if (!d.exists()) {
@@ -967,6 +1239,7 @@ HTTPRepository::failure() const
            http->cancelRequest(*rq, "Repository object deleted");
        }

+        flushHashCaches();
        directories.clear(); // wil delete them all
    }

@@ -986,154 +1259,6 @@ HTTPRepository::failure() const
        return r;
    }

-
-    class HashEntryWithPath
-    {
-    public:
-        HashEntryWithPath(const SGPath& p) : path(p.utf8Str()) {}
-        bool operator()(const HTTPRepoPrivate::HashCacheEntry& entry) const
-        { return entry.filePath == path; }
-    private:
-        std::string path;
-    };
-
-    std::string HTTPRepoPrivate::hashForPath(const SGPath& p)
-    {
-        HashCache::iterator it = std::find_if(hashes.begin(), hashes.end(), HashEntryWithPath(p));
-        if (it != hashes.end()) {
-            // ensure data on disk hasn't changed.
-            // we could also use the file type here if we were paranoid
-            if ((p.sizeInBytes() == it->lengthBytes) && (p.modTime() == it->modTime)) {
-                return it->hashHex;
-            }
-
-            // entry in the cache, but it's stale so remove and fall through
-            hashes.erase(it);
-        }
-
-        std::string hash = computeHashForPath(p);
-        updatedFileContents(p, hash);
-        return hash;
-    }
-
-    std::string HTTPRepoPrivate::computeHashForPath(const SGPath& p)
-    {
-        if (!p.exists())
-            return {};
-
-        sha1nfo info;
-        sha1_init(&info);
-
-        const int bufSize = 1024 * 1024;
-        char* buf = static_cast<char*>(malloc(bufSize));
-        if (!buf) {
-            sg_io_exception("Couldn't allocate SHA1 computation buffer");
-        }
-
-        size_t readLen;
-        SGBinaryFile f(p);
-        if (!f.open(SG_IO_IN)) {
-            free(buf);
-            throw sg_io_exception("Couldn't open file for compute hash", p);
-        }
-        while ((readLen = f.read(buf, bufSize)) > 0) {
-            sha1_write(&info, buf, readLen);
-        }
-
-        f.close();
-        free(buf);
-        std::string hashBytes((char*) sha1_result(&info), HASH_LENGTH);
-        return strutils::encodeHex(hashBytes);
-    }
-
-    void HTTPRepoPrivate::updatedFileContents(const SGPath& p, const std::string& newHash)
-    {
-        // remove the existing entry
-        auto it = std::find_if(hashes.begin(), hashes.end(), HashEntryWithPath(p));
-        if (it != hashes.end()) {
-            hashes.erase(it);
-            ++hashCacheDirty;
-        }
-
-        if (newHash.empty()) {
-            return; // we're done
-        }
-
-        // use a cloned SGPath and reset its caching to force one stat() call
-        SGPath p2(p);
-        p2.set_cached(false);
-        p2.set_cached(true);
-
-        HashCacheEntry entry;
-        entry.filePath = p.utf8Str();
-        entry.hashHex = newHash;
-        entry.modTime = p2.modTime();
-        entry.lengthBytes = p2.sizeInBytes();
-        hashes.push_back(entry);
-
-        ++hashCacheDirty ;
-    }
-
-    void HTTPRepoPrivate::writeHashCache()
-    {
-        if (hashCacheDirty == 0) {
-            return;
-        }
-
-        SGPath cachePath = basePath;
-        cachePath.append(".hashes");
-        sg_ofstream stream(cachePath, std::ios::out | std::ios::trunc | std::ios::binary);
-        HashCache::const_iterator it;
-        for (it = hashes.begin(); it != hashes.end(); ++it) {
-            stream << it->filePath << "*" << it->modTime << "*"
-            << it->lengthBytes << "*" << it->hashHex << "\n";
-        }
-        stream.close();
-        hashCacheDirty = 0;
-    }
-
-    void HTTPRepoPrivate::parseHashCache()
-    {
-        hashes.clear();
-        SGPath cachePath = basePath;
-        cachePath.append(".hashes");
-        if (!cachePath.exists()) {
-            return;
-        }
-
-        sg_ifstream stream(cachePath, std::ios::in);
-
-        while (!stream.eof()) {
-            std::string line;
-            std::getline(stream,line);
-            line = simgear::strutils::strip(line);
-            if( line.empty() || line[0] == '#' )
-                continue;
-
-			string_list tokens = simgear::strutils::split(line, "*");
-            if( tokens.size() < 4 ) {
-                SG_LOG(SG_TERRASYNC, SG_WARN, "invalid entry in '" << cachePath << "': '" << line << "' (ignoring line)");
-                continue;
-            }
-            const std::string nameData = simgear::strutils::strip(tokens[0]);
-            const std::string timeData = simgear::strutils::strip(tokens[1]);
-            const std::string sizeData = simgear::strutils::strip(tokens[2]);
-            const std::string hashData = simgear::strutils::strip(tokens[3]);
-
-            if (nameData.empty() || timeData.empty() || sizeData.empty() || hashData.empty() ) {
-                SG_LOG(SG_TERRASYNC, SG_WARN, "invalid entry in '" << cachePath << "': '" << line << "' (ignoring line)");
-                continue;
-            }
-
-            HashCacheEntry entry;
-            entry.filePath = nameData;
-            entry.hashHex = hashData;
-            entry.modTime = strtol(timeData.c_str(), NULL, 10);
-            entry.lengthBytes = strtol(sizeData.c_str(), NULL, 10);
-            hashes.push_back(entry);
-        }
-    }
-
    class DirectoryWithPath
    {
    public:
@@ -1163,16 +1288,12 @@ HTTPRepository::failure() const
        if (it != directories.end()) {
          assert((*it)->absolutePath() == absPath);
          directories.erase(it);
-                } else {
-			// we encounter this code path when deleting an orphaned directory
-		}
+        } else {
+            // we encounter this code path when deleting an orphaned directory
+        }

-		Dir dir(absPath);
+        Dir dir(absPath);
 		bool result = dir.remove(true);
-
-		// update the hash cache too
-		updatedFileContents(absPath, std::string());
-
        return result;
    }

@@ -1208,17 +1329,10 @@ HTTPRepository::failure() const
        http->makeRequest(rr);
      }

-      // rate limit how often we write this, since otherwise
-      // it dominates the time on Windows. 256 seems about right,
-      // causes a write a few times a minute.
-      if (hashCacheDirty > 256) {
-        writeHashCache();
-      }
-
-      if (activeRequests.empty() && queuedRequests.empty()) {
-        isUpdating = false;
-        writeHashCache();
+      if (countDirtyHashCaches() > 32) {
+          flushHashCaches();
      }
+      checkForComplete();
    }

    void HTTPRepoPrivate::failedToGetRootIndex(HTTPRepository::ResultCode st)
@@ -1279,4 +1393,38 @@ HTTPRepository::failure() const
          failures.end());
    }

+    void HTTPRepoPrivate::scheduleUpdateOfChildren(HTTPDirectory* dir)
+    {
+      auto updateChildTask = [dir](const HTTPRepoPrivate *) {
+        dir->updateChildrenBasedOnHash();
+        return ProcessDone;
+      };
+
+      addTask(updateChildTask);
+    }
+
+    void HTTPRepoPrivate::addTask(RepoProcessTask task) {
+      pendingTasks.push_back(task);
+    }
+
+    int HTTPRepoPrivate::countDirtyHashCaches() const
+    {
+        int result = rootDir->isHashCacheDirty() ? 1 : 0;
+        for (const auto& dir : directories) {
+            if (dir->isHashCacheDirty()) {
+                ++result;
+            }
+        }
+
+        return result;
+    }
+
+    void HTTPRepoPrivate::flushHashCaches()
+    {
+        rootDir->writeHashCache();
+        for (const auto& dir : directories) {
+            dir->writeHashCache();
+        }
+    }
+
 } // of namespace simgear
@@ -62,6 +62,11 @@ public:

  virtual bool isDoingSync() const;

+  /**
+        @brief call this periodically to progress non-network tasks
+     */
+  void process();
+
  virtual ResultCode failure() const;

  virtual size_t bytesToDownload() const;
@@ -19,8 +19,11 @@

 #pragma once

+#include <deque>
+#include <functional>
 #include <memory>
 #include <string>
+#include <unordered_map>

 #include <simgear/io/HTTPClient.hxx>
 #include <simgear/misc/sg_path.hxx>
@@ -48,20 +51,11 @@ protected:
  size_t _contentSize = 0;
 };

-typedef SGSharedPtr<HTTPRepoGetRequest> RepoRequestPtr;
+using RepoRequestPtr = SGSharedPtr<HTTPRepoGetRequest>;

 class HTTPRepoPrivate {
 public:
-  struct HashCacheEntry {
-    std::string filePath;
-    time_t modTime;
-    size_t lengthBytes;
-    std::string hashHex;
-  };

-  typedef std::vector<HashCacheEntry> HashCache;
-  HashCache hashes;
-  int hashCacheDirty = 0;

  HTTPRepository::FailureVec failures;
  int maxPermittedFailures = 16;
@@ -89,18 +83,14 @@ public:
  HTTP::Request_ptr updateDir(HTTPDirectory *dir, const std::string &hash,
                              size_t sz);

-  std::string hashForPath(const SGPath &p);
-  void updatedFileContents(const SGPath &p, const std::string &newHash);
-  void parseHashCache();
-  std::string computeHashForPath(const SGPath &p);
-  void writeHashCache();
-
  void failedToGetRootIndex(HTTPRepository::ResultCode st);
  void failedToUpdateChild(const SGPath &relativePath,
                           HTTPRepository::ResultCode fileStatus);

  void updatedChildSuccessfully(const SGPath &relativePath);

+  void checkForComplete();
+
  typedef std::vector<RepoRequestPtr> RequestVector;
  RequestVector queuedRequests, activeRequests;

@@ -113,10 +103,24 @@ public:
  HTTPDirectory *getOrCreateDirectory(const std::string &path);
  bool deleteDirectory(const std::string &relPath, const SGPath &absPath);

+
  typedef std::vector<HTTPDirectory_ptr> DirectoryVector;
  DirectoryVector directories;

+  void scheduleUpdateOfChildren(HTTPDirectory *dir);
+
  SGPath installedCopyPath;
+
+  int countDirtyHashCaches() const;
+  void flushHashCaches();
+
+  enum ProcessResult { ProcessContinue, ProcessDone, ProcessFailed };
+
+  using RepoProcessTask = std::function<ProcessResult(HTTPRepoPrivate *repo)>;
+
+  void addTask(RepoProcessTask task);
+
+  std::deque<RepoProcessTask> pendingTasks;
 };

 } // namespace simgear
@@ -185,6 +185,9 @@ gzfilebuf::setcompressionstrategy( int comp_strategy )

 z_off_t
 gzfilebuf::approxOffset() {
+    #ifdef __OpenBSD__
+    z_off_t res = 0;
+    #else
    z_off_t res = gzoffset(file);

    if (res == -1) {
@@ -201,7 +204,7 @@ gzfilebuf::approxOffset() {
        SG_LOG( SG_GENERAL, SG_ALERT, errMsg );
        throw sg_io_exception(errMsg);
    }
-
+    #endif
    return res;
 }

@@ -42,6 +42,7 @@

 #include <simgear/bucket/newbucket.hxx>
 #include <simgear/misc/sg_path.hxx>
+#include <simgear/misc/strutils.hxx>
 #include <simgear/math/SGGeometry.hxx>
 #include <simgear/structure/exception.hxx>

@@ -563,6 +564,14 @@ bool SGBinObject::read_bin( const SGPath& file ) {
    // read headers
    unsigned int header;
    sgReadUInt( fp, &header );
+
+    if (sgReadError()) {
+        int code = 0;
+        const char* gzErrorString = gzerror(fp, &code);
+        gzclose(fp);
+        throw sg_io_exception("Unable to read BTG header: " + string{gzErrorString} + ", code =" + std::to_string(code), sg_location(file));
+    }
+
    if ( ((header & 0xFF000000) >> 24) == 'S' &&
         ((header & 0x00FF0000) >> 16) == 'G' ) {

@@ -31,6 +31,10 @@ using TestApi = simgear::HTTP::TestApi;

 std::string dataForFile(const std::string& parentName, const std::string& name, int revision)
 {
+    if (name == "zeroByteFile") {
+        return {};
+    }
+
    std::ostringstream os;
    // random content but which definitely depends on our tree location
    // and revision.
@@ -409,6 +413,7 @@ void waitForUpdateComplete(HTTP::Client* cl, HTTPRepository* repo)
        cl->update();
        testServer.poll();

+        repo->process();
        if (!repo->isDoingSync()) {
            return;
        }
@@ -423,6 +428,7 @@ void runForTime(HTTP::Client *cl, HTTPRepository *repo, int msec = 15) {
  while (start.elapsedMSec() < msec) {
    cl->update();
    testServer.poll();
+    repo->process();
    SGTimeStamp::sleepForMSec(1);
  }
 }
@@ -444,6 +450,7 @@ void testBasicClone(HTTP::Client* cl)
    verifyFileState(p, "fileA");
    verifyFileState(p, "dirA/subdirA/fileAAA");
    verifyFileState(p, "dirC/subdirA/subsubA/fileCAAA");
+    verifyFileState(p, "dirA/subdirA/zeroByteFile");

    global_repo->findEntry("fileA")->revision++;
    global_repo->findEntry("dirB/subdirA/fileBAA")->revision++;
@@ -909,6 +916,8 @@ int main(int argc, char* argv[])
    global_repo->defineFile("dirA/fileAC");
    global_repo->defineFile("dirA/subdirA/fileAAA");
    global_repo->defineFile("dirA/subdirA/fileAAB");
+    global_repo->defineFile("dirA/subdirA/zeroByteFile");
+
    global_repo->defineFile("dirB/subdirA/fileBAA");
    global_repo->defineFile("dirB/subdirA/fileBAB");
    global_repo->defineFile("dirB/subdirA/fileBAC");
@@ -31,6 +31,8 @@
 #include <simgear/sg_inlines.h>
 #include <simgear/io/sg_file.hxx>
 #include <simgear/misc/sg_dir.hxx>
+#include <simgear/misc/strutils.hxx>
+
 #include <simgear/io/iostreams/sgstream.hxx>
 #include <simgear/debug/logstream.hxx>
 #include <simgear/package/unzip.h>
@@ -509,32 +511,34 @@ public:
 		const size_t BUFFER_SIZE = 1024 * 1024;
 		void* buf = malloc(BUFFER_SIZE);

-		try {
-			int result = unzGoToFirstFile(zip);
-			if (result != UNZ_OK) {
-				throw sg_exception("failed to go to first file in archive");
-			}
-
-			while (true) {
-				extractCurrentFile(zip, (char*)buf, BUFFER_SIZE);
-				if (state == FILTER_STOPPED) {
-					break;
-				}
-
-				result = unzGoToNextFile(zip);
-				if (result == UNZ_END_OF_LIST_OF_FILE) {
-					break;
-				}
-				else if (result != UNZ_OK) {
-					throw sg_io_exception("failed to go to next file in the archive");
-				}
-			}
-			state = END_OF_ARCHIVE;
-		}
-		catch (sg_exception&) {
+        int result = unzGoToFirstFile(zip);
+        if (result != UNZ_OK) {
+            SG_LOG(SG_IO, SG_ALERT, outer->rootPath() << "failed to go to first file in archive:" << result);
            state = BAD_ARCHIVE;
+            free(buf);
+            unzClose(zip);
+            return;
        }
-        
+
+        while (true) {
+            extractCurrentFile(zip, (char*)buf, BUFFER_SIZE);
+            if (state == FILTER_STOPPED) {
+                state = END_OF_ARCHIVE;
+                break;
+            }
+
+            result = unzGoToNextFile(zip);
+            if (result == UNZ_END_OF_LIST_OF_FILE) {
+                state = END_OF_ARCHIVE;
+                break;
+            } else if (result != UNZ_OK) {
+                SG_LOG(SG_IO, SG_ALERT, outer->rootPath() << "failed to go to next file in archive:" << result);
+                state = BAD_ARCHIVE;
+                break;
+            }
+        }
+
+
        free(buf);
        unzClose(zip);
    }
@@ -592,7 +596,7 @@ public:

 		outFile.open(path, std::ios::binary | std::ios::trunc | std::ios::out);
 		if (outFile.fail()) {
-			throw sg_io_exception("failed to open output file for writing", path);
+			throw sg_io_exception("failed to open output file for writing:" + strutils::error_string(errno), path);
 		}

 		while (!eof) {
@@ -70,6 +70,11 @@ public:
 		Stop
 	};

+    SGPath rootPath() const
+    {
+        return _rootPath;
+    }
+
 protected:


@@ -29,6 +29,13 @@ public:
  /// Default constructor, initializes the instance to lat = lon = elev = 0
  SGGeod(void);

+  /**
+    return an SGGeod for which isValid() returns false.
+    This is necessaerby becuase for historical reasons, ther defaulrt constructor above initialsies to zero,zero,zero
+    which *is*
+   */
+  static SGGeod invalid();
+
  /// Factory from angular values in radians and elevation is 0
  static SGGeod fromRad(double lon, double lat);
  /// Factory from angular values in degrees and elevation is 0
@@ -663,3 +663,8 @@ SGGeodesy::radialIntersection(const SGGeod& a, double aRadial,
  result = SGGeod::fromGeoc(r);
  return true;
 }
+
+SGGeod SGGeod::invalid()
+{
+    return SGGeod::fromDeg(-999.9, -999.0);
+}
@@ -248,7 +248,7 @@ void naFreeContext(naContext c)
    // than I have right now. So instead I'm clearing the stack tops here, so
    // a freed context looks the same as a new one returned by initContext.

-    c->fTop = c->opTop = c->markTop = 0;
+    c->fTop = c->opTop = c->markTop = c->ntemps = 0;

    c->nextFree = globals->freeContexts;
    globals->freeContexts = c;
@@ -63,7 +63,7 @@ namespace nasal

    class NasalMainLoopRecipient : public simgear::Emesary::IReceiver {
    public:
-        NasalMainLoopRecipient() : receiveCount(0) {
+        NasalMainLoopRecipient() : receiveCount(0), Active(false), CanWait(false) {
            simgear::Emesary::GlobalTransmitter::instance()->Register(*this);
        }
        virtual ~NasalMainLoopRecipient() {
@@ -100,7 +100,14 @@ naRef naNewHash(struct Context* c)

 naRef naNewCode(struct Context* c)
 {
-    return naNew(c, T_CODE);
+    naRef r = naNew(c, T_CODE);
+    // naNew can return a previously used naCode. naCodeGen will init
+    // all these members but a GC can occur inside naCodeGen, so we see
+    // partially initalized state here. To avoid this, clear out the values
+    // which mark() cares about.
+    PTR(r).code->srcFile = naNil();
+    PTR(r).code->nConstants = 0;
+    return r;
 }

 naRef naNewCCode(struct Context* c, naCFunction fptr)
@@ -643,16 +643,53 @@ void Catalog::processAlternate(SGPropertyNode_ptr alt)
        return;
    }

-    // we have an alternate ID, and it's differnt from our ID, so let's
+    // we have an alternate ID, and it's different from our ID, so let's
    // define a new catalog
    if (!altId.empty()) {
-        SG_LOG(SG_GENERAL, SG_INFO, "Adding new catalog:" << altId << " as version alternate for " << id());
-        // new catalog being added
-        createFromUrl(root(), altUrl);
-
-        // and we can go idle now
+      // don't auto-re-add Catalogs the user has explicilty rmeoved, that would
+      // suck
+      const auto removedByUser = root()->explicitlyRemovedCatalogs();
+      auto it = std::find(removedByUser.begin(), removedByUser.end(), altId);
+      if (it != removedByUser.end()) {
        changeStatus(Delegate::FAIL_VERSION);
        return;
+      }
+
+      SG_LOG(SG_GENERAL, SG_WARN,
+             "Adding new catalog:" << altId << " as version alternate for "
+                                   << id());
+      // new catalog being added
+      auto newCat = createFromUrl(root(), altUrl);
+
+      bool didRun = false;
+      newCat->m_migratedFrom = this;
+
+      auto migratePackagesCb = [didRun](Catalog *c) mutable {
+        // removing callbacks is awkward, so use this
+        // flag to only run once. (and hence, we need to be mutable)
+        if (didRun)
+          return;
+
+        if (c->status() == Delegate::STATUS_REFRESHED) {
+          didRun = true;
+
+          string_list existing;
+          for (const auto &pack : c->migratedFrom()->installedPackages()) {
+            existing.push_back(pack->id());
+          }
+
+          const int count = c->markPackagesForInstallation(existing);
+          SG_LOG(
+              SG_GENERAL, SG_INFO,
+              "Marked " << count
+                        << " packages from previous catalog for installation");
+        }
+      };
+
+      newCat->addStatusCallback(migratePackagesCb);
+      // and we can go idle now
+      changeStatus(Delegate::FAIL_VERSION);
+      return;
    }

    SG_LOG(SG_GENERAL, SG_INFO, "Migrating catalog " << id() << " to new URL:" << altUrl);
@@ -661,6 +698,26 @@ void Catalog::processAlternate(SGPropertyNode_ptr alt)
    root()->makeHTTPRequest(dl);
 }

+int Catalog::markPackagesForInstallation(const string_list &packageIds) {
+  int result = 0;
+
+  for (const auto &id : packageIds) {
+    auto ourPkg = getPackageById(id);
+    if (!ourPkg)
+      continue;
+
+    auto existing = ourPkg->existingInstall();
+    if (!existing) {
+      ourPkg->markForInstall();
+      ++result;
+    }
+  } // of outer package ID candidates iteration
+
+  return result;
+}
+
+CatalogRef Catalog::migratedFrom() const { return m_migratedFrom; }
+
 } // of namespace pkg

 } // of namespace simgear
@@ -23,6 +23,7 @@
 #include <map>

 #include <simgear/misc/sg_path.hxx>
+#include <simgear/misc/strutils.hxx>
 #include <simgear/props/props.hxx>

 #include <simgear/structure/SGReferenced.hxx>
@@ -93,7 +94,7 @@ public:

    /**
     * retrieve all the packages in the catalog which are installed
-     * and have a pendig update
+     * and have a pending update
     */
    PackageList packagesNeedingUpdate() const;

@@ -151,7 +152,32 @@ public:
    
    bool isUserEnabled() const;
    void setUserEnabled(bool b);
-private:
+
+    /**
+     * Given a list of package IDs, mark all which exist in this package,
+     * for installation. ANy packahe IDs not present in this catalog,
+     * will be ignored.
+     *
+     * @result The number for packages newly marked for installation.
+     */
+    int markPackagesForInstallation(const string_list &packageIds);
+
+    /**
+     * When a catalog is added due to migration, this will contain the
+     * Catalog which triggered the add. Usually this will be a catalog
+     * corresponding to an earlier version.
+     *
+     * Note it's only valid at the time, the migration actually took place;
+     * when the new catalog is loaded from disk, this value will return
+     * null.
+     *
+     * This is intended to allow Uis to show a 'catalog was migrated'
+     * feedback, when they see a catalog refresh, which has a non-null
+     * value of this method.
+     */
+    CatalogRef migratedFrom() const;
+
+  private:
    Catalog(Root* aRoot);

    class Downloader;
@@ -197,6 +223,8 @@ private:
    PackageWeakMap m_variantDict;

    function_list<Callback> m_statusCallbacks;
+
+    CatalogRef m_migratedFrom;
 };

 } // of namespace pkg
@@ -827,7 +827,10 @@ void testVersionMigrateToId(HTTP::Client* cl)
        
        it = std::find(enabledCats.begin(), enabledCats.end(), altCat);
        SG_VERIFY(it != enabledCats.end());
-        
+
+        SG_CHECK_EQUAL(altCat->packagesNeedingUpdate().size(),
+                       1); // should be the 737
+
        // install a parallel package from the new catalog
        pkg::PackageRef p2 = root->getPackageById("org.flightgear.test.catalog-alt.b737-NG");
        SG_CHECK_EQUAL(p2->id(), "b737-NG");
@@ -840,8 +843,8 @@ void testVersionMigrateToId(HTTP::Client* cl)
        pkg::PackageRef p3 = root->getPackageById("b737-NG");
        SG_CHECK_EQUAL(p2, p3);
    }
-    
-    // test that re-init-ing doesn't mirgate again
+
+    // test that re-init-ing doesn't migrate again
    {
        pkg::RootRef root(new pkg::Root(rootPath, "7.5"));
        root->setHTTPClient(cl);
@@ -1184,6 +1187,128 @@ void testMirrorsFailure(HTTP::Client* cl)
    
 }

+void testMigrateInstalled(HTTP::Client *cl) {
+  SGPath rootPath(simgear::Dir::current().path());
+  rootPath.append("pkg_migrate_installed");
+  simgear::Dir pd(rootPath);
+  pd.removeChildren();
+
+  pkg::RootRef root(new pkg::Root(rootPath, "8.1.2"));
+  root->setHTTPClient(cl);
+
+  pkg::CatalogRef oldCatalog, newCatalog;
+
+  {
+    oldCatalog = pkg::Catalog::createFromUrl(
+        root.ptr(), "http://localhost:2000/catalogTest1/catalog.xml");
+    waitForUpdateComplete(cl, root);
+
+    pkg::PackageRef p1 =
+        root->getPackageById("org.flightgear.test.catalog1.b747-400");
+    p1->install();
+    auto p2 = root->getPackageById("org.flightgear.test.catalog1.c172p");
+    p2->install();
+    auto p3 = root->getPackageById("org.flightgear.test.catalog1.b737-NG");
+    p3->install();
+    waitForUpdateComplete(cl, root);
+  }
+
+  {
+    newCatalog = pkg::Catalog::createFromUrl(
+        root.ptr(), "http://localhost:2000/catalogTest2/catalog.xml");
+    waitForUpdateComplete(cl, root);
+
+    string_list existing;
+    for (const auto &pack : oldCatalog->installedPackages()) {
+      existing.push_back(pack->id());
+    }
+
+    SG_CHECK_EQUAL(4, existing.size());
+
+    int result = newCatalog->markPackagesForInstallation(existing);
+    SG_CHECK_EQUAL(2, result);
+    SG_CHECK_EQUAL(2, newCatalog->packagesNeedingUpdate().size());
+
+    auto p1 = root->getPackageById("org.flightgear.test.catalog2.b737-NG");
+    auto ins = p1->existingInstall();
+    SG_CHECK_EQUAL(0, ins->revsion());
+  }
+
+  {
+    root->scheduleAllUpdates();
+    waitForUpdateComplete(cl, root);
+
+    SG_CHECK_EQUAL(0, newCatalog->packagesNeedingUpdate().size());
+
+    auto p1 = root->getPackageById("org.flightgear.test.catalog2.b737-NG");
+    auto ins = p1->existingInstall();
+    SG_CHECK_EQUAL(ins->revsion(), p1->revision());
+  }
+}
+
+void testDontMigrateRemoved(HTTP::Client *cl) {
+  global_catalogVersion = 2; // version which has migration info
+  SGPath rootPath(simgear::Dir::current().path());
+  rootPath.append("cat_dont_migrate_id");
+  simgear::Dir pd(rootPath);
+  pd.removeChildren();
+
+  // install and mnaully remove the alt catalog
+
+  {
+    pkg::RootRef root(new pkg::Root(rootPath, "8.1.2"));
+    root->setHTTPClient(cl);
+
+    pkg::CatalogRef c = pkg::Catalog::createFromUrl(
+        root.ptr(), "http://localhost:2000/catalogTest1/catalog-alt.xml");
+    waitForUpdateComplete(cl, root);
+
+    root->removeCatalogById("org.flightgear.test.catalog-alt");
+  }
+
+  // install the migration catalog
+  {
+    pkg::RootRef root(new pkg::Root(rootPath, "8.1.2"));
+    root->setHTTPClient(cl);
+
+    pkg::CatalogRef c = pkg::Catalog::createFromUrl(
+        root.ptr(), "http://localhost:2000/catalogTest1/catalog.xml");
+    waitForUpdateComplete(cl, root);
+    SG_VERIFY(c->isEnabled());
+  }
+
+  // change version to an alternate one
+  {
+    pkg::RootRef root(new pkg::Root(rootPath, "7.5"));
+
+    auto removed = root->explicitlyRemovedCatalogs();
+    auto j = std::find(removed.begin(), removed.end(),
+                       "org.flightgear.test.catalog-alt");
+    SG_VERIFY(j != removed.end());
+
+    root->setHTTPClient(cl);
+
+    // this would tirgger migration, but we blocked it
+    root->refresh(true);
+    waitForUpdateComplete(cl, root);
+
+    pkg::CatalogRef cat = root->getCatalogById("org.flightgear.test.catalog1");
+    SG_VERIFY(!cat->isEnabled());
+    SG_CHECK_EQUAL(cat->status(), pkg::Delegate::FAIL_VERSION);
+    SG_CHECK_EQUAL(cat->id(), "org.flightgear.test.catalog1");
+    SG_CHECK_EQUAL(cat->url(),
+                   "http://localhost:2000/catalogTest1/catalog.xml");
+
+    auto enabledCats = root->catalogs();
+    auto it = std::find(enabledCats.begin(), enabledCats.end(), cat);
+    SG_VERIFY(it == enabledCats.end());
+
+    // check the new catalog
+    auto altCat = root->getCatalogById("org.flightgear.test.catalog-alt");
+    SG_VERIFY(altCat.get() == nullptr);
+  }
+}
+
 int main(int argc, char* argv[])
 {
    sglog().setLogLevels( SG_ALL, SG_WARN );
@@ -1228,7 +1353,11 @@ int main(int argc, char* argv[])
    testInstallBadPackage(&cl);
    
    testMirrorsFailure(&cl);
-    
+
+    testMigrateInstalled(&cl);
+
+    testDontMigrateRemoved(&cl);
+
    cerr << "Successfully passed all tests!" << endl;
    return EXIT_SUCCESS;
 }
@@ -198,6 +198,11 @@ InstallRef Package::install()
 {
    InstallRef ins = existingInstall();
    if (ins) {
+      // if there's updates, treat this as a 'start update' request
+      if (ins->hasUpdate()) {
+        m_catalog->root()->scheduleToUpdate(ins);
+      }
+
        return ins;
    }

@@ -210,13 +215,39 @@ InstallRef Package::install()
    return ins;
 }

+InstallRef Package::markForInstall() {
+  InstallRef ins = existingInstall();
+  if (ins) {
+    return ins;
+  }
+
+  const auto pd = pathOnDisk();
+
+  Dir dir(pd);
+  if (!dir.create(0700)) {
+    SG_LOG(SG_IO, SG_ALERT,
+           "Package::markForInstall: couldn't create directory at:" << pd);
+    return {};
+  }
+
+  ins = new Install{this, pd};
+  _install_cb(this, ins); // not sure if we should trigger the callback for this
+
+  // repeat for dependencies to be kind
+  for (auto dep : dependencies()) {
+    dep->markForInstall();
+  }
+
+  return ins;
+}
+
 InstallRef Package::existingInstall(const InstallCallback& cb) const
 {
    InstallRef install;
    try {
        install = m_catalog->root()->existingInstallForPackage(const_cast<Package*>(this));
    } catch (std::exception& ) {
-        return InstallRef();
+      return {};
    }

  if( cb )
@@ -61,6 +61,13 @@ public:
    InstallRef
    existingInstall(const InstallCallback& cb = InstallCallback()) const;

+    /**
+     * Mark this package for installation, but don't actually start the
+     * download process. This creates the on-disk placeholder, so
+     * the package will appear an eededing to be updated.
+     */
+    InstallRef markForInstall();
+
    bool isInstalled() const;

    /**
@@ -133,7 +133,9 @@ public:

    void fireRefreshStatus(CatalogRef catalog, Delegate::StatusCode status)
    {
-        for (auto d : delegates) {
+        // take a copy of delegates since firing this can modify the data
+        const auto currentDelegates = delegates;
+        for (auto d : currentDelegates) {
            d->catalogRefreshed(catalog, status);
        }
    }
@@ -283,6 +285,32 @@ public:
        fireDataForThumbnail(url, reinterpret_cast<const uint8_t*>(bytes.data()), bytes.size());
    }

+    void writeRemovedCatalogsFile() const {
+      SGPath p = path / "RemovedCatalogs";
+      sg_ofstream stream(p, std::ios::out | std::ios::trunc | std::ios::binary);
+      for (const auto &cid : manuallyRemovedCatalogs) {
+        stream << cid << "\n";
+      }
+      stream.close();
+    }
+
+    void loadRemovedCatalogsFile() {
+      manuallyRemovedCatalogs.clear();
+      SGPath p = path / "RemovedCatalogs";
+      if (!p.exists())
+        return;
+
+      sg_ifstream stream(p, std::ios::in);
+      while (!stream.eof()) {
+        std::string line;
+        std::getline(stream, line);
+        const auto trimmed = strutils::strip(line);
+        if (!trimmed.empty()) {
+          manuallyRemovedCatalogs.push_back(trimmed);
+        }
+      } // of lines iteration
+    }
+
    DelegateVec delegates;

    SGPath path;
@@ -312,6 +340,9 @@ public:

    typedef std::map<PackageRef, InstallRef> InstallCache;
    InstallCache m_installs;
+
+    /// persistent list of catalogs the user has manually removed
+    string_list manuallyRemovedCatalogs;
 };


@@ -400,6 +431,8 @@ Root::Root(const SGPath& aPath, const std::string& aVersion) :
        thumbsCacheDir.create(0755);
    }

+    d->loadRemovedCatalogsFile();
+
    for (SGPath c : dir.children(Dir::TYPE_DIR | Dir::NO_DOT_OR_DOTDOT)) {
        // note this will set the catalog status, which will insert into
        // disabled catalogs automatically if necesary
@@ -621,6 +654,13 @@ void Root::scheduleToUpdate(InstallRef aInstall)
    }
 }

+void Root::scheduleAllUpdates() {
+  auto toBeUpdated = packagesNeedingUpdate(); // make a copy
+  for (const auto &u : toBeUpdated) {
+    scheduleToUpdate(u->existingInstall());
+  }
+}
+
 bool Root::isInstallQueued(InstallRef aInstall) const
 {
    auto it = std::find(d->updateDeque.begin(), d->updateDeque.end(), aInstall);
@@ -684,12 +724,6 @@ void Root::catalogRefreshStatus(CatalogRef aCat, Delegate::StatusCode aReason)
    auto catIt = d->catalogs.find(aCat->id());
    d->fireRefreshStatus(aCat, aReason);

-    if (aReason == Delegate::STATUS_IN_PROGRESS) {
-        d->refreshing.insert(aCat);
-    } else {
-        d->refreshing.erase(aCat);
-    }
-
    if (aCat->isUserEnabled() &&
        (aReason == Delegate::STATUS_REFRESHED) && 
        (catIt == d->catalogs.end())) 
@@ -723,6 +757,17 @@ void Root::catalogRefreshStatus(CatalogRef aCat, Delegate::StatusCode aReason)
        }
    } // of catalog is disabled

+    // remove from refreshing /after/ checking for enable / disabled, since for
+    // new catalogs, the reference in d->refreshing might be our /only/
+    // reference to the catalog. Once the refresh is done (either failed or
+    // succeeded) the Catalog will be in either d->catalogs or
+    // d->disabledCatalogs
+    if (aReason == Delegate::STATUS_IN_PROGRESS) {
+      d->refreshing.insert(aCat);
+    } else {
+      d->refreshing.erase(aCat);
+    }
+
    if (d->refreshing.empty()) {
        d->fireRefreshStatus(CatalogRef(), Delegate::STATUS_REFRESHED);
        d->firePackagesChanged();
@@ -783,6 +828,9 @@ bool Root::removeCatalogById(const std::string& aId)
            << "failed to remove directory");
    }

+    d->manuallyRemovedCatalogs.push_back(aId);
+    d->writeRemovedCatalogsFile();
+
    // notify that a catalog is being removed
    d->firePackagesChanged();

@@ -854,6 +902,10 @@ void Root::unregisterInstall(InstallRef ins)
    d->fireFinishUninstall(ins->package());
 }

+string_list Root::explicitlyRemovedCatalogs() const {
+  return d->manuallyRemovedCatalogs;
+}
+
 } // of namespace pkg

 } // of namespace simgear
@@ -155,7 +155,22 @@ public:
    void requestThumbnailData(const std::string& aUrl);
    
    bool isInstallQueued(InstallRef aInstall) const;
-private:
+
+    /**
+     * Mark all 'to be updated' packages for update now
+     */
+    void scheduleAllUpdates();
+
+    /**
+     * @brief list of catalog IDs, the user has explicitly removed via
+     * removeCatalogById(). This is important to allow the user to opt-out
+     * of migrated packages.
+     *
+     * This information is stored in a helper file, in the root directory
+     */
+    string_list explicitlyRemovedCatalogs() const;
+
+  private:
    friend class Install;
    friend class Catalog;    
    friend class Package;
@@ -0,0 +1,86 @@
+<?xml version="1.0"?>
+
+<PropertyList>
+    <id>org.flightgear.test.catalog2</id>
+    <description>Second test catalog</description>
+    <url>http://localhost:2000/catalogTest2/catalog.xml</url>
+    <catalog-version>4</catalog-version>
+
+    <version>8.1.*</version>
+    <version>8.0.0</version>
+    <version>8.2.0</version>
+
+    <package>
+        <id>alpha</id>
+        <name>Alpha package</name>
+        <revision type="int">8</revision>
+        <file-size-bytes type="int">593</file-size-bytes>
+
+        <md5>a469c4b837f0521db48616cfe65ac1ea</md5>
+        <url>http://localhost:2000/catalogTest1/alpha.zip</url>
+
+        <dir>alpha</dir>
+
+    </package>
+
+
+    <package>
+        <id>b737-NG</id>
+        <name>Boeing 737 NG</name>
+        <dir>b737NG</dir>
+        <description>A popular twin-engined narrow body jet</description>
+        <revision type="int">111</revision>
+        <file-size-bytes type="int">860</file-size-bytes>
+
+        <tag>boeing</tag>
+        <tag>jet</tag>
+        <tag>ifr</tag>
+
+      <!-- not within a localized element -->
+          <de>
+            <description>German description of B737NG XYZ</description>
+          </de>
+          <fr>
+            <description>French description of B737NG</description>
+          </fr>
+
+        <rating>
+          <FDM type="int">5</FDM>
+          <systems type="int">5</systems>
+          <model type="int">4</model>
+          <cockpit type="int">4</cockpit>
+        </rating>
+
+        <md5>a94ca5704f305b90767f40617d194ed6</md5>
+        <url>http://localhost:2000/mirrorA/b737.tar.gz</url>
+        <url>http://localhost:2000/mirrorB/b737.tar.gz</url>
+        <url>http://localhost:2000/mirrorC/b737.tar.gz</url>
+      </package>
+
+    <package>
+        <id>b747-400</id>
+        <name>Boeing 747-400</name>
+        <dir>b744</dir>
+        <description>A popular four-engined wide-body jet</description>
+        <revision type="int">111</revision>
+        <file-size-bytes type="int">860</file-size-bytes>
+
+        <tag>boeing</tag>
+        <tag>jet</tag>
+        <tag>ifr</tag>
+
+  
+
+        <rating>
+          <FDM type="int">5</FDM>
+          <systems type="int">5</systems>
+          <model type="int">4</model>
+          <cockpit type="int">4</cockpit>
+        </rating>
+
+        <md5>4d3f7417d74f811aa20ccc4f35673d20</md5>
+        <!-- this URL will sometimes fail, on purpose -->
+        <url>http://localhost:2000/catalogTest1/b747.tar.gz</url>
+      </package>
+
+</PropertyList>
@@ -59,6 +59,12 @@ void AtomicChangeListener::fireChangeListeners()
    listeners.clear();
 }

+void AtomicChangeListener::clearPendingChanges()
+{
+    auto& listeners = ListenerListSingleton::instance()->listeners;
+    listeners.clear();
+}
+
 void AtomicChangeListener::valueChangedImplementation()
 {
    if (!_dirty) {
@@ -52,7 +52,17 @@ public:
    bool isDirty() { return _dirty; }
    bool isValid() { return _valid; }
    virtual void unregister_property(SGPropertyNode* node) override;
+
    static void fireChangeListeners();
+
+    /**
+     * @brief Ensure we've deleted any pending changes. 
+     * 
+     * This is important in shutdown and reset, to avoid holding
+     * property listeners around after the property tree is destroyed
+     */
+    static void clearPendingChanges();
+
 private:
    virtual void valueChangedImplementation() override;
    virtual void valuesChanged();
@@ -19,6 +19,10 @@
 #include <sstream>
 #include <iomanip>
 #include <iterator>
+#include <exception> // can't use sg_exception becuase of PROPS_STANDALONE
+#include <mutex>
+#include <thread>
+
 #include <stdio.h>
 #include <string.h>

@@ -51,11 +55,15 @@ using namespace simgear;

 struct SGPropertyNodeListeners
 {
+  /* Protect _num_iterators and _items. We use a recursive mutex to allow
+  nested access to work as normal. */
+  std::recursive_mutex  _rmutex;
+
  /* This keeps a count of the current number of nested invocations of
  forEachListener(). If non-zero, other code higher up the stack is iterating
  _items[] so for example code must not erase items in the vector. */
  int _num_iterators = 0;
-  
+
  std::vector<SGPropertyChangeListener *> _items;
 };

@@ -558,23 +566,15 @@ find_node (SGPropertyNode * current,
  }
 }
 #else
-template<typename Range>
-SGPropertyNode*
-find_node (SGPropertyNode * current,
-           const Range& path,
-           bool create,
-           int last_index = -1)
-{
+template <typename Range>
+SGPropertyNode *find_node(SGPropertyNode *current, const Range &path,
+                          bool create, int last_index = -1) {
  using namespace boost;
-  typedef split_iterator<typename range_result_iterator<Range>::type>
-    PathSplitIterator;
-
-  PathSplitIterator itr
-    = make_split_iterator(path, first_finder("/", is_equal()));
+  auto itr = make_split_iterator(path, first_finder("/", is_equal()));
  if (*path.begin() == '/')
    return find_node_aux(current->getRootNode(), itr, create, last_index);
-   else
-     return find_node_aux(current, itr, create, last_index);
+  else
+    return find_node_aux(current, itr, create, last_index);
 }
 #endif

@@ -2406,6 +2406,7 @@ SGPropertyNode::addChangeListener (SGPropertyChangeListener * listener,
  if (_listeners == 0)
    _listeners = new SGPropertyNodeListeners;

+  std::lock_guard<std::recursive_mutex> lock(_listeners->_rmutex);
  /* If there's a nullptr entry (a listener that was unregistered), we
  overwrite it. This ensures that listeners that routinely unregister+register
  themselves don't make _listeners->_items grow unnecessarily. Otherwise simply
@@ -2429,9 +2430,13 @@ SGPropertyNode::removeChangeListener (SGPropertyChangeListener * listener)
 {
  if (_listeners == 0)
    return;
+  /* We use a std::unique_lock rather than a std::lock_guard because we may
+  need to unlock early. */
+  std::unique_lock<std::recursive_mutex> lock(_listeners->_rmutex);
  vector<SGPropertyChangeListener*>::iterator it =
    find(_listeners->_items.begin(), _listeners->_items.end(), listener);
  if (it != _listeners->_items.end()) {
+    assert(_listeners->_num_iterators >= 0);
    if (_listeners->_num_iterators) {
      /* _listeners._items is currently being iterated further up the stack in
      this thread by one or more nested invocations of forEachListener(), so
@@ -2450,6 +2455,7 @@ SGPropertyNode::removeChangeListener (SGPropertyChangeListener * listener)
      _listeners->_items.erase(it);
      listener->unregister_property(this);
      if (_listeners->_items.empty()) {
+        lock.unlock();
        delete _listeners;
        _listeners = 0;
      }
@@ -2511,9 +2517,11 @@ static void forEachListener(
    )
 {
  if (!_listeners) return;
-  
+
+  std::lock_guard<std::recursive_mutex> lock(_listeners->_rmutex);
+  assert(_listeners->_num_iterators >= 0);
  _listeners->_num_iterators += 1;
-  
+
  /* We need to use an index here when iterating _listeners->_items, not an
  iterator. This is because a listener may add new listeners, causing the
  vector to be reallocated, which would invalidate any iterator. */
@@ -2528,10 +2536,12 @@ static void forEachListener(
      }
    }
  }
-  
+
  _listeners->_num_iterators -= 1;
-  
+  assert(_listeners->_num_iterators >= 0);
+
  if (_listeners->_num_iterators == 0) {
+
    /* Remove any items that have been set to nullptr. */
    _listeners->_items.erase(
        std::remove(_listeners->_items.begin(), _listeners->_items.end(), (SGPropertyChangeListener*) nullptr),
@@ -2547,6 +2557,7 @@ static void forEachListener(
 int SGPropertyNode::nListeners() const
 {
  if (!_listeners) return 0;
+  std::lock_guard<std::recursive_mutex> lock(_listeners->_rmutex);
  int   n = 0;
  for (auto listener: _listeners->_items) {
    if (listener)   n += 1;
@@ -76,7 +76,7 @@
 #include <simgear/structure/SGExpression.hxx>
 #include <simgear/props/props_io.hxx>
 #include <simgear/props/vectorPropTemplates.hxx>
-
+#include <simgear/debug/ErrorReportingCallback.hxx>
 #include <simgear/io/iostreams/sgstream.hxx>

 namespace simgear
@@ -928,7 +928,8 @@ void ShaderProgramBuilder::buildAttribute(Effect* effect, Pass* pass,
        // FIXME orig: const string& shaderName = shaderKey.first;
        string shaderName = shaderKey.first;
        Shader::Type stype = (Shader::Type)shaderKey.second;
-        if (getPropertyRoot()->getBoolValue("/sim/version/compositor-support", false) &&
+        const bool compositorEnabled = getPropertyRoot()->getBoolValue("/sim/version/compositor-support", false);
+        if (compositorEnabled &&
            shaderName.substr(0, shaderName.find("/")) == "Shaders") {
            shaderName = "Compositor/" + shaderName;
        }
@@ -936,7 +937,9 @@ void ShaderProgramBuilder::buildAttribute(Effect* effect, Pass* pass,
        if (fileName.empty())
        {
            SG_LOG(SG_INPUT, SG_ALERT, "Could not locate shader" << shaderName);
-
+            if (!compositorEnabled) {
+                reportError("Missing shader", shaderName);
+            }

            throw BuilderException(string("couldn't find shader ") +
                shaderName);
@@ -1481,10 +1484,8 @@ void mergeSchemesFallbacks(Effect *effect, const SGReaderWriterOptions *options)

 // Walk the techniques property tree, building techniques and
 // passes.
-static std::mutex  realizeTechniques_lock;
 bool Effect::realizeTechniques(const SGReaderWriterOptions* options)
 {
-    std::lock_guard<std::mutex> g(realizeTechniques_lock);
    if (getPropertyRoot()->getBoolValue("/sim/version/compositor-support", false))
        mergeSchemesFallbacks(this, options);

@@ -273,11 +273,19 @@ bool setAttrs(const TexTuple& attrs, Texture* tex,
        options->setLoadOriginHint(SGReaderWriterOptions::LoadOriginHint::ORIGIN_EFFECTS_NORMALIZED);
    else
        options->setLoadOriginHint(SGReaderWriterOptions::LoadOriginHint::ORIGIN_EFFECTS);
-#if OSG_VERSION_LESS_THAN(3,4,2)
-    result = osgDB::readImageFile(imageName, options);
-#else
-    result = osgDB::readRefImageFile(imageName, options);
-#endif
+
+    try {
+    #if OSG_VERSION_LESS_THAN(3,4,2)
+        result = osgDB::readImageFile(imageName, options);
+    #else
+        result = osgDB::readRefImageFile(imageName, options);
+    #endif
+    } catch (std::bad_alloc& ba) {
+        SG_LOG(SG_GL, SG_ALERT, "Bad allocation loading:" << imageName);
+        // todo: report low memory warning
+        return false;
+    }
+
    options->setLoadOriginHint(origLOH);
    osg::ref_ptr<osg::Image> image;
    if (result.success())
@@ -393,6 +393,10 @@ ModelRegistry::readImage(const string& fileName,
                        isEffect = true;
 //                        can_compress = false;
                    }
+                    else if (sgoptC && !transparent && sgoptC->getLoadOriginHint() == SGReaderWriterOptions::LoadOriginHint::ORIGIN_CANVAS) {
+                        SG_LOG(SG_IO, SG_INFO, "From Canvas " + absFileName + " will generate mipmap only");
+                        can_compress = false;
+                    }
                    if (can_compress)
                    {
                        std::string pot_message;
@@ -1135,16 +1135,8 @@ void SpinAnimCallback::operator()(osg::Node* node, osg::NodeVisitor* nv)
        double intPart;
        double rot = modf(rotation, &intPart);
        double angle = rot * 2.0 * osg::PI;
-        const SGVec3d& sgcenter = transform->getCenter();
-        const SGVec3d& sgaxis = transform->getAxis();
-        Matrixd mat = Matrixd::translate(-sgcenter[0], -sgcenter[1], -sgcenter[2])
-            * Matrixd::rotate(angle, sgaxis[0], sgaxis[1], sgaxis[2])
-            * Matrixd::translate(sgcenter[0], sgcenter[1], sgcenter[2])
-            * *cv->getModelViewMatrix();
-        ref_ptr<RefMatrix> refmat = new RefMatrix(mat);
-        cv->pushModelViewMatrix(refmat.get(), transform->getReferenceFrame());
+        transform->setAngleRad(angle);
        traverse(transform, nv);
-        cv->popModelViewMatrix();
    } else {
        traverse(transform, nv);
    }
@@ -294,7 +294,7 @@ ReaderWriterSPT::createTree(const BucketBox& bucketBox, const LocalOptions& opti
 osg::ref_ptr<osg::Node>
 ReaderWriterSPT::createPagedLOD(const BucketBox& bucketBox, const LocalOptions& options) const
 {
-    osg::PagedLOD* pagedLOD = new osg::PagedLOD;
+    osg::ref_ptr<osg::PagedLOD> pagedLOD = new osg::PagedLOD;

    pagedLOD->setCenterMode(osg::PagedLOD::USER_DEFINED_CENTER);
    SGSpheref sphere = bucketBox.getBoundingSphere();
@@ -212,7 +212,8 @@ struct ReaderWriterSTG::_ModelBin {
            STGObjectsQuadtree quadtree((GetModelLODCoord()), (AddModelLOD()));
            quadtree.buildQuadTree(_objectStaticList.begin(), _objectStaticList.end());
            osg::ref_ptr<osg::Group> group = quadtree.getRoot();
-            group->setName("STG-group-A");
+            string group_name = string("STG-group-A ").append(_bucket.gen_index_str());
+            group->setName(group_name);
            group->setDataVariance(osg::Object::STATIC);

            simgear::AirportSignBuilder signBuilder(_options->getMaterialLib(), _bucket.get_center());
@@ -586,10 +587,12 @@ struct ReaderWriterSTG::_ModelBin {
    {
        osg::ref_ptr<SGReaderWriterOptions> options;
        options = SGReaderWriterOptions::copyOrCreate(opt);
+        float pagedLODExpiry = atoi(options->getPluginStringData("SimGear::PAGED_LOD_EXPIRY").c_str());

        osg::ref_ptr<osg::Group> terrainGroup = new osg::Group;
        terrainGroup->setDataVariance(osg::Object::STATIC);
-        terrainGroup->setName("terrain");
+        std::string terrain_name = string("terrain ").append(bucket.gen_index_str());
+        terrainGroup->setName(terrain_name);

        if (_foundBase) {
            for (auto stgObject : _objectList) {
@@ -637,11 +640,13 @@ struct ReaderWriterSTG::_ModelBin {
        } else {
            osg::PagedLOD* pagedLOD = new osg::PagedLOD;
            pagedLOD->setCenterMode(osg::PagedLOD::USE_BOUNDING_SPHERE_CENTER);
-            pagedLOD->setName("pagedObjectLOD");
+            std::string name = string("pagedObjectLOD ").append(bucket.gen_index_str());
+            pagedLOD->setName(name);

            // This should be visible in any case.
            // If this is replaced by some lower level of detail, the parent LOD node handles this.
            pagedLOD->addChild(terrainGroup, 0, std::numeric_limits<float>::max());
+            pagedLOD->setMinimumExpiryTime(0, pagedLODExpiry);

            // we just need to know about the read file callback that itself holds the data
            osg::ref_ptr<DelayLoadReadFileCallback> readFileCallback = new DelayLoadReadFileCallback;
@@ -658,10 +663,11 @@ struct ReaderWriterSTG::_ModelBin {

            // Objects may end up displayed up to 2x the object range.
            pagedLOD->setRange(pagedLOD->getNumChildren(), 0, 2.0 * _object_range_rough);
+            pagedLOD->setMinimumExpiryTime(pagedLOD->getNumChildren(), pagedLODExpiry);
            pagedLOD->setRadius(SG_TILE_RADIUS);
-            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile PagedLOD Center: " << pagedLOD->getCenter().x() << "," << pagedLOD->getCenter().y() << "," << pagedLOD->getCenter().z() );
-            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile PagedLOD Range: " << (2.0 * _object_range_rough));
-            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile PagedLOD Radius: " << SG_TILE_RADIUS);
+            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile " << bucket.gen_index_str() << " PagedLOD Center: " << pagedLOD->getCenter().x() << "," << pagedLOD->getCenter().y() << "," << pagedLOD->getCenter().z() );
+            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile " << bucket.gen_index_str() << " PagedLOD Range: " << (2.0 * _object_range_rough));
+            SG_LOG( SG_TERRAIN, SG_DEBUG, "Tile " << bucket.gen_index_str() << " PagedLOD Radius: " << SG_TILE_RADIUS);
            return pagedLOD;
        }
    }
@@ -441,6 +441,10 @@ typedef QuadTreeBuilder<LOD*, SGBuildingBin::BuildingInstance, MakeBuildingLeaf,
      if (hash_pos != std::string::npos)
          line.resize(hash_pos);

+      if (line.empty()) {
+          continue; // skip blank lines
+      }
+
      // and process further
      std::stringstream in(line);

@@ -41,6 +41,7 @@
 #include <simgear/scene/material/EffectGeode.hxx>
 #include <simgear/scene/material/mat.hxx>
 #include <simgear/scene/material/matlib.hxx>
+#include <simgear/scene/model/BoundingVolumeBuildVisitor.hxx>
 #include <simgear/scene/util/OsgMath.hxx>
 #include <simgear/scene/util/VectorArrayAdapter.hxx>
 #include <simgear/scene/util/SGNodeMasks.hxx>
@@ -340,5 +341,10 @@ osg::Node* SGOceanTile(const SGBucket& b, SGMaterialLib *matlib, int latPoints,
    transform->addChild(geode);
    transform->setNodeMask( ~(simgear::CASTSHADOW_BIT | simgear::MODELLIGHT_BIT) );

+    // Create a BVH at this point.  This is normally provided by the file loader, but as we create the 
+    // geometry programmatically, no file loader is involved.
+    BoundingVolumeBuildVisitor bvhBuilder(false);
+    transform->accept(bvhBuilder);
+    
    return transform;
 }
@@ -334,6 +334,10 @@ public:
    void runInternal();
    void updateSyncSlot(SyncSlot& slot);

+    void beginSyncAirports(SyncSlot& slot);
+    void beginSyncTile(SyncSlot& slot);
+    void beginNormalSync(SyncSlot& slot);
+
    void drainWaitingTiles();

    // commond helpers between both internal and external models
@@ -547,6 +551,7 @@ void SGTerraSync::WorkerThread::run()
 void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
 {
    if (slot.repository.get()) {
+        slot.repository->process();
        if (slot.repository->isDoingSync()) {
 #if 1
            if (slot.stamp.elapsedMSec() > (int)slot.nextWarnTimeout) {
@@ -567,6 +572,15 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
            notFound(slot.currentItem);
        } else if (res != HTTPRepository::REPO_NO_ERROR) {
            fail(slot.currentItem);
+
+            // in case the Airports_archive download fails, create the
+            // directory, so that next sync, we do a manual sync
+            if ((slot.currentItem._type == SyncItem::AirportData) && slot.isNewDirectory) {
+                SG_LOG(SG_TERRASYNC, SG_ALERT, "Failed to download Airports_archive, will download discrete files next time");
+                simgear::Dir d(_local_dir + "/Airports");
+                d.create(0755);
+                _completedTiles.erase(slot.currentItem._dir);
+            }
        } else {
            updated(slot.currentItem, slot.isNewDirectory);
            SG_LOG(SG_TERRASYNC, SG_DEBUG, "sync of " << slot.repository->baseUrl() << " finished ("
@@ -588,47 +602,14 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
        SGPath path(_local_dir);
        path.append(slot.currentItem._dir);
        slot.isNewDirectory = !path.exists();
-        if (slot.isNewDirectory) {
-            int rc = path.create_dir( 0755 );
-            if (rc) {
-                SG_LOG(SG_TERRASYNC,SG_ALERT,
-                       "Cannot create directory '" << path << "', return code = " << rc );
-                fail(slot.currentItem);
-                return;
-            }
-        } // of creating directory step
-
-        // optimise initial Airport download
-        if (slot.isNewDirectory &&
-            (slot.currentItem._type == SyncItem::AirportData)) {
-          SG_LOG(SG_TERRASYNC, SG_INFO, "doing Airports download via tarball");
-
-          // we want to sync the 'root' TerraSync dir, but not all of it, just
-          // the Airports_archive.tar.gz file so we use our TerraSync local root
-          // as the path (since the archive will add Airports/)
-          slot.repository.reset(new HTTPRepository(_local_dir, &_http));
-          slot.repository->setBaseUrl(_httpServer + "/");
-
-          // filter callback to *only* sync the Airport_archive tarball,
-          // and ensure no other contents are touched
-          auto f = [](const HTTPRepository::SyncItem &item) {
-            if (!item.directory.empty())
-              return false;
-            return (item.filename.find("Airports_archive.") == 0);
-          };
-
-          slot.repository->setFilter(f);
+        const auto type = slot.currentItem._type;

+        if (type == SyncItem::AirportData) {
+            beginSyncAirports(slot);
+        } else if (type == SyncItem::Tile) {
+            beginSyncTile(slot);
        } else {
-          slot.repository.reset(new HTTPRepository(path, &_http));
-          slot.repository->setBaseUrl(_httpServer + "/" +
-                                      slot.currentItem._dir);
-        }
-
-        if (_installRoot.exists()) {
-            SGPath p = _installRoot;
-            p.append(slot.currentItem._dir);
-            slot.repository->setInstalledCopyPath(p);
+            beginNormalSync(slot);
        }

        try {
@@ -651,6 +632,96 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot)
    }
 }

+void SGTerraSync::WorkerThread::beginSyncAirports(SyncSlot& slot)
+{
+    if (!slot.isNewDirectory) {
+        beginNormalSync(slot);
+        return;
+    }
+
+    SG_LOG(SG_TERRASYNC, SG_INFO, "doing Airports download via tarball");
+
+    // we want to sync the 'root' TerraSync dir, but not all of it, just
+    // the Airports_archive.tar.gz file so we use our TerraSync local root
+    // as the path (since the archive will add Airports/)
+    slot.repository.reset(new HTTPRepository(_local_dir, &_http));
+    slot.repository->setBaseUrl(_httpServer);
+
+    // filter callback to *only* sync the Airport_archive tarball,
+    // and ensure no other contents are touched
+    auto f = [](const HTTPRepository::SyncItem& item) {
+        if (!item.directory.empty())
+            return false;
+        return (item.filename.find("Airports_archive.") == 0);
+    };
+
+    slot.repository->setFilter(f);
+}
+
+void SGTerraSync::WorkerThread::beginSyncTile(SyncSlot& slot)
+{
+    // avoid 404 requests by doing a sync which excludes all paths
+    // except our tile path. In the case of a missing 1x1 tile, we will
+    // stop becuase all directories are filtered out, which is what we want
+
+    auto comps = strutils::split(slot.currentItem._dir, "/");
+    if (comps.size() != 3) {
+        SG_LOG(SG_TERRASYNC, SG_ALERT, "Bad tile path:" << slot.currentItem._dir);
+        beginNormalSync(slot);
+        return;
+    }
+
+    const auto tileCategory = comps.front();
+    const auto tenByTenDir = comps.at(1);
+    const auto oneByOneDir = comps.at(2);
+
+    const auto path = SGPath::fromUtf8(_local_dir) / tileCategory;
+    slot.repository.reset(new HTTPRepository(path, &_http));
+    slot.repository->setBaseUrl(_httpServer + "/" + tileCategory);
+
+    if (_installRoot.exists()) {
+      SGPath p = _installRoot / tileCategory;
+      slot.repository->setInstalledCopyPath(p);
+    }
+
+    const auto dirPrefix = tenByTenDir + "/" + oneByOneDir;
+
+    // filter callback to *only* sync the 1x1 dir we want, if it exists
+    // if doesn't, we'll simply stop, which is what we want
+    auto f = [tenByTenDir, oneByOneDir, dirPrefix](const HTTPRepository::SyncItem& item) {
+        // only allow the specific 10x10 and 1x1 dirs we want
+        if (item.directory.empty()) {
+            return item.filename == tenByTenDir;
+        } else if (item.directory == tenByTenDir) {
+            return item.filename == oneByOneDir;
+        }
+
+        // allow arbitrary children below dirPrefix, including sub-dirs
+        if (item.directory.find(dirPrefix) == 0) {
+            return true;
+        }
+
+        SG_LOG(SG_TERRASYNC, SG_ALERT, "Tile sync: saw weird path:" << item.directory << " file " << item.filename);
+        return false;
+    };
+
+    slot.repository->setFilter(f);
+}
+
+void SGTerraSync::WorkerThread::beginNormalSync(SyncSlot& slot)
+{
+    SGPath path(_local_dir);
+    path.append(slot.currentItem._dir);
+    slot.repository.reset(new HTTPRepository(path, &_http));
+    slot.repository->setBaseUrl(_httpServer + "/" + slot.currentItem._dir);
+
+    if (_installRoot.exists()) {
+      SGPath p = _installRoot;
+      p.append(slot.currentItem._dir);
+      slot.repository->setInstalledCopyPath(p);
+    }
+}
+
 void SGTerraSync::WorkerThread::runInternal()
 {
    while (!_stop) {
@@ -46,6 +46,7 @@ public:
        ORIGIN_EFFECTS,
        ORIGIN_EFFECTS_NORMALIZED,
        ORIGIN_SPLASH_SCREEN,
+        ORIGIN_CANVAS,
    };

    //SGReaderWriterOptions* cloneOptions(const osg::CopyOp& copyop = osg::CopyOp::SHALLOW_COPY) const { return static_cast<SGReaderWriterOptions*>(clone(copyop)); }
@@ -380,7 +380,8 @@ namespace simgear
 ALvoid* loadWAVFromFile(const SGPath& path, unsigned int& format, ALsizei& size, ALfloat& freqf, unsigned int& block_align)
 {
  if (!path.exists()) {
-    throw sg_io_exception("loadWAVFromFile: file not found", path);
+    SG_LOG(SG_IO, SG_DEV_ALERT, "loadWAVFromFile: file not found:" << path);
+    return nullptr;
  }

  Buffer b;
@@ -395,13 +396,15 @@ ALvoid* loadWAVFromFile(const SGPath& path, unsigned int& format, ALsizei& size,
  fd = gzopen(ps.c_str(), "rb");
 #endif
  if (!fd) {
-    throw sg_io_exception("loadWAVFromFile: unable to open file", path);
+    SG_LOG(SG_IO, SG_DEV_ALERT, "loadWAVFromFile: unable to open file:" << path);
+    return nullptr;
  }

  try {
      loadWavFile(fd, &b);
  } catch (sg_exception& e) {
-      throw sg_io_exception(e.getFormattedMessage() + "\nfor: " + path.str());
+      SG_LOG(SG_IO, SG_DEV_ALERT, "loadWAVFromFile:" << e.getFormattedMessage() << "\nfor: " << path);
+      return nullptr;
  }

  ALvoid* data = b.data;
@@ -813,13 +813,14 @@ bool SGSoundMgr::load( const std::string &samplepath,

    auto data = simgear::loadWAVFromFile(samplepath, format, size, freqf, blocksz);
    freq = (ALsizei)freqf;
-    if (data == nullptr) {
-        throw sg_io_exception("Failed to load wav file", sg_location(samplepath));
+    if (!data) {
+        return false;
    }

    if (format == AL_FORMAT_STEREO8 || format == AL_FORMAT_STEREO16) {
         free(data);
-        throw sg_io_exception("Warning: STEREO files are not supported for 3D audio effects: " + samplepath);
+        SG_LOG(SG_IO, SG_DEV_ALERT, "Warning: STEREO files are not supported for 3D audio effects: " << samplepath);
+        return false;
    }

    *dbuf = (void *)data;
@@ -0,0 +1,86 @@
+
+
+#if 0
+
+set bindings for action seperate from defintion ?
+    - in XML, especially aircraft XML
+
+define actions from command / Nasal
+add behaviours from Nasal 
+
+define keymapping
+    - manager of keybindings defined against actions?
+    - for a toggle or enum, define behaviour
+        - each key repeat cycles
+        - alternate key to go the other way (G/shift-G)
+
+release bindings for momentary actions:
+    button up / key-up
+
+send activate, release
+
+    send deactivate, release for 'alternate' action
+
+#endif
+
+
+void SGAction::setValueExpression()
+{
+    // watch all the properties
+}
+
+void SGAction::setValueCondition()
+{
+    // 
+}
+
+void SGAction::updateProperties()
+{
+    //
+    _node->setBoolValue("enabled", isEnabled());
+    switch (_type) {
+    case Momentary:
+    case Toggle:
+        _node->setBoolValue("value", getValue());
+        break;
+
+    case Enumerated: 
+        if (!_valueEnumeration.empty()) {
+            // map to the string value
+            _node->setStringValue("value", _valueEnumeration.at(getValue()));
+        } else {
+            // set as an integer
+            _node->setIntValue("value", getValue());
+        }
+    }
+
+    // set description
+}
+
+bool SGAction::isEnabled()
+{
+    if (_enableCondition) {
+
+    } else {
+        return _enabled;
+    }
+
+    updateProperties();
+}
+
+int SGAction::getValue()
+{
+    if (type == Enumerated) {
+        if (_valueExpression) {
+            // invoke it
+        }
+    } else {
+        if (_valueCondition) {
+            return _valueCondition.test();
+        }
+    }
+
+    return _value;
+}
+
+// commands enable-action, disable-action
@@ -441,7 +441,7 @@ void StateMachine::initFromPlist(SGPropertyNode* desc, SGPropertyNode* root)
        std::string nm = stateDesc->getStringValue("name");

        if (nm.empty()) {
-          SG_LOG(SG_GENERAL, SG_ALERT, "No name found for state in branch " << path);
+          SG_LOG(SG_GENERAL, SG_DEV_ALERT, "No name found for state in branch " << path);
          throw sg_exception("No name element in state");
        }

@@ -464,8 +464,8 @@ void StateMachine::initFromPlist(SGPropertyNode* desc, SGPropertyNode* root)
        std::string target_id = tDesc->getStringValue("target");

        if (nm.empty()) {
-          SG_LOG(SG_GENERAL, SG_ALERT, "No name found for transition in branch " << path);
-          throw sg_exception("No name element in transition");
+          SG_LOG(SG_GENERAL, SG_DEV_WARN, "No name found for transition in branch " << path);
+          nm = "transition-to-" + target_id;
        }

        if (target_id.empty()) {
@@ -30,9 +30,10 @@
 #include "subsystem_mgr.hxx"
 #include "commands.hxx"

-#include <simgear/props/props.hxx>
-#include <simgear/math/SGMath.hxx>
 #include "SGSmplstat.hxx"
+#include <simgear/debug/ErrorReportingCallback.hxx>
+#include <simgear/math/SGMath.hxx>
+#include <simgear/props/props.hxx>

 const int SG_MAX_SUBSYSTEM_EXCEPTIONS = 4;
 const char SUBSYSTEM_NAME_SEPARATOR = '.';
@@ -828,8 +829,18 @@ SGSubsystemGroup::Member::update (double delta_time_sec)
      if (++exceptionCount > SG_MAX_SUBSYSTEM_EXCEPTIONS) {
        SG_LOG(SG_GENERAL, SG_ALERT, "(exceptionCount=" << exceptionCount <<
          ", suspending)");
+        simgear::reportError("suspending subsystem after too many errors:" + name);
        subsystem->suspend();
      }
+    } catch (std::bad_alloc& ba) {
+        // attempting to track down source of these on Sentry.io
+        SG_LOG(SG_GENERAL, SG_ALERT, "caught bad_alloc processing subsystem:" << name);
+        simgear::reportError("caught bad_alloc processing subsystem:" + name);
+
+        if (++exceptionCount > SG_MAX_SUBSYSTEM_EXCEPTIONS) {
+            SG_LOG(SG_GENERAL, SG_ALERT, "(exceptionCount=" << exceptionCount << ", suspending)");
+            subsystem->suspend();
+        }
    }
 }
Author	SHA1	Message	Date
Automatic Release Builder	ca5f66da9f	new version: 2020.3.6	2021-01-20 13:01:25 +00:00
James Turner	92d053d850	Tolerate blank lines in buildings lists	2021-01-20 11:35:33 +00:00
Colin Geniet	001ae80723	Animations: Fix spin for Compositor lights The 'spin' animation has a strange behaviour: it pushes a temporary rotation matrix on the transformation stack during cull traversal, and removes it thereafter. Thus, the rotation matrix is missing outside of cull traversal, e.g. when the position of Compositor lights is computed. The same issue was fixed for the 'rotate' animation by commit `e202d4e4a5` (this mentions broken 'picking' animations as a different manifestation of the same issue). Fix this by setting the angle of the persistent SGRotateTransform, instead of creating a temporary rotation matrix.	2021-01-15 15:45:26 +00:00
James Turner	58d9a6d0b5	Sentry: adjust reporting of missing shaders Reduce noise on the backend when a shader is missing, by special- casing how we report it.	2021-01-05 12:58:46 +00:00
Scott Giese	261316cb45	Support Boost v1.75 Resolved a breaking change.	2021-01-02 21:57:54 -06:00
Stuart Buchanan	2ad4d2e672	Fix hang on "Loading Scenery" over the sea Previously attempting to start on an ocean tile with static models present cause FG to spin on "Loading Scenery". This was caused by not creating a BVH for an ocean tile in the specific case where there was STG models, due to a PagedLOD node being inserted in the scene graph. So FG never thought the scenery was loaded sufficient to place the aircraft. By explicitly creating a BVH for ocean tiles the problem is fixed. Candidate for 2020.X	2021-01-01 22:41:47 +00:00
James Turner	1d9fa929fa	Launhcer: fix crashes adding a catalog Traversing a container which is modified causes crashes, take a copy during traversal for firePackageStatus. Sentry-Id: FLIGHTGEAR-CJF Sentry-Id: FLIGHTGEAR-CJ5	2021-01-01 22:41:42 +00:00
James Turner	26bb6236a0	Effect builder: catch allocation failures Fail on bad_alloc of reading texture images, instead of crashing.	2020-12-28 15:08:49 +00:00
Scott Giese	b2acf5a623	fix: sky and cockpit to turn black Metar sky condition without base layer height (e.g. FEW///) is the cause of this issue. Randomize the base layer height when the actual height is unknown.	2020-12-26 10:53:49 -06:00
James Turner	bea88bb3a9	Fix BTG error reporting Use gzerror, not strerror, for these codes.	2020-12-21 15:31:36 +00:00
Automatic Release Builder	def2af2bdd	new version: 2020.3.5	2020-12-17 13:48:00 +00:00
Fernando García Liñán	6f6d705f22	Add support for anisotropic filtering in Canvas	2020-12-17 13:00:01 +00:00
Richard Harrison	e9c33104d3	DDS-TC add option to exclude all Canvas orignated images.	2020-12-17 12:59:26 +00:00
Richard Harrison	985897f8ba	Emesary : change to use scoped lock - scoped locks using lock_guard are better - because of this the exception handler can also be removed; although this was originally intended to manage the locks it never did work properly with C++ exceptions.	2020-12-17 12:57:53 +00:00
Richard Harrison	4251b28e88	Emesary: add missing property initialisation to constructor	2020-12-17 12:56:49 +00:00
James Turner	76540a211c	SGGeod: add static constructor of an invalid Geod Use this to allow explicitly initializing a value which isValid will return false for.	2020-12-17 12:55:30 +00:00
James Turner	904fc5a7dd	Fix Nasal GC errors on tests/reset Ensure the Context temps are cleared, and when recycling an naCode, ensure old values are cleared explicily. Sentry-Id: FLIGHTGEAR-Y	2020-12-17 12:55:25 +00:00
Stuart Buchanan	4aebc159d5	Use ref_ptr for ReaderWriterSPT	2020-12-07 17:18:32 +00:00
Richard Harrison	f89227fc1a	Remove mutex lock on realizeTechniques This was causing a deadlock - and I can't quite remember what problem it was intended to solve so it is best to remove it.	2020-12-07 17:17:44 +00:00
James Turner	f50f383cc0	Reporting of std::bad_alloc in Subsystem::update Trying to trace down our bad-alloc exception, starting with the simplest place for now.	2020-12-07 17:16:58 +00:00
James Turner	706ab387de	Add reporting callback option to SimGear Allows us to trigger an error logging callback explicitly, which can be used to drive Sentry.io on the FlightGear side.	2020-12-07 17:16:47 +00:00
James Turner	cddfdb7d1d	TerraSync: stronger fix for handling 0-byte files Change logic so we create an empty file for such cases, i.e exactly matching the repository. This simplifies logic in downstream code, compared with not creating a local file. Add a test-case to cover this Modify TerraSync to detect a failure of Airports_archive downloading, and fall back to file-by-file updating.	2020-12-07 17:15:51 +00:00
James Turner	3c64578848	HTTPRepository: don’t crash on empty files Fix some additional crash cases around 0-length files	2020-12-07 17:15:38 +00:00
James Turner	e1fe9b45e0	Unzip: adjust error reporting mechanism Don’t use local exception throw+catch to report failures in extracting a zip archive, since this generates noise in Sentry.	2020-12-07 17:15:30 +00:00
James Turner	8fdc1d306f	State-machines: don’t require name for transitions Allow anonymous transitions, since the name is purely informational (unlike for states). Sentry-Id: FLIGHTGEAR-9H	2020-12-07 17:15:17 +00:00
Automatic Release Builder	6e0c39bb68	new version: 2020.3.4	2020-11-30 11:17:52 +00:00
Julian Smith	f20b416cfe	simgear/props/props.cxx: use rmutex to protect SGPropertyNodeListeners. Also added asserts to check _num_iterators always >= 0.	2020-11-29 20:16:46 +00:00
Scott Giese	47e06b5216	METAR: mitigate wind sensor failures	2020-11-29 16:25:01 +00:00
James Turner	bfcdf22705	TerraSync: fix crashes with null file return Not sure how this is happening, but, check for a null file object in FileGetRequest::onDone.	2020-11-29 16:23:44 +00:00
Stuart Buchanan	d0db407faa	Set minimum expiry time on STG nodes.	2020-11-29 16:23:40 +00:00
Automatic Release Builder	d95b1c0441	new version: 2020.3.3	2020-11-12 11:14:23 +00:00
James Turner	837ba86d57	DNSClient: own requests, and cancel them on timeout Fixes crashes where a request times-out, but then is completed by UDN sometime afterwards, with a free-d object. Have the DNS::Client own requests, and be able to retrieve the udns_query to cancel them, in the timeout case. Fixes a couple of Sentry reports.	2020-11-12 09:39:58 +00:00
Automatic Release Builder	0cb1b463e1	Catalogs: fix ownership of new Catalogs When doing the initial download of a Catalog, ensure we still keep an owning ref to it.	2020-11-11 21:18:57 +00:00
Automatic Release Builder	8a772c8edd	new version: 2020.3.2	2020-11-04 22:08:28 +00:00
James Turner	03bdad0a10	Try to capture cause of BTG load failures	2020-11-04 11:52:58 +00:00
James Turner	537776e1f8	TerraSync: fix local-file copying Avoid downloading data corresponding to files shipped in FGData.	2020-11-04 11:22:48 +00:00
James Turner	f34a4a304e	Untar: log error details when output create fails Trying to understand why writing to the output files fails for some users. Sentry-Id: FLIGHTGEAR-SS	2020-11-03 21:26:03 +00:00
James Turner	a59c4e2c8b	TerraSync: make tarball extraction asynchronous	2020-11-03 17:49:19 +00:00
Julian Smith	46f4967f6e	Allow use of old zlib-1.2.3 on OpenBSD. As of 2020-08-01, OpenBSD's system zlib is 1.2.3 which doesn't have gzoffset(). However we can get away with this by making gzfilebuf::approxOffset() always return zero.	2020-11-01 11:12:00 +00:00
James Turner	9305417207	Terrasync: tarball extraction, use larger buffer Use a 1MB buffer, 2kByte is very 1979 :)	2020-11-01 11:11:31 +00:00
James Turner	11da8b33f9	TerraSync: switch to per-directory hash caching Avoid a central hash cache becoming enormous, now we use the selective download scheme for the tile dirs. Hash name is changed to co-exist with older versions.	2020-11-01 11:11:27 +00:00
James Turner	c7b320eb55	Fix Airports/ initial sync	2020-11-01 11:11:23 +00:00
James Turner	72b2eb0ebf	TerraSync: avoid 404s to probe missing tiles Use the top-level dirIndex files to determine if a 1x1 tile dir exists, instead of proving the server via a 404. This reduces the number of requests we make considerably, which is … important.	2020-11-01 11:11:19 +00:00
James Turner	ec3829addb	TerraSync: validate local dirs incrementally Add a process() method to HTTPRepository, and use this to incrementally validate subdirs after the .dirIndex is received. This avoids large pauses of the TerraSync thread, when all of Airports/ is validated at once.	2020-11-01 11:11:15 +00:00
James Turner	96bafef3f3	TerraSync: use an unordered_map for the hash cache Linear-scan is a bit slow in debug builds, for the large Airports/ tree; switch to an unordered_map. Will back-port to the LTS once tested a bit more.	2020-11-01 11:11:12 +00:00
James Turner	3ff3bd0a6c	Props: allow flushing the atomic change listener Trying to narrow down causes of the ‘unregister listeners crashes on shutdown’ reports.	2020-11-01 11:11:07 +00:00
Automatic Release Builder	78d073a0f0	new version: 2020.3.1	2020-10-26 09:09:22 +00:00
Scott Giese	d62796c19d	METAR: When Wind unit not specified, default to knots. FLIGHTGEAR-F6 resolved	2020-10-26 08:58:06 +00:00
Scott Giese	7ea7ff43fc	METAR Wind Sensor Failure. FLIGHTGEAR-F9 resolved.	2020-10-26 08:57:24 +00:00
James Turner	dafd185595	Fix for crash reported by Michael Danilov In case uDNS returns a NULL txt pointer, don’t try to create a std::string from it, since this will crash, See: https://sourceforge.net/p/flightgear/codetickets/2398/	2020-10-26 08:54:18 +00:00
Automatic Release Builder	1568ed8b97	Catalog migration: migrate packages too When doing a catalog migration to a new ID (eg, 2018 -> 2020), also mark the installed packages for installation, on the new catalog. Related to this, when manually removing a catalog, record this fact, so we don’t re-add it automatically due to migration. Add unit-tests covering both of these cases.	2020-10-22 21:27:47 +01:00
Automatic Release Builder	444e2ffb2d	Sound: readWAV: avoid common exceptions. Avoid exceptions for the common ‘file not found’ case, and instead return false / nullptr. Erik says it’s fine.	2020-10-15 17:00:30 +01:00
@@ -1 +1 @@
 .3.0
 .3.6