Packages: randomise mirror URL selection

Also add retry on mirrors when a download fails.

simgear/package/CatalogTest.cxx

@@ -62,6 +62,8 @@ unsigned int global_catalogVersion = 0;
 bool global_failRequests = false;
 bool global_fail747Request = true;
 
+int global_737RequestsToFailCount = 0;
+
 class TestPackageChannel : public TestServerChannel
 {
 public:
@@ -106,6 +108,15 @@ public:
             }
         }
         
+        if ((path.find("/mirror") == 0) && (path.find("/b737.tar.gz") == 8)) {
+            if (global_737RequestsToFailCount > 0) {
+                sendErrorResponse(404, false, "Mirror failure");
+                --global_737RequestsToFailCount;
+                return;
+            }
+            path = "/catalogTest1/b737.tar.gz";
+        }
+        
         localPath.append(path);
 
       //  SG_LOG(SG_IO, SG_INFO, "local path is:" << localPath.str());
@@ -329,7 +340,9 @@ int parseTest()
         SG_VERIFY(p3->matches(queryText.ptr()));
     }
     
-    
+    string_list urls = p3->downloadUrls();
+    SG_CHECK_EQUAL(urls.size(), 3);
+    SG_CHECK_EQUAL(urls.at(1), "http://localhost:2000/mirrorB/b737.tar.gz");
 
     delete root;
     return EXIT_SUCCESS;
@@ -1136,9 +1149,44 @@ void testInstallBadPackage(HTTP::Client* cl)
     SG_CHECK_EQUAL(ins->path(), rootPath / "org.flightgear.test.catalog1" / "Aircraft" / "b744");
 }
 
+void testMirrorsFailure(HTTP::Client* cl)
+{
+    global_catalogVersion = 0;
+    // there's three mirrors defined, so se tthe first twom attempts to fail
+    global_737RequestsToFailCount = 2;
+    
+    SGPath rootPath(simgear::Dir::current().path());
+    rootPath.append("pkg_install_mirror_fail");
+    simgear::Dir pd(rootPath);
+    pd.removeChildren();
+
+    pkg::RootRef root(new pkg::Root(rootPath, "8.1.2"));
+    // specify a test dir
+    root->setHTTPClient(cl);
+
+    pkg::CatalogRef c = pkg::Catalog::createFromUrl(root.ptr(), "http://localhost:2000/catalogTest1/catalog.xml");
+    waitForUpdateComplete(cl, root);
+
+    pkg::PackageRef p1 = root->getPackageById("org.flightgear.test.catalog1.b737-NG");
+    pkg::InstallRef ins = p1->install();
+    
+    bool didFail = false;
+    ins->fail([&didFail, &ins](pkg::Install* ourInstall) {
+        SG_CHECK_EQUAL(ins, ourInstall);
+        didFail = true;
+    });
+    
+    waitForUpdateComplete(cl, root);
+    SG_VERIFY(p1->isInstalled());
+    SG_VERIFY(!didFail);
+    SG_VERIFY(p1->existingInstall() == ins);
+    SG_CHECK_EQUAL(ins->status(), pkg::Delegate::STATUS_SUCCESS);
+    
+}
+
 int main(int argc, char* argv[])
 {
-   // sglog().setLogLevels( SG_ALL, SG_DEBUG );
+    sglog().setLogLevels( SG_ALL, SG_WARN );
 
     HTTP::Client cl;
     cl.setMaxConnections(1); 
@@ -1150,7 +1198,7 @@ int main(int argc, char* argv[])
     parseTest();
 
     parseInvalidTest();
-    
+
     testInstallPackage(&cl);
 
     testUninstall(&cl);
@@ -1179,6 +1227,8 @@ int main(int argc, char* argv[])
 
     testInstallBadPackage(&cl);
     
-    SG_LOG(SG_GENERAL, SG_INFO, "Successfully passed all tests!");
+    testMirrorsFailure(&cl);
+    
+    cerr << "Successfully passed all tests!" << endl;
     return EXIT_SUCCESS;
 }

simgear/package/Install.cxx

@@ -18,8 +18,8 @@
 #include <simgear_config.h>
 #include <simgear/package/Install.hxx>
 
-#include <boost/foreach.hpp>
 #include <fstream>
+#include <cstdlib>
 
 #include <simgear/package/unzip.h>
 #include <simgear/package/md5.h>
@@ -43,21 +43,17 @@ namespace pkg {
 class Install::PackageArchiveDownloader : public HTTP::Request
 {
 public:
-    PackageArchiveDownloader(InstallRef aOwner) :
+    PackageArchiveDownloader(InstallRef aOwner, const string_list& urls) :
         HTTP::Request("" /* dummy URL */),
         m_owner(aOwner)
     {
-        m_urls = m_owner->package()->downloadUrls();
+        m_urls = urls;
-        if (m_urls.empty()) {
+        if (urls.empty()) {
-            throw sg_exception("no package download URLs");
+            m_urls = m_owner->package()->downloadUrls();
         }
 
-       // if (m_owner->package()->properties()->hasChild("archive-type")) {
+        selectMirrorUrl();
-       //     setArchiveTypeFromExtension(m_owner->package()->properties()->getStringValue("archive-type"));
-        //}
         
-        // TODO randomise order of m_urls
-
         m_extractPath = aOwner->path().dir();
         m_extractPath.append("_extract_" + aOwner->package()->md5());
 
@@ -93,12 +89,19 @@ public:
         return (m_downloaded * 100) / responseLength();
     }
 protected:
-    virtual std::string url() const
+    void selectMirrorUrl()
     {
-        return m_urls.front();
+        const int randomizedIndex = rand() % m_urls.size();
+        m_activeURL = m_urls.at(randomizedIndex);
+        m_urls.erase(m_urls.begin() + randomizedIndex);
+    }
+    
+    virtual std::string url() const override
+    {
+        return m_activeURL;
     }
 
-    virtual void responseHeadersComplete()
+    void responseHeadersComplete() override
     {
         Request::responseHeadersComplete();
 
@@ -112,7 +115,7 @@ protected:
         m_owner->startDownload();
     }
 
-    virtual void gotBodyData(const char* s, int n)
+    void gotBodyData(const char* s, int n) override
     {
 		const uint8_t* ubytes = (uint8_t*) s;
         SG_MD5Update(&m_md5, ubytes, n);
@@ -121,10 +124,10 @@ protected:
 		m_extractor->extractBytes(ubytes, n);
     }
 
-    virtual void onDone()
+    void onDone() override
     {
         if (responseCode() != 200) {
-            SG_LOG(SG_GENERAL, SG_ALERT, "download failure:" << responseCode() <<
+            SG_LOG(SG_GENERAL, SG_WARN, "download failure:" << responseCode() <<
                    "\n\t" << url());
             Delegate::StatusCode code = Delegate::FAIL_DOWNLOAD;
             if (responseCode() == 404) {
@@ -141,7 +144,7 @@ protected:
           strutils::encodeHex(digest, MD5_DIGEST_LENGTH);
 
         if (hex_md5 != m_owner->package()->md5()) {
-            SG_LOG(SG_GENERAL, SG_ALERT, "md5 verification failed:\n"
+            SG_LOG(SG_GENERAL, SG_WARN, "md5 verification failed:\n"
                 << "\t" << hex_md5 << "\n\t"
                 << m_owner->package()->md5() << "\n\t"
                 << "downloading from:" << url());
@@ -196,7 +199,7 @@ protected:
         m_owner->installResult(Delegate::STATUS_SUCCESS);
     }
 
-    virtual void onFail()
+    void onFail() override
     {
         if (responseCode() == -1) {
             doFailure(Delegate::USER_CANCELLED);
@@ -212,6 +215,18 @@ private:
         if (dir.exists()) {
             dir.remove(true /* recursive */);
         }
+        
+        const auto canRetry = (aReason == Delegate::FAIL_NOT_FOUND) || (aReason == Delegate::FAIL_DOWNLOAD);
+        if (canRetry && !m_urls.empty()) {
+            SG_LOG(SG_GENERAL, SG_WARN, "archive download failed from:" << m_activeURL
+                   << "\n\twill retry with next mirror");
+            // becuase selectMirrorUrl erased the active URL from m_urls,
+            // this new request will select one of the other mirrors
+            auto retryDownload = new PackageArchiveDownloader(m_owner, m_urls);
+            m_owner->m_download.reset(retryDownload);
+            m_owner->package()->catalog()->root()->makeHTTPRequest(retryDownload);
+            return;
+        }
 
         // TODO - try other mirrors
         m_owner->m_download.reset(); // ensure we get cleaned up
@@ -220,6 +235,7 @@ private:
 
     
     InstallRef m_owner;
+    string m_activeURL;
     string_list m_urls;
     SG_MD5_CTX m_md5;
     SGPath m_extractPath;
@@ -283,7 +299,7 @@ void Install::startUpdate()
         return; // already active
     }
 
-    m_download = new PackageArchiveDownloader(this);
+    m_download = new PackageArchiveDownloader(this, {});
     m_package->catalog()->root()->makeHTTPRequest(m_download);
     m_package->catalog()->root()->startInstall(this);
 }
@@ -343,6 +359,7 @@ void Install::cancelDownload()
     }
 
     m_package->catalog()->root()->cancelDownload(this);
+    m_download.clear();
 }
 
 SGPath Install::primarySetPath() const

simgear/package/Root.cxx

@@ -660,7 +660,7 @@ void Root::startNext(InstallRef aCurrent)
 void Root::finishInstall(InstallRef aInstall, Delegate::StatusCode aReason)
 {
     if (aReason != Delegate::STATUS_SUCCESS) {
-        SG_LOG(SG_GENERAL, SG_ALERT, "failed to install package:"
+        SG_LOG(SG_GENERAL, SG_WARN, "failed to install package:"
                << aInstall->package()->id() << ":" << aReason);
     }
 

simgear/package/catalogTest1/catalog-alt.xml

@@ -123,7 +123,9 @@
         </rating>
 
         <md5>a94ca5704f305b90767f40617d194ed6</md5>
-        <url>http://localhost:2000/catalogTest1/b737.tar.gz</url>
+          <url>http://localhost:2000/mirrorA/b737.tar.gz</url>
+        <url>http://localhost:2000/mirrorB/b737.tar.gz</url>
+        <url>http://localhost:2000/mirrorC/b737.tar.gz</url>
 
       </package>
 

simgear/package/catalogTest1/catalog-v10.xml

@@ -163,7 +163,9 @@
         </rating>
 
         <md5>a94ca5704f305b90767f40617d194ed6</md5>
-        <url>http://localhost:2000/catalogTest1/b737.tar.gz</url>
+        <url>http://localhost:2000/mirrorA/b737.tar.gz</url>
+        <url>http://localhost:2000/mirrorB/b737.tar.gz</url>
+        <url>http://localhost:2000/mirrorC/b737.tar.gz</url>
       </package>
 
     <package>

simgear/package/catalogTest1/catalog-v2.xml

@@ -150,7 +150,10 @@
         </rating>
 
         <md5>a94ca5704f305b90767f40617d194ed6</md5>
-        <url>http://localhost:2000/catalogTest1/b737.tar.gz</url>
+        
+        <url>http://localhost:2000/mirrorA/b737.tar.gz</url>
+        <url>http://localhost:2000/mirrorB/b737.tar.gz</url>
+        <url>http://localhost:2000/mirrorC/b737.tar.gz</url>
 
         <preview>
           <type>exterior</type>

simgear/package/catalogTest1/catalog-v9.xml

@@ -163,7 +163,9 @@
         </rating>
 
         <md5>a94ca5704f305b90767f40617d194ed6</md5>
-        <url>http://localhost:2000/catalogTest1/b737.tar.gz</url>
+        <url>http://localhost:2000/mirrorA/b737.tar.gz</url>
+        <url>http://localhost:2000/mirrorB/b737.tar.gz</url>
+        <url>http://localhost:2000/mirrorC/b737.tar.gz</url>
       </package>
 
     <package>

simgear/package/catalogTest1/catalog.xml

@@ -191,7 +191,9 @@
         </rating>
 
         <md5>a94ca5704f305b90767f40617d194ed6</md5>
-        <url>http://localhost:2000/catalogTest1/b737.tar.gz</url>
+        <url>http://localhost:2000/mirrorA/b737.tar.gz</url>
+        <url>http://localhost:2000/mirrorB/b737.tar.gz</url>
+        <url>http://localhost:2000/mirrorC/b737.tar.gz</url>
       </package>
 
     <package>