From 40725a76ab9d672715dab46afb717d2a5d2a5bce Mon Sep 17 00:00:00 2001 From: James Turner Date: Sat, 21 Mar 2020 18:31:08 +0000 Subject: [PATCH] Packages: randomise mirror URL selection Also add retry on mirrors when a download fails. --- simgear/package/CatalogTest.cxx | 58 ++++++++++++++++++-- simgear/package/Install.cxx | 55 ++++++++++++------- simgear/package/Root.cxx | 2 +- simgear/package/catalogTest1/catalog-alt.xml | 4 +- simgear/package/catalogTest1/catalog-v10.xml | 4 +- simgear/package/catalogTest1/catalog-v2.xml | 5 +- simgear/package/catalogTest1/catalog-v9.xml | 4 +- simgear/package/catalogTest1/catalog.xml | 4 +- 8 files changed, 107 insertions(+), 29 deletions(-) diff --git a/simgear/package/CatalogTest.cxx b/simgear/package/CatalogTest.cxx index 6dae43f7..edd33f34 100644 --- a/simgear/package/CatalogTest.cxx +++ b/simgear/package/CatalogTest.cxx @@ -62,6 +62,8 @@ unsigned int global_catalogVersion = 0; bool global_failRequests = false; bool global_fail747Request = true; +int global_737RequestsToFailCount = 0; + class TestPackageChannel : public TestServerChannel { public: @@ -106,6 +108,15 @@ public: } } + if ((path.find("/mirror") == 0) && (path.find("/b737.tar.gz") == 8)) { + if (global_737RequestsToFailCount > 0) { + sendErrorResponse(404, false, "Mirror failure"); + --global_737RequestsToFailCount; + return; + } + path = "/catalogTest1/b737.tar.gz"; + } + localPath.append(path); // SG_LOG(SG_IO, SG_INFO, "local path is:" << localPath.str()); @@ -329,7 +340,9 @@ int parseTest() SG_VERIFY(p3->matches(queryText.ptr())); } - + string_list urls = p3->downloadUrls(); + SG_CHECK_EQUAL(urls.size(), 3); + SG_CHECK_EQUAL(urls.at(1), "http://localhost:2000/mirrorB/b737.tar.gz"); delete root; return EXIT_SUCCESS; @@ -1136,9 +1149,44 @@ void testInstallBadPackage(HTTP::Client* cl) SG_CHECK_EQUAL(ins->path(), rootPath / "org.flightgear.test.catalog1" / "Aircraft" / "b744"); } +void testMirrorsFailure(HTTP::Client* cl) +{ + global_catalogVersion = 0; + // there's three mirrors defined, so se tthe first twom attempts to fail + global_737RequestsToFailCount = 2; + + SGPath rootPath(simgear::Dir::current().path()); + rootPath.append("pkg_install_mirror_fail"); + simgear::Dir pd(rootPath); + pd.removeChildren(); + + pkg::RootRef root(new pkg::Root(rootPath, "8.1.2")); + // specify a test dir + root->setHTTPClient(cl); + + pkg::CatalogRef c = pkg::Catalog::createFromUrl(root.ptr(), "http://localhost:2000/catalogTest1/catalog.xml"); + waitForUpdateComplete(cl, root); + + pkg::PackageRef p1 = root->getPackageById("org.flightgear.test.catalog1.b737-NG"); + pkg::InstallRef ins = p1->install(); + + bool didFail = false; + ins->fail([&didFail, &ins](pkg::Install* ourInstall) { + SG_CHECK_EQUAL(ins, ourInstall); + didFail = true; + }); + + waitForUpdateComplete(cl, root); + SG_VERIFY(p1->isInstalled()); + SG_VERIFY(!didFail); + SG_VERIFY(p1->existingInstall() == ins); + SG_CHECK_EQUAL(ins->status(), pkg::Delegate::STATUS_SUCCESS); + +} + int main(int argc, char* argv[]) { - // sglog().setLogLevels( SG_ALL, SG_DEBUG ); + sglog().setLogLevels( SG_ALL, SG_WARN ); HTTP::Client cl; cl.setMaxConnections(1); @@ -1150,7 +1198,7 @@ int main(int argc, char* argv[]) parseTest(); parseInvalidTest(); - + testInstallPackage(&cl); testUninstall(&cl); @@ -1179,6 +1227,8 @@ int main(int argc, char* argv[]) testInstallBadPackage(&cl); - SG_LOG(SG_GENERAL, SG_INFO, "Successfully passed all tests!"); + testMirrorsFailure(&cl); + + cerr << "Successfully passed all tests!" << endl; return EXIT_SUCCESS; } diff --git a/simgear/package/Install.cxx b/simgear/package/Install.cxx index 8ab574c9..117d4021 100644 --- a/simgear/package/Install.cxx +++ b/simgear/package/Install.cxx @@ -18,8 +18,8 @@ #include #include -#include #include +#include #include #include @@ -43,21 +43,17 @@ namespace pkg { class Install::PackageArchiveDownloader : public HTTP::Request { public: - PackageArchiveDownloader(InstallRef aOwner) : + PackageArchiveDownloader(InstallRef aOwner, const string_list& urls) : HTTP::Request("" /* dummy URL */), m_owner(aOwner) { - m_urls = m_owner->package()->downloadUrls(); - if (m_urls.empty()) { - throw sg_exception("no package download URLs"); + m_urls = urls; + if (urls.empty()) { + m_urls = m_owner->package()->downloadUrls(); } - // if (m_owner->package()->properties()->hasChild("archive-type")) { - // setArchiveTypeFromExtension(m_owner->package()->properties()->getStringValue("archive-type")); - //} + selectMirrorUrl(); - // TODO randomise order of m_urls - m_extractPath = aOwner->path().dir(); m_extractPath.append("_extract_" + aOwner->package()->md5()); @@ -93,12 +89,19 @@ public: return (m_downloaded * 100) / responseLength(); } protected: - virtual std::string url() const + void selectMirrorUrl() { - return m_urls.front(); + const int randomizedIndex = rand() % m_urls.size(); + m_activeURL = m_urls.at(randomizedIndex); + m_urls.erase(m_urls.begin() + randomizedIndex); + } + + virtual std::string url() const override + { + return m_activeURL; } - virtual void responseHeadersComplete() + void responseHeadersComplete() override { Request::responseHeadersComplete(); @@ -112,7 +115,7 @@ protected: m_owner->startDownload(); } - virtual void gotBodyData(const char* s, int n) + void gotBodyData(const char* s, int n) override { const uint8_t* ubytes = (uint8_t*) s; SG_MD5Update(&m_md5, ubytes, n); @@ -121,10 +124,10 @@ protected: m_extractor->extractBytes(ubytes, n); } - virtual void onDone() + void onDone() override { if (responseCode() != 200) { - SG_LOG(SG_GENERAL, SG_ALERT, "download failure:" << responseCode() << + SG_LOG(SG_GENERAL, SG_WARN, "download failure:" << responseCode() << "\n\t" << url()); Delegate::StatusCode code = Delegate::FAIL_DOWNLOAD; if (responseCode() == 404) { @@ -141,7 +144,7 @@ protected: strutils::encodeHex(digest, MD5_DIGEST_LENGTH); if (hex_md5 != m_owner->package()->md5()) { - SG_LOG(SG_GENERAL, SG_ALERT, "md5 verification failed:\n" + SG_LOG(SG_GENERAL, SG_WARN, "md5 verification failed:\n" << "\t" << hex_md5 << "\n\t" << m_owner->package()->md5() << "\n\t" << "downloading from:" << url()); @@ -196,7 +199,7 @@ protected: m_owner->installResult(Delegate::STATUS_SUCCESS); } - virtual void onFail() + void onFail() override { if (responseCode() == -1) { doFailure(Delegate::USER_CANCELLED); @@ -212,6 +215,18 @@ private: if (dir.exists()) { dir.remove(true /* recursive */); } + + const auto canRetry = (aReason == Delegate::FAIL_NOT_FOUND) || (aReason == Delegate::FAIL_DOWNLOAD); + if (canRetry && !m_urls.empty()) { + SG_LOG(SG_GENERAL, SG_WARN, "archive download failed from:" << m_activeURL + << "\n\twill retry with next mirror"); + // becuase selectMirrorUrl erased the active URL from m_urls, + // this new request will select one of the other mirrors + auto retryDownload = new PackageArchiveDownloader(m_owner, m_urls); + m_owner->m_download.reset(retryDownload); + m_owner->package()->catalog()->root()->makeHTTPRequest(retryDownload); + return; + } // TODO - try other mirrors m_owner->m_download.reset(); // ensure we get cleaned up @@ -220,6 +235,7 @@ private: InstallRef m_owner; + string m_activeURL; string_list m_urls; SG_MD5_CTX m_md5; SGPath m_extractPath; @@ -283,7 +299,7 @@ void Install::startUpdate() return; // already active } - m_download = new PackageArchiveDownloader(this); + m_download = new PackageArchiveDownloader(this, {}); m_package->catalog()->root()->makeHTTPRequest(m_download); m_package->catalog()->root()->startInstall(this); } @@ -343,6 +359,7 @@ void Install::cancelDownload() } m_package->catalog()->root()->cancelDownload(this); + m_download.clear(); } SGPath Install::primarySetPath() const diff --git a/simgear/package/Root.cxx b/simgear/package/Root.cxx index ac19af07..be6254c7 100644 --- a/simgear/package/Root.cxx +++ b/simgear/package/Root.cxx @@ -660,7 +660,7 @@ void Root::startNext(InstallRef aCurrent) void Root::finishInstall(InstallRef aInstall, Delegate::StatusCode aReason) { if (aReason != Delegate::STATUS_SUCCESS) { - SG_LOG(SG_GENERAL, SG_ALERT, "failed to install package:" + SG_LOG(SG_GENERAL, SG_WARN, "failed to install package:" << aInstall->package()->id() << ":" << aReason); } diff --git a/simgear/package/catalogTest1/catalog-alt.xml b/simgear/package/catalogTest1/catalog-alt.xml index 54d2edc2..013ed8c6 100644 --- a/simgear/package/catalogTest1/catalog-alt.xml +++ b/simgear/package/catalogTest1/catalog-alt.xml @@ -123,7 +123,9 @@ a94ca5704f305b90767f40617d194ed6 - http://localhost:2000/catalogTest1/b737.tar.gz + http://localhost:2000/mirrorA/b737.tar.gz + http://localhost:2000/mirrorB/b737.tar.gz + http://localhost:2000/mirrorC/b737.tar.gz diff --git a/simgear/package/catalogTest1/catalog-v10.xml b/simgear/package/catalogTest1/catalog-v10.xml index c86e8f79..e9e873f4 100644 --- a/simgear/package/catalogTest1/catalog-v10.xml +++ b/simgear/package/catalogTest1/catalog-v10.xml @@ -163,7 +163,9 @@ a94ca5704f305b90767f40617d194ed6 - http://localhost:2000/catalogTest1/b737.tar.gz + http://localhost:2000/mirrorA/b737.tar.gz + http://localhost:2000/mirrorB/b737.tar.gz + http://localhost:2000/mirrorC/b737.tar.gz diff --git a/simgear/package/catalogTest1/catalog-v2.xml b/simgear/package/catalogTest1/catalog-v2.xml index fa416115..df33611c 100644 --- a/simgear/package/catalogTest1/catalog-v2.xml +++ b/simgear/package/catalogTest1/catalog-v2.xml @@ -150,7 +150,10 @@ a94ca5704f305b90767f40617d194ed6 - http://localhost:2000/catalogTest1/b737.tar.gz + + http://localhost:2000/mirrorA/b737.tar.gz + http://localhost:2000/mirrorB/b737.tar.gz + http://localhost:2000/mirrorC/b737.tar.gz exterior diff --git a/simgear/package/catalogTest1/catalog-v9.xml b/simgear/package/catalogTest1/catalog-v9.xml index 2dcd789e..d0246444 100644 --- a/simgear/package/catalogTest1/catalog-v9.xml +++ b/simgear/package/catalogTest1/catalog-v9.xml @@ -163,7 +163,9 @@ a94ca5704f305b90767f40617d194ed6 - http://localhost:2000/catalogTest1/b737.tar.gz + http://localhost:2000/mirrorA/b737.tar.gz + http://localhost:2000/mirrorB/b737.tar.gz + http://localhost:2000/mirrorC/b737.tar.gz diff --git a/simgear/package/catalogTest1/catalog.xml b/simgear/package/catalogTest1/catalog.xml index eb975620..76070b80 100644 --- a/simgear/package/catalogTest1/catalog.xml +++ b/simgear/package/catalogTest1/catalog.xml @@ -191,7 +191,9 @@ a94ca5704f305b90767f40617d194ed6 - http://localhost:2000/catalogTest1/b737.tar.gz + http://localhost:2000/mirrorA/b737.tar.gz + http://localhost:2000/mirrorB/b737.tar.gz + http://localhost:2000/mirrorC/b737.tar.gz