From 34c215ad83e2da69910f1906a882c04cf58e7921 Mon Sep 17 00:00:00 2001 From: James Turner Date: Tue, 18 Jul 2017 18:01:10 +0100 Subject: [PATCH] Persistent thumbnail cache for aircraft packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This should reduce the load on Ibiblio when browsing aircraft, the package system caches thumbnails for seven days. Also we detect for certain failure conditions and retry, to deal with Ibiblio’s rate- limiting mechanism. --- simgear/misc/sg_path.cxx | 20 ++++ simgear/misc/sg_path.hxx | 5 + simgear/package/Delegate.hxx | 1 + simgear/package/Root.cxx | 184 +++++++++++++++++++++++++++++------ 4 files changed, 182 insertions(+), 28 deletions(-) diff --git a/simgear/misc/sg_path.cxx b/simgear/misc/sg_path.cxx index 7d17d3c2..05af794f 100644 --- a/simgear/misc/sg_path.cxx +++ b/simgear/misc/sg_path.cxx @@ -1054,3 +1054,23 @@ bool SGPath::permissionsAllowsWrite() const return _permission_checker ? _permission_checker(*this).write : true; } +//------------------------------------------------------------------------------ +std::string SGPath::fileUrl() const +{ + // we should really URL encode the names here? + if (isAbsolute()) { + // check for a windows drive letter +#if defined(SG_WINDOWS) + if (isalpha(path.front())) { + // file URLs on Windows must look like file:///C:/Foo/Bar + return "file:///" + utf8Str(); + } +#endif + // the leading directory seperator of the path becomes the required + // third slash in this case. + return "file://" + utf8Str(); + } else { + SG_LOG(SG_GENERAL, SG_WARN, "Cannot convert relative path to a URL:" << path); + return {}; + } +} diff --git a/simgear/misc/sg_path.hxx b/simgear/misc/sg_path.hxx index 9f3dfc0f..38b391a6 100644 --- a/simgear/misc/sg_path.hxx +++ b/simgear/misc/sg_path.hxx @@ -281,6 +281,11 @@ public: */ SGPath dirPath() const; + /* + * return path as a file:// URI + */ + std::string fileUrl() const; + enum StandardLocation { HOME, diff --git a/simgear/package/Delegate.hxx b/simgear/package/Delegate.hxx index df59256c..9e75b4de 100644 --- a/simgear/package/Delegate.hxx +++ b/simgear/package/Delegate.hxx @@ -53,6 +53,7 @@ public: FAIL_FILESYSTEM, ///< unknown filesystem error occurred FAIL_VERSION, ///< version check mismatch FAIL_NOT_FOUND, ///< package URL returned a 404 + FAIL_HTTP_FORBIDDEN, ///< URL returned a 403. Marked specially to catch rate-limiting STATUS_REFRESHED, USER_CANCELLED } StatusCode; diff --git a/simgear/package/Root.cxx b/simgear/package/Root.cxx index 866dffae..f38f7e93 100644 --- a/simgear/package/Root.cxx +++ b/simgear/package/Root.cxx @@ -30,27 +30,50 @@ #include #include #include +#include +#include #include #include #include #include +const int SECONDS_PER_DAY = 24 * 60 * 60; + namespace simgear { +namespace { + std::string hashForUrl(const std::string& d) + { + sha1nfo info; + sha1_init(&info); + sha1_write(&info, d.data(), d.size()); + return strutils::encodeHex(sha1_result(&info), HASH_LENGTH); + } +} // of anonymous namespace + namespace pkg { typedef std::map CatalogDict; typedef std::vector DelegateVec; -typedef std::map MemThumbnailCache; typedef std::deque StringDeque; class Root::ThumbnailDownloader : public HTTP::Request { public: - ThumbnailDownloader(Root::RootPrivate* aOwner, const std::string& aUrl) : - HTTP::Request(aUrl), - m_owner(aOwner) + ThumbnailDownloader(Root::RootPrivate* aOwner, + const std::string& aUrl, const std::string& aRealUrl = std::string()) : + HTTP::Request(aUrl), + m_owner(aOwner), + m_realUrl(aRealUrl) { + if (m_realUrl.empty()) { + m_realUrl = aUrl; + } + } + + std::string realUrl() const + { + return m_realUrl; } protected: @@ -64,6 +87,7 @@ protected: private: Root::RootPrivate* m_owner; std::string m_buffer; + std::string m_realUrl; }; class Root::RootPrivate @@ -71,7 +95,7 @@ class Root::RootPrivate public: RootPrivate() : http(NULL), - maxAgeSeconds(60 * 60 * 24) + maxAgeSeconds(SECONDS_PER_DAY) { } @@ -123,20 +147,38 @@ public: void thumbnailDownloadComplete(HTTP::Request_ptr request, Delegate::StatusCode status, const std::string& bytes) { - std::string u(request->url()); + auto dl = static_cast(request.get()); + std::string u = dl->realUrl(); if (status == Delegate::STATUS_SUCCESS) { - thumbnailCache[u] = bytes; - fireDataForThumbnail(u, bytes); + thumbnailCache[u].requestPending = false; + fireDataForThumbnail(u, reinterpret_cast(bytes.data()), bytes.size()); + + // if this was a network load, rather than a re-load from the disk cache, + // then persist to disk now. + if (strutils::starts_with(request->url(), "http")) { + addToPersistentCache(u, bytes); + } + } else if (status == Delegate::FAIL_HTTP_FORBIDDEN) { + // treat this as rate-limiting failure, at least from some mirrors + // (eg Ibiblio) and retry up to the max count + const int retries = (thumbnailCache[u].retryCount++); + if (retries < 3) { + SG_LOG(SG_IO, SG_INFO, "Download failed for: " << u << ", will retry"); + thumbnailCache[u].requestPending = true; + pendingThumbnails.push_back(u); + } + } else { + // any other failure. + thumbnailCache[u].requestPending = false; } downloadNextPendingThumbnail(); } - void fireDataForThumbnail(const std::string& aUrl, const std::string& bytes) + void fireDataForThumbnail(const std::string& aUrl, const uint8_t* bytes, size_t size) { - const uint8_t* data = reinterpret_cast(bytes.data()); for (auto d : delegates) { - d->dataForThumbnail(aUrl, bytes.size(), data); + d->dataForThumbnail(aUrl, size, bytes); } } @@ -165,6 +207,73 @@ public: } } + void addToPersistentCache(const std::string& url, const std::string& imageBytes) + { + std::string hash = hashForUrl(url); + // append the correct file suffix + auto pos = url.rfind('.'); + if (pos == std::string::npos) { + return; + } + + SGPath cachePath = path / "ThumbnailCache" / (hash + url.substr(pos)); + sg_ofstream fstream(cachePath, std::ios::out | std::ios::trunc); + fstream.write(imageBytes.data(), imageBytes.size()); + fstream.close(); + } + + bool checkPersistentCache(const std::string& url) + { + std::string hash = hashForUrl(url); + // append the correct file suffix + auto pos = url.rfind('.'); + if (pos == std::string::npos) { + return false; + } + + SGPath cachePath = path / "ThumbnailCache" / (hash + url.substr(pos)); + if (!cachePath.exists()) { + return false; + } + + // check age, if it's too old, expire and download again + int age = time(nullptr) - cachePath.modTime(); + if (age > SECONDS_PER_DAY * 7) { // cache for seven days + SG_LOG(SG_IO, SG_INFO, "expiring old cached thumbnail " << url); + cachePath.remove(); + return false; + } + + queueLoadFromPersistentCache(url, cachePath); + return true; + } + + void queueLoadFromPersistentCache(const std::string& url, const SGPath& path) + { + assert(path.exists()); + + auto it = thumbnailCache.find(url); + if (it == thumbnailCache.end()) { + ThumbnailCacheEntry entry; + entry.pathOnDisk = path; + it = thumbnailCache.insert(it, std::make_pair(url, entry)); + } else { + assert(it->second.pathOnDisk == path); + } + + if (it->second.requestPending) { + return; // all done + } + + it->second.requestPending = true; + auto dl = new Root::ThumbnailDownloader(this, path.fileUrl(), url); + if (http) { + http->makeRequest(dl); + } else { + httpPendingRequests.push_back(dl); + } + } + DelegateVec delegates; SGPath path; @@ -182,7 +291,15 @@ public: HTTP::Request_ptr thumbnailDownloadRequest; StringDeque pendingThumbnails; - MemThumbnailCache thumbnailCache; + + struct ThumbnailCacheEntry + { + int retryCount = 0; + bool requestPending = false; + SGPath pathOnDisk; + }; + + std::map thumbnailCache; typedef std::map InstallCache; InstallCache m_installs; @@ -191,16 +308,19 @@ public: void Root::ThumbnailDownloader::onDone() { + if (simgear::strutils::starts_with(url(), "file://")) { + m_owner->thumbnailDownloadComplete(this, Delegate::STATUS_SUCCESS, m_buffer); + return; + } + if (responseCode() != 200) { - SG_LOG(SG_GENERAL, SG_ALERT, "thumbnail download failure:" << url()); - m_owner->thumbnailDownloadComplete(this, Delegate::FAIL_DOWNLOAD, std::string()); + auto status = (responseCode() == 403) ? Delegate::FAIL_HTTP_FORBIDDEN : Delegate::FAIL_DOWNLOAD; + SG_LOG(SG_NETWORK, SG_INFO, "thumbnail download failure: " << url() << " with reason " << responseCode()); + m_owner->thumbnailDownloadComplete(this, status, std::string()); return; } m_owner->thumbnailDownloadComplete(this, Delegate::STATUS_SUCCESS, m_buffer); - //time(&m_owner->m_retrievedTime); - //m_owner->writeTimestamp(); - //m_owner->refreshComplete(Delegate::STATUS_REFRESHED); } SGPath Root::path() const @@ -264,10 +384,14 @@ Root::Root(const SGPath& aPath, const std::string& aVersion) : Dir dir(aPath); if (!dir.exists()) { dir.create(0755); - return; } - BOOST_FOREACH(SGPath c, dir.children(Dir::TYPE_DIR | Dir::NO_DOT_OR_DOTDOT)) { + Dir thumbsCacheDir(aPath / "ThumbnailCache"); + if (!thumbsCacheDir.exists()) { + thumbsCacheDir.create(0755); + } + + for (SGPath c : dir.children(Dir::TYPE_DIR | Dir::NO_DOT_OR_DOTDOT)) { CatalogRef cat = Catalog::createFromPath(this, c); if (cat) { if (cat->status() == Delegate::STATUS_SUCCESS) { @@ -614,17 +738,21 @@ bool Root::removeCatalogById(const std::string& aId) void Root::requestThumbnailData(const std::string& aUrl) { - MemThumbnailCache::iterator it = d->thumbnailCache.find(aUrl); + auto it = d->thumbnailCache.find(aUrl); if (it == d->thumbnailCache.end()) { - // insert into cache to mark as pending - d->pendingThumbnails.push_front(aUrl); - d->thumbnailCache[aUrl] = std::string(); - d->downloadNextPendingThumbnail(); - } else if (!it->second.empty()) { - // already loaded, fire data synchronously - d->fireDataForThumbnail(aUrl, it->second); + bool cachedOnDisk = d->checkPersistentCache(aUrl); + if (cachedOnDisk) { + // checkPersistentCache will insert the entry and schedule + } else { + d->pendingThumbnails.push_front(aUrl); + d->thumbnailCache[aUrl] = RootPrivate::ThumbnailCacheEntry(); + d->thumbnailCache[aUrl].requestPending = true; + d->downloadNextPendingThumbnail(); + } } else { - // in cache but empty data, still fetching + if (!it->second.requestPending && it->second.pathOnDisk.exists()) { + d->queueLoadFromPersistentCache(aUrl, it->second.pathOnDisk); + } } }