diff --git a/simgear/misc/sg_path.cxx b/simgear/misc/sg_path.cxx index 7d17d3c2..05af794f 100644 --- a/simgear/misc/sg_path.cxx +++ b/simgear/misc/sg_path.cxx @@ -1054,3 +1054,23 @@ bool SGPath::permissionsAllowsWrite() const return _permission_checker ? _permission_checker(*this).write : true; } +//------------------------------------------------------------------------------ +std::string SGPath::fileUrl() const +{ + // we should really URL encode the names here? + if (isAbsolute()) { + // check for a windows drive letter +#if defined(SG_WINDOWS) + if (isalpha(path.front())) { + // file URLs on Windows must look like file:///C:/Foo/Bar + return "file:///" + utf8Str(); + } +#endif + // the leading directory seperator of the path becomes the required + // third slash in this case. + return "file://" + utf8Str(); + } else { + SG_LOG(SG_GENERAL, SG_WARN, "Cannot convert relative path to a URL:" << path); + return {}; + } +} diff --git a/simgear/misc/sg_path.hxx b/simgear/misc/sg_path.hxx index 9f3dfc0f..38b391a6 100644 --- a/simgear/misc/sg_path.hxx +++ b/simgear/misc/sg_path.hxx @@ -281,6 +281,11 @@ public: */ SGPath dirPath() const; + /* + * return path as a file:// URI + */ + std::string fileUrl() const; + enum StandardLocation { HOME, diff --git a/simgear/package/Delegate.hxx b/simgear/package/Delegate.hxx index df59256c..9e75b4de 100644 --- a/simgear/package/Delegate.hxx +++ b/simgear/package/Delegate.hxx @@ -53,6 +53,7 @@ public: FAIL_FILESYSTEM, ///< unknown filesystem error occurred FAIL_VERSION, ///< version check mismatch FAIL_NOT_FOUND, ///< package URL returned a 404 + FAIL_HTTP_FORBIDDEN, ///< URL returned a 403. Marked specially to catch rate-limiting STATUS_REFRESHED, USER_CANCELLED } StatusCode; diff --git a/simgear/package/Root.cxx b/simgear/package/Root.cxx index 866dffae..f38f7e93 100644 --- a/simgear/package/Root.cxx +++ b/simgear/package/Root.cxx @@ -30,27 +30,50 @@ #include #include #include +#include +#include #include #include #include #include +const int SECONDS_PER_DAY = 24 * 60 * 60; + namespace simgear { +namespace { + std::string hashForUrl(const std::string& d) + { + sha1nfo info; + sha1_init(&info); + sha1_write(&info, d.data(), d.size()); + return strutils::encodeHex(sha1_result(&info), HASH_LENGTH); + } +} // of anonymous namespace + namespace pkg { typedef std::map CatalogDict; typedef std::vector DelegateVec; -typedef std::map MemThumbnailCache; typedef std::deque StringDeque; class Root::ThumbnailDownloader : public HTTP::Request { public: - ThumbnailDownloader(Root::RootPrivate* aOwner, const std::string& aUrl) : - HTTP::Request(aUrl), - m_owner(aOwner) + ThumbnailDownloader(Root::RootPrivate* aOwner, + const std::string& aUrl, const std::string& aRealUrl = std::string()) : + HTTP::Request(aUrl), + m_owner(aOwner), + m_realUrl(aRealUrl) { + if (m_realUrl.empty()) { + m_realUrl = aUrl; + } + } + + std::string realUrl() const + { + return m_realUrl; } protected: @@ -64,6 +87,7 @@ protected: private: Root::RootPrivate* m_owner; std::string m_buffer; + std::string m_realUrl; }; class Root::RootPrivate @@ -71,7 +95,7 @@ class Root::RootPrivate public: RootPrivate() : http(NULL), - maxAgeSeconds(60 * 60 * 24) + maxAgeSeconds(SECONDS_PER_DAY) { } @@ -123,20 +147,38 @@ public: void thumbnailDownloadComplete(HTTP::Request_ptr request, Delegate::StatusCode status, const std::string& bytes) { - std::string u(request->url()); + auto dl = static_cast(request.get()); + std::string u = dl->realUrl(); if (status == Delegate::STATUS_SUCCESS) { - thumbnailCache[u] = bytes; - fireDataForThumbnail(u, bytes); + thumbnailCache[u].requestPending = false; + fireDataForThumbnail(u, reinterpret_cast(bytes.data()), bytes.size()); + + // if this was a network load, rather than a re-load from the disk cache, + // then persist to disk now. + if (strutils::starts_with(request->url(), "http")) { + addToPersistentCache(u, bytes); + } + } else if (status == Delegate::FAIL_HTTP_FORBIDDEN) { + // treat this as rate-limiting failure, at least from some mirrors + // (eg Ibiblio) and retry up to the max count + const int retries = (thumbnailCache[u].retryCount++); + if (retries < 3) { + SG_LOG(SG_IO, SG_INFO, "Download failed for: " << u << ", will retry"); + thumbnailCache[u].requestPending = true; + pendingThumbnails.push_back(u); + } + } else { + // any other failure. + thumbnailCache[u].requestPending = false; } downloadNextPendingThumbnail(); } - void fireDataForThumbnail(const std::string& aUrl, const std::string& bytes) + void fireDataForThumbnail(const std::string& aUrl, const uint8_t* bytes, size_t size) { - const uint8_t* data = reinterpret_cast(bytes.data()); for (auto d : delegates) { - d->dataForThumbnail(aUrl, bytes.size(), data); + d->dataForThumbnail(aUrl, size, bytes); } } @@ -165,6 +207,73 @@ public: } } + void addToPersistentCache(const std::string& url, const std::string& imageBytes) + { + std::string hash = hashForUrl(url); + // append the correct file suffix + auto pos = url.rfind('.'); + if (pos == std::string::npos) { + return; + } + + SGPath cachePath = path / "ThumbnailCache" / (hash + url.substr(pos)); + sg_ofstream fstream(cachePath, std::ios::out | std::ios::trunc); + fstream.write(imageBytes.data(), imageBytes.size()); + fstream.close(); + } + + bool checkPersistentCache(const std::string& url) + { + std::string hash = hashForUrl(url); + // append the correct file suffix + auto pos = url.rfind('.'); + if (pos == std::string::npos) { + return false; + } + + SGPath cachePath = path / "ThumbnailCache" / (hash + url.substr(pos)); + if (!cachePath.exists()) { + return false; + } + + // check age, if it's too old, expire and download again + int age = time(nullptr) - cachePath.modTime(); + if (age > SECONDS_PER_DAY * 7) { // cache for seven days + SG_LOG(SG_IO, SG_INFO, "expiring old cached thumbnail " << url); + cachePath.remove(); + return false; + } + + queueLoadFromPersistentCache(url, cachePath); + return true; + } + + void queueLoadFromPersistentCache(const std::string& url, const SGPath& path) + { + assert(path.exists()); + + auto it = thumbnailCache.find(url); + if (it == thumbnailCache.end()) { + ThumbnailCacheEntry entry; + entry.pathOnDisk = path; + it = thumbnailCache.insert(it, std::make_pair(url, entry)); + } else { + assert(it->second.pathOnDisk == path); + } + + if (it->second.requestPending) { + return; // all done + } + + it->second.requestPending = true; + auto dl = new Root::ThumbnailDownloader(this, path.fileUrl(), url); + if (http) { + http->makeRequest(dl); + } else { + httpPendingRequests.push_back(dl); + } + } + DelegateVec delegates; SGPath path; @@ -182,7 +291,15 @@ public: HTTP::Request_ptr thumbnailDownloadRequest; StringDeque pendingThumbnails; - MemThumbnailCache thumbnailCache; + + struct ThumbnailCacheEntry + { + int retryCount = 0; + bool requestPending = false; + SGPath pathOnDisk; + }; + + std::map thumbnailCache; typedef std::map InstallCache; InstallCache m_installs; @@ -191,16 +308,19 @@ public: void Root::ThumbnailDownloader::onDone() { + if (simgear::strutils::starts_with(url(), "file://")) { + m_owner->thumbnailDownloadComplete(this, Delegate::STATUS_SUCCESS, m_buffer); + return; + } + if (responseCode() != 200) { - SG_LOG(SG_GENERAL, SG_ALERT, "thumbnail download failure:" << url()); - m_owner->thumbnailDownloadComplete(this, Delegate::FAIL_DOWNLOAD, std::string()); + auto status = (responseCode() == 403) ? Delegate::FAIL_HTTP_FORBIDDEN : Delegate::FAIL_DOWNLOAD; + SG_LOG(SG_NETWORK, SG_INFO, "thumbnail download failure: " << url() << " with reason " << responseCode()); + m_owner->thumbnailDownloadComplete(this, status, std::string()); return; } m_owner->thumbnailDownloadComplete(this, Delegate::STATUS_SUCCESS, m_buffer); - //time(&m_owner->m_retrievedTime); - //m_owner->writeTimestamp(); - //m_owner->refreshComplete(Delegate::STATUS_REFRESHED); } SGPath Root::path() const @@ -264,10 +384,14 @@ Root::Root(const SGPath& aPath, const std::string& aVersion) : Dir dir(aPath); if (!dir.exists()) { dir.create(0755); - return; } - BOOST_FOREACH(SGPath c, dir.children(Dir::TYPE_DIR | Dir::NO_DOT_OR_DOTDOT)) { + Dir thumbsCacheDir(aPath / "ThumbnailCache"); + if (!thumbsCacheDir.exists()) { + thumbsCacheDir.create(0755); + } + + for (SGPath c : dir.children(Dir::TYPE_DIR | Dir::NO_DOT_OR_DOTDOT)) { CatalogRef cat = Catalog::createFromPath(this, c); if (cat) { if (cat->status() == Delegate::STATUS_SUCCESS) { @@ -614,17 +738,21 @@ bool Root::removeCatalogById(const std::string& aId) void Root::requestThumbnailData(const std::string& aUrl) { - MemThumbnailCache::iterator it = d->thumbnailCache.find(aUrl); + auto it = d->thumbnailCache.find(aUrl); if (it == d->thumbnailCache.end()) { - // insert into cache to mark as pending - d->pendingThumbnails.push_front(aUrl); - d->thumbnailCache[aUrl] = std::string(); - d->downloadNextPendingThumbnail(); - } else if (!it->second.empty()) { - // already loaded, fire data synchronously - d->fireDataForThumbnail(aUrl, it->second); + bool cachedOnDisk = d->checkPersistentCache(aUrl); + if (cachedOnDisk) { + // checkPersistentCache will insert the entry and schedule + } else { + d->pendingThumbnails.push_front(aUrl); + d->thumbnailCache[aUrl] = RootPrivate::ThumbnailCacheEntry(); + d->thumbnailCache[aUrl].requestPending = true; + d->downloadNextPendingThumbnail(); + } } else { - // in cache but empty data, still fetching + if (!it->second.requestPending && it->second.pathOnDisk.exists()) { + d->queueLoadFromPersistentCache(aUrl, it->second.pathOnDisk); + } } }