diff --git a/simgear/io/HTTPClient.cxx b/simgear/io/HTTPClient.cxx index 1c94ccfb..d0f414b7 100644 --- a/simgear/io/HTTPClient.cxx +++ b/simgear/io/HTTPClient.cxx @@ -202,7 +202,17 @@ void Client::update(int waitTimeout) } if (doProcess) { + bool ok = false; + SG_LOG(SG_IO, SG_DEBUG, "msg->data.result=" << msg->data.result); if (msg->data.result == 0) { + ok = true; + } + else if (req->range() != "" && (msg->data.result == 206 || msg->data.result == 416)) { + SG_LOG(SG_IO, SG_DEBUG, "req->range() is set (to '" << req->range() << "')" + << " so treating as ok: msg->data.result=" << msg->data.result); + ok = true; + } + if (ok) { req->responseComplete(); } else { SG_LOG(SG_IO, SG_WARN, @@ -255,6 +265,10 @@ void Client::makeRequest(const Request_ptr& r) curl_easy_setopt(curlRequest, CURLOPT_WRITEDATA, r.get()); curl_easy_setopt(curlRequest, CURLOPT_HEADERFUNCTION, requestHeaderCallback); curl_easy_setopt(curlRequest, CURLOPT_HEADERDATA, r.get()); + unsigned long bps = r->getMaxBytesPerSec(); + if (bps != 0) { + curl_easy_setopt(curlRequest, CURLOPT_MAX_RECV_SPEED_LARGE, bps); + } #if !defined(CURL_MAX_READ_SIZE) const int CURL_MAX_READ_SIZE = 512 * 1024; @@ -285,6 +299,12 @@ void Client::makeRequest(const Request_ptr& r) } } + std::string range = r->range(); + if (range != "") { + curl_easy_setopt(curlRequest, CURLOPT_RANGE, range.c_str()); + SG_LOG(SG_GENERAL, SG_DEBUG, "Have added CURLOPT_RANGE: '" << range << "'"); + } + const std::string method = strutils::lowercase (r->method()); if (method == "get") { curl_easy_setopt(curlRequest, CURLOPT_HTTPGET, 1); diff --git a/simgear/io/HTTPFileRequest.cxx b/simgear/io/HTTPFileRequest.cxx index 135d7fc5..018e06da 100644 --- a/simgear/io/HTTPFileRequest.cxx +++ b/simgear/io/HTTPFileRequest.cxx @@ -28,28 +28,64 @@ namespace HTTP { //---------------------------------------------------------------------------- - FileRequest::FileRequest(const std::string& url, const std::string& path): + FileRequest::FileRequest(const std::string& url, const std::string& path, bool append): Request(url, "GET"), - _filename(path) + _filename(path), + _append(append), + _callback() { + if (append && _filename.isFile()) { + size_t size = _filename.sizeInBytes(); + if (size) { + // Only download remaining bytes. + char buffer[32]; + snprintf(buffer, sizeof(buffer), "%zi-", size); + setRange(buffer); + } + } + } + void FileRequest::setCallback(std::function callback) + { + _callback = callback; } //---------------------------------------------------------------------------- void FileRequest::responseHeadersComplete() { Request::responseHeadersComplete(); + SG_LOG(SG_GENERAL, SG_BULK, "FileRequest::responseHeadersComplete()" + << " responseCode()=" << responseCode() + << " responseReason()=" << responseReason() + << " _append=" << _append + ); - if( responseCode() != 200 ) + bool ok = false; + if (responseCode() == 200) { + ok = true; + } + else if (_append && (responseCode() == 206 || responseCode() == 416)) { + /* See comments for simgear::HTTP::Client::setRange(). */ + SG_LOG(SG_IO, SG_ALERT, "_append is true so treating response code as success: " + << responseCode()); + ok = true; + } + if (!ok) { + SG_LOG(SG_GENERAL, SG_ALERT, "failure. calling setFailure()." + << " responseCode()=" << responseCode() + << " responseReason()=" << responseReason() + ); return setFailure(responseCode(), responseReason()); + } if( !_filename.isNull() ) { // TODO validate path? (would require to expose fgValidatePath somehow to // simgear) _filename.create_dir(0755); - - _file.open(_filename, std::ios::binary | std::ios::trunc | std::ios::out); + auto flags = std::ios::binary | std::ios::out | (_append ? std::ios::app : std::ios::trunc); + SG_LOG(SG_GENERAL, SG_DEBUG, "attempting to open file. _append=" << _append); + _file.open(_filename, flags); } if( !_file ) @@ -57,7 +93,7 @@ namespace HTTP SG_LOG ( SG_IO, - SG_WARN, + SG_ALERT, "HTTP::FileRequest: failed to open file '" << _filename << "'" ); } @@ -78,12 +114,18 @@ namespace HTTP } _file.write(s, n); + if (_callback) { + _callback(s, n); + } } //---------------------------------------------------------------------------- void FileRequest::onAlways() { _file.close(); + if (_callback) { + _callback(nullptr, 0); + } } } // namespace HTTP diff --git a/simgear/io/HTTPFileRequest.hxx b/simgear/io/HTTPFileRequest.hxx index b725c9ba..94ebecf2 100644 --- a/simgear/io/HTTPFileRequest.hxx +++ b/simgear/io/HTTPFileRequest.hxx @@ -25,6 +25,8 @@ #include +#include + namespace simgear { namespace HTTP @@ -42,12 +44,25 @@ namespace HTTP * * @param url Adress to download from * @param path Path to file for saving response + * @append If true we assume any existing file is partial download + * and use simgear::HTTP::Request::setRange() to download + * and append any remaining data. We don't currently + * attempt to use an If-Range validator to protect against + * any existing file containing incorrect data. */ - FileRequest(const std::string& url, const std::string& path); + FileRequest(const std::string& url, const std::string& path, bool append=false); + + /* + * Set callback for each chunk of data we receive. Called with (nullptr, + * 0) when download has completed (successfully or unsuccesfully). + */ + void setCallback(std::function callback); protected: SGPath _filename; sg_ofstream _file; + bool _append; + std::function _callback; virtual void responseHeadersComplete(); virtual void gotBodyData(const char* s, int n); diff --git a/simgear/io/HTTPRequest.cxx b/simgear/io/HTTPRequest.cxx index 0825486d..b9e6c6e1 100644 --- a/simgear/io/HTTPRequest.cxx +++ b/simgear/io/HTTPRequest.cxx @@ -127,6 +127,12 @@ void Request::setUrl(const std::string& url) _url = url; } +//------------------------------------------------------------------------------ +void Request::setRange(const std::string& range) +{ + _range = range; +} + //------------------------------------------------------------------------------ void Request::requestStart() { diff --git a/simgear/io/HTTPRequest.hxx b/simgear/io/HTTPRequest.hxx index 010ea2b8..d9742323 100644 --- a/simgear/io/HTTPRequest.hxx +++ b/simgear/io/HTTPRequest.hxx @@ -126,11 +126,34 @@ public: void setBodyData( const SGPropertyNode* data ); virtual void setUrl(const std::string& url); + + /* + * Set Range header, e.g. "1234-" to skip first 1234 bytes. + * + * If a range is specified, we treat http response codes 206 'Partial + * Content' and 416 'Range Not Satisfiable' both as success. + * + * The latter is unfortunate - e.g. with range='1234-' it is returned if + * the remote file is exactly 1234 bytes long, which for example will occur + * if we are updating a file that was fully downloaded previously. + */ + virtual void setRange(const std::string& range); virtual std::string method() const { return _method; } virtual std::string url() const { return _url; } + virtual std::string range() const + { return _range; } + + /* + * Limits download speed using CURLOPT_MAX_RECV_SPEED_LARGE. + */ + void setMaxBytesPerSec(unsigned long maxBytesPerSec) + { _maxBytesPerSec = maxBytesPerSec; } + + unsigned long getMaxBytesPerSec() const + { return _maxBytesPerSec; } Client* http() const { return _client; } @@ -245,6 +268,7 @@ public: std::string _method; std::string _url; + std::string _range; StringMap _request_headers; std::string _request_data; std::string _request_media_type; @@ -263,6 +287,7 @@ public: ReadyState _ready_state; bool _willClose; bool _connectionCloseHeader; + unsigned long _maxBytesPerSec = 0; }; typedef SGSharedPtr Request_ptr;