simgear/io/: exteded http client support to allow replaying of recordings from a URL.

simgear/io/HTTPClient.cxx
    Support range in HTTPRequest's. Set CURLOPT_MAX_RECV_SPEED_LARGE if request's
    getMaxBytesPerSec() is non-zero.
simgear/io/HTTPFileRequest.cxx
simgear/io/HTTPFileRequest.hxx
    Added 'bool append' to constructor args; if specified we assume that, if
    the output file already exists, it is from an interrupted download, and we
    use an http Range header to append any remaining data.
    Support a std::function callback when new data is available.
    Added setMaxBytesPerSec() to limit download bandwidth; useful for testing.
simgear/io/HTTPRequest.cxx
simgear/io/HTTPRequest.hxx
   Added support for setting Range header.

In both simgear/io/HTTPClient.cxx and simgear/io/HTTPRequest.cxx, if
a range is specified, we treat http results 206 'Partial Content' and
416 'Range Not Satisfiable' as success.

simgear/io/HTTPClient.cxx

@@ -202,7 +202,17 @@ void Client::update(int waitTimeout)
           }
 
           if (doProcess) {
+            bool ok = false;
+            SG_LOG(SG_IO, SG_DEBUG, "msg->data.result=" << msg->data.result);
             if (msg->data.result == 0) {
+              ok = true;
+            }
+            else if (req->range() != "" && (msg->data.result == 206 || msg->data.result == 416)) {
+              SG_LOG(SG_IO, SG_DEBUG, "req->range() is set (to '" << req->range() << "')"
+                      << " so treating as ok: msg->data.result=" << msg->data.result);
+              ok = true;
+            }
+            if (ok) {
               req->responseComplete();
             } else {
               SG_LOG(SG_IO, SG_WARN,
@@ -255,6 +265,10 @@ void Client::makeRequest(const Request_ptr& r)
     curl_easy_setopt(curlRequest, CURLOPT_WRITEDATA, r.get());
     curl_easy_setopt(curlRequest, CURLOPT_HEADERFUNCTION, requestHeaderCallback);
     curl_easy_setopt(curlRequest, CURLOPT_HEADERDATA, r.get());
+    unsigned long bps = r->getMaxBytesPerSec();
+    if (bps != 0) {
+        curl_easy_setopt(curlRequest, CURLOPT_MAX_RECV_SPEED_LARGE, bps);
+    }
 
 #if !defined(CURL_MAX_READ_SIZE)
 	const int CURL_MAX_READ_SIZE = 512 * 1024;
@@ -285,6 +299,12 @@ void Client::makeRequest(const Request_ptr& r)
       }
     }
 
+    std::string range = r->range();
+    if (range != "") {
+        curl_easy_setopt(curlRequest, CURLOPT_RANGE, range.c_str());
+        SG_LOG(SG_GENERAL, SG_DEBUG, "Have added CURLOPT_RANGE: '" << range << "'");
+    }
+
     const std::string method = strutils::lowercase (r->method());
     if (method == "get") {
       curl_easy_setopt(curlRequest, CURLOPT_HTTPGET, 1);

simgear/io/HTTPFileRequest.cxx

@@ -28,28 +28,64 @@ namespace HTTP
 {
 
   //----------------------------------------------------------------------------
-  FileRequest::FileRequest(const std::string& url, const std::string& path):
+  FileRequest::FileRequest(const std::string& url, const std::string& path, bool append):
     Request(url, "GET"),
-    _filename(path)
+    _filename(path),
+    _append(append),
+    _callback()
   {
+    if (append && _filename.isFile()) {
+      size_t size = _filename.sizeInBytes();
+      if (size) {
+        // Only download remaining bytes.
+        char buffer[32];
+        snprintf(buffer, sizeof(buffer), "%zi-", size);
+        setRange(buffer);
+      }
+    }
+  }
 
+  void FileRequest::setCallback(std::function<void(const void* data, size_t numbytes)> callback)
+  {
+    _callback = callback;
   }
 
   //----------------------------------------------------------------------------
   void FileRequest::responseHeadersComplete()
   {
     Request::responseHeadersComplete();
+    SG_LOG(SG_GENERAL, SG_BULK, "FileRequest::responseHeadersComplete()"
+        << " responseCode()=" << responseCode()
+        << " responseReason()=" << responseReason()
+        << " _append=" << _append
+        );
 
-    if( responseCode() != 200 )
+    bool ok = false;
+    if (responseCode() == 200) {
+      ok = true;
+    }
+    else if (_append && (responseCode() == 206 || responseCode() == 416)) {
+      /* See comments for simgear::HTTP::Client::setRange(). */
+      SG_LOG(SG_IO, SG_ALERT, "_append is true so treating response code as success: "
+          << responseCode());
+      ok = true;
+    }
+    if (!ok) {
+      SG_LOG(SG_GENERAL, SG_ALERT, "failure. calling setFailure()."
+          << " responseCode()=" << responseCode()
+          << " responseReason()=" << responseReason()
+          );
       return setFailure(responseCode(), responseReason());
+    }
 
     if( !_filename.isNull() )
     {
       // TODO validate path? (would require to expose fgValidatePath somehow to
       //      simgear)
       _filename.create_dir(0755);
-
+      auto flags = std::ios::binary | std::ios::out | (_append ? std::ios::app : std::ios::trunc);
-      _file.open(_filename, std::ios::binary | std::ios::trunc | std::ios::out);
+      SG_LOG(SG_GENERAL, SG_DEBUG, "attempting to open file. _append=" << _append);
+      _file.open(_filename, flags);
     }
 
     if( !_file )
@@ -57,7 +93,7 @@ namespace HTTP
       SG_LOG
       (
         SG_IO,
-        SG_WARN,
+        SG_ALERT,
         "HTTP::FileRequest: failed to open file '" << _filename << "'"
       );
     }
@@ -78,12 +114,18 @@ namespace HTTP
     }
 
     _file.write(s, n);
+    if (_callback) {
+      _callback(s, n);
+    }
   }
 
   //----------------------------------------------------------------------------
   void FileRequest::onAlways()
   {
     _file.close();
+    if (_callback) {
+      _callback(nullptr, 0);
+    }
   }
 
 } // namespace HTTP

simgear/io/HTTPFileRequest.hxx

@@ -25,6 +25,8 @@
 
 #include <simgear/io/iostreams/sgstream.hxx>
 
+#include <functional>
+
 namespace simgear
 {
 namespace HTTP
@@ -42,12 +44,25 @@ namespace HTTP
        *
        * @param url     Adress to download from
        * @param path    Path to file for saving response
+       * @append        If true we assume any existing file is partial download
+       *                and use simgear::HTTP::Request::setRange() to download
+       *                and append any remaining data. We don't currently
+       *                attempt to use an If-Range validator to protect against
+       *                any existing file containing incorrect data.
        */
-      FileRequest(const std::string& url, const std::string& path);
+      FileRequest(const std::string& url, const std::string& path, bool append=false);
+      
+      /*
+       * Set callback for each chunk of data we receive. Called with (nullptr,
+       * 0) when download has completed (successfully or unsuccesfully).
+       */
+      void setCallback(std::function<void (const void* data, size_t numbytes)> callback);
 
     protected:
       SGPath _filename;
       sg_ofstream _file;
+      bool _append;
+      std::function<void(const void* data, size_t numbytes)> _callback;
 
       virtual void responseHeadersComplete();
       virtual void gotBodyData(const char* s, int n);

simgear/io/HTTPRequest.cxx

@@ -127,6 +127,12 @@ void Request::setUrl(const std::string& url)
   _url = url;
 }
 
+//------------------------------------------------------------------------------
+void Request::setRange(const std::string& range)
+{
+  _range = range;
+}
+
 //------------------------------------------------------------------------------
 void Request::requestStart()
 {

simgear/io/HTTPRequest.hxx

@@ -126,11 +126,34 @@ public:
     void setBodyData( const SGPropertyNode* data );
 
     virtual void setUrl(const std::string& url);
+    
+    /*
+     * Set Range header, e.g. "1234-" to skip first 1234 bytes.
+     *
+     * If a range is specified, we treat http response codes 206 'Partial
+     * Content' and 416 'Range Not Satisfiable' both as success.
+     *
+     * The latter is unfortunate - e.g. with range='1234-' it is returned if
+     * the remote file is exactly 1234 bytes long, which for example will occur
+     * if we are updating a file that was fully downloaded previously.
+     */
+    virtual void setRange(const std::string& range);
 
     virtual std::string method() const
         { return _method; }
     virtual std::string url() const
         { return _url; }
+    virtual std::string range() const
+        { return _range; }
+    
+    /*
+     * Limits download speed using CURLOPT_MAX_RECV_SPEED_LARGE.
+     */
+    void setMaxBytesPerSec(unsigned long maxBytesPerSec)
+        { _maxBytesPerSec = maxBytesPerSec; }
+    
+    unsigned long getMaxBytesPerSec() const
+        { return _maxBytesPerSec; }
 
     Client* http() const
     { return _client; }
@@ -245,6 +268,7 @@ public:
 
     std::string   _method;
     std::string   _url;
+    std::string   _range;
     StringMap     _request_headers;
     std::string   _request_data;
     std::string   _request_media_type;
@@ -263,6 +287,7 @@ public:
     ReadyState    _ready_state;
     bool          _willClose;
     bool          _connectionCloseHeader;
+    unsigned long _maxBytesPerSec = 0;
 };
 
 typedef SGSharedPtr<Request> Request_ptr;