FGFS-4.1/simgear
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

simgear/io/: exteded http client support to allow replaying of recordings from a URL.

Browse Source 
simgear/io/HTTPClient.cxx
    Support range in HTTPRequest's. Set CURLOPT_MAX_RECV_SPEED_LARGE if request's
    getMaxBytesPerSec() is non-zero.
simgear/io/HTTPFileRequest.cxx
simgear/io/HTTPFileRequest.hxx
    Added 'bool append' to constructor args; if specified we assume that, if
    the output file already exists, it is from an interrupted download, and we
    use an http Range header to append any remaining data.
    Support a std::function callback when new data is available.
    Added setMaxBytesPerSec() to limit download bandwidth; useful for testing.
simgear/io/HTTPRequest.cxx
simgear/io/HTTPRequest.hxx
   Added support for setting Range header.

In both simgear/io/HTTPClient.cxx and simgear/io/HTTPRequest.cxx, if
a range is specified, we treat http results 206 'Partial Content' and
416 'Range Not Satisfiable' as success.
This commit is contained in:
Julian Smith
2021-02-15 15:50:50 +00:00
parent 7f83686853
commit f9ecf455ba
5 changed files with 115 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
simgear/io/HTTPClient.cxx
View File

@@ -202,7 +202,17 @@ void Client::update(int waitTimeout)
}
if (doProcess) {
bool ok = false;
SG_LOG(SG_IO, SG_DEBUG, "msg->data.result=" << msg->data.result);
if (msg->data.result == 0) {
ok = true;
}
else if (req->range() != "" && (msg->data.result == 206 || msg->data.result == 416)) {
SG_LOG(SG_IO, SG_DEBUG, "req->range() is set (to '" << req->range() << "')"
<< " so treating as ok: msg->data.result=" << msg->data.result);
ok = true;
}
if (ok) {
req->responseComplete();
} else {
SG_LOG(SG_IO, SG_WARN,
@@ -255,6 +265,10 @@ void Client::makeRequest(const Request_ptr& r)
curl_easy_setopt(curlRequest, CURLOPT_WRITEDATA, r.get());
curl_easy_setopt(curlRequest, CURLOPT_HEADERFUNCTION, requestHeaderCallback);
curl_easy_setopt(curlRequest, CURLOPT_HEADERDATA, r.get());
unsigned long bps = r->getMaxBytesPerSec();
if (bps != 0) {
curl_easy_setopt(curlRequest, CURLOPT_MAX_RECV_SPEED_LARGE, bps);
}
#if !defined(CURL_MAX_READ_SIZE)
const int CURL_MAX_READ_SIZE = 512 * 1024;
@@ -285,6 +299,12 @@ void Client::makeRequest(const Request_ptr& r)
}
}
std::string range = r->range();
if (range != "") {
curl_easy_setopt(curlRequest, CURLOPT_RANGE, range.c_str());
SG_LOG(SG_GENERAL, SG_DEBUG, "Have added CURLOPT_RANGE: '" << range << "'");
}
const std::string method = strutils::lowercase (r->method());
if (method == "get") {
curl_easy_setopt(curlRequest, CURLOPT_HTTPGET, 1);

54
simgear/io/HTTPFileRequest.cxx
View File

@@ -28,28 +28,64 @@ namespace HTTP
{
//----------------------------------------------------------------------------
FileRequest::FileRequest(const std::string& url, const std::string& path):
FileRequest::FileRequest(const std::string& url, const std::string& path, bool append):
Request(url, "GET"),
_filename(path)
_filename(path),
_append(append),
_callback()
{
if (append && _filename.isFile()) {
size_t size = _filename.sizeInBytes();
if (size) {
// Only download remaining bytes.
char buffer[32];
snprintf(buffer, sizeof(buffer), "%zi-", size);
setRange(buffer);
}
}
}
void FileRequest::setCallback(std::function<void(const void* data, size_t numbytes)> callback)
{
_callback = callback;
}
//----------------------------------------------------------------------------
void FileRequest::responseHeadersComplete()
{
Request::responseHeadersComplete();
SG_LOG(SG_GENERAL, SG_BULK, "FileRequest::responseHeadersComplete()"
<< " responseCode()=" << responseCode()
<< " responseReason()=" << responseReason()
<< " _append=" << _append
);
if( responseCode() != 200 )
bool ok = false;
if (responseCode() == 200) {
ok = true;
}
else if (_append && (responseCode() == 206 || responseCode() == 416)) {
/* See comments for simgear::HTTP::Client::setRange(). */
SG_LOG(SG_IO, SG_ALERT, "_append is true so treating response code as success: "
<< responseCode());
ok = true;
}
if (!ok) {
SG_LOG(SG_GENERAL, SG_ALERT, "failure. calling setFailure()."
<< " responseCode()=" << responseCode()
<< " responseReason()=" << responseReason()
);
return setFailure(responseCode(), responseReason());
}
if( !_filename.isNull() )
{
// TODO validate path? (would require to expose fgValidatePath somehow to
// simgear)
_filename.create_dir(0755);
_file.open(_filename, std::ios::binary | std::ios::trunc | std::ios::out);
auto flags = std::ios::binary | std::ios::out | (_append ? std::ios::app : std::ios::trunc);
SG_LOG(SG_GENERAL, SG_DEBUG, "attempting to open file. _append=" << _append);
_file.open(_filename, flags);
}
if( !_file )
@@ -57,7 +93,7 @@ namespace HTTP
SG_LOG
(
SG_IO,
SG_WARN,
SG_ALERT,
"HTTP::FileRequest: failed to open file '" << _filename << "'"
);
}
@@ -78,12 +114,18 @@ namespace HTTP
}
_file.write(s, n);
if (_callback) {
_callback(s, n);
}
}
//----------------------------------------------------------------------------
void FileRequest::onAlways()
{
_file.close();
if (_callback) {
_callback(nullptr, 0);
}
}
} // namespace HTTP

17
simgear/io/HTTPFileRequest.hxx
View File

@@ -25,6 +25,8 @@
#include <simgear/io/iostreams/sgstream.hxx>
#include <functional>
namespace simgear
{
namespace HTTP
@@ -42,12 +44,25 @@ namespace HTTP
*
* @param url Adress to download from
* @param path Path to file for saving response
* @append If true we assume any existing file is partial download
* and use simgear::HTTP::Request::setRange() to download
* and append any remaining data. We don't currently
* attempt to use an If-Range validator to protect against
* any existing file containing incorrect data.
*/
FileRequest(const std::string& url, const std::string& path);
FileRequest(const std::string& url, const std::string& path, bool append=false);
/*
* Set callback for each chunk of data we receive. Called with (nullptr,
* 0) when download has completed (successfully or unsuccesfully).
*/
void setCallback(std::function<void (const void* data, size_t numbytes)> callback);
protected:
SGPath _filename;
sg_ofstream _file;
bool _append;
std::function<void(const void* data, size_t numbytes)> _callback;
virtual void responseHeadersComplete();
virtual void gotBodyData(const char* s, int n);

6
simgear/io/HTTPRequest.cxx
View File

@@ -127,6 +127,12 @@ void Request::setUrl(const std::string& url)
_url = url;
}
//------------------------------------------------------------------------------
void Request::setRange(const std::string& range)
{
_range = range;
}
//------------------------------------------------------------------------------
void Request::requestStart()
{

25
simgear/io/HTTPRequest.hxx
View File

@@ -126,11 +126,34 @@ public:
void setBodyData( const SGPropertyNode* data );
virtual void setUrl(const std::string& url);
/*
* Set Range header, e.g. "1234-" to skip first 1234 bytes.
*
* If a range is specified, we treat http response codes 206 'Partial
* Content' and 416 'Range Not Satisfiable' both as success.
*
* The latter is unfortunate - e.g. with range='1234-' it is returned if
* the remote file is exactly 1234 bytes long, which for example will occur
* if we are updating a file that was fully downloaded previously.
*/
virtual void setRange(const std::string& range);
virtual std::string method() const
{ return _method; }
virtual std::string url() const
{ return _url; }
virtual std::string range() const
{ return _range; }
/*
* Limits download speed using CURLOPT_MAX_RECV_SPEED_LARGE.
*/
void setMaxBytesPerSec(unsigned long maxBytesPerSec)
{ _maxBytesPerSec = maxBytesPerSec; }
unsigned long getMaxBytesPerSec() const
{ return _maxBytesPerSec; }
Client* http() const
{ return _client; }
@@ -245,6 +268,7 @@ public:
std::string _method;
std::string _url;
std::string _range;
StringMap _request_headers;
std::string _request_data;
std::string _request_media_type;
@@ -263,6 +287,7 @@ public:
ReadyState _ready_state;
bool _willClose;
bool _connectionCloseHeader;
unsigned long _maxBytesPerSec = 0;
};
typedef SGSharedPtr<Request> Request_ptr;