Update HTTP code to support HTTP/1.0 responses, eg metarproxy

simgear/io/HTTPClient.cxx

@@ -38,23 +38,16 @@ class Connection : public NetChat
 public:
     Connection(Client* pr) :
         client(pr),
-        state(STATE_IDLE)
+        state(STATE_CLOSED),
+        port(DEFAULT_HTTP_PORT)
     {
         setTerminator("\r\n");
     }
     
-    bool connectToHost(const string& host, short port)
+    void setServer(const string& h, short p)
     {
-        if (!open()) {
+        host = h;
-            SG_LOG(SG_ALL, SG_WARN, "HTTP::Connection: connectToHost: open() failed");
+        port = p;
-            return false;
-        }
-        
-        if (connect(host.c_str(), port) != 0) {
-            return false;
-        }
-        
-        return true;
     }
     
     // socket-level errors
@@ -62,6 +55,7 @@ public:
     {        
         NetChat::handleError(error);
         if (activeRequest) {
+            SG_LOG(SG_IO, SG_INFO, "HTTP socket error");
             activeRequest->setFailure(error, "socket error");
             activeRequest = NULL;
         }
@@ -69,6 +63,19 @@ public:
         state = STATE_SOCKET_ERROR;
     }
     
+    virtual void handleClose()
+    {
+        if ((state == STATE_GETTING_BODY) && activeRequest) {
+        // force state here, so responseComplete can avoid closing the 
+        // socket again
+            state =  STATE_CLOSED;
+            responseComplete();
+        }
+        
+        state = STATE_CLOSED;
+        NetChat::handleClose();
+    }
+    
     void queueRequest(const Request_ptr& r)
     {
         if (!activeRequest) {
@@ -80,9 +87,17 @@ public:
     
     void startRequest(const Request_ptr& r)
     {
+        if (state == STATE_CLOSED) {
+            if (!connectToHost()) {
+                return;
+            }
+            
+            state = STATE_IDLE;
+        }
+        
         activeRequest = r;
         state = STATE_IDLE;
-        bodyTransferSize = 0;
+        bodyTransferSize = -1;
         chunkedTransfer = false;
         
         stringstream headerData;
@@ -106,7 +121,12 @@ public:
 
     // TODO - add request body support for PUT, etc operations
 
-        push(headerData.str().c_str());
+        bool ok = push(headerData.str().c_str());
+        if (!ok) {
+            SG_LOG(SG_IO, SG_WARN, "HTTP writing to socket failed");
+            state = STATE_SOCKET_ERROR;
+            return;
+        }        
     }
     
     virtual void collectIncomingData(const char* s, int n)
@@ -169,6 +189,23 @@ public:
         return (state == STATE_SOCKET_ERROR);
     }
 private:
+    bool connectToHost()
+    {
+        SG_LOG(SG_IO, SG_INFO, "HTTP connecting to " << host << ":" << port);
+        
+        if (!open()) {
+            SG_LOG(SG_ALL, SG_WARN, "HTTP::Connection: connectToHost: open() failed");
+            return false;
+        }
+        
+        if (connect(host.c_str(), port) != 0) {
+            return false;
+        }
+        
+        return true;
+    }
+    
+    
     void processHeader()
     {
         string h = strutils::simplify(buffer);
@@ -177,12 +214,11 @@ private:
             
             if (chunkedTransfer) {
                 state = STATE_GETTING_CHUNKED;
-            } else if (bodyTransferSize > 0) {
-                state = STATE_GETTING_BODY;
-                setByteCount(bodyTransferSize);
             } else {
-                responseComplete();
+                setByteCount(bodyTransferSize); // may be -1, that's fine
+                state = STATE_GETTING_BODY;
             }
+            
             return;
         }
         
@@ -272,9 +308,18 @@ private:
     {
         activeRequest->responseComplete();
         client->requestFinished(this);
-        activeRequest = NULL;
         
-        state = STATE_IDLE;
+        bool doClose = activeRequest->closeAfterComplete();
+        activeRequest = NULL;
+        if (state == STATE_GETTING_BODY) {
+            state = STATE_IDLE;
+            if (doClose) {
+            // this will bring us into handleClose() above, which updates
+            // state to STATE_CLOSED
+                close();
+            }
+        }
+        
         setTerminator("\r\n");
         
         if (!queuedRequests.empty()) {
@@ -293,12 +338,15 @@ private:
         STATE_GETTING_CHUNKED,
         STATE_GETTING_CHUNKED_BYTES,
         STATE_GETTING_TRAILER,
-        STATE_SOCKET_ERROR
+        STATE_SOCKET_ERROR,
+        STATE_CLOSED             ///< connection should be closed now
     };
     
     Client* client;
     Request_ptr activeRequest;
     ConnectionState state;
+    string host;
+    short port;
     std::string buffer;
     int bodyTransferSize;
     SGTimeStamp idleTime;
@@ -320,6 +368,8 @@ void Client::update()
     for (; it != _connections.end(); ) {
         if (it->second->hasIdleTimeout() || it->second->hasError()) {
         // connection has been idle for a while, clean it up
+        // (or has an error condition, again clean it up)
+            SG_LOG(SG_IO, SG_INFO, "cleaning up " << it->second);
             ConnectionDict::iterator del = it++;
             delete del->second;
             _connections.erase(del);
@@ -344,18 +394,7 @@ void Client::makeRequest(const Request_ptr& r)
     
     if (_connections.find(connectionId) == _connections.end()) {
         Connection* con = new Connection(this);
-        bool ok = con->connectToHost(host, port);
+        con->setServer(host, port);
-        if (!ok) {
-        // since NetChannel connect is non-blocking, this failure
-        // path is unlikely, but still checked for.
-            SG_LOG(SG_IO, SG_WARN, "unable to connect to host:" 
-                << host << " (port:" << port << ")");
-            delete con;
-            
-            r->setFailure(-1, "unable to connect to host");
-            return;
-        }
-        
         _connections[connectionId] = con;
     }
     

simgear/io/HTTPClient.hxx

@@ -37,6 +37,7 @@ private:
     void requestFinished(Connection* con);
     
     friend class Connection;
+    friend class Request;
     
     std::string _userAgent;
     std::string _proxy;

simgear/io/HTTPRequest.cxx

@@ -18,9 +18,11 @@ extern const int DEFAULT_HTTP_PORT;
 Request::Request(const string& url, const string method) :
     _method(method),
     _url(url),
+    _responseVersion(HTTP_VERSION_UNKNOWN),
     _responseStatus(0),
     _responseLength(0),
-    _receivedBodyBytes(0)
+    _receivedBodyBytes(0),
+    _willClose(false)
 {
     
 }
@@ -52,17 +54,21 @@ void Request::responseStart(const string& r)
     string_list parts = strutils::split(r, NULL, maxSplit);
     if (parts.size() != 3) {
         SG_LOG(SG_IO, SG_WARN, "HTTP::Request: malformed response start:" << r);
-        _responseStatus = 400;
+        setFailure(400, "malformed HTTP response header");
-        _responseReason = "bad HTTP response header";
         return;
     }
     
+    _responseVersion = decodeVersion(parts[0]);    
     _responseStatus = strutils::to_int(parts[1]);
     _responseReason = parts[2];
 }
 
 void Request::responseHeader(const string& key, const string& value)
 {
+    if (key == "connection") {
+        _willClose = (value.find("close") >= 0);
+    }
+    
     _responseHeaders[key] = value;
 }
 
@@ -185,6 +191,20 @@ void Request::failed()
     // no-op in base class
 }
 
+Request::HTTPVersion Request::decodeVersion(const string& v)
+{
+    if (v == "HTTP/1.1") return HTTP_1_1;
+    if (v == "HTTP/1.0") return HTTP_1_0;
+    if (strutils::starts_with(v, "HTTP/0.")) return HTTP_0_x;
+    return HTTP_VERSION_UNKNOWN;
+}
+
+bool Request::closeAfterComplete() const
+{
+// for non HTTP/1.1 connections, assume server closes
+    return _willClose || (_responseVersion != HTTP_1_1);
+}
+
 } // of namespace HTTP
 
 } // of namespace simgear

simgear/io/HTTPRequest.hxx

@@ -50,6 +50,20 @@ public:
      */
     unsigned int responseBytesReceived() const
         { return _receivedBodyBytes; }
+        
+    enum HTTPVersion {
+        HTTP_VERSION_UNKNOWN = 0,
+        HTTP_0_x, // 0.9 or similar
+        HTTP_1_0,
+        HTTP_1_1
+    };
+    
+    HTTPVersion responseVersion() const
+        { return _responseVersion; }
+    
+    static HTTPVersion decodeVersion(const std::string& v);
+    
+    bool closeAfterComplete() const;
 protected:
     Request(const std::string& url, const std::string method = "GET");
 
@@ -68,10 +82,12 @@ private:
 
     std::string _method;
     std::string _url;
+    HTTPVersion _responseVersion;
     int _responseStatus;
     std::string _responseReason;
     unsigned int _responseLength;
     unsigned int _receivedBodyBytes;
+    bool _willClose;
     
     typedef std::map<std::string, std::string> HeaderDict;
     HeaderDict _responseHeaders; 

simgear/io/httpget.cxx

@@ -5,12 +5,14 @@
 #include <unistd.h> // for STDOUT_FILENO
 #include <iostream>
 #include <boost/foreach.hpp>
+#include <signal.h>
 
 #include <simgear/io/sg_file.hxx>
 #include <simgear/io/HTTPClient.hxx>
 #include <simgear/io/HTTPRequest.hxx>
 #include <simgear/io/sg_netChannel.hxx>
 #include <simgear/misc/strutils.hxx>
+#include <simgear/misc/sg_sleep.hxx>
 
 using namespace simgear;
 using std::cout;
@@ -126,12 +128,13 @@ int main(int argc, char* argv[])
         if (colonPos >= 0) {
             proxyHost = proxy.substr(0, colonPos);
             proxyPort = strutils::to_int(proxy.substr(colonPos + 1));
-            cout << proxyHost << " " << proxyPort << endl;
         }
         
         cl.setProxy(proxyHost, proxyPort, proxyAuth);
     }
 
+    signal(SIGPIPE, SIG_IGN);
+
     if (!outFile) {
         outFile = new SGFile(STDOUT_FILENO);
     }
@@ -150,7 +153,8 @@ int main(int argc, char* argv[])
     cl.makeRequest(req);
     
     while (!req->complete()) {
-        NetChannel::poll(100);
+        cl.update();
+        sleepForMSec(100);
     }
         
     if (req->responseCode() != 200) {

simgear/io/test_HTTP.cxx

@@ -147,7 +147,7 @@ public:
         if (path == "/test1") {
             string contentStr(BODY1);
             stringstream d;
-            d << "HTTP1.1 " << 200 << " " << reasonForCode(200) << "\r\n";
+            d << "HTTP/1.1 " << 200 << " " << reasonForCode(200) << "\r\n";
             d << "Content-Length:" << contentStr.size() << "\r\n";
             d << "\r\n"; // final CRLF to terminate the headers
             d << contentStr;
@@ -156,7 +156,7 @@ public:
             sendBody2();
         } else if (path == "/testchunked") {
             stringstream d;
-            d << "HTTP1.1 " << 200 << " " << reasonForCode(200) << "\r\n";
+            d << "HTTP/1.1 " << 200 << " " << reasonForCode(200) << "\r\n";
             d << "Transfer-Encoding:chunked\r\n";
             d << "\r\n";
             d << "8\r\n"; // first chunk
@@ -172,47 +172,70 @@ public:
         } else if (path == "http://www.google.com/test2") {
             // proxy test
             if (requestHeaders["host"] != "www.google.com") {
-                sendErrorResponse(400);
+                sendErrorResponse(400, true, "bad destination");
             }
             
             if (requestHeaders["proxy-authorization"] != string()) {
-                sendErrorResponse(401); // shouldn't supply auth
+                sendErrorResponse(401, false, "bad auth"); // shouldn't supply auth
             }
             
             sendBody2();
         } else if (path == "http://www.google.com/test3") {
             // proxy test
             if (requestHeaders["host"] != "www.google.com") {
-                sendErrorResponse(400);
+                sendErrorResponse(400, true, "bad destination");
             }
 
             if (requestHeaders["proxy-authorization"] != "ABCDEF") {
-                sendErrorResponse(401); // forbidden
+                sendErrorResponse(401, false, "bad auth"); // forbidden
             }
 
             sendBody2();
+        } else if (path == "/test_1_0") {
+            string contentStr(BODY1);
+            stringstream d;
+            d << "HTTP/1.0 " << 200 << " " << reasonForCode(200) << "\r\n";
+            d << "\r\n"; // final CRLF to terminate the headers
+            d << contentStr;
+            push(d.str().c_str());
+            closeWhenDone();
+        } else if (path == "/test_close") {
+            string contentStr(BODY1);
+            stringstream d;
+            d << "HTTP/1.1 " << 200 << " " << reasonForCode(200) << "\r\n";
+            d << "Connection: close\r\n";
+            d << "\r\n"; // final CRLF to terminate the headers
+            d << contentStr;
+            push(d.str().c_str());
+            closeWhenDone();
         } else {
-            sendErrorResponse(404);
+            sendErrorResponse(404, true, "");
         }
     }
     
     void sendBody2()
     {
         stringstream d;
-        d << "HTTP1.1 " << 200 << " " << reasonForCode(200) << "\r\n";
+        d << "HTTP/1.1 " << 200 << " " << reasonForCode(200) << "\r\n";
         d << "Content-Length:" << body2Size << "\r\n";
         d << "\r\n"; // final CRLF to terminate the headers
         push(d.str().c_str());
         bufferSend(body2, body2Size);
     }
     
-    void sendErrorResponse(int code)
+    void sendErrorResponse(int code, bool close, string content)
     {
         cerr << "sending error " << code << " for " << path << endl;
         stringstream headerData;
-        headerData << "HTTP1.1 " << code << " " << reasonForCode(code) << "\r\n";
+        headerData << "HTTP/1.1 " << code << " " << reasonForCode(code) << "\r\n";
+        headerData << "Content-Length:" << content.size() << "\r\n";
         headerData << "\r\n"; // final CRLF to terminate the headers
         push(headerData.str().c_str());
+        push(content.c_str());
+        
+        if (close) {
+            closeWhenDone();
+        }
     }
     
     string reasonForCode(int code) 
@@ -252,7 +275,7 @@ public:
     {
         simgear::IPAddress addr ;
         int handle = accept ( &addr ) ;
-        cout << "did accept from " << addr.getHost() << ":" << addr.getPort() << endl;
+        //cout << "did accept from " << addr.getHost() << ":" << addr.getPort() << endl;
         TestServerChannel* chan = new TestServerChannel();
         chan->setHandle(handle);
     }
@@ -359,6 +382,30 @@ int main(int argc, char* argv[])
         COMPARE(tr->responseLength(), 0);
     }
 
+    cout << "done1" << endl;
+// test HTTP/1.0
+    {
+        TestRequest* tr = new TestRequest("http://localhost:2000/test_1_0");
+        HTTP::Request_ptr own(tr);
+        cl.makeRequest(tr);
+        waitForComplete(tr);
+        COMPARE(tr->responseCode(), 200);
+        COMPARE(tr->responseLength(), strlen(BODY1));
+        COMPARE(tr->bodyData, string(BODY1));
+    }
+
+    cout << "done2" << endl;
+// test HTTP/1.1 Connection::close
+    {
+        TestRequest* tr = new TestRequest("http://localhost:2000/test_close");
+        HTTP::Request_ptr own(tr);
+        cl.makeRequest(tr);
+        waitForComplete(tr);
+        COMPARE(tr->responseCode(), 200);
+        COMPARE(tr->responseLength(), strlen(BODY1));
+        COMPARE(tr->bodyData, string(BODY1));
+    }
+    cout << "done3" << endl;
 // test connectToHost failure
 /*
     {