From e9ea5e90366d485ca29db4aa013d38cd9fef3fc9 Mon Sep 17 00:00:00 2001 From: Erik Hofman Date: Tue, 24 May 2016 14:51:17 +0200 Subject: [PATCH 01/14] Revert the check fro std::isnan() and isnan() --- simgear/math/SGMisc.hxx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/simgear/math/SGMisc.hxx b/simgear/math/SGMisc.hxx index c317b431..79030536 100644 --- a/simgear/math/SGMisc.hxx +++ b/simgear/math/SGMisc.hxx @@ -153,10 +153,10 @@ public: /// Use with care: allways code that you do not need to use that! static bool isNaN(const T& v) { -#ifdef HAVE_ISNAN - return (isnan(v) != 0); -#elif defined HAVE_STD_ISNAN +#ifdef HAVE_STD_ISNAN return std::isnan(v); +#elif defined HAVE_ISNAN + return (isnan(v) != 0); #else // Use that every compare involving a NaN returns false // But be careful, some usual compiler switches like for example From 91f184caff8c97a5c6eaa7a23ff2ef071be43928 Mon Sep 17 00:00:00 2001 From: Erik Hofman Date: Tue, 24 May 2016 17:32:22 +0200 Subject: [PATCH 02/14] Do not expose both std::isnan() and isnan() --- simgear/sound/sample_group.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simgear/sound/sample_group.cxx b/simgear/sound/sample_group.cxx index 70df9aa1..dbbb22ab 100644 --- a/simgear/sound/sample_group.cxx +++ b/simgear/sound/sample_group.cxx @@ -33,7 +33,7 @@ #include "soundmgr_openal_private.hxx" #include "sample_group.hxx" -#ifdef HAVE_STD_ISNAN +#if defined(HAVE_STD_ISNAN) && !defined(HAVE_ISNAN) using std::isnan; #endif From 857f7c9e611a20e3106267ab8cefd1a8900827a7 Mon Sep 17 00:00:00 2001 From: Erik Hofman Date: Tue, 24 May 2016 21:37:12 +0200 Subject: [PATCH 03/14] Revert to c++98 --- CMakeLists.txt | 6 +++--- simgear/package/untar.cxx | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2299c7ad..a67d65e9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,14 +22,14 @@ set(CMAKE_OSX_DEPLOYMENT_TARGET 10.7) # only relevant for building shared libs but let's set it regardless set(CMAKE_OSX_RPATH 1) -# Set the C++ standard to C++11 to avoid compilation errors on GCC 6 (which +# Set the C++ standard to C++98 to avoid compilation errors on GCC 6 (which # defaults to C++14). if(CMAKE_VERSION VERSION_LESS "3.1") if(CMAKE_COMPILER_IS_GNUCXX) - set (CMAKE_CXX_FLAGS "-std=c++11 ${CMAKE_CXX_FLAGS}") + set (CMAKE_CXX_FLAGS "-std=gnu++98 ${CMAKE_CXX_FLAGS}") endif() else() - set (CMAKE_CXX_STANDARD 11) + set (CMAKE_CXX_STANDARD 98) endif() project(SimGear) diff --git a/simgear/package/untar.cxx b/simgear/package/untar.cxx index 1ae4f08d..ee8b7178 100644 --- a/simgear/package/untar.cxx +++ b/simgear/package/untar.cxx @@ -19,7 +19,7 @@ #include #include -#include +#include #include #include @@ -246,7 +246,7 @@ public: } // reject absolute paths - if (p.front() == '/') { + if (p.at(0) == '/') { return false; } From 87c2427cfe1b5708fb869bab7579af5cee70966f Mon Sep 17 00:00:00 2001 From: Erik Hofman Date: Wed, 25 May 2016 09:08:52 +0200 Subject: [PATCH 04/14] MSVC 2013 and later define std::isnan --- simgear/compiler.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simgear/compiler.h b/simgear/compiler.h index 795747d5..4e2bea29 100644 --- a/simgear/compiler.h +++ b/simgear/compiler.h @@ -63,7 +63,7 @@ #ifdef _MSC_VER # define bcopy(from, to, n) memcpy(to, from, n) -# if _MSC_VER >= 1200 // msvc++ 6.0 or greater +# if _MSC_VER >= 1200 && _MSC_VER < 1800 // msvc++ 6.0 up to MSVC2013 # define isnan _isnan # define snprintf _snprintf # if _MSC_VER < 1500 From b3d95c075426f27bc6105b19156f235be2325890 Mon Sep 17 00:00:00 2001 From: Erik Hofman Date: Wed, 25 May 2016 14:30:06 +0200 Subject: [PATCH 05/14] Probably a better fix --- simgear/compiler.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/simgear/compiler.h b/simgear/compiler.h index 4e2bea29..0f0c8347 100644 --- a/simgear/compiler.h +++ b/simgear/compiler.h @@ -63,8 +63,10 @@ #ifdef _MSC_VER # define bcopy(from, to, n) memcpy(to, from, n) -# if _MSC_VER >= 1200 && _MSC_VER < 1800 // msvc++ 6.0 up to MSVC2013 -# define isnan _isnan +# if _MSC_VER >= 1200 // msvc++ 6.0 up to MSVC2013 +# if _MSC_VER < 1800 +# define isnan _isnan +# endif # define snprintf _snprintf # if _MSC_VER < 1500 # define vsnprintf _vsnprintf From 49bd96c55d924c1bff4b8dad8f7a10864f1fa2f9 Mon Sep 17 00:00:00 2001 From: James Turner Date: Wed, 25 May 2016 20:08:20 +0100 Subject: [PATCH 06/14] Guard usage of CURLMOPT_MAX_TOTAL_CONNECTIONS Check the Curl patch version to avoid breaking on older sustems. --- simgear/io/HTTPClient.cxx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/simgear/io/HTTPClient.cxx b/simgear/io/HTTPClient.cxx index 2d7aebb0..4fcb4d7b 100644 --- a/simgear/io/HTTPClient.cxx +++ b/simgear/io/HTTPClient.cxx @@ -80,7 +80,9 @@ public: // see https://curl.haxx.se/libcurl/c/CURLMOPT_PIPELINING.html // we request HTTP 1.1 pipelining curl_multi_setopt(curlMulti, CURLMOPT_PIPELINING, 1 /* aka CURLPIPE_HTTP1 */); +#if (LIBCURL_VERSION_MINOR >= 30) curl_multi_setopt(curlMulti, CURLMOPT_MAX_TOTAL_CONNECTIONS, (long) maxConnections); +#endif curl_multi_setopt(curlMulti, CURLMOPT_MAX_PIPELINE_LENGTH, (long) maxPipelineDepth); curl_multi_setopt(curlMulti, CURLMOPT_MAX_HOST_CONNECTIONS, @@ -138,7 +140,9 @@ Client::~Client() void Client::setMaxConnections(unsigned int maxCon) { d->maxConnections = maxCon; +#if (LIBCURL_VERSION_MINOR >= 30) curl_multi_setopt(d->curlMulti, CURLMOPT_MAX_TOTAL_CONNECTIONS, (long) maxCon); +#endif } void Client::setMaxHostConnections(unsigned int maxHostCon) From 2b15b6b8adce06107866fff0028503c1b92b0de4 Mon Sep 17 00:00:00 2001 From: Maciej Mrozowski Date: Thu, 26 May 2016 02:24:19 +0200 Subject: [PATCH 07/14] Add SYSTEM_UDNS CMake option. Remove spurious EXPAT_LIBRARIES linking when using bundled expat. --- 3rdparty/CMakeLists.txt | 2 +- CMakeLists.txt | 19 ++++++++++++----- CMakeModules/FindUdns.cmake | 42 +++++++++++++++++++++++++++++++++++++ simgear/CMakeLists.txt | 11 +++++++++- 4 files changed, 67 insertions(+), 7 deletions(-) create mode 100644 CMakeModules/FindUdns.cmake diff --git a/3rdparty/CMakeLists.txt b/3rdparty/CMakeLists.txt index e9b52823..a03a82d0 100644 --- a/3rdparty/CMakeLists.txt +++ b/3rdparty/CMakeLists.txt @@ -4,6 +4,6 @@ endif() add_subdirectory(utf8) -if (ENABLE_DNS) +if (ENABLE_DNS AND NOT SYSTEM_UDNS) add_subdirectory(udns) endif() diff --git a/CMakeLists.txt b/CMakeLists.txt index a67d65e9..7de1a0a9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,12 +114,14 @@ endif() if (NOT MSVC) option(SIMGEAR_SHARED "Set to ON to build SimGear as a shared library/framework" OFF) -option(SYSTEM_EXPAT "Set to ON to build SimGear using the system libExpat" OFF) +option(SYSTEM_EXPAT "Set to ON to build SimGear using the system expat library" OFF) +option(SYSTEM_UDNS "Set to ON to build SimGear using the system udns library" OFF) else() # Building SimGear DLLs is currently not supported for MSVC. set(SIMGEAR_SHARED OFF) -# Using a system expat is currently not supported for MSVC - it would require shared simgear (DLL). +# Using external 3rd party libraries is currently not supported for MSVC - it would require shared simgear (DLL). set(SYSTEM_EXPAT OFF) +set(SYSTEM_UDNS OFF) endif() option(SIMGEAR_HEADLESS "Set to ON to build SimGear without GUI/graphics support" OFF) @@ -426,9 +428,16 @@ endif() install (FILES ${PROJECT_BINARY_DIR}/simgear/simgear_config.h DESTINATION include/simgear/) include_directories(3rdparty/utf8/source) -if (ENABLE_DNS) - message(STATUS "DNS resolver: ENABLED") - include_directories(3rdparty/udns) + +if(ENABLE_DNS) + if(SYSTEM_UDNS) + message(STATUS "Requested to use system udns library, forcing SIMGEAR_SHARED to true") + set(SIMGEAR_SHARED ON) + find_package(Udns REQUIRED) + else() + message(STATUS "DNS resolver: ENABLED") + include_directories(3rdparty/udns) + endif() else() message(STATUS "DNS resolver: DISABLED") endif() diff --git a/CMakeModules/FindUdns.cmake b/CMakeModules/FindUdns.cmake new file mode 100644 index 00000000..a4363610 --- /dev/null +++ b/CMakeModules/FindUdns.cmake @@ -0,0 +1,42 @@ +# - Try to find UDNS library +# Once done this will define +# +# UDNS_FOUND - system has UDNS +# UDNS_INCLUDE_DIRS - the UDNS include directory +# UDNS_LIBRARIES - Link these to use UDNS +# UDNS_DEFINITIONS - Compiler switches required for using UDNS +# +# Copyright (c) 2016 Maciej Mrozowski +# +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# http://www.boost.org/LICENSE_1_0.txt) + + +if (UDN_LIBRARIES AND UDN_INCLUDE_DIRS) + # in cache already + set(UDNS_FOUND TRUE) +else () + set(UDNS_DEFINITIONS "") + + find_path(UDNS_INCLUDE_DIRS NAMES udns.h) + find_library(UDNS_LIBRARIES NAMES udns) + + if (UDNS_INCLUDE_DIRS AND UDNS_LIBRARIES) + set(UDNS_FOUND TRUE) + endif () + + if (UDNS_FOUND) + if (NOT Udns_FIND_QUIETLY) + message(STATUS "Found UDNS: ${UDNS_LIBRARIES}") + endif () + else () + if (Udns_FIND_REQUIRED) + message(FATAL_ERROR "Could not find UDNS") + endif () + endif () + + # show the UDNS_INCLUDE_DIRS and UDNS_LIBRARIES variables only in the advanced view + mark_as_advanced(UDNS_INCLUDE_DIRS UDNS_LIBRARIES) + +endif () diff --git a/simgear/CMakeLists.txt b/simgear/CMakeLists.txt index 59768334..516ea6cd 100644 --- a/simgear/CMakeLists.txt +++ b/simgear/CMakeLists.txt @@ -128,11 +128,20 @@ target_link_libraries(SimGearCore ${ZLIB_LIBRARY} ${RT_LIBRARY} ${DL_LIBRARY} - ${EXPAT_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${COCOA_LIBRARY} ${CURL_LIBRARIES}) +if(SYSTEM_EXPAT) + target_link_libraries(SimGearCore + ${EXPAT_LIBRARIES}) +endif() + +if(ENABLE_DNS AND SYSTEM_UDNS) + target_link_libraries(SimGearCore + ${UDNS_LIBRARIES}) +endif() + if(NOT SIMGEAR_HEADLESS) target_link_libraries(SimGearScene SimGearCore From 9d2df12ab8b97351b1c91c9411b9f2808a1d4db5 Mon Sep 17 00:00:00 2001 From: James Turner Date: Wed, 25 May 2016 20:27:24 +0100 Subject: [PATCH 08/14] Remove old terraysnc backend methods. Only in-process HTTP access is supported now, SVN and rsync and removed. This is to allow changes for better use of the HTTP API. --- simgear/scene/tsync/terrasync.cxx | 381 +++++------------------------- simgear/scene/tsync/terrasync.hxx | 4 +- 2 files changed, 64 insertions(+), 321 deletions(-) diff --git a/simgear/scene/tsync/terrasync.cxx b/simgear/scene/tsync/terrasync.cxx index 412c3add..7edb6c7d 100644 --- a/simgear/scene/tsync/terrasync.cxx +++ b/simgear/scene/tsync/terrasync.cxx @@ -59,23 +59,14 @@ #include #include #include -#include #include #include #include #include -static const bool svn_built_in_available = true; - using namespace simgear; using namespace std; -const char* rsync_cmd = - "rsync --verbose --archive --delete --perms --owner --group"; - -const char* svn_options = - "checkout -q"; - namespace UpdateInterval { // interval in seconds to allow an update to repeat after a successful update (=daily) @@ -89,6 +80,7 @@ typedef map TileAgeCache; /////////////////////////////////////////////////////////////////////////////// // helper functions /////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////// + string stripPath(string path) { // svn doesn't like trailing white-spaces or path separators - strip them! @@ -199,52 +191,15 @@ static unsigned int syncSlotForType(SyncItem::Type ty) } } -/////////////////////////////////////////////////////////////////////////////// -// Base server query -/////////////////////////////////////////////////////////////////////////////// -class ServerSelectQuery : public HTTP::Request +/////////////////////////////////////////////////////////////////////////////// +// SGTerraSync::WorkerThread ////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// +class SGTerraSync::WorkerThread : public SGThread { public: - ServerSelectQuery() : - HTTP::Request("http://scenery.flightgear.org/svn-server", "GET") - { - } - - std::string svnUrl() const - { - std::string s = simgear::strutils::strip(m_url); - if (s.at(s.length() - 1) == '/') { - s = s.substr(0, s.length() - 1); - } - - return s; - } -protected: - virtual void gotBodyData(const char* s, int n) - { - m_url.append(std::string(s, n)); - } - - virtual void onFail() - { - SG_LOG(SG_TERRASYNC, SG_ALERT, "Failed to query TerraSync SVN server"); - HTTP::Request::onFail(); - } - -private: - std::string m_url; -}; - - -/////////////////////////////////////////////////////////////////////////////// -// SGTerraSync::SvnThread ///////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////////////////////// -class SGTerraSync::SvnThread : public SGThread -{ -public: - SvnThread(); - virtual ~SvnThread( ) { stop(); } + WorkerThread(); + virtual ~WorkerThread( ) { stop(); } void stop(); bool start(); @@ -255,23 +210,17 @@ public: bool hasNewTiles() { return !_freshTiles.empty();} SyncItem getNewTile() { return _freshTiles.pop_front();} - void setSvnServer(string server) { _svn_server = stripPath(server);} - void setSvnDataServer(string server) { _svn_data_server = stripPath(server);} void setHTTPServer(const std::string& server) { _httpServer = stripPath(server); } - void setExtSvnUtility(string svn_util) { _svn_command = simgear::strutils::strip(svn_util);} - void setRsyncServer(string server) { _rsync_server = simgear::strutils::strip(server);} void setLocalDir(string dir) { _local_dir = stripPath(dir);} string getLocalDir() { return _local_dir;} - void setUseSvn(bool use_svn) { _use_svn = use_svn;} void setAllowedErrorCount(int errors) {_allowed_errors = errors;} void setCachePath(const SGPath& p) {_persistentCachePath = p;} void setCacheHits(unsigned int hits) {_cache_hits = hits;} - void setUseBuiltin(bool built_in) { _use_built_in = built_in;} volatile bool _active; volatile bool _running; @@ -290,11 +239,6 @@ public: private: virtual void run(); - // external model run and helpers - void runExternal(); - void syncPathExternal(const SyncItem& next); - bool runExternalSyncCommand(const char* dir); - // internal mode run and helpers void runInternal(); void updateSyncSlot(SyncSlot& slot); @@ -308,7 +252,6 @@ private: void fail(SyncItem failedItem); void notFound(SyncItem notFoundItem); - bool _use_built_in; HTTP::Client _http; SyncSlot _syncSlots[NUM_SYNC_SLOTS]; @@ -320,17 +263,12 @@ private: TileAgeCache _notFoundItems; SGBlockingDeque _freshTiles; - bool _use_svn; - string _svn_server; - string _svn_data_server; - string _svn_command; - string _rsync_server; string _local_dir; SGPath _persistentCachePath; string _httpServer; }; -SGTerraSync::SvnThread::SvnThread() : +SGTerraSync::WorkerThread::WorkerThread() : _active(false), _running(false), _busy(false), @@ -343,15 +281,13 @@ SGTerraSync::SvnThread::SvnThread() : _cache_hits(0), _transfer_rate(0), _total_kb_downloaded(0), - _use_built_in(true), _is_dirty(false), - _stop(false), - _use_svn(true) + _stop(false) { _http.setUserAgent("terrascenery-" SG_STRINGIZE(SIMGEAR_VERSION)); } -void SGTerraSync::SvnThread::stop() +void SGTerraSync::WorkerThread::stop() { // drop any pending requests waitingTiles.clear(); @@ -367,7 +303,7 @@ void SGTerraSync::SvnThread::stop() _running = false; } -bool SGTerraSync::SvnThread::start() +bool SGTerraSync::WorkerThread::start() { if (_running) return false; @@ -403,18 +339,6 @@ bool SGTerraSync::SvnThread::start() return false; } - _use_svn |= _use_built_in; - - - if ((!_use_svn)&&(_rsync_server=="")) - { - SG_LOG(SG_TERRASYNC,SG_ALERT, - "Cannot start scenery download. Rsync scenery server is undefined."); - _fail_count++; - _stalled = true; - return false; - } - _fail_count = 0; _updated_tile_count = 0; _success_count = 0; @@ -423,20 +347,7 @@ bool SGTerraSync::SvnThread::start() _stalled = false; _running = true; - string status; - - if (_use_svn && _use_built_in) - status = "Using built-in SVN support. "; - else if (_use_svn) - { - status = "Using external SVN utility '"; - status += _svn_command; - status += "'. "; - } - else - { - status = "Using RSYNC. "; - } + string status = "Using built-in HTTP support."; // not really an alert - but we want to (always) see this message, so user is // aware we're downloading scenery (and using bandwidth). @@ -449,61 +360,7 @@ bool SGTerraSync::SvnThread::start() return true; } -bool SGTerraSync::SvnThread::runExternalSyncCommand(const char* dir) -{ - ostringstream buf; - SGPath localPath( _local_dir ); - localPath.append( dir ); - - if (_use_svn) - { - buf << "\"" << _svn_command << "\" " - << svn_options << " " - << "\"" << _svn_server << "/" << dir << "\" " - << "\"" << localPath.str_native() << "\""; - } else { - buf << rsync_cmd << " " - << "\"" << _rsync_server << "/" << dir << "/\" " - << "\"" << localPath.str_native() << "/\""; - } - - string command; -#ifdef SG_WINDOWS - // windows command line parsing is just lovely... - // to allow white spaces, the system call needs this: - // ""C:\Program Files\something.exe" somearg "some other arg"" - // Note: whitespace strings quoted by a pair of "" _and_ the - // entire string needs to be wrapped by "" too. - // The svn url needs forward slashes (/) as a path separator while - // the local path needs windows-native backslash as a path separator. - command = "\"" + buf.str() + "\""; -#else - command = buf.str(); -#endif - SG_LOG(SG_TERRASYNC,SG_DEBUG, "sync command '" << command << "'"); - -#ifdef SG_WINDOWS - // tbd: does Windows support "popen"? - int rc = system( command.c_str() ); -#else - FILE* pipe = popen( command.c_str(), "r"); - int rc=-1; - // wait for external process to finish - if (pipe) - rc = pclose(pipe); -#endif - - if (rc) - { - SG_LOG(SG_TERRASYNC,SG_ALERT, - "Failed to synchronize directory '" << dir << "', " << - "error code= " << rc); - return false; - } - return true; -} - -void SGTerraSync::SvnThread::run() +void SGTerraSync::WorkerThread::run() { _active = true; initCompletedTilesPersistentCache(); @@ -566,106 +423,15 @@ void SGTerraSync::SvnThread::run() SG_LOG(SG_TERRASYNC,SG_INFO, "picking entry # " << idx << ", server is " << _httpServer ); } } - if( _httpServer.empty() ) { // don't resolve SVN server is HTTP server is set - if ( _svn_server.empty()) { - SG_LOG(SG_TERRASYNC,SG_INFO, "Querying closest TerraSync server"); - ServerSelectQuery* ssq = new ServerSelectQuery; - HTTP::Request_ptr req = ssq; - _http.makeRequest(req); - while (!req->isComplete()) { - _http.update(20); - } - if (req->readyState() == HTTP::Request::DONE) { - _svn_server = ssq->svnUrl(); - SG_LOG(SG_TERRASYNC,SG_INFO, "Closest TerraSync server:" << _svn_server); - } else { - SG_LOG(SG_TERRASYNC,SG_WARN, "Failed to query closest TerraSync server"); - } - } else { - SG_LOG(SG_TERRASYNC,SG_INFO, "Explicit: TerraSync server:" << _svn_server); - } - - if (_svn_server.empty()) { -#if WE_HAVE_A_DEFAULT_SERVER_BUT_WE_DONT_THIS_URL_IS_NO_LONGER_VALID - // default value - _svn_server = "http://foxtrot.mgras.net:8080/terrascenery/trunk/data/Scenery"; -#endif - } - } - - if (_use_built_in) { - runInternal(); - } else { - runExternal(); - } + runInternal(); _active = false; _running = false; _is_dirty = true; } -void SGTerraSync::SvnThread::runExternal() -{ - while (!_stop) - { - SyncItem next = waitingTiles.pop_front(); - if (_stop) - break; - - SyncItem::Status cacheStatus = isPathCached(next); - if (cacheStatus != SyncItem::Invalid) { - _cache_hits++; - SG_LOG(SG_TERRASYNC, SG_DEBUG, - "Cache hit for: '" << next._dir << "'"); - next._status = cacheStatus; - _freshTiles.push_back(next); - _is_dirty = true; - continue; - } - - syncPathExternal(next); - - if ((_allowed_errors >= 0)&& - (_consecutive_errors >= _allowed_errors)) - { - _stalled = true; - _stop = true; - } - } // of thread running loop -} - -void SGTerraSync::SvnThread::syncPathExternal(const SyncItem& next) -{ - _busy = true; - SGPath path( _local_dir ); - path.append( next._dir ); - bool isNewDirectory = !path.exists(); - - try { - if (isNewDirectory) { - int rc = path.create_dir( 0755 ); - if (rc) { - SG_LOG(SG_TERRASYNC,SG_ALERT, - "Cannot create directory '" << path << "', return code = " << rc ); - throw sg_exception("Cannot create directory for terrasync", path.str()); - } - } - - if (!runExternalSyncCommand(next._dir.c_str())) { - throw sg_exception("Running external sync command failed"); - } - } catch (sg_exception& e) { - fail(next); - _busy = false; - return; - } - - updated(next, isNewDirectory); - _busy = false; -} - -void SGTerraSync::SvnThread::updateSyncSlot(SyncSlot &slot) +void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot) { if (slot.repository.get()) { if (slot.repository->isDoingSync()) { @@ -680,7 +446,7 @@ void SGTerraSync::SvnThread::updateSyncSlot(SyncSlot &slot) } // check result - SVNRepository::ResultCode res = slot.repository->failure(); + AbstractRepository::ResultCode res = slot.repository->failure(); if (res == AbstractRepository::REPO_ERROR_NOT_FOUND) { notFound(slot.currentItem); } else if (res != AbstractRepository::REPO_NO_ERROR) { @@ -714,18 +480,9 @@ void SGTerraSync::SvnThread::updateSyncSlot(SyncSlot &slot) } } // of creating directory step - string serverUrl(_svn_server); - if (!_httpServer.empty()) { - slot.repository.reset(new HTTPRepository(path, &_http)); - serverUrl = _httpServer; - } else { - if (slot.currentItem._type == SyncItem::AIData) { - serverUrl = _svn_data_server; - } - slot.repository.reset(new SVNRepository(path, &_http)); - } + slot.repository.reset(new HTTPRepository(path, &_http)); + slot.repository->setBaseUrl(_httpServer + "/" + slot.currentItem._dir); - slot.repository->setBaseUrl(serverUrl + "/" + slot.currentItem._dir); try { slot.repository->update(); } catch (sg_exception& e) { @@ -744,7 +501,7 @@ void SGTerraSync::SvnThread::updateSyncSlot(SyncSlot &slot) } } -void SGTerraSync::SvnThread::runInternal() +void SGTerraSync::WorkerThread::runInternal() { while (!_stop) { try { @@ -794,7 +551,7 @@ void SGTerraSync::SvnThread::runInternal() } // of thread running loop } -SyncItem::Status SGTerraSync::SvnThread::isPathCached(const SyncItem& next) const +SyncItem::Status SGTerraSync::WorkerThread::isPathCached(const SyncItem& next) const { TileAgeCache::const_iterator ii = _completedTiles.find( next._dir ); if (ii == _completedTiles.end()) { @@ -816,7 +573,7 @@ SyncItem::Status SGTerraSync::SvnThread::isPathCached(const SyncItem& next) cons return (ii->second > now) ? SyncItem::Cached : SyncItem::Invalid; } -void SGTerraSync::SvnThread::fail(SyncItem failedItem) +void SGTerraSync::WorkerThread::fail(SyncItem failedItem) { time_t now = time(0); _consecutive_errors++; @@ -829,7 +586,7 @@ void SGTerraSync::SvnThread::fail(SyncItem failedItem) _is_dirty = true; } -void SGTerraSync::SvnThread::notFound(SyncItem item) +void SGTerraSync::WorkerThread::notFound(SyncItem item) { // treat not found as authorative, so use the same cache expiry // as succesful download. Important for MP models and similar so @@ -844,7 +601,7 @@ void SGTerraSync::SvnThread::notFound(SyncItem item) writeCompletedTilesPersistentCache(); } -void SGTerraSync::SvnThread::updated(SyncItem item, bool isNewDirectory) +void SGTerraSync::WorkerThread::updated(SyncItem item, bool isNewDirectory) { time_t now = time(0); _consecutive_errors = 0; @@ -863,7 +620,7 @@ void SGTerraSync::SvnThread::updated(SyncItem item, bool isNewDirectory) writeCompletedTilesPersistentCache(); } -void SGTerraSync::SvnThread::initCompletedTilesPersistentCache() +void SGTerraSync::WorkerThread::initCompletedTilesPersistentCache() { if (!_persistentCachePath.exists()) { return; @@ -896,7 +653,7 @@ void SGTerraSync::SvnThread::initCompletedTilesPersistentCache() } } -void SGTerraSync::SvnThread::writeCompletedTilesPersistentCache() const +void SGTerraSync::WorkerThread::writeCompletedTilesPersistentCache() const { // cache is disabled if (_persistentCachePath.isNull()) { @@ -931,11 +688,11 @@ void SGTerraSync::SvnThread::writeCompletedTilesPersistentCache() const // SGTerraSync //////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////// SGTerraSync::SGTerraSync() : - _svnThread(NULL), + _workerThread(NULL), _bound(false), _inited(false) { - _svnThread = new SvnThread(); + _workerThread = new WorkerThread(); _log = new BufferedLogCallback(SG_TERRASYNC, SG_INFO); _log->truncateAt(255); @@ -944,8 +701,8 @@ SGTerraSync::SGTerraSync() : SGTerraSync::~SGTerraSync() { - delete _svnThread; - _svnThread = NULL; + delete _workerThread; + _workerThread = NULL; sglog().removeCallback(_log); delete _log; _tiedProperties.Untie(); @@ -965,56 +722,42 @@ void SGTerraSync::init() _inited = true; assert(_terraRoot); - _terraRoot->setBoolValue("built-in-svn-available",svn_built_in_available); + _terraRoot->setBoolValue("built-in-svn-available", true); reinit(); } void SGTerraSync::shutdown() { - _svnThread->stop(); + _workerThread->stop(); } void SGTerraSync::reinit() { // do not reinit when enabled and we're already up and running if ((_terraRoot->getBoolValue("enabled",false))&& - (_svnThread->_active && _svnThread->_running)) + (_workerThread->_active && _workerThread->_running)) { return; } - _svnThread->stop(); + _workerThread->stop(); if (_terraRoot->getBoolValue("enabled",false)) { - _svnThread->setSvnServer(_terraRoot->getStringValue("svn-server","")); std::string httpServer(_terraRoot->getStringValue("http-server","")); - _svnThread->setHTTPServer(httpServer); - _svnThread->setSvnDataServer(_terraRoot->getStringValue("svn-data-server","")); - _svnThread->setRsyncServer(_terraRoot->getStringValue("rsync-server","")); - _svnThread->setLocalDir(_terraRoot->getStringValue("scenery-dir","")); - _svnThread->setAllowedErrorCount(_terraRoot->getIntValue("max-errors",5)); - _svnThread->setUseBuiltin(_terraRoot->getBoolValue("use-built-in-svn",true)); + _workerThread->setHTTPServer(httpServer); + _workerThread->setLocalDir(_terraRoot->getStringValue("scenery-dir","")); + _workerThread->setAllowedErrorCount(_terraRoot->getIntValue("max-errors",5)); + _workerThread->setCacheHits(_terraRoot->getIntValue("cache-hit", 0)); - if (httpServer.empty()) { - // HTTP doesn't benefit from using the persistent cache - _svnThread->setCachePath(SGPath(_terraRoot->getStringValue("cache-path",""))); - } else { - SG_LOG(SG_TERRASYNC, SG_INFO, "HTTP repository selected, disabling persistent cache"); - } - - _svnThread->setCacheHits(_terraRoot->getIntValue("cache-hit", 0)); - _svnThread->setUseSvn(_terraRoot->getBoolValue("use-svn",true)); - _svnThread->setExtSvnUtility(_terraRoot->getStringValue("ext-svn-utility","svn")); - - if (_svnThread->start()) + if (_workerThread->start()) { syncAirportsModels(); } } - _stalledNode->setBoolValue(_svnThread->_stalled); + _stalledNode->setBoolValue(_workerThread->_stalled); } void SGTerraSync::bind() @@ -1024,17 +767,17 @@ void SGTerraSync::bind() } _bound = true; - _tiedProperties.Tie( _terraRoot->getNode("busy", true), (bool*) &_svnThread->_busy ); - _tiedProperties.Tie( _terraRoot->getNode("active", true), (bool*) &_svnThread->_active ); - _tiedProperties.Tie( _terraRoot->getNode("update-count", true), (int*) &_svnThread->_success_count ); - _tiedProperties.Tie( _terraRoot->getNode("error-count", true), (int*) &_svnThread->_fail_count ); - _tiedProperties.Tie( _terraRoot->getNode("tile-count", true), (int*) &_svnThread->_updated_tile_count ); - _tiedProperties.Tie( _terraRoot->getNode("cache-hits", true), (int*) &_svnThread->_cache_hits ); - _tiedProperties.Tie( _terraRoot->getNode("transfer-rate-bytes-sec", true), (int*) &_svnThread->_transfer_rate ); + _tiedProperties.Tie( _terraRoot->getNode("busy", true), (bool*) &_workerThread->_busy ); + _tiedProperties.Tie( _terraRoot->getNode("active", true), (bool*) &_workerThread->_active ); + _tiedProperties.Tie( _terraRoot->getNode("update-count", true), (int*) &_workerThread->_success_count ); + _tiedProperties.Tie( _terraRoot->getNode("error-count", true), (int*) &_workerThread->_fail_count ); + _tiedProperties.Tie( _terraRoot->getNode("tile-count", true), (int*) &_workerThread->_updated_tile_count ); + _tiedProperties.Tie( _terraRoot->getNode("cache-hits", true), (int*) &_workerThread->_cache_hits ); + _tiedProperties.Tie( _terraRoot->getNode("transfer-rate-bytes-sec", true), (int*) &_workerThread->_transfer_rate ); // use kbytes here because propety doesn't support 64-bit and we might conceivably // download more than 2G in a single session - _tiedProperties.Tie( _terraRoot->getNode("downloaded-kbytes", true), (int*) &_svnThread->_total_kb_downloaded ); + _tiedProperties.Tie( _terraRoot->getNode("downloaded-kbytes", true), (int*) &_workerThread->_total_kb_downloaded ); _terraRoot->getNode("busy", true)->setAttribute(SGPropertyNode::WRITE,false); _terraRoot->getNode("active", true)->setAttribute(SGPropertyNode::WRITE,false); @@ -1045,13 +788,13 @@ void SGTerraSync::bind() _terraRoot->getNode("use-svn", true)->setAttribute(SGPropertyNode::USERARCHIVE,false); // stalled is used as a signal handler (to connect listeners triggering GUI pop-ups) _stalledNode = _terraRoot->getNode("stalled", true); - _stalledNode->setBoolValue(_svnThread->_stalled); + _stalledNode->setBoolValue(_workerThread->_stalled); _stalledNode->setAttribute(SGPropertyNode::PRESERVE,true); } void SGTerraSync::unbind() { - _svnThread->stop(); + _workerThread->stop(); _tiedProperties.Untie(); _bound = false; _inited = false; @@ -1064,11 +807,11 @@ void SGTerraSync::unbind() void SGTerraSync::update(double) { static SGBucket bucket; - if (_svnThread->isDirty()) + if (_workerThread->isDirty()) { - if (!_svnThread->_active) + if (!_workerThread->_active) { - if (_svnThread->_stalled) + if (_workerThread->_stalled) { SG_LOG(SG_TERRASYNC,SG_ALERT, "Automatic scenery download/synchronization stalled. Too many errors."); @@ -1079,12 +822,12 @@ void SGTerraSync::update(double) SG_LOG(SG_TERRASYNC,SG_ALERT, "Automatic scenery download/synchronization has stopped."); } - _stalledNode->setBoolValue(_svnThread->_stalled); + _stalledNode->setBoolValue(_workerThread->_stalled); } - while (_svnThread->hasNewTiles()) + while (_workerThread->hasNewTiles()) { - SyncItem next = _svnThread->getNewTile(); + SyncItem next = _workerThread->getNewTile(); if ((next._type == SyncItem::Tile) || (next._type == SyncItem::AIData)) { _activeTileDirs.erase(next._dir); @@ -1093,7 +836,7 @@ void SGTerraSync::update(double) } } -bool SGTerraSync::isIdle() {return _svnThread->isIdle();} +bool SGTerraSync::isIdle() {return _workerThread->isIdle();} void SGTerraSync::syncAirportsModels() { @@ -1106,12 +849,12 @@ void SGTerraSync::syncAirportsModels() ostringstream dir; dir << "Airports/" << synced_other; SyncItem w(dir.str(), SyncItem::AirportData); - _svnThread->request( w ); + _workerThread->request( w ); } } SyncItem w("Models", SyncItem::SharedModels); - _svnThread->request( w ); + _workerThread->request( w ); } void SGTerraSync::syncAreaByPath(const std::string& aPath) @@ -1126,7 +869,7 @@ void SGTerraSync::syncAreaByPath(const std::string& aPath) _activeTileDirs.insert(dir); SyncItem w(dir, SyncItem::Tile); - _svnThread->request( w ); + _workerThread->request( w ); } } @@ -1139,7 +882,7 @@ bool SGTerraSync::scheduleTile(const SGBucket& bucket) bool SGTerraSync::isTileDirPending(const std::string& sceneryDir) const { - if (!_svnThread->_running) { + if (!_workerThread->_running) { return false; } @@ -1162,13 +905,13 @@ void SGTerraSync::scheduleDataDir(const std::string& dataDir) _activeTileDirs.insert(dataDir); SyncItem w(dataDir, SyncItem::AIData); - _svnThread->request( w ); + _workerThread->request( w ); } bool SGTerraSync::isDataDirPending(const std::string& dataDir) const { - if (!_svnThread->_running) { + if (!_workerThread->_running) { return false; } diff --git a/simgear/scene/tsync/terrasync.hxx b/simgear/scene/tsync/terrasync.hxx index 8150872d..2ee6772b 100644 --- a/simgear/scene/tsync/terrasync.hxx +++ b/simgear/scene/tsync/terrasync.hxx @@ -84,10 +84,10 @@ protected: void syncAirportsModels(); - class SvnThread; + class WorkerThread; private: - SvnThread* _svnThread; + WorkerThread* _workerThread; SGPropertyNode_ptr _terraRoot; SGPropertyNode_ptr _stalledNode; SGPropertyNode_ptr _cacheHits; From 3e2f37418af2ef8f1a160a3d4e86d7481ec2e78c Mon Sep 17 00:00:00 2001 From: James Turner Date: Wed, 25 May 2016 20:36:04 +0100 Subject: [PATCH 09/14] Remove SVN sync code. --- simgear/io/CMakeLists.txt | 11 - simgear/io/DAVMultiStatus.cxx | 402 ---------------------- simgear/io/DAVMultiStatus.hxx | 143 -------- simgear/io/SVNDirectory.cxx | 400 ---------------------- simgear/io/SVNDirectory.hxx | 109 ------ simgear/io/SVNReportParser.cxx | 605 --------------------------------- simgear/io/SVNReportParser.hxx | 57 ---- simgear/io/SVNRepository.cxx | 386 --------------------- simgear/io/SVNRepository.hxx | 59 ---- simgear/io/http_svn.cxx | 51 --- 10 files changed, 2223 deletions(-) delete mode 100644 simgear/io/DAVMultiStatus.cxx delete mode 100644 simgear/io/DAVMultiStatus.hxx delete mode 100644 simgear/io/SVNDirectory.cxx delete mode 100644 simgear/io/SVNDirectory.hxx delete mode 100644 simgear/io/SVNReportParser.cxx delete mode 100644 simgear/io/SVNReportParser.hxx delete mode 100644 simgear/io/SVNRepository.cxx delete mode 100644 simgear/io/SVNRepository.hxx delete mode 100644 simgear/io/http_svn.cxx diff --git a/simgear/io/CMakeLists.txt b/simgear/io/CMakeLists.txt index 00d072d5..71213551 100644 --- a/simgear/io/CMakeLists.txt +++ b/simgear/io/CMakeLists.txt @@ -19,10 +19,6 @@ set(HEADERS HTTPMemoryRequest.hxx HTTPRequest.hxx AbstractRepository.hxx - DAVMultiStatus.hxx - SVNRepository.hxx - SVNDirectory.hxx - SVNReportParser.hxx HTTPRepository.hxx ) @@ -43,10 +39,6 @@ set(SOURCES HTTPMemoryRequest.cxx HTTPRequest.cxx AbstractRepository.cxx - DAVMultiStatus.cxx - SVNRepository.cxx - SVNDirectory.cxx - SVNReportParser.cxx HTTPRepository.cxx ) @@ -59,9 +51,6 @@ simgear_component(io io "${SOURCES}" "${HEADERS}") if(ENABLE_TESTS) -add_executable(http_svn http_svn.cxx) -target_link_libraries(http_svn ${TEST_LIBS}) - add_executable(test_sock socktest.cxx) target_link_libraries(test_sock ${TEST_LIBS}) diff --git a/simgear/io/DAVMultiStatus.cxx b/simgear/io/DAVMultiStatus.cxx deleted file mode 100644 index 6a82a0c5..00000000 --- a/simgear/io/DAVMultiStatus.cxx +++ /dev/null @@ -1,402 +0,0 @@ -// DAVMultiStatus.cxx -- parser for WebDAV MultiStatus XML data -// -// Copyright (C) 2012 James Turner -// -// This program is free software; you can redistribute it and/or -// modify it under the terms of the GNU General Public License as -// published by the Free Software Foundation; either version 2 of the -// License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -#ifdef HAVE_CONFIG_H -# include -#endif - -#include "DAVMultiStatus.hxx" - -#include -#include -#include -#include -#include - -#include - -#include "simgear/debug/logstream.hxx" -#include "simgear/misc/strutils.hxx" -#include "simgear/structure/exception.hxx" - -#ifdef SYSTEM_EXPAT -# include -#else -# include "sg_expat.h" -#endif - -using std::string; - -using namespace simgear; - -#define DAV_NS "DAV::" -#define SUBVERSION_DAV_NS "http://subversion.tigris.org/xmlns/dav/" - -const char* DAV_MULTISTATUS_TAG = DAV_NS "multistatus"; -const char* DAV_RESPONSE_TAG = DAV_NS "response"; -const char* DAV_PROPSTAT_TAG = DAV_NS "propstat"; -const char* DAV_PROP_TAG = DAV_NS "prop"; - -const char* DAV_HREF_TAG = DAV_NS "href"; -const char* DAV_RESOURCE_TYPE_TAG = DAV_NS "resourcetype"; -const char* DAV_CONTENT_TYPE_TAG = DAV_NS "getcontenttype"; -const char* DAV_CONTENT_LENGTH_TAG = DAV_NS "getcontentlength"; -const char* DAV_VERSIONNAME_TAG = DAV_NS "version-name"; -const char* DAV_COLLECTION_TAG = DAV_NS "collection"; -const char* DAV_VCC_TAG = DAV_NS "version-controlled-configuration"; - -const char* SUBVERSION_MD5_CHECKSUM_TAG = SUBVERSION_DAV_NS ":md5-checksum"; - -DAVResource::DAVResource(const string& href) : - _type(Unknown), - _url(href), - _container(NULL) -{ - assert(!href.empty()); - if (strutils::ends_with(href, "/")) { - _url = href.substr(0, _url.size() - 1); - } -} - -void DAVResource::setVersionName(const std::string& aVersion) -{ - _versionName = aVersion; -} - -void DAVResource::setVersionControlledConfiguration(const std::string& vcc) -{ - _vcc = vcc; -} - -void DAVResource::setMD5(const std::string& md5Hex) -{ - _md5 = md5Hex; -} - -std::string DAVResource::name() const -{ - string::size_type index = _url.rfind('/'); - if (index != string::npos) { - return _url.substr(index + 1); - } - - throw sg_exception("bad DAV resource HREF:" + _url); -} - -//////////////////////////////////////////////////////////////////////////// - -DAVCollection::DAVCollection(const string& href) : - DAVResource(href) -{ - _type = DAVResource::Collection; -} - -DAVCollection::~DAVCollection() -{ - BOOST_FOREACH(DAVResource* c, _contents) { - delete c; - } -} - -void DAVCollection::addChild(DAVResource *res) -{ - assert(res); - if (res->container() == this) { - return; - } - - assert(res->container() == NULL); - assert(std::find(_contents.begin(), _contents.end(), res) == _contents.end()); - assert(strutils::starts_with(res->url(), _url)); - assert(childWithUrl(res->url()) == NULL); - - res->_container = this; - _contents.push_back(res); -} - -void DAVCollection::removeChild(DAVResource* res) -{ - assert(res); - assert(res->container() == this); - - res->_container = NULL; - DAVResourceList::iterator it = std::find(_contents.begin(), _contents.end(), res); - assert(it != _contents.end()); - _contents.erase(it); -} - -DAVCollection* -DAVCollection::createChildCollection(const std::string& name) -{ - DAVCollection* child = new DAVCollection(urlForChildWithName(name)); - addChild(child); - return child; -} - -DAVResourceList DAVCollection::contents() const -{ - return _contents; -} - -DAVResource* DAVCollection::childWithUrl(const string& url) const -{ - if (url.empty()) - return NULL; - - BOOST_FOREACH(DAVResource* c, _contents) { - if (c->url() == url) { - return c; - } - } - - return NULL; -} - -DAVResource* DAVCollection::childWithName(const string& name) const -{ - return childWithUrl(urlForChildWithName(name)); -} - -std::string DAVCollection::urlForChildWithName(const std::string& name) const -{ - return url() + "/" + name; -} - -/////////////////////////////////////////////////////////////////////////////// - -class DAVMultiStatus::DAVMultiStatusPrivate -{ -public: - DAVMultiStatusPrivate() : - parserInited(false), - valid(false) - { - rootResource = NULL; - } - - void startElement (const char * name) - { - if (tagStack.empty()) { - if (strcmp(name, DAV_MULTISTATUS_TAG)) { - SG_LOG(SG_TERRASYNC, SG_WARN, "root element is not " << - DAV_MULTISTATUS_TAG << ", got:" << name); - } else { - - } - } else { - // not at the root element - if (tagStack.back() == DAV_MULTISTATUS_TAG) { - if (strcmp(name, DAV_RESPONSE_TAG)) { - SG_LOG(SG_TERRASYNC, SG_WARN, "multistatus child is not response: saw:" - << name); - } - } - - if (tagStack.back() == DAV_RESOURCE_TYPE_TAG) { - if (!strcmp(name, DAV_COLLECTION_TAG)) { - currentElementType = DAVResource::Collection; - } else { - currentElementType = DAVResource::Unknown; - } - } - } - - tagStack.push_back(name); - if (!strcmp(name, DAV_RESPONSE_TAG)) { - currentElementType = DAVResource::Unknown; - currentElementUrl.clear(); - currentElementMD5.clear(); - currentVersionName.clear(); - currentVCC.clear(); - } - } - - void endElement (const char * name) - { - assert(tagStack.back() == name); - tagStack.pop_back(); - - if (!strcmp(name, DAV_RESPONSE_TAG)) { - // finish complete response - currentElementUrl = strutils::strip(currentElementUrl); - - DAVResource* res = NULL; - if (currentElementType == DAVResource::Collection) { - DAVCollection* col = new DAVCollection(currentElementUrl); - res = col; - } else { - res = new DAVResource(currentElementUrl); - } - - res->setVersionName(strutils::strip(currentVersionName)); - res->setVersionControlledConfiguration(currentVCC); - if (rootResource && - strutils::starts_with(currentElementUrl, rootResource->url())) - { - static_cast(rootResource)->addChild(res); - } - - if (!rootResource) { - rootResource = res; - } - } - } - - void data (const char * s, int length) - { - if (tagStack.back() == DAV_HREF_TAG) { - if (tagN(1) == DAV_RESPONSE_TAG) { - currentElementUrl += string(s, length); - } else if (tagN(1) == DAV_VCC_TAG) { - currentVCC += string(s, length); - } - } else if (tagStack.back() == SUBVERSION_MD5_CHECKSUM_TAG) { - currentElementMD5 = string(s, length); - } else if (tagStack.back() == DAV_VERSIONNAME_TAG) { - currentVersionName = string(s, length); - } else if (tagStack.back() == DAV_CONTENT_LENGTH_TAG) { - std::istringstream is(string(s, length)); - is >> currentElementLength; - } - } - - void pi (const char * target, const char * data) {} - - string tagN(const unsigned int n) const - { - size_t sz = tagStack.size(); - if (n >= sz) { - return string(); - } - - return tagStack[sz - (1 + n)]; - } - - bool parserInited; - bool valid; - XML_Parser xmlParser; - DAVResource* rootResource; - - // in-flight data - string_list tagStack; - DAVResource::Type currentElementType; - string currentElementUrl, - currentVersionName, - currentVCC; - int currentElementLength; - string currentElementMD5; -}; - - -//////////////////////////////////////////////////////////////////////// -// Static callback functions for Expat. -//////////////////////////////////////////////////////////////////////// - -#define VISITOR static_cast(userData) - -static void -start_element (void * userData, const char * name, const char ** atts) -{ - VISITOR->startElement(name); -} - -static void -end_element (void * userData, const char * name) -{ - VISITOR->endElement(name); -} - -static void -character_data (void * userData, const char * s, int len) -{ - VISITOR->data(s, len); -} - -static void -processing_instruction (void * userData, - const char * target, - const char * data) -{ - VISITOR->pi(target, data); -} - -#undef VISITOR - -/////////////////////////////////////////////////////////////////////////////// - -DAVMultiStatus::DAVMultiStatus() : -_d(new DAVMultiStatusPrivate) -{ - -} - -DAVMultiStatus::~DAVMultiStatus() -{ - -} - -void DAVMultiStatus::parseXML(const char* data, int size) -{ - if (!_d->parserInited) { - _d->xmlParser = XML_ParserCreateNS(0, ':'); - XML_SetUserData(_d->xmlParser, _d.get()); - XML_SetElementHandler(_d->xmlParser, start_element, end_element); - XML_SetCharacterDataHandler(_d->xmlParser, character_data); - XML_SetProcessingInstructionHandler(_d->xmlParser, processing_instruction); - _d->parserInited = true; - } - - if (!XML_Parse(_d->xmlParser, data, size, false)) { - SG_LOG(SG_TERRASYNC, SG_WARN, "DAV parse error:" << XML_ErrorString(XML_GetErrorCode(_d->xmlParser)) - << " at line:" << XML_GetCurrentLineNumber(_d->xmlParser) - << " column " << XML_GetCurrentColumnNumber(_d->xmlParser)); - - XML_ParserFree(_d->xmlParser); - _d->parserInited = false; - _d->valid = false; - } -} - -void DAVMultiStatus::finishParse() -{ - if (_d->parserInited) { - if (!XML_Parse(_d->xmlParser, NULL, 0, true)) { - SG_LOG(SG_TERRASYNC, SG_WARN, "DAV parse error:" << XML_ErrorString(XML_GetErrorCode(_d->xmlParser)) - << " at line:" << XML_GetCurrentLineNumber(_d->xmlParser) - << " column " << XML_GetCurrentColumnNumber(_d->xmlParser)); - _d->valid = false; - } else { - _d->valid = true; - } - XML_ParserFree(_d->xmlParser); - } - - _d->parserInited = false; -} - -DAVResource* DAVMultiStatus::resource() -{ - return _d->rootResource; -} - -bool DAVMultiStatus::isValid() const -{ - return _d->valid; -} - - diff --git a/simgear/io/DAVMultiStatus.hxx b/simgear/io/DAVMultiStatus.hxx deleted file mode 100644 index d415e5ae..00000000 --- a/simgear/io/DAVMultiStatus.hxx +++ /dev/null @@ -1,143 +0,0 @@ -// DAVMultiStatus.hxx -- parser for WebDAV MultiStatus XML data -// -// Copyright (C) 2012 James Turner -// -// This program is free software; you can redistribute it and/or -// modify it under the terms of the GNU General Public License as -// published by the Free Software Foundation; either version 2 of the -// License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - - -#ifndef SG_IO_DAVMULTISTATUS_HXX -#define SG_IO_DAVMULTISTATUS_HXX - -#include -#include -#include // for auto_ptr - -namespace simgear -{ - -class DAVCollection; - -class DAVResource -{ -public: - DAVResource(const std::string& url); - virtual ~DAVResource() { } - - typedef enum { - Unknown = 0, - Collection = 1 - } Type; - - const Type type() const - { return _type; } - - const std::string& url() const - { return _url; } - - std::string name() const; - - /** - * SVN servers use this field to expose the head revision - * of the resource, which is useful - */ - const std::string& versionName() const - { return _versionName; } - - void setVersionName(const std::string& aVersion); - - DAVCollection* container() const - { return _container; } - - virtual bool isCollection() const - { return false; } - - void setVersionControlledConfiguration(const std::string& vcc); - const std::string& versionControlledConfiguration() const - { return _vcc; } - - void setMD5(const std::string& md5Hex); - const std::string& md5() const - { return _md5; } -protected: - friend class DAVCollection; - - Type _type; - std::string _url; - std::string _versionName; - std::string _vcc; - std::string _md5; - DAVCollection* _container; -}; - -typedef std::vector DAVResourceList; - -class DAVCollection : public DAVResource -{ -public: - DAVCollection(const std::string& url); - virtual ~DAVCollection(); - - DAVResourceList contents() const; - - void addChild(DAVResource* res); - void removeChild(DAVResource* res); - - DAVCollection* createChildCollection(const std::string& name); - - /** - * find the collection member with the specified URL, or return NULL - * if no such member of this collection exists. - */ - DAVResource* childWithUrl(const std::string& url) const; - - /** - * find the collection member with the specified name, or return NULL - */ - DAVResource* childWithName(const std::string& name) const; - - /** - * wrapper around URL manipulation - */ - std::string urlForChildWithName(const std::string& name) const; - - virtual bool isCollection() const - { return true; } -private: - DAVResourceList _contents; -}; - -class DAVMultiStatus -{ -public: - DAVMultiStatus(); - ~DAVMultiStatus(); - - // incremental XML parsing - void parseXML(const char* data, int size); - - void finishParse(); - - bool isValid() const; - - DAVResource* resource(); - - class DAVMultiStatusPrivate; -private: - std::auto_ptr _d; -}; - -} // of namespace simgear - -#endif // of SG_IO_DAVMULTISTATUS_HXX diff --git a/simgear/io/SVNDirectory.cxx b/simgear/io/SVNDirectory.cxx deleted file mode 100644 index 267d3120..00000000 --- a/simgear/io/SVNDirectory.cxx +++ /dev/null @@ -1,400 +0,0 @@ - -#include "SVNDirectory.hxx" - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using std::string; -using std::cout; -using std::endl; -using namespace simgear; - -typedef std::vector RequestVector; -typedef std::map DAVResourceMap; - - -const char* DAV_CACHE_NAME = ".terrasync_cache"; -const char* CACHE_VERSION_4_TOKEN = "terrasync-cache-4"; - -// important: with the Google servers, setting this higher than '1' causes -// server internal errors (500, the connection is closed). In other words we -// can only specify update report items one level deep at most and no more. -// (the root and its direct children, not NOT grand-children) -const unsigned int MAX_UPDATE_REPORT_DEPTH = 1; - -enum LineState -{ - LINESTATE_HREF = 0, - LINESTATE_VERSIONNAME -}; - -SVNDirectory::SVNDirectory(SVNRepository *r, const SGPath& path) : - localPath(path), - dav(NULL), - repo(r), - _doingUpdateReport(false), - _parent(NULL) -{ - if (path.exists()) { - parseCache(); - } - - // don't create dir here, repo might not exist at all -} - -SVNDirectory::SVNDirectory(SVNDirectory* pr, DAVCollection* col) : - dav(col), - repo(pr->repository()), - _doingUpdateReport(false), - _parent(pr) -{ - assert(col->container()); - assert(!col->url().empty()); - assert(_parent); - - localPath = pr->fsDir().file(col->name()); - if (!localPath.exists()) { - Dir d(localPath); - d.create(0755); - writeCache(); - } else { - parseCache(); - } -} - -SVNDirectory::~SVNDirectory() -{ - // recursive delete our child directories - BOOST_FOREACH(SVNDirectory* d, _children) { - delete d; - } -} - -void SVNDirectory::parseCache() -{ - SGPath p(localPath); - p.append(DAV_CACHE_NAME); - if (!p.exists()) { - return; - } - - char href[1024]; - char versionName[128]; - LineState lineState = LINESTATE_HREF; - std::ifstream file(p.c_str()); - if (!file.is_open()) { - SG_LOG(SG_TERRASYNC, SG_WARN, "unable to open cache file for reading:" << p); - return; - } - bool doneSelf = false; - - file.getline(href, 1024); - if (strcmp(CACHE_VERSION_4_TOKEN, href)) { - SG_LOG(SG_TERRASYNC, SG_WARN, "invalid cache file [missing header token]:" << p << " '" << href << "'"); - return; - } - - std::string vccUrl; - file.getline(href, 1024); - vccUrl = href; - - while (!file.eof()) { - if (lineState == LINESTATE_HREF) { - file.getline(href, 1024); - lineState = LINESTATE_VERSIONNAME; - } else { - assert(lineState == LINESTATE_VERSIONNAME); - file.getline(versionName, 1024); - lineState = LINESTATE_HREF; - char* hrefPtr = href; - - if (!doneSelf) { - if (!dav) { - dav = new DAVCollection(hrefPtr); - dav->setVersionName(versionName); - } else { - assert(string(hrefPtr) == dav->url()); - } - - if (!vccUrl.empty()) { - dav->setVersionControlledConfiguration(vccUrl); - } - - _cachedRevision = versionName; - doneSelf = true; - } else { - DAVResource* child = parseChildDirectory(hrefPtr)->collection(); - string s = strutils::strip(versionName); - if (!s.empty()) { - child->setVersionName(versionName); - } - } // of done self test - } // of line-state switching - } // of file get-line loop -} - -void SVNDirectory::writeCache() -{ - SGPath p(localPath); - if (!p.exists()) { - Dir d(localPath); - d.create(0755); - } - - p.append(string(DAV_CACHE_NAME) + ".new"); - - std::ofstream file(p.c_str(), std::ios::trunc); -// first, cache file version header - file << CACHE_VERSION_4_TOKEN << '\n'; - -// second, the repository VCC url - file << dav->versionControlledConfiguration() << '\n'; - -// third, our own URL, and version - file << dav->url() << '\n' << _cachedRevision << '\n'; - - BOOST_FOREACH(DAVResource* child, dav->contents()) { - if (child->isCollection()) { - file << child->name() << '\n' << child->versionName() << "\n"; - } - } // of child iteration - - file.close(); - -// approximately atomic delete + rename operation - SGPath cacheName(localPath); - cacheName.append(DAV_CACHE_NAME); - p.rename(cacheName); -} - -void SVNDirectory::setBaseUrl(const string& url) -{ - if (_parent) { - SG_LOG(SG_TERRASYNC, SG_ALERT, "setting base URL on non-root directory " << url); - return; - } - - if (dav && (url == dav->url())) { - return; - } - - dav = new DAVCollection(url); -} - -std::string SVNDirectory::url() const -{ - if (!_parent) { - return repo->baseUrl(); - } - - return _parent->url() + "/" + name(); -} - -std::string SVNDirectory::name() const -{ - return dav->name(); -} - -DAVResource* -SVNDirectory::addChildFile(const std::string& fileName) -{ - DAVResource* child = NULL; - child = new DAVResource(dav->urlForChildWithName(fileName)); - dav->addChild(child); - - writeCache(); - return child; -} - -SVNDirectory* -SVNDirectory::addChildDirectory(const std::string& dirName) -{ - if (dav->childWithName(dirName)) { - // existing child, let's remove it - deleteChildByName(dirName); - } - - DAVCollection* childCol = dav->createChildCollection(dirName); - SVNDirectory* child = new SVNDirectory(this, childCol); - childCol->setVersionName(child->cachedRevision()); - _children.push_back(child); - writeCache(); - return child; -} - -SVNDirectory* -SVNDirectory::parseChildDirectory(const std::string& dirName) -{ - assert(!dav->childWithName(dirName)); - DAVCollection* childCol = dav->createChildCollection(dirName); - SVNDirectory* child = new SVNDirectory(this, childCol); - childCol->setVersionName(child->cachedRevision()); - _children.push_back(child); - return child; -} - -void SVNDirectory::deleteChildByName(const std::string& nm) -{ - DAVResource* child = dav->childWithName(nm); - if (!child) { - return; - } - - SGPath path = fsDir().file(nm); - - if (child->isCollection()) { - Dir d(path); - bool ok = d.remove(true); - if (!ok) { - SG_LOG(SG_TERRASYNC, SG_ALERT, "SVNDirectory::deleteChildByName: failed to remove dir:" - << nm << " at path:\n\t" << path); - } - - DirectoryList::iterator it = findChildDir(nm); - if (it != _children.end()) { - SVNDirectory* c = *it; - delete c; - _children.erase(it); - } - } else { - bool ok = path.remove(); - if (!ok) { - SG_LOG(SG_TERRASYNC, SG_ALERT, "SVNDirectory::deleteChildByName: failed to remove path:" << nm - << " at path:\n\t" << path); - } - } - - dav->removeChild(child); - delete child; - - writeCache(); -} - -bool SVNDirectory::isDoingSync() const -{ - if (_doingUpdateReport) { - return true; - } - - BOOST_FOREACH(SVNDirectory* child, _children) { - if (child->isDoingSync()) { - return true; - } // of children - } - - return false; -} - -void SVNDirectory::beginUpdateReport() -{ - _doingUpdateReport = true; - _cachedRevision.clear(); - writeCache(); -} - -void SVNDirectory::updateReportComplete() -{ - _cachedRevision = dav->versionName(); - _doingUpdateReport = false; - writeCache(); - - SVNDirectory* pr = parent(); - if (pr) { - pr->writeCache(); - } -} - -SVNRepository* SVNDirectory::repository() const -{ - return repo; -} - -void SVNDirectory::mergeUpdateReportDetails(unsigned int depth, - string_list& items) -{ - // normal, easy case: we are fully in-sync at a revision - if (!_cachedRevision.empty()) { - std::ostringstream os; - os << "" - << repoPath() << ""; - items.push_back(os.str()); - return; - } - - Dir d(localPath); - if (depth >= MAX_UPDATE_REPORT_DEPTH) { - d.removeChildren(); - return; - } - - PathList cs = d.children(Dir::NO_DOT_OR_DOTDOT | Dir::INCLUDE_HIDDEN | Dir::TYPE_DIR); - BOOST_FOREACH(SGPath path, cs) { - SVNDirectory* c = child(path.file()); - if (!c) { - // ignore this child, if it's an incomplete download, - // it will be over-written on the update anyway - //std::cerr << "unknown SVN child" << path << std::endl; - } else { - // recurse down into children - c->mergeUpdateReportDetails(depth+1, items); - } - } // of child dir iteration -} - -std::string SVNDirectory::repoPath() const -{ - if (!_parent) { - return "/"; - } - - // find the length of the repository base URL, then - // trim that off our repo URL - job done! - size_t baseUrlLen = repo->baseUrl().size(); - return dav->url().substr(baseUrlLen + 1); -} - -SVNDirectory* SVNDirectory::parent() const -{ - return _parent; -} - -SVNDirectory* SVNDirectory::child(const std::string& dirName) const -{ - BOOST_FOREACH(SVNDirectory* d, _children) { - if (d->name() == dirName) { - return d; - } - } - - return NULL; -} - -DirectoryList::iterator -SVNDirectory::findChildDir(const std::string& dirName) -{ - DirectoryList::iterator it; - for (it=_children.begin(); it != _children.end(); ++it) { - if ((*it)->name() == dirName) { - return it; - } - } - return it; -} - -simgear::Dir SVNDirectory::fsDir() const -{ - return Dir(localPath); -} diff --git a/simgear/io/SVNDirectory.hxx b/simgear/io/SVNDirectory.hxx deleted file mode 100644 index 1d7122cc..00000000 --- a/simgear/io/SVNDirectory.hxx +++ /dev/null @@ -1,109 +0,0 @@ -// DAVCollectionMirror.hxx - mirror a DAV collection to the local filesystem -// -// Copyright (C) 2013 James Turner -// -// This program is free software; you can redistribute it and/or -// modify it under the terms of the GNU General Public License as -// published by the Free Software Foundation; either version 2 of the -// License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - - -#ifndef SG_IO_DAVCOLLECTIONMIRROR_HXX -#define SG_IO_DAVCOLLECTIONMIRROR_HXX - -#include -#include -#include - -#include -#include -#include - -namespace simgear { - -class Dir; -namespace HTTP { class Request; } - -// forward decls -class DAVMirror; -class SVNRepository; -class SVNDirectory; - -typedef std::vector DirectoryList; - -class SVNDirectory -{ -public: - // init from local - SVNDirectory(SVNRepository *repo, const SGPath& path); - ~SVNDirectory(); - - void setBaseUrl(const std::string& url); - - // init from a collection - SVNDirectory(SVNDirectory* pr, DAVCollection* col); - - void beginUpdateReport(); - void updateReportComplete(); - - bool isDoingSync() const; - - std::string url() const; - - std::string name() const; - - DAVResource* addChildFile(const std::string& fileName); - SVNDirectory* addChildDirectory(const std::string& dirName); - - // void updateChild(DAVResource* child); - void deleteChildByName(const std::string& name); - - SGPath fsPath() const - { return localPath; } - - simgear::Dir fsDir() const; - - std::string repoPath() const; - - SVNRepository* repository() const; - DAVCollection* collection() const - { return dav; } - - std::string cachedRevision() const - { return _cachedRevision; } - - void mergeUpdateReportDetails(unsigned int depth, string_list& items); - - SVNDirectory* parent() const; - SVNDirectory* child(const std::string& dirName) const; -private: - - void parseCache(); - void writeCache(); - - DirectoryList::iterator findChildDir(const std::string& dirName); - SVNDirectory* parseChildDirectory(const std::string& dirName); - - SGPath localPath; - DAVCollection* dav; - SVNRepository* repo; - - std::string _cachedRevision; - bool _doingUpdateReport; - - SVNDirectory* _parent; - DirectoryList _children; -}; - -} // of namespace simgear - -#endif // of SG_IO_DAVCOLLECTIONMIRROR_HXX diff --git a/simgear/io/SVNReportParser.cxx b/simgear/io/SVNReportParser.cxx deleted file mode 100644 index 5ee3d63f..00000000 --- a/simgear/io/SVNReportParser.cxx +++ /dev/null @@ -1,605 +0,0 @@ -// SVNReportParser -- parser for SVN report XML data -// -// Copyright (C) 2012 James Turner -// -// This program is free software; you can redistribute it and/or -// modify it under the terms of the GNU General Public License as -// published by the Free Software Foundation; either version 2 of the -// License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -#ifdef HAVE_CONFIG_H -# include -#endif - -#include "SVNReportParser.hxx" - -#include -#include -#include -#include -#include -#include - -#include - -#include "simgear/misc/sg_path.hxx" -#include "simgear/misc/sg_dir.hxx" -#include "simgear/debug/logstream.hxx" -#include "simgear/xml/easyxml.hxx" -#include "simgear/misc/strutils.hxx" -#include "simgear/package/md5.h" - -#ifdef SYSTEM_EXPAT -# include -#else -# include "sg_expat.h" -#endif - -#include "SVNDirectory.hxx" -#include "SVNRepository.hxx" -#include "DAVMultiStatus.hxx" - -using std::cout; -using std::cerr; -using std::endl; -using std::string; - -using namespace simgear; - -#define DAV_NS "DAV::" -#define SVN_NS "svn::" -#define SUBVERSION_DAV_NS "http://subversion.tigris.org/xmlns/dav/" - -namespace { - - #define MAX_ENCODED_INT_LEN 10 - - static size_t - decode_size(unsigned char* &p, - const unsigned char *end) - { - if (p + MAX_ENCODED_INT_LEN < end) - end = p + MAX_ENCODED_INT_LEN; - /* Decode bytes until we're done. */ - size_t result = 0; - - while (p < end) { - result = (result << 7) | (*p & 0x7f); - if (((*p++ >> 7) & 0x1) == 0) { - break; - } - } - - return result; - } - - static bool - try_decode_size(unsigned char* &p, - const unsigned char *end) - { - if (p + MAX_ENCODED_INT_LEN < end) - end = p + MAX_ENCODED_INT_LEN; - - while (p < end) { - if (((*p++ >> 7) & 0x1) == 0) { - return true; - } - } - - return false; - } - -// const char* SVN_UPDATE_REPORT_TAG = SVN_NS "update-report"; - // const char* SVN_TARGET_REVISION_TAG = SVN_NS "target-revision"; - const char* SVN_OPEN_DIRECTORY_TAG = SVN_NS "open-directory"; - const char* SVN_OPEN_FILE_TAG = SVN_NS "open-file"; - const char* SVN_ADD_DIRECTORY_TAG = SVN_NS "add-directory"; - const char* SVN_ADD_FILE_TAG = SVN_NS "add-file"; - const char* SVN_TXDELTA_TAG = SVN_NS "txdelta"; - const char* SVN_SET_PROP_TAG = SVN_NS "set-prop"; - const char* SVN_PROP_TAG = SVN_NS "prop"; - const char* SVN_DELETE_ENTRY_TAG = SVN_NS "delete-entry"; - - const char* SVN_DAV_MD5_CHECKSUM = SUBVERSION_DAV_NS ":md5-checksum"; - - const char* DAV_HREF_TAG = DAV_NS "href"; - const char* DAV_CHECKED_IN_TAG = DAV_NS "checked-in"; - - - const int svn_txdelta_source = 0; - const int svn_txdelta_target = 1; - const int svn_txdelta_new = 2; - - const size_t DELTA_HEADER_SIZE = 4; - - /** - * helper struct to decode and store the SVN delta header - * values - */ - struct SVNDeltaWindow - { - public: - - static bool isWindowComplete(unsigned char* buffer, size_t bytes) - { - unsigned char* p = buffer; - unsigned char* pEnd = p + bytes; - // if we can't decode five sizes, certainly incomplete - for (int i=0; i<5; i++) { - if (!try_decode_size(p, pEnd)) { - return false; - } - } - - p = buffer; - // ignore these three - decode_size(p, pEnd); - decode_size(p, pEnd); - decode_size(p, pEnd); - size_t instructionLen = decode_size(p, pEnd); - size_t newLength = decode_size(p, pEnd); - size_t headerLength = p - buffer; - - return (bytes >= (instructionLen + newLength + headerLength)); - } - - SVNDeltaWindow(unsigned char* p) : - headerLength(0), - _ptr(p) - { - sourceViewOffset = decode_size(p, p+20); - sourceViewLength = decode_size(p, p+20); - targetViewLength = decode_size(p, p+20); - instructionLength = decode_size(p, p+20); - newLength = decode_size(p, p+20); - - headerLength = p - _ptr; - _ptr = p; - } - - bool apply(std::vector& output, std::istream& source) - { - unsigned char* pEnd = _ptr + instructionLength; - unsigned char* newData = pEnd; - - while (_ptr < pEnd) { - int op = ((*_ptr >> 6) & 0x3); - if (op >= 3) { - SG_LOG(SG_IO, SG_INFO, "SVNDeltaWindow: bad opcode:" << op); - return false; - } - - int length = *_ptr++ & 0x3f; - int offset = 0; - - if (length == 0) { - length = decode_size(_ptr, pEnd); - } - - if (length == 0) { - SG_LOG(SG_IO, SG_INFO, "SVNDeltaWindow: malformed stream, 0 length" << op); - return false; - } - - // if op != new, decode another size value - if (op != svn_txdelta_new) { - offset = decode_size(_ptr, pEnd); - } - - if (op == svn_txdelta_target) { - // this is inefficent, but ranges can overlap. - while (length > 0) { - output.push_back(output[offset++]); - --length; - } - } else if (op == svn_txdelta_new) { - output.insert(output.end(), newData, newData + length); - newData += length; - } else if (op == svn_txdelta_source) { - source.seekg(offset); - char* sourceBuf = (char*) malloc(length); - assert(sourceBuf); - source.read(sourceBuf, length); - output.insert(output.end(), sourceBuf, sourceBuf + length); - free(sourceBuf); - } else { - SG_LOG(SG_IO, SG_WARN, "bad opcode logic"); - return false; - } - } // of instruction loop - - return true; - } - - size_t size() const - { - return headerLength + instructionLength + newLength; - } - - unsigned int sourceViewOffset; - size_t sourceViewLength, - targetViewLength; - size_t headerLength, - instructionLength, - newLength; - -private: - unsigned char* _ptr; - }; - - -} // of anonymous namespace - -class SVNReportParser::SVNReportParserPrivate -{ -public: - SVNReportParserPrivate(SVNRepository* repo) : - tree(repo), - status(AbstractRepository::REPO_NO_ERROR), - parserInited(false), - currentPath(repo->fsBase()) - { - inFile = false; - currentDir = repo->rootDir(); - } - - ~SVNReportParserPrivate() - { - } - - void startElement (const char * name, const char** attributes) - { - if (status != AbstractRepository::REPO_NO_ERROR) { - return; - } - - ExpatAtts attrs(attributes); - tagStack.push_back(name); - if (!strcmp(name, SVN_TXDELTA_TAG)) { - txDeltaData.clear(); - } else if (!strcmp(name, SVN_ADD_FILE_TAG)) { - string fileName(attrs.getValue("name")); - SGPath filePath(currentDir->fsDir().file(fileName)); - currentPath = filePath; - inFile = true; - } else if (!strcmp(name, SVN_OPEN_FILE_TAG)) { - string fileName(attrs.getValue("name")); - SGPath filePath(Dir(currentPath).file(fileName)); - currentPath = filePath; - - if (!filePath.exists()) { - fail(AbstractRepository::REPO_ERROR_FILE_NOT_FOUND); - return; - } - - inFile = true; - } else if (!strcmp(name, SVN_ADD_DIRECTORY_TAG)) { - string dirName(attrs.getValue("name")); - Dir d(currentDir->fsDir().file(dirName)); - if (d.exists()) { - // policy decision : if we're doing an add, wipe the existing - d.remove(true); - } - - currentDir = currentDir->addChildDirectory(dirName); - currentPath = currentDir->fsPath(); - currentDir->beginUpdateReport(); - //cout << "addDir:" << currentPath << endl; - } else if (!strcmp(name, SVN_SET_PROP_TAG)) { - setPropName = attrs.getValue("name"); - setPropValue.clear(); - } else if (!strcmp(name, SVN_DAV_MD5_CHECKSUM)) { - md5Sum.clear(); - } else if (!strcmp(name, SVN_OPEN_DIRECTORY_TAG)) { - string dirName; - if (attrs.getValue("name")) { - dirName = string(attrs.getValue("name")); - } - openDirectory(dirName); - } else if (!strcmp(name, SVN_DELETE_ENTRY_TAG)) { - string entryName(attrs.getValue("name")); - deleteEntry(entryName); - } else if (!strcmp(name, DAV_CHECKED_IN_TAG) || - !strcmp(name, DAV_HREF_TAG) || - !strcmp(name, SVN_PROP_TAG)) { - // don't warn on these ones - } else { - //SG_LOG(SG_IO, SG_WARN, "SVNReportParser: unhandled tag:" << name); - } - } // of startElement - - void openDirectory(const std::string& dirName) - { - if (dirName.empty()) { - // root directory, we shall assume - currentDir = tree->rootDir(); - } else { - assert(currentDir); - currentDir = currentDir->child(dirName); - } - - assert(currentDir); - currentPath = currentDir->fsPath(); - currentDir->beginUpdateReport(); - } - - void deleteEntry(const std::string& entryName) - { - currentDir->deleteChildByName(entryName); - } - - bool decodeTextDelta(const SGPath& outputPath) - { - std::vector output, decoded; - strutils::decodeBase64(txDeltaData, decoded); - size_t bytesToDecode = decoded.size(); - - unsigned char* p = decoded.data(); - if (memcmp(p, "SVN\0", DELTA_HEADER_SIZE) != 0) { - return false; // bad header - } - - bytesToDecode -= DELTA_HEADER_SIZE; - p += DELTA_HEADER_SIZE; - std::ifstream source; - source.open(outputPath.c_str(), std::ios::in | std::ios::binary); - - while (bytesToDecode > 0) { - if (!SVNDeltaWindow::isWindowComplete(p, bytesToDecode)) { - SG_LOG(SG_IO, SG_WARN, "SVN txdelta broken window"); - return false; - } - - SVNDeltaWindow window(p); - assert(bytesToDecode >= window.size()); - window.apply(output, source); - bytesToDecode -= window.size(); - p += window.size(); - } - - source.close(); - - std::ofstream f; - f.open(outputPath.c_str(), - std::ios::out | std::ios::trunc | std::ios::binary); - f.write((char*) output.data(), output.size()); - - // compute MD5 while we have the file in memory - memset(&md5Context, 0, sizeof(SG_MD5_CTX)); - SG_MD5Init(&md5Context); - SG_MD5Update(&md5Context, (unsigned char*) output.data(), output.size()); - unsigned char digest[MD5_DIGEST_LENGTH]; - SG_MD5Final(digest, &md5Context); - decodedFileMd5 = strutils::encodeHex(digest, MD5_DIGEST_LENGTH); - - return true; - } - - void endElement (const char * name) - { - if (status != SVNRepository::REPO_NO_ERROR) { - return; - } - - assert(tagStack.back() == name); - tagStack.pop_back(); - - if (!strcmp(name, SVN_TXDELTA_TAG)) { - if (!decodeTextDelta(currentPath)) { - fail(SVNRepository::SVN_ERROR_TXDELTA); - } - } else if (!strcmp(name, SVN_ADD_FILE_TAG)) { - finishFile(currentPath); - } else if (!strcmp(name, SVN_OPEN_FILE_TAG)) { - finishFile(currentPath); - } else if (!strcmp(name, SVN_ADD_DIRECTORY_TAG)) { - // pop directory - currentPath = currentPath.dir(); - currentDir->updateReportComplete(); - currentDir = currentDir->parent(); - } else if (!strcmp(name, SVN_SET_PROP_TAG)) { - if (setPropName == "svn:entry:committed-rev") { - revision = strutils::to_int(setPropValue); - currentVersionName = setPropValue; - if (!inFile) { - // for directories we have the resource already - // for adding files, we might not; we set the version name - // above when ending the add/open-file element - currentDir->collection()->setVersionName(currentVersionName); - } - } - } else if (!strcmp(name, SVN_DAV_MD5_CHECKSUM)) { - // validate against (presumably) just written file - if (decodedFileMd5 != md5Sum) { - fail(SVNRepository::REPO_ERROR_CHECKSUM); - } - } else if (!strcmp(name, SVN_OPEN_DIRECTORY_TAG)) { - currentDir->updateReportComplete(); - if (currentDir->parent()) { - // pop the collection stack - currentDir = currentDir->parent(); - } - - currentPath = currentDir->fsPath(); - } else { - // std::cout << "element:" << name; - } - } - - void finishFile(const SGPath& path) - { - currentPath = path.dir(); - inFile = false; - } - - void data (const char * s, int length) - { - if (status != SVNRepository::REPO_NO_ERROR) { - return; - } - - if (tagStack.back() == SVN_SET_PROP_TAG) { - setPropValue.append(s, length); - } else if (tagStack.back() == SVN_TXDELTA_TAG) { - txDeltaData.append(s, length); - } else if (tagStack.back() == SVN_DAV_MD5_CHECKSUM) { - md5Sum.append(s, length); - } - } - - void pi (const char * target, const char * data) {} - - string tagN(const unsigned int n) const - { - size_t sz = tagStack.size(); - if (n >= sz) { - return string(); - } - - return tagStack[sz - (1 + n)]; - } - - void fail(SVNRepository::ResultCode err) - { - status = err; - } - - SVNRepository* tree; - DAVCollection* rootCollection; - SVNDirectory* currentDir; - SVNRepository::ResultCode status; - - bool parserInited; - XML_Parser xmlParser; - -// in-flight data - string_list tagStack; - string currentVersionName; - string txDeltaData; - SGPath currentPath; - bool inFile; - - unsigned int revision; - SG_MD5_CTX md5Context; - string md5Sum, decodedFileMd5; - std::string setPropName, setPropValue; -}; - - -//////////////////////////////////////////////////////////////////////// -// Static callback functions for Expat. -//////////////////////////////////////////////////////////////////////// - -#define VISITOR static_cast(userData) - -static void -start_element (void * userData, const char * name, const char ** atts) -{ - VISITOR->startElement(name, atts); -} - -static void -end_element (void * userData, const char * name) -{ - VISITOR->endElement(name); -} - -static void -character_data (void * userData, const char * s, int len) -{ - VISITOR->data(s, len); -} - -static void -processing_instruction (void * userData, - const char * target, - const char * data) -{ - VISITOR->pi(target, data); -} - -#undef VISITOR - -/////////////////////////////////////////////////////////////////////////////// - -SVNReportParser::SVNReportParser(SVNRepository* repo) : - _d(new SVNReportParserPrivate(repo)) -{ - -} - -SVNReportParser::~SVNReportParser() -{ -} - -SVNRepository::ResultCode -SVNReportParser::innerParseXML(const char* data, int size) -{ - if (_d->status != SVNRepository::REPO_NO_ERROR) { - return _d->status; - } - - bool isEnd = (data == NULL); - if (!XML_Parse(_d->xmlParser, data, size, isEnd)) { - SG_LOG(SG_IO, SG_INFO, "SVN parse error:" << XML_ErrorString(XML_GetErrorCode(_d->xmlParser)) - << " at line:" << XML_GetCurrentLineNumber(_d->xmlParser) - << " column " << XML_GetCurrentColumnNumber(_d->xmlParser)); - - XML_ParserFree(_d->xmlParser); - _d->parserInited = false; - return SVNRepository::SVN_ERROR_XML; - } else if (isEnd) { - XML_ParserFree(_d->xmlParser); - _d->parserInited = false; - } - - return _d->status; -} - -SVNRepository::ResultCode -SVNReportParser::parseXML(const char* data, int size) -{ - if (_d->status != SVNRepository::REPO_NO_ERROR) { - return _d->status; - } - - if (!_d->parserInited) { - _d->xmlParser = XML_ParserCreateNS(0, ':'); - XML_SetUserData(_d->xmlParser, _d.get()); - XML_SetElementHandler(_d->xmlParser, start_element, end_element); - XML_SetCharacterDataHandler(_d->xmlParser, character_data); - XML_SetProcessingInstructionHandler(_d->xmlParser, processing_instruction); - _d->parserInited = true; - } - - return innerParseXML(data, size); -} - -SVNRepository::ResultCode SVNReportParser::finishParse() -{ - if (_d->status != SVNRepository::REPO_NO_ERROR) { - return _d->status; - } - - return innerParseXML(NULL, 0); -} - -std::string SVNReportParser::etagFromRevision(unsigned int revision) -{ - // etags look like W/"7//", hopefully this is stable - // across different servers and similar - std::ostringstream os; - os << "W/\"" << revision << "//"; - return os.str(); -} - - diff --git a/simgear/io/SVNReportParser.hxx b/simgear/io/SVNReportParser.hxx deleted file mode 100644 index 1e94751b..00000000 --- a/simgear/io/SVNReportParser.hxx +++ /dev/null @@ -1,57 +0,0 @@ -// SVNReportParser -- parser for SVN report XML data -// -// Copyright (C) 2012 James Turner -// -// This program is free software; you can redistribute it and/or -// modify it under the terms of the GNU General Public License as -// published by the Free Software Foundation; either version 2 of the -// License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - - -#ifndef SG_IO_SVNREPORTPARSER_HXX -#define SG_IO_SVNREPORTPARSER_HXX - -#include -#include // for auto_ptr - -#include "SVNRepository.hxx" - -class SGPath; - -namespace simgear -{ - -class SVNRepository; - -class SVNReportParser -{ -public: - SVNReportParser(SVNRepository* repo); - ~SVNReportParser(); - - // incremental XML parsing - SVNRepository::ResultCode parseXML(const char* data, int size); - - SVNRepository::ResultCode finishParse(); - - static std::string etagFromRevision(unsigned int revision); - - class SVNReportParserPrivate; -private: - SVNRepository::ResultCode innerParseXML(const char* data, int size); - - std::auto_ptr _d; -}; - -} // of namespace simgear - -#endif // of SG_IO_SVNREPORTPARSER_HXX diff --git a/simgear/io/SVNRepository.cxx b/simgear/io/SVNRepository.cxx deleted file mode 100644 index 033fa9ef..00000000 --- a/simgear/io/SVNRepository.cxx +++ /dev/null @@ -1,386 +0,0 @@ -// DAVMirrorTree -- mirror a DAV tree to the local file-system -// -// Copyright (C) 2012 James Turner -// -// This program is free software; you can redistribute it and/or -// modify it under the terms of the GNU General Public License as -// published by the Free Software Foundation; either version 2 of the -// License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -#include "SVNRepository.hxx" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "simgear/debug/logstream.hxx" -#include "simgear/misc/strutils.hxx" -#include -#include -#include -#include -#include -#include - -using std::cout; -using std::cerr; -using std::endl; -using std::string; - -namespace simgear -{ - -typedef std::vector RequestVector; - -class SVNRepoPrivate -{ -public: - SVNRepoPrivate(SVNRepository* parent) : - p(parent), - isUpdating(false), - status(SVNRepository::REPO_NO_ERROR) - { ; } - - SVNRepository* p; // link back to outer - SVNDirectory* rootCollection; - HTTP::Client* http; - std::string baseUrl; - std::string vccUrl; - std::string targetRevision; - bool isUpdating; - SVNRepository::ResultCode status; - - void svnUpdateDone() - { - isUpdating = false; - } - - void updateFailed(HTTP::Request* req, SVNRepository::ResultCode err) - { - SG_LOG(SG_TERRASYNC, SG_WARN, "SVN: failed to update from:" << req->url() - << "\n(repository:" << p->baseUrl() << ")"); - isUpdating = false; - status = err; - } - - void propFindComplete(HTTP::Request* req, DAVCollection* col); - void propFindFailed(HTTP::Request* req, SVNRepository::ResultCode err); -}; - - -namespace { // anonmouse - - string makeAbsoluteUrl(const string& url, const string& base) - { - if (strutils::starts_with(url, "http://")) - return url; // already absolute - - assert(strutils::starts_with(base, "http://")); - int schemeEnd = base.find("://"); - int hostEnd = base.find('/', schemeEnd + 3); - if (hostEnd < 0) { - return url; - } - - return base.substr(0, hostEnd) + url; - } - - // keep the responses small by only requesting the properties we actually - // care about; the ETag, length and MD5-sum - const char* PROPFIND_REQUEST_BODY = - "" - "" - "" - "" - "" - "" - "" - ""; - - class PropFindRequest : public HTTP::Request - { - public: - PropFindRequest(SVNRepoPrivate* repo) : - Request(repo->baseUrl, "PROPFIND"), - _repo(repo) - { - assert(repo); - requestHeader("Depth") = "0"; - setBodyData( PROPFIND_REQUEST_BODY, - "application/xml; charset=\"utf-8\"" ); - } - - protected: - virtual void responseHeadersComplete() - { - if (responseCode() == 207) { - // fine - } else if (responseCode() == 404) { - _repo->propFindFailed(this, SVNRepository::REPO_ERROR_NOT_FOUND); - } else { - SG_LOG(SG_TERRASYNC, SG_WARN, "request for:" << url() << - " return code " << responseCode()); - _repo->propFindFailed(this, SVNRepository::REPO_ERROR_SOCKET); - _repo = NULL; - } - - Request::responseHeadersComplete(); - } - - virtual void onDone() - { - if (responseCode() == 207) { - _davStatus.finishParse(); - if (_davStatus.isValid()) { - _repo->propFindComplete(this, (DAVCollection*) _davStatus.resource()); - } else { - _repo->propFindFailed(this, SVNRepository::REPO_ERROR_SOCKET); - } - } - } - - virtual void gotBodyData(const char* s, int n) - { - if (responseCode() != 207) { - return; - } - _davStatus.parseXML(s, n); - } - - virtual void onFail() - { - HTTP::Request::onFail(); - if (_repo) { - _repo->propFindFailed(this, SVNRepository::REPO_ERROR_SOCKET); - _repo = NULL; - } - } - - private: - SVNRepoPrivate* _repo; - DAVMultiStatus _davStatus; - }; - -class UpdateReportRequest: - public HTTP::Request -{ -public: - UpdateReportRequest(SVNRepoPrivate* repo, - const std::string& aVersionName, - bool startEmpty) : - HTTP::Request("", "REPORT"), - _parser(repo->p), - _repo(repo), - _failed(false) - { - setUrl(repo->vccUrl); - std::string request = - "\n" - "\n" - "" + repo->baseUrl + "\n" - "unknown\n" - "\n"; - - if( !startEmpty ) - { - string_list entries; - _repo->rootCollection->mergeUpdateReportDetails(0, entries); - BOOST_FOREACH(string e, entries) - { - request += e + "\n"; - } - } - - request += ""; - - setBodyData(request, "application/xml; charset=\"utf-8\""); - } - -protected: - virtual void onDone() - { - if (_failed) { - return; - } - - if (responseCode() == 200) { - SVNRepository::ResultCode err = _parser.finishParse(); - if (err) { - _repo->updateFailed(this, err); - _failed = true; - } else { - _repo->svnUpdateDone(); - } - } else if (responseCode() == 404) { - _repo->updateFailed(this, SVNRepository::REPO_ERROR_NOT_FOUND); - _failed = true; - } else { - SG_LOG(SG_TERRASYNC, SG_WARN, "SVN: request for:" << url() << - " got HTTP status " << responseCode()); - _repo->updateFailed(this, SVNRepository::REPO_ERROR_HTTP); - _failed = true; - } - } - - virtual void gotBodyData(const char* s, int n) - { - if (_failed) { - return; - } - - if (responseCode() != 200) { - return; - } - - SVNRepository::ResultCode err = _parser.parseXML(s, n); - if (err) { - _failed = true; - SG_LOG(SG_IO, SG_WARN, this << ": SVN: request for:" << url() << " failed:" << err); - _repo->updateFailed(this, err); - _repo = NULL; - } - } - - virtual void onFail() - { - HTTP::Request::onFail(); - if (_repo) { - _repo->updateFailed(this, SVNRepository::REPO_ERROR_SOCKET); - _repo = NULL; - } - } -private: - SVNReportParser _parser; - SVNRepoPrivate* _repo; - bool _failed; -}; - -} // anonymous - -SVNRepository::SVNRepository(const SGPath& base, HTTP::Client *cl) : - _d(new SVNRepoPrivate(this)) -{ - _d->http = cl; - _d->rootCollection = new SVNDirectory(this, base); - _d->baseUrl = _d->rootCollection->url(); -} - -SVNRepository::~SVNRepository() -{ - delete _d->rootCollection; -} - -void SVNRepository::setBaseUrl(const std::string &url) -{ - _d->baseUrl = url; - _d->rootCollection->setBaseUrl(url); -} - -std::string SVNRepository::baseUrl() const -{ - return _d->baseUrl; -} - -HTTP::Client* SVNRepository::http() const -{ - return _d->http; -} - -SGPath SVNRepository::fsBase() const -{ - return _d->rootCollection->fsPath(); -} - -bool SVNRepository::isBare() const -{ - if (!fsBase().exists() || Dir(fsBase()).isEmpty()) { - return true; - } - - if (_d->vccUrl.empty()) { - return true; - } - - return false; -} - -void SVNRepository::update() -{ - _d->status = REPO_NO_ERROR; - if (_d->targetRevision.empty() || _d->vccUrl.empty()) { - _d->isUpdating = true; - PropFindRequest* pfr = new PropFindRequest(_d.get()); - http()->makeRequest(pfr); - return; - } - - if (_d->targetRevision == rootDir()->cachedRevision()) { - SG_LOG(SG_TERRASYNC, SG_DEBUG, baseUrl() << " in sync at version " << _d->targetRevision); - _d->isUpdating = false; - return; - } - - _d->isUpdating = true; - UpdateReportRequest* urr = new UpdateReportRequest(_d.get(), - _d->targetRevision, isBare()); - http()->makeRequest(urr); -} - -bool SVNRepository::isDoingSync() const -{ - if (_d->status != REPO_NO_ERROR) { - return false; - } - - return _d->isUpdating || _d->rootCollection->isDoingSync(); -} - -SVNDirectory* SVNRepository::rootDir() const -{ - return _d->rootCollection; -} - -SVNRepository::ResultCode -SVNRepository::failure() const -{ - return _d->status; -} - -/////////////////////////////////////////////////////////////////////////// - -void SVNRepoPrivate::propFindComplete(HTTP::Request* req, DAVCollection* c) -{ - targetRevision = c->versionName(); - vccUrl = makeAbsoluteUrl(c->versionControlledConfiguration(), baseUrl); - rootCollection->collection()->setVersionControlledConfiguration(vccUrl); - p->update(); -} - -void SVNRepoPrivate::propFindFailed(HTTP::Request *req, SVNRepository::ResultCode err) -{ - if (err != SVNRepository::REPO_ERROR_NOT_FOUND) { - SG_LOG(SG_TERRASYNC, SG_WARN, "PropFind failed for:" << req->url()); - } - - isUpdating = false; - status = err; -} - -} // of namespace simgear diff --git a/simgear/io/SVNRepository.hxx b/simgear/io/SVNRepository.hxx deleted file mode 100644 index 3ab4fedc..00000000 --- a/simgear/io/SVNRepository.hxx +++ /dev/null @@ -1,59 +0,0 @@ -// DAVMirrorTree.hxx - mirror a DAV tree to the local file system -// -// Copyright (C) 2012 James Turner -// -// This program is free software; you can redistribute it and/or -// modify it under the terms of the GNU General Public License as -// published by the Free Software Foundation; either version 2 of the -// License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - - -#ifndef SG_IO_SVN_REPOSITORY_HXX -#define SG_IO_SVN_REPOSITORY_HXX - -#include -#include - -namespace simgear { - -class SVNDirectory; -class SVNRepoPrivate; - -class SVNRepository : public AbstractRepository -{ -public: - - SVNRepository(const SGPath& root, HTTP::Client* cl); - virtual ~SVNRepository(); - - SVNDirectory* rootDir() const; - virtual SGPath fsBase() const; - - virtual void setBaseUrl(const std::string& url); - virtual std::string baseUrl() const; - - virtual HTTP::Client* http() const; - - virtual void update(); - - virtual bool isDoingSync() const; - - virtual ResultCode failure() const; -private: - bool isBare() const; - - std::auto_ptr _d; -}; - -} // of namespace simgear - -#endif // of SG_IO_SVN_REPOSITORY_HXX diff --git a/simgear/io/http_svn.cxx b/simgear/io/http_svn.cxx deleted file mode 100644 index f39072d1..00000000 --- a/simgear/io/http_svn.cxx +++ /dev/null @@ -1,51 +0,0 @@ -#include -#include -#include - -#include -#include - - -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -using namespace simgear; -using std::cout; -using std::endl; -using std::cerr; -using std::string; - -HTTP::Client* httpClient; - -int main(int argc, char* argv[]) -{ - sglog().setLogLevels( SG_ALL, SG_INFO ); - HTTP::Client cl; - httpClient = &cl; - - - SGPath p("/Users/jmt/Desktop/traffic"); - SVNRepository airports(p, &cl); - // airports.setBaseUrl("http://svn.goneabitbursar.com/testproject1"); -// airports.setBaseUrl("http://terrascenery.googlecode.com/svn/trunk/data/Scenery/Models"); - - airports.setBaseUrl("http://fgfs.goneabitbursar.com/fgfsai/trunk/AI/Traffic"); - -// airports.setBaseUrl("http://terrascenery.googlecode.com/svn/trunk/data/Scenery/Airports"); - airports.update(); - - while (airports.isDoingSync()) { - cl.update(100); - } - - cout << "all done!" << endl; - return EXIT_SUCCESS; -} \ No newline at end of file From 7d59dd977fbc3efb6582a83056600d5652d5b005 Mon Sep 17 00:00:00 2001 From: James Turner Date: Wed, 25 May 2016 20:39:15 +0100 Subject: [PATCH 10/14] AbstractRepository interface removed. --- simgear/io/AbstractRepository.cxx | 34 --------------- simgear/io/AbstractRepository.hxx | 72 ------------------------------- simgear/io/CMakeLists.txt | 2 - simgear/io/HTTPRepository.cxx | 38 ++++++++-------- simgear/io/HTTPRepository.hxx | 18 +++++++- simgear/io/http_repo_sync.cxx | 2 +- simgear/io/test_repository.cxx | 4 +- simgear/scene/tsync/terrasync.cxx | 8 ++-- 8 files changed, 42 insertions(+), 136 deletions(-) delete mode 100644 simgear/io/AbstractRepository.cxx delete mode 100644 simgear/io/AbstractRepository.hxx diff --git a/simgear/io/AbstractRepository.cxx b/simgear/io/AbstractRepository.cxx deleted file mode 100644 index e9e53efc..00000000 --- a/simgear/io/AbstractRepository.cxx +++ /dev/null @@ -1,34 +0,0 @@ -// AbstractRepository.cxx -- abstract API for TerraSync remote -// -// Copyright (C) 2016 James Turner -// -// This program is free software; you can redistribute it and/or -// modify it under the terms of the GNU General Public License as -// published by the Free Software Foundation; either version 2 of the -// License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -#include "AbstractRepository.hxx" - -namespace simgear -{ - -AbstractRepository::~AbstractRepository() -{ - -} - -size_t AbstractRepository::bytesStillToDownload() const -{ - return 0; -} - -} // of namespace simgear diff --git a/simgear/io/AbstractRepository.hxx b/simgear/io/AbstractRepository.hxx deleted file mode 100644 index 8d586930..00000000 --- a/simgear/io/AbstractRepository.hxx +++ /dev/null @@ -1,72 +0,0 @@ -// AbstractRepository.hxx - API for terrasyc to access remote server -// -// Copyright (C) 2016 James Turner -// -// This program is free software; you can redistribute it and/or -// modify it under the terms of the GNU General Public License as -// published by the Free Software Foundation; either version 2 of the -// License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - - -#ifndef SG_IO_ABSTRACT_REPOSITORY_HXX -#define SG_IO_ABSTRACT_REPOSITORY_HXX - -#include - -#include - -namespace simgear { - - namespace HTTP { - class Client; - } - -class AbstractRepository -{ -public: - - virtual ~AbstractRepository(); - - virtual SGPath fsBase() const = 0; - - virtual void setBaseUrl(const std::string& url) =0; - virtual std::string baseUrl() const = 0;; - - virtual HTTP::Client* http() const = 0; - - virtual void update() = 0; - - virtual bool isDoingSync() const = 0; - - virtual size_t bytesStillToDownload() const; - - enum ResultCode { - REPO_NO_ERROR = 0, - REPO_ERROR_NOT_FOUND, - REPO_ERROR_SOCKET, - SVN_ERROR_XML, - SVN_ERROR_TXDELTA, - REPO_ERROR_IO, - REPO_ERROR_CHECKSUM, - REPO_ERROR_FILE_NOT_FOUND, - REPO_ERROR_HTTP, - REPO_PARTIAL_UPDATE - }; - - virtual ResultCode failure() const = 0; - protected: - -}; - -} // of namespace simgear - -#endif // of SG_IO_ABSTRACT_REPOSITORY_HXX diff --git a/simgear/io/CMakeLists.txt b/simgear/io/CMakeLists.txt index 71213551..682a4f6a 100644 --- a/simgear/io/CMakeLists.txt +++ b/simgear/io/CMakeLists.txt @@ -18,7 +18,6 @@ set(HEADERS HTTPFileRequest.hxx HTTPMemoryRequest.hxx HTTPRequest.hxx - AbstractRepository.hxx HTTPRepository.hxx ) @@ -38,7 +37,6 @@ set(SOURCES HTTPFileRequest.cxx HTTPMemoryRequest.cxx HTTPRequest.cxx - AbstractRepository.cxx HTTPRepository.cxx ) diff --git a/simgear/io/HTTPRepository.cxx b/simgear/io/HTTPRepository.cxx index f2dcc943..dc4dc6ea 100644 --- a/simgear/io/HTTPRepository.cxx +++ b/simgear/io/HTTPRepository.cxx @@ -94,7 +94,7 @@ public: struct Failure { SGPath path; - AbstractRepository::ResultCode error; + HTTPRepository::ResultCode error; }; typedef std::vector FailureList; @@ -104,7 +104,7 @@ public: hashCacheDirty(false), p(parent), isUpdating(false), - status(AbstractRepository::REPO_NO_ERROR), + status(HTTPRepository::REPO_NO_ERROR), totalDownloaded(0) { ; } @@ -115,7 +115,7 @@ public: std::string baseUrl; SGPath basePath; bool isUpdating; - AbstractRepository::ResultCode status; + HTTPRepository::ResultCode status; HTTPDirectory* rootDir; size_t totalDownloaded; @@ -130,9 +130,9 @@ public: std::string computeHashForPath(const SGPath& p); void writeHashCache(); - void failedToGetRootIndex(AbstractRepository::ResultCode st); + void failedToGetRootIndex(HTTPRepository::ResultCode st); void failedToUpdateChild(const SGPath& relativePath, - AbstractRepository::ResultCode fileStatus); + HTTPRepository::ResultCode fileStatus); typedef std::vector RequestVector; RequestVector queuedRequests, @@ -237,7 +237,7 @@ public: std::sort(children.begin(), children.end()); } - void failedToUpdate(AbstractRepository::ResultCode status) + void failedToUpdate(HTTPRepository::ResultCode status) { if (_relativePath.isNull()) { // root dir failed @@ -365,7 +365,7 @@ public: fpath.append(file); if (it->hash != hash) { - _repository->failedToUpdateChild(_relativePath, AbstractRepository::REPO_ERROR_CHECKSUM); + _repository->failedToUpdateChild(_relativePath, HTTPRepository::REPO_ERROR_CHECKSUM); } else { _repository->updatedFileContents(fpath, hash); _repository->totalDownloaded += sz; @@ -375,7 +375,7 @@ public: } void didFailToUpdateFile(const std::string& file, - AbstractRepository::ResultCode status) + HTTPRepository::ResultCode status) { SGPath fpath(_relativePath); fpath.append(file); @@ -580,7 +580,7 @@ size_t HTTPRepository::bytesDownloaded() const return result; } -AbstractRepository::ResultCode +HTTPRepository::ResultCode HTTPRepository::failure() const { if ((_d->status == REPO_NO_ERROR) && !_d->failures.empty()) { @@ -634,10 +634,10 @@ HTTPRepository::failure() const SG_LOG(SG_TERRASYNC, SG_DEBUG, "got file " << fileName << " in " << _directory->absolutePath()); } else if (responseCode() == 404) { SG_LOG(SG_TERRASYNC, SG_WARN, "terrasync file not found on server: " << fileName << " for " << _directory->absolutePath()); - _directory->didFailToUpdateFile(fileName, AbstractRepository::REPO_ERROR_FILE_NOT_FOUND); + _directory->didFailToUpdateFile(fileName, HTTPRepository::REPO_ERROR_FILE_NOT_FOUND); } else { SG_LOG(SG_TERRASYNC, SG_WARN, "terrasync file download error on server: " << fileName << " for " << _directory->absolutePath() << ": " << responseCode() ); - _directory->didFailToUpdateFile(fileName, AbstractRepository::REPO_ERROR_HTTP); + _directory->didFailToUpdateFile(fileName, HTTPRepository::REPO_ERROR_HTTP); } _directory->repository()->finishedRequest(this); @@ -651,7 +651,7 @@ HTTPRepository::failure() const } if (_directory) { - _directory->didFailToUpdateFile(fileName, AbstractRepository::REPO_ERROR_SOCKET); + _directory->didFailToUpdateFile(fileName, HTTPRepository::REPO_ERROR_SOCKET); _directory->repository()->finishedRequest(this); } } @@ -701,7 +701,7 @@ HTTPRepository::failure() const if (responseCode() == 200) { std::string hash = strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH); if (!_targetHash.empty() && (hash != _targetHash)) { - _directory->failedToUpdate(AbstractRepository::REPO_ERROR_CHECKSUM); + _directory->failedToUpdate(HTTPRepository::REPO_ERROR_CHECKSUM); _directory->repository()->finishedRequest(this); return; } @@ -739,12 +739,12 @@ HTTPRepository::failure() const _directory->updateChildrenBasedOnHash(); SG_LOG(SG_TERRASYNC, SG_INFO, "after update of:" << _directory->absolutePath() << " child update took:" << st.elapsedMSec()); } catch (sg_exception& ) { - _directory->failedToUpdate(AbstractRepository::REPO_ERROR_IO); + _directory->failedToUpdate(HTTPRepository::REPO_ERROR_IO); } } else if (responseCode() == 404) { - _directory->failedToUpdate(AbstractRepository::REPO_ERROR_FILE_NOT_FOUND); + _directory->failedToUpdate(HTTPRepository::REPO_ERROR_FILE_NOT_FOUND); } else { - _directory->failedToUpdate(AbstractRepository::REPO_ERROR_HTTP); + _directory->failedToUpdate(HTTPRepository::REPO_ERROR_HTTP); } _directory->repository()->finishedRequest(this); @@ -753,7 +753,7 @@ HTTPRepository::failure() const virtual void onFail() { if (_directory) { - _directory->failedToUpdate(AbstractRepository::REPO_ERROR_SOCKET); + _directory->failedToUpdate(HTTPRepository::REPO_ERROR_SOCKET); _directory->repository()->finishedRequest(this); } } @@ -1021,14 +1021,14 @@ HTTPRepository::failure() const } } - void HTTPRepoPrivate::failedToGetRootIndex(AbstractRepository::ResultCode st) + void HTTPRepoPrivate::failedToGetRootIndex(HTTPRepository::ResultCode st) { SG_LOG(SG_TERRASYNC, SG_WARN, "Failed to get root of repo:" << baseUrl); status = st; } void HTTPRepoPrivate::failedToUpdateChild(const SGPath& relativePath, - AbstractRepository::ResultCode fileStatus) + HTTPRepository::ResultCode fileStatus) { Failure f; f.path = relativePath; diff --git a/simgear/io/HTTPRepository.hxx b/simgear/io/HTTPRepository.hxx index 4a355441..0b82f069 100644 --- a/simgear/io/HTTPRepository.hxx +++ b/simgear/io/HTTPRepository.hxx @@ -20,16 +20,30 @@ #ifndef SG_IO_HTTP_REPOSITORY_HXX #define SG_IO_HTTP_REPOSITORY_HXX -#include #include +#include +#include + namespace simgear { class HTTPRepoPrivate; -class HTTPRepository : public AbstractRepository +class HTTPRepository { public: + enum ResultCode { + REPO_NO_ERROR = 0, + REPO_ERROR_NOT_FOUND, + REPO_ERROR_SOCKET, + SVN_ERROR_XML, + SVN_ERROR_TXDELTA, + REPO_ERROR_IO, + REPO_ERROR_CHECKSUM, + REPO_ERROR_FILE_NOT_FOUND, + REPO_ERROR_HTTP, + REPO_PARTIAL_UPDATE + }; HTTPRepository(const SGPath& root, HTTP::Client* cl); virtual ~HTTPRepository(); diff --git a/simgear/io/http_repo_sync.cxx b/simgear/io/http_repo_sync.cxx index f8186b4f..decd5ea0 100644 --- a/simgear/io/http_repo_sync.cxx +++ b/simgear/io/http_repo_sync.cxx @@ -73,7 +73,7 @@ int main(int argc, char* argv[]) SGTimeStamp::sleepForMSec(100); } - if (repo->failure() != AbstractRepository::REPO_NO_ERROR) { + if (repo->failure() != HTTPRepository::REPO_NO_ERROR) { cerr << "got response:" << repo->failure() << endl; return EXIT_FAILURE; } diff --git a/simgear/io/test_repository.cxx b/simgear/io/test_repository.cxx index 760223d0..f179c5d2 100644 --- a/simgear/io/test_repository.cxx +++ b/simgear/io/test_repository.cxx @@ -532,7 +532,7 @@ void testAbandonMissingFiles(HTTP::Client* cl) repo->setBaseUrl("http://localhost:2000/repo"); repo->update(); waitForUpdateComplete(cl, repo.get()); - if (repo->failure() != AbstractRepository::REPO_PARTIAL_UPDATE) { + if (repo->failure() != HTTPRepository::REPO_PARTIAL_UPDATE) { throw sg_exception("Bad result from missing files test"); } @@ -556,7 +556,7 @@ void testAbandonCorruptFiles(HTTP::Client* cl) repo->setBaseUrl("http://localhost:2000/repo"); repo->update(); waitForUpdateComplete(cl, repo.get()); - if (repo->failure() != AbstractRepository::REPO_PARTIAL_UPDATE) { + if (repo->failure() != HTTPRepository::REPO_PARTIAL_UPDATE) { throw sg_exception("Bad result from corrupt files test"); } diff --git a/simgear/scene/tsync/terrasync.cxx b/simgear/scene/tsync/terrasync.cxx index 7edb6c7d..54644a3a 100644 --- a/simgear/scene/tsync/terrasync.cxx +++ b/simgear/scene/tsync/terrasync.cxx @@ -160,7 +160,7 @@ public: SyncItem currentItem; bool isNewDirectory; std::queue queue; - std::auto_ptr repository; + std::auto_ptr repository; SGTimeStamp stamp; bool busy; ///< is the slot working or idle @@ -446,10 +446,10 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot) } // check result - AbstractRepository::ResultCode res = slot.repository->failure(); - if (res == AbstractRepository::REPO_ERROR_NOT_FOUND) { + HTTPRepository::ResultCode res = slot.repository->failure(); + if (res == HTTPRepository::REPO_ERROR_NOT_FOUND) { notFound(slot.currentItem); - } else if (res != AbstractRepository::REPO_NO_ERROR) { + } else if (res != HTTPRepository::REPO_NO_ERROR) { fail(slot.currentItem); } else { updated(slot.currentItem, slot.isNewDirectory); From 43ebde9914bfa33e5a347413565fce36e8c7e891 Mon Sep 17 00:00:00 2001 From: James Turner Date: Wed, 25 May 2016 20:54:26 +0100 Subject: [PATCH 11/14] Expose pending count from terrasync Total and expose HTTP pending download value as a new property under terrasync-root. --- simgear/scene/tsync/terrasync.cxx | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/simgear/scene/tsync/terrasync.cxx b/simgear/scene/tsync/terrasync.cxx index 54644a3a..62aa4905 100644 --- a/simgear/scene/tsync/terrasync.cxx +++ b/simgear/scene/tsync/terrasync.cxx @@ -163,7 +163,7 @@ public: std::auto_ptr repository; SGTimeStamp stamp; bool busy; ///< is the slot working or idle - + unsigned int pendingKBytes; unsigned int nextWarnTimeout; }; @@ -236,6 +236,8 @@ public: // kbytes, not bytes, because bytes might overflow 2^31 volatile int _total_kb_downloaded; + unsigned int _totalKbPending; + private: virtual void run(); @@ -281,6 +283,7 @@ SGTerraSync::WorkerThread::WorkerThread() : _cache_hits(0), _transfer_rate(0), _total_kb_downloaded(0), + _totalKbPending(0), _is_dirty(false), _stop(false) { @@ -441,6 +444,7 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot) SG_LOG(SG_TERRASYNC, SG_INFO, "HTTP request count:" << _http.hasActiveRequests()); slot.nextWarnTimeout += 10000; } + slot.pendingKBytes = slot.repository->bytesToDownload(); #endif return; // easy, still working } @@ -460,6 +464,7 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot) // whatever happened, we're done with this repository instance slot.busy = false; slot.repository.reset(); + slot.pendingKBytes = 0; } // init and start sync of the next repository @@ -497,6 +502,8 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot) slot.nextWarnTimeout = 20000; slot.stamp.stamp(); slot.busy = true; + slot.pendingKBytes = slot.repository->bytesToDownload(); + SG_LOG(SG_TERRASYNC, SG_INFO, "sync of " << slot.repository->baseUrl() << " started, queue size is " << slot.queue.size()); } } @@ -535,12 +542,16 @@ void SGTerraSync::WorkerThread::runInternal() } bool anySlotBusy = false; + unsigned int newPendingCount = 0; + // update each sync slot in turn for (unsigned int slot=0; slot < NUM_SYNC_SLOTS; ++slot) { updateSyncSlot(_syncSlots[slot]); + newPendingCount += _syncSlots[slot].pendingKBytes; anySlotBusy |= _syncSlots[slot].busy; } + _totalKbPending = newPendingCount; // approximately atomic update _busy = anySlotBusy; if (!anySlotBusy) { // wait on the blocking deque here, otherwise we spin @@ -779,6 +790,9 @@ void SGTerraSync::bind() // download more than 2G in a single session _tiedProperties.Tie( _terraRoot->getNode("downloaded-kbytes", true), (int*) &_workerThread->_total_kb_downloaded ); + _tiedProperties.Tie( _terraRoot->getNode("pending-kbytes", true), (int*) &_workerThread->_totalKbPending ); + + _terraRoot->getNode("busy", true)->setAttribute(SGPropertyNode::WRITE,false); _terraRoot->getNode("active", true)->setAttribute(SGPropertyNode::WRITE,false); _terraRoot->getNode("update-count", true)->setAttribute(SGPropertyNode::WRITE,false); From 8211f1c482d5d9141c62b56fce8f888eee22f2b2 Mon Sep 17 00:00:00 2001 From: James Turner Date: Wed, 25 May 2016 21:46:16 +0100 Subject: [PATCH 12/14] Allow updating a Catalog URL explicitly. Needed for fallback Catalog URL support. --- simgear/package/Catalog.cxx | 12 ++++++++++-- simgear/package/Catalog.hxx | 6 ++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/simgear/package/Catalog.cxx b/simgear/package/Catalog.cxx index 2ce17d5e..8e912ff9 100644 --- a/simgear/package/Catalog.cxx +++ b/simgear/package/Catalog.cxx @@ -144,7 +144,7 @@ protected: " to \n\t" << url); // update the URL and kick off a new request - m_owner->m_url = url; + m_owner->setUrl(url); Downloader* dl = new Downloader(m_owner, url); m_owner->root()->makeHTTPRequest(dl); } else { @@ -437,6 +437,14 @@ std::string Catalog::url() const return m_url; } +void Catalog::setUrl(const std::string &url) +{ + m_url = url; + if (m_status == Delegate::FAIL_NOT_FOUND) { + m_status = Delegate::FAIL_UNKNOWN; + } +} + std::string Catalog::name() const { return getLocalisedString(m_props, "name"); @@ -505,8 +513,8 @@ std::string Catalog::getLocalisedString(const SGPropertyNode* aRoot, const char* void Catalog::refreshComplete(Delegate::StatusCode aReason) { - changeStatus(aReason); m_refreshRequest.reset(); + changeStatus(aReason); } void Catalog::changeStatus(Delegate::StatusCode newStatus) diff --git a/simgear/package/Catalog.hxx b/simgear/package/Catalog.hxx index 11928b1c..8c6e2efc 100644 --- a/simgear/package/Catalog.hxx +++ b/simgear/package/Catalog.hxx @@ -106,6 +106,12 @@ public: std::string url() const; + /** + * update the URL of a package. Does not trigger a refresh, but resets + * error state if the previous URL was not found. + */ + void setUrl(const std::string& url); + std::string name() const; std::string description() const; From 00f42481379e213c85c0a0e7057fb09db25e852c Mon Sep 17 00:00:00 2001 From: James Turner Date: Tue, 29 Mar 2016 18:11:56 +0100 Subject: [PATCH 13/14] Partial update support for HTTP repos. Not used yet, we set full-update mode on repositories for the moment. Pending Terrasync change will enable partial mode. --- simgear/io/HTTPRepository.cxx | 261 +++++++++++++++++++++--- simgear/io/HTTPRepository.hxx | 7 + simgear/io/test_repository.cxx | 320 +++++++++++++++++++++++++++++- simgear/scene/tsync/terrasync.cxx | 1 + 4 files changed, 556 insertions(+), 33 deletions(-) diff --git a/simgear/io/HTTPRepository.cxx b/simgear/io/HTTPRepository.cxx index dc4dc6ea..d87fe3f4 100644 --- a/simgear/io/HTTPRepository.cxx +++ b/simgear/io/HTTPRepository.cxx @@ -57,8 +57,6 @@ namespace simgear { } - virtual void cancel(); - size_t contentSize() const { return _contentSize; @@ -104,6 +102,7 @@ public: hashCacheDirty(false), p(parent), isUpdating(false), + updateEverything(false), status(HTTPRepository::REPO_NO_ERROR), totalDownloaded(0) { ; } @@ -115,10 +114,14 @@ public: std::string baseUrl; SGPath basePath; bool isUpdating; + bool updateEverything; + string_list updatePaths; HTTPRepository::ResultCode status; HTTPDirectory* rootDir; size_t totalDownloaded; + void updateWaiting(); + HTTP::Request_ptr updateFile(HTTPDirectory* dir, const std::string& name, size_t sz); HTTP::Request_ptr updateDir(HTTPDirectory* dir, const std::string& hash, @@ -192,10 +195,12 @@ class HTTPDirectory typedef std::vector ChildInfoList; ChildInfoList children; + public: HTTPDirectory(HTTPRepoPrivate* repo, const std::string& path) : _repository(repo), - _relativePath(path) + _relativePath(path), + _state(DoNotUpdate) { assert(repo); @@ -232,6 +237,8 @@ public: fpath.append(".dirindex"); _repository->updatedFileContents(fpath, hash); + _state = Updated; + children.clear(); parseDirIndex(children); std::sort(children.begin(), children.end()); @@ -239,6 +246,7 @@ public: void failedToUpdate(HTTPRepository::ResultCode status) { + _state = UpdateFailed; if (_relativePath.isNull()) { // root dir failed _repository->failedToGetRootIndex(status); @@ -249,7 +257,11 @@ public: void updateChildrenBasedOnHash() { - //SG_LOG(SG_TERRASYNC, SG_DEBUG, "updated children for:" << relativePath()); + // if we got here for a dir which is still updating or excluded + // from updates, just bail out right now. + if (_state != Updated) { + return; + } string_list indexNames = indexChildren(), toBeUpdated, orphans; @@ -284,6 +296,9 @@ public: SGPath p(relativePath()); p.append(it->file()); HTTPDirectory* childDir = _repository->getOrCreateDirectory(p.str()); + if (childDir->_state == NotUpdated) { + childDir->_state = Updated; + } childDir->updateChildrenBasedOnHash(); } } @@ -301,6 +316,101 @@ public: scheduleUpdates(toBeUpdated); } + void markAsUpToDate() + { + _state = Updated; + } + + void markAsUpdating() + { + assert(_state == NotUpdated); + _state = HTTPDirectory::UpdateInProgress; + } + + void markAsEnabled() + { + // assert because this should only get invoked on newly created + // directory objects which are inside the sub-tree(s) to be updated + assert(_state == DoNotUpdate); + _state = NotUpdated; + } + + void markSubtreeAsNeedingUpdate() + { + if (_state == Updated) { + _state = NotUpdated; // reset back to not-updated + } + + ChildInfoList::iterator cit; + for (cit = children.begin(); cit != children.end(); ++cit) { + if (cit->type == ChildInfo::DirectoryType) { + SGPath p(relativePath()); + p.append(cit->name); + HTTPDirectory* childDir = _repository->getOrCreateDirectory(p.str()); + childDir->markSubtreeAsNeedingUpdate(); + } + } // of child iteration + } + + void markSubtreeAsEnabled() + { + if (_state == DoNotUpdate) { + markAsEnabled(); + } + + ChildInfoList::iterator cit; + for (cit = children.begin(); cit != children.end(); ++cit) { + if (cit->type == ChildInfo::DirectoryType) { + SGPath p(relativePath()); + p.append(cit->name); + HTTPDirectory* childDir = _repository->getOrCreateDirectory(p.str()); + childDir->markSubtreeAsEnabled(); + } + } // of child iteration + } + + + void markAncestorChainAsEnabled() + { + if (_state == DoNotUpdate) { + markAsEnabled(); + } + + if (_relativePath.isNull()) { + return; + } + + std::string prPath = _relativePath.dir(); + if (prPath.empty()) { + _repository->rootDir->markAncestorChainAsEnabled(); + } else { + HTTPDirectory* prDir = _repository->getOrCreateDirectory(prPath); + prDir->markAncestorChainAsEnabled(); + } + } + + void updateIfWaiting(const std::string& hash, size_t sz) + { + if (_state == NotUpdated) { + _repository->updateDir(this, hash, sz); + return; + } + + if ((_state == DoNotUpdate) || (_state == UpdateInProgress)) { + return; + } + + ChildInfoList::iterator cit; + for (cit = children.begin(); cit != children.end(); ++cit) { + if (cit->type == ChildInfo::DirectoryType) { + SGPath p(relativePath()); + p.append(cit->name); + HTTPDirectory* childDir = _repository->getOrCreateDirectory(p.str()); + childDir->updateIfWaiting(cit->hash, cit->sizeInBytes); + } + } // of child iteration + } + void removeOrphans(const string_list& orphans) { string_list::const_iterator it; @@ -337,6 +447,11 @@ public: SGPath p(relativePath()); p.append(*it); HTTPDirectory* childDir = _repository->getOrCreateDirectory(p.str()); + if (childDir->_state == DoNotUpdate) { + SG_LOG(SG_TERRASYNC, SG_WARN, "scheduleUpdate, child:" << *it << " is marked do not update so skipping"); + continue; + } + _repository->updateDir(childDir, cit->hash, cit->sizeInBytes); } } @@ -365,11 +480,12 @@ public: fpath.append(file); if (it->hash != hash) { + // we don't erase the file on a hash mismatch, becuase if we're syncing during the + // middle of a server-side update, the downloaded file may actually become valid. _repository->failedToUpdateChild(_relativePath, HTTPRepository::REPO_ERROR_CHECKSUM); } else { _repository->updatedFileContents(fpath, hash); _repository->totalDownloaded += sz; - //SG_LOG(SG_TERRASYNC, SG_INFO, "did update:" << fpath); } // of hash matches } // of found in child list } @@ -495,7 +611,16 @@ private: HTTPRepoPrivate* _repository; SGPath _relativePath; // in URL and file-system space + typedef enum + { + NotUpdated, + UpdateInProgress, + Updated, + UpdateFailed, + DoNotUpdate + } State; + State _state; }; HTTPRepository::HTTPRepository(const SGPath& base, HTTP::Client *cl) : @@ -533,14 +658,38 @@ SGPath HTTPRepository::fsBase() const void HTTPRepository::update() { - if (_d->isUpdating) { + _d->rootDir->markSubtreeAsNeedingUpdate(); + _d->updateWaiting(); +} + +void HTTPRepository::setEntireRepositoryMode() +{ + if (!_d->updateEverything) { + // this is a one-way decision + _d->updateEverything = true; + } + + // probably overkill but not expensive so let's check everything + // we have in case someone did something funky and switched from partial + // to 'whole repo' updating. + _d->rootDir->markSubtreeAsEnabled(); +} + + +void HTTPRepository::addSubpath(const std::string& relPath) +{ + if (_d->updateEverything) { + SG_LOG(SG_TERRASYNC, SG_WARN, "called HTTPRepository::addSubpath but updating everything"); return; } - _d->status = REPO_NO_ERROR; - _d->isUpdating = true; - _d->failures.clear(); - _d->updateDir(_d->rootDir, std::string(), 0); + _d->updatePaths.push_back(relPath); + + HTTPDirectory* dir = _d->getOrCreateDirectory(relPath); + dir->markSubtreeAsEnabled(); + dir->markAncestorChainAsEnabled(); + + _d->updateWaiting(); } bool HTTPRepository::isDoingSync() const @@ -590,12 +739,6 @@ HTTPRepository::failure() const return _d->status; } - void HTTPRepoGetRequest::cancel() - { - _directory->repository()->http->cancelRequest(this, "Reposiotry cancelled"); - _directory = 0; - } - class FileGetRequest : public HTTPRepoGetRequest { public: @@ -605,7 +748,6 @@ HTTPRepository::failure() const { pathInRepo = _directory->absolutePath(); pathInRepo.append(fileName); - //SG_LOG(SG_TERRASYNC, SG_INFO, "will GET file " << url()); } protected: @@ -628,6 +770,7 @@ HTTPRepository::failure() const virtual void onDone() { file->close(); + if (responseCode() == 200) { std::string hash = strutils::encodeHex(sha1_result(&hashContext), HASH_LENGTH); _directory->didUpdateFile(fileName, hash, contentSize()); @@ -649,7 +792,7 @@ HTTPRepository::failure() const if (pathInRepo.exists()) { pathInRepo.remove(); } - + if (_directory) { _directory->didFailToUpdateFile(fileName, HTTPRepository::REPO_ERROR_SOCKET); _directory->repository()->finishedRequest(this); @@ -676,7 +819,6 @@ HTTPRepository::failure() const _targetHash(targetHash) { sha1_init(&hashContext); - //SG_LOG(SG_TERRASYNC, SG_INFO, "will GET dir " << url()); } void setIsRootDir() @@ -725,8 +867,8 @@ HTTPRepository::failure() const of.write(body.data(), body.size()); of.close(); _directory->dirIndexUpdated(hash); - - //SG_LOG(SG_TERRASYNC, SG_INFO, "updated dir index " << _directory->absolutePath()); + } else { + _directory->markAsUpToDate(); } _directory->repository()->totalDownloaded += contentSize(); @@ -778,15 +920,18 @@ HTTPRepository::failure() const HTTPRepoPrivate::~HTTPRepoPrivate() { + // take a copy since cancelRequest will fail and hence remove + // remove activeRequests, invalidating any iterator to it. + RequestVector copyOfActive(activeRequests); + RequestVector::iterator rq; + for (rq = copyOfActive.begin(); rq != copyOfActive.end(); ++rq) { + http->cancelRequest(*rq, "Repository object deleted"); + } + DirectoryVector::iterator it; for (it=directories.begin(); it != directories.end(); ++it) { delete *it; } - - RequestVector::iterator r; - for (r=activeRequests.begin(); r != activeRequests.end(); ++r) { - (*r)->cancel(); - } } HTTP::Request_ptr HTTPRepoPrivate::updateFile(HTTPDirectory* dir, const std::string& name, size_t sz) @@ -799,6 +944,7 @@ HTTPRepository::failure() const HTTP::Request_ptr HTTPRepoPrivate::updateDir(HTTPDirectory* dir, const std::string& hash, size_t sz) { + dir->markAsUpdating(); RepoRequestPtr r(new DirGetRequest(dir, hash)); r->setContentSize(sz); makeRequest(r); @@ -966,6 +1112,25 @@ HTTPRepository::failure() const HTTPDirectory* d = new HTTPDirectory(this, path); directories.push_back(d); + if (updateEverything) { + d->markAsEnabled(); + } else { + string_list::const_iterator s; + bool shouldUpdate = false; + + for (s = updatePaths.begin(); s != updatePaths.end(); ++s) { + size_t minLen = std::min(path.size(), s->size()); + if (s->compare(0, minLen, path, 0, minLen) == 0) { + shouldUpdate = true; + break; + } + } // of paths iteration + + if (shouldUpdate) { + d->markAsEnabled(); + } + } + return d; } @@ -1002,10 +1167,11 @@ HTTPRepository::failure() const void HTTPRepoPrivate::finishedRequest(const RepoRequestPtr& req) { RequestVector::iterator it = std::find(activeRequests.begin(), activeRequests.end(), req); - if (it == activeRequests.end()) { - throw sg_exception("lost request somehow", req->url()); + // in some cases, for example a checksum failure, we clear the active + // and queued request vectors, so the ::find above can fail + if (it != activeRequests.end()) { + activeRequests.erase(it); } - activeRequests.erase(it); if (!queuedRequests.empty()) { RepoRequestPtr rr = queuedRequests.front(); @@ -1030,6 +1196,27 @@ HTTPRepository::failure() const void HTTPRepoPrivate::failedToUpdateChild(const SGPath& relativePath, HTTPRepository::ResultCode fileStatus) { + if (fileStatus == HTTPRepository::REPO_ERROR_CHECKSUM) { + // stop updating, and mark repository as failed, becuase this + // usually indicates we need to start a fresh update from the + // root. + // (we could issue a retry here, but we leave that to higher layers) + status = fileStatus; + + queuedRequests.clear(); + + RequestVector copyOfActive(activeRequests); + RequestVector::iterator rq; + for (rq = copyOfActive.begin(); rq != copyOfActive.end(); ++rq) { + //SG_LOG(SG_TERRASYNC, SG_DEBUG, "cancelling request for:" << (*rq)->url()); + http->cancelRequest(*rq, "Repository updated failed"); + } + + + SG_LOG(SG_TERRASYNC, SG_WARN, "failed to update repository:" << baseUrl + << ", possibly modified during sync"); + } + Failure f; f.path = relativePath; f.error = fileStatus; @@ -1038,6 +1225,22 @@ HTTPRepository::failure() const SG_LOG(SG_TERRASYNC, SG_WARN, "failed to update entry:" << relativePath << " code:" << fileStatus); } + void HTTPRepoPrivate::updateWaiting() + { + if (!isUpdating) { + status = HTTPRepository::REPO_NO_ERROR; + isUpdating = true; + failures.clear(); + } + // find to-be-updated sub-trees and kick them off + rootDir->updateIfWaiting(std::string(), 0); + + // maybe there was nothing to do + if (activeRequests.empty()) { + status = HTTPRepository::REPO_NO_ERROR; + isUpdating = false; + } + } } // of namespace simgear diff --git a/simgear/io/HTTPRepository.hxx b/simgear/io/HTTPRepository.hxx index 0b82f069..7d46c3dd 100644 --- a/simgear/io/HTTPRepository.hxx +++ b/simgear/io/HTTPRepository.hxx @@ -57,6 +57,13 @@ public: virtual void update(); + /** + * set if we should sync the entire repository + */ + void setEntireRepositoryMode(); + + void addSubpath(const std::string& relPath); + virtual bool isDoingSync() const; virtual ResultCode failure() const; diff --git a/simgear/io/test_repository.cxx b/simgear/io/test_repository.cxx index f179c5d2..da47d005 100644 --- a/simgear/io/test_repository.cxx +++ b/simgear/io/test_repository.cxx @@ -19,6 +19,8 @@ #include #include #include +#include + #include using namespace simgear; @@ -68,9 +70,12 @@ public: int requestCount; bool getWillFail; bool returnCorruptData; + std::auto_ptr accessCallback; void clearRequestCounts(); + void clearFailFlags(); + void setGetWillFail(bool b) { getWillFail = b; @@ -113,7 +118,7 @@ public: if (path.empty()) { return this; } - + string_list pathParts = strutils::split(path, "/"); TestRepoEntry* entry = childEntry(pathParts.front()); if (pathParts.size() == 1) { @@ -213,6 +218,18 @@ void TestRepoEntry::clearRequestCounts() } } +void TestRepoEntry::clearFailFlags() +{ + getWillFail = false; + returnCorruptData = false; + + if (isDir) { + for (size_t i=0; iclearFailFlags(); + } + } +} + TestRepoEntry* global_repo = NULL; class TestRepositoryChannel : public TestServerChannel @@ -249,6 +266,10 @@ public: return; } + if (entry->accessCallback.get()) { + (*entry->accessCallback)(); + } + if (entry->getWillFail) { sendErrorResponse(404, false, "entry marked to fail explicitly:" + repoPath); return; @@ -316,6 +337,15 @@ void verifyFileState(const SGPath& fsRoot, const std::string& relPath) } } +void verifyFileNotPresent(const SGPath& fsRoot, const std::string& relPath) +{ + SGPath p(fsRoot); + p.append(relPath); + if (p.exists()) { + throw sg_error("Present file system entry", relPath); + } +} + void verifyRequestCount(const std::string& relPath, int count) { TestRepoEntry* entry = global_repo->findEntry(relPath); @@ -370,9 +400,10 @@ void testBasicClone(HTTP::Client* cl) p.append("http_repo_basic"); simgear::Dir pd(p); pd.removeChildren(); - + repo.reset(new HTTPRepository(p, cl)); repo->setBaseUrl("http://localhost:2000/repo"); + repo->setEntireRepositoryMode(); repo->update(); waitForUpdateComplete(cl, repo.get()); @@ -410,6 +441,7 @@ void testModifyLocalFiles(HTTP::Client* cl) repo.reset(new HTTPRepository(p, cl)); repo->setBaseUrl("http://localhost:2000/repo"); + repo->setEntireRepositoryMode(); repo->update(); waitForUpdateComplete(cl, repo.get()); @@ -451,6 +483,7 @@ void testMergeExistingFileWithoutDownload(HTTP::Client* cl) repo.reset(new HTTPRepository(p, cl)); repo->setBaseUrl("http://localhost:2000/repo"); + repo->setEntireRepositoryMode(); createFile(p, "dirC/fileCB", 4); // should match createFile(p, "dirC/fileCC", 3); // mismatch @@ -493,6 +526,7 @@ void testLossOfLocalFiles(HTTP::Client* cl) repo.reset(new HTTPRepository(p, cl)); repo->setBaseUrl("http://localhost:2000/repo"); + repo->setEntireRepositoryMode(); repo->update(); waitForUpdateComplete(cl, repo.get()); verifyFileState(p, "dirB/subdirA/fileBAA"); @@ -530,6 +564,7 @@ void testAbandonMissingFiles(HTTP::Client* cl) repo.reset(new HTTPRepository(p, cl)); repo->setBaseUrl("http://localhost:2000/repo"); + repo->setEntireRepositoryMode(); repo->update(); waitForUpdateComplete(cl, repo.get()); if (repo->failure() != HTTPRepository::REPO_PARTIAL_UPDATE) { @@ -554,18 +589,285 @@ void testAbandonCorruptFiles(HTTP::Client* cl) repo.reset(new HTTPRepository(p, cl)); repo->setBaseUrl("http://localhost:2000/repo"); + repo->setEntireRepositoryMode(); + repo->update(); waitForUpdateComplete(cl, repo.get()); - if (repo->failure() != HTTPRepository::REPO_PARTIAL_UPDATE) { + if (repo->failure() != HTTPRepository::REPO_ERROR_CHECKSUM) { throw sg_exception("Bad result from corrupt files test"); } + repo.reset(); + if (cl->hasActiveRequests()) { + cl->debugDumpRequests(); + throw sg_exception("Connection still has requests active"); + } + std::cout << "Passed test: detect corrupted download" << std::endl; } +void testPartialUpdateBasic(HTTP::Client* cl) +{ + std::auto_ptr repo; + SGPath p(simgear::Dir::current().path()); + p.append("http_repo_partial_update"); + simgear::Dir pd(p); + if (pd.exists()) { + pd.removeChildren(); + } + + global_repo->clearRequestCounts(); + global_repo->clearFailFlags(); + global_repo->defineFile("dirA/subdirF/fileAFA"); + global_repo->defineFile("dirA/subdirF/fileAFB"); + global_repo->defineFile("dirA/subdirH/fileAHA"); + global_repo->defineFile("dirA/subdirH/fileAHB"); + + global_repo->defineFile("dirG/subdirA/subsubA/fileGAAB"); + +// request subdir of A + repo.reset(new HTTPRepository(p, cl)); + repo->setBaseUrl("http://localhost:2000/repo"); + repo->addSubpath("dirA/subdirF"); + waitForUpdateComplete(cl, repo.get()); + + verifyFileState(p, "dirA/subdirF/fileAFA"); + verifyFileState(p, "dirA/subdirF/fileAFB"); + + verifyFileState(p, "fileA"); // files are always synced + verifyFileState(p, "dirA/fileAB"); + verifyFileNotPresent(p, "dirB/subdirB/fileBBB"); + verifyFileNotPresent(p, "dirD"); + verifyFileNotPresent(p, "dirA/subdirH/fileAHB"); + + verifyRequestCount("dirA", 1); + verifyRequestCount("dirA/fileAA", 1); + verifyRequestCount("dirA/subdirF", 1); + verifyRequestCount("dirA/subdirF/fileAFA", 1); + verifyRequestCount("dirA/subdirF/fileAFB", 1); + verifyRequestCount("dirB", 0); + verifyRequestCount("dirG", 0); + +// now request dir B + repo->addSubpath("dirB"); + waitForUpdateComplete(cl, repo.get()); + + verifyFileState(p, "dirA/subdirF/fileAFB"); + verifyFileState(p, "dirB/subdirB/fileBBA"); + verifyFileState(p, "dirB/subdirB/fileBBB"); + + verifyRequestCount("dirB", 1); + verifyRequestCount("dirB/subdirA/fileBAC", 1); + verifyRequestCount("dirA", 1); + verifyRequestCount("dirA/fileAA", 1); + verifyRequestCount("dirG", 0); + +// widen subdir to parent + repo->addSubpath("dirA"); + waitForUpdateComplete(cl, repo.get()); + + verifyFileState(p, "dirA/subdirH/fileAHA"); + verifyFileState(p, "dirA/subdirH/fileAHB"); + + verifyRequestCount("dirA", 1); + verifyRequestCount("dirB/subdirA/fileBAC", 1); + verifyRequestCount("dirA/subdirF/fileAFA", 1); + +// request an already fetched subdir - should be a no-op + repo->addSubpath("dirB/subdirB"); + waitForUpdateComplete(cl, repo.get()); + + verifyRequestCount("dirB", 1); + verifyRequestCount("dirB/subdirB/fileBBB", 1); + +// add new / modify files inside + global_repo->defineFile("dirA/subdirF/fileAFC"); + global_repo->defineFile("dirA/subdirF/fileAFD"); + repo->update(); + waitForUpdateComplete(cl, repo.get()); + + if (global_repo->requestCount != 2) { + throw sg_exception("Bad root request count"); + } + + verifyFileState(p, "dirA/subdirF/fileAFC"); + verifyFileState(p, "dirA/subdirF/fileAFD"); + + std::cout << "Passed test: basic partial clone and update" << std::endl; +} + +void testPartialUpdateExisting(HTTP::Client* cl) +{ + std::auto_ptr repo; + SGPath p(simgear::Dir::current().path()); + p.append("http_repo_partial_update_existing"); + simgear::Dir pd(p); + if (pd.exists()) { + pd.removeChildren(); + } + + global_repo->clearRequestCounts(); + global_repo->clearFailFlags(); + +// full update to sync everything + repo.reset(new HTTPRepository(p, cl)); + repo->setBaseUrl("http://localhost:2000/repo"); + repo->setEntireRepositoryMode(); + repo->update(); + waitForUpdateComplete(cl, repo.get()); + +// new repo for partial + global_repo->clearRequestCounts(); + repo.reset(new HTTPRepository(p, cl)); + repo->setBaseUrl("http://localhost:2000/repo"); + repo->addSubpath("dirA/subdirF"); + waitForUpdateComplete(cl, repo.get()); + + if (global_repo->requestCount != 1) { + throw sg_exception("Bad root request count"); + } + + verifyRequestCount("dirA", 0); + verifyRequestCount("dirA/fileAA", 0); + verifyRequestCount("dirA/subdirF", 0); + verifyRequestCount("dirA/subdirF/fileAFA", 0); + verifyRequestCount("dirA/subdirF/fileAFB", 0); + +// and request more dirs + // this is a good simulation of terrasync requesting more subdirs of + // an already created and in sync tree. should not generate any more + // network trip + repo->addSubpath("dirC"); + + verifyFileState(p, "dirC/subdirA/subsubA/fileCAAA"); + verifyRequestCount("dirC/subdirA/subsubA/fileCAAA", 0); + + if (global_repo->requestCount != 1) { + throw sg_exception("Bad root request count"); + } + + std::cout << "Passed test: partial update of existing" << std::endl; +} + +void modifyBTree() +{ + std::cout << "Modifying sub-tree" << std::endl; + + global_repo->findEntry("dirB/subdirA/fileBAC")->revision++; + global_repo->defineFile("dirB/subdirZ/fileBZA"); + global_repo->findEntry("dirB/subdirB/fileBBB")->revision++; +} + +void testPartialUpdateWidenWhileInProgress(HTTP::Client* cl) +{ + std::auto_ptr repo; + SGPath p(simgear::Dir::current().path()); + p.append("http_repo_partial_update_widen"); + simgear::Dir pd(p); + if (pd.exists()) { + pd.removeChildren(); + } + + global_repo->clearRequestCounts(); + global_repo->clearFailFlags(); + + // full update to sync everything + repo.reset(new HTTPRepository(p, cl)); + repo->setBaseUrl("http://localhost:2000/repo"); + + repo->addSubpath("dirA/subdirF"); + repo->addSubpath("dirB/subdirB"); + waitForUpdateComplete(cl, repo.get()); + + verifyRequestCount("dirA/subdirF", 1); + if (global_repo->requestCount != 1) { + throw sg_exception("Bad root request count"); + } + + repo->addSubpath("dirA"); + repo->addSubpath("dirB"); + repo->addSubpath("dirC"); + + waitForUpdateComplete(cl, repo.get()); + + // should not request the root again + verifyRequestCount("dirA/subdirF", 1); + if (global_repo->requestCount != 1) { + throw sg_exception("Bad root request count"); + } + + verifyFileState(p, "dirA/subdirF/fileAFA"); + verifyFileState(p, "dirC/subdirA/subsubA/fileCAAA"); + + std::cout << "Passed test: partial update with widen" << std::endl; +} + +void testServerModifyDuringSync(HTTP::Client* cl) +{ + std::auto_ptr repo; + SGPath p(simgear::Dir::current().path()); + p.append("http_repo_server_modify_during_sync"); + simgear::Dir pd(p); + if (pd.exists()) { + pd.removeChildren(); + } + + global_repo->clearRequestCounts(); + global_repo->clearFailFlags(); + + repo.reset(new HTTPRepository(p, cl)); + repo->setBaseUrl("http://localhost:2000/repo"); + repo->setEntireRepositoryMode(); + + global_repo->findEntry("dirA/fileAA")->accessCallback.reset(make_callback(&modifyBTree)); + + repo->update(); + waitForUpdateComplete(cl, repo.get()); + + global_repo->findEntry("dirA/fileAA")->accessCallback.reset(); + + if (repo->failure() != HTTPRepository::REPO_ERROR_CHECKSUM) { + throw sg_exception("Bad result from corrupt files test"); + } + + std::cout << "Passed test modify server during sync" << std::endl; + +} + +void testDestroyDuringSync(HTTP::Client* cl) +{ + std::auto_ptr repo; + SGPath p(simgear::Dir::current().path()); + p.append("http_repo_destory_during_sync"); + simgear::Dir pd(p); + if (pd.exists()) { + pd.removeChildren(); + } + + global_repo->clearRequestCounts(); + global_repo->clearFailFlags(); + + repo.reset(new HTTPRepository(p, cl)); + repo->setBaseUrl("http://localhost:2000/repo"); + repo->setEntireRepositoryMode(); + + repo->update(); + + // would ideally spin slightly here + + repo.reset(); + + if (cl->hasActiveRequests()) { + throw sg_exception("destory of repo didn't clean up requests"); + } + + std::cout << "Passed test destory during sync" << std::endl; +} + + int main(int argc, char* argv[]) { - sglog().setLogLevels( SG_ALL, SG_INFO ); + sglog().setLogLevels( SG_ALL, SG_DEBUG ); HTTP::Client cl; cl.setMaxConnections(1); @@ -581,6 +883,8 @@ int main(int argc, char* argv[]) global_repo->defineFile("dirB/subdirA/fileBAA"); global_repo->defineFile("dirB/subdirA/fileBAB"); global_repo->defineFile("dirB/subdirA/fileBAC"); + global_repo->defineFile("dirB/subdirB/fileBBA"); + global_repo->defineFile("dirB/subdirB/fileBBB"); global_repo->defineFile("dirC/subdirA/subsubA/fileCAAA"); testBasicClone(&cl); @@ -595,5 +899,13 @@ int main(int argc, char* argv[]) testAbandonCorruptFiles(&cl); + testPartialUpdateBasic(&cl); + testPartialUpdateExisting(&cl); + testPartialUpdateWidenWhileInProgress(&cl); + + testServerModifyDuringSync(&cl); + + testDestroyDuringSync(&cl); + return 0; } diff --git a/simgear/scene/tsync/terrasync.cxx b/simgear/scene/tsync/terrasync.cxx index 62aa4905..da49af48 100644 --- a/simgear/scene/tsync/terrasync.cxx +++ b/simgear/scene/tsync/terrasync.cxx @@ -486,6 +486,7 @@ void SGTerraSync::WorkerThread::updateSyncSlot(SyncSlot &slot) } // of creating directory step slot.repository.reset(new HTTPRepository(path, &_http)); + slot.repository->setEntireRepositoryMode(); slot.repository->setBaseUrl(_httpServer + "/" + slot.currentItem._dir); try { From 1b0289b11fc1ff22f93386648caf126b927618fe Mon Sep 17 00:00:00 2001 From: James Turner Date: Thu, 26 May 2016 23:36:03 +0100 Subject: [PATCH 14/14] HTTP: Always use absolute paths for hashes Ensure we only compute / retrieve hashes for absolute paths. Fixes duplicate paths in repository hash cache. --- simgear/io/HTTPRepository.cxx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/simgear/io/HTTPRepository.cxx b/simgear/io/HTTPRepository.cxx index d87fe3f4..05635dac 100644 --- a/simgear/io/HTTPRepository.cxx +++ b/simgear/io/HTTPRepository.cxx @@ -233,7 +233,7 @@ public: void dirIndexUpdated(const std::string& hash) { - SGPath fpath(_relativePath); + SGPath fpath(absolutePath()); fpath.append(".dirindex"); _repository->updatedFileContents(fpath, hash); @@ -476,7 +476,7 @@ public: if (it == children.end()) { SG_LOG(SG_TERRASYNC, SG_WARN, "updated file but not found in dir:" << _relativePath << " " << file); } else { - SGPath fpath(_relativePath); + SGPath fpath(absolutePath()); fpath.append(file); if (it->hash != hash) { @@ -588,7 +588,7 @@ private: ok = _repository->deleteDirectory(fpath.str()); } else { // remove the hash cache entry - _repository->updatedFileContents(fpath, std::string()); + _repository->updatedFileContents(p, std::string()); ok = p.remove(); } @@ -1146,7 +1146,7 @@ HTTPRepository::failure() const delete d; // update the hash cache too - updatedFileContents(path, std::string()); + updatedFileContents(d->absolutePath(), std::string()); return result; }