new version: 2016.2.0

- avoid confusing people by having both the old and new logic
  in the same place.

.gitignore

@@ -12,7 +12,6 @@ sgBuild
 image
 build
 macflightgear
-fgdata
 src
 tmp
 Makefile
@@ -20,3 +19,4 @@ CMakeFiles
 archivebuild
 osgbuild
 CMakeCache.txt
+*.pyc

.gitmodules

@@ -1,16 +1,20 @@
 [submodule "simgear"]
 	path = simgear
-	url = git://gitorious.org/fg/simgear.git
+	url = git://git.code.sf.net/p/flightgear/simgear
 	branch = next
 [submodule "flightgear"]
 	path = flightgear
-	url = git://gitorious.org/fg/flightgear.git
+	url = git://git.code.sf.net/p/flightgear/flightgear
 	branch = next
 [submodule "fgrun"]
 	path = fgrun
-	url = git://gitorious.org/fg/fgrun.git
+	url = git://git.code.sf.net/p/flightgear/fgrun
 	branch = next
-[submodule "maclauncher"]
-	path = maclauncher
-	url = git://gitorious.org/fg/maclauncher.git
+[submodule "fgdata"]
+	path = fgdata
+	url = git://git.code.sf.net/p/flightgear/fgdata
+	branch = next
+[submodule "windows-3rd-party"]
+	path = windows-3rd-party
+	url = git://git.code.sf.net/p/flightgear/windows-3rd-party
 	branch = master

CMakeLists.txt

@@ -1,65 +1,41 @@
-cmake_minimum_required (VERSION 2.6.4)
+cmake_minimum_required (VERSION 2.8.0)
 
 include (ExternalProject)
 
 project(FlightGear-Meta)
 
-if(NOT CMAKE_INSTALL_PREFIX)
+if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
     set(CMAKE_INSTALL_PREFIX ${CMAKE_BINARY_DIR}/install)
-    message(STATUS "Default install dir to ${CMAKE_INSTALL_PREFIX}")
-endif()
-
-if (UNIX)
-    set(BOOST_BOOTSTRAP "./bootstrap.sh" --prefix=${CMAKE_INSTALL_PREFIX})
-else()
-    set(BOOST_BOOTSTRAP "bootstrap.bat")
+    message(STATUS "Default install dir set to ${CMAKE_INSTALL_PREFIX}")
 endif()
 
 set(SG_DEPS OSG)
 set(FG_DEPS SimGear)
 set(SG_CMAKE_ARGS "")
 set(FG_CMAKE_ARGS "")
-set(FGRUN_CMAKE_ARGS "")
 
-if(APPLE)
-    set(BOOST_ARGS link=static stage --with-system)
-    ExternalProject_Add(Boost
-        PREFIX ${CMAKE_BINARY_DIR}
-        SVN_REPOSITORY http://svn.boost.org/svn/boost/tags/release/Boost_1_52_0
-        UPDATE_COMMAND ${BOOST_BOOTSTRAP}
-        CONFIGURE_COMMAND ""
-        BUILD_COMMAND ./bjam ${BOOST_ARGS}
-        BUILD_IN_SOURCE 1
-        INSTALL_COMMAND ./b2 install ${BOOST_ARGS})
-              
-    list(APPEND SG_DEPS Boost)
-endif()  # of Apple
 
 set(OSG_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX})
 set(SG_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX})
 set(FG_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX})
 set(PLIB_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX})
 set(RTI_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX})
-set(FGRUN_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX})
+
 
 # OpenSceneGraph configuration
-set(OSG_SOURCE http://www.openscenegraph.org/downloads/developer_releases/OpenSceneGraph-3.2.0.zip)
+#set(OSG_SOURCE http://www.openscenegraph.org/downloads/developer_releases/OpenSceneGraph-3.2.0.zip)
+set(OSG_SOURCE https://github.com/openscenegraph/osg.git)
+set(OSG_TAG OpenSceneGraph-3.2)
 
 if (APPLE)
-    set(SDKROOT /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.7.sdk)
-    # force disable Qt and Jasper, and ensure an SDKROOT is set, or
-    # osgViewer system detection goes wrongh
-    set(OSG_CMAKE_ARGS -DCMAKE_OSX_SYSROOT=${SDKROOT} 
+    # force disable Qt and Jasper
+    set(OSG_CMAKE_ARGS
         -DOSG_USE_QT=0 
         -DJASPER_LIBRARY= 
-        -DSDL_LIBRARY:FILEPATH=
-        -DCMAKE_OSX_ARCHITECTURES=x86_64
-        -DCMAKE_OSX_DEPLOYMENT_TARGET=10.7
     )
     # OSG with some patches applied for Mac
-    # set(OSG_SOURCE GIT_REPOSITORY git://gitorious.org/+flightgear-developers/openscenegraph/mac-release-osg.git)
-    # set(OSG_SOURCE URL http://www.openscenegraph.org/downloads/developer_releases/OpenSceneGraph-3.2.0.zip)
-
+    set(OSG_SOURCE https://github.com/zakalawe/osg.git)
+    set(OSG_TAG fgfs-osg-32)
 elseif(MSVC)
     set(OSG_MSVC "msvc")
     if (${MSVC_VERSION} EQUAL 1700)
@@ -104,64 +80,31 @@ else()
     # normal OSG
 endif()
 
-if (MSVC)
-    # download 3rdparty dependencies zip, including boost
-    ExternalProject_Add(WinDeps
-        SVN_REPOSITORY http://fgfs.goneabitbursar.com/fgwin3rdparty/trunk/msvc100/${RDPARTY_DIR}
-        # extract to current root
-        SOURCE_DIR ${CMAKE_BINARY_DIR}/winDeps/${RDPARTY_DIR}
-        BINARY_DIR ${CMAKE_BINARY_DIR}
-        UPDATE_COMMAND ""
-        CONFIGURE_COMMAND ""
-        BUILD_COMMAND ${CMAKE_SOURCE_DIR}/installWinDeps.bat
-        INSTALL_COMMAND ""
-    )
-
-    set(BOOST_ARGS link=static stage --with-system)
-    set(Boost_Version 1.54.0)
-    string(REPLACE "." "_" Boost_Version_Underscore ${Boost_Version})
-    ExternalProject_Add(Boost
-        PREFIX ${CMAKE_BINARY_DIR}
-        URL http://downloads.sourceforge.net/project/boost/boost/${Boost_Version}/boost_${Boost_Version_Underscore}.zip
-        URL_MD5 78a35834c45220a6164310e280abe675
-        UPDATE_COMMAND ""
-        CONFIGURE_COMMAND ${BOOST_BOOTSTRAP}
-        BUILD_COMMAND bjam --with-program_options ${BOOST_ARGS}
-        BUILD_IN_SOURCE 1
-        INSTALL_COMMAND ""
-    )
-
-    list(APPEND SG_DEPS Boost)
-    list(APPEND SG_CMAKE_ARGS -DBOOST_ROOT=${CMAKE_BINARY_DIR}/src/Boost)
-    list(APPEND FG_CMAKE_ARGS -DBOOST_ROOT=${CMAKE_BINARY_DIR}/src/Boost)
-    list(APPEND FG_CMAKE_ARGS -DPNG_LIBRARY=${CMAKE_BINARY_DIR}/${RDPARTY_DIR}/lib/libpng16.lib)
-    list(APPEND FGRUN_CMAKE_ARGS -DBOOST_ROOT=${CMAKE_BINARY_DIR}/src/Boost)
-endif(MSVC) # of Windows
 
 ExternalProject_Add(OSG
     DEPENDS ${OSG_DEPS}
     PREFIX ${CMAKE_BINARY_DIR}
-    URL ${OSG_SOURCE}
-    URL_HASH MD5=4980f8692712a24d4c99f363f80c6814
+    GIT_REPOSITORY ${OSG_SOURCE}
+    GIT_TAG ${OSG_TAG}
     BINARY_DIR osgbuild
     CMAKE_ARGS ${OSG_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${OSG_INSTALL_PREFIX}
 )
 
 # Because OSG install the libraries in lib64/ instead of lib/
+
 if (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
     ExternalProject_Add_Step(OSG after_install
-        COMMAND ${CMAKE_COMMAND} -E copy_directory ${OSG_INSTALL_PREFIX}/lib64 ${OSG_INSTALL_PREFIX}/lib
-        COMMAND ${CMAKE_COMMAND} -E remove_directory ${OSG_INSTALL_PREFIX}/lib64
+        COMMAND ${CMAKE_COMMAND} -E create_symlink ${OSG_INSTALL_PREFIX}/lib64 ${OSG_INSTALL_PREFIX}/lib
         DEPENDEES install
     )
-endif(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+endif()
 
 # FIXME install of OpenRTI is failing on Windows, files in PREFIX/share which
 # are ending up in C:/Program Files/OpenRTI
-if (NOT MSVC)
+if (FALSE)
     ExternalProject_Add(OpenRTI
         PREFIX ${CMAKE_BINARY_DIR}
-        DOWNLOAD_COMMAND GIT_REPOSITORY git://gitorious.org/openrti/openrti.git
+        DOWNLOAD_COMMAND GIT_REPOSITORY git://git.code.sf.net/p/openrti/OpenRTI
         BINARY_DIR rtibuild 
         CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${RTI_INSTALL_PREFIX}
     )
@@ -205,23 +148,6 @@ if (NOT WIN32)
     list(APPEND FG_DEPS PLIB)
 endif()
 
-# Only compile FGRun for Windows
-if (MSVC)
-    list(APPEND FGRUN_CMAKE_ARGS -DMSVC_3RDPARTY_ROOT:PATH=${CMAKE_BINARY_DIR})
-    list(APPEND FGRUN_CMAKE_ARGS -DFLTK_FLUID_EXECUTABLE:FILEPATH=${CMAKE_BINARY_DIR}/${RDPARTY_DIR}/bin/fluid.exe)
-    list(APPEND FGRUN_CMAKE_ARGS -DGETTEXT_MSGFMT_EXECUTABLE:FILEPATH==${CMAKE_BINARY_DIR}/${RDPARTY_DIR}/bin/msgfmt.exe)
-    list(APPEND FGRUN_CMAKE_ARGS -DGETTEXT_MSGMERGE_EXECUTABLE:FILEPATH=${CMAKE_BINARY_DIR}/${RDPARTY_DIR}/bin/msgmerge.exe)
-    ExternalProject_Add(FGRun
-        PREFIX ${CMAKE_BINARY_DIR}
-        DEPENDS ${FG_DEPS}
-        DOWNLOAD_COMMAND ""    # no need to download
-        UPDATE_COMMAND ""      # or update.
-        SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}/fgrun
-        BINARY_DIR fgrunbuild
-        CMAKE_ARGS ${FGRUN_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${FGRUN_INSTALL_PREFIX} -DCMAKE_PREFIX_PATH=${OSG_INSTALL_PREFIX} -DADDITIONAL_LIBRARY_PATHS=${SG_INSTALL_PREFIX}
-    )
-endif()
-
 ExternalProject_Add(FlightGear
     PREFIX ${CMAKE_BINARY_DIR}
     DEPENDS ${FG_DEPS}

FlightGear.iss

@@ -37,21 +37,23 @@
 AppId=FlightGear
 AppName=FlightGear
 AppPublisher=The FlightGear Team
-OutputBaseFilename=fgsetup-{#FGVersion}
+OutputBaseFilename=FlightGear-{#FGVersion}{#FGDetails}
 AppVerName=FlightGear v{#FGVersion}
 AppPublisherURL=http://www.flightgear.org
 AppSupportURL=http://www.flightgear.org
 AppUpdatesURL=http://www.flightgear.org
 DefaultDirName={pf}\FlightGear {#FGVersion}
+UsePreviousAppDir=no
 DefaultGroupName=FlightGear {#FGVersion}
+UsePreviousGroup=no
 LicenseFile=X:\flightgear\COPYING
 Uninstallable=yes
 SetupIconFile=X:\flightgear\package\flightgear.ico
 VersionInfoVersion={#FGVersion}.0
-InfoBeforeFile=X:\flightgear\package\Win32-Inno\infobefore.txt
-WizardImageFile=X:\flightgear\package\Win32-Inno\setupimg.bmp
+InfoBeforeFile=X:\flightgear\package\windows\infobefore.txt
+WizardImageFile=X:\flightgear\package\windows\setupimg.bmp
 WizardImageStretch=No
-WizardSmallImageFile=X:\flightgear\package\Win32-Inno\setupsmall.bmp
+WizardSmallImageFile=X:\flightgear\package\windows\setupsmall.bmp
 VersionInfoCompany=The FlightGear Team
 UninstallDisplayIcon={app}\bin\fgfs.exe
 ArchitecturesInstallIn64BitMode=x64
@@ -67,22 +69,13 @@ Name: "insoal"; Description: "Install OpenAL (the sound engine)"
 ; NOTE: run subst X: F:\ (or whatever path the expanded tree resides at)
 ;Source: "X:\*.txt"; DestDir: "{app}"; Flags: ignoreversion
 ; 32 bits install
-Source: "X:\install\msvc100\FlightGear\bin\fgfs.exe"; DestDir: "{app}\bin"; Flags: ignoreversion; Check: not Is64BitInstallMode
-Source: "X:\install\msvc100\FlightGear\bin\fgadmin.exe"; DestDir: "{app}\bin"; Flags: ignoreversion; Check: not Is64BitInstallMode
-Source: "X:\install\msvc100\FlightGear\bin\terrasync.exe"; DestDir: "{app}\bin"; Flags: ignoreversion; Check: not Is64BitInstallMode
-Source: "X:\install\msvc100\FlightGear\bin\js_demo.exe"; DestDir: "{app}\bin"; Flags: ignoreversion; Check: not Is64BitInstallMode
-Source: "X:\install\msvc100\FlightGear\bin\fgjs.exe"; DestDir: "{app}\bin"; Flags: ignoreversion; Check: not Is64BitInstallMode
-Source: "X:\install\msvc100\FlightGear\bin\GPSsmooth.exe"; DestDir: "{app}\bin"; Flags: ignoreversion; Check: not Is64BitInstallMode
-Source: "X:\install\msvc100\FlightGear\bin\UGsmooth.exe"; DestDir: "{app}\bin"; Flags: ignoreversion; Check: not Is64BitInstallMode
-Source: "X:\install\msvc100\FlightGear\bin\MIDGsmooth.exe"; DestDir: "{app}\bin"; Flags: ignoreversion; Check: not Is64BitInstallMode
-Source: "X:\install\msvc100\FlightGear\bin\metar.exe"; DestDir: "{app}\bin"; Flags: ignoreversion; Check: not Is64BitInstallMode
-Source: "X:\install\msvc100\FlightGear\bin\yasim.exe"; DestDir: "{app}\bin"; Flags: ignoreversion; Check: not Is64BitInstallMode
-Source: "X:\install\msvc100\FlightGear\bin\fgcom.exe"; DestDir: "{app}\bin"; Flags: ignoreversion; Check: not Is64BitInstallMode
+Source: "X:\install\msvc100\FlightGear\bin\*"; DestDir: "{app}\bin"; Flags: ignoreversion recursesubdirs; Check: not Is64BitInstallMode
 Source: "X:\install\msvc100\FGRun\bin\fgrun.exe"; DestDir: "{app}\bin"; Flags: ignoreversion ; Check: not Is64BitInstallMode
 Source: "X:\install\msvc100\FGRun\share\locale\*"; DestDir: "{app}\bin\locale"; Flags: ignoreversion recursesubdirs; Check: not Is64BitInstallMode
 Source: "X:\3rdParty\bin\zlib.dll"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
 Source: "X:\3rdParty\bin\OpenAL32.dll"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
 Source: "X:\3rdParty\bin\libpng16.dll"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
+Source: "X:\3rdParty\bin\libintl-8.dll"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
 Source: "X:\3rdParty\bin\CrashRpt1402.dll"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
 Source: "X:\3rdParty\bin\crashrpt_lang.ini"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
 Source: "X:\3rdParty\bin\CrashSender1402.exe"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
@@ -90,23 +83,14 @@ Source: "{#VCInstallDir}\redist\x86\Microsoft.VC100.CRT\*.dll"; DestDir:  "{app}
 Source: "X:\3rdParty\bin\vcredist_x86.exe"; DestDir: "{app}\bin"; Flags: skipifsourcedoesntexist; Check: not Is64BitInstallMode
 
 ; 64 bits install
-Source: "X:\install\msvc100-64\FlightGear\bin\fgfs.exe"; DestDir: "{app}\bin"; Flags: ignoreversion; Check: Is64BitInstallMode
-Source: "X:\install\msvc100-64\FlightGear\bin\fgadmin.exe"; DestDir: "{app}\bin"; Flags: ignoreversion; Check: Is64BitInstallMode
-Source: "X:\install\msvc100-64\FlightGear\bin\terrasync.exe"; DestDir: "{app}\bin"; Flags: ignoreversion; Check: Is64BitInstallMode
-Source: "X:\install\msvc100-64\FlightGear\bin\js_demo.exe"; DestDir: "{app}\bin"; Flags: ignoreversion; Check: Is64BitInstallMode
-Source: "X:\install\msvc100-64\FlightGear\bin\fgjs.exe"; DestDir: "{app}\bin"; Flags: ignoreversion; Check: Is64BitInstallMode
-Source: "X:\install\msvc100-64\FlightGear\bin\GPSsmooth.exe"; DestDir: "{app}\bin"; Flags: ignoreversion; Check: Is64BitInstallMode
-Source: "X:\install\msvc100-64\FlightGear\bin\UGsmooth.exe"; DestDir: "{app}\bin"; Flags: ignoreversion; Check: Is64BitInstallMode
-Source: "X:\install\msvc100-64\FlightGear\bin\MIDGsmooth.exe"; DestDir: "{app}\bin"; Flags: ignoreversion; Check: Is64BitInstallMode
-Source: "X:\install\msvc100-64\FlightGear\bin\metar.exe"; DestDir: "{app}\bin"; Flags: ignoreversion; Check: Is64BitInstallMode
-Source: "X:\install\msvc100-64\FlightGear\bin\yasim.exe"; DestDir: "{app}\bin"; Flags: ignoreversion; Check: Is64BitInstallMode
-Source: "X:\install\msvc100-64\FlightGear\bin\fgcom.exe"; DestDir: "{app}\bin"; Flags: ignoreversion; Check: Is64BitInstallMode
+Source: "X:\install\msvc100-64\FlightGear\bin\*"; DestDir: "{app}\bin"; Flags: ignoreversion recursesubdirs; Check: Is64BitInstallMode
 Source: "X:\install\msvc100-64\FGRun\bin\fgrun.exe"; DestDir: "{app}\bin"; Flags: ignoreversion; Check: Is64BitInstallMode
 Source: "X:\install\msvc100-64\FGRun\share\locale\*"; DestDir: "{app}\bin\locale"; Flags: ignoreversion recursesubdirs; Check: Is64BitInstallMode
 Source: "X:\3rdParty.x64\bin\zlib.dll"; DestDir: "{app}\bin"; Check: Is64BitInstallMode
 Source: "X:\3rdParty.x64\bin\OpenAL32.dll"; DestDir: "{app}\bin"; Check: Is64BitInstallMode
 Source: "X:\3rdParty.x64\bin\vcredist_x64.exe"; DestDir: "{app}\bin"; Flags: skipifsourcedoesntexist; Check: Is64BitInstallMode
 Source: "X:\3rdParty.x64\bin\libpng16.dll"; DestDir: "{app}\bin"; Check: Is64BitInstallMode
+Source: "X:\3rdParty.x64\bin\libintl-8.dll"; DestDir: "{app}\bin"; Check: Is64BitInstallMode
 Source: "X:\3rdParty.x64\bin\CrashRpt1402.dll"; DestDir: "{app}\bin"; Check: Is64BitInstallMode
 Source: "X:\3rdParty.x64\bin\crashrpt_lang.ini"; DestDir: "{app}\bin"; Check: Is64BitInstallMode
 Source: "X:\3rdParty.x64\bin\CrashSender1402.exe"; DestDir: "{app}\bin"; Check: Is64BitInstallMode
@@ -115,14 +99,12 @@ Source: "{#VCInstallDir}\redist\x64\Microsoft.VC100.CRT\*.dll"; DestDir: "{app}\
 ; 32/64 bits install
 ;NOTE: FGPanel has no 64 bits equivalent, so we are using the 32 bits binary for 32&64 bits OS
 Source: "X:\install\msvc100\FlightGear\bin\fgpanel.exe"; DestDir: "{app}\bin"; Flags: ignoreversion
-Source: "X:\install\msvc100\FlightGear\share\flightgear\positions.txt"; DestDir: "{app}\share\flightgear"
-Source: "X:\install\msvc100\FlightGear\share\flightgear\special_frequencies.txt"; DestDir: "{app}\share\flightgear"
 Source: "X:\3rdParty\bin\oalinst.exe"; DestDir: "{app}\bin"; Flags: ignoreversion skipifsourcedoesntexist
 
-; NOTE: tar.gz file uses name 'fgdata', to avoid renaming it, look for both names.
-; assuming no setup has both names and hence we don't package twice :)
-Source: "X:\data\*.*"; DestDir: "{app}\data"; Flags: ignoreversion recursesubdirs skipifsourcedoesntexist
+; Include the base package
+#if IncludeData == "TRUE"
 Source: "X:\fgdata\*.*"; DestDir: "{app}\data"; Flags: ignoreversion recursesubdirs skipifsourcedoesntexist
+#endif
 
 ; 32 bits install
 Source: "{#OSGInstallDir}\bin\osg{#OSGSoNumber}-osg.dll"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
@@ -207,16 +189,17 @@ Name: "{userdocs}\FlightGear\TerraSync"; Permissions: everyone-modify; Check: no
 Name: "{userdocs}\FlightGear\Custom Scenery"; Permissions: everyone-modify; Check: not DirExists(ExpandConstant('{userdocs}\FlightGear\Custom Scenery'))
 
 [Icons]
-Name: "{userdesktop}\FlightGear {#FGVersion}"; Filename: "{app}\bin\fgrun.exe"; WorkingDir: "{app}"; Tasks: desktopicon;
-Name: "{group}\FlightGear Launcher"; Filename: "{app}\bin\fgrun.exe"; WorkingDir: "{app}";
-Name: "{group}\FlightGear Manual"; Filename: "{app}\data\Docs\getstart.pdf"
-Name: "{group}\FlightGear Documentation"; Filename: "{app}\data\Docs\index.html"
+Name: "{userdesktop}\FlightGear {#FGVersion}"; Filename: "{app}\bin\fgfs.exe"; Parameters: "--launcher"; WorkingDir: "{app}"; Tasks: desktopicon;
+Name: "{group}\FlightGear Launcher"; Filename: "{app}\bin\fgfs.exe"; Parameters: "--launcher"; WorkingDir: "{app}";
+Name: "{group}\FlightGear Manual"; Filename: "{app}\fgdata\Docs\getstart.pdf"
+Name: "{group}\FlightGear Documentation"; Filename: "{app}\fgdata\Docs\index.html"
 Name: "{group}\Flightgear Wiki"; Filename: "http://wiki.flightgear.org"
+Name: "{group}\Tools\FGRun"; Filename: "{app}\bin\fgrun.exe"; WorkingDir: "{app}";
 Name: "{group}\Tools\Install & Uninstall Scenery"; Filename: "{app}\bin\fgadmin.exe"; WorkingDir: "{app}"
 Name: "{group}\Tools\TerraSync"; Filename: "{app}\bin\terrasync.exe"; Parameters: "-S -p 5505 -d ""{userdocs}\FlightGear\TerraSync"""; WorkingDir: "{app}"
 Name: "{group}\Tools\Uninstall FlightGear"; Filename: "{uninstallexe}"
 Name: "{group}\Tools\js_demo"; Filename: "{app}\bin\js_demo.exe"
-Name: "{group}\Tools\fgjs"; Filename: "cmd"; Parameters: "/k fgjs.exe ""--fg-root={app}\data"""; WorkingDir: "{app}\bin"
+Name: "{group}\Tools\fgjs"; Filename: "cmd"; Parameters: "/k fgjs.exe ""--fg-root={app}\fgdata"""; WorkingDir: "{app}\bin"
 Name: "{group}\Tools\GPSsmooth"; Filename: "cmd"; Parameters: "/k ""{app}\bin\GPSsmooth.exe"" -h"; WorkingDir: "{app}\bin"
 Name: "{group}\Tools\UGsmooth"; Filename: "cmd"; Parameters: "/k ""{app}\bin\UGsmooth.exe"" -h"; WorkingDir: "{app}\bin"
 Name: "{group}\Tools\MIDGsmooth"; Filename: "cmd"; Parameters: "/k ""{app}\bin\MIDGsmooth.exe"" -h"; WorkingDir: "{app}\bin"
@@ -224,8 +207,8 @@ Name: "{group}\Tools\metar"; Filename: "cmd"; Parameters: "/k ""{app}\bin\metar.
 Name: "{group}\Tools\yasim"; Filename: "cmd"; Parameters: "/k ""{app}\bin\yasim.exe"" -h"; WorkingDir: "{app}\bin"
 Name: "{group}\Tools\fgpanel"; Filename: "cmd"; Parameters: "/k ""{app}\bin\fgpanel.exe"" -h"; WorkingDir: "{app}\bin"
 Name: "{group}\Tools\FGCom"; Filename: "{app}\bin\fgcom.exe"; WorkingDir: "{app}\bin"
-Name: "{group}\Tools\FGCom-testing"; Filename: "{app}\bin\fgcom.exe"; Parameters: "-f910"; WorkingDir: "{app}\bin"
-Name: "{group}\Tools\Explore Documentation Folder"; Filename: "{app}\data\Docs"
+Name: "{group}\Tools\FGCom-testing"; Filename: "{app}\bin\fgcom.exe"; Parameters: "--frequency=910"; WorkingDir: "{app}\bin"
+Name: "{group}\Tools\Explore Documentation Folder"; Filename: "{app}\fgdata\Docs"
 
 [Run]
 filename: "cmd.exe"; WorkingDir: "{app}\bin"; Parameters: "/C del msvc*.dll"; Check: FileExists(ExpandConstant('{app}\bin\vcredist_x86.exe'))
@@ -234,11 +217,25 @@ filename: "{app}\bin\vcredist_x86.exe"; WorkingDir: "{app}\bin"; Parameters: "/p
 filename: "{app}\bin\vcredist_x64.exe"; WorkingDir: "{app}\bin"; Parameters: "/passive /norestart"; Description: "Installing MS Visual C++ runtime components"; Check: Is64BitInstallMode and FileExists(ExpandConstant('{app}\bin\vcredist_x64.exe'))
 filename: "{app}\bin\oalinst.exe"; WorkingDir: "{app}\bin"; Description: "Installing OpenAL"; Check: IsTaskSelected('insoal') and FileExists(ExpandConstant('{app}\bin\oalinst.exe'))
 ; Put installation directory into the fgrun.prefs
-filename: "{app}\bin\fgrun.exe"; WorkingDir: "{app}\bin"; Parameters: "--silent ""--fg-exe={app}\bin\fgfs.exe"" ""--fg-root={app}\data"" ""--fg-scenery={userdocs}\FlightGear\Custom Scenery;{app}\data\Scenery;{userdocs}\FlightGear\TerraSync"" ""--fg-aircraft={userdocs}\FlightGear\Aircraft"" ""--terrasync-dir={userdocs}\FlightGear\TerraSync"" --version={#FGVersion}"
+filename: "{app}\bin\fgrun.exe"; WorkingDir: "{app}\bin"; Parameters: "--silent ""--fg-exe={app}\bin\fgfs.exe"" ""--fg-root={app}\fgdata"" ""--fg-scenery={userdocs}\FlightGear\Custom Scenery;{app}\fgdata\Scenery;{userdocs}\FlightGear\TerraSync"" ""--fg-aircraft={userdocs}\FlightGear\Aircraft"" ""--terrasync-dir={userdocs}\FlightGear\TerraSync"" --version={#FGVersion}"
 ; Put installation and source directories into the fgadmin.prefs
 filename: "{app}\bin\fgadmin.exe"; WorkingDir: "{app}\bin"; Parameters: "--silent ""--install-source={src}\..\Scenery"" ""--scenery-dest={userdocs}\FlightGear\Custom Scenery"""
 
 [Code]
+const
+  NET_FW_SCOPE_ALL = 0;
+  NET_FW_IP_VERSION_ANY = 2;
+  NET_FW_ACTION_ALLOW = 1;
+  NET_FW_RULE_DIR_ALL = 0;
+  NET_FW_RULE_DIR_IN = 1;
+  NET_FW_RULE_DIR_OUT = 2;
+  NET_FW_IP_PROTOCOL_ALL = 0;
+  NET_FW_IP_PROTOCOL_TCP = 6;
+  NET_FW_IP_PROTOCOL_UDP = 17;
+  NET_FW_PROFILE2_DOMAIN = 1;
+  NET_FW_PROFILE2_PRIVATE = 2;
+  NET_FW_PROFILE2_PUBLIC = 4;
+
 procedure URLLabelOnClick(Sender: TObject);
 var
   ErrorCode: Integer;
@@ -274,3 +271,109 @@ begin
   Result := S;
 end;
 
+procedure AddBasicFirewallException(AppName, FileName: String);
+var
+  FirewallObject: variant;
+  RuleObject: variant;
+begin
+  try
+    FirewallObject := CreateOleObject('HNetCfg.FwMgr');
+    RuleObject := CreateOleObject('HNetCfg.FwAuthorizedApplication');
+    RuleObject.ProcessImageFileName := FileName;
+    RuleObject.Name := AppName;
+    RuleObject.Scope := NET_FW_SCOPE_ALL;
+    RuleObject.IpVersion := NET_FW_IP_VERSION_ANY;
+    RuleObject.Enabled := true;
+    FirewallObject.LocalPolicy.CurrentProfile.AuthorizedApplications.Add(RuleObject);
+  except
+  end;
+end;
+
+procedure AddAdvancedFirewallException(AppName, AppDescription, FileName: String; Protocol: Integer; LocalPorts, RemotePorts: String; Direction: Integer);
+var
+  FirewallObject: variant;
+  RuleObject: variant;
+begin
+  try
+    FirewallObject := CreateOleObject('HNetCfg.FwPolicy2');
+    RuleObject := CreateOleObject('HNetCfg.FWRule');
+    RuleObject.Name := AppName;
+    RuleObject.Description := AppDescription;
+    RuleObject.ApplicationName := FileName;
+    if (Protocol <> NET_FW_IP_PROTOCOL_ALL) then
+      RuleObject.Protocol := Protocol;
+    if (LocalPorts <> '') then
+      RuleObject.LocalPorts := LocalPorts;
+    if (RemotePorts <> '') then
+      RuleObject.RemotePorts := RemotePorts;
+    if (Direction <> NET_FW_RULE_DIR_ALL) then
+      RuleObject.Direction := Direction;
+    RuleObject.Enabled := true;
+    RuleObject.Grouping := 'FlightGear';
+    RuleObject.Profiles := NET_FW_PROFILE2_DOMAIN + NET_FW_PROFILE2_PRIVATE + NET_FW_PROFILE2_PUBLIC;
+    RuleObject.Action := NET_FW_ACTION_ALLOW;
+    RuleObject.RemoteAddresses := '*';
+    FirewallObject.Rules.Add(RuleObject);
+  except
+  end;
+end;
+
+procedure RemoveFirewallException(AppName, FileName: String);
+var
+  FirewallObject: variant;
+  Version: TWindowsVersion;
+begin
+  GetWindowsVersionEx(Version);
+  try
+    if (Version.Major >= 6) then
+      begin
+        FirewallObject := CreateOleObject('HNetCfg.FwPolicy2');
+        FirewallObject.Rules.Remove(AppName);
+      end
+    else if (Version.Major = 5) and (((Version.Minor = 1) and (Version.ServicePackMajor >= 2)) or ((Version.Minor = 2) and (Version.ServicePackMajor >= 1))) then
+      begin
+        FirewallObject := CreateOleObject('HNetCfg.FwMgr');
+        FirewallObject.LocalPolicy.CurrentProfile.AuthorizedApplications.Remove(FileName);
+      end;
+  except
+  end;
+end;
+
+procedure CurStepChanged(CurStep: TSetupStep);
+var
+  Version: TWindowsVersion;
+begin
+  if CurStep = ssPostInstall then
+    begin
+      GetWindowsVersionEx(Version);
+      if (Version.Major >= 6) then
+        begin
+          { IN and OUT rules must be specified separately, otherwise the firewall will create only the IN rule }
+          AddAdvancedFirewallException('FlightGear', 'Allows FlightGear to send and receive data over the multiplayer network and to get METARs.', ExpandConstant('{app}') + '\bin\fgfs.exe', NET_FW_IP_PROTOCOL_ALL, '', '', NET_FW_RULE_DIR_IN);
+          AddAdvancedFirewallException('FlightGear', 'Allows FlightGear to send and receive data over the multiplayer network and to get METARs.', ExpandConstant('{app}') + '\bin\fgfs.exe', NET_FW_IP_PROTOCOL_ALL, '', '', NET_FW_RULE_DIR_OUT);
+          AddAdvancedFirewallException('FlightGear METAR Utility', 'Allows the FlightGear METAR utility to receive METARs.', ExpandConstant('{app}') + '\bin\metar.exe', NET_FW_IP_PROTOCOL_TCP, '', '80', NET_FW_RULE_DIR_OUT);
+          AddAdvancedFirewallException('FlightGear TerraSync', 'Allows TerraSync to download additional scenery while FlightGear is running.', ExpandConstant('{app}') + '\bin\terrasync.exe', NET_FW_IP_PROTOCOL_ALL, '', '', NET_FW_RULE_DIR_IN);
+          AddAdvancedFirewallException('FlightGear TerraSync', 'Allows TerraSync to download additional scenery while FlightGear is running.', ExpandConstant('{app}') + '\bin\terrasync.exe', NET_FW_IP_PROTOCOL_ALL, '', '', NET_FW_RULE_DIR_OUT);
+          AddAdvancedFirewallException('FlightGear FGCom', 'Allows FGCom to establish a connection to FlightGear and the VoIP server for voice ATC communication.', ExpandConstant('{app}') + '\bin\fgcom.exe', NET_FW_IP_PROTOCOL_ALL, '', '', NET_FW_RULE_DIR_IN);
+          AddAdvancedFirewallException('FlightGear FGCom', 'Allows FGCom to establish a connection to FlightGear and the VoIP server for voice ATC communication.', ExpandConstant('{app}') + '\bin\fgcom.exe', NET_FW_IP_PROTOCOL_ALL, '', '', NET_FW_RULE_DIR_OUT);
+        end
+      else if (Version.Major = 5) and (((Version.Minor = 1) and (Version.ServicePackMajor >= 2)) or ((Version.Minor = 2) and (Version.ServicePackMajor >= 1))) then
+        begin
+          { The Windows XP/Server 2003 firewall does not block outgoing connections at all, so only listening processes should be added }
+          AddBasicFirewallException('FlightGear', ExpandConstant('{app}') + '\bin\fgfs.exe');
+          AddBasicFirewallException('FlightGear TerraSync', ExpandConstant('{app}') + '\bin\terrasync.exe');
+          AddBasicFirewallException('FlightGear FGCom', ExpandConstant('{app}') + '\bin\fgcom.exe');
+        end;
+    end;
+end;
+
+procedure CurUninstallStepChanged(CurUninstallStep: TUninstallStep);
+begin
+  if CurUninstallStep = usPostUninstall then
+    begin
+      RemoveFirewallException('FlightGear', ExpandConstant('{app}') + '\bin\fgfs.exe');
+      RemoveFirewallException('FlightGear METAR Utility', ExpandConstant('{app}') + '\bin\metar.exe');
+      RemoveFirewallException('FlightGear TerraSync', ExpandConstant('{app}') + '\bin\terrasync.exe');
+      RemoveFirewallException('FlightGear FGCom', ExpandConstant('{app}') + '\bin\fgcom.exe');
+    end;
+end;

Info.plist.in

@@ -13,7 +13,7 @@
 	<key>CFBundleSignature</key>
 	<string>????</string>
 	<key>CFBundleExecutable</key>
-	<string>FlightGear</string>
+	<string>fgfs</string>
 	<key>CFBundleIdentifier</key>
 	<string>org.flightgear.FlightGear</string>
 	<key>CFBundleVersion</key>
@@ -21,13 +21,20 @@
 	<key>CFBundleShortVersionString</key>
 	<string><%= fgVersion %></string>
 	<key>LSMinimumSystemVersion</key>
-	<string>10.6.0</string>
+	<string>10.7.0</string>
 	<key>NSMainNibFile</key>
 	<string>MainMenu</string>
 	<key>NSPrincipalClass</key>
 	<string>NSApplication</string>
     <key>NSHumanReadableCopyright</key>
     <string>©1996-<%= fgCurrentYear%>, The FlightGear Project</string>
+
+    <!-- when launched via LaunchServices, run the launcher GUI -->
+    <key>LSEnvironment</key>
+    <dict>
+        <key>FG_LAUNCHER</key>
+        <string>1</string>
+    </dict>
 </dict>
 </plist>
 

base-package.rules

@@ -1,38 +0,0 @@
-- .git
-- *.xcf
-- *.tex
-
-+ /fgdata/Aircraft/Generic
-+ /fgdata/Aircraft/Instruments
-+ /fgdata/Aircraft/Instruments-3d
-
-+ /fgdata/Aircraft/UIUC
-+ /fgdata/Aircraft/c172p
-+ /fgdata/Aircraft/777
-+ /fgdata/Aircraft/777-200
-+ /fgdata/Aircraft/b1900d
-+ /fgdata/Aircraft/ufo
-+ /fgdata/Aircraft/CitationX
-+ /fgdata/Aircraft/ZLT-NT
-+ /fgdata/Aircraft/dhc2
-+ /fgdata/Aircraft/Cub
-+ /fgdata/Aircraft/sopwithCamel
-+ /fgdata/Aircraft/f-14b
-+ /fgdata/Aircraft/ASK13
-+ /fgdata/Aircraft/bo105
-+ /fgdata/Aircraft/Dragonfly
-+ /fgdata/Aircraft/SenecaII
-+ /fgdata/Aircraft/A6M2
-
-- /fgdata/Aircraft/*
-
-- /fgdata/Textures.high/*.new
-- /fgdata/Textures.high/*.orig
-- /fgdata/Textures.high/*.save
-- /fgdata/Textures/Unused
-- /fgdata/Textures/*.orig
-- /fgdata/Docs/source
-- /fgdata/Models/Airspace
-- /fgdata/Models/MNUAV
-
-

build_release_linux.sh

@@ -9,13 +9,12 @@ VERSION=`cat flightgear/version`
 
 #####################################################################################
 # remove old and create fresh build directories
-rm -rf sgBuild
-rm -rf fgBuild
+cd $WORKSPACE
 mkdir -p sgBuild
 mkdir -p fgBuild
 mkdir -p output
 rm -rf output/*
-rm -rf $WORKSPACE/dist/include/simgear $WORKSPACE/dist/libSim* $WORKSPACE/dist/libsg*.a
+rm -rf dist/*
 
 #####################################################################################
 echo "Starting on SimGear"
@@ -60,15 +59,5 @@ cp flightgear-*.tar.bz2 ../output/.
 echo "Assembling base package"
 cd $WORKSPACE
 
-echo "Syncing base packages files from sphere.telascience.org"
-
-# a: archive mode
-# z: compress
-# delete: 'delete extraneous files from dest dirs'; avoid bug 1344
-# filter: use the rules in our rules file
-rsync -az --delete \
- --filter 'merge base-package.rules' \
- -e ssh jturner@sphere.telascience.org:/home/jturner/fgdata .
-
 tar cjf output/FlightGear-$VERSION-data.tar.bz2 fgdata/
 

build_release_windows.bat

@@ -0,0 +1,142 @@
+
+IF NOT DEFINED WORKSPACE SET WORKSPACE=%~dp0
+
+REM 32bits
+md build-sg32
+md build-fg32
+md build-fgrun32
+cd build-sg32
+cmake ..\simgear -G "Visual Studio 10" ^
+                 -DMSVC_3RDPARTY_ROOT=%WORKSPACE% ^
+                 -DCMAKE_INSTALL_PREFIX:PATH=%WORKSPACE%/install/msvc100/SimGear ^
+                 -DBOOST_ROOT=%WORKSPACE%/Boost
+cmake --build . --config Release --target INSTALL
+
+cd ..\build-fg32
+cmake ..\flightgear -G "Visual Studio 10" ^
+                    -DMSVC_3RDPARTY_ROOT=%WORKSPACE% ^
+                    -DCMAKE_INSTALL_PREFIX:PATH=%WORKSPACE%/install/msvc100/FlightGear ^
+                    -DCMAKE_PREFIX_PATH=%QT5SDK32% ^
+                    -DPNG_LIBRARY=%WORKSPACE%/3rdParty/lib/libpng16.lib ^
+                    -DFLTK_FLUID_EXECUTABLE=%WORKSPACE%/3rdParty/bin/fluid.exe ^
+                    -DBOOST_ROOT=%WORKSPACE%/Boost
+cmake --build . --config Release --target INSTALL
+
+cd ..\build-fgrun32
+cmake ..\fgrun -G "Visual Studio 10" ^
+               -DMSVC_3RDPARTY_ROOT:PATH=%WORKSPACE% ^
+               -DCMAKE_INSTALL_PREFIX:PATH=%WORKSPACE%/install/msvc100/FGRun ^
+               -DFLTK_FLUID_EXECUTABLE:FILEPATH=%WORKSPACE%/3rdParty/bin/fluid.exe ^
+               -DGETTEXT_MSGFMT_EXECUTABLE:FILEPATH=%WORKSPACE%/3rdParty/bin/msgfmt.exe ^
+               -DGETTEXT_MSGMERGE_EXECUTABLE:FILEPATH=%WORKSPACE%/3rdParty/bin/msgmerge.exe ^
+               -DBOOST_ROOT=%WORKSPACE%/Boost
+cmake --build . --config Release --target INSTALL
+
+cd ..
+
+REM 64 bits
+md build-sg64
+md build-fg64
+md build-fgrun64
+cd build-sg64
+cmake ..\SimGear -G "Visual Studio 10 Win64" ^
+                 -DMSVC_3RDPARTY_ROOT=%WORKSPACE% ^
+                 -DCMAKE_INSTALL_PREFIX:PATH=%WORKSPACE%/install/msvc100-64/SimGear ^
+                 -DBOOST_ROOT=%WORKSPACE%/Boost
+cmake --build . --config Release --target INSTALL
+
+cd ..\build-fg64
+cmake ..\flightgear -G "Visual Studio 10 Win64" ^
+                    -DMSVC_3RDPARTY_ROOT=%WORKSPACE% ^
+                    -DCMAKE_INSTALL_PREFIX:PATH=%WORKSPACE%/install/msvc100-64/FlightGear ^
+                    -DCMAKE_PREFIX_PATH=%QT5SDK64% ^
+                    -DFLTK_FLUID_EXECUTABLE=%WORKSPACE%/3rdParty/bin/fluid.exe ^
+                    -DBOOST_ROOT=%WORKSPACE%/Boost ^
+                    -DWITH_FGPANEL=OFF ^
+                    -DENABLE_PROFILE=OFF
+cmake --build . --config Release --target INSTALL
+
+cd ..\build-fgrun64
+cmake ..\fgrun -G "Visual Studio 10 Win64" ^
+               -DMSVC_3RDPARTY_ROOT:PATH=%WORKSPACE% ^
+               -DCMAKE_INSTALL_PREFIX:PATH=%WORKSPACE%/install/msvc100-64/FGRun ^
+               -DFLTK_FLUID_EXECUTABLE:FILEPATH=%WORKSPACE%/3rdParty/bin/fluid.exe ^
+               -DGETTEXT_MSGFMT_EXECUTABLE:FILEPATH=%WORKSPACE%/3rdParty/bin/msgfmt.exe ^
+               -DGETTEXT_MSGMERGE_EXECUTABLE:FILEPATH=%WORKSPACE%/3rdParty/bin/msgmerge.exe ^
+               -DBOOST_ROOT=%WORKSPACE%/Boost
+cmake --build . --config Release --target INSTALL
+
+cd ..
+
+REM Qt5 deployment
+%QT5SDK32%\bin\windeployqt --release --list target %WORKSPACE%/install/msvc100/FlightGear/bin/fgfs.exe
+%QT5SDK64%\bin\windeployqt --release --list target %WORKSPACE%/install/msvc100-64/FlightGear/bin/fgfs.exe
+
+REM build setup
+ECHO Packaging root is %WORKSPACE%
+
+subst X: /D
+subst X: %WORKSPACE%.
+
+REM ensure output dir is clean since we upload the entirety of it
+rmdir /S /Q output
+
+REM archiving PDB files
+copy %WORKSPACE%\build-fg32\src\Main\RelWithDebInfo\fgfs.pdb %WORKSPACE%\Output\fgfs-x86-%BUILD_NUMBER%.pdb
+copy %WORKSPACE%\build-fg64\src\Main\RelWithDebInfo\fgfs.pdb %WORKSPACE%\Output\fgfs-x64-%BUILD_NUMBER%.pdb
+
+REM indirect way to get command output into an environment variable
+set PATH=%WORKSPACE%\install\msvc100\OpenSceneGraph\bin;%PATH%
+osgversion --so-number > %TEMP%\osg-so-number.txt
+osgversion --version-number > %TEMP%\osg-version.txt
+osgversion --openthreads-soversion-number > %TEMP%\openthreads-so-number.txt
+
+SET /P FLIGHTGEAR_VERSION=<flightgear\version
+SET /P OSG_VERSION=<%TEMP%\osg-version.txt
+SET /P OSG_SO_NUMBER=<%TEMP%\osg-so-number.txt
+SET /P OT_SO_NUMBER=<%TEMP%\openthreads-so-number.txt
+
+IF %IS_NIGHTLY_BUILD% EQU 1 (
+  REM FlightGear nightly: with fgdata, output filename would be "FlightGear-x.x.x-nightly-full.exe"
+  CALL :writeBaseConfig
+  CALL :writeNightlyFullConfig
+  iscc FlightGear.iss
+
+  REM FlightGear nightly: without fgdata, output filename would be "FlightGear-x.x.x-nightly.exe"
+  CALL :writeBaseConfig
+  CALL :writeNightlyDietConfig
+  iscc FlightGear.iss
+) ELSE (
+  REM FlightGear release: with fgdata, output filename would be "FlightGear-x.x.x.exe"
+  CALL :writeBaseConfig
+  CALL :writeReleaseConfig
+  iscc FlightGear.iss
+)
+GOTO End
+
+:writeBaseConfig
+ECHO #define FGVersion "%FLIGHTGEAR_VERSION%" > InstallConfig.iss
+ECHO #define OSGVersion "%OSG_VERSION%" >> InstallConfig.iss
+ECHO #define OSGSoNumber "%OSG_SO_NUMBER%" >> InstallConfig.iss
+ECHO #define OTSoNumber "%OT_SO_NUMBER%" >> InstallConfig.iss
+GOTO End
+
+:writeReleaseConfig
+CALL :writeBaseConfig
+ECHO #define FGDetails "" >> InstallConfig.iss
+ECHO #define IncludeData "TRUE" >> InstallConfig.iss
+GOTO End
+
+:writeNightlyFullConfig
+CALL :writeBaseConfig
+ECHO #define FGDetails "-nightly-full" >> InstallConfig.iss
+ECHO #define IncludeData "TRUE" >> InstallConfig.iss
+GOTO End
+
+:writeNightlyDietConfig
+CALL :writeBaseConfig
+ECHO #define FGDetails "-nightly" >> InstallConfig.iss
+ECHO #define IncludeData "FALSE" >> InstallConfig.iss
+GOTO End
+
+:End

catalogFilenames.py

@@ -1,454 +0,0 @@
-aircraft = {
-"14bis":"14bis_20120506.zip",
-"21":"21_20101206.zip",
-"707":"707_20140103.zip",
-"717":"717_20140116.zip",
-"727-230":"727-230_20120714.zip",
-"737-100":"737-100_20120506.zip",
-"737-200":"737-200_20120506.zip",
-"737-300":"737-300_20140116.zip",
-"737NG":"737NG_20140116.zip",
-"747-200":"747-200_2.1.zip",
-"747-400":"747-400_20140216.zip",
-"747":"747_20070714.zip",
-"757-200":"757-200_20140116.zip",
-"767-300":"767-300_20100100.zip",
-"777-200":"777-200_20111215.zip",
-"777":"777_20140216.zip",
-"787":"787_20140116.zip",
-"A-10":"A-10_20140116.zip",
-"A-26-Invader":"A-26-Invader_20130214.zip",
-"A-6E":"A-6E_20131204.zip",
-"A24-Viking":"A24-Viking_20120506.zip",
-"A300":"A300_20120717.zip",
-"A320-family":"A320-family_20140116.zip",
-"A340-600":"A340-600_20080400.zip",
-"A380":"A380_1.2.zip",
-"A6M2":"A6M2_20120717.zip",
-"AG-14":"AG-14_20130810.zip",
-"AN-225":"AN-225_20130823.zip",
-"ANT-20":"ANT-20_20120506.zip",
-"AR-234":"AR-234_20120821.zip",
-"ASK13":"ASK13_20100401.zip",
-"ASK21-MI":"ASK21-MI_20120506.zip",
-"ASK21":"ASK21_20070701.zip",
-"ATC-ML":"ATC-ML_20131014.zip",
-"ATC":"ATC_20120717.zip",
-"ATI-Resolution":"ATI-Resolution_0.0.zip",
-"ATR-72-500":"ATR-72-500_20120717.zip",
-"AVRO-IV-Triplane":"AVRO-IV-Triplane_20130805.zip",
-"Aero-Commander":"Aero-Commander_20130313.zip",
-"Aerocar":"Aerocar_20130805.zip",
-"Aerostar-700":"Aerostar-700_20120610.zip",
-"Aichi-D3A":"Aichi-D3A_20120506.zip",
-"AirCrane":"AirCrane_20131204.zip",
-"Airco-DH2":"Airco-DH2_20120506.zip",
-"Albatross":"Albatross_20130823.zip",
-"Allegro-2000":"Allegro-2000_20120506.zip",
-"Alouette-III":"Alouette-III_20131117.zip",
-"Alouette-II":"Alouette-II_20120506.zip",
-"Alphajet":"Alphajet_20130225.zip",
-"Antoinette":"Antoinette_20130220.zip",
-"Antonov-An-12":"Antonov-An-12_20131220.zip",
-"Arsenal-VG33":"Arsenal-VG33_0.1.zip",
-"Arup-S2":"Arup-S2_20120506.zip",
-"Avro-Arrow":"Avro-Arrow_20120506.zip",
-"Avro-Lancaster":"Avro-Lancaster_20120817.zip",
-"B-17":"B-17_20130706.zip",
-"B-1B":"B-1B_20130823.zip",
-"B-24-Liberator":"B-24-Liberator_20131227.zip",
-"B-25":"B-25_20130810.zip",
-"B-2":"B-2_20130823.zip",
-"B-36D-Peacemaker":"B-36D-Peacemaker_20120506.zip",
-"B-52F":"B-52F_20130823.zip",
-"BAC-TSR2":"BAC-TSR2_20120717.zip",
-"BAe-125":"BAe-125_20120305.zip",
-"BV-141":"BV-141_20131015.zip",
-"BV-170":"BV-170_20130411.zip",
-"Beechcraft-Staggerwing":"Beechcraft-Staggerwing_20130303.zip",
-"Bell-222X":"Bell-222X_20120508.zip",
-"Bell-P-39":"Bell-P-39_20120821.zip",
-"Bell-P-59":"Bell-P-59_20120817.zip",
-"Bell-X1":"Bell-X1_20120114.zip",
-"Bernard-HV220":"Bernard-HV220_20090506.zip",
-"Bleriot-SPAD-S.510":"Bleriot-SPAD-S.510_20131226.zip",
-"Boeing-247":"Boeing-247_20130903.zip",
-"Boeing314":"Boeing314_1.6.zip",
-"Bombardier-415":"Bombardier-415_20131203.zip",
-"Br-761":"Br-761_20130123.zip",
-"Brabazon":"Brabazon_20120821.zip",
-"Breda-B.Z.308":"Breda-B.Z.308_20130102.zip",
-"Breguet-XIX":"Breguet-XIX_20100829.zip",
-"Buccaneer":"Buccaneer_0.1.zip",
-"Bugatti":"Bugatti_20120524.zip",
-"Burnelli-CBY-3":"Burnelli-CBY-3_20120506.zip",
-"C-160-Transall":"C-160-Transall_20120905.zip",
-"C-2A":"C-2A_20120506.zip",
-"C130":"C130_20130810.zip",
-"C460":"C460_20130614.zip",
-"C561":"C561_20130614.zip",
-"C684":"C684_20131025.zip",
-"CRJ-200":"CRJ-200_2.zip",
-"CRJ700-family":"CRJ700-family_1.1.0.zip",
-"Campini-Caproni-N1":"Campini-Caproni-N1_20130115.zip",
-"CanberraBI8":"CanberraBI8_20101204.zip",
-"Cap10B":"Cap10B_20130824.zip",
-"Cap10C":"Cap10C_20131202.zip",
-"Caravelle":"Caravelle_20120506.zip",
-"Carreidas":"Carreidas_20131025.zip",
-"Caudron-G3":"Caudron-G3_20120506.zip",
-"Caudron-G4":"Caudron-G4_20120506.zip",
-"Cessna-208-Caravan":"Cessna-208-Caravan_20130301.zip",
-"Cessna-421-Golden-Eagle":"Cessna-421-Golden-Eagle_20120506.zip",
-"Cessna337":"Cessna337_v20120429.zip",
-"Citation-Bravo":"Citation-Bravo_20131025.zip",
-"CitationX":"CitationX_20131025.zip",
-"Citation":"Citation_20130305.zip",
-"Coanda-1910":"Coanda-1910_20130129.zip",
-"Commonwealth-Ca-12":"Commonwealth-Ca-12_20120817.zip",
-"ComperSwift":"ComperSwift_20120717.zip",
-"Concorde":"Concorde_2.11.zip",
-"Convair-XFY-1-Pogo":"Convair-XFY-1-Pogo_20120506.zip",
-"Cub":"Cub_20120508.zip",
-"Curtiss-Jenny":"Curtiss-Jenny_20120906.zip",
-"Curtiss-P40":"Curtiss-P40_20130221.zip",
-"D510":"D510_20130916.zip",
-"D520":"D520_20120506.zip",
-"DC-10-30":"DC-10-30_20140116.zip",
-"DG-101G":"DG-101G_2013-01-01.zip",
-"DH-106":"DH-106_20120506.zip",
-"DH-88":"DH-88_20131026.zip",
-"DH-89":"DH-89_20120905.zip",
-"DH-91":"DH-91_20120506.zip",
-"DO-228":"DO-228_20130824.zip",
-"DO-26":"DO-26_20100521.zip",
-"DO-335":"DO-335_20130216.zip",
-"DO-X":"DO-X_0.2.0.zip",
-"DR400-dauphin":"DR400-dauphin_20140105.zip",
-"DR400":"DR400_20131129.zip",
-"Dauphin":"Dauphin_20140109.zip",
-"Deperdussin":"Deperdussin_20101124.zip",
-"DerKleineUhu":"DerKleineUhu_0.1.zip",
-"Deuche":"Deuche_20120506.zip",
-"Diamond-Da40":"Diamond-Da40_20130326.zip",
-"Diamond-Da42":"Diamond-Da42_20120506.zip",
-"Douglas-Dc3":"Douglas-Dc3_20130107.zip",
-"Douglas-Dolphin":"Douglas-Dolphin_20131029.zip",
-"Dragonfly":"Dragonfly_1.0.zip",
-"Dromader":"Dromader_31-02-2011.zip",
-"E-2C":"E-2C_20131014.zip",
-"E3B":"E3B_20130313.zip",
-"Etrich-Taube":"Etrich-Taube_20120207.zip",
-"F-106-dart":"F-106-dart_20130808.zip",
-"F-117":"F-117_20130317.zip",
-"F-35B":"F-35B_20120717.zip",
-"F-86":"F-86_20131216.zip",
-"F-8E-Crusader":"F-8E-Crusader_20130810.zip",
-"F4U":"F4U_20120717.zip",
-"F6F-Hellcat":"F6F-Hellcat_20120821.zip",
-"F7F-Tigercat":"F7F-Tigercat_20130107.zip",
-"F80C":"F80C_20120717.zip",
-"FK9MK2":"FK9MK2_20120506.zip",
-"Fairchild-C119":"Fairchild-C119_20120817.zip",
-"Fairchild-Metroliner":"Fairchild-Metroliner_20110321.zip",
-"Fairey-Gannet":"Fairey-Gannet_20131122.zip",
-"Falcon-50":"Falcon-50_20140116.zip",
-"Farman-III":"Farman-III_20131014.zip",
-"Farman-IV":"Farman-IV_20130213.zip",
-"Focke-Wulf-F19-ente":"Focke-Wulf-F19-ente_20130103.zip",
-"Fokker-Eindecker-EIII":"Fokker-Eindecker-EIII_20110301.zip",
-"Fokker-S-11":"Fokker-S-11_20120506.zip",
-"Fokker-Spin":"Fokker-Spin_20130805.zip",
-"Ford-Trimotor":"Ford-Trimotor_20120506.zip",
-"Fouga-Magister":"Fouga-Magister_20120501.zip",
-"Fw200":"Fw200_20120506.zip",
-"Fw61":"Fw61_20091025.zip",
-"G-164":"G-164_20131014.zip",
-"Gee-Bee":"Gee-Bee_20130711.zip",
-"Gloster-Meteor":"Gloster-Meteor_20120821.zip",
-"Gloster-Whittle":"Gloster-Whittle_20120821.zip",
-"Gotha-G-V":"Gotha-G-V_20131007.zip",
-"Grob-Astir":"Grob-Astir_20131209.zip",
-"Grob-G109":"Grob-G109_20131104.zip",
-"Grob-G115":"Grob-G115_20131209.zip",
-"Grob-Gf200":"Grob-Gf200_20140102.zip",
-"H-21C-Shawnee":"H-21C-Shawnee_20120717.zip",
-"H1-Racer":"H1-Racer_20130531.zip",
-"H4-Hercules":"H4-Hercules_1.3.zip",
-"HM-14":"HM-14_20120506.zip",
-"HS-P-75":"HS-P-75_20130912.zip",
-"HUP-Retriever":"HUP-Retriever_20120717.zip",
-"Handley-Page-Halifax":"Handley-Page-Halifax_20120506.zip",
-"Hansajet":"Hansajet_0.1.zip",
-"Hawker-Siddeley-Nimrod":"Hawker-Siddeley-Nimrod_20120506.zip",
-"He-111":"He-111_20131022.zip",
-"He162":"He162_20131022.zip",
-"Heinkel-He-111":"Heinkel-He-111_20120817.zip",
-"Heinkel-He-177":"Heinkel-He-177_20130805.zip",
-"Heinkel-He-178":"Heinkel-He-178_20101128.zip",
-"Heinkel-He-280":"Heinkel-He-280_20120825.zip",
-"HondaJet":"HondaJet_20130620.zip",
-"Hornet":"Hornet_20101110.zip",
-"Horsa":"Horsa_20131022.zip",
-"Horten-Ho-IX":"Horten-Ho-IX_20130208.zip",
-"Hughes-XF11":"Hughes-XF11_20120506.zip",
-"Hunter":"Hunter_20131025.zip",
-"Hurricane":"Hurricane_20130823.zip",
-"IAR80":"IAR80_20120912.zip",
-"IL-2":"IL-2_20110311.zip",
-"JA37":"JA37_1.21.zip",
-"JAS39-Gripen":"JAS39-Gripen_20120506.zip",
-"JT-5B":"JT-5B_20130330.zip",
-"Jaguar":"Jaguar_4.81.zip",
-"Ju-87":"Ju-87_20130712.zip",
-"Ju-88":"Ju-88_20120717.zip",
-"Junkers-F13":"Junkers-F13_20130516.zip",
-"Junkers-G.38ce":"Junkers-G.38ce_20120506.zip",
-"Junkers-W34":"Junkers-W34_20130713.zip",
-"K-7":"K-7_20120506.zip",
-"K5Y1":"K5Y1_0.1.zip",
-"KC135":"KC135_20131022.zip",
-"KM":"KM_20100603.zip",
-"Ka-50":"Ka-50_20130418.zip",
-"Katana":"Katana_20130322.zip",
-"Ki-84":"Ki-84_20120717.zip",
-"L-1011-500":"L-1011-500_20120506.zip",
-"LATE-29":"LATE-29_20120506.zip",
-"La-5":"La-5_20120506.zip",
-"Lancair-235":"Lancair-235_20130810.zip",
-"Late-631":"Late-631_20120829.zip",
-"Leduc-022":"Leduc-022_20120821.zip",
-"Lightning":"Lightning_20140116.zip",
-"Lionceau":"Lionceau_20130512.zip",
-"Lockheed-Martin-FA-22A-Raptor":"Lockheed-Martin-FA-22A-Raptor_20131022.zip",
-"Lockheed-NF104A":"Lockheed-NF104A_20110316.zip",
-"Lockheed-NF104":"Lockheed-NF104_20110604.zip",
-"Lockheed-P38":"Lockheed-P38_20131202.zip",
-"Lockheed-SR71":"Lockheed-SR71_20120507.zip",
-"Lockheed-U-2S":"Lockheed-U-2S_20131104.zip",
-"Lockheed-Vega":"Lockheed-Vega_20100324.zip",
-"Lockheed1049":"Lockheed1049_1.6.zip",
-"Lockheed1049h":"Lockheed1049h_1.0.zip",
-"Long-EZ":"Long-EZ_20120506.zip",
-"Lynx-WG13":"Lynx-WG13_20120717.zip",
-"M-XE":"M-XE_20120506.zip",
-"MB326":"MB326_20131022.zip",
-"MD11":"MD11_20121126.zip",
-"ME-209-V1":"ME-209-V1_20130416.zip",
-"ME-262HGIII":"ME-262HGIII_20120506.zip",
-"ME-262":"ME-262_20120821.zip",
-"MPCarrier":"MPCarrier_0.1.zip",
-"MRJ":"MRJ_20090817.zip",
-"MS-406":"MS-406_20130806.zip",
-"Macchi-Castoldi-MC72":"Macchi-Castoldi-MC72_20131016.zip",
-"Macchi-M33":"Macchi-M33_20120506.zip",
-"Martin-Baker-MB5":"Martin-Baker-MB5_20120813.zip",
-"Messerschmitt-BF110":"Messerschmitt-BF110_20130329.zip",
-"Messerschmitt-Libelle":"Messerschmitt-Libelle_20130511.zip",
-"Messerschmitt-P1101":"Messerschmitt-P1101_20130315.zip",
-"MiG-15":"MiG-15_20121210.zip",
-"MiG-21bis":"MiG-21bis_20120717.zip",
-"Mig-29":"Mig-29_20140116.zip",
-"Mil-Mi-12":"Mil-Mi-12_20120216.zip",
-"Mil-Mi-24":"Mil-Mi-24_20140116.zip",
-"Mirage-2000":"Mirage-2000_20131025.zip",
-"MirageIII":"MirageIII_20131008.zip",
-"MirageIV":"MirageIV_20131008.zip",
-"Mirage_F1":"Mirage_F1_20131022.zip",
-"Nakajima-B5N":"Nakajima-B5N_20120506.zip",
-"Nieuport-11":"Nieuport-11_20130729.zip",
-"Noratlas":"Noratlas_20120717.zip",
-"Nord-2502":"Nord-2502_20130605.zip",
-"Nordstern":"Nordstern_20120813.zip",
-"North-American-T28D-Trojan":"North-American-T28D-Trojan_20140113.zip",
-"North-American-T6-Texan":"North-American-T6-Texan_20130915.zip",
-"Northrop-P61":"Northrop-P61_20130107.zip",
-"Northrop-xb35":"Northrop-xb35_20130823.zip",
-"OH-1":"OH-1_0.1.zip",
-"OV10":"OV10_20140116.zip",
-"P-38-Lightning":"P-38-Lightning_20120717.zip",
-"P180":"P180_20131022.zip",
-"P47":"P47_20131022.zip",
-"PBY-Catalina":"PBY-Catalina_20140107.zip",
-"PC-12":"PC-12_20121208.zip",
-"PC-21":"PC-21_20130810.zip",
-"PC-6":"PC-6_20130810.zip",
-"PC-9M":"PC-9M_20131025.zip",
-"PaperAirplane":"PaperAirplane_20101229.zip",
-"Percival-Mew-Gull":"Percival-Mew-Gull_20130919.zip",
-"Piaggio-Pegna-Pc-7":"Piaggio-Pegna-Pc-7_20131018.zip",
-"Pioneer-200":"Pioneer-200_20120506.zip",
-"Piper-PA-18":"Piper-PA-18_20120506.zip",
-"Piper-PA-32":"Piper-PA-32_20120506.zip",
-"Polikarpov-I16":"Polikarpov-I16_20120506.zip",
-"Pond-Racer":"Pond-Racer_20120506.zip",
-"Potez-630":"Potez-630_20100313.zip",
-"Pterodactyl":"Pterodactyl_20100502.zip",
-"Quickie":"Quickie_20120506.zip",
-"R22":"R22_20120903.zip",
-"R44":"R44_20130426.zip",
-"RAF-S-E-5":"RAF-S-E-5_20120506.zip",
-"RV-6A":"RV-6A_0.1.zip",
-"Rascal":"Rascal_0.1.zip",
-"Ryan-Navion":"Ryan-Navion_20130103.zip",
-"Ryan-PT22":"Ryan-PT22_20110503.zip",
-"Ryan-SoSL":"Ryan-SoSL_20120325.zip",
-"S-51-Dragonfly":"S-51-Dragonfly_20120717.zip",
-"SM-55":"SM-55_20120506.zip",
-"SPAD-VII":"SPAD-VII_20120506.zip",
-"SR71-BlackBird":"SR71-BlackBird_20110614.zip",
-"SU-37":"SU-37_20130823.zip",
-"SaabJ35Draken":"SaabJ35Draken_20071217.zip",
-"SaabJ35F":"SaabJ35F_0.9.1.zip",
-"Saunders-Roe-Princess":"Saunders-Roe-Princess_1.3.zip",
-"SeaVixen":"SeaVixen_20130524.zip",
-"SenecaII":"SenecaII_1.0.zip",
-"Short-Stirling":"Short-Stirling_20120506.zip",
-"Short_Empire":"Short_Empire_20140113.zip",
-"Sikorsky-76C":"Sikorsky-76C_20120723.zip",
-"Sikorsky-S38":"Sikorsky-S38_20120506.zip",
-"Sikorsky-S58":"Sikorsky-S58_20131213.zip",
-"Skyranger":"Skyranger_20131205.zip",
-"Skyvan":"Skyvan_20131127.zip",
-"Socata-ST10":"Socata-ST10_20130107.zip",
-"SpaceShip-One":"SpaceShip-One_20120506.zip",
-"Spitfire":"Spitfire_20130310.zip",
-"Stampe-SV4":"Stampe-SV4_20121113.zip",
-"Starship":"Starship_20130923.zip",
-"Stearman":"Stearman_20130817.zip",
-"Stieglitz":"Stieglitz_20091025.zip",
-"Stiletto":"Stiletto_20110917.zip",
-"Storch":"Storch_20130313.zip",
-"Submarine_Scout":"Submarine_Scout_20130115.zip",
-"Super-Etendard":"Super-Etendard_20131008.zip",
-"Super-Frelon":"Super-Frelon_20120506.zip",
-"Supermarine-S.6B":"Supermarine-S.6B_20130312.zip",
-"Superwal":"Superwal_20100330.zip",
-"T-4":"T-4_20130620.zip",
-"T-50":"T-50_20120717.zip",
-"T37":"T37_20101230.zip",
-"T38":"T38_20110911.zip",
-"TBM-Avenger":"TBM-Avenger_20120506.zip",
-"TU-114":"TU-114_20120717.zip",
-"TU-95":"TU-95_20120506.zip",
-"Tecnam-P92":"Tecnam-P92_20120506.zip",
-"Tigre":"Tigre_20130823.zip",
-"Tu-134":"Tu-134_20131220.zip",
-"UH-1":"UH-1_20130810.zip",
-"UH-60":"UH-60_20120718.zip",
-"V22-Osprey":"V22-Osprey_0.2.zip",
-"VMX22-Osprey":"VMX22-Osprey_0.59-28.08.2010.zip",
-"Velocity-XL":"Velocity-XL_20120506.zip",
-"Vickers-Vimy":"Vickers-Vimy_20120506.zip",
-"Vostok-1":"Vostok-1_20120717.zip",
-"Westland-Whirlwind":"Westland-Whirlwind_20130930.zip",
-"X15":"X15_20101128.zip",
-"XB-70":"XB-70_20120730.zip",
-"YF-23":"YF-23_20120717.zip",
-"YS-11":"YS-11_0.1.zip",
-"Yak-18T":"Yak-18T_20121126.zip",
-"Yak-36":"Yak-36_20121225.zip",
-"YardStik":"YardStik_0.0.zip",
-"ZF_Navy_free_balloon":"ZF_Navy_free_balloon_20130107.zip",
-"ZLT-NT":"ZLT-NT_20130923.zip",
-"ZivkoEdge":"ZivkoEdge_0.2.zip",
-"Zlin-50lx":"Zlin-50lx_20131125.zip",
-"a4":"a4_20130706.zip",
-"airwaveXtreme150":"airwaveXtreme150_July-2013.zip",
-"an2":"an2_20131014.zip",
-"apache":"apache_20110119.zip",
-"as332":"as332_20091217.zip",
-"asw20":"asw20_July-2013.zip",
-"b1900d":"b1900d_20130911.zip",
-"b26":"b26_20130620.zip",
-"b29":"b29_20130401.zip",
-"beaufighter":"beaufighter_20120717.zip",
-"beech99":"beech99_20081221.zip",
-"bf109":"bf109_20130615.zip",
-"bleriot-XI":"bleriot-XI_20130715.zip",
-"bluebird":"bluebird_10.4.zip",
-"bo105":"bo105_20140116.zip",
-"bocian":"bocian_20131104.zip",
-"c150":"c150_20140116.zip",
-"c172p":"c172p_20140112.zip",
-"c172r":"c172r_20130209.zip",
-"c182":"c182_20130209.zip",
-"c182rg":"c182rg_20130209.zip",
-"c310":"c310_20130209.zip",
-"c310u3a":"c310u3a_20130209.zip",
-"ch47":"ch47_0.1.zip",
-"ch53e":"ch53e_0.1.zip",
-"colditz":"colditz_20090126.zip",
-"couzinet70":"couzinet70_20120506.zip",
-"cri-cri":"cri-cri_20080206.zip",
-"dc2":"dc2_20130209.zip",
-"dc3":"dc3_20110905.zip",
-"dc6":"dc6_20131014.zip",
-"dc8-63":"dc8-63_0.02.zip",
-"dc8-73":"dc8-73_0.1.zip",
-"dhc1":"dhc1_20120506.zip",
-"dhc2":"dhc2_20130315.zip",
-"dhc3":"dhc3_20120506.zip",
-"dhc4":"dhc4_20101230.zip",
-"dhc6":"dhc6_20130202.zip",
-"dhc8":"dhc8_20120714.zip",
-"eastbourne_mono":"eastbourne_mono_20130805.zip",
-"ec130":"ec130_20140117.zip",
-"ec135":"ec135_0.6.zip",
-"ercoupe":"ercoupe_20130313.zip",
-"eurofighter":"eurofighter_1.4.zip",
-"f-14b":"f-14b_20121124.zip",
-"f15c":"f15c_20101022.zip",
-"f16":"f16_20120812.zip",
-"f18":"f18_20101026.zip",
-"fa223":"fa223_20120717.zip",
-"fkdr1":"fkdr1_20130313.zip",
-"flash2a":"flash2a_20130313.zip",
-"fokker100":"fokker100_20140116.zip",
-"fokker50":"fokker50_20130810.zip",
-"followme":"followme_20110613.zip",
-"fw190":"fw190_20120717.zip",
-"harrier":"harrier_20130823.zip",
-"j22":"j22_20101022.zip",
-"j7w":"j7w_20120717.zip",
-"jeep":"jeep_20120717.zip",
-"jetman":"jetman_20110613.zip",
-"ju52":"ju52_20131204.zip",
-"kodiak":"kodiak_20101223.zip",
-"l39":"l39_20100310.zip",
-"marchetti":"marchetti_20050614.zip",
-"me163":"me163_20131022.zip",
-"me323":"me323_20131022.zip",
-"mirage2000":"mirage2000_20101024.zip",
-"mosquito":"mosquito_20130620.zip",
-"ogel":"ogel_20130209.zip",
-"ornithopter":"ornithopter_20070710.zip",
-"p51d":"p51d_20140111.zip",
-"pa22":"pa22_20090703.zip",
-"pa24-250":"pa24-250_20121128.zip",
-"pa28-161":"pa28-161_20120717.zip",
-"paraglider":"paraglider_20130620.zip",
-"payen-pa100":"payen-pa100_20101117.zip",
-"pittss1c":"pittss1c_20120717.zip",
-"pushback":"pushback_20120717.zip",
-"rallye-MS893":"rallye-MS893_20131016.zip",
-"santa":"santa_20130216.zip",
-"seahawk":"seahawk_20131025.zip",
-"sf25b":"sf25b_20120222.zip",
-"sgs233":"sgs233_20120717.zip",
-"shuttle":"shuttle_20080808.zip",
-"sm79":"sm79_20131022.zip",
-"snowplow":"snowplow_1.0.zip",
-"sopwithCamel":"sopwithCamel_20131008.zip",
-"spitfireIX":"spitfireIX_20120717.zip",
-"sr20":"sr20_20120717.zip",
-"superguppySGT":"superguppySGT_20110613.zip",
-"tu154":"tu154_20081221.zip",
-"tu154b":"tu154b_20130805.zip",
-"ufo":"ufo_20140125.zip",
-"victor":"victor_20140116.zip",
-"vulcanb2":"vulcanb2_20140116.zip",
-"wrightFlyer1903":"wrightFlyer1903_20070710.zip",
-"x24b":"x24b_20090126.zip",
-"yak3":"yak3_20131022.zip"
-}

catalogTags.py

@@ -1,12 +1,13 @@
 aircraftTypeTags = [
-    "ga", "fighter", "helicopter", "glider", "spaceship", "bomber", "groundvehicle", 
-    "tanker", "cargo", "passenger", "bizjet", "trainer", "airship", "balloon"
+    "ga", "fighter", "helicopter", "glider", "spaceship", "bomber", "groundvehicle",
+    "tanker", "cargo", "transport", "bizjet", "trainer", "airship", "balloon"
 ]
 
 manufacturerTags = [
-    "boeing", "cessna", "diamond", "douglas", "bell", "piper", 
+    "boeing", "cessna", "diamond", "douglas", "bell", "piper",
     "airbus", "vickers", "lockheed", "fokker",
-    "embrarer", "bombardier", "pilatus"
+    "embraer", "bombardier", "pilatus", "robin",
+    "eurocopter"
 ]
 
 eraTags = [
@@ -21,6 +22,8 @@ eraTags = [
     "1960s",
     "1970s",
     "1980s",
+    "1990s",
+    "2000s",
     "gulfwar1",
     "gulfwar2"
 ]
@@ -52,7 +55,8 @@ featureTags = [
     "etops",
     "floats",
     "amphibious",
-    "supersonic"
+    "airship",
+    "aerobatic"
 ]
 
 propulsionTags = [
@@ -62,12 +66,15 @@ propulsionTags = [
     "supercharged",
     "turboprop",
     "jet", "afterburner", "rocket",
-    "electric"
+    "electric",
+    "twin-engine",
+    "single-engine"
 ]
 
 simFeatureTags = [
     "tow",
-    "dual-controls"
+    "dual-controls",
+    "rembrandt"
 ]
 
 tags = aircraftTypeTags + manufacturerTags + eraTags + simFeatureTags + propulsionTags + featureTags

create_catalog.py

@@ -1,134 +0,0 @@
-#!/usr/bin/python
-
-import os, sys, re
-import urllib
-import hashlib # for MD5
-
-import catalogFilenames
-import catalogTags
-import sgprops
-
-fgRoot = sys.argv[1]
-aircraftDir = os.path.join(fgRoot, 'Aircraft')
-
-catalogProps = sgprops.Node()
-catalogProps.addChild('version').value = '3.1.0'
-catalogProps.addChild('id').value = 'org.flightgear.default'
-catalogProps.addChild('license').value = 'GPL'
-catalogProps.addChild('url').value = "http://fgfs.goneabitbursar.com/pkg/3.1.0/default-catalog.xml"
-
-catalogProps.addChild('description').value = "Aircraft developed and maintained by the FlightGear project"
-
-de = catalogProps.addChild('de')
-# de.addChild('description').value = "<German translation of catalog description>"
-
-fr = catalogProps.addChild('fr')
-
-urls = [
-        "http://flightgear.wo0t.de/",
-        "http://ftp.icm.edu.pl/packages/flightgear/",
-        "http://mirrors.ibiblio.org/pub/mirrors/flightgear/ftp/",
-        "http://ftp.igh.cnrs.fr/pub/flightgear/ftp/",
-        "http://ftp.linux.kiev.ua/pub/fgfs/",
-        "http://fgfs.physra.net/ftp/"
-]
-
-thumbs = [
-    "http://www.flightgear.org/thumbs/v3.0/{acft}.jpg"
-]
-    
-standardTagSet = frozenset(catalogTags.tags)
-def isNonstandardTag(t):
-    return t not in standardTagSet
-    
-# create the download cache dir if require
-
-cacheDir = '.catalog_cache'
-if not os.path.isdir(cacheDir):
-    print "Creating catalog cache dir"
-    os.mkdir(cacheDir)
-    
-    
-for d in os.listdir(aircraftDir):
-    acftDirPath = os.path.join(aircraftDir, d)
-    if not os.path.isdir(acftDirPath):
-        continue
-        
-    setFilePath = None
-
-    # find the first set file 
-    # FIXME - way to designate the primary file
-    for f in os.listdir(acftDirPath):
-        if f.endswith("-set.xml"):
-            setFilePath = os.path.join(acftDirPath, f)
-            break
-            
-    if setFilePath is None:
-        print "No -set.xml file found in",acftDirPath,"will be skipped"
-        continue
-    
-    try:
-        props = sgprops.readProps(setFilePath, dataDirPath = fgRoot)
-        sim = props.getNode("sim")
-     
-        pkgNode = catalogProps.addChild('package')
-        
-        # basic / mandatory values
-        pkgNode.addChild('id').value = d
-        pkgNode.addChild('name').value = sim.getValue('description')
-        
-        longDesc = sim.getValue('long-description')
-        if longDesc is not None:
-            pkgNode.addChild('description').value = longDesc
-            
-        # copy all the standard values
-        for p in ['status', 'author', 'license']:
-            v = sim.getValue(p)
-            if v is not None:
-                pkgNode.addChild(p).value = v
-            
-        # ratings
-        if sim.hasChild('rating'):
-            pkgRatings = pkgNode.addChild('rating')
-            for r in ['FDM', 'systems', 'cockpit', 'model']:
-                pkgRatings.addChild(r).value = sim.getValue('rating/' + r, 0)
-            
-        # copy tags
-        if sim.hasChild('tags'):
-            for c in sim.getChild('tags').getChildren('tag'):
-                if isNonstandardTag(c.value):
-                    print "Skipping non-standard tag:", c.value
-                else:
-                    pkgNode.addChild('tag').value = c.value
-                
-        # create download and thumbnail URLs
-        s = "{url}Aircraft-3.0/"
-        if d not in catalogFilenames.aircraft:
-            print "filename not found for:",d
-            raise RuntimeError("filename not found for:" + d)
-        s += catalogFilenames.aircraft[d]
-        
-        for u in urls:
-            pkgNode.addChild("url").value = s.format(url=u)
-        
-        for t in thumbs:
-            pkgNode.addChild("thumbnail").value = t.format(acft=d)
-        
-        cachedZip = os.path.join(cacheDir, catalogFilenames.aircraft[d])
-        if not os.path.exists(cachedZip):
-            # download the zip
-            url = s.format(url=urls[0])
-            print "Downloading ", url
-            urllib.urlretrieve(url, cachedZip)
-        #else:
-        #    print "Using cached zip for", d
-            
-        zipFile = open(cachedZip, 'r')
-        
-        digest = hashlib.md5(zipFile.read()).hexdigest()
-        pkgNode.addChild("md5").value = digest
-        pkgNode.addChild("file-size-bytes").value = os.path.getsize(cachedZip)
-    except:
-        print "Failure processing:", setFilePath
-        
-catalogProps.write("catalog.xml")        

fgdata_checkers.py

@@ -8,15 +8,25 @@ import os.path
 import re
 from collections import defaultdict
 import subprocess
+import multiprocessing
 import math
 import tarfile
+import tempfile
 import gzip
+import shutil
+import time
+try:
+    devnull=subprocess.DEVNULL#hide annoying nvcompress messages
+except (AttributeError,NameError):#pre-3.3 Python
+    devnull=None
 def path_join(*args):
-    """Unlike plain os.path.join, this doesn't add a trailing / if the last component is empty"""
-    return os.path.normpath(os.path.join(*args))
+    """Unlike plain os.path.join, this always uses forward slashes, and doesn't add a trailing / if the last component is empty"""
+    return os.path.normpath(os.path.join(*args)).replace('\\','/')
 def rfilelist(path,exclude_dirs=[]):
     """Dict of files/sizes in path, including those in any subdirectories (as relative paths)"""
     files=defaultdict(int)
+    if not os.path.exists(path):
+        return files
     dirs=[""]
     while dirs:
         cdir=dirs.pop()
@@ -65,16 +75,16 @@ def files_used(pattern,path,exclude_dirs=[],filelist=None,filetypes=None,relativ
             tex=texfind.search(line)
             if tex:
                 if relative_path:
-                    textures.append(os.path.normpath(path_join(os.path.dirname(file),tex.group(1).replace('\\','/'))))
+                    textures.append(os.path.normpath(path_join(os.path.dirname(file),tex.group(1).replace('\\','/'))).replace('\\','/'))
                 else:
-                    textures.append(os.path.normpath(tex.group(1).replace('\\','/')))
+                    textures.append(os.path.normpath(tex.group(1).replace('\\','/')).replace('\\','/'))
     return textures
 def find_unused_textures(basedir,output_lists=True,grep_check=False,output_rsync_rules=False,output_comparison_strips=False, output_removal_commands=False,return_used_noregions=False):
     """Checks if any textures are unused (wasting space), and if any textures are only available as .dds (not recommended in the source repository, as it is a lossy-compressed format)
 
 Set basedir to your fg-root, and enable the kind(s) of output you want:
 output_lists prints lists of unused textures, and of dds-only textures
-grep_check checks for possible use outside the normal directories; requires Unix shell
+grep_check checks for possible use outside the normal directories; requires Unix shell and assumes side-by-side fgdata,flightgear,simgear
 output_rsync_rules prints rsync rules for excluding unused textures from the release flightgear-data.  Warning: if you use this, re-run this script regularly, in case they start being used
 output_comparison_strips creates thumbnail strips, unused_duplicate.png/unused_dds.png/high_low.png, for visually checking whether same-name textures are the same (remove the unused one entirely) or different (move it to Unused); requires imagemagick or graphicsmagick
 output_removal_commands creates another script, delete_unused_textures.sh, which will remove unused textures when run in a Unix shell"""
@@ -117,6 +127,9 @@ output_removal_commands creates another script, delete_unused_textures.sh, which
     low_unneeded_nondup=low_unneeded&set(known_highlow_mismatch)
     def image_check_strip(basedir,index_fname,ilist1,ilist2=None,size=128):
         """Generate two rows of thumbnails, for easy visual comparison (between the two lists given, or if a single list is given, between low and high resolution)"""
+        if not ilist1:
+            print(index_fname," empty, skipping")
+            return
         if ilist2 is None:
             ipairs=[[path_join(basedir,'Textures',f),path_join(basedir,'Textures.high',f)] for f in ilist1]
         else:
@@ -162,6 +175,12 @@ output_removal_commands creates another script, delete_unused_textures.sh, which
     if output_comparison_strips:
         image_check_strip(basedir,"unused_duplicate.png",unused_duplicate,["Terrain"+f[14:] for f in unused_duplicate])
         image_check_strip(basedir,"unused_dds.png",unused_dds,[f[:-4]+".png" for f in unused_dds])
+        dds_skip=set(['Runway/rwy-normalmap.dds','Water/perlin-noise-nm.dds','Water/water_sine_nmap.dds','Water/waves-ver10-nm.dds'])
+        used_dds_withpng=set(f for f in (high_textures|low_textures) if (f[-4:]==".dds" and f[:-4]+".png" in (high_textures|low_textures)))-unused_dds-dds_skip
+        print(".dds omitted from comparison strip (normal maps etc): ",dds_skip)
+        #used_dds_withpng different: p{a,c}_taxiway.dds and possibly more runway markings (darker),sand{4,5,6}.dds(less high-freq noise),water.dds(more high-freq noise),grass_rwy.dds (has stripes),cropwood.dds,irrcrop.dds,shrub.dds,herbtundra.dds,cropgrass.dds(unrelated)
+        #water-reflection.{png,dds} really are plain white (so do match), not an alpha map
+        image_check_strip(basedir,"used_dds.png",used_dds_withpng,[f[:-4]+".png" for f in used_dds_withpng])
         image_check_strip(basedir,"high_low.png",high_textures&low_textures)
         #image_check_strip(basedir,"high_low2.png",[f for f in high_textures&low_textures if (f[0:14]=="Terrain.winter" or "_taxiway." in f or "lava" in f or "sand" in f)],size=512)#closer look at the doubtful cases
     if output_lists:
@@ -171,11 +190,11 @@ output_removal_commands creates another script, delete_unused_textures.sh, which
         unused_f=[os.path.basename(f) for f in unused]
         all_f=[os.path.basename(f) for f in (high_textures|low_textures)]
         print("\n\nPossible use outside main search:")#used to set false_positives
-        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=.git","-e","("+")|(".join(unused)+")","/home/palmer/fs_dev/git/fgdata","/home/palmer/fs_dev/git/flightgear","/home/palmer/fs_dev/git/simgear"])#everywhere using full names
-        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=Textures.high","--exclude-dir=Models","--exclude-dir=Materials","--exclude-dir=Effects","--exclude-dir=.git","-e","("+")|(".join(all_f)+")","/home/palmer/fs_dev/git/fgdata","/home/palmer/fs_dev/git/flightgear","/home/palmer/fs_dev/git/simgear"])#restricted (to avoid false positives from Terrain.winter vs Terrain) using filenames
-        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=Textures.high","--exclude-dir=Models","--exclude-dir=Materials","--exclude-dir=Effects","--exclude-dir=.git","-e",'[."\']dds',"/home/palmer/fs_dev/git/fgdata","/home/palmer/fs_dev/git/flightgear","/home/palmer/fs_dev/git/simgear"])#check for programmatic .png -> .dds swap; none found
+        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=.git","-e","("+")|(".join(unused)+")",basedir,path_join(basedir,"../flightgear"),path_join(basedir,"../simgear")])#everywhere using full names
+        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=Textures.high","--exclude-dir=Models","--exclude-dir=Materials","--exclude-dir=Effects","--exclude-dir=.git","-e","("+")|(".join(all_f)+")",basedir,path_join(basedir,"../flightgear"),path_join(basedir,"../simgear")])#restricted (to avoid false positives from Terrain.winter vs Terrain) using filenames
+        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=Textures.high","--exclude-dir=Models","--exclude-dir=Materials","--exclude-dir=Effects","--exclude-dir=.git","-e",'[."\']dds',basedir,path_join(basedir,"../flightgear"),path_join(basedir,"../simgear")])#check for programmatic .png -> .dds swap; none found
         print("\n\nUse of sourceless textures:")
-        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=.git","-e","("+")|(".join(sourceless)+")","/home/palmer/fs_dev/git/fgdata","/home/palmer/fs_dev/git/flightgear","/home/palmer/fs_dev/git/simgear"])
+        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=.git","-e","("+")|(".join(sourceless)+")",basedir,path_join(basedir,"../flightgear"),path_join(basedir,"../simgear")])
     if output_rsync_rules:
         print("\n\nFull flightgear-data:\n")
         rsync_rules(basedir,unused)
@@ -240,54 +259,171 @@ def find_locally_unused_models(basedir):
     missing=set(f for f in (used_models|used_textures) if ((f.startswith('Models') and f not in models_allfiles.keys()) or not os.path.isfile(path_join(basedir,f))))
     print("used\n",sorted(used_models),"\nsize=",t_size(used_models),"\n\n",sorted(used_textures),"\nsize=",t_size(used_textures),"\n\nunused\n",sorted(unused),"\nsize=",t_size(unused),"\n\nmissing\n",sorted(missing),"\nsize=",t_size(missing))
 
-def size_by_type(path,exclude_dirs=[]):
-    """Dict of total file size by file extension"""
+class FilesetSizes:
+    def __init__(self):
+        self.count=0
+        self.ncsize=0
+        self.csize=0
+    @property
+    def size(self):
+        if self.csize>0:
+            return self.csize
+        else:
+            return self.ncsize
+    def __str__(self):
+        if self.csize>0:
+            return "{0:10,} {1:15,} {2:15,}".format(self.count,self.ncsize,self.csize)
+        else:
+            return "{0:10,} {1:15,}".format(self.count,self.ncsize)
+    def __add__(self,other):
+        result=FilesetSizes()
+        result.count=self.count+other.count
+        result.ncsize=self.ncsize+other.ncsize
+        result.csize=self.csize+other.csize
+        return result
+def size_by_category(path,exclude_dirs,keyfn,compressed_size=False):
+    """Total size of files, in each category returned from keyfn"""
     files=rfilelist(path,exclude_dirs)
-    size_totals=defaultdict(int)
-    for filename,size in files.items():
-        file_ext=os.path.splitext(filename)[1]
-        if file_ext==".gz":
-            file_ext=os.path.splitext(os.path.splitext(filename)[0])[1]+file_ext
-        size_totals[file_ext]=size_totals[file_ext]+size
-    return size_totals
-def size_by_size(path,exclude_dirs=[],exts=[".png",".dds",".rgb"]):
-    """Dict of total file size by individual file size range, of given extensions (empty list for all files)"""
-    files=rfilelist(path,exclude_dirs)
-    size_totals=defaultdict(int)
-    for filename,size in files.items():
-        file_ext=os.path.splitext(filename)[1]
-        if (not exts) or (file_ext in exts):
-            size_totals[2**math.frexp(size)[1]]=size_totals[2**math.frexp(size)[1]]+size
-    return size_totals
-def fgdata_size(path,dirs_to_list=["AI/Aircraft","AI/Traffic","Aircraft","Models","Scenery","Textures","Textures.high"],exclude_dirs=None,compressed_size=False,num_types=3):
-    if dirs_to_list is None:
-        dirs_to_list=[d for d in os.listdir(path) if os.path.isdir(path_join(path,d))]
+    result=defaultdict(FilesetSizes)
+    gzdir=tempfile.TemporaryDirectory()
+    gzpath=gzdir.name
+    gzcount=0
+    for file,size in files.items():
+        cat=keyfn(file,size)
+        if cat is None:
+            continue
+        result[cat].count=result[cat].count+1
+        result[cat].ncsize=result[cat].ncsize+size
+        if compressed_size:
+            try:
+                result[cat].targz.add(path_join(path,file))
+            except AttributeError:
+                result[cat].targzname=path_join(gzpath,str(gzcount)+".tar.gz")
+                result[cat].targz=tarfile.open(result[cat].targzname,mode="w:gz")
+                gzcount=gzcount+1
+                result[cat].targz.add(path_join(path,file))
+    if compressed_size:
+        for cat in result:
+            result[cat].targz.close()
+            result[cat].csize=os.path.getsize(result[cat].targzname)
+    return result
+def fgdata_size(path,by_size=False,exclude_dirs=None,compressed_size=False,min_size=1e6,include_aircraft=['UIUC','777','777-200','b1900d','CitationX','ZLT-NT','dhc2','Cub','sopwithCamel','f-14b','ASK13','bo105','Dragonfly','SenecaII','A6M2'],sections={"Aircraft/c172p":"Aircraft-base","Aircraft/Generic":"Aircraft-base","Aircraft/Instruments":"Aircraft-base","Aircraft/Instruments-3d":"Aircraft-base","Aircraft/ufo":"Aircraft-base","Aircraft/777":"Aircraft-777","Aircraft/777-200":"Aircraft-777","Aircraft":"Aircraft-other","Textures":"Textures","Textures.high":"Textures","AI/Aircraft":"AI-aircraft","AI":"AI-other","Scenery":"scenery","Models":"models"}):
+    if any('\\' in s for s in sections):
+        raise ValueError("sections always uses forward slashes")
+    if "" not in sections:
+        sections[""]="Other"
     if exclude_dirs is None:
         if os.path.exists(path_join(path,".git")):
-            exclude_dirs=[".git","Aircraft"]
+            exclude_dirs=[".git","Textures/Unused"]+[path_join("Aircraft",d) for d in os.listdir(path_join(path,"Aircraft")) if d not in ["c172p","ufo","Generic","Instruments","Instruments-3d"]+include_aircraft]
         else:
             exclude_dirs=[]
-    total_compressed_size=0
-    exclude_list=[[]]*len(dirs_to_list)+[dirs_to_list+exclude_dirs]+[exclude_dirs]
-    names_list=dirs_to_list+["other","all"]
-    for n,dir1 in enumerate(dirs_to_list+["",""]):
-        size_totals=size_by_type(path_join(path,dir1),exclude_list[n])
-        print(names_list[n],sorted(size_totals.items(),key=lambda x:-x[1])[:num_types],"total",sum(size_totals.values()))
-        if compressed_size:
-            if names_list[n]=="all":
-                print("compressed size",total_compressed_size)
-                continue
-            targz=tarfile.open("fgdata_sizetest_temp.tar.gz",mode="w:gz")
-            for file in rfilelist(path_join(path,dir1),exclude_list[n]):
-                targz.add(path_join(path,dir1,file))
-            targz.close()
-            print("compressed size",os.path.getsize("fgdata_sizetest_temp.tar.gz"))
-            total_compressed_size=total_compressed_size+os.path.getsize("fgdata_sizetest_temp.tar.gz")
+    if by_size:
+        keyfn=lambda file,size: ("texture",math.frexp(size)[1]) if os.path.splitext(file)[1] in [".png",".dds",".rgb",".jpg"] else ("nontexture",math.frexp(size)[1])
+    else:
+        def keyfn(file,size):
+            file_ext=os.path.splitext(file)[1]
+            if file_ext==".gz":
+                file_ext=os.path.splitext(os.path.splitext(file)[0])[1]+file_ext
+            file0=file.replace('\\','/')
+            section=sections[max((s for s in sections if file0.startswith(s)),key=len)]
+            return (section,file_ext)
+    result=size_by_category(path,exclude_dirs,keyfn,compressed_size)
+    #print(result)
+    totals1=defaultdict(FilesetSizes)
+    totals2=defaultdict(FilesetSizes)
+    other=defaultdict(FilesetSizes)
+    for cat in result:
+        totals1[cat[0]]=totals1[cat[0]]+result[cat]
+        totals2[cat[1]]=totals2[cat[1]]+result[cat]
+        if (not by_size) and (result[cat].size<min_size):
+            other[cat[0]]=other[cat[0]]+result[cat]
+    print("{0:>10} {1:>15} {2:>15}".format("Count","Size","Compr.size"))
+    for cat0,r0 in sorted(totals1.items(),key=lambda p:-p[1].size):
+        print(cat0)
+        for cat,r in sorted([(c[1],r) for (c,r) in result.items() if (c[0]==cat0 and (by_size or r.size>min_size))],key=(lambda p:p[0]) if by_size else (lambda p:-p[1].size)):
+            print(r,cat)
+        if not by_size:
+            print(other[cat0],"other")
+        print(totals1[cat0],"all")
+    print("All")
+    for cat,r in sorted([(c,r) for (c,r) in totals2.items() if (by_size or r.size>min_size)],key=(lambda p:p[0]) if by_size else (lambda p:-p[1].size)):
+        print(r,cat)
+    if not by_size:
+        print(sum((r for r in totals2.values() if r.size<min_size),FilesetSizes()),"other")
+    print(sum(totals1.values(),FilesetSizes()),"all")
+def create_reduced_file(input_path,output_path,temp_path,cdir,file,downsample_this,compress_this,compress_names_find, compressed_format,texture_filetypes,binary_types,fclass):
+    """Process a single file in create_reduced_fgdata.  (Separate function to allow parallel processing)"""
+    retcode=0
+    if downsample_this or compress_this:
+        image_type=texture_filetypes[os.path.splitext(file)[1]]
+        output_image_type=compressed_format if compress_this else os.path.splitext(file)[1]
+        output_file=os.path.splitext(file)[0]+output_image_type
+        output_image_type=texture_filetypes[output_image_type]
+        if "{0}" in output_path and fclass=="base-textures":#downsampled in base-textures, full resolution in extra-textures
+            shutil.copy(path_join(input_path,cdir,file),path_join(output_path.format("extra-textures"),cdir,file))
+        if output_image_type=="DDS":# in Ubuntu, neither imagemagick nor graphicsmagick can write .dds
+            #doesn't work (for dds -> smaller dds): subprocess.call(["nvzoom","-s","0.5","-f","box",path_join(input_path,cdir,file),path_join(output_path.format(fclass),cdir,file)])
+            if subprocess.call(["convert",image_type+":"+path_join(input_path,cdir,file)]+(["-flip"] if ((image_type=="DDS")!=(output_image_type=="DDS")) else [])+(["-sample","50%"] if downsample_this else [])+[path_join(temp_path,cdir,os.path.splitext(file)[0]+".png")]):#fails on some DDS formats, so just copy them
+                shutil.copy(path_join(input_path,cdir,file),path_join(output_path.format(fclass),cdir,file))
+                print(path_join(cdir,file),"unsupported type, probably normal map")
+                if compress_this:
+                    raise TypeError#copy will have the wrong name
+            else:
+                try:
+                    image_properties=subprocess.check_output(["identify","-verbose",path_join(temp_path,cdir,os.path.splitext(file)[0]+".png")])
+                except subprocess.CalledProcessError as err:
+                    print("identify error on",path_join(cdir,file),"after ",["convert",image_type+":"+path_join(input_path,cdir,file)]+(["-flip"] if ((image_type=="DDS")!=(output_image_type=="DDS")) else [])+(["-sample","50%"] if downsample_this else [])+[path_join(temp_path,cdir,os.path.splitext(file)[0]+".png")],["identify","-verbose",path_join(temp_path,cdir,os.path.splitext(file)[0]+".png")],err)
+                    raise
+                has_alpha=b"Alpha" in image_properties
+                needs_alpha=has_alpha
+                if has_alpha and re.search(rb"Alpha:\s*min: 255 \(1\)",image_properties):
+                    print(path_join(cdir,file),"has always-255 alpha")
+                    needs_alpha=False
+                retcode=subprocess.call(["nvcompress","-bc3" if needs_alpha else "-bc1",path_join(temp_path,cdir,os.path.splitext(file)[0]+".png"),path_join(output_path.format(fclass),cdir,output_file)],stdout=devnull)
+        else:
+            retcode=subprocess.call(["convert",image_type+":"+path_join(input_path,cdir,file)]+(["-sample","50%"] if downsample_this else [])+[output_image_type+":"+path_join(output_path.format(fclass),cdir,output_file)])#we use sample rather than an averaging filter to not break mask/rotation/... maps
+    else:#not to be downsampled/compressed
+        if os.path.splitext(file)[1] in binary_types:#just copy
+            shutil.copy(path_join(input_path,cdir,file),path_join(output_path.format(fclass),cdir,file))
+        else:#texture name replacement
+            file_in=open(path_join(input_path,cdir,file),'rb')
+            file_out=open(path_join(output_path.format(fclass),cdir,file),'wb')
+            file_str=file_in.read(None)
+            file_in.close()
+            (file_strout,num_matches)=compress_names_find.subn(lambda mf: os.path.splitext(mf.group(0))[0]+(compressed_format.encode('utf-8')),file_str)
+            file_out.write(file_strout)
+            file_out.close()
+            #if ((os.path.splitext(file)[1] not in textureuser_types) and num_matches>0):
+                #print("Warning: ",num_matches," unexpected use(s) in ",path_join(cdir,file))
+            #if compress_names_find0.search(file_strout):
+                #print("Warning: unreplaced match(es) in ",path_join(cdir,file),compress_names_find0.search(file_strout).group(0))
+            """Warning: unreplaced match(es) in... correct rejections of match within a filename:
+                Aircraft/Instruments-3d/AN-APS-13.ac b'panel.png'
+                Aircraft/Instruments-3d/magneto-switch/mag_switch.ac b'black.png'
+                Nasal/canvas/map/Images/chart_symbols.svg b'wash.png'
+                Models/Airport/blast-deflector49m.ac b'generic.png'
+                Models/Airport/blast-deflector63m.ac b'generic.png'
+                Models/Industrial/oilrig09.ac b'yellow.png'
+                Models/Industrial/oilrig10.ac b'yellow.png'
+                Models/Industrial/oilrig09.ac.before-color-change b'yellow.png'
+                Models/Industrial/oilrig10.ac.before-color-change b'yellow.png'
+                Models/Maritime/Civilian/Tanker.ac b'black.png'
+                Models/Transport/flatcar.xml b'evergreen.png'
+                Models/Commercial/tower-grey-black.ac b'black.png'
+                Materials/base/materials-base.xml b'yellow.png'
+                
+                Warning: unexpected use(s) in...
+                Docs/README.local_weather.html (the only one that looke like an actual problem; hence, Docs is now skipped)
+                Nasal/canvas/map/Images/chart_symbols.svg (probably inkscape:export-filename, which are creator-specific absolute paths anyway, but now skipped)
+                oilrig09.ac.before-color-change,oilrig10.ac.before-color-change,stbd_coaming_panel.ac.bak (presumably backup files)
+                """
+    if retcode:
+        print("Error ",retcode," in ",path_join(cdir,file))
 
-def create_reduced_fgdata(input_path,output_path,split_textures=True,exclude_parts=[],include_aircraft=['UIUC','777','777-200','b1900d','CitationX','ZLT-NT','dhc2','Cub','sopwithCamel','f-14b','ASK13','bo105','Dragonfly','SenecaII','A6M2'],dirs_to_downsample=(),downsample_min_filesize=30000):
+def create_reduced_fgdata(input_path,output_path,reject_positional_args=None,split_textures=False,exclude_parts=[],include_aircraft=['UIUC','777','777-200','b1900d','CitationX','ZLT-NT','dhc2','Cub','sopwithCamel','f-14b','ASK13','bo105','Dragonfly','SenecaII','A6M2'],dirs_to_downsample=(),dirs_to_compress=(),compressed_format=".dds",downsample_min_filesize=1e5,compress_min_filesize=3e4,use_ready_compressed=True):
     """Create a smaller, reduced-quality flightgear-data package
-Can downsample textures 50% and/or omit sections
-Requires Unix shell; downsampling requires imagemagick or graphicsmagick (for convert) and libnvtt-bin (for nvcompress)
+Can downsample textures 50%, change texture format, and/or omit sections (region-specific textures, aircraft, AI traffic)
+Downsampling and format change require imagemagick or graphicsmagick (for convert) and libnvtt-bin (for nvcompress)
 
 Optional parts, use exclude_parts to omit:
 ai: no background traffic, but tankers etc do still work
@@ -296,13 +432,39 @@ extra-textures (requires split_textures=True): no region-specific textures
 The c172p and ufo are always included; other aircraft are added by include_aircraft
 
 Texture downsampling: textures in dirs_to_downsample and larger than downsample_min_filesize downsampled 50%
-Example: dirs_to_downsample=("Textures.high/Terrain","Textures.high/Trees","Textures.high/Terrain.winter","AI/Aircraft","Models"),downsample_min_filesize=30000
+Texture format conversion: textures in dirs_to_compress and larger than compress_min_filesize converted to compressed_format
+use_ready_compressed determines what happens if a same-basename file in compressed_format already exists: True uses the already-compressed one, False uses the uncompressed one, None keeps both
+Suggested dirs_to_downsample:
+3.2: ('Textures.high/Terrain','Textures.high/Trees','Textures.high/Terrain.winter','AI/Aircraft','Models')
+3.3: ('Textures/Terrain','Textures/Trees','Textures/Terrain.winter','AI/Aircraft','Models')
+To do "everything" (a few are always skipped due to potential breakage), use dirs_to_compress=('',)
 
-To put each section in its own directory use {0} in output_path, e.g.
-python3 -c "import fgdata_checkers; fgdata_checkers.create_reduced_fgdata(input_path='/home/palmer/fs_dev/git/fgdata',output_path='/home/palmer/fs_dev/flightgear/data_split/debian/flightgear-data-{0}/usr/share/games/flightgear',include_aircraft=['UIUC','b1900d','CitationX','ZLT-NT','dhc2','Cub','sopwithCamel','f-14b','ASK13','bo105','Dragonfly','SenecaII','A6M2'])"
+To put each section in its own directory (e.g. for building a Debian-style flightgear-data-* set of packages) use {0} in output_path, e.g.
+python3 -c "import fgdata_checkers; fgdata_checkers.create_reduced_fgdata(input_path='/home/rnpalmer/fs_dev/git/fgdata',output_path='/home/rnpalmer/fs_dev/flightgear/data_split/debian/flightgear-data-{0}/usr/share/games/flightgear',include_aircraft=['UIUC','b1900d','CitationX','ZLT-NT','dhc2','Cub','sopwithCamel','f-14b','ASK13','bo105','Dragonfly','SenecaII','A6M2'])"
 This creates separate preferences-regions.xml and preferences-noregions.xml files for with and without regional textures; you need to handle symlinking preferences.xml to the correct one
 """
-    texture_filetypes={".png":"PNG",".dds":"DDS"}#,".rgb":"SGI" loses cloud transparency
+    start_time=time.time()
+    if reject_positional_args is not None:
+        raise TypeError("Keyword arguments only please: this is not a stable API")
+    if use_ready_compressed not in (True,False,None):
+        raise TypeError("invalid use_ready_compressed setting")
+    texture_filetypes={".png":"PNG",".dds":"DDS",".jpg":"JPEG"}#,".rgb":"SGI" loses cloud transparency
+    if compressed_format not in texture_filetypes:
+        raise ValueError("Invalid compressed_format (include the .)")
+    textureuser_types={".eff",".xml",".ac",".nas"}
+    binary_types={".png",".dds",".rgb",".RGB",".jpg",".wav",".WAV",".btg.gz",".zip",".tar.gz"}#don't search these for texture name replacement
+    """Textures named directly in flightgear/simgear code:
+gui/images/shadow.png,gui/cursor-spin-cw.png (probably safest to treat this as gui/*, they're all small)
+Textures/Globe/world.topo.bathy.200407.3x4096x2048.png
+Textures/buildings.png,Textures/buildings-lightmap.png
+Textures/Sky/*
+Textures/Splash*.png
+unknown.rgb (probably Textures/ or Textures/Terrain/, neither exists)
+Aircraft/Instruments/Textures/nd-symbols.png (doesn't actually exist),Aircraft/Instruments/Textures/compass-ribbon.rgb,Aircraft/Instruments/Textures/od_wxradar.rgb,Aircraft/Instruments/Textures/od_wxradar.rgb,Aircraft/Instruments/Textures/wxecho.rgb,Aircraft/Instruments/Textures/od_groundradar.rgb (doesn't actually exist)
+also, Aircraft/{Instruments,Instruments-3d,Generic} may be used by downloaded aircraft, and Docs images are used in .html
+Nasal (Canvas map) probably wouldn't break anything, but guessing it's a bad idea visually"""
+    no_compress_pattern=re.compile(r'mask|light|relief|nmap|nm\.|normal|dudv|^Splash[0-9].png$|^buildings.png$|^buildings-lightmap.png$|^world.topo.bathy.200407.3x4096x2048.png$|^ocean_depth_1.png$')#edge blurring from lossy compression may break masks (converting ocean_depth_1.png is known to give black water near shore at effects>=4), and this script doesn't know how to create DDS normal maps
+    no_compress_dirs=("gui","Docs","webgui","Nasal","Textures/Sky","Aircraft/Instruments","Aircraft/Instruments-3d","Aircraft/Generic")
     exclude_dirs=[".git","Textures/Unused"]
     exclude_unnamed_subdirs=["Aircraft"]#these are a separate mechanism from subtree_class/exclude_parts mostly to save time (subtree_class still fully scans excluded directories because the class may change again further down the tree, e.g. AI/Aircraft ai -> performancedb.xml base; these don't)
     subtree_class={"Aircraft/c172p":"base","Aircraft/Generic":"base","Aircraft/Instruments":"base","Aircraft/Instruments-3d":"base","Aircraft/ufo":"base","Textures":"textures","Textures.high":"textures","AI/Aircraft":"ai","AI/Traffic":"ai","AI/Aircraft/performancedb.xml":"base","Scenery":"scenery","Models":"models"}
@@ -314,12 +476,41 @@ This creates separate preferences-regions.xml and preferences-noregions.xml file
         base_texture_files=[]
         for t in find_unused_textures(input_path,return_used_noregions=True):
             base_texture_files.extend([path_join("Textures",t),path_join("Textures.high",t)])
-    #if os.path.exists(path_join(input_path,".git")):
-        #print(input_path,"appears to be a git clone; this will work, but the result will be larger than starting from a standard flightgear-data package.\nTo create this use (adjusting paths as necessary) rsync -av --filter=\"merge /home/palmer/fs_dev/git/fgmeta/base-package.rules\" ~/fs_dev/git/fgdata ~/fs_dev/flightgear/data_full")
+    #no longer a significant problem with exclude_dirs: if os.path.exists(path_join(input_path,".git")):
+        #print(input_path,"appears to be a git clone; this will work, but the result will be larger than starting from a standard flightgear-data package.\nTo create this use (adjusting paths as necessary) rsync -av --filter=\"merge /home/rnpalmer/fs_dev/git/fgmeta/base-package.rules\" ~/fs_dev/git/fgdata ~/fs_dev/flightgear/data_full")
     if os.path.exists(output_path.format("base")) and os.listdir(output_path.format("base")):
         print("output path",output_path,"non-empty, aborting to avoid data loss\nIf you did want to lose its previous contents, run:\nrm -r",output_path,"\nthen re-run this script")
         return
+    if compressed_format==".jpg":
+        print("Warning: selected compression format does not support transparency")
+    compress_names=set()
+    if dirs_to_compress:#need this preliminary pass to get names to change in .xml,etc
+        no_compress_names=set()
+        dirs={"":"base"}
+        while dirs:
+            cdir,cclass=dirs.popitem()
+            cdirfiles=os.listdir(path_join(input_path,cdir))
+            for file in cdirfiles:
+                fclass=subtree_class.get(path_join(cdir,file),cclass)
+                if os.path.isdir(path_join(input_path,cdir,file)):
+                    if (path_join(cdir,file) not in exclude_dirs) and (cdir not in exclude_unnamed_subdirs or path_join(cdir,file) in subtree_class):
+                        dirs[path_join(cdir,file)]=fclass
+                else:#file
+                    compress_this=cdir.startswith(dirs_to_compress) and (os.path.splitext(file)[1] in texture_filetypes) and (os.path.splitext(file)[1]!=compressed_format) and (os.path.getsize(path_join(input_path,cdir,file))>compress_min_filesize) and not no_compress_pattern.search(file) and not cdir.startswith(no_compress_dirs) and (file not in no_compress_names) and ((use_ready_compressed is not None) or not os.path.exists(path_join(input_path,cdir,os.path.splitext(file)[0]+compressed_format)))
+                    if compress_this:
+                        compress_names.add(file)
+                    else:
+                        no_compress_names.add(file)
+                        compress_names.discard(file)#if there are two with the same name in different directories, compress both or neither, to simplify name replacement
+    compress_names_find=re.compile(('(?<=["\'>/\\\\ \\n])('+'|'.join(re.escape(f) for f in compress_names)+')($|(?=["\'< \\n]))').encode('utf-8'))
+    compress_names_find0=re.compile(('|'.join(re.escape(f) for f in compress_names)).encode('utf-8'))
+    #print(compress_names,"\n\n",no_compress_names,"\n\n",'(?<=["\'>/\\\\ \\n])('+'|'.join(re.escape(f) for f in compress_names)+')($|(?=["\'< \\n]))',"\n\n",'|'.join(re.escape(f) for f in compress_names),"\n\n")
+    print("Starting conversion...",len(compress_names),"files to change format, runtime so far=",int(time.time()-start_time),"sec")
     dirs={"":"base"}
+    subprocess_pool=multiprocessing.Pool(processes=8)
+    subprocess_list=[]
+    temp_dir=tempfile.TemporaryDirectory()
+    temp_path=temp_dir.name
     while dirs:
         cdir,cclass=dirs.popitem()
         cdirfiles=os.listdir(path_join(input_path,cdir))
@@ -337,25 +528,29 @@ This creates separate preferences-regions.xml and preferences-noregions.xml file
                 if fclass in exclude_parts:
                     continue
                 if not os.path.exists(path_join(output_path.format(fclass),cdir)):
-                    subprocess.call(["mkdir","-p",path_join(output_path.format(fclass),cdir)])
-                if (cdir.startswith(dirs_to_downsample)) and (os.path.splitext(file)[1] in texture_filetypes) and (os.path.getsize(path_join(input_path,cdir,file))>downsample_min_filesize):
-                    image_type=texture_filetypes[os.path.splitext(file)[1]]
-                    if "{0}" in output_path and fclass=="base-textures":#downsampled in base-textures, full resolution in extra-textures
-                        if not os.path.exists(path_join(output_path.format("extra-textures"),cdir)):
-                            subprocess.call(["mkdir","-p",path_join(output_path.format("extra-textures"),cdir)])
-                        subprocess.call(["cp",path_join(input_path,cdir,file),path_join(output_path.format("extra-textures"),cdir,file)])
-                    if image_type=="DDS":# in Ubuntu, neither imagemagick nor graphicsmagick can write .dds
-                        #doesn't work subprocess.call(["nvzoom","-s","0.5","-f","box",path_join(input_path,cdir,file),path_join(output_path.format(fclass),cdir,file)])
-                        if subprocess.call(["convert",image_type+":"+path_join(input_path,cdir,file),"-sample","50%","temp_reduced_size.png"]):#fails on normal maps, so just copy them
-                            subprocess.call(["cp",path_join(input_path,cdir,file),path_join(output_path.format(fclass),cdir,file)])
-                        else:
-                            subprocess.call(["nvcompress","-bc3","temp_reduced_size.png",path_join(output_path.format(fclass),cdir,file)])
-                    else:
-                        subprocess.call(["convert",image_type+":"+path_join(input_path,cdir,file),"-sample","50%",image_type+":"+path_join(output_path.format(fclass),cdir,file)])#we use sample rather than an averaging filter to not break mask/rotation/... maps
-                else:#not to be downsampled
-                    subprocess.call(["cp",path_join(input_path,cdir,file),path_join(output_path.format(fclass),cdir,file)])
+                    os.makedirs(path_join(output_path.format(fclass),cdir))#errors out if the directory does exist, so calling it in the per-file subprocess would be a race condition
+                if "{0}" in output_path and fclass=="base-textures" and not os.path.exists(path_join(output_path.format("extra-textures"),cdir)):#downsampled in base-textures, full resolution in extra-textures
+                    os.makedirs(path_join(output_path.format("extra-textures"),cdir))
+                if not os.path.exists(path_join(temp_path,cdir)):
+                    os.makedirs(path_join(temp_path,cdir))
+                downsample_this=(cdir.startswith(dirs_to_downsample)) and (os.path.splitext(file)[1] in texture_filetypes) and (os.path.getsize(path_join(input_path,cdir,file))>downsample_min_filesize)
+                compress_this=(file in compress_names)
+                if compress_this and use_ready_compressed==True and os.path.exists(path_join(input_path,cdir,os.path.splitext(file)[0]+compressed_format)):
+                    continue
+                if (not compress_this) and use_ready_compressed==False and os.path.splitext(file)[1]==compressed_format and any((os.path.exists(path_join(input_path,cdir,os.path.splitext(file)[0]+f)) and os.path.splitext(file)[0]+f in compress_names) for f in texture_filetypes if f!=os.path.splitext(file)[1]):
+                    continue
+                subprocess_list.append(subprocess_pool.apply_async(create_reduced_file,args=(input_path,output_path,temp_path,cdir,file,downsample_this,compress_this,compress_names_find, compressed_format,texture_filetypes,binary_types,fclass)))
+    print(len(subprocess_list),"file tasks started...runtime so far=",int(time.time()-start_time),"sec\n(Not a linear progress indicator: they are different lengths.)")
+    last_report=time.time()-100#print first report immediately
+    for s0 in subprocess_list:
+        if time.time()>last_report+60:
+            print("Waiting for",len([s for s in subprocess_list if not s.ready()]),"file tasks...runtime so far=",int(time.time()-start_time),"sec")
+            last_report=time.time()
+        s0.get()
+    subprocess_pool.close()
+    subprocess_pool.join()
     if "{0}" in output_path:
-        subprocess.call(["mv",path_join(output_path.format("base"),"preferences.xml"),path_join(output_path.format("base"),"preferences-regions.xml")])
+        os.rename(path_join(output_path.format("base"),"preferences.xml"),path_join(output_path.format("base"),"preferences-regions.xml"))
     if "extra-textures" in exclude_parts or "{0}" in output_path:
         prefs_in=open(path_join(input_path,"preferences.xml"),'r')
         prefs_out=open(path_join(output_path.format("base"),"preferences-noregions.xml" if "{0}" in output_path else "preferences.xml"),'w')
@@ -364,6 +559,7 @@ This creates separate preferences-regions.xml and preferences-noregions.xml file
         prefs_str=prefs_str.replace("Materials/regions/materials.xml","Materials/default/materials.xml")#turn off regional textures
         prefs_out.write(prefs_str)
         prefs_out.close()
+    print("Total runtime=",int(time.time()-start_time),"sec")
 def check_text_encoding(path,filelist=None,binary_types=(".png",".dds",".rgb",".RGB",".jpg",".wav",".WAV",".btg.gz",".xcf.gz",".xcf",".XCF","Thumbs.db",".blend",".bmp",".gif", ".3ds",".3DS",".pdf",".ttf",".txf",".htsvoice",".odt",".ods",".xls",".mp3",".zip",".tar.gz"),exclude_dirs=[".git","Timezone"]):
     """filelist is intended for quick testing: see fgdata_nonascii_filelist.py"""
     def err_context(err):

git_catalog_repository.py

@@ -0,0 +1,50 @@
+# git diff --quiet e5f841bc84d31fee339191a59b8746cb4eb8074c -- ./Aircraft/
+
+import subprocess
+import os, sgprops
+
+class GITCatalogRepository:
+    def __init__(self, node, singleAircraft = False):
+        self._path = node.getValue("path")
+
+        if not os.path.exists(os.path.join(self._path , ".git")):
+            raise RuntimeError("not a Git directory:" + self._path )
+
+        self._usesSubmodules = node.getValue("uses-submodules", False)
+        self._singleAircraft = singleAircraft
+
+        self._currentRevision = subprocess.check_output(["git", "rev-parse", "HEAD"],
+            cwd = self._path)
+
+        self._aircraftPath = None
+        if node.hasChild("scan-suffix"):
+            self._aircraftPath = os.path.join(path, node.getValue("scan-suffix"))
+
+    @property
+    def path(self):
+        return self._path
+
+    @property
+    def aircraftPath(self):
+        return self._aircraftPath
+
+    def hasPathChanged(self, path, oldRev):
+        diffArgs = ["git", "diff", "--quiet", oldRev, "--"]
+        if not (self._usesSubmodules and self._singleAircraft):
+            diffArgs.append(path)
+
+        return subprocess.call(diffArgs, cwd = self._path)
+
+    def update(self):
+        subprocess.call(["git", "pull"])
+        self._currentRevision = subprocess.check_output(["git", "rev-parse", "HEAD"],
+                cwd = self._path)
+
+        if self._usesSubmodules:
+            subprocess.call(["git", "submodule", "update"], cwd = self._path)
+
+    def scmRevisionForPath(self, path):
+        if self._usesSubmodules:
+            return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd = self._path)
+
+        return self._currentRevision

git_discrete_repository.py

@@ -0,0 +1,34 @@
+# git diff --quiet e5f841bc84d31fee339191a59b8746cb4eb8074c -- ./Aircraft/
+ 
+import subprocess
+import os
+import sgprops
+
+import git_catalog_repository
+
+class GitDiscreteSCM:
+    def __init__(self, node):
+        
+        configNode = node.parent
+        
+        self._repos = {}
+        
+        # iterate over aicraft paths finding repositories
+        for g in config.getChildren("aircraft-dir"):     
+            repo = GITCatalogRepository(g, useSubmodules = False, 
+                singleAircraft = True)
+        
+
+    def hasPathChanged(self, path, oldRev):
+        
+        return self._repos[path].hasPathChanged(path, oldRev)
+
+    def update(self):
+        for r in self._repos:
+            r.update()
+
+    def scmRevisionForPath(self, path):
+        return self._repos[path].scmRevisionForPath(path)
+        
+
+ 

hudson_mac_build_release.sh

@@ -15,59 +15,47 @@ mkdir -p output
 rm -rf output/*
 rm -rf $WORKSPACE/dist/include/simgear $WORKSPACE/dist/libSim* $WORKSPACE/dist/libsg*.a
 
+PATH=$PATH:$QTPATH
+echo "Build path is: $PATH"
+
 ###############################################################################
 echo "Starting on SimGear"
 pushd sgBuild
-cmake -DCMAKE_INSTALL_PREFIX:PATH=$WORKSPACE/dist -G Xcode ../simgear
+cmake -DCMAKE_INSTALL_PREFIX:PATH=$WORKSPACE/dist -DCMAKE_BUILD_TYPE=RelWithDebInfo ../simgear
 
 # compile
-xcodebuild -configuration RelWithDebInfo -target install  build
+make
 
 if [ $? -ne '0' ]; then
     echo "make simgear failed"
     exit 1
 fi
 
+make install
+
+
 popd
 
 ################################################################################
 echo "Starting on FlightGear"
 pushd fgBuild
-cmake -DCMAKE_INSTALL_PREFIX:PATH=$WORKSPACE/dist -G Xcode ../flightgear
+cmake -DFG_NIGHTLY=1 -DCMAKE_INSTALL_PREFIX:PATH=$WORKSPACE/dist -DCMAKE_BUILD_TYPE=RelWithDebInfo ../flightgear
 
-xcodebuild -configuration RelWithDebInfo -target install  build
+make
 
 if [ $? -ne '0' ]; then
     echo "make flightgear failed"
     exit 1
 fi
 
+make install
+
 popd
 
 chmod +x $WORKSPACE/dist/bin/osgversion
 
-################################################################################
-echo "Building Macflightgear launcher"
-
-#OSX_TARGET="10.6"
-# JMT - disabling setting the sysroot since it's breaking things on 
-# current build slave. Sinc ethe slave runs 10.6 natively, we don't 
-# actually need to set these. Real solution would be to use a proper
-# build system for the Mac-launcher of course.
-# -mmacosx-version-min=$OSX_TARGET -isysroot $SDK_PATH
-
-pushd maclauncher/FlightGearOSX
-
-# compile the stub executable
-gcc -o FlightGear main.m \
-    -framework Cocoa -framework RubyCocoa -framework Foundation -framework AppKit
-
-popd
 
 ################################################################################
-echo "Syncing base packages files from sphere.telascience.org"
-rsync -avz --filter 'merge base-package.rules' \
- -e ssh jturner@sphere.telascience.org:/home/jturner/fgdata .
 
 # run the unlock script now - we need to do this right before code-signing,
 # or the keychain may automatically re-lock after some period of time

hudson_mac_package_release.rb

@@ -7,6 +7,19 @@ include FileUtils
 $osgLibs = ['osgFX', 'osgParticle', 'osg', 'osgGA', 'osgText', 'osgUtil', 'osgSim', 'osgViewer', 'osgDB']
 $osgPlugins = ['ac', 'osg', 'freetype', 'imageio', 'rgb', 'txf', 'mdl', '3ds']
 
+# from http://drawingablank.me/blog/ruby-boolean-typecasting.html
+class String
+  def to_bool
+    return true if self == true || self =~ (/^(true|t|yes|y|1)$/i)
+    return false if self == false || self.blank? || self =~ (/^(false|f|no|n|0)$/i)
+    raise ArgumentError.new("invalid value for Boolean: \"#{self}\"")
+  end
+end
+
+class NilClass
+  def to_bool; false; end
+end
+
 def runOsgVersion(option)
   env = "export DYLD_LIBRARY_PATH=#{Dir.pwd}/dist/lib"
   bin = Dir.pwd + "/dist/bin/osgversion"
@@ -19,41 +32,35 @@ $openThreadsSoVersion=runOsgVersion('openthreads-soversion-number')
 
 $codeSignIdentity = ENV['FG_CODESIGN_IDENTITY']
 puts "Code signing identity is #{$codeSignIdentity}"
-
 puts "osgVersion=#{osgVersion}, so-number=#{$osgSoVersion}"
 
-def fix_install_names(object)
-  #puts "fixing install names for #{object}"
-  
-  $osgLibs.each do |l|
-    oldName = "lib#{l}.#{$osgSoVersion}.dylib"
-    newName = "@executable_path/../Frameworks/#{oldName}"
-    `install_name_tool -change #{oldName} #{newName} #{object}`
-  end
-  
-  oldName = "libOpenThreads.#{$openThreadsSoVersion}.dylib"
-  newName= "@executable_path/../Frameworks/#{oldName}"
-  `install_name_tool -change #{oldName} #{newName} #{object}`
-end
+$isRelease = ENV['FG_IS_RELEASE'].to_bool
+puts "Is-release? : ##{$isRelease}"
 
 $prefixDir=Dir.pwd + "/dist"
 dmgDir=Dir.pwd + "/image"
 srcDir=Dir.pwd + "/flightgear"
 
-def code_sign(path)
-  puts "Signing #{path}"
-  `codesign -s "#{$codeSignIdentity}" #{path}`
-end
-
 puts "Erasing previous image dir"
 `rm -rf #{dmgDir}`
 
+bundle=dmgDir + "/FlightGear.app"
+
+# run macdeployt before we rename the bundle, otherwise it
+# can't find the bundle executable
+puts "Running macdeployqt on the bundle to copy Qt libraries"
+`macdeployqt #{$prefixDir}/fgfs.app`
+
+puts "Moving & renaming app bundle"
+`mkdir -p #{dmgDir}`
+`mv #{$prefixDir}/fgfs.app #{bundle}`
+
 bundle=dmgDir + "/FlightGear.app"
 contents=bundle + "/Contents"
 macosDir=contents + "/MacOS"
 $frameworksDir=contents +"/Frameworks"
 resourcesDir=contents+"/Resources"
-osgPluginsDir=contents+"/PlugIns/osgPlugins-#{osgVersion}"
+osgPluginsDir=contents+"/PlugIns/osgPlugins"
 
 # for writing copyright year to Info.plist
 t = Time.new
@@ -62,7 +69,13 @@ fgCurrentYear = t.year
 fgVersion = File.read("#{srcDir}/version").strip
 volName="\"FlightGear #{fgVersion}\""
 
-dmgPath = Dir.pwd + "/output/FlightGear-#{fgVersion}.dmg"
+if $isRelease
+  dmgPath = "" # no 'lite' build for release candidates
+  dmgFullPath = Dir.pwd + "/output/FlightGear-#{fgVersion}.dmg"
+else
+  dmgPath = Dir.pwd + "/output/FlightGear-#{fgVersion}-nightly.dmg"
+  dmgFullPath = Dir.pwd + "/output/FlightGear-#{fgVersion}-nightly-full.dmg"
+end
 
 puts "Creating directory structure"
 `mkdir -p #{macosDir}`
@@ -70,24 +83,22 @@ puts "Creating directory structure"
 `mkdir -p #{resourcesDir}`
 `mkdir -p #{osgPluginsDir}`
 
-puts "Copying binaries"
-`cp #{$prefixDir}/fgfs.app/Contents/MacOS/fgfs #{macosDir}/fgfs`
-bins = ['fgjs', 'fgcom', 'fgviewer']
+
+puts "Copying auxilliary binaries"
+bins = ['fgjs', 'fgcom']
 bins.each do |b|
   if !File.exist?("#{$prefixDir}/bin/#{b}")
     next
   end
-  
+
   outPath = "#{macosDir}/#{b}"
   `cp #{$prefixDir}/bin/#{b} #{outPath}`
-  fix_install_names(outPath)
 end
 
 puts "copying libraries"
 $osgLibs.each do |l|
   libFile = "lib#{l}.#{$osgSoVersion}.dylib"
   `cp #{$prefixDir}/lib/#{libFile} #{$frameworksDir}`
-  fix_install_names("#{$frameworksDir}/#{libFile}")
 end
 
 # and not forgetting OpenThreads
@@ -95,17 +106,8 @@ libFile = "libOpenThreads.#{$openThreadsSoVersion}.dylib"
 `cp #{$prefixDir}/lib/#{libFile} #{$frameworksDir}`
 
 $osgPlugins.each do |p|
-  pluginFile = "osgdb_#{p}.so"
-  `cp #{$prefixDir}/lib/osgPlugins-#{osgVersion}/#{pluginFile} #{osgPluginsDir}`
-  fix_install_names("#{osgPluginsDir}/#{pluginFile}")
-end
-
-# Macflightgear launcher
-puts "Copying Macflightgear launcher files"
-
-Dir.chdir "maclauncher/FlightGearOSX" do
-  `cp FlightGear #{macosDir}`
-  `rsync -a *.rb *.lproj *.sh *.tiff *.html #{resourcesDir}`
+  pluginFile = "osgdb_#{p}.dylib"
+  `cp #{$prefixDir}/lib/osgPlugins/#{pluginFile} #{osgPluginsDir}`
 end
 
 if File.exist?("#{$prefixDir}/bin/fgcom-data")
@@ -125,24 +127,32 @@ File.open("#{contents}/Info.plist", 'w') { |f|
 `cp #{srcDir}/COPYING #{dmgDir}`
 
 # move documentation to a public place
-`mv fgdata/Docs/FGShortRef.pdf "#{dmgDir}/Quick Reference.pdf"`
-`mv fgdata/Docs/getstart.pdf "#{dmgDir}/Getting Started.pdf"`
+`cp fgdata/Docs/FGShortRef.pdf "#{dmgDir}/Quick Reference.pdf"`
+`cp fgdata/Docs/getstart.pdf "#{dmgDir}/Getting Started.pdf"`
+
+createArgs = "-format UDBZ -imagekey bzip2-level=9 -quiet -volname #{volName}"
+
+
+if !$isRelease
+  # create the 'lite' DMG without the base files
+
+  # code sign the entire bundle once complete - v2 code-signing
+  puts "Signing #{bundle}"
+  `codesign --deep -s "#{$codeSignIdentity}" #{bundle}`
+  puts "Creating DMG"
+
+  `rm #{dmgPath}`
+  `hdiutil create -srcfolder #{dmgDir} #{createArgs} #{dmgPath}`
+end
+
+puts "Creating full image with data"
 
 puts "Copying base package files into the image"
 `rsync -a fgdata/ #{resourcesDir}/data`
 
-# code sign all executables in MacOS dir. Do this last since reource
-# changes will invalidate the signature!
-Dir.foreach(macosDir) do |b|
-    if b == '.' or b == '..' then
-        next
-    end
-  code_sign("#{macosDir}/#{b}")
-end
+# re-sign the entire bundle
+puts "Re-signing full #{bundle}"
+`codesign --force --deep -s "#{$codeSignIdentity}" #{bundle}`
 
-puts "Creating DMG"
-
-createArgs = "-format UDBZ -imagekey bzip2-level=9 -quiet -volname #{volName}"
-
-`rm #{dmgPath}`
-`hdiutil create -srcfolder #{dmgDir} #{createArgs} #{dmgPath}`
+`rm #{dmgFullPath}`
+`hdiutil create -srcfolder #{dmgDir} #{createArgs} #{dmgFullPath}`

hudson_win_build32.bat

@@ -1,22 +0,0 @@
-IF NOT DEFINED WORKSPACE SET WORKSPACE=%~dp0
-
-SET /P SIMGEAR_VERSION=<%WORKSPACE%\simgear\version
-ECHO #define SIMGEAR_VERSION "%SIMGEAR_VERSION%" > %WORKSPACE%\simgear\simgear\version.h
-
-rem set PATH=%PATH%;D:\Program Files (x86)\CMake 2.8\bin
-rem call "C:\Program Files (x86)\Microsoft Visual Studio 10.0\Common7\Tools\vsvars32.bat"
-
-md build-sg
-md build-fg
-md build-fgrun
-cd build-sg
-cmake ..\simgear -G "Visual Studio 10" -DMSVC_3RDPARTY_ROOT=%WORKSPACE% -DCMAKE_INSTALL_PREFIX:PATH=%WORKSPACE%/install/msvc100/SimGear -DBOOST_ROOT=%WORKSPACE%/Boost
-cmake --build . --config Release --target INSTALL
-
-cd ..\build-fg
-cmake ..\flightgear -G "Visual Studio 10" -DMSVC_3RDPARTY_ROOT=%WORKSPACE% -DCMAKE_INSTALL_PREFIX:PATH=%WORKSPACE%/install/msvc100/FlightGear -DPNG_LIBRARY=%WORKSPACE%/3rdParty/lib/libpng16.lib -DFLTK_FLUID_EXECUTABLE=%WORKSPACE%/3rdParty/bin/fluid.exe -DBOOST_ROOT=%WORKSPACE%/Boost
-cmake --build . --config Release --target INSTALL
-
-cd ..\build-fgrun
-cmake ..\fgrun -G "Visual Studio 10" -DMSVC_3RDPARTY_ROOT:PATH=%WORKSPACE% -DCMAKE_INSTALL_PREFIX:PATH=%WORKSPACE%/install/msvc100/FGRun -DFLTK_FLUID_EXECUTABLE:FILEPATH=%WORKSPACE%/3rdParty/bin/fluid.exe -DGETTEXT_MSGFMT_EXECUTABLE:FILEPATH=%WORKSPACE%/3rdParty/bin/msgfmt.exe -DGETTEXT_MSGMERGE_EXECUTABLE:FILEPATH=%WORKSPACE%/3rdParty/bin/msgmerge.exe -DBOOST_ROOT=%WORKSPACE%/Boost
-cmake --build . --config Release --target INSTALL

hudson_win_build64.bat

@@ -1,24 +0,0 @@
-IF NOT DEFINED WORKSPACE SET WORKSPACE=%~dp0
-
-
-SET /P SIMGEAR_VERSION=<%WORKSPACE%\simgear\version
-ECHO #define SIMGEAR_VERSION "%SIMGEAR_VERSION%" > %WORKSPACE%\simgear\simgear\version.h
-
-rem ECHO #define SIMGEAR_VERSION "2.9.0" > %WORKSPACE%\simgear\simgear\version.h
-rem set PATH=%PATH%;D:\Program Files (x86)\CMake 2.8\bin
-rem call "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat" amd64
-
-md build-sg64
-md build-fg64
-md build-fgrun64
-cd build-sg64
-cmake ..\SimGear -G "Visual Studio 10 Win64" -DMSVC_3RDPARTY_ROOT=%WORKSPACE% -DCMAKE_INSTALL_PREFIX:PATH=%WORKSPACE%/install/msvc100-64/SimGear -DBOOST_ROOT=%WORKSPACE%/Boost
-cmake --build . --config Release --target INSTALL
-
-cd ..\build-fg64
-cmake ..\flightgear -G "Visual Studio 10 Win64" -DMSVC_3RDPARTY_ROOT=%WORKSPACE% -DCMAKE_INSTALL_PREFIX:PATH=%WORKSPACE%/install/msvc100-64/FlightGear -DFLTK_FLUID_EXECUTABLE=%WORKSPACE%/3rdParty/bin/fluid.exe -DBOOST_ROOT=%WORKSPACE%/Boost -DWITH_FGPANEL=OFF -DENABLE_PROFILE=OFF
-cmake --build . --config Release --target INSTALL
-
-cd ..\build-fgrun64
-cmake ..\fgrun -G "Visual Studio 10 Win64" -DMSVC_3RDPARTY_ROOT:PATH=%WORKSPACE% -DCMAKE_INSTALL_PREFIX:PATH=%WORKSPACE%/install/msvc100-64/FGRun -DFLTK_FLUID_EXECUTABLE:FILEPATH=%WORKSPACE%/3rdParty/bin/fluid.exe -DGETTEXT_MSGFMT_EXECUTABLE:FILEPATH=%WORKSPACE%/3rdParty/bin/msgfmt.exe -DGETTEXT_MSGMERGE_EXECUTABLE:FILEPATH=%WORKSPACE%/3rdParty/bin/msgmerge.exe -DBOOST_ROOT=%WORKSPACE%/Boost
-cmake --build . --config Release --target INSTALL

hudson_win_package_release.bat

@@ -1,40 +0,0 @@
-ECHO OFF
-
-IF NOT DEFINED WORKSPACE SET WORKSPACE=%~dp0
-ECHO Packaging root is %WORKSPACE%
-
-subst X: /D
-subst X: %WORKSPACE%.
-
-REM construct information file to be read by Inno-setup
-
-
-set PATH=%WORKSPACE%\install\msvc100\OpenSceneGraph\bin;%PATH%
-
-REM add 7-zip to the PATH
-set PATH=%PATH%;C:\Program Files\7-zip
-
-REM indirect way to get command output into an environment variable
-osgversion --so-number > %TEMP%\osg-so-number.txt
-osgversion --version-number > %TEMP%\osg-version.txt
-osgversion --openthreads-soversion-number > %TEMP%\openthreads-so-number.txt
-
-SET /P FLIGHTGEAR_VERSION=<flightgear\version
-SET /P OSG_VERSION=<%TEMP%\osg-version.txt
-SET /P OSG_SO_NUMBER=<%TEMP%\osg-so-number.txt
-SET /P OT_SO_NUMBER=<%TEMP%\openthreads-so-number.txt
-
-ECHO #define FGVersion "%FLIGHTGEAR_VERSION%" > InstallConfig.iss
-ECHO #define OSGVersion "%OSG_VERSION%" >> InstallConfig.iss
-ECHO #define OSGSoNumber "%OSG_SO_NUMBER%" >> InstallConfig.iss
-ECHO #define OTSoNumber "%OT_SO_NUMBER%" >> InstallConfig.iss
-
-set DATA_FILE=FlightGear-%FLIGHTGEAR_VERSION%-data
-
-REM extract the data files
-7z e -aoa %DATA_FILE%.tar.bz2 && 7z x -aoa %DATA_FILE%.tar
-
-REM run Inno-setup!
-REM use iscc instead of compil32 for better error reporting
-
-iscc FlightGear.iss

installWinDeps.bat

@@ -1,6 +1,7 @@
 REM ExternalProject can't cleanly extract a zip into an existing directory
 REM Instead we extract to a subdir, and then move the directories we want
 REM using this bat file.
+REM This script is used by the SuperBuild CMakeLists.txt
 
 echo %CD%
 IF EXIST winDeps/3rdParty (

maintain_catalog.py

@@ -0,0 +1,226 @@
+#!/usr/bin/python
+
+import os, sys, re, glob, shutil
+import subprocess
+import sgprops
+import argparse
+import urllib2
+import package as pkg
+
+import svn_catalog_repository
+import git_catalog_repository
+import git_discrete_repository
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--clean", help="Regenerate every package",
+     action="store_true")
+parser.add_argument("--update", help="Update/pull SCM source",
+     action="store_true")
+parser.add_argument("--force-dirty", dest="forcedirty",
+     help="Mark every package as dirty", action="store_true")
+parser.add_argument("--no-update",
+     dest = "noupdate",
+     help="Disable updating from SCM source",
+     action="store_true")
+parser.add_argument("--no-upload",
+     dest = "noupload",
+     help="Disable uploading to destination server",
+     action="store_true")
+parser.add_argument("dir", help="Catalog directory")
+args = parser.parse_args()
+
+CATALOG_VERSION = 4
+includePaths = []
+packages = {}
+
+def scanPackages(scmRepo):
+    result = []
+    globPath = scmRepo.aircraftPath
+    if globPath is None:
+        return result
+
+    print "Scanning", globPath
+    print os.getcwd()
+    for d in glob.glob(globPath):
+        # check dir contains at least one -set.xml file
+        if len(glob.glob(os.path.join(d, "*-set.xml"))) == 0:
+            print "no -set.xml in", d
+            continue
+
+        result.append(pkg.PackageData(d, scmRepo))
+
+    return result
+
+def initScmRepository(node):
+    scmType = node.getValue("type")
+    if (scmType == "svn"):
+        return svn_catalog_repository.SVNCatalogRepository(node)
+    elif (scmType == "git"):
+        return git_catalog_repository.GITCatalogRepository(node)
+    elif (scmType == "git-discrete"):
+        return git_discrete_repository.GitDiscreteSCM(node)
+    elif (scmType == None):
+        raise RuntimeError("No scm/type defined in catalog configuration")
+    else:
+        raise RuntimeError("Unspported SCM type:" + scmType)
+
+def initRepositories():
+    repositories = []
+
+    for scm in config.getChildren("scm"):
+        scmRepo = initScmRepository(scm)
+        if args.update or (not args.noupdate and scm.getValue("update")):
+            scmRepo.update()
+        # presumably include repos in parse path
+        # TODO: make this configurable
+        includePaths.append(scmRepo.path)
+        repositories.append(scmRepo)
+
+    return repositories
+
+def processUpload(node, outputPath):
+    if args.noupload or not node.getValue("enabled", True):
+        print "Upload disabled"
+        return
+
+    uploadType = node.getValue("type")
+    if (uploadType == "rsync"):
+        subprocess.call(["rsync", node.getValue("args", "-az"), ".",
+            node.getValue("remote")],
+        cwd = outputPath)
+    elif (uploadType == "rsync-ssh"):
+        print "Doing rsync upload to:", node.getValue("remote")
+        subprocess.call(["rsync", node.getValue("args", "-azve"),
+            "ssh", ".",
+            node.getValue("remote")],
+            cwd = outputPath)
+    elif (uploadType == "scp"):
+        subprocess.call(["scp", node.getValue("args", "-r"), ".",
+            node.getValue("remote")],
+            cwd = outputPath)
+    else:
+        raise RuntimeError("Unsupported upload type:" + uploadType)
+
+def parseExistingCatalog():
+    global existingCatalogPath
+    global previousCatalog
+
+    # contains existing catalog
+    existingCatalogPath = os.path.join(outPath, 'catalog.xml')
+
+    if not os.path.exists(existingCatalogPath):
+        url = config.getValue("template/url")
+        print "Attempting downloading from", url
+        try:
+        # can happen on new or from clean, try to pull current
+        # catalog from the upload location
+            response = urllib2.urlopen(url, timeout = 5)
+            content = response.read()
+            f = open(existingCatalogPath, 'w' )
+            f.write( content )
+            f.close()
+            print "...worked"
+        except urllib2.URLError as e:
+            print "Downloading current catalog failed", e, "from", url
+
+rootDir = args.dir
+if not os.path.isabs(rootDir):
+    rootDir = os.path.abspath(rootDir)
+os.chdir(rootDir)
+
+configPath = 'catalog.config.xml'
+if not os.path.exists(configPath):
+    raise RuntimeError("no config file found at:" + configPath)
+
+config = sgprops.readProps(configPath)
+
+# out path
+outPath = config.getValue('output-dir')
+if outPath is None:
+    # default out path
+    outPath = os.path.join(rootDir, "output")
+elif not os.path.isabs(outPath):
+    outPath = os.path.join(rootDir, "output")
+
+if args.clean:
+    print "Cleaning output"
+    shutil.rmtree(outPath)
+
+if not os.path.exists(outPath):
+    os.mkdir(outPath)
+
+thumbnailPath = os.path.join(outPath, config.getValue('thumbnail-dir', "thumbnails"))
+if not os.path.exists(thumbnailPath):
+    os.mkdir(thumbnailPath)
+
+thumbnailUrls = list(t.value for t in config.getChildren("thumbnail-url"))
+
+for i in config.getChildren("include-dir"):
+    if not os.path.exists(i.value):
+        print "Skipping missing include path:", i.value
+        continue
+    includePaths.append(i.value)
+
+parseExistingCatalog()
+repositories = initRepositories()
+
+for scm in repositories:
+    for p in scanPackages(scm):
+        try:
+            p.scanSetXmlFiles(includePaths)
+            packages[p.id] = p
+        except:
+            print "Skipping SCM package due to exception:", p.path
+
+if os.path.exists(existingCatalogPath):
+    try:
+        previousCatalog = sgprops.readProps(existingCatalogPath)
+    except:
+        print "Previous catalog is malformed"
+        previousCatalog = sgprops.Node()
+
+    for p in previousCatalog.getChildren("package"):
+        pkgId = p.getValue("id")
+        if not pkgId in packages.keys():
+            print "Orphaned old package:", pkgId
+            continue
+
+        packages[pkgId].setPreviousData(p)
+else:
+    print "No previous catalog"
+
+catalogNode = sgprops.Node("catalog")
+sgprops.copy(config.getChild("template"), catalogNode)
+
+catalogNode.getChild("catalog-version", create = True).value = CATALOG_VERSION
+mirrorUrls = list(m.value for m in config.getChildren("mirror"))
+
+packagesToGenerate = []
+for p in packages.values():
+    if p.isSourceModified or args.forcedirty:
+        packagesToGenerate.append(p)
+    else:
+        p.useExistingCatalogData()
+
+excludeFilePath = os.path.join(rootDir, "zip-excludes.lst")
+
+# def f(x):
+#     x.generateZip(outPath)
+#     x.extractThumbnails(thumbnailPath)
+#     return True
+#
+# p = Pool(8)
+# print(p.map(f,packagesToGenerate))
+
+for p in packagesToGenerate:
+   p.generateZip(outPath, excludeFilePath)
+   p.extractThumbnails(thumbnailPath)
+
+print "Creating catalog"
+for p in packages.values():
+    catalogNode.addChild(p.packageNode(mirrorUrls, thumbnailUrls[0]))
+
+catalogNode.write(os.path.join(outPath, "catalog.xml"))
+
+for up in config.getChildren("upload"):
+    processUpload(up, outPath)

package.py

@@ -0,0 +1,239 @@
+import os, subprocess
+import sgprops
+import hashlib # for MD5
+import shutil # for copy2
+import catalogTags
+
+standardTagSet = frozenset(catalogTags.tags)
+def isNonstandardTag(t):
+    return t not in standardTagSet
+
+thumbnailNames = ["thumbnail.png", "thumbnail.jpg"]
+
+class VariantData:
+    def __init__(self, path, node):
+        #self._primary = primary
+        self._path = path
+        self._name = node.getValue("sim/description")
+        if (not self._name):
+            print "Missing description for " + path
+            self._name = "Missing description:" + self.id
+
+        # ratings
+
+        # seperate thumbnails
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def id(self):
+        return self._path[:-8] # "remove -set.xml" (8 chars)
+
+    @property
+    def catalogNode(self):
+        n = sgprops.Node("variant")
+        n.addChild("id").value = self.id
+        n.addChild("name").value = self.name
+        return n
+
+class PackageData:
+    def __init__(self, path, scmRepo):
+        self._path = path
+        self._scm = scmRepo
+        self._previousSCMRevision = None
+        self._previousRevision = 0
+        self._thumbnails = []
+        self._variants = {}
+        self._revision = 0
+        self._md5 = None
+        self._fileSize = 0
+        self._primarySetXmlPath = None
+        self._node = sgprops.Node("package")
+
+    def setPreviousData(self, node):
+        self._previousRevision = node.getValue("revision")
+        self._previousMD5 = node.getValue("md5")
+        self._previousSCMRevision = node.getValue("scm-revision")
+        self._fileSize = int(node.getValue("file-size-bytes"))
+
+    @property
+    def id(self):
+        return self._primarySetXmlPath
+
+    @property
+    def thumbnails(self):
+        return self._thumbnails
+
+    @property
+    def path(self):
+        return self._path
+
+    @property
+    def variants(self):
+        return self._variants
+
+    @property
+    def scmRevision(self):
+        currentRev = self._scm.scmRevisionForPath(self.path)
+        if (currentRev is None):
+            raise RuntimeError("Unable to query SCM revision of files")
+
+        return currentRev
+
+    @property
+    def isSourceModified(self):
+        if (self._previousSCMRevision == None):
+            return True
+
+        if (self._previousSCMRevision == self.scmRevision):
+            return False
+
+        return True
+
+    def scanSetXmlFiles(self, includes):
+        foundPrimary = False
+        foundMultiple = False
+
+        for f in os.listdir(self.path):
+            if not f.endswith("-set.xml"):
+                continue
+
+            p = os.path.join(self.path, f)
+            node = sgprops.readProps(p, includePaths = includes)
+            if not node.hasChild("sim"):
+                continue
+
+            simNode = node.getChild("sim")
+            # honour variosu exclusion flags
+            if (simNode.getValue("exclude-from-catalog", False) or simNode.getValue("exclude-from-gui", False)):
+                continue
+
+            primary = simNode.getValue("variant-of", None)
+            if primary:
+                if not primary in self.variants:
+                    self._variants[primary] = []
+                self._variants[primary].append(VariantData(f, node))
+                continue
+
+            if foundPrimary:
+                if not foundMultiple:
+                    print "Multiple primary -set.xml files at:" + self.path
+                    print "\t" + p
+                    foundMultiple = True
+                continue
+            else:
+                foundPrimary = True;
+                self._primarySetXmlPath = f[:-8] # trim -set.xml
+
+            self.parsePrimarySetNode(simNode)
+
+            for n in thumbnailNames:
+                if os.path.exists(os.path.join(self.path, n)):
+                    self._thumbnails.append(n)
+
+        if not foundPrimary:
+            raise RuntimeError("No primary -set.xml found at:" + self.path)
+
+
+
+    def parsePrimarySetNode(self, sim):
+
+        # basic / mandatory values
+        self._node.addChild('name').value = sim.getValue('description')
+
+        longDesc = sim.getValue('long-description')
+        if longDesc is not None:
+            self._node.addChild('description').value = longDesc
+
+        # copy all the standard values
+        for p in ['status', 'author', 'license']:
+            v = sim.getValue(p)
+            if v is not None:
+                self._node.addChild(p).value = v
+
+        # ratings
+        if sim.hasChild('rating'):
+            pkgRatings = self._node.addChild('rating')
+            for r in ['FDM', 'systems', 'cockpit', 'model']:
+                pkgRatings.addChild(r).value = sim.getValue('rating/' + r, 0)
+
+        # copy tags
+        if sim.hasChild('tags'):
+            for c in sim.getChild('tags').getChildren('tag'):
+                if isNonstandardTag(c.value):
+                    print "Skipping non-standard tag:", c.value, self.path
+                else:
+                    self._node.addChild('tag').value = c.value
+
+        for t in sim.getChildren("thumbnail"):
+            self._thumbnails.append(t.value)
+
+    def validate(self):
+        for t in self._thumbnails:
+            if not os.path.exists(os.path.join(self.path, t)):
+                raise RuntimeError("missing thumbnail:" + t);
+
+    def generateZip(self, outDir, globalExcludePath):
+        self._revision = self._previousRevision + 1
+
+        baseName = os.path.basename(self.path)
+        zipName = baseName + ".zip"
+        zipFilePath = os.path.join(outDir, zipName)
+
+        os.chdir(os.path.dirname(self.path))
+
+        print "Creating zip", zipFilePath
+        # anything we can do to make this faster?
+
+        zipArgs = ['zip', '--quiet', '-r']
+        if os.path.exists(globalExcludePath):
+            zipArgs += [ "-x@" + globalExcludePath]
+
+        excludePath = os.path.join(self.path, 'package-exclude.lst')
+        if (os.path.exists(excludePath)):
+            print self.id, "has zip exclude list"
+            zipArgs += ["-x@" + excludePath]
+
+        zipArgs += [zipFilePath, baseName]
+        subprocess.call(zipArgs)
+
+        zipFile = open(zipFilePath, 'r')
+        self._md5 = hashlib.md5(zipFile.read()).hexdigest()
+        self._fileSize = os.path.getsize(zipFilePath)
+
+    def useExistingCatalogData(self):
+        self._md5 = self._previousMD5
+
+    def packageNode(self, mirrorUrls, thumbnailUrl):
+        self._node.addChild("id").value = self.id
+        self._node.getChild("md5", create = True).value = self._md5
+        self._node.getChild("file-size-bytes", create = True).value = self._fileSize
+        self._node.getChild("revision", create = True).value = int(self._revision)
+        self._node.getChild("scm-revision", create = True).value = self.scmRevision
+
+        baseName = os.path.basename(self.path)
+        self._node.getChild("dir", create = True).value = baseName
+        zipName = baseName + ".zip"
+
+        for m in mirrorUrls:
+            self._node.addChild("url").value = m + "/" + zipName
+
+        for t in self._thumbnails:
+            self._node.addChild("thumbnail-path").value = t
+            self._node.addChild("thumbnail").value = thumbnailUrl + "/" + self.id + "_" + t
+
+        for pr in self._variants:
+            for vr in self._variants[pr]:
+                self._node.addChild(vr.catalogNode)
+
+        return self._node
+
+    def extractThumbnails(self, thumbnailDir):
+        for t in self._thumbnails:
+            fullName = self.id + "_" + t
+            shutil.copy2(os.path.join(self._path, t),
+                         os.path.join(thumbnailDir, fullName)
+                         )
+            # TODO : verify image format, size and so on

post_upload.py

@@ -0,0 +1,128 @@
+#!/usr/bin/python
+
+# this file runs on the download server (download.flightgear.org)
+# from the Jenkins upload-via-ssh jobs. It ensures that only complete
+# uploads are visible (and mirrored to SF).
+
+import os, sys, re, fnmatch
+from subprocess import call
+
+suffix = '.dmg'
+if sys.argv[1] == 'windows':
+    suffix = '.exe'
+if sys.argv[1] == 'linux':
+    suffix = '.tar.bz2'
+
+isReleaseCandidate = False
+if len(sys.argv) > 2 and sys.argv[2] == 'release':
+    isReleaseCandidate = True
+
+allSuffix = '*' + suffix
+
+print "Wildcard pattern is:" + allSuffix
+pattern = r'\w+-(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)([\w-]*)' + suffix
+sourceForgeUserHost = "jmturner@frs.sourceforge.net"
+sftpCommandFile = "sftp-commands"
+symbolDir = "/home/jenkins/symbols"
+
+if isReleaseCandidate:
+    publicRoot = "/var/www/html/builds/rc"
+    incomingDir = "/home/jenkins/incoming"
+    sourceForgePath = "/home/frs/project/f/fl/flightgear/release-candidate/"
+else:
+    publicRoot = "/var/www/html/builds/nightly"
+    incomingDir = "/home/jenkins/nightly-incoming"
+    sourceForgePath = "/home/frs/project/f/fl/flightgear/unstable/"
+
+os.chdir(publicRoot)
+
+def findFileVersion(dir):
+    for file in os.listdir(dir):
+        if fnmatch.fnmatch(file, allSuffix):
+            m = re.match(pattern, file)
+            if (m is not None):
+                return (m.group('major'), m.group('minor'), m.group('patch'))
+
+    return None
+
+incomingVer = findFileVersion(incomingDir)
+if incomingVer is None:
+    print "No incoming files found matching " + allSuffix
+    exit()
+
+existingVer = findFileVersion('.')
+
+# if files in dest location mis-match the version, archive them
+# and re-create the symlinks
+
+versionChange = (existingVer != incomingVer)
+
+oldFiles = []
+incomingFiles = []
+newFiles = []
+
+if versionChange:
+    print "Version number changing"
+
+    for file in os.listdir('.'):
+        if fnmatch.fnmatch(file, allSuffix):
+            if not os.path.islink(file):
+                oldFiles.append(file)
+            os.remove(file)
+
+for file in os.listdir(incomingDir):
+    if fnmatch.fnmatch(file, allSuffix):
+        incomingFiles.append(file)
+
+# copy and symlink
+for file in incomingFiles:
+    # move it to the public location
+    srcFile = os.path.join(incomingDir, file)
+
+    outFile = file
+    # insert -rc before suffix
+    if isReleaseCandidate:
+        m = re.match(r'(\w+-\d+\.\d+\.\d+[\w-]*)' + suffix, file)
+        outFile = m.group(1) + '-rc' + suffix
+        print "RC out name is " + outFile
+
+    os.rename(srcFile, outFile)
+    newFiles.append(outFile)
+
+    if not isReleaseCandidate:
+        # symlink for stable web URL
+        m = re.match(r'(\w+)-\d+\.\d+\.\d+-([\w-]+)' + suffix, file)
+        latestName = m.group(1) + '-latest-' + m.group(2) + suffix
+
+        print "Creating symlink from " + file + " to " + latestName
+        if os.path.exists(latestName):
+            print "\tremoving existing target"
+            os.remove(latestName)
+        os.symlink(file, latestName)
+
+
+# remove files from SF
+if len(oldFiles) > 0:
+    f = open(sftpCommandFile, 'w')
+    f.write("cd " + sourceForgePath + '\n')
+    for file in oldFiles:
+        print "Removing file " + file + " from SourceForge"
+        f.write("rm " + file + '\n')
+    f.write("bye\n")
+    f.close()
+
+    call(["sftp", "-b", sftpCommandFile, sourceForgeUserHost])
+    os.remove(sftpCommandFile)
+
+# upload to SourceForge
+for file in newFiles:
+    print "Uploading " + file + " to SourceForge"
+    call(["scp", file, sourceForgeUserHost + ":" + sourceForgePath + file])
+
+if sys.argv[1] == 'windows':
+    print "Archiving PDB files"
+    for file in os.listdir(incomingDir):
+        if fnmatch.fnmatch(file, "*.pdb"):
+            srcFile = os.path.join(incomingDir, file)
+            outFile = os.path.join(symbolDir, file)
+            os.rename(srcFile, outFile)

sgprops.py

@@ -11,9 +11,9 @@ class Node(object):
         self._parent = parent
         self._name = name
         self._value = None
-        self._index = 0
+        self._index = index
         self._children = []
-        
+
     @property
     def value(self):
         return self._value
@@ -21,21 +21,21 @@ class Node(object):
     @value.setter
     def value(self, v):
         self._value = v
-    
+
     @property
     def name(self):
         return self._name
-        
+
     @property
     def index(self):
         return self._index
-        
-    @property 
+
+    @property
     def parent(self):
         return self._parent
-        
+
     def getChild(self, n, i=None, create = False):
-     
+
         if i is None:
             i = 0
             # parse name as foo[999] if necessary
@@ -43,72 +43,79 @@ class Node(object):
             if m is not None:
                 n = m.group(1)
                 i = int(m.group(2))
-                
+
         for c in self._children:
             if (c.name == n) and (c.index == i):
                 return c
-                
+
         if create:
             c = Node(n, i, self)
             self._children.append(c)
             return c
         else:
             raise IndexError("no such child:" + str(n) + " index=" + str(i))
-            
+
     def addChild(self, n):
-        i = 0
-        
-        # find first free index
-        usedIndices = frozenset(c.index for c in self.getChildren(n))
-        while i < 1000:
-            if i not in usedIndices:
-                 break
-            i += 1
-        
+        # adding an existing instance
+        if isinstance(n, Node):
+            n._parent = self
+            n._index = self.firstUnusedIndex(n.name)
+            self._children.append(n)
+            return n
+
+        i = self.firstUnusedIndex(n)
         # create it via getChild
         return self.getChild(n, i, create=True)
-            
+
+    def firstUnusedIndex(self, n):
+        usedIndices = frozenset(c.index for c in self.getChildren(n))
+        i = 0
+        while i < 1000:
+            if i not in usedIndices:
+                 return i
+            i += 1
+        raise RuntimeException("too many children with name:" + n)
+
     def hasChild(self, nm):
         for c in self._children:
             if (c.name == nm):
                 return True
-                
+
         return False
-            
+
     def getChildren(self, n = None):
         if n is None:
             return self._children
-        
+
         return [c for c in self._children if c.name == n]
-            
+
     def getNode(self, path, cr = False):
         axes = path.split('/')
         nd = self
         for ax in axes:
             nd = nd.getChild(ax, create = cr)
-            
+
         return nd
-        
+
     def getValue(self, path, default = None):
         try:
             nd = self.getNode(path)
             return nd.value
         except:
             return default
-            
+
     def write(self, path):
         root = self._createXMLElement('PropertyList')
-        
         t = ET.ElementTree(root)
-        t.write(path, 'utf-8')
-        
+        t.write(path, 'utf-8', xml_declaration = True)
+
     def _createXMLElement(self, nm = None):
         if nm is None:
             nm = self.name
-            
+
         n = ET.Element(nm)
-        
-        # value and type specification 
+
+        # value and type specification
         try:
             if self._value is not None:
                 if isinstance(self._value, basestring):
@@ -126,98 +133,145 @@ class Node(object):
                         n.set('type', "bool")
         except UnicodeEncodeError:
             print "Encoding error with", self._value, type(self._value)
-            
+
         # index in parent
         if (self.index != 0):
-            n.set('n', self.index)
-            
+            n.set('n', str(self.index))
+
         # children
         for c in self._children:
             n.append(c._createXMLElement())
-            
+
         return n;
-        
+
 
 class PropsHandler(handler.ContentHandler):
-    def __init__(self, root = None, path = None, dataDirPath = None):
+    def __init__(self, root = None, path = None, includePaths = []):
         self._root = root
         self._path = path
         self._basePath = os.path.dirname(path)
-        self._dataDirPath = dataDirPath
+        self._includes = includePaths
         self._locator = None
-        
+
         if root is None:
             # make a nameless root node
             self._root = Node("", 0)
         self._current = self._root
-            
+
     def setDocumentLocator(self, loc):
-        self._locator = loc    
-        
-    def startElement(self, name, attrs):    
-        self._content = ''
+        self._locator = loc
+
+    def startElement(self, name, attrs):
+        self._content = None
         if (name == 'PropertyList'):
             return
-            
-        index = 0
+
         if 'n' in attrs.keys():
-            index = int(attrs['n'])
-            
-        self._current = self._current.getChild(name, index, create=True)
-            
+            try:
+                index = int(attrs['n'])
+            except:
+                print "Invalid index at line:", self._locator.getLineNumber(), "of", self._path
+                self._current = self._current.addChild(name)
+                return
+
+            self._current = self._current.getChild(name, index, create=True)
+        else:
+            self._current = self._current.addChild(name)
+
+
         if 'include' in attrs.keys():
             self.handleInclude(attrs['include'])
-            
+
         self._currentTy = None;
         if 'type' in attrs.keys():
             self._currentTy = attrs['type']
-                     
+
     def handleInclude(self, includePath):
         if includePath.startswith('/'):
             includePath = includePath[1:]
-   
+
         p = os.path.join(self._basePath, includePath)
         if not os.path.exists(p):
-            p = os.path.join(self._dataDirPath, includePath)
-            if not os.path.exists(p):
+            found = False
+            for i in self._includes:
+                p = os.path.join(i, includePath)
+                if os.path.exists(p):
+                    found = True
+                    break
+
+            if not found:
                 raise RuntimeError("include file not found", includePath, "at line", self._locator.getLineNumber())
-                
-        readProps(p, self._current, self._dataDirPath)
-            
+
+        readProps(p, self._current, self._includes)
+
     def endElement(self, name):
         if (name == 'PropertyList'):
             return
-            
+
         try:
             # convert and store value
             self._current.value = self._content
             if self._currentTy == "int":
-                self._current.value = int(self._content)
-            if self._currentTy is "bool":
-                self._current.value = bool(self._content)
-            if self._currentTy is "double":
-                self._current.value = float(self._content)
+                self._current.value = int(self._content) if self._content is not None else 0
+            if self._currentTy == "bool":
+                self._current.value = self.parsePropsBool(self._content)
+            if self._currentTy == "double":
+                if self._content is None:
+                    self._current.value = 0.0
+                else:
+                    if self._content.endswith('f'):
+                        self._content = self._content[:-1]
+                    self._current.value = float(self._content)
         except:
             print "Parse error for value:", self._content, "at line:", self._locator.getLineNumber(), "of:", self._path
-            
+
         self._current = self._current.parent
-            
+        self._content = None
+        self._currentTy = None
+
+
+    def parsePropsBool(self, content):
+        if content == "True" or content == "true":
+            return True
+
+        if content == "False" or content == "false":
+            return False
+
+        try:
+            icontent = int(content)
+            if icontent is not None:
+                if icontent == 0:
+                    return False
+                else:
+                    return True;
+        except:
+            return False
+
     def characters(self, content):
+        if self._content is None:
+            self._content = ''
         self._content += content
-            
+
     def endDocument(self):
         pass
-        
+
     @property
     def root(self):
         return self._root
 
-    
-def readProps(path, root = None, dataDirPath = None):
+def readProps(path, root = None, includePaths = []):
     parser = make_parser()
     locator = expatreader.ExpatLocator( parser )
-    h = PropsHandler(root, path, dataDirPath)
+    h = PropsHandler(root, path, includePaths)
     h.setDocumentLocator(locator)
     parser.setContentHandler(h)
     parser.parse(path)
-    return h.root
+    return h.root
+
+def copy(src, dest):
+    dest.value = src.value
+
+    # recurse over children
+    for c in src.getChildren() :
+        dc = dest.getChild(c.name, i = c.index, create = True)
+        copy(c, dc)

svn_catalog_repository.py

@@ -0,0 +1,41 @@
+
+import subprocess, os, sgprops
+import xml.etree.cElementTree as ET
+
+class SVNCatalogRepository:
+    def __init__(self, node):
+        path = node.getValue("path")
+        if not os.path.exists(path):
+            raise RuntimeError("No directory at:" + path)
+
+        self._path = path
+        xml = subprocess.check_output(["svn", "info", "--xml", path])
+        root = ET.fromstring(xml)
+
+        if (root.find(".//repository/root") == None):
+            raise RuntimeError("Not an SVN repository:" + path)
+
+        self._aircraftPath = None
+        if node.hasChild("scan-suffix"):
+            self._aircraftPath = os.path.join(path, node.getValue("scan-suffix"))
+
+    @property
+    def path(self):
+        return self._path
+
+    @property
+    def aircraftPath(self):
+        return self._aircraftPath
+
+    def hasPathChanged(self, path, oldRevision):
+        return self.scmRevisionForPath(path) != oldRevision
+
+    def scmRevisionForPath(self, path):
+        xml = subprocess.check_output(["svn", "info", "--xml", path])
+        root = ET.fromstring(xml)
+        commit = root.find(".//entry/commit")
+        return commit.get('revision', 0)
+
+    def update(self):
+        print "SVN update of", self._path
+        subprocess.call(["svn", "update", self._path])

template_catalog.xml

@@ -0,0 +1,52 @@
+<?xml version="1.0"?>
+<!--
+Template catalog - copy and modify for your site as required
+-->
+<PropertyList>
+    
+    <!--
+    catalogProps.addChild('version').value = '3.1.0'
+    catalogProps.addChild('id').value = 'org.flightgear.default'
+    catalogProps.addChild('license').value = 'GPL'
+    catalogProps.addChild('url').value = "http://fgfs.goneabitbursar.com/pkg/3.1.0/default-catalog.xml"
+
+    catalogProps.addChild('description').value = "Aircraft developed and maintained by the FlightGear project"
+
+    de = catalogProps.addChild('de')
+    # de.addChild('description').value = "<German translation of catalog description>"
+
+    fr = catalogProps.addChild('fr')
+    -->
+        
+        <version>3.4.*</version>
+        <version>3.5.*</version>
+        <version>3.6.*</version>
+        
+        <id>org.myorganisation.hangar</id>
+        <license>GPL</license>
+        <url>http://some.stable.url.com/foo/bar/catalog.xml</url>
+        
+        <description>A collection of interesting aircraft with some features
+        </description>
+        
+        <de>
+            <description>Au Deutsch</description>
+        </de>
+        
+        <fr>
+            <description>Francais</description>
+        </fr>
+        
+        <mirror>http://some.url/</mirror>
+        <!--  <mirror>another mirror</mirror> -->
+        
+        
+        <thumbnails>http://some.url/images</thumbnails>
+        
+        <git-repository>git://some.git.repo/</git-repository>
+        
+        <repository-prefix>Aircraft</repository-prefix>
+        
+</PropertyList>
+
+    

version

@@ -0,0 +1 @@
+2016.2.0

webhooks.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0"?>
+<!--
+  The <regex> field is evaluated by preg_match
+  http://php.net/manual/en/function.preg-match.php
+-->
+<PropertyList>
+  <webhook>
+    <regex>/^Merge branch/</regex>
+    <description>Detect merge branch when rebase has been ommited</description>
+  </webhook>
+  <webhook>
+    <regex>/^test$/</regex>
+    <description>Detect for dummy message</description>
+  </webhook>
+  <webhook>
+    <regex>/blabla/</regex>
+    <description>Detect for bad commit description</description>
+  </webhook>
+</PropertyList>