Changes to create a debug build

The former is an advanced configuration tool for users' joysticks.

.gitattributes

@@ -0,0 +1,23 @@
+# Set default behaviour, in case users don't have core.autocrlf set.
+* text=auto
+
+.gitattributes                 text export-ignore
+.gitignore                     text export-ignore
+*.py                           text
+*.txt                          text
+*.rst                          text
+*.xml                          text
+*.desktop                      text
+*.svg                          text
+*.jpg                          binary
+*.png                          binary
+*.gif                          binary
+*.bat                          text eol=crlf
+AUTHORS                        text
+COPYING                        text
+COPYING.*                      text
+README.*                       text
+Makefile                       text
+ChangeLog                      text
+ChangeLog.*                    text
+/download_and_compile.sh       text ident

.gitignore

@@ -1,6 +1,6 @@
 dist
 .DS_Store
-install
+install*/
 3rdParty
 3rdParty.x64
 boost_1_44_0
@@ -21,3 +21,4 @@ osgbuild
 CMakeCache.txt
 *.pyc
 build-*
+testOutput*

.gitmodules

@@ -1,20 +1,24 @@
 [submodule "simgear"]
 	path = simgear
-	url = https://git.code.sf.net/p/flightgear/simgear
+	url = ../simgear
 	branch = next
 [submodule "flightgear"]
 	path = flightgear
-	url = https://git.code.sf.net/p/flightgear/flightgear
+	url = ../flightgear
 	branch = next
 [submodule "fgrun"]
 	path = fgrun
-	url = https://git.code.sf.net/p/flightgear/fgrun
+	url = ../fgrun
 	branch = next
 [submodule "fgdata"]
 	path = fgdata
-	url = git://git.code.sf.net/p/flightgear/fgdata
+	url = ../fgdata
 	branch = next
 [submodule "windows-3rd-party"]
 	path = windows-3rd-party
-	url = https://git.code.sf.net/p/flightgear/windows-3rd-party
+	url = ../windows-3rd-party
 	branch = master
+[submodule "getstart"]
+	path = getstart
+	url = ../getstart
+	branch = next

FlightGear.iss

@@ -20,18 +20,13 @@
 
 #include "InstallConfig.iss"
 
-#if GetEnv("VSINSTALLDIR") == ""
-  #define VSInstallDir "C:\Program Files (x86)\Microsoft Visual Studio 14.0"
-#else
-  #define VSInstallDir GetEnv("VSINSTALLDIR")
-#endif
-
-#define VCInstallDir VSInstallDir + "\VC"
 #define InstallDir32 "X:\install\msvc140"
+#define InstallCompositor32 "X:\install\msvc140\compositor"
 #define OSGInstallDir InstallDir32 + "\OpenSceneGraph"
 #define OSGPluginsDir OSGInstallDir + "\bin\osgPlugins-" + OSGVersion
 
 #define InstallDir64 "X:\install\msvc140-64"
+#define InstallCompositor64 "X:\install\msvc140-64\compositor"
 #define OSG64InstallDir InstallDir64 + "\OpenSceneGraph"
 #define OSG64PluginsDir OSG64InstallDir + "\bin\osgPlugins-" + OSGVersion
 
@@ -39,7 +34,7 @@
 
 ; we copy everything in install/<arch>/bin except these, which aren't
 ; useful to the end-user to ship
-#define ExcludedBinaries "*smooth.exe,metar.exe,js_demo.exe"
+#define ExcludedBinaries "*smooth.exe,metar.exe"
 
 [Setup]
 AppId=FlightGear
@@ -81,6 +76,7 @@ Name: "desktopicon"; Description: "Create a &desktop icon"; GroupDescription: "A
 ; 32 bits install
 
 Source: "{#InstallDir32}\bin\*.*"; DestDir: "{app}\bin"; Excludes: "{#ExcludedBinaries}"; Flags: ignoreversion recursesubdirs; Check: not Is64BitInstallMode
+Source: "{#InstallCompositor32}\bin\fgfs.exe"; DestDir: "{app}\bin"; DestName: "fgfs-compositor.exe"; Excludes: "{#ExcludedBinaries}"; Flags: ignoreversion recursesubdirs; Check: not Is64BitInstallMode
 ;locale only exists for fgrun - which has been disabled
 ;Source: "{#InstallDir32}\share\locale\*"; DestDir: "{app}\bin\locale"; Flags: ignoreversion recursesubdirs; Check: not Is64BitInstallMode
 
@@ -89,13 +85,14 @@ Source: "{#ThirdPartyDir}\3rdParty\bin\OpenAL32.dll"; DestDir: "{app}\bin"; Chec
 Source: "{#ThirdPartyDir}\3rdParty\bin\libpng.dll"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
 Source: "{#ThirdPartyDir}\3rdParty\bin\libcurl.dll"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
 Source: "{#ThirdPartyDir}\3rdParty\bin\libintl-8.dll"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
-Source: "{#ThirdPartyDir}\3rdParty\bin\CrashRpt1403.dll"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
+Source: "{#ThirdPartyDir}\3rdParty\bin\sentry.dll"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
-Source: "{#ThirdPartyDir}\3rdParty\bin\crashrpt_lang.ini"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
+Source: "{#ThirdPartyDir}\3rdParty\bin\crashpad_handler.exe"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
-Source: "{#ThirdPartyDir}\3rdParty\bin\CrashSender1403.exe"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
+Source: "{#ThirdPartyDir}\3rdParty\bin\dbus-1-3.dll"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
-Source: "{#VCInstallDir}\redist\x86\Microsoft.VC140.CRT\*.dll"; DestDir:  "{app}\bin"; Check: not Is64BitInstallMode
+Source: "{#ThirdPartyDir}\3rdParty\bin\event_core.dll"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
 
 ; 64 bits install
 Source: "{#InstallDir64}\bin\*.*"; DestDir: "{app}\bin"; Excludes: "{#ExcludedBinaries}"; Flags: ignoreversion recursesubdirs; Check: Is64BitInstallMode
+Source: "{#InstallCompositor64}\bin\fgfs.exe"; DestDir: "{app}\bin"; DestName: "fgfs-compositor.exe"; Excludes: "{#ExcludedBinaries}"; Flags: ignoreversion recursesubdirs; Check: Is64BitInstallMode
 ;locale only exists for fgrun - which has been disabled
 ;Source: "{#InstallDir64}\share\locale\*"; DestDir: "{app}\bin\locale"; Flags: ignoreversion recursesubdirs; Check: Is64BitInstallMode
 
@@ -104,10 +101,10 @@ Source: "{#ThirdPartyDir}\3rdParty.x64\bin\OpenAL32.dll"; DestDir: "{app}\bin";
 Source: "{#ThirdPartyDir}\3rdParty.x64\bin\libpng.dll"; DestDir: "{app}\bin"; Check: Is64BitInstallMode
 Source: "{#ThirdPartyDir}\3rdParty.x64\bin\libcurl.dll"; DestDir: "{app}\bin"; Check: Is64BitInstallMode
 Source: "{#ThirdPartyDir}\3rdParty.x64\bin\libintl-8.dll"; DestDir: "{app}\bin"; Check: Is64BitInstallMode
-Source: "{#ThirdPartyDir}\3rdParty.x64\bin\CrashRpt1403.dll"; DestDir: "{app}\bin"; Check: Is64BitInstallMode
+Source: "{#ThirdPartyDir}\3rdParty.x64\bin\sentry.dll"; DestDir: "{app}\bin"; Check: Is64BitInstallMode
-Source: "{#ThirdPartyDir}\3rdParty.x64\bin\crashrpt_lang.ini"; DestDir: "{app}\bin"; Check: Is64BitInstallMode
+Source: "{#ThirdPartyDir}\3rdParty.x64\bin\crashpad_handler.exe"; DestDir: "{app}\bin"; Check: Is64BitInstallMode
-Source: "{#ThirdPartyDir}\3rdParty.x64\bin\CrashSender1403.exe"; DestDir: "{app}\bin"; Check: Is64BitInstallMode
+Source: "{#ThirdPartyDir}\3rdParty.x64\bin\dbus-1-3.dll"; DestDir: "{app}\bin"; Check: Is64BitInstallMode
-Source: "{#VCInstallDir}\redist\x64\Microsoft.VC140.CRT\*.dll"; DestDir: "{app}\bin"; Check: Is64BitInstallMode
+Source: "{#ThirdPartyDir}\3rdParty.x64\bin\event_core.dll"; DestDir: "{app}\bin"; Check: Is64BitInstallMode
 
 ; Include the base package
 #if IncludeData == "TRUE"
@@ -199,6 +196,7 @@ Name: "{userdocs}\FlightGear\Custom Scenery"; Permissions: everyone-modify; Chec
 [Icons]
 Name: "{userdesktop}\FlightGear {#FGVersion}"; Filename: "{app}\bin\fgfs.exe"; Parameters: "--launcher"; WorkingDir: "{app}\bin"; Tasks: desktopicon;
 Name: "{group}\FlightGear"; Filename: "{app}\bin\fgfs.exe"; Parameters: "--launcher"; WorkingDir: "{app}\bin";
+Name: "{group}\FlightGear - Compositor"; Filename: "{app}\bin\fgfs-compositor.exe"; Parameters: "--launcher"; WorkingDir: "{app}\bin";
 Name: "{group}\FlightGear Manual"; Filename: "{app}\data\Docs\getstart.pdf"
 Name: "{group}\FlightGear Documentation"; Filename: "{app}\data\Docs\index.html"
 Name: "{group}\Flightgear Wiki"; Filename: "http://wiki.flightgear.org"
@@ -328,6 +326,66 @@ begin
   end;
 end;
 
+var
+  UninstallCheckCleanPage: TNewNotebookPage;
+  UninstallBackButton: TNewButton;
+  UninstallNextButton: TNewButton;
+  DoCleanCheckbox : TNewCheckBox;
+  CleanHelp : TNewStaticText;
+
+procedure InitializeUninstallProgressForm();
+begin
+  UninstallProgressForm
+
+  UninstallCheckCleanPage := TNewNotebookPage.Create(UninstallProgressForm);
+  UninstallCheckCleanPage.Notebook := UninstallProgressForm.InnerNotebook;
+  UninstallCheckCleanPage.Parent := UninstallProgressForm.InnerNotebook;
+  UninstallCheckCleanPage.Align := alClient
+
+  DoCleanCheckbox := TNewCheckBox.Create(UninstallProgressForm);
+  DoCleanCheckbox.Parent := UninstallCheckCleanPage;
+  DoCleanCheckbox.Caption := 'Remove all settings, downloaded scenery and aircraft';
+  DoCleanCheckbox.Left := ScaleX(10);
+  DoCleanCheckbox.Top := ScaleY(10);
+
+  DoCleanCheckbox.Width := UninstallProgressForm.InnerNotebook.Width - ScaleX(20)
+  DoCleanCheckbox.Height := ScaleY(30)
+
+  CleanHelp := TNewStaticText.Create(UninstallProgressForm);
+  CleanHelp.Parent := UninstallCheckCleanPage;
+  CleanHelp.Top := DoCleanCheckbox.Top + DoCleanCheckbox.Height + ScaleY(10);
+  CleanHelp.Left := DoCleanCheckbox.Left;
+  CleanHelp.Width := DoCleanCheckbox.Width;
+  CleanHelp.Height := CleanHelp.AdjustHeight();
+
+  CleanHelp.WordWrap := True;
+  CleanHelp.Caption := 'FlightGear stores some settings in your user folder. In addition, ' +
+       'scenery or aircraft data may have been downloaded to the download directory. ' +
+       'To completely remove all these files, select this option.';
+
+  UninstallProgressForm.InnerNotebook.ActivePage := UninstallCheckCleanPage;
+
+   UninstallNextButton := TNewButton.Create(UninstallProgressForm);
+   UninstallNextButton.Caption := 'Next';
+    UninstallNextButton.Parent := UninstallProgressForm;
+    UninstallNextButton.Left :=
+      UninstallProgressForm.CancelButton.Left -
+      UninstallProgressForm.CancelButton.Width -
+      ScaleX(10);
+    UninstallNextButton.Top := UninstallProgressForm.CancelButton.Top;
+    UninstallNextButton.Width := UninstallProgressForm.CancelButton.Width;
+    UninstallNextButton.Height := UninstallProgressForm.CancelButton.Height;
+     UninstallNextButton.ModalResult := mrOk;
+
+    UninstallProgressForm.CancelButton.Enabled := True;
+    UninstallProgressForm.CancelButton.ModalResult := mrCancel;
+
+    if UninstallProgressForm.ShowModal = mrCancel then Abort;
+
+    UninstallProgressForm.InnerNotebook.ActivePage := UninstallProgressForm.InstallingPage;
+end;
+
+
 procedure CurStepChanged(CurStep: TSetupStep);
 var
   Version: TWindowsVersion;
@@ -353,7 +411,18 @@ begin
 end;
 
 procedure CurUninstallStepChanged(CurUninstallStep: TUninstallStep);
+     var ResultCode: Integer;
 begin
+  if CurUninstallStep = usUninstall then
+  begin
+     if DoCleanCheckbox.Checked = True then
+     begin
+         Log('Running clean uninstall');
+         Exec(ExpandConstant('{app}\bin\fgfs.exe'), '--uninstall', '', SW_HIDE, ewWaitUntilTerminated, ResultCode);
+         Log('clean uninstall completed');
+     end;
+  end;
+
   if CurUninstallStep = usPostUninstall then
     begin
       RemoveFirewallException('FlightGear', ExpandConstant('{app}') + '\bin\fgfs.exe');

Info.plist.in

@@ -15,7 +15,7 @@
 	<key>CFBundleExecutable</key>
 	<string>fgfs</string>
 	<key>CFBundleIdentifier</key>
-	<string>org.flightgear.FlightGear</string>
+	<string><%= fgBundleIdentifier %></string>
 	<key>CFBundleVersion</key>
 	<string><%= fgVersion %></string>
 	<key>CFBundleShortVersionString</key>

Nasa2FGearthview/README.md

@@ -1,96 +1,123 @@
 # Nasa2FGearthview
 A bash-script to convert NASA satellite images to ready-to-use
-textures for FG's EarthView using ImageMagick.
+textures for FG's EarthView using ImageMagick and normalmap
 
-For info about FGearthview, see the forum thread:
+You can get "normalmap" there:
+  https://github.com/planrich/normalmap
+
+For info about FGearthview, see the forum thread:\
   https://forum.flightgear.org/viewtopic.php?f=6&t=15754
-or this FG-wiki-page:
+
+or this FG-wiki-page:\
   http://wiki.flightgear.org/Earthview
 
+### Caution!
+Don't use this script on a server! It will most likely cause
+Denial-of-service (DoS). When working on these huge images, the
+harddisk throughput will cease occasionally and CPU / RAM usage will
+spike tremendously! So, only use this script on your home desktop
+computer, or if you don't mind several long server-outages...
+
 
 ------------------------------------
-About:
+## About:
 
 This script runs on Linux (maybe Mac also?) in a Bash
 (Bourne Again Shell) - Windows is not supported (by the nature of the
 script). Maybe it works on windows as well, I don't know, feel free
 to try, and please let me know! :)
 
-This will download the raw images from http://visibleearth.nasa.gov - 
-their server is not very fast, so I provide an alternative download
-location: https://musicchris.de/download/FG/EarthView/raw-data-NASA.7z
-This one is much quicker! If you really want the images directly from
-NASA, then provide "nasa" to the script (see below)
-
 In the end you will have 8 world-textures in .png and .dds format.
 Generally .dds is better in performance, but it won't work on some
 graphics cards. If this is the case for you, then try the .png files.
-For further information see:
+For further information see:\
 http://wiki.flightgear.org/index.php?title=DDS_Textures_in_FlightGear&redirect=no
 
-If you also converted the clouds, then you'll also find 8 cloud-
+If you also converted the clouds and the height maps, then you'll also
-textures in the format .png. Because the .dds-format has trouble with
+find 8 cloud- and 8 height textures (as well as their conversion to
-rendering heavy alpha images, which is because of it's compression
+normal maps) in the format .png. Because the .dds-format has trouble
-algorythm [1], I think it's useless to also build faulty files.
+with rendering heavy alpha images, which is because of it's
-However, this is not entirely true! It is possible to switch off the
+compression algorythm [1], I think it's useless to also build faulty
-.dds/DXT compression. But this results in huge files and is rather
+files.  However, this is not entirely true! It is possible to switch
-heavy on the GPU's RAM.
+off the .dds/DXT compression. But this results in huge files and is
+rather heavy on the GPU's RAM.
 
 Buckaroo has created a nice overview on dds-compression:
 [1] http://www.buckarooshangar.com/flightgear/tut_dds.html
 
 ------------------------------------
-Installation and usage:
+## Installation and usage:
 
 Simply copy "convert.sh" into a folder of your liking and run it:
 
-$ ./convert.sh
+```shell
+./convert.sh
+```
 
 This will show a help text, since you didn't specify any target(s).
 Possible targets are:
 * world
 * clouds
+* heights
 * all
 
 Additionally, there are some options you could specify (further
 explained below):
 * 1k | 2k | 4k | 8k | 16k
-* nasa
+* download | no-download
-* no-download
+* world
+* clouds
+* heights
 * cleanup
 * rebuild
 * check
 
 So your call could look sth like this:
 
-$ ./convert.sh world no-download cleanup 8k
+```shell
+./convert.sh world download alt cleanup 8k
+```
 
 
 ------------------------------------
-Requirements:
+## Requirements:
 
 WARNING!
 
-This script uses a *lot* disk space! Make sure you have at least 90GB
+This script uses a *lot* disk space! In my last test run, which
-available!
+generated all maps in all resolutions, the disk usage was about 330GB!
+Beware!\
+Also, I wouldn't recommend doing this on a SSD! While SSDs are
+generally faster, they also get more wear-and-tear when write such
+huge files. So this script might cause your SSD to die earlier as it
+should. Generally speaking, this won't kill your SSD, but it might
+cause it to die earlier. HDDs are much more robust in that respect.
 
 Also, this script will run for a *very long* time! It might be best to
 let it run over night - your computer might become unresponsive from
 time to time, due to the heavy CPU and memory load, which tends to
 occur, when converting 54000x27000 images. ;-)
-I also recommend to deactivate swapping!
-  $ sudo swapoff -a
-To reactivate swapping do:
-  $ sudo swapon -a
 
-This script relies on wget and imagemagick. Both are easily installed
+I also recommend to deactivate swapping!
-by your systems package-management-system.
+```shell
-(On Debian/Ubuntu this is "apt-get")
+  sudo swapoff -a
+ ```
+To reactivate swapping do:
+```shell
+  $ sudo swapon -a
+```
+
+This script relies on wget, ImageMagick and, for converting the height
+maps to normal maps, on "normalmap". Some of these programs are easily
+installed by your systems package-management-system.\ (On
+Debian/Ubuntu this is "apt-get").
 
 So, on Debian for instance, you only need to put the following into
 the console:
 
-  $ sudo apt-get install wget imagemagick
+```shell
+sudo apt-get install wget imagemagick
+```
 
 Depending on your distro, the package names might differ slightly! Use
 a search engine of your choice to find out, how the packages are named
@@ -98,55 +125,77 @@ in your distro!
 
 You may want to check:
 
-  $ apt search imagemagick
+```shell
+apt search imagemagick
+```
 
+### IMPORTANT!
+Check out your ```/etc/ImageMagick-6/policy.xml```
+On some distros, there are limits set, which will cause IM to abort
+the conversion of images larger than
+[rediculously small images](https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=860763).
+Edit and set to our needs:
+* width: at least 55000
+* height: at least 55000
+* area: less than your free RAM
+
+### Normalmap
+For normalmap, you can download and compile it from
+  https://github.com/planrich/normalmap
+
+You can install the binary into your system, or just copy it next to
+convert.sh - both should work.
 
 ------------------------------------
-Targets:
+## Targets:
 
-world
+**world**\
         Generates the world tiles, needed to run FG with EarthView.
-	You will find the results in output/[$resolution]/*. Copy
+        You will find the results in output/[$resolution]/\*. Copy
-	these into $FGDATA/Models/Astro/*. More about the installation
+        these into $FGDATA/Models/Astro/\*. More about the installation
         of these textures can be found here:
         http://wiki.flightgear.org/Earthview#Customization
 
-clouds
+**clouds**\
         Generates the cloud tiles, needed to run FG with EarthView.
         The locations are the same as the other textures mentioned
         above. Note that clouds are only available with up to 8k
         resolution, due to the available data at NASA.
 
-all
+**heights**\
+        Generates the height tiles, which are then converted to the
+        normal maps needed to run FG with EarthView. The locations are
+        the same as the other textures mentioned above. Note that
+        heights are only available with up to 8k resolution, due to the
+        available data at NASA.
+
+**all**\
         Converts everything needed for a full-blown earthview texture
         set. Does the same as:
-	  $ ./convert.sh world clouds
+        ```./convert.sh world clouds heights```
 
 
-Options:
+## Options:
 
-1k | 2k | 4k | 8k | 16k
+**1k | 2k | 4k | 8k | 16k**\
         Lets you specify a desired resolution of the textures.
         Possible values are 1k, 2k, 4k, 8k and 16k. If nothing is
         specified, the script will generate all of the resolutions.
-	16k is only available for earth textures.
+        16k is recommended only for earth textures, it will induce
+        oversampling from clouds and height maps.
 
-nasa
+**download**\
-	Causes the script to download directly from 
+        Causes the script to download the needed data, this is the
-	http://visibleearth.nasa.gov . If omitted the script will
+        default behavior (and can be omitted).
-	download from
-	https://musicchris.de/download/FG/EarthView/raw-data-NASA.7z
-	which is much faster!
-	Uses wget either way.
 
-no-download
+**no-download**\
         Causes the script to skip the download function. If you
         already have the source images, then you don't need to
         re-download them. (About 2.4GB!)
         If omitted, the script will download the source images from
-	https://musicchris.de/download/FG/EarthView/raw-data-NASA.7z
+        the default location.
 
-cleanup
+**cleanup**\
         Deletes the temporary files created during texture generation.
         These can be found in tmp/
         Note: if for some reason you later want some other resolution,
@@ -155,13 +204,13 @@ cleanup
         Frees up a lot of disk-space! Which would have to be
         regenerated if needed again.
 
-rebuild
+**rebuild**\
         Deletes only the temporary files of the given target. So if
-	you call './convert.sh rebuild world' the script will delete
+        you call ```./convert.sh rebuild world``` the script will delete
         all corresponding temp-files of the target world, which will
         trigger a complete regeneration of the relevant (instead of
         skipping existing files)
 
-check
+**check**\
         Creates mosaics of the tiles, so you can look at them and see
         if all went well.

Nasa2FGearthview/convert.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
 
-# convert.sh - a script to convert NASA sattelite images into textures for use with FGearthview (orbital rendering)
+# convert.sh - a script to convert NASA sattelite images into textures
-# Copyright (C) 2016 Chris Schwarz <chris@musicchris.de>
+# for use with FGearthview (orbital rendering)
+# Copyright (C) 2016 chris_blues <chris@musicchris.de>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -16,9 +17,12 @@
 # You should have received a copy of the GNU General Public License along
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# v0.12: Chris Ringeval (eatdirt):
+# -add normalmapping output + small improvements
 
 
-VERSION="v0.11"
+VERSION="v0.13"
 
 # make sure the script halts on error
 set -e
@@ -29,21 +33,21 @@ function showHelp
    echo "https://github.com/chris-blues/Nasa2FGearthview"
    echo
    echo "Usage:"
-   echo "./convert.sh [ download world clouds 8k cleanup rebuild ]"
+   echo "./convert.sh [ download no-download world clouds heights"
+   echo "               1k 2k 4k 8k 16k cleanup rebuild ]"
    echo
-   echo "* Append \"nasa\" to download the needed images from NASA"
-   echo "  -> This will download ca 2.4GB of data!"
-   echo "  -> wget can continue interrupted downloads!"
-   echo "  If omitted, it will download from my server, which is a lot"
-   echo "  faster. See README for details."
    echo "* Append \"no-download\" to the command to skip the download"
    echo "  process alltogether. Only makes sense if you already got"
    echo "  the necessary data."
    echo "* Append \"world\" to the command to generate the world tiles"
    echo "* Append \"clouds\" to the command to generate cloud tiles"
-   echo "* Append \"all\" to the command to generate both - world and"
+   echo "* Append \"heights\" to the command to generate height tiles"
-   echo "  clouds"
+   echo "  and the normalmaps needed by EarthView. Notice that you need"
-   echo "* Append the size of the tiles (1k, 2k, 4, 8k, 16k). If you"
+   echo "  the normalmap binary to be installed. You can get it from:"
+   echo "  https://github.com/planrich/normalmap"
+   echo "* Append \"all\" to the command to generate all - world, clouds"
+   echo "  and heights"
+   echo "* Append the size of the tiles (1k, 2k, 4k, 8k, 16k). If you"
    echo "  don't pass a resolution, then all resolutions will be"
    echo "  generated."
    echo "* Append \"cleanup\" to delete all temporary files in tmp/"
@@ -59,7 +63,7 @@ function showHelp
    echo "  Useful if the source files have changed."
    echo "* Append \"check\" to let check the results. This will create"
    echo "  mosaics of the existing tiles. If no target is specified,"
-   echo "  all 3 layers will be built: clouds world and nightlights."
+   echo "  all layers will be built: clouds, heights, world and nightlights."
    echo
    echo "If, for some reason, the script aborts, then it will try to"
    echo "skip the already completed steps, so you don't have to wait"
@@ -102,11 +106,11 @@ if [ $1 == "-h" ] ; then showHelp ; fi
 ################################
 for ARG in "$@"
 do
-  if [ $ARG == "nasa" ] ; then DOWNLOAD="true" ; DL_LOCATION="NASA" ;  echo "Downloading from visibleearth.nasa.gov" ; fi
   if [ $ARG == "no-download" ] ; then DOWNLOAD="false" ; echo "Skipping the download process" ; fi
   if [ $ARG == "world" ] ; then WORLD="true" ; fi
   if [ $ARG == "clouds" ] ; then CLOUDS="true" ; fi
-  if [ $ARG == "all" ] ; then WORLD="true" ; CLOUDS="true" ; fi
+  if [ $ARG == "heights" ] ; then HEIGHTS="true" ; fi
+  if [ $ARG == "all" ] ; then WORLD="true" ; CLOUDS="true" ; HEIGHTS="true" ; fi
   if [ $ARG == "1k" ] ; then RESOLUTION="1024" ; fi
   if [ $ARG == "2k" ] ; then RESOLUTION="2048" ; fi
   if [ $ARG == "4k" ] ; then RESOLUTION="4096" ; fi
@@ -119,17 +123,22 @@ done
 if [ -z $DOWNLOAD ] ; then DOWNLOAD="true" ; fi
 if [ -z $WORLD ] ; then WORLD="false" ; fi
 if [ -z $CLOUDS ] ; then CLOUDS="false" ; fi
+if [ -z $HEIGHTS ] ; then HEIGHTS="false" ; fi
 if [ -z $CLEANUP ] ; then CLEANUP="false" ; fi
 if [ -z $REBUILD ] ; then REBUILD="false" ; fi
 if [ -z $BUILDCHECKS ] ; then BUILDCHECKS="false" ; fi
 
 CHECKWORLD=$WORLD
 CHECKCLOUDS=$CLOUDS
+CHECKHEIGHTS=$HEIGHTS
 
 
 ########################
 ## Set some variables ##
 ########################
+
+DL_LOCATION="NASA"
+
 mkdir -p tmp
 export MAGICK_TMPDIR=${PWD}/tmp
 echo "tmp-dir: $MAGICK_TMPDIR"
@@ -139,21 +148,50 @@ TIME=$(date +"%Y-%m-%d_%H:%M:%S")
 LOGFILE_GENERAL="logs/${TIME}.log"
 LOGFILE_TIME="logs/${TIME}.time.log"
 
-URLS_WORLD="http://eoimages.gsfc.nasa.gov/images/imagerecords/74000/74117/world.200408.3x21600x21600.A1.png
+#command line gimp plugin from https://github.com/planrich/normalmap
-http://eoimages.gsfc.nasa.gov/images/imagerecords/74000/74117/world.200408.3x21600x21600.A2.png
+#higher filters (5x5) create too sharp features (no rescaling, I
-http://eoimages.gsfc.nasa.gov/images/imagerecords/74000/74117/world.200408.3x21600x21600.B1.png
+#assume earthview do its own normalization)
-http://eoimages.gsfc.nasa.gov/images/imagerecords/74000/74117/world.200408.3x21600x21600.B2.png
+NORMALBIN="normalmap"
-http://eoimages.gsfc.nasa.gov/images/imagerecords/74000/74117/world.200408.3x21600x21600.C1.png
+NORMALOPTS="-s 1 -f FILTER_PREWITT_3x3"
-http://eoimages.gsfc.nasa.gov/images/imagerecords/74000/74117/world.200408.3x21600x21600.C2.png
-http://eoimages.gsfc.nasa.gov/images/imagerecords/74000/74117/world.200408.3x21600x21600.D1.png
-http://eoimages.gsfc.nasa.gov/images/imagerecords/74000/74117/world.200408.3x21600x21600.D2.png
-http://eoimages.gsfc.nasa.gov/images/imagerecords/79000/79765/dnb_land_ocean_ice.2012.54000x27000_geo.tif"
 
-URLS_CLOUDS="http://eoimages.gsfc.nasa.gov/images/imagerecords/57000/57747/cloud.E.2001210.21600x21600.png
+URLS_WORLD="https://eoimages.gsfc.nasa.gov/images/imagerecords/74000/74117/world.200408.3x21600x21600.A1.png
-http://eoimages.gsfc.nasa.gov/images/imagerecords/57000/57747/cloud.W.2001210.21600x21600.png"
+https://eoimages.gsfc.nasa.gov/images/imagerecords/74000/74117/world.200408.3x21600x21600.A2.png
+https://eoimages.gsfc.nasa.gov/images/imagerecords/74000/74117/world.200408.3x21600x21600.B1.png
+https://eoimages.gsfc.nasa.gov/images/imagerecords/74000/74117/world.200408.3x21600x21600.B2.png
+https://eoimages.gsfc.nasa.gov/images/imagerecords/74000/74117/world.200408.3x21600x21600.C1.png
+https://eoimages.gsfc.nasa.gov/images/imagerecords/74000/74117/world.200408.3x21600x21600.C2.png
+https://eoimages.gsfc.nasa.gov/images/imagerecords/74000/74117/world.200408.3x21600x21600.D1.png
+https://eoimages.gsfc.nasa.gov/images/imagerecords/74000/74117/world.200408.3x21600x21600.D2.png
+https://eoimages.gsfc.nasa.gov/images/imagerecords/79000/79765/dnb_land_ocean_ice.2012.54000x27000_geo.tif"
+
+URLS_CLOUDS="https://eoimages.gsfc.nasa.gov/images/imagerecords/57000/57747/cloud.E.2001210.21600x21600.png
+https://eoimages.gsfc.nasa.gov/images/imagerecords/57000/57747/cloud.W.2001210.21600x21600.png"
+
+URLS_HEIGHTS="https://eoimages.gsfc.nasa.gov/images/imagerecords/73000/73934/gebco_08_rev_elev_A1_grey_geo.tif
+https://eoimages.gsfc.nasa.gov/images/imagerecords/73000/73934/gebco_08_rev_elev_A2_grey_geo.tif
+https://eoimages.gsfc.nasa.gov/images/imagerecords/73000/73934/gebco_08_rev_elev_B1_grey_geo.tif
+https://eoimages.gsfc.nasa.gov/images/imagerecords/73000/73934/gebco_08_rev_elev_B2_grey_geo.tif
+https://eoimages.gsfc.nasa.gov/images/imagerecords/73000/73934/gebco_08_rev_elev_C1_grey_geo.tif
+https://eoimages.gsfc.nasa.gov/images/imagerecords/73000/73934/gebco_08_rev_elev_C2_grey_geo.tif
+https://eoimages.gsfc.nasa.gov/images/imagerecords/73000/73934/gebco_08_rev_elev_D1_grey_geo.tif
+https://eoimages.gsfc.nasa.gov/images/imagerecords/73000/73934/gebco_08_rev_elev_D2_grey_geo.tif"
+
+if ! [ -x "$(command -v $NORMALBIN)" ]
+  then
+    if ! [ -x "./${NORMALBIN}" ]
+      then
+        echo ">>>>>>>>>>>>  Error: $NORMALBIN binary not found! <<<<<<<<<<<<<"
+        echo "You can get it from: https://github.com/planrich/normalmap"
+        HEIGHTS="false"
+      else
+        NORMALBIN="./${NORMALBIN}"
+    fi
+  else
+    echo ">>>>>>>>>>>>  Error: $NORMALBIN binary not found! <<<<<<<<<<<<<"
+    echo "You can get it from: https://github.com/planrich/normalmap"
+    HEIGHTS="false"
+fi
 
-ALTERNATE_URL="https://musicchris.de/download/FG/EarthView/raw-data-NASA.7z"
-ALTERNATE_FILENAME="raw-data-NASA.7z"
 
 if [ -z $RESOLUTION ]
   then
@@ -162,7 +200,7 @@ if [ -z $RESOLUTION ]
 4096
 8192
 16384"
-    NO_RESOLUTION_GIVEN="true"
+    NO_RESOLUTION_GIVEN="false"
     RESOLUTION_MAX="16384"
 fi
 if [ -z $RESOLUTION_MAX ] ; then RESOLUTION_MAX=$RESOLUTION ; fi
@@ -236,6 +274,18 @@ function rebuild
       rm tmp/cloud*
      }
    fi
+
+    if [ $HEIGHTS == "true" ]
+   then
+     {
+      echo
+      echo "#########################################"
+      echo "## Removing tmp-files of target heights##"
+      echo "#########################################"
+      rm tmp/height*
+     }
+   fi
+
   }
 
 function cleanUp
@@ -244,9 +294,10 @@ function cleanUp
    echo "############################"
    echo "## Removing all tmp-files ##"
    echo "############################"
-   rm -rv tmp/night*
+   rm -rvf tmp/night*
-   rm -rv tmp/world*
+   rm -rvf tmp/world*
-   rm -rv tmp/clouds*
+   rm -rvf tmp/cloud*
+   rm -rvf tmp/height*
   }
 
 function prettyTime
@@ -312,13 +363,18 @@ function downloadImages
   {
    echo | tee -a $LOGFILE_GENERAL
    echo "###################################################" | tee -a $LOGFILE_GENERAL
-   if [ ! -z $DL_LOCATION ]
+   if [ -z $DL_LOCATION ]
-     then echo "## Downloading images from visibleearth.nasa.gov ##" | tee -a $LOGFILE_GENERAL
+   then
-     else echo "##    Downloading images from  musicchris.de     ##" | tee -a $LOGFILE_GENERAL
+       DL_LOCATION="NASA"
    fi
+
+   if [ $DL_LOCATION == "NASA" ]
+   then
+       echo "## Downloading images from visibleearth.nasa.gov ##" | tee -a $LOGFILE_GENERAL
+   fi
+
    echo "###################################################" | tee -a $LOGFILE_GENERAL
-   if [ -z $DL_LOCATION ] ; then f=$ALTERNATE_URL ; DL_LOCATION="musicchris.de" ; fi
+
-   FILENAME=$(echo $f | sed 's@.*/@@')
    if [ $WORLD == "true" ]
    then
      if [ $DL_LOCATION == "NASA" ]
@@ -335,7 +391,13 @@ function downloadImages
      fi
    fi
 
-   if [ $DL_LOCATION == "musicchris.de" ] ; then downloadMusicchris ; fi
+   if [ $HEIGHTS == "true" ]
+   then
+     if [ $DL_LOCATION == "NASA" ]
+     then
+       downloadHeights
+     fi
+   fi
   }
 
 function downloadWorld
@@ -345,10 +407,23 @@ function downloadWorld
    for f in $URLS_WORLD
    do
      FILENAME=$(echo $f | sed 's@.*/@@')
-     wget --output-document=input/$FILENAME --continue --show-progress $f 2>> $LOGFILE_GENERAL
+     wget --output-document=input/$FILENAME --continue --show-progress $f | tee -a $LOGFILE_GENERAL 2>> $LOGFILE_GENERAL
    done
   }
 
+
+function downloadHeights
+  {
+   mkdir -p input
+   echo "Downloading height tiles..." | tee -a $LOGFILE_GENERAL
+   for f in $URLS_HEIGHTS
+   do
+     FILENAME=$(echo $f | sed 's@.*/@@')
+     wget --output-document=input/$FILENAME --continue --show-progress $f | tee -a $LOGFILE_GENERAL 2>> $LOGFILE_GENERAL
+   done
+  }
+
+
 function downloadClouds
   {
    mkdir -p input
@@ -356,24 +431,10 @@ function downloadClouds
    for f in $URLS_CLOUDS
    do
      FILENAME=$(echo $f | sed 's@.*/@@')
-     wget --output-document=input/$FILENAME --continue --show-progress $f 2>> $LOGFILE_GENERAL
+     wget --output-document=input/$FILENAME --continue --show-progress $f | tee -a $LOGFILE_GENERAL 2>> $LOGFILE_GENERAL
    done
   }
 
-function downloadMusicchris
-  {
-   mkdir -p input
-   echo "Downloading raw images... (ca 2.2 GB)" | tee -a $LOGFILE_GENERAL
-   for p in 1 2 3 4 5
-   do
-     wget --output-document=input/${FILENAME}.00${p} --continue --show-progress $ALTERNATE_URL | tee -a $LOGFILE_GENERAL 2>> $LOGFILE_GENERAL
-   done
-   echo "Unpacking raw images..." | tee -a $LOGFILE_GENERAL
-   cd input
-   7z e -bt -y raw-data-NASA.7z 2>> $LOGFILE_GENERAL
-   cd ..
-  }
-
 function generateWorld
   {
    STARTTIME=$(date +%s)
@@ -419,7 +480,7 @@ function generateWorld
    let "RESIZE_H = ( $RESOLUTION_MAX - ( 2 * $BORDER_WIDTH ) ) * 2"
    if [ ! -s "tmp/nightlights_${RESIZE_W}x${RESIZE_H}.mpc" ]
    then
-     for r in 16384 8192 4096 2048
+     for r in $RESOLUTION
      do
        if [ $r -le $RESOLUTION_MAX ]
        then continue
@@ -553,7 +614,7 @@ function generateWorld
    do
      NASA2FG $t
      FOUND_BIGGER_WORLD_PICTURE="false"
-     unset $TIMESAVER_SIZE
+     unset TIMESAVER_SIZE
      if [ ! -s "tmp/world_seamless_${IMAGE_BORDERLESS}_${DEST}.mpc" ]
      then
        {
@@ -825,6 +886,10 @@ function generateWorld
         echo
         echo "--> Writing output/${r}/world_${t}.dds @ ${r}x${r}"
         # set -x
+
+        if [ ! -s "output/${r}/world_${t}.dds" ]
+        then
+
           convert \
             -monitor \
              tmp/world_${RESOLUTION_MAX}_done_${t}.mpc \
@@ -834,7 +899,15 @@ function generateWorld
              output/${r}/world_${t}.dds
           set +x
           echo
+
+        else echo "=> Skipping existing file: output/${r}/world_${t}.dds" | tee -a $LOGFILE_GENERAL | tee -a $LOGFILE_TIME
+        fi
+
         echo "--> Writing output/${r}/world_${t}.png @ ${r}x${r}"
+
+        if [ ! -s "output/${r}/world_${t}.png" ]
+        then
+
           # set -x
           convert \
             -monitor \
@@ -843,6 +916,10 @@ function generateWorld
              output/${r}/world_${t}.png
           set +x
           echo
+
+        else echo "=> Skipping existing file: output/${r}/world_${t}.png" | tee -a $LOGFILE_GENERAL | tee -a $LOGFILE_TIME
+
+        fi
        }
      done
 
@@ -874,7 +951,11 @@ function generateClouds
   {
    if [ -z $STARTTIME ] ; then STARTTIME=$(date +%s) ; fi
    # maximum cloud-tile resolution is 8192, since we have no big enough source files...
+
+   if [[ $NO_RESOLUTION_GIVEN == "true" ]]; then
        if [ $RESOLUTION_MAX -eq 16384 ] ; then RESOLUTION_MAX=8192 ; fi
+   fi
+
    let "BORDER_WIDTH = $RESOLUTION_MAX / 128"
    let "IMAGE_BORDERLESS = $RESOLUTION_MAX - ( 2 * $BORDER_WIDTH )"
    let "IMAGE_WITH_BORDER = $RESOLUTION_MAX - $BORDER_WIDTH - 1"
@@ -1071,7 +1152,9 @@ W"
    echo "## propagate last pixels to the edge ##" | tee -a $LOGFILE_GENERAL
    echo "#######################################" | tee -a $LOGFILE_GENERAL
 
+   if [[ $NO_RESOLUTION_GIVEN == "true" ]]; then
        if [ $RESOLUTION_MAX -eq 16384 ] ; then RESOLUTION_MAX=8192 ; fi
+   fi
    let "BORDER_WIDTH = $RESOLUTION_MAX / 128"
    let "IMAGE_BORDERLESS = $RESOLUTION_MAX - ( 2 * $BORDER_WIDTH )"
    let "IMAGE_WITH_BORDER = $RESOLUTION_MAX - $BORDER_WIDTH - 1"
@@ -1190,23 +1273,22 @@ W"
      for r in $RESOLUTION
      do
        {
-        if [ $r -eq 16384 ]
-          then
-            if [ $NO_RESOLUTION_GIVEN == "true" ]
-              then
-                continue
-	      else
-	        r=8192
-	    fi
-	fi
         mkdir -p output/$r
-        echo
+
+        if [ ! -s "output/${r}/clouds_${t}.png" ]
+        then
+
           echo "--> Writing output/${r}/clouds_${t}.png @ ${r}x${r}" | tee -a $LOGFILE_GENERAL
           convert \
             -monitor \
              tmp/clouds_${RESOLUTION_MAX}_${t}_done.mpc \
             -resize ${r}x${r} \
              output/${r}/clouds_${t}.png
+          echo
+
+        else echo "=> Skipping existing file: output/${r}/clouds_${t}.png" | tee -a $LOGFILE_GENERAL | tee -a $LOGFILE_TIME
+        fi
+
        }
      done
 
@@ -1231,6 +1313,322 @@ W"
    echo "Overall processing time: $DAYS d, $HOURS h, $MINUTES m, $SECS s" | tee -a $LOGFILE_GENERAL | tee -a $LOGFILE_TIME
   }
 
+
+
+function generateHeights
+  {
+  if [ -z $STARTTIME ] ; then STARTTIME=$(date +%s) ; fi
+   echo | tee -a $LOGFILE_GENERAL
+   echo "################################" | tee -a $LOGFILE_GENERAL
+   echo "####    Processing Heights  ####" | tee -a $LOGFILE_GENERAL
+   echo "################################" | tee -a $LOGFILE_GENERAL
+   echo | tee -a $LOGFILE_GENERAL
+
+   if [[ $NO_RESOLUTION_GIVEN == "true" ]]; then
+       if [ $RESOLUTION_MAX -eq 16384 ] ; then RESOLUTION_MAX=8192 ; fi
+   fi
+
+   let "BORDER_WIDTH = $RESOLUTION_MAX / 128"
+   let "IMAGE_BORDERLESS = $RESOLUTION_MAX - ( 2 * $BORDER_WIDTH )"
+   let "IMAGE_WITH_BORDER = $RESOLUTION_MAX - $BORDER_WIDTH - 1"
+   let "IMAGE_WITH_BORDER_POS = $RESOLUTION_MAX - $BORDER_WIDTH"
+   let "SIZE = 2 * $IMAGE_BORDERLESS"
+
+   mkdir -p tmp
+   mkdir -p output
+
+
+   echo "################################################" | tee -a $LOGFILE_GENERAL
+   echo "## Resize the NASA-Originals to ${RESOLUTION_MAX}-(2*${BORDER_WIDTH}) ##" | tee -a $LOGFILE_GENERAL
+   echo "################################################" | tee -a $LOGFILE_GENERAL
+   for t in $NASA
+   do
+     NASA2FG $t
+     FOUND_BIGGER_WORLD_PICTURE="false"
+     unset TIMESAVER_SIZE
+     if [ ! -s "tmp/heights_seamless_${IMAGE_BORDERLESS}_${DEST}.mpc" ]
+     then
+       {
+        for r in $RESOLUTION
+        do
+          if [ $r -le $RESOLUTION_MAX ]
+          then
+            continue
+          fi
+          let "IMAGE_SIZE = $r - ( 2 * ( $r / 128 ) )"
+          let "I_W = $IMAGE_SIZE * 4"
+          let "I_H = $IMAGE_SIZE * 2"
+          if [ -s tmp/heights_seamless_${IMAGE_SIZE}_${DEST}.mpc ]
+          then
+            if [ $IMAGE_SIZE -ge $IMAGE_BORDERLESS ]
+            then
+              FOUND_BIGGER_WORLD_PICTURE="true"
+              TIMESAVER_SIZE="$IMAGE_SIZE"
+            fi
+          fi
+        done
+        if [ $FOUND_BIGGER_WORLD_PICTURE != "true" ]
+        then
+            # set -x
+              convert \
+                -monitor \
+                -limit memory 32 \
+                -limit map 32 \
+                input/gebco_08_rev_elev_${t}_grey_geo.tif \
+                -resize ${IMAGE_BORDERLESS}x${IMAGE_BORDERLESS} \
+                tmp/heights_seamless_${IMAGE_BORDERLESS}_${DEST}.mpc
+              set +x
+        else
+          echo "==> Timesaver:) Using existing file: tmp/heights_seamless_${TIMESAVER_SIZE}_${DEST}.mpc -> tmp/heights_seamless_${IMAGE_BORDERLESS}_${DEST}.mpc" | tee -a $LOGFILE_GENERAL
+           set -x
+          convert \
+            -monitor \
+            -limit memory 32 \
+            -limit map 32 \
+            tmp/heights_seamless_${TIMESAVER_SIZE}_${DEST}.mpc \
+            -resize ${IMAGE_BORDERLESS}x${IMAGE_BORDERLESS} \
+            tmp/heights_seamless_${IMAGE_BORDERLESS}_${DEST}.mpc
+          set +x
+        fi
+       }
+     else echo "=> Skipping existing file: tmp/heights_seamless_${IMAGE_BORDERLESS}_${DEST}.mpc" | tee -a $LOGFILE_GENERAL | tee -a $LOGFILE_TIME
+     fi
+   done
+   # 3h, 12m, 9s
+   if [ -z $LASTTIME ] ; then LASTTIME=$STARTTIME ; fi
+   echo "input/gebco_08_rev_elev_[A-D][12]_grey_geo.tif -> tmp/heights_seamless_${IMAGE_BORDERLESS}_[NS][1-4].mpc" >> $LOGFILE_TIME
+   getProcessingTime
+
+   echo | tee -a $LOGFILE_GENERAL
+   echo "#####################################" | tee -a $LOGFILE_GENERAL
+   echo "## Put a ${BORDER_WIDTH}px border to each side ##" | tee -a $LOGFILE_GENERAL
+   echo "#####################################" | tee -a $LOGFILE_GENERAL
+   for t in $TILES
+   do
+     if [ ! -s "tmp/heights_seams_${RESOLUTION_MAX}_${t}_emptyBorder.mpc" ]
+     then
+       # set -x
+       convert \
+         -monitor \
+         tmp/heights_seamless_${IMAGE_BORDERLESS}_${t}.mpc \
+         -bordercolor none \
+         -border ${BORDER_WIDTH} \
+         tmp/heights_seams_${RESOLUTION_MAX}_${t}_emptyBorder.mpc
+       set +x
+       echo
+     fi
+     if [ ! -s "tmp/heights_seams_${RESOLUTION_MAX}_${t}.mpc" ]
+     then
+       # set -x
+       cp tmp/heights_seams_${RESOLUTION_MAX}_${t}_emptyBorder.mpc tmp/heights_seams_${RESOLUTION_MAX}_${t}.mpc
+       cp tmp/heights_seams_${RESOLUTION_MAX}_${t}_emptyBorder.cache tmp/heights_seams_${RESOLUTION_MAX}_${t}.cache
+       set +x
+     else echo "=> Skipping existing file: tmp/heights_seams_${RESOLUTION_MAX}_${t}.mpc" | tee -a $LOGFILE_GENERAL | tee -a $LOGFILE_TIME
+     fi
+   done
+   # 11m, 24s
+   echo "-> tmp/heights_seams_${RESOLUTION_MAX}_[NS][1-4]_emptyBorder.mpc -> tmp/heights_seams_${RESOLUTION_MAX}_[NS][1-4].mpc" >> $LOGFILE_TIME
+   getProcessingTime
+
+   echo | tee -a $LOGFILE_GENERAL
+   echo "######################################################" | tee -a $LOGFILE_GENERAL
+   echo "## crop borderline pixels and propagate to the edge ##" | tee -a $LOGFILE_GENERAL
+   echo "######################################################" | tee -a $LOGFILE_GENERAL
+
+   if [[ $NO_RESOLUTION_GIVEN == "true" ]]; then
+       if [ $RESOLUTION_MAX -eq 16384 ] ; then RESOLUTION_MAX=8192 ; fi
+   fi
+   let "BORDER_WIDTH = $RESOLUTION_MAX / 128"
+   let "IMAGE_BORDERLESS = $RESOLUTION_MAX - ( 2 * $BORDER_WIDTH )"
+   let "IMAGE_WITH_BORDER = $RESOLUTION_MAX - $BORDER_WIDTH - 1"
+   let "IMAGE_WITH_BORDER_POS = $RESOLUTION_MAX - $BORDER_WIDTH"
+   let "SIZE = 2 * $IMAGE_BORDERLESS"
+
+   CROP_TOP="${IMAGE_BORDERLESS}x1+${BORDER_WIDTH}+${BORDER_WIDTH}"
+   CROP_RIGHT="1x${IMAGE_BORDERLESS}+${IMAGE_WITH_BORDER}+${BORDER_WIDTH}"
+   CROP_BOTTOM="${IMAGE_BORDERLESS}x1+${BORDER_WIDTH}+${IMAGE_WITH_BORDER}"
+   CROP_LEFT="1x${IMAGE_BORDERLESS}+${BORDER_WIDTH}+${BORDER_WIDTH}"
+   CROP_TOPLEFT="1x1+${BORDER_WIDTH}+${BORDER_WIDTH}"
+   CROP_TOPRIGHT="1x1+${IMAGE_WITH_BORDER}+${BORDER_WIDTH}"
+   CROP_BOTTOMRIGHT="1x1+${IMAGE_WITH_BORDER}+${IMAGE_WITH_BORDER}"
+   CROP_BOTTOMLEFT="1x1+${BORDER_WIDTH}+${IMAGE_WITH_BORDER}"
+
+   ## HORIZ meaning a horizontal bar, like the one on top
+   HORIZ_RESIZE="${IMAGE_BORDERLESS}x${BORDER_WIDTH}"
+   VERT_RESIZE="${BORDER_WIDTH}x${IMAGE_BORDERLESS}"
+
+   POS_TOP="+${BORDER_WIDTH}+0"
+   POS_RIGHT="+${IMAGE_WITH_BORDER_POS}+${BORDER_WIDTH}"
+   POS_BOTTOM="+${BORDER_WIDTH}+${IMAGE_WITH_BORDER_POS}"
+   POS_LEFT="+0+${BORDER_WIDTH}"
+
+   for t in $TILES
+   do
+     if [ ! -s "tmp/heights_${RESOLUTION_MAX}_done_${t}.mpc" ]
+     then
+     for b in $BORDERS
+     do
+       {
+        if [ $b == "top" ]
+        then
+          CROP=$CROP_TOP
+          RESIZE=$HORIZ_RESIZE
+          POSITION=$POS_TOP
+          CROPCORNER=$CROP_TOPRIGHT
+          CORNER_POS="+${IMAGE_WITH_BORDER_POS}+0"
+          CORNER_NAME="topRight"
+        fi
+        if [ $b == "right" ]
+        then
+          CROP=$CROP_RIGHT
+          RESIZE=$VERT_RESIZE
+          POSITION=$POS_RIGHT
+          CROPCORNER=$CROP_BOTTOMRIGHT
+          CORNER_POS="+${IMAGE_WITH_BORDER_POS}+${IMAGE_WITH_BORDER_POS}"
+          CORNER_NAME="bottomRight"
+        fi
+        if [ $b == "bottom" ]
+        then
+          CROP=$CROP_BOTTOM
+          RESIZE=$HORIZ_RESIZE
+          POSITION=$POS_BOTTOM
+          CROPCORNER=$CROP_BOTTOMLEFT
+          CORNER_POS="+0+${IMAGE_WITH_BORDER_POS}"
+          CORNER_NAME="bottomLeft"
+        fi
+        if [ $b == "left" ]
+        then
+          CROP=$CROP_LEFT
+          RESIZE=$VERT_RESIZE
+          POSITION=$POS_LEFT
+          CROPCORNER=$CROP_TOPLEFT
+          CORNER_POS="+0+0"
+          CORNER_NAME="topLeft"
+        fi
+        echo
+        # set -x
+        convert \
+          -monitor \
+          tmp/heights_seams_${RESOLUTION_MAX}_${t}_emptyBorder.mpc \
+          -crop $CROP \
+          -resize $RESIZE\! \
+          tmp/heights_${RESOLUTION_MAX}_${t}_seam_${b}.mpc
+        convert \
+          -monitor \
+          tmp/heights_seams_${RESOLUTION_MAX}_${t}_emptyBorder.mpc \
+          -crop $CROPCORNER \
+          -resize ${BORDER_WIDTH}x${BORDER_WIDTH}\! \
+          tmp/heights_${RESOLUTION_MAX}_${t}_seam_${CORNER_NAME}.mpc
+        convert \
+          -monitor \
+          tmp/heights_seams_${RESOLUTION_MAX}_${t}.mpc \
+          tmp/heights_${RESOLUTION_MAX}_${t}_seam_${b}.mpc \
+          -geometry $POSITION \
+          -composite \
+          tmp/heights_seams_${RESOLUTION_MAX}_${t}.mpc
+        echo
+        convert \
+          -monitor \
+          tmp/heights_seams_${RESOLUTION_MAX}_${t}.mpc \
+          tmp/heights_${RESOLUTION_MAX}_${t}_seam_${CORNER_NAME}.mpc \
+          -geometry $CORNER_POS \
+          -composite \
+          tmp/heights_seams_${RESOLUTION_MAX}_${t}.mpc
+        set +x
+        echo
+       }
+     done
+     echo
+     # set -x
+     cp -v tmp/heights_seams_${RESOLUTION_MAX}_${t}.mpc tmp/heights_${RESOLUTION_MAX}_done_${t}.mpc | tee -a $LOGFILE_GENERAL
+     cp -v tmp/heights_seams_${RESOLUTION_MAX}_${t}.cache tmp/heights_${RESOLUTION_MAX}_done_${t}.cache | tee -a $LOGFILE_GENERAL
+     set +x
+
+     else echo "=> Skipping existing file: tmp/heights_${RESOLUTION_MAX}_done_${t}.mpc" | tee -a $LOGFILE_GENERAL | tee -a $LOGFILE_TIME
+     fi
+
+  done
+  # 37m, 6s
+  echo "-> tmp/heights_seams_${RESOLUTION_MAX}_[NS][1-4].mpc -> tmp/heights_${RESOLUTION_MAX}_done_[NS][1-4].mpc" >> $LOGFILE_TIME
+  getProcessingTime
+
+  for t in $TILES
+   do
+     echo | tee -a $LOGFILE_GENERAL
+     echo "#############################" | tee -a $LOGFILE_GENERAL
+     echo "## Final output of tile $t ##" | tee -a $LOGFILE_GENERAL
+     echo "##       and normalmapping ##" | tee -a $LOGFILE_GENERAL
+     echo "#############################" | tee -a $LOGFILE_GENERAL
+
+     for r in $RESOLUTION
+     do
+       {
+        mkdir -p output/$r
+        set +x
+        echo
+        echo "--> Writing output/${r}/heights_${t}.png @ ${r}x${r}"
+        # set -x
+
+        if [ ! -s "output/${r}/heights_${t}.png" ]
+        then
+
+          convert \
+            -monitor \
+             tmp/heights_${RESOLUTION_MAX}_done_${t}.mpc \
+            -resize ${r}x${r} \
+             output/${r}/heights_${t}.png
+          echo
+
+        else echo "=> Skipping existing file: tmp/heights_${RESOLUTION_MAX}_done_${t}.mpc" | tee -a $LOGFILE_GENERAL | tee -a $LOGFILE_TIME
+        fi
+
+        echo "--> Writing output/${r}/normalmap_earth_${t}.png @ ${r}x${r}"
+        if [ ! -s "output/${r}/normalmap_earth_${t}.png" ]
+        then
+
+          $NORMALBIN $NORMALOPTS output/${r}/heights_${t}.png output/${r}/normalmap_earth_${t}.png
+
+        else echo "=> Skipping existing file: output/${r}/normalmap_earth_${t}.png" | tee -a $LOGFILE_GENERAL | tee -a $LOGFILE_TIME
+        fi
+
+        set +x
+        echo
+       }
+     done
+
+     echo | tee -a $LOGFILE_GENERAL
+     echo "Heights and Normal $t [ done ]" | tee -a $LOGFILE_GENERAL
+     echo | tee -a $LOGFILE_GENERAL
+
+   done
+   echo "###############################" | tee -a $LOGFILE_GENERAL
+   echo "####    Heights: [ done ]  ####" | tee -a $LOGFILE_GENERAL
+   echo "###############################" | tee -a $LOGFILE_GENERAL
+   # 2h, 19m, 7s
+   # Overall processing time: 44089 s
+   # Overall processing time: 0 d, 2 h, 19 m, 7 s
+
+   echo "-> output/<\$RESOLUTIONS>/heights_[NS][1-4].png" >> $LOGFILE_TIME
+   getProcessingTime
+   echo
+   if [ $STARTTIME -eq $ENDTIME ]
+     then SECS=0
+     else let "SECS = $ENDTIME - $STARTTIME"
+   fi
+   echo "Overall processing time: $SECS s" | tee -a $LOGFILE_GENERAL
+   prettyTime
+   echo "Overall processing time: $DAYS d, $HOURS h, $MINUTES m, $SECS s" | tee -a $LOGFILE_GENERAL
+  }
+
+
+
+
+
+
+
+
+
+
 function checkResults
   {
    echo | tee -a $LOGFILE_GENERAL
@@ -1251,7 +1649,7 @@ function checkResults
    let "HEIGHT = 2 * $RES"
    echo "Lowest available resolution is: $RES" | tee -a $LOGFILE_GENERAL
 
-   if [ $CHECKCLOUDS == "true" ]
+   if [[ $CHECKCLOUDS == "true" ]]
    then
      {
       echo "checking clouds..." | tee -a $LOGFILE_GENERAL
@@ -1292,7 +1690,51 @@ function checkResults
      }
    fi
 
-   if [ $CHECKWORLD == "true" ]
+
+   if [[ $CHECKHEIGHTS == "true" ]]
+   then
+     {
+      echo "checking heights..." | tee -a $LOGFILE_GENERAL
+      echo | tee -a $LOGFILE_GENERAL
+
+      echo "Creating canvas ${WIDTH}x${HEIGHT}" | tee -a $LOGFILE_GENERAL
+      convert \
+        -size ${WIDTH}x${HEIGHT} \
+        xc:Black \
+        -alpha on \
+        check_heights.png
+
+      POS=0
+      for t in 1 2 3 4
+      do
+        convert \
+          -monitor \
+          check_heights.png \
+          output/${RES}/heights_N${t}.png \
+          -geometry +${POS}+0 \
+          -composite \
+          check_heights.png
+        echo
+        convert \
+          -monitor \
+          check_heights.png \
+          output/${RES}/heights_S${t}.png \
+          -geometry +${POS}+${RES} \
+          -composite \
+          check_heights.png
+        echo
+        let "POS += $RES"
+      done
+      mogrify \
+        -monitor \
+        -resize 4096x2048 \
+        check_heights.png
+     }
+   fi
+
+
+
+   if [[ $CHECKWORLD == "true" ]]
    then
      {
       echo "checking world..." | tee -a $LOGFILE_GENERAL
@@ -1366,6 +1808,7 @@ echo $TIME | tee -a $LOGFILE_TIME
 echo | tee -a $LOGFILE_GENERAL
 printf "Target:     " | tee -a $LOGFILE_GENERAL | tee -a $LOGFILE_TIME
 if [ $CLOUDS == "true" ] ; then printf "clouds " | tee -a $LOGFILE_GENERAL | tee -a $LOGFILE_TIME ; fi
+if [ $HEIGHTS == "true" ] ; then printf "heights " | tee -a $LOGFILE_GENERAL | tee -a $LOGFILE_TIME ; fi
 if [ $WORLD == "true" ] ;  then printf "world " | tee -a $LOGFILE_GENERAL | tee -a $LOGFILE_TIME ; fi
 echo | tee -a $LOGFILE_GENERAL | tee -a $LOGFILE_TIME
 echo "Will work in ${RESOLUTION_MAX}x${RESOLUTION_MAX} resolution and will output" | tee -a $LOGFILE_GENERAL
@@ -1377,12 +1820,13 @@ echo "--------------------------------------------------------------" | tee -a $
 echo | tee -a $LOGFILE_GENERAL
 
 
-if [ $REBUILD == "true" ] ; then rebuild ; fi
+if [[ $REBUILD == "true" ]] ; then rebuild ; fi
-if [ $DOWNLOAD == "true" ] ; then downloadImages ; fi
+if [[ $DOWNLOAD == "true" ]] ; then downloadImages ; fi
-if [ $WORLD == "true" ] ;  then generateWorld ; fi
+if [[ $WORLD == "true" ]] ;  then generateWorld ; fi
-if [ $CLOUDS == "true" ] ; then generateClouds ; fi
+if [[ $CLOUDS == "true" ]] ; then generateClouds ; fi
-if [ $BUILDCHECKS == "true" ] ; then checkResults ; fi
+if [[ $HEIGHTS == "true" ]]; then generateHeights; fi
-if [ $CLEANUP == "true" ] ; then cleanUp ; fi
+if [[ $BUILDCHECKS == "true" ]] ; then checkResults ; fi
+if [[ $CLEANUP == "true" ]] ; then cleanUp ; fi
 
 
 

build_release_linux.sh

@@ -7,6 +7,10 @@ fi
 
 VERSION=`cat flightgear/version`
 
+#####################################################################################
+# ensure fgrcc can run when linked against libSimGearCore, for example
+export LD_LIBRARY_PATH=$WORKSPACE/dist/lib64:$WORKSPACE/dist/lib:$LD_LIBRARY_PATH
+
 #####################################################################################
 # remove old and create fresh build directories
 cd $WORKSPACE
@@ -14,7 +18,6 @@ mkdir -p sgBuild
 mkdir -p fgBuild
 mkdir -p output
 rm -rf output/*
-rm -rf dist/*
 
 #####################################################################################
 echo "Starting on SimGear"
@@ -38,7 +41,7 @@ cp simgear-*.tar.bz2 ../output/.
 #####################################################################################
 echo "Starting on FlightGear"
 cd ../fgBuild
-cmake -DCMAKE_INSTALL_PREFIX:PATH=$WORKSPACE/dist -DSIMGEAR_SHARED:BOOL="ON" -DFG_BUILD_TYPE=Release ../flightgear
+cmake -DCMAKE_INSTALL_PREFIX:PATH=$WORKSPACE/dist -DSIMGEAR_SHARED:BOOL="ON" -DENABLE_SWIFT:BOOL=ON -DFG_BUILD_TYPE=Release ../flightgear
 
 # compile
 make

build_release_windows.bat

@@ -1,5 +1,6 @@
 
 IF NOT DEFINED WORKSPACE SET WORKSPACE=%~dp0
+IF NOT DEFINED IS_NIGHTLY_BUILD SET IS_NIGHTLY_BUILD=1
 
 IF %IS_NIGHTLY_BUILD% EQU 1 (
   SET FGBUILDTYPE=Nightly
@@ -19,10 +20,13 @@ SET OSG64=%WORKSPACE%\install\msvc140-64\OpenSceneGraph
 REM 32bits
 md build-sg32
 md build-fg32
+md build-fg32-compositor
+
 cd build-sg32
 cmake ..\simgear -G "Visual Studio 14" ^
                  -DMSVC_3RDPARTY_ROOT=%WORKSPACE%/windows-3rd-party/msvc140 ^
                  -DBOOST_ROOT=%WORKSPACE%/windows-3rd-party ^
+                 -DOSG_FSTREAM_EXPORT_FIXED=1 ^
                  -DCMAKE_PREFIX_PATH:PATH=%OSG32% ^
                  -DCMAKE_INSTALL_PREFIX:PATH=%WORKSPACE%/install/msvc140
 cmake --build . --config RelWithDebInfo --target INSTALL
@@ -31,10 +35,24 @@ cd ..\build-fg32
 cmake ..\flightgear -G "Visual Studio 14" ^
                     -DMSVC_3RDPARTY_ROOT=%WORKSPACE%/windows-3rd-party/msvc140 ^
                     -DCMAKE_INSTALL_PREFIX:PATH=%WORKSPACE%/install/msvc140 ^
-                    -DCMAKE_PREFIX_PATH:PATH=%WORKSPACE%/install/msvc140/OpenSceneGraph ^
                     -DBOOST_ROOT=%WORKSPACE%/windows-3rd-party ^
+                    -DOSG_FSTREAM_EXPORT_FIXED=1 ^
                     -DCMAKE_PREFIX_PATH=%QT5SDK32%;%OSG32% ^
-                    -DFG_BUILD_TYPE=%FGBUILDTYPE%
+                    -DFG_BUILD_TYPE=%FGBUILDTYPE% ^
+                    -DENABLE_SWIFT:BOOL=ON ^
+                    -DENABLE_COMPOSITOR:BOOL=OFF
+cmake --build . --config RelWithDebInfo --target INSTALL
+
+cd ..\build-fg32-compositor
+cmake ..\flightgear -G "Visual Studio 14" ^
+                    -DMSVC_3RDPARTY_ROOT=%WORKSPACE%/windows-3rd-party/msvc140 ^
+                    -DCMAKE_INSTALL_PREFIX:PATH=%WORKSPACE%/install/msvc140/compositor ^
+                    -DCMAKE_PREFIX_PATH:PATH=%QT5SDK32%;%OSG32%;%WORKSPACE%/install/msvc140/ ^
+                    -DBOOST_ROOT=%WORKSPACE%/windows-3rd-party ^
+                    -DOSG_FSTREAM_EXPORT_FIXED=1 ^
+                    -DFG_BUILD_TYPE=%FGBUILDTYPE% ^
+                    -DENABLE_SWIFT:BOOL=ON ^
+                    -DENABLE_COMPOSITOR:BOOL=ON
 cmake --build . --config RelWithDebInfo --target INSTALL
 
 cd ..
@@ -42,11 +60,13 @@ cd ..
 REM 64 bits
 md build-sg64
 md build-fg64
+md build-fg64-compositor
 
 cd build-sg64
 cmake ..\SimGear -G "Visual Studio 14 Win64" ^
                  -DMSVC_3RDPARTY_ROOT=%WORKSPACE%/windows-3rd-party/msvc140 ^
                  -DBOOST_ROOT=%WORKSPACE%/windows-3rd-party ^
+                 -DOSG_FSTREAM_EXPORT_FIXED=1 ^
                  -DCMAKE_PREFIX_PATH:PATH=%OSG64% ^
                  -DCMAKE_INSTALL_PREFIX:PATH=%WORKSPACE%/install/msvc140-64
 cmake --build . --config RelWithDebInfo --target INSTALL
@@ -57,14 +77,31 @@ cmake ..\flightgear -G "Visual Studio 14 Win64" ^
                     -DBOOST_ROOT=%WORKSPACE%/windows-3rd-party ^
                     -DCMAKE_INSTALL_PREFIX:PATH=%WORKSPACE%/install/msvc140-64 ^
                     -DCMAKE_PREFIX_PATH=%QT5SDK64%;%OSG64% ^
-                    -DFG_BUILD_TYPE=%FGBUILDTYPE%
+                    -DOSG_FSTREAM_EXPORT_FIXED=1 ^
+                    -DFG_BUILD_TYPE=%FGBUILDTYPE% ^
+                    -DENABLE_SWIFT:BOOL=ON ^
+                    -DENABLE_COMPOSITOR:BOOL=OFF
 cmake --build . --config RelWithDebInfo --target INSTALL
 
+cd ..\build-fg64-compositor
+cmake ..\flightgear -G "Visual Studio 14 Win64" ^
+                    -DMSVC_3RDPARTY_ROOT=%WORKSPACE%/windows-3rd-party/msvc140 ^
+                    -DBOOST_ROOT=%WORKSPACE%/windows-3rd-party ^
+                    -DCMAKE_INSTALL_PREFIX:PATH=%WORKSPACE%/install/msvc140-64/compositor ^
+                    -DCMAKE_PREFIX_PATH=%QT5SDK64%;%OSG64%;%WORKSPACE%/install/msvc140-64/ ^
+                    -DOSG_FSTREAM_EXPORT_FIXED=1 ^
+                    -DFG_BUILD_TYPE=%FGBUILDTYPE% ^
+                    -DENABLE_SWIFT:BOOL=ON ^
+                    -DENABLE_COMPOSITOR:BOOL=ON
+cmake --build . --config RelWithDebInfo --target INSTALL
 cd ..
 
 REM Qt5 deployment
-%QT5SDK32%\bin\windeployqt --release --list target %WORKSPACE%/install/msvc140/bin/fgfs.exe
+SET QMLDIR=%WORKSPACE%/flightgear/src/GUI/qml
-%QT5SDK64%\bin\windeployqt --release --list target %WORKSPACE%/install/msvc140-64/bin/fgfs.exe
+%QT5SDK32%\bin\windeployqt --release --list target --qmldir %QMLDIR% %WORKSPACE%/install/msvc140/bin/fgfs.exe
+%QT5SDK32%\bin\windeployqt --release --list target --qmldir %QMLDIR% %WORKSPACE%/install/msvc140/compositor/bin/fgfs.exe
+%QT5SDK64%\bin\windeployqt --release --list target --qmldir %QMLDIR% %WORKSPACE%/install/msvc140-64/bin/fgfs.exe
+%QT5SDK64%\bin\windeployqt --release --list target --qmldir %QMLDIR% %WORKSPACE%/install/msvc140-64/compositor/bin/fgfs.exe
 
 REM build setup
 ECHO Packaging root is %WORKSPACE%
@@ -75,9 +112,16 @@ subst X: %WORKSPACE%.
 REM ensure output dir is clean since we upload the entirety of it
 rmdir /S /Q output
 
-REM archiving PDB files
+SET FGFS_PDB=src\Main\RelWithDebInfo\fgfs.pdb
-copy %WORKSPACE%\build-fg32\src\Main\RelWithDebInfo\fgfs.pdb %WORKSPACE%\Output\fgfs-x86-%BUILD_NUMBER%.pdb
+
-copy %WORKSPACE%\build-fg64\src\Main\RelWithDebInfo\fgfs.pdb %WORKSPACE%\Output\fgfs-x64-%BUILD_NUMBER%.pdb
+SET SENTRY_ORG=flightgear
+SET SENTRY_PROJECT=flightgear
+REM ensure SENTRY_AUTH_TOKEN is set in the environment
+
+sentry-cli upload-dif %WORKSPACE%\build-fg32\%FGFS_PDB%
+sentry-cli upload-dif %WORKSPACE%\build-fg64\%FGFS_PDB%
+sentry-cli upload-dif %WORKSPACE%\build-fg32-compositor\%FGFS_PDB%
+sentry-cli upload-dif %WORKSPACE%\build-fg64-compositor\%FGFS_PDB%
 
 REM indirect way to get command output into an environment variable
 set PATH=%OSG32%\bin;%PATH%

catalog/README.md

@@ -0,0 +1,63 @@
+Introduction
+============
+
+This is the directory containing the script for updating aircraft, or other
+craft, catalogs.  It will create the `catalog.xml` file used to add a hangar to
+FlightGear, as well as the zip archives of each craft in the hangar and the
+md5sums, thumbnails, and previews of these.  It consists of the script:
+
+* update-catalog.py
+
+And its Python modules:
+
+* catalog.py
+* catalogTags.py
+* sgprogs.py
+
+
+Usage
+=====
+
+The script can be run directly from this directory, or the script and its
+modules can be copied together and run from any location.  The steps to use
+these are:
+
+* Create an output directory where the catalog and zip files will be located.
+* Copy the configuration files `catalog.config.xml`, `template.xml`, and
+  `zip-excludes.lst` from one of the `*catalog*` example directories into the
+  output directory.
+* Modify these files as required.
+
+Then run the script with:
+
+`$ $FGMETA/catalog/update-catalog.py dir`
+
+where `dir` is the output directory.  The script will create the following
+files:
+
+* `md5sum.xml`:  A file containing checksums of all craft zip archives in the
+  base output directory.
+* `ftp/catalog.xml`:  The XML catalog to upload to a server and advertise to
+  FlightGear users.
+* `ftp/*.zip`:  The zip archives of each craft in the hangar.
+* `ftp/previews/*_Splashs/`:  A directory per craft containing the splash screen
+  graphics.
+* `ftp/thumbnails/`:  The collection of thumbnail graphics for the hangar.
+
+The `ftp` directory is to be uploaded to a server which can be configured via
+the `catalog.config.xml` file.
+
+
+Examples
+========
+
+A number of example configuration files are located in this directory.  These
+include:
+
+* `fgaddon-catalog/`:  The configuration files used for the [official FGAddon
+  hangar](http://wiki.flightgear.org/FGAddon).
+* `stable-2018-catalog/':  The configuration files used for the 2018 long term
+  stability release.
+* `single-craft-catalog-test/`:  Configuration files used for testing the
+  catalog and zip archive creation for a single craft.  These are for content
+  developers to test their craft.

catalog/catalog.py

@@ -0,0 +1,430 @@
+#!/usr/bin/python
+
+import argparse
+import datetime
+from fnmatch import fnmatch, translate
+import lxml.etree as ET
+import os
+from os.path import exists, join, relpath
+from os import F_OK, access, walk
+import re
+import sgprops
+import sys
+import catalogTags
+import zipfile
+
+CATALOG_VERSION = 4
+quiet = False
+verbose = False
+
+# The Python version.
+PY_VERSION = sys.version_info[0]
+
+# Python 2 and 3 compatibility.
+if PY_VERSION == 3:
+    long = int
+
+
+def warning(msg):
+    if not quiet:
+        print(msg)
+
+def log(msg):
+    if verbose:
+        print(msg)
+
+# xml node (robust) get text helper
+def get_xml_text(e):
+    if e != None and e.text != None:
+        return e.text
+    else:
+        return ''
+
+# return all available aircraft information from the set file as a
+# dict
+def scan_set_file(aircraft_dir, set_file, includes):
+    base_file = os.path.basename(set_file)
+    base_id = base_file[:-8]
+    set_path = os.path.join(aircraft_dir, set_file)
+
+    includes.append(aircraft_dir)
+    root_node = sgprops.readProps(set_path, includePaths = includes)
+
+    if not root_node.hasChild("sim"):
+        return None
+
+    sim_node = root_node.getChild("sim")
+    if sim_node == None:
+        return None
+
+    # allow -set.xml files to specifcially exclude themselves from
+    # the creation process, by setting <exclude-from-catalog>true</>
+    if (sim_node.getValue("exclude-from-catalog", False) == True):
+        return None
+
+    variant = {}
+    name = sim_node.getValue("description", None)
+    if (name == None or len(name) == 0):
+        warning("Set file " + set_file + " is missing a <description>, skipping")
+        return None
+
+    variant['name'] =  name
+    variant['status'] = sim_node.getValue("status", None)
+
+    if sim_node.hasChild('authors'):
+        # aircraft has structured authors data, handle that
+        variant['authors'] = sim_node.getChild('authors')
+    
+    # can have legacy author tag alongside new strucutred data for
+    # backwards FG compatability
+    if sim_node.hasChild('author'):
+        variant['author'] = sim_node.getValue("author", None)
+
+    if sim_node.hasChild('maintainers'):
+        variant['maintainers'] = sim_node.getChild('maintainers')
+
+    if sim_node.hasChild('urls'):
+        variant['urls'] = sim_node.getChild('urls')
+
+    if sim_node.hasChild('long-description'):
+        variant['description'] = sim_node.getValue("long-description", None)
+    variant['id'] = base_id
+
+    # allow -set.xml files to declare themselves as primary.
+    # we use this avoid needing a variant-of in every other -set.xml
+    variant['primary-set'] = sim_node.getValue('primary-set', False)
+
+    # extract and record previews for each variant
+    if sim_node.hasChild('previews'):
+        variant['previews'] = extract_previews(sim_node.getChild('previews'), aircraft_dir)
+
+    if sim_node.hasChild('rating'):
+        variant['rating'] = sim_node.getChild("rating")
+
+    if sim_node.hasChild('tags'):
+        variant['tags'] = extract_tags(sim_node.getChild('tags'), set_file)
+
+    if sim_node.hasChild('thumbnail'):
+        variant['thumbnail'] = sim_node.getValue("thumbnail", None)
+
+    variant['variant-of'] = sim_node.getValue("variant-of", None)
+
+    if sim_node.hasChild('minimum-fg-version'):
+        variant['minimum-fg-version'] = sim_node.getValue('minimum-fg-version', None)
+
+    #print("    %s" % variant)
+    return variant
+
+def extract_previews(previews_node, aircraft_dir):
+    result = []
+    for node in previews_node.getChildren("preview"):
+        previewType = node.getValue("type", None)
+        previewPath = node.getValue("path", None)
+
+        # check path exists in base-name-dir
+        fullPath = os.path.join(aircraft_dir, previewPath)
+        if not os.path.isfile(fullPath):
+            warning("Bad preview path, skipping:" + fullPath)
+            continue
+        result.append({'type':previewType, 'path':previewPath})
+
+    return result
+
+def extract_tags(tags_node, set_path):
+    result = []
+    for node in tags_node.getChildren("tag"):
+        tag = node.value
+        # check tag is in the allowed list
+        if not catalogTags.isValidTag(tag):
+            warning("Unknown tag value:" + tag + " in " + set_path)
+        result.append(tag)
+
+    return result
+
+# scan all the -set.xml files in an aircraft directory.  Returns a
+# package dict and a list of variants.
+def scan_aircraft_dir(aircraft_dir, includes):
+    setDicts = []
+    primaryAircraft = []
+    package = None
+
+    files = os.listdir(aircraft_dir)
+    for file in sorted(files, key=lambda s: s.lower()):
+        if file.endswith('-set.xml'):
+            # print('trying: %s' % file)
+            try:
+                d = scan_set_file(aircraft_dir, file, includes)
+                if d == None:
+                    continue
+            except:
+                print("Skipping set file since couldn't be parsed: %s %s" % os.path.join(aircraft_dir, file), sys.exc_info()[0])
+                continue
+
+            setDicts.append(d)
+            if d['primary-set']:
+                # ensure explicit primary-set aircraft goes first
+                primaryAircraft.insert(0, d)
+            elif d['variant-of'] == None:
+                primaryAircraft.append(d)
+
+    # print(setDicts)
+    if len(setDicts) == 0:
+        return None, None
+
+    # use the first one
+    if len(primaryAircraft) == 0:
+        print("Aircraft has no primary aircraft at all: %s" % aircraft_dir)
+        primaryAircraft = [setDicts[0]]
+
+    package = primaryAircraft[0]
+    if not 'thumbnail' in package:
+        if (os.path.exists(os.path.join(aircraft_dir, "thumbnail.jpg"))):
+            package['thumbnail'] = "thumbnail.jpg"
+
+    # variants is just all the set dicts except the first one
+    variants = setDicts
+    variants.remove(package)
+    return (package, variants)
+
+# create an xml node with text content
+def make_xml_leaf(name, text):
+    leaf = ET.Element(name)
+    if text != None:
+        if isinstance(text, (int, long)):
+            leaf.text = str(text)
+        else:
+            leaf.text = text
+    else:
+        leaf.text = ''
+    return leaf
+
+def append_preview_nodes(node, variant, download_base, package_name):
+    if not 'previews' in variant:
+        return
+
+    for preview in variant['previews']:
+        preview_node = ET.Element('preview')
+        preview_url = download_base + 'previews/' + package_name + '_' + preview['path']
+        preview_node.append( make_xml_leaf('type', preview['type']) )
+        preview_node.append( make_xml_leaf('url', preview_url) )
+        preview_node.append( make_xml_leaf('path', preview['path']) )
+        node.append(preview_node)
+
+def append_tag_nodes(node, variant):
+    if not 'tags' in variant:
+        return
+
+    for tag in variant['tags']:
+        node.append(make_xml_leaf('tag', tag))
+
+def append_author_nodes(node, info):
+    if 'authors' in info:
+        node.append(info['authors']._createXMLElement())
+    if 'author' in info:
+        # traditional single author string
+        node.append( make_xml_leaf('author', info['author']) )
+
+def make_aircraft_node(aircraftDirName, package, variants, downloadBase, mirrors):
+    #print("package: %s" % package)
+    #print("variants: %s" % variants)
+    package_node = ET.Element('package')
+    package_node.append( make_xml_leaf('name', package['name']) )
+    package_node.append( make_xml_leaf('status', package['status']) )
+
+    append_author_nodes(package_node, package)
+
+    if 'description' in package:
+        package_node.append( make_xml_leaf('description', package['description']) )
+
+    if 'minimum-fg-version' in package:
+        package_node.append( make_xml_leaf('minimum-fg-version', package['minimum-fg-version']) )
+
+    if 'rating' in package:
+        package_node.append(package['rating']._createXMLElement())
+       
+    package_node.append( make_xml_leaf('id', package['id']) )
+    for variant in variants:
+        variant_node = ET.Element('variant')
+        package_node.append(variant_node)
+        variant_node.append( make_xml_leaf('id', variant['id']) )
+        variant_node.append( make_xml_leaf('name', variant['name']) )
+        if 'description' in variant:
+            variant_node.append( make_xml_leaf('description', variant['description']) )
+
+        if 'author' in variant:
+            variant_node.append( make_xml_leaf('author', variant['author']) )
+
+        if 'thumbnail' in variant:
+            # note here we prefix with the package name, since the thumbnail path
+            # is assumed to be unique within the package
+            thumbUrl = downloadBase + "thumbnails/" + aircraftDirName + '_' + variant['thumbnail']
+            variant_node.append(make_xml_leaf('thumbnail', thumbUrl))
+            variant_node.append(make_xml_leaf('thumbnail-path', variant['thumbnail']))
+
+        variantOf = variant['variant-of']
+        if variantOf is None:
+            variant_node.append(make_xml_leaf('variant-of', '_primary_'))
+        else:
+            variant_node.append(make_xml_leaf('variant-of', variantOf))
+
+        append_preview_nodes(variant_node, variant, downloadBase, aircraftDirName)
+        append_tag_nodes(variant_node, variant)
+        append_author_nodes(variant_node, variant)
+
+    package_node.append( make_xml_leaf('dir', aircraftDirName) )
+
+    # primary URL is first
+    download_url = downloadBase + aircraftDirName + '.zip'
+    package_node.append( make_xml_leaf('url', download_url) )
+
+    for m in mirrors:
+        mu = m + aircraftDirName + '.zip'
+        package_node.append( make_xml_leaf('url', mu) )
+
+
+    if 'thumbnail' in package:
+        thumbnail_url = downloadBase + 'thumbnails/' + aircraftDirName + '_' + package['thumbnail']
+        package_node.append( make_xml_leaf('thumbnail', thumbnail_url) )
+        package_node.append( make_xml_leaf('thumbnail-path', package['thumbnail']))
+
+    append_preview_nodes(package_node, package, downloadBase, aircraftDirName)
+    append_tag_nodes(package_node, package)
+
+    if 'maintainers' in package:
+        package_node.append(package['maintainers']._createXMLElement())
+
+    if 'urls' in package:
+        package_node.append(package['urls']._createXMLElement())
+
+    return package_node
+
+
+def make_aircraft_zip(repo_path, craft_name, zip_file, global_zip_excludes, verbose=True):
+    """Create a zip archive of the given aircraft."""
+
+    # Printout.
+    if verbose:
+        print("Zip file creation: %s.zip" % craft_name)
+
+    # Go to the directory of crafts to catalog.
+    savedir = os.getcwd()
+    os.chdir(repo_path)
+
+    # Clear out the old file.
+    if exists(zip_file):
+        os.remove(zip_file)
+
+    # Use the Python zipfile module to create the zip file.
+    zip_handle = zipfile.ZipFile(zip_file, 'w', zipfile.ZIP_DEFLATED)
+
+    # Find a per-craft exclude list.
+    craft_path = join(repo_path, craft_name)
+    exclude_file = join(craft_path, 'zip-excludes.lst')
+    if exists(exclude_file):
+        if verbose:
+            print("Found the craft specific exclusion list '%s'" % exclude_file)
+
+    # Otherwise use the catalog default exclusion list.
+    else:
+        exclude_file = global_zip_excludes
+
+    # Process the exclusion list and find all matching file names.
+    blacklist = fetch_zip_exclude_list(craft_name, craft_path, exclude_file)
+
+    # Walk over all craft files.
+    print_format = "  %-30s '%s'"
+    for root, dirs, files in walk(craft_path):
+        # Loop over the files.
+        for file in files:
+            # The directory and relative and absolute paths.
+            dir = relpath(root, start=repo_path)
+            full_path = join(root, file)
+            rel_path = relpath(full_path, start=repo_path)
+
+            # Skip blacklist files or directories.
+            skip = False
+            if file == 'zip-excludes.lst':
+                if verbose:
+                    print(print_format % ("Skipping the file:", join(dir, 'zip-excludes.lst')))
+                skip = True
+            if dir in blacklist:
+                if verbose:
+                    print(print_format % ("Skipping the file:", join(dir, file)))
+                skip = True
+            for name in blacklist:
+                if fnmatch(rel_path, name):
+                    if verbose:
+                        print(print_format % ("Skipping the file:", rel_path))
+                    skip = True
+                    break
+            if skip:
+                continue
+
+            # Otherwise add the file.
+            zip_handle.write(rel_path)
+
+    # Clean up.
+    os.chdir(savedir)
+    zip_handle.close()
+
+
+def fetch_zip_exclude_list(name, path, exclude_path):
+    """Use Unix style path regular expression to find all files to exclude."""
+
+    # Init.
+    blacklist = []
+    file = open(exclude_path)
+    exclude_list = file.readlines()
+    file.close()
+    old_path = os.getcwd()
+    os.chdir(path)
+
+    # Process each exclusion path or regular expression, converting to Python RE objects.
+    reobj_list = []
+    for i in range(len(exclude_list)):
+        reobj_list.append(re.compile(translate(exclude_list[i].strip())))
+
+    # Recursively loop over all files, finding the ones to exclude.
+    for root, dirs, files in walk(path):
+        for file in files:
+            full_path = join(root, file)
+            rel_path = join(name, relpath(full_path, start=path))
+
+            # Skip Unix shell-style wildcard matches
+            for i in range(len(reobj_list)):
+                if reobj_list[i].match(rel_path):
+                    blacklist.append(rel_path)
+                    break
+
+    # Return to the original path.
+    os.chdir(old_path)
+
+    # Return the list.
+    return blacklist
+
+
+def parse_config_file(parser=None, file_name=None):
+    """Test and parse the catalog configuration file."""
+
+    # Check for the file.
+    if not access(file_name, F_OK):
+        print("CatalogError: The catalog configuration file '%s' cannot be found." % file_name)
+        sys.exit(1)
+
+    # Parse the XML and return the root node.
+    config = ET.parse(file_name, parser)
+    return config.getroot()
+
+
+def parse_template_file(parser=None, file_name=None):
+    """Test and parse the catalog configuration file."""
+
+    # Check for the file.
+    if not access(file_name, F_OK):
+        print("CatalogError: The catalog template file '%s' cannot be found." % file_name)
+        sys.exit(1)
+
+    # Parse the XML and return the template node.
+    template = ET.parse(file_name, parser)
+    template_root = template.getroot()
+    return template_root.find('template')

catalog/catalogTags.py

@@ -1,80 +1,179 @@
 aircraftTypeTags = [
-    "ga", "fighter", "helicopter", "glider", "spaceship", "bomber", "groundvehicle",
+    "aerobatic",
-    "tanker", "cargo", "transport", "bizjet", "trainer", "airship", "balloon"
+    "airship",
+    "balloon",
+    "bizjet",
+    "bomber",
+    "cargo",
+    "carrier",
+    "fighter",
+    "ga",
+    "glider",
+    "groundvehicle",
+    "helicopter",
+    "interceptor",
+    "passenger",
+    "racer",
+    "spaceship",
+    "tanker",
+    "trainer",
+    "transport",
+    "ultralight",
+    "unpowered",
+    "uav",
+    "reconnaissance",
+    "seacraft",
+    "crop-duster",
+    "bush-plane"
 ]
 
 manufacturerTags = [
-    "boeing", "cessna", "diamond", "douglas", "bell", "piper",
+    "airbus",
-    "airbus", "vickers", "lockheed", "fokker",
+    "antonov",
-    "embraer", "bombardier", "pilatus", "robin",
+    "arado",
-    "eurocopter"
+    "atr",
+    "avro",
+    "bae",
+    "bell",
+    "bleriot",
+    "boeing",
+    "bombardier",
+    "caudron",
+    "cessna",
+    "consolidated",
+    "dassault",
+    "diamond",
+    "dornier",
+    "douglas",
+    "embraer",
+    "eurocopter",
+    "fairchild",
+    "fairey",
+    "focke-wulf",
+    "fokker",
+    "general-dynamics",
+    "gotha",
+    "grumman",
+    "handley-page",
+    "hawker",
+    "heinkel",
+    "ilyushin",
+    "junkers",
+    "kawasaki",
+    "lockheed",
+    "mc-donnell-douglas",
+    "messerschmitt",
+    "mikoyan-gurevich",
+    "mitsubishi",
+    "north-american",
+    "northrop",
+    "pilatus",
+    "piper",
+    "republic",
+    "robin",
+    "rockwell",
+    "saab",
+    "short",
+    "sopwith",
+    "spad",
+    "sukhoi",
+    "supermarine",
+    "tupolev",
+    "vickers",
+    "vought",
+    "wright",
+    "yakovlev"
 ]
 
 eraTags = [
-    "early-pioneers",
+    "1910s",
-    "ww1",
     "1920s",
     "1930s",
-    "golden-age",
+    "1940s",
-    "ww2",
-    "coldwar", "vietnam",
     "1950s",
     "1960s",
     "1970s",
     "1980s",
     "1990s",
     "2000s",
+    "2010s",
+    "coldwar",
+    "early-pioneers",
+    "golden-age",
     "gulfwar1",
-    "gulfwar2"
+    "gulfwar2",
+    "vietnam",
+    "ww1",
+    "ww2"
 ]
 
 featureTags = [
-    "ifr",
+    "aerobatic",
-    "retractable-gear",
-    "fixed-gear",
-    "tail-dragger",
-    "seaplane",
-    "vtol",
-    "stol",
-    "experimental",
-    "prototype",
-    "fictional",
-    "biplane",
-    "triplane",
-    "supersonic",
-    "t-tail",
-    "v-tail",
-    "high-wing",
-    "cannard",
-    "tail-hook",
-    "refuel",
-    "delta",
-    "variable-geometry",
-    "glass-cockpit",
-    "hud",
-    "etops",
-    "floats",
-    "amphibious",
     "airship",
-    "aerobatic"
+    "amphibious",
+    "biplane",
+    "canard",
+    "castering-wheel",
+    "combat",
+    "delta",
+    "etops",
+    "experimental",
+    "fictional",
+    "fixed-gear",
+    "floats",
+    "glass-cockpit",
+    "low-wing",
+    "mid-wing",
+    "high-wing",
+    "h-tail",
+    "hud",
+    "ifr",
+    "lifting-body",
+    "pressurised",
+    "prototype",
+    "refuel",
+    "retractable-gear",
+    "seaplane",
+    "skis",
+    "stol",
+    "supersonic",
+    "supercharger",
+    "t-tail",
+    "tail-dragger",
+    "tricycle",
+    "tail-hook",
+    "triplane",
+    "twin-boom",
+    "v-tail",
+    "variable-geometry",
+    "vtol",
+    "wing-fold"
+    "water-drop"
 ]
 
 propulsionTags = [
-    "piston", "radial",
+    "afterburner",
     "diesel",
-    "variable-pitch",
+    "electric",
+    "jet",
+    "propeller",
+    "piston",
+    "radial",
+    "rocket",
+    "single-engine",
     "supercharged",
     "turboprop",
-    "jet", "afterburner", "rocket",
-    "electric",
     "twin-engine",
-    "single-engine"
+    "four-engine",
+    "variable-pitch",
+    "fixed-pitch"
 ]
 
 simFeatureTags = [
-    "tow",
     "dual-controls",
-    "rembrandt"
+    "rembrandt",
+    "tow",
+    "wildfire"
 ]
 
 tags = aircraftTypeTags + manufacturerTags + eraTags + simFeatureTags + propulsionTags + featureTags

catalog/check_aircraft.py

@@ -0,0 +1,73 @@
+#!/usr/bin/python
+
+import argparse
+import os
+import sgprops
+
+def check_meta_data(aircraft_dir, set_file, includes):
+    base_file = os.path.basename(set_file)
+    base_id = base_file[:-8]
+    set_path = os.path.join(aircraft_dir, set_file)
+
+    includes.append(aircraft_dir)
+    root_node = sgprops.readProps(set_path, includePaths = includes)
+
+    if not root_node.hasChild("sim"):
+        print "-set.xml has no <sim> node:", set_path
+        return
+
+    sim_node = root_node.getChild("sim")
+    if not sim_node.hasChild('description'):
+        print "-set.xml missing <description>:", set_path
+
+    if not sim_node.hasChild('long-description'):
+        print "-set.xml missing <long-description>:", set_path
+
+    if not sim_node.hasChild('authors'):
+        print "-set.xml is missing structured <authors> data:", set_path
+
+    if not sim_node.hasChild('tags'):
+        print "-set.xml does not define any tags", set_path
+
+    # check for non-standard tags
+
+    if not sim_node.hasChild('thumbnail'):
+        print "-set.xml does not define a thumbnail", set_path
+
+    # check thumbnail size and format
+
+    if not sim_node.hasChild('rating'):
+        print "-set.xml does not define any ratings", set_path
+
+    if not sim_node.hasChild('minimum-fg-version'):
+        print "-set.xml does not define a minimum FG version", set_path
+
+# check all the -set.xml files in an aircraft directory.  
+def check_aircraft_dir(d, includes):
+    if not os.path.isdir(d):
+        return
+
+    files = os.listdir(d)
+    for file in sorted(files, key=lambda s: s.lower()):
+        if file.endswith('-set.xml'):
+            check_meta_data(d, file, includes)
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--include", help="Include directory to validate -set.xml parsing",
+                    action="append", dest='include', default=[])
+parser.add_argument("dir", nargs='+', help="Aircraft directory")
+args = parser.parse_args()
+
+for d in args.dir:
+    if not os.path.isdir(d):
+        print "Skipping missing directory:", d
+
+    names = os.listdir(d)
+    for name in sorted(names, key=lambda s: s.lower()):
+        # if name in skip_list:
+        #     print "skipping:", name
+        #     continue
+
+        acftDir = os.path.join(d, name)
+        check_aircraft_dir(acftDir, args.include)
+    

catalog/fgaddon-catalog-ukmirror/catalog.config.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<PropertyList>
+  <local-output>/home/fgaddon/output/Aircraft-trunk</local-output>
+  <download-url n="0">http://mirrors.ibiblio.org/flightgear/ftp/Aircraft-trunk/</download-url>
+  <download-url n="1">http://ukmirror.flightgear.org/fgaddon/Aircraft-trunk/</download-url>
+  <download-url n="2">https://cdn.merspieler.tk/fgaddon/Aircraft-trunk/</download-url>
+
+  <thumbnail-url n="0">http://mirrors.ibiblio.org/flightgear/ftp/Aircraft-trunk/thumbnails</thumbnail-url>
+  <thumbnail-url n="1">http://ukmirror.flightgear.org/fgaddon/Aircraft-trunk/thumbnails</thumbnail-url>
+  <thumbnail-url n="2">https://cdn.merspieler.tk/fgaddon/Aircraft-trunk/thumbnails</thumbnail-url>
+
+  <scm>
+    <type>svn</type>
+    <path>/home/fgaddon/fgaddon-trunk/Aircraft</path>
+    <skip>NTPS</skip>
+    <skip>c172</skip>
+    <skip>tu134</skip>
+  </scm>
+    <include-dir>/home/fgaddon/fgdata</include-dir>
+    <include-dir>/home/fgaddon/fgaddon-trunk</include-dir>
+    
+  <!-- <scm>
+       <type>git</type>
+       <update type="bool">false</update>
+       <path>/Users/jmt/FGFS/fgdata</path>
+       <url>git://git.code.sf.net/p/flightgear/fgdata</url>
+       </scm> -->
+  <upload n="0">
+    <enabled type="bool">true</enabled>
+    <type>rsync-ssh</type>
+    <remote>ibiblio:/public/mirrors/flightgear/ftp/Aircraft</remote>
+  </upload>
+</PropertyList>

catalog/fgaddon-catalog-ukmirror/template.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<PropertyList>
+  <template>
+	  <version n="0">2019.*</version>
+	  <version n="1">2018.*</version>
+	  <version n="2">2020.1.*</version>
+	  <version n="3">2020.2.*</version>
+
+    <id>org.flightgear.fgaddon.trunk</id>
+    <license>GPL</license>
+    <url>http://mirrors.ibiblio.org/flightgear/ftp/Aircraft-trunk/catalog.xml</url>
+    <name>FlightGear aircraft distribution from fgaddon</name>
+    <description>This hangar provides aircraft officially supported and maintained by the FlightGear project, 
+    under a free-software license. It tracks latest aircraft developments in the FGaddon repository.</description>
+    <!-- <de>
+      <description>Auf Deutsch</description>
+    </de>
+    <fr>
+      <description>En Francais</description>
+    </fr> -->
+
+ </template>
+</PropertyList>

catalog/fgaddon-catalog-ukmirror/zip-excludes.lst

@@ -0,0 +1,11 @@
+*/.svn/*
+*.xcf*
+*.XCF*
+*.blend*
+*.psd
+*~
+*/dev/*
+*/Dev/*
+*/development/*
+*/src/*
+*/Resources/*

catalog/fgaddon-catalog/template.xml

@@ -6,13 +6,13 @@
     <version n="1">3.5.*</version>
     <version n="2">3.6.*</version>
     <version n="3">3.7.*</version>
-    <version n="4">2016.1.*</version>
     <version n="5">2016.*.*</version>
     <version n="6">2017.*.*</version>
+    <version n="7">2018.*.*</version>
     <id>org.flightgear.fgaddon</id>
     <license>GPL</license>
     <url>http://mirrors.ibiblio.org/flightgear/ftp/Aircraft/catalog.xml</url>
-    <name>FlightGear Aircraft Distribution From fgaddon</name>
+    <name>FlightGear aircraft distribution from fgaddon</name>
     <description>This hangar provides aircraft officially supported and maintained by the FlightGear project, under a free-software license.</description>
     <de>
       <description>Auf Deutsch</description>

catalog/fgaddon-catalog/zip-excludes.lst

@@ -1,5 +1,11 @@
 */.svn/*
-*.xcf
+*.xcf*
-*.blend
+*.XCF*
+*.blend*
 *.psd
 *~
+*/dev/*
+*/Dev/*
+*/development/*
+*/src/*
+*/Resources/*

catalog/legacy-catalog/catalog.config.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<PropertyList>
+  <local-output>/home/fgaddon/output/Aircraft</local-output>
+  <download-url n="0">http://mirrors.ibiblio.org/flightgear/ftp/Aircraft/</download-url>
+  <download-url n="1">http://ukmirror.flightgear.org/fgaddon/Aircraft/</download-url>
+
+  <thumbnail-url n="0">http://mirrors.ibiblio.org/flightgear/ftp/Aircraft/thumbnails</thumbnail-url>
+  <thumbnail-url n="1">http://ukmirror.flightgear.org/fgaddon/Aircraft/thumbnails</thumbnail-url>
+
+  <scm>
+    <type>svn</type>
+    <path>/home/fgaddon/fgaddon-legacy/Aircraft</path>
+    <skip>NTPS</skip>
+    <skip>c172</skip>
+    <skip>tu134</skip>
+  </scm>
+    <include-dir>/home/fgaddon/fgdata-legacy</include-dir>
+    <include-dir>/home/fgaddon/fgaddon-legacy</include-dir>
+
+  <upload n="0">
+    <enabled type="bool">true</enabled>
+    <type>rsync-ssh</type>
+    <remote>ibiblio:/public/mirrors/flightgear/ftp/Aircraft-legacy</remote>
+  </upload>
+</PropertyList>

catalog/legacy-catalog/template.xml

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<PropertyList>
+  <template>
+    <version n="0">3.4.*</version>
+    <version n="1">3.5.*</version>
+    <version n="2">3.6.*</version>
+    <version n="3">3.7.*</version>
+    <version n="5">2016.*.*</version>
+    <version n="6">2017.*.*</version>
+    <version n="7">2018.*.*</version>
+
+    <!-- the legacy catalog has to have this ID, becuase this is what older FG
+    clients expect -->
+    <id>org.flightgear.fgaddon</id>
+    <license>GPL</license>
+    <url>http://mirrors.ibiblio.org/flightgear/ftp/Aircraft/catalog.xml</url>
+    <name>FlightGear aircraft(legacy support)</name>
+    <description>This hangar provides aircraft officially supported and maintained by the 
+    FlightGear project, under a free-software license. Aircraft are intended for
+    use with FlightGear versions 2018.2 and older.</description>
+
+<!--    <alternate-version>
+      <version>2019.*</version>
+      <version>2018.4.*</version>
+      <url>http://mirrors.ibiblio.org/flightgear/ftp/Aircraft-trunk/catalog.xml</url>
+      <id>org.flightgear.fgaddon.trunk</id>
+    </alternate-version>
+
+    <alternate-version>
+      <version>2018.3.*</version>
+      <url>http://mirrors.ibiblio.org/flightgear/ftp/Aircraft-2018/catalog.xml</url>
+      <id>org.flightgear.fgaddon.stable_2018</id>
+    </alternate-version>
+-->
+
+  </template>
+</PropertyList>

catalog/legacy-catalog/zip-excludes.lst

@@ -0,0 +1,5 @@
+*/.svn/*
+*.xcf
+*.blend
+*.psd
+*~

catalog/sgprops.py

@@ -2,7 +2,8 @@
 from xml.sax import make_parser, handler, expatreader
 
 # ElementTree for writing
-import xml.etree.cElementTree as ET
+#import xml.etree.cElementTree as ET
+import lxml.etree as ET
 
 import re, os
 
@@ -118,7 +119,7 @@ class Node(object):
         # value and type specification
         try:
             if self._value is not None:
-                if isinstance(self._value, basestring):
+                if isinstance(self._value, str):
                     # don't call str() on strings, breaks the
                     # encoding
                     n.text = self._value
@@ -132,7 +133,9 @@ class Node(object):
                     elif isinstance(self._value, bool):
                         n.set('type', "bool")
         except UnicodeEncodeError:
-            print "Encoding error with", self._value, type(self._value)
+            print("Encoding error with %s %s" % (self._value, type(self._value)))
+        except:
+            print("Unexpected exception in sgprops._createXMLElement():", sys.exc_info()[0])
 
         # index in parent
         if (self.index != 0):
@@ -191,7 +194,7 @@ class PropsHandler(handler.ContentHandler):
             try:
                 index = int(attrs['n'])
             except:
-                print "Invalid index at line:", self._locator.getLineNumber(), "of", self._path
+                print("Invalid index at line: %s of %s" % (self._locator.getLineNumber(), self._path))
                 raise IndexError("Invalid index at line:", self._locator.getLineNumber(), "of", self._path)
 
             currentState.recordExplicitIndex(name, index)
@@ -248,7 +251,7 @@ class PropsHandler(handler.ContentHandler):
                         self._content = self._content[:-1]
                     self._current.value = float(self._content)
         except:
-            print "Parse error for value:", self._content, "at line:", self._locator.getLineNumber(), "of:", self._path
+            print("Parse error for value: %s at line: %s of: %s" % (self._content, self._locator.getLineNumber(), self._path))
 
         self._current = self._current.parent
         self._content = None

catalog/single-craft-catalog-test/README

@@ -0,0 +1,6 @@
+Configuration files for testing the catalog updating script on a single aircraft.  This is useful for seeing what will be packed into the zip archive.  To use these:
+
+* Copy all *.xml and *.lst files into a local testing directory X.
+* Modify the *.xml files:
+    * Replace all instances of '/data/fgaddon_catalog_p51d/' with the directory X.
+    * Update any other tags as necessary.

catalog/single-craft-catalog-test/catalog.config.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--Template catalog - copy and modify for your site as required-->
+<PropertyList>
+  <local-output>/data/fgaddon-catalog_p51d/ftp</local-output>
+  <download-url n="0">/data/fgaddon-catalog_p51d/ftp/Aircraft/</download-url>
+  <scm>
+    <type>svn</type>
+    <update type="bool">false</update>
+    <path>/flightgear/flightgear-fgaddon/Aircraft</path>
+    <include>p51d</include>
+  </scm>
+</PropertyList>

catalog/single-craft-catalog-test/template.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--Template catalog - copy and modify for your site as required-->
+<PropertyList>
+  <template>
+    <version n="0">HEAD</version>
+    <id>localhost</id>
+    <license>GPL</license>
+    <url>/data/fgaddon-catalog_p51d/ftp/Aircraft/catalog.xml</url>
+    <name>p51d FGAddon aircraft test</name>
+    <description>This is a test of the catalog updating script to allow aircraft authors to see what will be present in the zip archive of their aircraft.</description>
+  </template>
+</PropertyList>

catalog/single-craft-catalog-test/zip-excludes.lst

@@ -0,0 +1,11 @@
+*/.svn/*
+*.xcf*
+*.XCF*
+*.blend*
+*.psd
+*~
+*/dev/*
+*/Dev/*
+*/development/*
+*/src/*
+*/Resources/*

catalog/stable-2018-catalog/catalog.config.xml

@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--Template catalog - copy and modify for your site as required-->
+<PropertyList>
+	<local-output>/home/fgaddon/output/Aircraft-2018</local-output>
+  <download-url n="0">http://mirrors.ibiblio.org/flightgear/ftp/Aircraft-2018/</download-url>
+  <download-url n="1">http://ukmirror.flightgear.org/fgaddon/Aircraft-2018/</download-url>
+  <download-url n="2">https://cdn.merspieler.tk/fgaddon/Aircraft-2018/</download-url>
+
+  <thumbnail-url n="0">http://mirrors.ibiblio.org/flightgear/ftp/Aircraft-2018/thumbnails</thumbnail-url>
+  <thumbnail-url n="1">http://ukmirror.flightgear.org/official/Aircraft-2018/thumbnails</thumbnail-url>
+  <thumbnail-url n="2">https://cdn.merspieler.tk/fgaddon/Aircraft-2018/thumbnails</thumbnail-url>
+
+
+  <!-- share zips with the trunk catalog where possible -->
+  <share-output>/home/fgaddon/output/Aircraft</share-output>
+  <share-md5-sums>/home/fgaddon/fgmeta/catalog/fgaddon-catalog-ukmirror/md5sum.xml</share-md5-sums>
+
+  <scm>
+    <type>svn</type>
+    <path>/home/fgaddon/fgaddon-release-2018.3/Aircraft</path>
+    <skip>NTPS</skip>
+    <skip>c172</skip>
+    <skip>tu134</skip>
+  </scm>
+
+    <include-dir>/home/fgaddon/fgdata-release_2018.3</include-dir>
+    <include-dir>/home/fgaddon/fgaddon-release-2018.3</include-dir>
+    
+  <upload n="0">
+    <enabled type="bool">true</enabled>
+    <type>rsync-ssh</type>
+    <remote>ibiblio:/public/mirrors/flightgear/ftp/Aircraft-2018</remote>
+  </upload>
+</PropertyList>

catalog/stable-2018-catalog/template.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<PropertyList>
+  <template>
+	  <version n="0">2018.*</version>
+    <id>org.flightgear.fgaddon.stable_2018</id>
+    <license>GPL</license>
+    <url>http://mirrors.ibiblio.org/flightgear/ftp/Aircraft-2018/catalog.xml</url>
+    <name>FlightGear aircraft distribution for 2018.x versions</name>
+    <description>This hangar provides aircraft officially supported and maintained 
+    by the FlightGear project, under a free-software license. These aircraft
+    are compatible with FlightGear versions from 2018.</description>
+    
+   </template>
+</PropertyList>

catalog/stable-2018-catalog/zip-excludes.lst

@@ -0,0 +1,5 @@
+*/.svn/*
+*.xcf
+*.blend
+*.psd
+*~

catalog/testData/Aircraft/c150/c150-set.xml

@@ -0,0 +1,8 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<PropertyList>
+  <sim>
+    <name>c150</name>
+    <description>Cessna 150</description>
+  </sim>
+
+</PropertyList>

catalog/testData/Aircraft/c150/zip-excludes.lst

@@ -0,0 +1 @@
+*.xcf*

catalog/testData/Aircraft/c172/c172-set.xml

@@ -0,0 +1,9 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<PropertyList>
+  <sim>
+    <name>c172</name>
+    <description>Cessna 172P</description>
+    <author>Wilbur Wright</author>
+  </sim>
+
+</PropertyList>

catalog/testData/Aircraft/dc3/.svn/entries

@@ -0,0 +1 @@
+12

catalog/testData/Aircraft/dc3/dc3-set.xml

@@ -0,0 +1,21 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<PropertyList>
+  <sim>
+    <name>dc3</name>
+    <description>Douglas DC-3</description>
+    <author>Donald Douglas</author>
+    <authors>
+        <author>
+            <name>Donald Douglas</name>
+            <nick>dd</nick>
+            <email>dd@douglas.com</email>
+            <description>Made all the stuff</description>
+        </author>
+    </authors>
+    <urls>
+        <support>https://bugtracker.douglas.com/</support>
+        <home-page>http://www.douglas.com</home-page>
+    </urls>
+  </sim>
+
+</PropertyList>

catalog/testData/Aircraft/f16/f16-common.xml

@@ -0,0 +1,24 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<PropertyList>
+  <sim include="settings-common.xml">
+    <authors>
+      <author n="0">
+        <name>Wilbur Wright</name>
+        <email>ww@wright.com</email>
+        <nick>wilburw</nick>
+        <description>Model, FDM and cockpit</description>
+      </author>
+      <author n="1">
+        <name>Orville Wright</name>
+        <description>Testing and systems</description>
+      </author>
+    </authors>
+    <tags>
+      <tag>fighter</tag>
+      <tag>1980s</tag>
+      <tag>glass-cockpit</tag>
+    </tags>
+    <minimum-fg-version>2017.4</minimum-fg-version>
+  </sim>
+
+</PropertyList>

catalog/testData/Aircraft/f16/f16-excluded-set.xml

@@ -0,0 +1,11 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<PropertyList include="f16-common.xml">
+  <sim>
+    <exclude-from-catalog type="bool">true</exclude-from-catalog> 
+    <name>f16-excluded</name>
+    <description>Fine</description>
+    <long-description>Blah blah blah</long-description>
+    <variant-of>f16a</variant-of>
+  </sim>
+
+</PropertyList>

catalog/testData/Aircraft/f16/f16-trainer-set.xml

@@ -0,0 +1,11 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<PropertyList include="f16-common.xml">
+  <sim>
+    <name>f16-trainer</name>
+    <description>F16 Trainer</description>
+    <long-description>Twin-seat trainer version of the F16</long-description>
+
+
+  </sim>
+
+</PropertyList>

catalog/testData/Aircraft/f16/f16a-set.xml

@@ -0,0 +1,36 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<PropertyList include="f16-common.xml">
+  <sim>
+    <name>f16a</name>
+    <description>F16-A</description>
+    <long-description>The F16 is compact, light-weight multi-role fighter used around the world</long-description>
+
+    <primary-set type="bool">true</primary-set>
+    
+    <rating>
+      <FDM type="int">3</FDM>
+      <systems type="int">1</systems>
+      <cockpit type="int">2</cockpit>
+      <model type="int">5</model>
+    </rating>
+
+    <previews>
+        <preview>
+          <type>exterior</type>
+          <splash type="bool">true</splash>
+          <path>Previews/exterior-1.png</path>
+        </preview>
+        <preview>
+          <type>exterior</type>
+          <splash type="bool">true</splash>
+          <path>Previews/exterior-f16a-2.png</path>
+        </preview>
+        <preview>
+          <type>panel</type>
+          <splash type="bool">false</splash>
+          <path>Previews/cockpit.png</path>
+        </preview>
+      </previews>
+  </sim>
+
+</PropertyList>

catalog/testData/Aircraft/f16/f16b-set.xml

@@ -0,0 +1,19 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<PropertyList include="f16-common.xml">
+  <sim>
+    <name>f16b</name>
+    <description>F16-B</description>
+    <long-description>The F16-B is an upgraded version of the F16A.</long-description>
+    <variant-of>f16a</variant-of>
+
+    <authors n="0">
+      <author n="0">
+        <name>James T Kirk</name>
+        <email>shatner@enterprise.com</email>
+        <nick>starlover</nick>
+        <description>Everything</description>
+      </author>
+    </authors>
+  </sim>
+
+</PropertyList>

catalog/testData/Aircraft/f16/f16broken-set.xml

@@ -0,0 +1,11 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<PropertyList include="f16-common.xml">
+  <sim>
+    <name>f16broken</name>
+    <!-- <description></description> -->
+    <description></description>
+    <long-description>Blah blah blah</long-description>
+    <variant-of>f16a</variant-of>
+  </sim>
+
+</PropertyList>

catalog/testData/Aircraft/f16/f16c-set.xml

@@ -0,0 +1,10 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<PropertyList include="f16-common.xml">
+  <sim>
+    <name>f16c</name>
+    <description>F16-C</description>
+    <long-description>The F16-C is an upgraded version of the F16A.</long-description>
+    <variant-of>f16a</variant-of>
+  </sim>
+
+</PropertyList>

catalog/testData/OtherDir/settings-common.xml

@@ -0,0 +1,12 @@
+<?xml version='1.0' encoding='UTF-8'?>
+
+<!-- this file exists to test including + overlaying in the set XML -->
+
+<PropertyList>
+    <views>
+      <view n="0">
+        <foo>dsdddd</foo>
+      </view>
+    </views>
+
+</PropertyList>

catalog/test_catalog.py

@@ -0,0 +1,323 @@
+#!/usr/bin/python
+
+import unittest
+import sgprops
+import os
+from os.path import join
+import catalog
+import lxml.etree as ET
+from shutil import rmtree
+from tempfile import mkdtemp
+import zipfile
+
+
+catalog.quiet = True
+
+class UpdateCatalogTests(unittest.TestCase):
+    def test_scan_set(self):
+        info = catalog.scan_set_file("testData/Aircraft/f16", "f16a-set.xml", ["testData/OtherDir"])
+        self.assertEqual(info['id'], 'f16a')
+        self.assertEqual(info['name'], 'F16-A')
+        self.assertEqual(info['primary-set'], True)
+        self.assertEqual(info['variant-of'], None)
+       # self.assertEqual(info['rating_FDM'], 3)
+      #  self.assertEqual(info['rating_model'], 5)
+
+        ratings = info['rating']
+        self.assertEqual(ratings.getValue('FDM'), 3)
+        self.assertEqual(ratings.getValue('model'), 5)
+
+        self.assertEqual(len(info['tags']), 3)
+        self.assertEqual(info['minimum-fg-version'], '2017.4')
+
+        authors = info['authors']
+        self.assertNotIn('author', info)
+        self.assertEqual(len(authors.getChildren()), 2)
+
+        self.assertEqual(authors.getValue('author[0]/name'), 'Wilbur Wright')
+        self.assertEqual(authors.getValue('author[0]/nick'), 'wilburw')
+        self.assertEqual(authors.getValue('author[0]/email'), 'ww@wright.com')
+        self.assertEqual(authors.getValue('author[1]/name'), 'Orville Wright')
+
+
+    def test_scan_dir(self):
+        (pkg, variants) = catalog.scan_aircraft_dir("testData/Aircraft/f16", ["testData/OtherDir"])
+
+        self.assertEqual(pkg['id'], 'f16a')
+        f16trainer = next(v for v in variants if v['id'] == 'f16-trainer')
+        self.assertEqual(len(variants), 3)
+        self.assertEqual(pkg['minimum-fg-version'], '2017.4')
+
+        # test variant relatonship between
+        self.assertEqual(pkg['variant-of'], None)
+        self.assertEqual(pkg['primary-set'], True)
+
+        self.assertEqual(f16trainer['variant-of'], None)
+        self.assertEqual(f16trainer['primary-set'], False)
+
+        f16b = next(v for v in variants if v['id'] == 'f16b')
+        self.assertEqual(f16b['variant-of'], 'f16a')
+        self.assertEqual(f16b['primary-set'], False)
+
+        authorsArray = f16b['authors']
+        self.assertNotIn('author', f16b)
+
+        self.assertEqual(authorsArray.getValue('author[0]/name'), 'James T Kirk')
+        self.assertEqual(authorsArray.getValue('author[0]/nick'), 'starlover')
+
+        f16c = next(v for v in variants if v['id'] == 'f16c')
+        self.assertEqual(f16c['variant-of'], 'f16a')
+        self.assertEqual(f16c['primary-set'], False)
+
+        authors = f16c['authors']
+        self.assertNotIn('author', f16c)
+        self.assertEqual(len(authors.getChildren()), 2)
+
+    # test some older constructs for compat
+    def test_scan_dir_legacy(self):
+        (pkg, variants) = catalog.scan_aircraft_dir("testData/Aircraft/c172", [])
+
+        self.assertEqual(pkg['id'], 'c172')
+        self.assertEqual(pkg['author'], 'Wilbur Wright')
+
+    def test_extract_previews(self):
+        info = catalog.scan_set_file("testData/Aircraft/f16", "f16a-set.xml", ["testData/OtherDir"])
+        previews = info['previews']
+        self.assertEqual(len(previews), 3)
+        self.assertEqual(2, len([p for p in previews if p['type'] == 'exterior']))
+        self.assertEqual(1, len([p for p in previews if p['type'] == 'panel']))
+        self.assertEqual(1, len([p for p in previews if p['path'] == 'Previews/exterior-1.png']))
+
+    def test_extract_tags(self):
+        info = catalog.scan_set_file("testData/Aircraft/f16", "f16a-set.xml", ["testData/OtherDir"])
+        tags = info['tags']
+
+    def test_node_creation(self):
+        (pkg, variants) = catalog.scan_aircraft_dir("testData/Aircraft/f16", ["testData/OtherDir"])
+
+        catalog_node = ET.Element('PropertyList')
+        catalog_root = ET.ElementTree(catalog_node)
+
+        pkgNode = catalog.make_aircraft_node('f16', pkg, variants, "http://foo.com/testOutput/", [])
+        catalog_node.append(pkgNode)
+
+        # write out so we can parse using sgprops
+        # yes we are round-tripping via the disk, if you can improve
+        # then feel free..
+        if not os.path.isdir("testOutput"):
+            os.mkdir("testOutput")
+
+        cat_file = os.path.join("testOutput", 'catalog_fragment.xml')
+        catalog_root.write(cat_file, encoding='utf-8', xml_declaration=True)
+
+        parsed = sgprops.readProps(cat_file)
+        parsedPkgNode = parsed.getChild("package")
+
+        self.assertEqual(parsedPkgNode.name, "package");
+
+        self.assertEqual(parsedPkgNode.getValue('id'), pkg['id']);
+        self.assertEqual(parsedPkgNode.getValue('dir'), 'f16');
+        self.assertEqual(parsedPkgNode.getValue('url'), 'http://foo.com/testOutput/f16.zip');
+        self.assertEqual(parsedPkgNode.getValue('thumbnail'), 'http://foo.com/testOutput/thumbnails/f16_thumbnail.jpg');
+        self.assertEqual(parsedPkgNode.getValue('thumbnail-path'), 'thumbnail.jpg');
+
+        self.assertEqual(parsedPkgNode.getValue('name'), pkg['name']);
+        self.assertEqual(parsedPkgNode.getValue('description'), pkg['description']);
+
+        self.assertEqual(parsedPkgNode.getValue('minimum-fg-version'), "2017.4");
+
+        parsedVariants = parsedPkgNode.getChildren("variant")
+        self.assertEqual(len(parsedVariants), 3)
+
+        # verify rating copying
+        self.assertEqual(parsedPkgNode.getValue('rating/FDM'), 3)
+        self.assertEqual(parsedPkgNode.getValue('rating/cockpit'), 2)
+        self.assertEqual(parsedPkgNode.getValue('rating/model'), 5)
+
+
+# author data verification
+        self.assertFalse(parsedPkgNode.hasChild('author'));
+        parsedAuthors = parsedPkgNode.getChild("authors").getChildren('author')
+
+        self.assertEqual(len(parsedAuthors), 2)
+        author1 = parsedAuthors[0]
+        self.assertEqual(author1.getValue("name"), "Wilbur Wright")
+        self.assertEqual(author1.getValue("nick"), "wilburw")
+        self.assertEqual(author1.getValue("email"), "ww@wright.com")
+
+        author2 = parsedAuthors[1]
+        self.assertEqual(author2.getValue("name"), "Orville Wright")
+
+        f16ANode = parsedPkgNode
+        self.assertEqual(f16ANode.getValue('name'), 'F16-A');
+
+        for index, pv in enumerate(parsedVariants):
+            var = variants[index]
+            self.assertEqual(pv.getValue('name'), var['name']);
+            self.assertEqual(pv.getValue('description'), var['description']);
+
+            if (var['id'] == 'f16-trainer'):
+                self.assertEqual(pv.getValue('variant-of'), '_primary_')
+            #    self.assertEqual(pv.getValue('author'), "Wilbur Wright");
+            elif (var['id'] == 'f16b'):
+                self.assertEqual(pv.getValue('variant-of'), 'f16a')
+                self.assertEqual(pv.getValue('description'), 'The F16-B is an upgraded version of the F16A.')
+
+                # variant author verification
+                parsedAuthors = pv.getChild("authors").getChildren('author')
+                author1 = parsedAuthors[0]
+                self.assertEqual(author1.getValue("name"), "James T Kirk")
+                self.assertEqual(author1.getValue("nick"), "starlover")
+                self.assertEqual(author1.getValue("email"), "shatner@enterprise.com")
+                self.assertEqual(author1.getValue("description"), "Everything")
+
+    def test_node_creation2(self):
+        (pkg, variants) = catalog.scan_aircraft_dir("testData/Aircraft/dc3", ["testData/OtherDir"])
+
+        catalog_node = ET.Element('PropertyList')
+        catalog_root = ET.ElementTree(catalog_node)
+
+        pkgNode = catalog.make_aircraft_node('dc3', pkg, variants, "http://foo.com/testOutput/", [])
+        catalog_node.append(pkgNode)
+
+        if not os.path.isdir("testOutput"):
+            os.mkdir("testOutput")
+
+        cat_file = os.path.join("testOutput", 'catalog_fragment2.xml')
+        catalog_root.write(cat_file, encoding='utf-8', xml_declaration=True)
+
+        parsed = sgprops.readProps(cat_file)
+        parsedPkgNode = parsed.getChild("package")
+
+        self.assertEqual(parsedPkgNode.name, "package");
+
+        self.assertEqual(parsedPkgNode.getValue('id'), pkg['id']);
+        self.assertEqual(parsedPkgNode.getValue('dir'), 'dc3');
+        self.assertEqual(parsedPkgNode.getValue('url'), 'http://foo.com/testOutput/dc3.zip');
+      
+        self.assertEqual(parsedPkgNode.getValue('author'), 'Donald Douglas');
+
+        parsedAuthors = parsedPkgNode.getChild("authors").getChildren('author')
+        self.assertEqual(len(parsedAuthors), 1)
+        author1 = parsedAuthors[0]
+        self.assertEqual(author1.getValue("name"), "Donald Douglas")
+        self.assertEqual(author1.getValue("nick"), "dd")
+        self.assertEqual(author1.getValue("email"), "dd@douglas.com")
+
+        urls = parsedPkgNode.getChild('urls')
+        self.assertEqual(urls.getValue('home-page'), 'http://www.douglas.com')
+
+    def test_minimalAircraft(self):
+        # test an aircraft with a deliberately spartan -set.xml file with
+        # most interesting data missing
+        (pkg, variants) = catalog.scan_aircraft_dir("testData/Aircraft/c150", ["testData/OtherDir"])
+
+        catalog_node = ET.Element('PropertyList')
+        catalog_root = ET.ElementTree(catalog_node)
+
+        pkgNode = catalog.make_aircraft_node('c150', pkg, variants, "http://foo.com/testOutput/", [])
+        catalog_node.append(pkgNode)
+
+        if not os.path.isdir("testOutput2"):
+            os.mkdir("testOutput2")
+
+        cat_file = os.path.join("testOutput2", 'catalog_fragment.xml')
+        catalog_root.write(cat_file, encoding='utf-8', xml_declaration=True)
+
+        parsed = sgprops.readProps(cat_file)
+        parsedPkgNode = parsed.getChild("package")
+
+        self.assertEqual(parsedPkgNode.getValue('id'), pkg['id'])
+        self.assertEqual(parsedPkgNode.getValue('dir'), 'c150')
+        self.assertEqual(parsedPkgNode.getValue('url'), 'http://foo.com/testOutput/c150.zip')
+        self.assertFalse(parsedPkgNode.hasChild('thumbnail'))
+        self.assertFalse(parsedPkgNode.hasChild('thumbnail-path'));
+
+        self.assertEqual(parsedPkgNode.getValue('name'), pkg['name']);
+        self.assertFalse(parsedPkgNode.hasChild('description'));
+        self.assertFalse(parsedPkgNode.hasChild('author'));
+        self.assertFalse(parsedPkgNode.hasChild('minimum-fg-version'));
+        self.assertFalse(parsedPkgNode.hasChild('variant'));
+
+
+class ZipTests(unittest.TestCase):
+    """Specific craft zip file creation tests."""
+
+    def check_zip(self, file_name, expected_content=None):
+        """General checks for the zip file."""
+
+        # Check for file existence.
+        self.assert_(os.access(file_name, os.F_OK))
+
+        # Check the contents.
+        file = zipfile.ZipFile(file_name)
+        zip_contents = file.namelist()
+        if len(zip_contents) != len(expected_content):
+            print("Zip contents:\n    %s" % zip_contents)
+            print("Expected contents:\n    %s" % expected_content)
+            self.assertEqual(len(zip_contents), len(expected_content))
+        for i in range(len(zip_contents)):
+            self.assertEqual(zip_contents[i], expected_content[i])
+
+
+    def setUp(self):
+        """Common set up for these system tests."""
+
+        # Store the current directory.
+        self._cwd = os.getcwd()
+
+        # Create a temporary directory for dumping files.
+        self.tmpdir = mkdtemp()
+
+
+    def tearDown(self):
+        """Delete temp files."""
+
+        # Force return to the correct directory.
+        os.chdir(self._cwd)
+
+        # Remove temporary file (if there is a deletion failure, continue to allow the test suite to survive).
+        try:
+            rmtree(self.tmpdir)
+        except:
+            pass
+
+        # Remove the variable.
+        del self.tmpdir
+
+
+    def test_zip_creation(self):
+        """Test the creation of a basic craft zip archive."""
+
+        # Create a basic zip file.
+        name = "c172"
+        catalog.make_aircraft_zip(join(os.getcwd(), "testData/Aircraft"), name, join(self.tmpdir, name+'.zip'), join(os.getcwd(), 'fgaddon-catalog/zip-excludes.lst'), verbose=False)
+
+        # Checks.
+        self.check_zip(join(self.tmpdir, name+'.zip'), expected_content=['c172/c172-set.xml'])
+
+
+    def test_zip_exclusion_global(self):
+        """Test file exclusion in a craft zip archive using the global catalog exclusion list."""
+
+        # Create a basic zip file.
+        name = "dc3"
+        catalog.make_aircraft_zip(join(os.getcwd(), "testData/Aircraft"), name, join(self.tmpdir, name+'.zip'), join(os.getcwd(), 'fgaddon-catalog/zip-excludes.lst'), verbose=False)
+
+        # Checks.
+        self.check_zip(join(self.tmpdir, name+'.zip'), expected_content=['dc3/dc3-set.xml'])
+
+
+    def test_zip_exclusion_local(self):
+        """Test file exclusion in a craft zip archive using a local catalog exclusion list."""
+
+        # Create a basic zip file.
+        name = "c150"
+        catalog.make_aircraft_zip(join(os.getcwd(), "testData/Aircraft"), name, join(self.tmpdir, name+'.zip'), join(os.getcwd(), 'testData/Aircraft/c150/zip-excludes.lst'), verbose=False)
+
+        # Checks.
+        self.check_zip(join(self.tmpdir, name+'.zip'), expected_content=['c150/c150-set.xml', 'c150/Resources/crazy_20Gb_file'])
+
+
+if __name__ == '__main__':
+    unittest.main()

catalog/update-catalog.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#! /usr/bin/env python3
 
 import argparse
 import datetime
@@ -12,9 +12,15 @@ import time
 import sgprops
 import sys
 import catalogTags
+import catalog
+from catalog import make_aircraft_node, make_aircraft_zip, parse_config_file, parse_template_file
+
 
 CATALOG_VERSION = 4
 
+# The Python version.
+PY_VERSION = sys.version_info[0]
+
 parser = argparse.ArgumentParser()
 parser.add_argument("--update", help="Update/pull SCM source",
                     action="store_true")
@@ -23,10 +29,13 @@ parser.add_argument("--no-update",
                     action="store_true")
 parser.add_argument("--clean", help="Force regeneration of all zip files",
                     action="store_true")
+parser.add_argument("--quiet", help="Only print warnings and errors",
+                    action="store_true")
 parser.add_argument("dir", help="Catalog directory")
 args = parser.parse_args()
 
 includes = []
+mirrors = [] # mirror base URLs
 
 # xml node (robust) get text helper
 def get_xml_text(e):
@@ -35,145 +44,15 @@ def get_xml_text(e):
     else:
         return ''
 
-# create an xml node with text content
-def make_xml_leaf(name, text):
-    leaf = ET.Element(name)
-    if text != None:
-        if isinstance(text, (int, long)):
-            leaf.text = str(text)
-        else:
-            leaf.text = text
-    else:
-        leaf.text = ''
-    return leaf
-
-# return all available aircraft information from the set file as a
-# dict
-def scan_set_file(aircraft_dir, set_file):
-    global includes
-
-    base_file = os.path.basename(set_file)
-    base_id = base_file[:-8]
-    set_path = os.path.join(aircraft_dir, set_file)
-
-    local_includes = includes
-    local_includes.append(aircraft_dir)
-    root_node = sgprops.readProps(set_path, includePaths = local_includes)
-
-    if not root_node.hasChild("sim"):
-        return None
-
-    sim_node = root_node.getChild("sim")
-    if sim_node == None:
-        return None
-
-    variant = {}
-    variant['name'] = sim_node.getValue("description", None)
-    variant['status'] = sim_node.getValue("status", None)
-    variant['author'] = sim_node.getValue("author", None)
-    variant['description'] = sim_node.getValue("long-description", None)
-    variant['id'] = base_id
-
-    # allow -set.xml files to declare themselves as primary.
-    # we use this avoid needing a variant-of in every other -set.xml
-    variant['primary-set'] = sim_node.getValue('primary-set', False)
-
-    # extract and record previews for each variant
-    if sim_node.hasChild('previews'):
-        print "has previews ..."
-        variant['previews'] = extract_previews(sim_node.getChild('previews'), aircraft_dir)
-
-    if sim_node.hasChild('rating'):
-        rating_node = sim_node.getChild("rating")
-        variant['rating_FDM'] = rating_node.getValue("FDM", 0)
-        variant['rating_systems'] = rating_node.getValue("systems", 0)
-        variant['rating_cockpit'] = rating_node.getValue("cockpit", 0)
-        variant['rating_model'] = rating_node.getValue("model", 0)
-
-    if sim_node.hasChild('tags'):
-        variant['tags'] = extract_tags(sim_node.getChild('tags'), set_file)
-
-    variant['variant-of'] = sim_node.getValue("variant-of", None)
-    #print '    ', variant
-    return variant
-
-def extract_previews(previews_node, aircraft_dir):
-    result = []
-    for node in previews_node.getChildren("preview"):
-        previewType = node.getValue("type", None)
-        previewPath = node.getValue("path", None)
-
-        # check path exists in base-name-dir
-        fullPath = os.path.join(aircraft_dir, previewPath)
-        if not os.path.isfile(fullPath):
-            print "Bad preview path, skipping:" + fullPath
-            continue
-        result.append({'type':previewType, 'path':previewPath})
-
-    return result
-
-def extract_tags(tags_node, set_path):
-    result = []
-    for node in tags_node.getChildren("tag"):
-        tag = node.value
-        # check tag is in the allowed list
-        if not catalogTags.isValidTag(tag):
-            print "Unknown tag value:", tag, " in ", set_path
-        result.append(tag)
-
-    return result
-
-# scan all the -set.xml files in an aircraft directory.  Returns a
-# package dict and a list of variants.
-def scan_aircraft_dir(aircraft_dir):
-    # old way of finding the master aircraft: it's the only one whose
-    # variant-of is empty. All the others have an actual value
-    # newer alternative is to specify one -set.xml as the primary. All the
-    # others are therefore variants.
-    setDicts = []
-    found_master = False
-    package = None
-
-    files = os.listdir(aircraft_dir)
-    for file in sorted(files, key=lambda s: s.lower()):
-        if file.endswith('-set.xml'):
-            try:
-                d = scan_set_file(aircraft_dir, file)
-                if d == None:
-                    continue
-            except:
-                print "Skipping set file since couldn't be parsed:", os.path.join(aircraft_dir, file), sys.exc_info()[0]
-                continue
-
-            setDicts.append(d)
-            if d['primary-set']:
-                found_master = True
-                package = d
-
-    # didn't find a dict identified explicitly as the primary, look for one
-    # with an undefined variant-of
-    if not found_master:
-        for d in setDicts:
-            if d['variant-of'] == '':
-                found_master = True
-                package = d
-                break
-
-    if not found_master:
-        if len(setDicts) > 1:
-            print "Warning, no explicit primary set.xml in " + aircraft_dir
-        # use the first one
-        package = setDicts[0]
-
-    # variants is just all the set dicts except the master
-    variants = setDicts
-    variants.remove(package)
-    return (package, variants)
-
 # use svn commands to report the last change date within dir
 def last_change_date_svn(dir):
     command = [ 'svn', 'info', dir ]
     result = subprocess.check_output( command )
+
+    # Python 3 compatibility.
+    if PY_VERSION == 3:
+        result = result.decode('utf8')
+
     match = re.search('Last Changed Date: (\d+)\-(\d+)\-(\d+)', result)
     if match:
         rev_str = match.group(1) + match.group(2) + match.group(3)
@@ -197,39 +76,13 @@ def scan_dir_for_change_date_mtime(path):
                 maxsec = mtime
     return maxsec
 
-def make_aircraft_zip(repo_path, name, zip_file):
-    print "Updating:", name + '.zip'
-    savedir = os.getcwd()
-    os.chdir(repo_path)
-    if os.path.exists(zip_file):
-        os.remove(zip_file)
-    command = ['zip', '-rq', '-9']
-    if os.path.exists(zip_excludes):
-        command += ['-x@' + zip_excludes]
-    else:
-        print "warning: no zip-excludes.lst file provided", zip_excludes
-    command += [zip_file, name]
-    subprocess.call(command)
-    os.chdir(savedir)
 
 def get_md5sum(file):
-    f = open(file, 'r')
+    f = open(file, 'rb')
     md5sum = hashlib.md5(f.read()).hexdigest()
     f.close()
     return md5sum
 
-def append_preview_nodes(node, variant, download_base, package_name):
-    if not 'previews' in variant:
-        return
-
-    for preview in variant['previews']:
-        preview_node = ET.Element('preview')
-        preview_url = download_base + 'previews/' + package_name + '_' + preview['path']
-        preview_node.append( make_xml_leaf('type', preview['type']) )
-        preview_node.append( make_xml_leaf('url', preview_url) )
-        preview_node.append( make_xml_leaf('path', preview['path']) )
-        node.append(preview_node)
-
 def copy_previews_for_variant(variant, package_name, package_dir, previews_dir):
     if not 'previews' in variant:
         return
@@ -237,7 +90,7 @@ def copy_previews_for_variant(variant, package_name, package_dir, previews_dir):
     for preview in variant['previews']:
         preview_src = os.path.join(package_dir, preview['path'])
         preview_dst = os.path.join(previews_dir, package_name + '_' + preview['path'])
-        #print preview_src, preview_dst, preview['path']
+        #print(preview_src, preview_dst, preview['path'])
         dir = os.path.dirname(preview_dst)
         if not os.path.isdir(dir):
             os.makedirs(dir)
@@ -249,12 +102,114 @@ def copy_previews_for_package(package, variants, package_name, package_dir, prev
     for v in variants:
         copy_previews_for_variant(v, package_name, package_dir, previews_dir)
 
-def append_tag_nodes(node, variant):
+def copy_thumbnail_for_variant(variant, package_name, package_dir, thumbnails_dir):
-    if not 'tags' in variant:
+    if not 'thumbnail' in variant:
         return
 
-    for tag in variant['tags']:
+    thumb_src = os.path.join(package_dir, variant['thumbnail'])
-        node.append(make_xml_leaf('tag', tag))
+    thumb_dst = os.path.join(thumbnails_dir, package_name + '_' + variant['thumbnail'])
+
+    dir = os.path.dirname(thumb_dst)
+    if not os.path.isdir(dir):
+        os.makedirs(dir)
+    if os.path.exists(thumb_src):
+        shutil.copy2(thumb_src, thumb_dst)
+
+def copy_thumbnails_for_package(package, variants, package_name, package_dir, thumbnails_dir):
+    copy_thumbnail_for_variant(package, package_name, package_dir, thumbnails_dir)
+
+    # and now each variant in turn
+    for v in variants:
+        copy_thumbnail_for_variant(v, package_name, package_dir, thumbnails_dir)
+
+def process_aircraft_dir(name, repo_path):
+    global includes
+    global download_base
+    global output_dir
+    global valid_zips
+    global previews_dir
+    global mirrors
+
+    aircraft_dir = os.path.join(repo_path, name)
+    if not os.path.isdir(aircraft_dir):
+        return
+
+    (package, variants) = catalog.scan_aircraft_dir(aircraft_dir, includes)
+    if package == None:
+        if not args.quiet:
+            print("skipping: %s (no -set.xml files)" % name)
+        return
+
+    if not args.quiet:
+        print("%s:" % name)
+
+    package_node = make_aircraft_node(name, package, variants, download_base, mirrors)
+
+    download_url = download_base + name + '.zip'
+    if 'thumbnail' in package:
+        # this is never even used, but breaks the script by assuming
+        # all aircraft packages have thumbnails defined?
+        thumbnail_url = download_base + 'thumbnails/' + name + '_' + package['thumbnail']
+
+    # get cached md5sum if it exists
+    md5sum = get_xml_text(md5sum_root.find(str('aircraft_' + name)))
+
+    # now do the packaging and rev number stuff
+    dir_mtime = scan_dir_for_change_date_mtime(aircraft_dir)
+    if repo_type == 'svn':
+        rev = last_change_date_svn(aircraft_dir)
+    else:
+        d = datetime.datetime.utcfromtimestamp(dir_mtime)
+        rev = d.strftime("%Y%m%d")
+    package_node.append( catalog.make_xml_leaf('revision', rev) )
+    #print("rev: %s" % rev)
+    #print("dir mtime: %s" % dir_mtime)
+    zipfile = os.path.join( output_dir, name + '.zip' )
+    valid_zips.append(name + '.zip')
+    if not os.path.exists(zipfile) \
+       or dir_mtime > os.path.getmtime(zipfile) \
+       or args.clean:
+        # rebuild zip file
+        if not args.quiet:
+            print("updating: %s" % zipfile)
+        make_aircraft_zip(repo_path, name, zipfile, zip_excludes, verbose=not args.quiet)
+        md5sum = get_md5sum(zipfile)
+    else:
+        if not args.quiet:
+            print("(no change)")
+        if md5sum == "":
+            md5sum = get_md5sum(zipfile)
+    filesize = os.path.getsize(zipfile)
+    package_node.append( catalog.make_xml_leaf('md5', md5sum) )
+    package_node.append( catalog.make_xml_leaf('file-size-bytes', filesize) )
+
+    # handle md5sum cache
+    node = md5sum_root.find('aircraft_' + name)
+    if node != None:
+        node.text = md5sum
+    else:
+        md5sum_root.append( catalog.make_xml_leaf('aircraft_' + name, md5sum) )
+
+    # handle sharing
+    if share_md5sum_root != None:
+        sharedNode = share_md5sum_root.find(str('aircraft_' + name))
+        if node != None:
+            shared_md5 = get_xml_text(sharedNode)
+            if shared_md5 == md5sum:
+                if not args.quiet:
+                    print("Sharing zip with share catalog for: %s" % name)
+                os.remove(zipfile)
+                os.symlink(os.path.join( share_output_dir, name + '.zip' ), zipfile)
+
+
+    # handle thumbnails
+    copy_thumbnails_for_package(package, variants, name, aircraft_dir, thumbnail_dir)
+
+    catalog_node.append(package_node)
+
+    # copy previews for the package and variants into the
+    # output directory
+    copy_previews_for_package(package, variants, name, aircraft_dir, previews_dir)
 
 #def get_file_stats(file):
 #    f = open(file, 'r')
@@ -263,19 +218,12 @@ def append_tag_nodes(node, variant):
 #    return (md5, file_size)
 
 if not os.path.isdir(args.dir):
-    print "A valid catalog directory must be provided"
+    print("A valid catalog directory must be provided")
     exit(0)
 
 parser = ET.XMLParser(remove_blank_text=True)
-
+config_node = parse_config_file(parser=parser, file_name=os.path.join(args.dir, 'catalog.config.xml'))
-config_file = os.path.join(args.dir, 'catalog.config.xml')
+template_node = parse_template_file(parser=parser, file_name=os.path.join(args.dir, 'template.xml'))
-config = ET.parse(config_file, parser)
-config_node = config.getroot()
-
-template_file = os.path.join(args.dir, 'template.xml')
-template = ET.parse(template_file, parser)
-template_root = template.getroot()
-template_node = template_root.find('template')
 
 md5sum_file = os.path.join(args.dir, 'md5sum.xml')
 if os.path.exists(md5sum_file):
@@ -285,9 +233,32 @@ else:
     md5sum_root = ET.Element('PropertyList')
     md5sum_tree = ET.ElementTree(md5sum_root)
 
+# share .zip files with other output dirs
+share_output_dir = get_xml_text(config_node.find('share-output'))
+share_md5_file = get_xml_text(config_node.find('share-md5-sums'))
+if share_output_dir != '' and share_md5_file != '':
+    print("Output shared with: %s" % share_output_dir)
+    share_md5sum_tree = ET.parse(share_md5_file, parser)
+    share_md5sum_root = share_md5sum_tree.getroot()
+else:
+    share_md5sum_root = None
+
+# SCM providers
 scm_list = config_node.findall('scm')
 upload_node = config_node.find('upload')
-download_base = get_xml_text(config_node.find('download-url'))
+
+download_base = None
+for i in config_node.findall("download-url"):
+    url = get_xml_text(i)
+    if not url.endswith('/'):
+        url += '/'
+
+    if download_base == None:
+        # download_base is the first entry
+        download_base = url
+    else:
+        mirrors.append(url)
+
 output_dir = get_xml_text(config_node.find('local-output'))
 if output_dir == '':
     output_dir = os.path.join(args.dir, 'output')
@@ -308,31 +279,37 @@ zip_excludes = os.path.realpath(tmp)
 for i in config_node.findall("include-dir"):
     path = get_xml_text(i)
     if not os.path.exists(path):
-        print "Skipping missing include path:", path
+        print("Skipping missing include path: %s" % path)
         continue
     includes.append(path)
 
 # freshen repositories
 if args.no_update:
-    print 'Skipping repository updates.'
+    print('Skipping repository updates.')
 else:
     cwd = os.getcwd()
     for scm in scm_list:
+        # XML mandated skip, with command line override.
+        if not args.update:
+            skip = get_xml_text(scm.find('update'))
+            if skip == 'false':
+                continue
+
         repo_type = get_xml_text(scm.find('type'))
         repo_path = get_xml_text(scm.find('path'))
         includes.append(repo_path)
 
         if repo_type == 'svn':
-            print 'SVN update:', repo_path
+            print("SVN update: %s" % repo_path)
             subprocess.call(['svn', 'update', repo_path])
         elif repo_type == 'git':
-            print 'GIT pull:', repo_path
+            print("GIT pull: %s" % repo_path)
             os.chdir(repo_path)
             subprocess.call(['git','pull'])
         elif repo_type == 'no-scm':
-            print "No update of unmannaged files:", repo_path
+            print("No update of unmannaged files: %s" % repo_path)
         else:
-            print "Unknown scm type:", scm, repo_path
+            print("Unknown scm type: %s %s" % (scm, repo_path))
     os.chdir(cwd)
 
 # names of zip files we want (so we can identify/remove orphans)
@@ -354,126 +331,39 @@ for scm in scm_list:
     skip_list = []
     for s in skip_nodes:
         skip_list.append(get_xml_text(s))
-    print 'skip list:', skip_list
+
+    # Selective list of craft to include, overriding the skip list.
+    include_nodes = scm.findall('include')
+    include_list = []
+    for node in include_nodes:
+        include_list.append(get_xml_text(node))
+    if len(include_list):
+        skip_list = []
+
+    print("Skip list: %s" % skip_list)
+    print("Include list: %s" % include_list)
+
     names = os.listdir(repo_path)
     for name in sorted(names, key=lambda s: s.lower()):
-        if name in skip_list:
+        if name in skip_list or (len(include_list) and name not in include_list):
-            print "skipping:", name
+            if not args.quiet:
+                print("Skipping: %s" % name)
             continue
 
-        aircraft_dir = os.path.join(repo_path, name)
+        # process each aircraft in turn
-        if os.path.isdir(aircraft_dir):
+        # print("%s %s" % (name, repo_path))
-            print "%s:" % name,
+        process_aircraft_dir(name, repo_path)
-            (package, variants) = scan_aircraft_dir(aircraft_dir)
-            if package == None:
-                print "skipping:", name, "(no -set.xml files)"
-                continue
-            #print "package:", package
-            #print "variants:", variants
-            package_node = ET.Element('package')
-            package_node.append( make_xml_leaf('name', package['name']) )
-            package_node.append( make_xml_leaf('status', package['status']) )
-            package_node.append( make_xml_leaf('author', package['author']) )
-            package_node.append( make_xml_leaf('description', package['description']) )
-            if 'rating_FDM' in package or 'rating_systems' in package \
-               or 'rating_cockpit' in package or 'rating_model' in package:
-                rating_node = ET.Element('rating')
-                package_node.append(rating_node)
-                rating_node.append( make_xml_leaf('FDM',
-                                                  package['rating_FDM']) )
-                rating_node.append( make_xml_leaf('systems',
-                                                  package['rating_systems']) )
-                rating_node.append( make_xml_leaf('cockpit',
-                                                  package['rating_cockpit']) )
-                rating_node.append( make_xml_leaf('model',
-                                                  package['rating_model']) )
-            package_node.append( make_xml_leaf('id', package['id']) )
-            for variant in variants:
-                variant_node = ET.Element('variant')
-                package_node.append(variant_node)
-                variant_node.append( make_xml_leaf('id', variant['id']) )
-                variant_node.append( make_xml_leaf('name', variant['name']) )
-                if 'description' in variant:
-                    variant_node.append( make_xml_leaf('description', variant['description']) )
-
-                if 'author' in variant:
-                    variant_node.append( make_xml_leaf('author', variant['author']) )
-
-                append_preview_nodes(variant_node, variant, download_base, name)
-                append_tag_nodes(variant_node, variant)
-
-            package_node.append( make_xml_leaf('dir', name) )
-            if not download_base.endswith('/'):
-                download_base += '/'
-            download_url = download_base + name + '.zip'
-            thumbnail_url = download_base + 'thumbnails/' + name + '_thumbnail.jpg'
-            package_node.append( make_xml_leaf('url', download_url) )
-            package_node.append( make_xml_leaf('thumbnail', thumbnail_url) )
-
-            append_preview_nodes(package_node, package, download_base, name)
-            append_tag_nodes(package_node, package)
-
-            # todo: url (download), thumbnail (download url)
-
-            # get cached md5sum if it exists
-            md5sum = get_xml_text(md5sum_root.find(str('aircraft_' + name)))
-
-            # now do the packaging and rev number stuff
-            dir_mtime = scan_dir_for_change_date_mtime(aircraft_dir)
-            if repo_type == 'svn':
-                rev = last_change_date_svn(aircraft_dir)
-            else:
-                d = datetime.datetime.utcfromtimestamp(dir_mtime)
-                rev = d.strftime("%Y%m%d")
-            package_node.append( make_xml_leaf('revision', rev) )
-            #print "rev:", rev
-            #print "dir mtime:", dir_mtime
-            zipfile = os.path.join( output_dir, name + '.zip' )
-            valid_zips.append(name + '.zip')
-            if not os.path.exists(zipfile) \
-               or dir_mtime > os.path.getmtime(zipfile) \
-               or args.clean:
-                # rebuild zip file
-                print "updating:", zipfile
-                make_aircraft_zip(repo_path, name, zipfile)
-                md5sum = get_md5sum(zipfile)
-            else:
-                print "(no change)"
-                if md5sum == "":
-                    md5sum = get_md5sum(zipfile)
-            filesize = os.path.getsize(zipfile)
-            package_node.append( make_xml_leaf('md5', md5sum) )
-            package_node.append( make_xml_leaf('file-size-bytes', filesize) )
-
-            # handle md5sum cache
-            node = md5sum_root.find('aircraft_' + name)
-            if node != None:
-                node.text = md5sum
-            else:
-                md5sum_root.append( make_xml_leaf('aircraft_' + name, md5sum) )
-
-            # handle thumbnails
-            thumbnail_src = os.path.join(aircraft_dir, 'thumbnail.jpg')
-            thumbnail_dst = os.path.join(thumbnail_dir, name + '_thumbnail.jpg')
-            if os.path.exists(thumbnail_src):
-                shutil.copy2(thumbnail_src, thumbnail_dst)
-            catalog_node.append(package_node)
-            package_node.append( make_xml_leaf('thumbnail-path', 'thumbnail.jpg') )
-
-            # copy previews for the package and variants into the
-            # output directory
-            copy_previews_for_package(package, variants, name, aircraft_dir, previews_dir)
 
 # write out the master catalog file
 cat_file = os.path.join(output_dir, 'catalog.xml')
 catalog_root.write(cat_file, encoding='utf-8', xml_declaration=True, pretty_print=True)
 
 # write out the md5sum cache file
-print md5sum_file
+print(md5sum_file)
 md5sum_tree.write(md5sum_file, encoding='utf-8', xml_declaration=True, pretty_print=True)
 
 # look for orphaned zip files
 files = os.listdir(output_dir)
 for file in files:
     if file.endswith('.zip')and not file in valid_zips:
-        print "orphaned zip:", file
+        print("orphaned zip: %s" % file)

fg-from-scratch/.gitignore

@@ -0,0 +1,2 @@
+scratch-*/
+vcpkg-git/

fg-from-scratch/COPYING

@@ -0,0 +1,481 @@
+                  GNU LIBRARY GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
+
+ Copyright (C) 1991 Free Software Foundation, Inc.
+ 51 Franklin St, Fifth Floor, Boston, MA  02110-1301, USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+[This is the first released version of the library GPL.  It is
+ numbered 2 because it goes with version 2 of the ordinary GPL.]
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+Licenses are intended to guarantee your freedom to share and change
+free software--to make sure the software is free for all its users.
+
+  This license, the Library General Public License, applies to some
+specially designated Free Software Foundation software, and to any
+other libraries whose authors decide to use it.  You can use it for
+your libraries, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if
+you distribute copies of the library, or if you modify it.
+
+  For example, if you distribute copies of the library, whether gratis
+or for a fee, you must give the recipients all the rights that we gave
+you.  You must make sure that they, too, receive or can get the source
+code.  If you link a program with the library, you must provide
+complete object files to the recipients so that they can relink them
+with the library, after making changes to the library and recompiling
+it.  And you must show them these terms so they know their rights.
+
+  Our method of protecting your rights has two steps: (1) copyright
+the library, and (2) offer you this license which gives you legal
+permission to copy, distribute and/or modify the library.
+
+  Also, for each distributor's protection, we want to make certain
+that everyone understands that there is no warranty for this free
+library.  If the library is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original
+version, so that any problems introduced by others will not reflect on
+the original authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that companies distributing free
+software will individually obtain patent licenses, thus in effect
+transforming the program into proprietary software.  To prevent this,
+we have made it clear that any patent must be licensed for everyone's
+free use or not licensed at all.
+
+  Most GNU software, including some libraries, is covered by the ordinary
+GNU General Public License, which was designed for utility programs.  This
+license, the GNU Library General Public License, applies to certain
+designated libraries.  This license is quite different from the ordinary
+one; be sure to read it in full, and don't assume that anything in it is
+the same as in the ordinary license.
+
+  The reason we have a separate public license for some libraries is that
+they blur the distinction we usually make between modifying or adding to a
+program and simply using it.  Linking a program with a library, without
+changing the library, is in some sense simply using the library, and is
+analogous to running a utility program or application program.  However, in
+a textual and legal sense, the linked executable is a combined work, a
+derivative of the original library, and the ordinary General Public License
+treats it as such.
+
+  Because of this blurred distinction, using the ordinary General
+Public License for libraries did not effectively promote software
+sharing, because most developers did not use the libraries.  We
+concluded that weaker conditions might promote sharing better.
+
+  However, unrestricted linking of non-free programs would deprive the
+users of those programs of all benefit from the free status of the
+libraries themselves.  This Library General Public License is intended to
+permit developers of non-free programs to use free libraries, while
+preserving your freedom as a user of such programs to change the free
+libraries that are incorporated in them.  (We have not seen how to achieve
+this as regards changes in header files, but we have achieved it as regards
+changes in the actual functions of the Library.)  The hope is that this
+will lead to faster development of free libraries.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.  Pay close attention to the difference between a
+"work based on the library" and a "work that uses the library".  The
+former contains code derived from the library, while the latter only
+works together with the library.
+
+  Note that it is possible for a library to be covered by the ordinary
+General Public License rather than by this special one.
+
+                  GNU LIBRARY GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License Agreement applies to any software library which
+contains a notice placed by the copyright holder or other authorized
+party saying it may be distributed under the terms of this Library
+General Public License (also called "this License").  Each licensee is
+addressed as "you".
+
+  A "library" means a collection of software functions and/or data
+prepared so as to be conveniently linked with application programs
+(which use some of those functions and data) to form executables.
+
+  The "Library", below, refers to any such software library or work
+which has been distributed under these terms.  A "work based on the
+Library" means either the Library or any derivative work under
+copyright law: that is to say, a work containing the Library or a
+portion of it, either verbatim or with modifications and/or translated
+straightforwardly into another language.  (Hereinafter, translation is
+included without limitation in the term "modification".)
+
+  "Source code" for a work means the preferred form of the work for
+making modifications to it.  For a library, complete source code means
+all the source code for all modules it contains, plus any associated
+interface definition files, plus the scripts used to control compilation
+and installation of the library.
+
+  Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running a program using the Library is not restricted, and output from
+such a program is covered only if its contents constitute a work based
+on the Library (independent of the use of the Library in a tool for
+writing it).  Whether that is true depends on what the Library does
+and what the program that uses the Library does.
+  
+  1. You may copy and distribute verbatim copies of the Library's
+complete source code as you receive it, in any medium, provided that
+you conspicuously and appropriately publish on each copy an
+appropriate copyright notice and disclaimer of warranty; keep intact
+all the notices that refer to this License and to the absence of any
+warranty; and distribute a copy of this License along with the
+Library.
+
+  You may charge a fee for the physical act of transferring a copy,
+and you may at your option offer warranty protection in exchange for a
+fee.
+
+  2. You may modify your copy or copies of the Library or any portion
+of it, thus forming a work based on the Library, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) The modified work must itself be a software library.
+
+    b) You must cause the files modified to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    c) You must cause the whole of the work to be licensed at no
+    charge to all third parties under the terms of this License.
+
+    d) If a facility in the modified Library refers to a function or a
+    table of data to be supplied by an application program that uses
+    the facility, other than as an argument passed when the facility
+    is invoked, then you must make a good faith effort to ensure that,
+    in the event an application does not supply such function or
+    table, the facility still operates, and performs whatever part of
+    its purpose remains meaningful.
+
+    (For example, a function in a library to compute square roots has
+    a purpose that is entirely well-defined independent of the
+    application.  Therefore, Subsection 2d requires that any
+    application-supplied function or table used by this function must
+    be optional: if the application does not supply it, the square
+    root function must still compute square roots.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Library,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Library, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote
+it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Library.
+
+In addition, mere aggregation of another work not based on the Library
+with the Library (or with a work based on the Library) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may opt to apply the terms of the ordinary GNU General Public
+License instead of this License to a given copy of the Library.  To do
+this, you must alter all the notices that refer to this License, so
+that they refer to the ordinary GNU General Public License, version 2,
+instead of to this License.  (If a newer version than version 2 of the
+ordinary GNU General Public License has appeared, then you can specify
+that version instead if you wish.)  Do not make any other change in
+these notices.
+
+  Once this change is made in a given copy, it is irreversible for
+that copy, so the ordinary GNU General Public License applies to all
+subsequent copies and derivative works made from that copy.
+
+  This option is useful when you wish to copy part of the code of
+the Library into a program that is not a library.
+
+  4. You may copy and distribute the Library (or a portion or
+derivative of it, under Section 2) in object code or executable form
+under the terms of Sections 1 and 2 above provided that you accompany
+it with the complete corresponding machine-readable source code, which
+must be distributed under the terms of Sections 1 and 2 above on a
+medium customarily used for software interchange.
+
+  If distribution of object code is made by offering access to copy
+from a designated place, then offering equivalent access to copy the
+source code from the same place satisfies the requirement to
+distribute the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  5. A program that contains no derivative of any portion of the
+Library, but is designed to work with the Library by being compiled or
+linked with it, is called a "work that uses the Library".  Such a
+work, in isolation, is not a derivative work of the Library, and
+therefore falls outside the scope of this License.
+
+  However, linking a "work that uses the Library" with the Library
+creates an executable that is a derivative of the Library (because it
+contains portions of the Library), rather than a "work that uses the
+library".  The executable is therefore covered by this License.
+Section 6 states terms for distribution of such executables.
+
+  When a "work that uses the Library" uses material from a header file
+that is part of the Library, the object code for the work may be a
+derivative work of the Library even though the source code is not.
+Whether this is true is especially significant if the work can be
+linked without the Library, or if the work is itself a library.  The
+threshold for this to be true is not precisely defined by law.
+
+  If such an object file uses only numerical parameters, data
+structure layouts and accessors, and small macros and small inline
+functions (ten lines or less in length), then the use of the object
+file is unrestricted, regardless of whether it is legally a derivative
+work.  (Executables containing this object code plus portions of the
+Library will still fall under Section 6.)
+
+  Otherwise, if the work is a derivative of the Library, you may
+distribute the object code for the work under the terms of Section 6.
+Any executables containing that work also fall under Section 6,
+whether or not they are linked directly with the Library itself.
+
+  6. As an exception to the Sections above, you may also compile or
+link a "work that uses the Library" with the Library to produce a
+work containing portions of the Library, and distribute that work
+under terms of your choice, provided that the terms permit
+modification of the work for the customer's own use and reverse
+engineering for debugging such modifications.
+
+  You must give prominent notice with each copy of the work that the
+Library is used in it and that the Library and its use are covered by
+this License.  You must supply a copy of this License.  If the work
+during execution displays copyright notices, you must include the
+copyright notice for the Library among them, as well as a reference
+directing the user to the copy of this License.  Also, you must do one
+of these things:
+
+    a) Accompany the work with the complete corresponding
+    machine-readable source code for the Library including whatever
+    changes were used in the work (which must be distributed under
+    Sections 1 and 2 above); and, if the work is an executable linked
+    with the Library, with the complete machine-readable "work that
+    uses the Library", as object code and/or source code, so that the
+    user can modify the Library and then relink to produce a modified
+    executable containing the modified Library.  (It is understood
+    that the user who changes the contents of definitions files in the
+    Library will not necessarily be able to recompile the application
+    to use the modified definitions.)
+
+    b) Accompany the work with a written offer, valid for at
+    least three years, to give the same user the materials
+    specified in Subsection 6a, above, for a charge no more
+    than the cost of performing this distribution.
+
+    c) If distribution of the work is made by offering access to copy
+    from a designated place, offer equivalent access to copy the above
+    specified materials from the same place.
+
+    d) Verify that the user has already received a copy of these
+    materials or that you have already sent this user a copy.
+
+  For an executable, the required form of the "work that uses the
+Library" must include any data and utility programs needed for
+reproducing the executable from it.  However, as a special exception,
+the source code distributed need not include anything that is normally
+distributed (in either source or binary form) with the major
+components (compiler, kernel, and so on) of the operating system on
+which the executable runs, unless that component itself accompanies
+the executable.
+
+  It may happen that this requirement contradicts the license
+restrictions of other proprietary libraries that do not normally
+accompany the operating system.  Such a contradiction means you cannot
+use both them and the Library together in an executable that you
+distribute.
+
+  7. You may place library facilities that are a work based on the
+Library side-by-side in a single library together with other library
+facilities not covered by this License, and distribute such a combined
+library, provided that the separate distribution of the work based on
+the Library and of the other library facilities is otherwise
+permitted, and provided that you do these two things:
+
+    a) Accompany the combined library with a copy of the same work
+    based on the Library, uncombined with any other library
+    facilities.  This must be distributed under the terms of the
+    Sections above.
+
+    b) Give prominent notice with the combined library of the fact
+    that part of it is a work based on the Library, and explaining
+    where to find the accompanying uncombined form of the same work.
+
+  8. You may not copy, modify, sublicense, link with, or distribute
+the Library except as expressly provided under this License.  Any
+attempt otherwise to copy, modify, sublicense, link with, or
+distribute the Library is void, and will automatically terminate your
+rights under this License.  However, parties who have received copies,
+or rights, from you under this License will not have their licenses
+terminated so long as such parties remain in full compliance.
+
+  9. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Library or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Library (or any work based on the
+Library), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Library or works based on it.
+
+  10. Each time you redistribute the Library (or any work based on the
+Library), the recipient automatically receives a license from the
+original licensor to copy, distribute, link with or modify the Library
+subject to these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  11. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Library at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Library by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Library.
+
+If any portion of this section is held invalid or unenforceable under any
+particular circumstance, the balance of the section is intended to apply,
+and the section as a whole is intended to apply in other circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  12. If the distribution and/or use of the Library is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Library under this License may add
+an explicit geographical distribution limitation excluding those countries,
+so that distribution is permitted only in or among countries not thus
+excluded.  In such case, this License incorporates the limitation as if
+written in the body of this License.
+
+  13. The Free Software Foundation may publish revised and/or new
+versions of the Library General Public License from time to time.
+Such new versions will be similar in spirit to the present version,
+but may differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Library
+specifies a version number of this License which applies to it and
+"any later version", you have the option of following the terms and
+conditions either of that version or of any later version published by
+the Free Software Foundation.  If the Library does not specify a
+license version number, you may choose any version ever published by
+the Free Software Foundation.
+
+  14. If you wish to incorporate parts of the Library into other free
+programs whose distribution conditions are incompatible with these,
+write to the author to ask for permission.  For software which is
+copyrighted by the Free Software Foundation, write to the Free
+Software Foundation; we sometimes make exceptions for this.  Our
+decision will be guided by the two goals of preserving the free status
+of all derivatives of our free software and of promoting the sharing
+and reuse of software generally.
+
+                            NO WARRANTY
+
+  15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
+WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
+EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
+OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
+KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
+LIBRARY IS WITH YOU.  SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
+THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
+WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
+FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
+CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
+LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
+RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
+SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+           How to Apply These Terms to Your New Libraries
+
+  If you develop a new library, and you want it to be of the greatest
+possible use to the public, we recommend making it free software that
+everyone can redistribute and change.  You can do so by permitting
+redistribution under these terms (or, alternatively, under the terms of the
+ordinary General Public License).
+
+  To apply these terms, attach the following notices to the library.  It is
+safest to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least the
+"copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the library's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Library General Public
+    License as published by the Free Software Foundation; either
+    version 2 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Library General Public License for more details.
+
+    You should have received a copy of the GNU Library General Public
+    License along with this library; if not, write to the Free
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301, USA
+
+Also add information on how to contact you by electronic and paper mail.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the library, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the
+  library `Frob' (a library for tweaking knobs) written by James Random Hacker.
+
+  <signature of Ty Coon>, 1 April 1990
+  Ty Coon, President of Vice
+
+That's all there is to it!

fg-from-scratch/CustomSceneryProjects/Test/README.md

@@ -0,0 +1,56 @@
+## Test Scenery Project
+An example to demostrate the usage of the TerraGear Scenery tools.  By studying this example, you should gain a better understanding of the tool workflow.
+Copyright (C) 2018  Scott Giese (xDraconian) scttgs0@gmail.com
+
+### Purpose:
+Most users could benefit from the availability of a reference example to gain a working understanding of scenery generation.  I am hopeful you gain value from this example and i would like to see this lead toward encouraging more Windows users to get involved in contributing to scenery development.
+
+### Supported Platform:
+Windows 10
+
+### Prerequisites:
+	SimGear
+    TerraGear
+
+### Installation Instructions:
+
+None
+
+### Usage:
+Below is an explaination of the files provided:
+
+| File | Description |
+|-----:|------------|
+|**generate.cmd** | A command script containing calls to the various TerraGear tools to produce scenery for Oahu |
+|**test.apt.dat** | Contains all airport data for those located on Oahu |
+|**/data** | Folder containing files you provide to the TerraGear tools |
+|**/data/SRTM-3** | Terrain data for Oahu |
+|**/data/fg_* ** | Shapefiles.  These define the type of terrain (grass, lake, forest, etc.) - Items typically defined by a polygon area |
+|**/data/osm_* ** | OpenStreetMap files.  These represent roadways, rivers, etc. - Items typically defined by "lines" |
+| **/work** | Files produced by the TerraGear tools |
+| **/output** | Final resulting files.  These are the scenery files that will be loaded into FlightGear |
+
+Run the command script interactively.  No log is produced.
+
+	generate.cmd
+
+Runs the command script and routes STDOUT and STDERR to a log file. You will miss the prompts, so it is important that you monitor the log while it is running.
+
+	generate.cmd > generate.log 2>&1
+
+Options for monitoring the log file while the script is running:
+Start the script and then
+- Load the log file into Notepad++ (https://notepad-plus-plus.org/).
+ - Turn on feature "Monitor" via Notepad++
+- **[Preferred]** Load the log file into WinTail (http://www.baremetalsoft.com/wintail/)
+
+### Testing the Scenery:
+
+Configure fgfs or the Flightgear Launcher to look in **/CustomSeneryProjects/Test/output** for additional scenery files.
+Choose as a starting location from any of the airports present on Oahu (e.g. PHNL or PHNG.)
+
+Note: You will find that all the buildings, radio towers, smoke stacks, etc. are missing.  This is expected since your *output* folder doesn't contain any of these items.
+
+### Known Issues:
+
+There is a bug in the toolchain that prevents the airport BTG files from being copied to the */output* folder. You will need to copy these files manually until the issue is resolved. Copy the files you find in the */work/AirportObj* folder to your */output* folder.

fg-from-scratch/CustomSceneryProjects/Test/data/fg_Airport/fg_Airport.prj

@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]

fg-from-scratch/CustomSceneryProjects/Test/data/fg_Asphalt/fg_Asphalt.prj

@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]

fg-from-scratch/CustomSceneryProjects/Test/data/fg_Canal/fg_Canal.prj

@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]

fg-from-scratch/CustomSceneryProjects/Test/data/fg_Cemetery/fg_Cemetery.prj

@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]

fg-from-scratch/CustomSceneryProjects/Test/data/fg_Default/fg_Default.prj

@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]

fg-from-scratch/CustomSceneryProjects/Test/data/fg_Default/fg_Default.qpj

@@ -0,0 +1 @@
+GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]

fg-from-scratch/CustomSceneryProjects/Test/data/fg_Dirt/fg_Dirt.prj

@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]

fg-from-scratch/CustomSceneryProjects/Test/data/fg_Dirt/fg_Dirt.qpj

@@ -0,0 +1 @@
+GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]

fg-from-scratch/CustomSceneryProjects/Test/data/fg_Freeway/fg_Freeway.prj

@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]

fg-from-scratch/CustomSceneryProjects/Test/data/fg_GolfCourse/fg_GolfCourse.prj

@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]

fg-from-scratch/CustomSceneryProjects/Test/data/fg_Grassland/fg_Grassland.prj

@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]

fg-from-scratch/CustomSceneryProjects/Test/data/fg_Grassland/fg_Grassland.qpj

@@ -0,0 +1 @@
+GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]