Fix Mac packaging.

Aircraft for 3.4.1 package are at /home/jenkins/fgdata_340 now.

.gitignore

@@ -20,3 +20,7 @@ CMakeFiles
 archivebuild
 osgbuild
 CMakeCache.txt
+aircraft-data
+base_package
+extended-data
+

.gitmodules

@@ -1,16 +1,16 @@
 [submodule "simgear"]
 	path = simgear
-	url = git://gitorious.org/fg/simgear.git
-	branch = next
+	url = git://git.code.sf.net/p/flightgear/simgear
+	branch = release/3.4.0
 [submodule "flightgear"]
 	path = flightgear
-	url = git://gitorious.org/fg/flightgear.git
-	branch = next
+	url = git://git.code.sf.net/p/flightgear/flightgear
+	branch = release/3.4.0
 [submodule "fgrun"]
 	path = fgrun
-	url = git://gitorious.org/fg/fgrun.git
-	branch = next
-[submodule "maclauncher"]
-	path = maclauncher
-	url = git://gitorious.org/fg/maclauncher.git
-	branch = master
+	url = git://git.code.sf.net/p/flightgear/fgrun
+	branch = release/3.4.0
+[submodule "fgdata"]
+	path = fgdata
+	url = git://git.code.sf.net/p/flightgear/fgdata
+	branch = release/3.4.0

CMakeLists.txt

@@ -84,6 +84,14 @@ elseif(MSVC)
         -DACTUAL_3RDPARTY_DIR:PATH=${CMAKE_BINARY_DIR}/${RDPARTY_DIR}
         -DCMAKE_LIBRARY_PATH:STRING=${CMAKE_BINARY_DIR}/${RDPARTY_DIR}/lib
         -DCMAKE_INCLUDE_PATH:STRING=${CMAKE_BINARY_DIR}/${RDPARTY_DIR}/include;${CMAKE_BINARY_DIR}/${RDPARTY_DIR}/include/freetype
+        -DCURL_INCLUDE_DIR:PATH=${CMAKE_BINARY_DIR}/${RDPARTY_DIR}/include
+        -DTIFF_INCLUDE_DIR:PATH=${CMAKE_BINARY_DIR}/${RDPARTY_DIR}/include
+        -DGDAL_INCLUDE_DIR:PATH=${CMAKE_BINARY_DIR}/${RDPARTY_DIR}/include
+        -DGDAL_LIBRARY:FILEPATH=${CMAKE_BINARY_DIR}/${RDPARTY_DIR}/lib/gdal_i.lib
+        -DTIFF_LIBRARY:FILEPATH=${CMAKE_BINARY_DIR}/${RDPARTY_DIR}/lib/libtiff.lib
+        -DCURL_LIBRARY:FILEPATH=${CMAKE_BINARY_DIR}/${RDPARTY_DIR}/lib/libcurl_imp.lib
+        -DFREETYPE_LIBRARY:FILEPATH=${CMAKE_BINARY_DIR}/${RDPARTY_DIR}/lib/freetype.lib
+        -DFREETYPE_INCLUDE_DIR:PATH=${CMAKE_BINARY_DIR}/${RDPARTY_DIR}/include;${CMAKE_BINARY_DIR}/${RDPARTY_DIR}/include/freetype
     )
 
     # for compatability with MSVC directory layout

FlightGear.iss

@@ -43,15 +43,16 @@ AppPublisherURL=http://www.flightgear.org
 AppSupportURL=http://www.flightgear.org
 AppUpdatesURL=http://www.flightgear.org
 DefaultDirName={pf}\FlightGear {#FGVersion}
+UsePreviousAppDir=no
 DefaultGroupName=FlightGear {#FGVersion}
 LicenseFile=X:\flightgear\COPYING
 Uninstallable=yes
 SetupIconFile=X:\flightgear\package\flightgear.ico
 VersionInfoVersion={#FGVersion}.0
-InfoBeforeFile=X:\flightgear\package\Win32-Inno\infobefore.txt
-WizardImageFile=X:\flightgear\package\Win32-Inno\setupimg.bmp
+InfoBeforeFile=X:\flightgear\package\windows\infobefore.txt
+WizardImageFile=X:\flightgear\package\windows\setupimg.bmp
 WizardImageStretch=No
-WizardSmallImageFile=X:\flightgear\package\Win32-Inno\setupsmall.bmp
+WizardSmallImageFile=X:\flightgear\package\windows\setupsmall.bmp
 VersionInfoCompany=The FlightGear Team
 UninstallDisplayIcon={app}\bin\fgfs.exe
 ArchitecturesInstallIn64BitMode=x64
@@ -82,6 +83,10 @@ Source: "X:\install\msvc100\FGRun\bin\fgrun.exe"; DestDir: "{app}\bin"; Flags: i
 Source: "X:\install\msvc100\FGRun\share\locale\*"; DestDir: "{app}\bin\locale"; Flags: ignoreversion recursesubdirs; Check: not Is64BitInstallMode
 Source: "X:\3rdParty\bin\zlib.dll"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
 Source: "X:\3rdParty\bin\OpenAL32.dll"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
+Source: "X:\3rdParty\bin\libpng16.dll"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
+Source: "X:\3rdParty\bin\CrashRpt1402.dll"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
+Source: "X:\3rdParty\bin\crashrpt_lang.ini"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
+Source: "X:\3rdParty\bin\CrashSender1402.exe"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
 Source: "{#VCInstallDir}\redist\x86\Microsoft.VC100.CRT\*.dll"; DestDir:  "{app}\bin"; Check: not Is64BitInstallMode
 Source: "X:\3rdParty\bin\vcredist_x86.exe"; DestDir: "{app}\bin"; Flags: skipifsourcedoesntexist; Check: not Is64BitInstallMode
 
@@ -102,13 +107,15 @@ Source: "X:\install\msvc100-64\FGRun\share\locale\*"; DestDir: "{app}\bin\locale
 Source: "X:\3rdParty.x64\bin\zlib.dll"; DestDir: "{app}\bin"; Check: Is64BitInstallMode
 Source: "X:\3rdParty.x64\bin\OpenAL32.dll"; DestDir: "{app}\bin"; Check: Is64BitInstallMode
 Source: "X:\3rdParty.x64\bin\vcredist_x64.exe"; DestDir: "{app}\bin"; Flags: skipifsourcedoesntexist; Check: Is64BitInstallMode
+Source: "X:\3rdParty.x64\bin\libpng16.dll"; DestDir: "{app}\bin"; Check: Is64BitInstallMode
+Source: "X:\3rdParty.x64\bin\CrashRpt1402.dll"; DestDir: "{app}\bin"; Check: Is64BitInstallMode
+Source: "X:\3rdParty.x64\bin\crashrpt_lang.ini"; DestDir: "{app}\bin"; Check: Is64BitInstallMode
+Source: "X:\3rdParty.x64\bin\CrashSender1402.exe"; DestDir: "{app}\bin"; Check: Is64BitInstallMode
 Source: "{#VCInstallDir}\redist\x64\Microsoft.VC100.CRT\*.dll"; DestDir: "{app}\bin"; Check: Is64BitInstallMode
 
 ; 32/64 bits install
 ;NOTE: FGPanel has no 64 bits equivalent, so we are using the 32 bits binary for 32&64 bits OS
 Source: "X:\install\msvc100\FlightGear\bin\fgpanel.exe"; DestDir: "{app}\bin"; Flags: ignoreversion
-Source: "X:\install\msvc100\FlightGear\share\flightgear\positions.txt"; DestDir: "{app}\share\flightgear"
-Source: "X:\install\msvc100\FlightGear\share\flightgear\special_frequencies.txt"; DestDir: "{app}\share\flightgear"
 Source: "X:\3rdParty\bin\oalinst.exe"; DestDir: "{app}\bin"; Flags: ignoreversion skipifsourcedoesntexist
 
 ; NOTE: tar.gz file uses name 'fgdata', to avoid renaming it, look for both names.
@@ -116,6 +123,9 @@ Source: "X:\3rdParty\bin\oalinst.exe"; DestDir: "{app}\bin"; Flags: ignoreversio
 Source: "X:\data\*.*"; DestDir: "{app}\data"; Flags: ignoreversion recursesubdirs skipifsourcedoesntexist
 Source: "X:\fgdata\*.*"; DestDir: "{app}\data"; Flags: ignoreversion recursesubdirs skipifsourcedoesntexist
 
+; work-around for 3.4.1: look for additional aircraft data
+Source: "X:\extended-data\*.*"; DestDir: "{app}\data"; Flags: ignoreversion recursesubdirs skipifsourcedoesntexist
+
 ; 32 bits install
 Source: "{#OSGInstallDir}\bin\osg{#OSGSoNumber}-osg.dll"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
 Source: "{#OSGInstallDir}\bin\osg{#OSGSoNumber}-osgDB.dll"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
@@ -216,7 +226,7 @@ Name: "{group}\Tools\metar"; Filename: "cmd"; Parameters: "/k ""{app}\bin\metar.
 Name: "{group}\Tools\yasim"; Filename: "cmd"; Parameters: "/k ""{app}\bin\yasim.exe"" -h"; WorkingDir: "{app}\bin"
 Name: "{group}\Tools\fgpanel"; Filename: "cmd"; Parameters: "/k ""{app}\bin\fgpanel.exe"" -h"; WorkingDir: "{app}\bin"
 Name: "{group}\Tools\FGCom"; Filename: "{app}\bin\fgcom.exe"; WorkingDir: "{app}\bin"
-Name: "{group}\Tools\FGCom-testing"; Filename: "{app}\bin\fgcom.exe"; Parameters: "-f910"; WorkingDir: "{app}\bin"
+Name: "{group}\Tools\FGCom-testing"; Filename: "{app}\bin\fgcom.exe"; Parameters: "--frequency=910"; WorkingDir: "{app}\bin"
 Name: "{group}\Tools\Explore Documentation Folder"; Filename: "{app}\data\Docs"
 
 [Run]
@@ -231,6 +241,20 @@ filename: "{app}\bin\fgrun.exe"; WorkingDir: "{app}\bin"; Parameters: "--silent
 filename: "{app}\bin\fgadmin.exe"; WorkingDir: "{app}\bin"; Parameters: "--silent ""--install-source={src}\..\Scenery"" ""--scenery-dest={userdocs}\FlightGear\Custom Scenery"""
 
 [Code]
+const
+  NET_FW_SCOPE_ALL = 0;
+  NET_FW_IP_VERSION_ANY = 2;
+  NET_FW_ACTION_ALLOW = 1;
+  NET_FW_RULE_DIR_ALL = 0;
+  NET_FW_RULE_DIR_IN = 1;
+  NET_FW_RULE_DIR_OUT = 2;
+  NET_FW_IP_PROTOCOL_ALL = 0;
+  NET_FW_IP_PROTOCOL_TCP = 6;
+  NET_FW_IP_PROTOCOL_UDP = 17;
+  NET_FW_PROFILE2_DOMAIN = 1;
+  NET_FW_PROFILE2_PRIVATE = 2;
+  NET_FW_PROFILE2_PUBLIC = 4;
+
 procedure URLLabelOnClick(Sender: TObject);
 var
   ErrorCode: Integer;
@@ -266,3 +290,109 @@ begin
   Result := S;
 end;
 
+procedure AddBasicFirewallException(AppName, FileName: String);
+var
+  FirewallObject: variant;
+  RuleObject: variant;
+begin
+  try
+    FirewallObject := CreateOleObject('HNetCfg.FwMgr');
+    RuleObject := CreateOleObject('HNetCfg.FwAuthorizedApplication');
+    RuleObject.ProcessImageFileName := FileName;
+    RuleObject.Name := AppName;
+    RuleObject.Scope := NET_FW_SCOPE_ALL;
+    RuleObject.IpVersion := NET_FW_IP_VERSION_ANY;
+    RuleObject.Enabled := true;
+    FirewallObject.LocalPolicy.CurrentProfile.AuthorizedApplications.Add(RuleObject);
+  except
+  end;
+end;
+
+procedure AddAdvancedFirewallException(AppName, AppDescription, FileName: String; Protocol: Integer; LocalPorts, RemotePorts: String; Direction: Integer);
+var
+  FirewallObject: variant;
+  RuleObject: variant;
+begin
+  try
+    FirewallObject := CreateOleObject('HNetCfg.FwPolicy2');
+    RuleObject := CreateOleObject('HNetCfg.FWRule');
+    RuleObject.Name := AppName;
+    RuleObject.Description := AppDescription;
+    RuleObject.ApplicationName := FileName;
+    if (Protocol <> NET_FW_IP_PROTOCOL_ALL) then
+      RuleObject.Protocol := Protocol;
+    if (LocalPorts <> '') then
+      RuleObject.LocalPorts := LocalPorts;
+    if (RemotePorts <> '') then
+      RuleObject.RemotePorts := RemotePorts;
+    if (Direction <> NET_FW_RULE_DIR_ALL) then
+      RuleObject.Direction := Direction;
+    RuleObject.Enabled := true;
+    RuleObject.Grouping := 'FlightGear';
+    RuleObject.Profiles := NET_FW_PROFILE2_DOMAIN + NET_FW_PROFILE2_PRIVATE + NET_FW_PROFILE2_PUBLIC;
+    RuleObject.Action := NET_FW_ACTION_ALLOW;
+    RuleObject.RemoteAddresses := '*';
+    FirewallObject.Rules.Add(RuleObject);
+  except
+  end;
+end;
+
+procedure RemoveFirewallException(AppName, FileName: String);
+var
+  FirewallObject: variant;
+  Version: TWindowsVersion;
+begin
+  GetWindowsVersionEx(Version);
+  try
+    if (Version.Major >= 6) then
+      begin
+        FirewallObject := CreateOleObject('HNetCfg.FwPolicy2');
+        FirewallObject.Rules.Remove(AppName);
+      end
+    else if (Version.Major = 5) and (((Version.Minor = 1) and (Version.ServicePackMajor >= 2)) or ((Version.Minor = 2) and (Version.ServicePackMajor >= 1))) then
+      begin
+        FirewallObject := CreateOleObject('HNetCfg.FwMgr');
+        FirewallObject.LocalPolicy.CurrentProfile.AuthorizedApplications.Remove(FileName);
+      end;
+  except
+  end;
+end;
+
+procedure CurStepChanged(CurStep: TSetupStep);
+var
+  Version: TWindowsVersion;
+begin
+  if CurStep = ssPostInstall then
+    begin
+      GetWindowsVersionEx(Version);
+      if (Version.Major >= 6) then
+        begin
+          { IN and OUT rules must be specified separately, otherwise the firewall will create only the IN rule }
+          AddAdvancedFirewallException('FlightGear', 'Allows FlightGear to send and receive data over the multiplayer network and to get METARs.', ExpandConstant('{app}') + '\bin\fgfs.exe', NET_FW_IP_PROTOCOL_ALL, '', '', NET_FW_RULE_DIR_IN);
+          AddAdvancedFirewallException('FlightGear', 'Allows FlightGear to send and receive data over the multiplayer network and to get METARs.', ExpandConstant('{app}') + '\bin\fgfs.exe', NET_FW_IP_PROTOCOL_ALL, '', '', NET_FW_RULE_DIR_OUT);
+          AddAdvancedFirewallException('FlightGear METAR Utility', 'Allows the FlightGear METAR utility to receive METARs.', ExpandConstant('{app}') + '\bin\metar.exe', NET_FW_IP_PROTOCOL_TCP, '', '80', NET_FW_RULE_DIR_OUT);
+          AddAdvancedFirewallException('FlightGear TerraSync', 'Allows TerraSync to download additional scenery while FlightGear is running.', ExpandConstant('{app}') + '\bin\terrasync.exe', NET_FW_IP_PROTOCOL_ALL, '', '', NET_FW_RULE_DIR_IN);
+          AddAdvancedFirewallException('FlightGear TerraSync', 'Allows TerraSync to download additional scenery while FlightGear is running.', ExpandConstant('{app}') + '\bin\terrasync.exe', NET_FW_IP_PROTOCOL_ALL, '', '', NET_FW_RULE_DIR_OUT);
+          AddAdvancedFirewallException('FlightGear FGCom', 'Allows FGCom to establish a connection to FlightGear and the VoIP server for voice ATC communication.', ExpandConstant('{app}') + '\bin\fgcom.exe', NET_FW_IP_PROTOCOL_ALL, '', '', NET_FW_RULE_DIR_IN);
+          AddAdvancedFirewallException('FlightGear FGCom', 'Allows FGCom to establish a connection to FlightGear and the VoIP server for voice ATC communication.', ExpandConstant('{app}') + '\bin\fgcom.exe', NET_FW_IP_PROTOCOL_ALL, '', '', NET_FW_RULE_DIR_OUT);
+        end
+      else if (Version.Major = 5) and (((Version.Minor = 1) and (Version.ServicePackMajor >= 2)) or ((Version.Minor = 2) and (Version.ServicePackMajor >= 1))) then
+        begin
+          { The Windows XP/Server 2003 firewall does not block outgoing connections at all, so only listening processes should be added }
+          AddBasicFirewallException('FlightGear', ExpandConstant('{app}') + '\bin\fgfs.exe');
+          AddBasicFirewallException('FlightGear TerraSync', ExpandConstant('{app}') + '\bin\terrasync.exe');
+          AddBasicFirewallException('FlightGear FGCom', ExpandConstant('{app}') + '\bin\fgcom.exe');
+        end;
+    end;
+end;
+
+procedure CurUninstallStepChanged(CurUninstallStep: TUninstallStep);
+begin
+  if CurUninstallStep = usPostUninstall then
+    begin
+      RemoveFirewallException('FlightGear', ExpandConstant('{app}') + '\bin\fgfs.exe');
+      RemoveFirewallException('FlightGear METAR Utility', ExpandConstant('{app}') + '\bin\metar.exe');
+      RemoveFirewallException('FlightGear TerraSync', ExpandConstant('{app}') + '\bin\terrasync.exe');
+      RemoveFirewallException('FlightGear FGCom', ExpandConstant('{app}') + '\bin\fgcom.exe');
+    end;
+end;

Info.plist.in

@@ -13,7 +13,7 @@
 	<key>CFBundleSignature</key>
 	<string>????</string>
 	<key>CFBundleExecutable</key>
-	<string>FlightGear</string>
+	<string>fgfs</string>
 	<key>CFBundleIdentifier</key>
 	<string>org.flightgear.FlightGear</string>
 	<key>CFBundleVersion</key>
@@ -21,13 +21,20 @@
 	<key>CFBundleShortVersionString</key>
 	<string><%= fgVersion %></string>
 	<key>LSMinimumSystemVersion</key>
-	<string>10.6.0</string>
+	<string>10.7.0</string>
 	<key>NSMainNibFile</key>
 	<string>MainMenu</string>
 	<key>NSPrincipalClass</key>
 	<string>NSApplication</string>
     <key>NSHumanReadableCopyright</key>
     <string>©1996-<%= fgCurrentYear%>, The FlightGear Project</string>
+
+    <!-- when launched via LaunchServices, run the launcher GUI -->
+    <key>LSEnvironment</key>
+    <dict>
+        <key>FG_LAUNCHER</key>
+        <string>1</string>
+    </dict>
 </dict>
 </plist>
 

aircraft.rules

@@ -0,0 +1,21 @@
++ /Aircraft
++ /Aircraft/777
++ /Aircraft/777-200
++ /Aircraft/b1900d
++ /Aircraft/CitationX
++ /Aircraft/ZLT-NT
++ /Aircraft/dhc2
++ /Aircraft/Cub
++ /Aircraft/sopwithCamel
++ /Aircraft/f-14b
++ /Aircraft/ASK13
++ /Aircraft/bo105
++ /Aircraft/Dragonfly
++ /Aircraft/SenecaII
++ /Aircraft/A6M2
+- /Aircraft/*
++ /Models
++ /Scenery
++ /AI
+- /*
+

base-package.rules

@@ -2,33 +2,6 @@
 - *.xcf
 - *.tex
 
-+ /fgdata/Aircraft/Generic
-+ /fgdata/Aircraft/Instruments
-+ /fgdata/Aircraft/Instruments-3d
-
-+ /fgdata/Aircraft/UIUC
-+ /fgdata/Aircraft/c172p
-+ /fgdata/Aircraft/777
-+ /fgdata/Aircraft/777-200
-+ /fgdata/Aircraft/b1900d
-+ /fgdata/Aircraft/ufo
-+ /fgdata/Aircraft/CitationX
-+ /fgdata/Aircraft/ZLT-NT
-+ /fgdata/Aircraft/dhc2
-+ /fgdata/Aircraft/Cub
-+ /fgdata/Aircraft/sopwithCamel
-+ /fgdata/Aircraft/f-14b
-+ /fgdata/Aircraft/ASK13
-+ /fgdata/Aircraft/bo105
-+ /fgdata/Aircraft/Dragonfly
-+ /fgdata/Aircraft/SenecaII
-+ /fgdata/Aircraft/A6M2
-
-- /fgdata/Aircraft/*
-
-- /fgdata/Textures.high/*.new
-- /fgdata/Textures.high/*.orig
-- /fgdata/Textures.high/*.save
 - /fgdata/Textures/Unused
 - /fgdata/Textures/*.orig
 - /fgdata/Docs/source

catalogTags.py

@@ -1,12 +1,12 @@
 aircraftTypeTags = [
     "ga", "fighter", "helicopter", "glider", "spaceship", "bomber", "groundvehicle", 
-    "tanker", "cargo", "passenger", "bizjet", "trainer", "airship", "balloon"
+    "tanker", "cargo", "transport", "bizjet", "trainer", "airship", "balloon"
 ]
 
 manufacturerTags = [
     "boeing", "cessna", "diamond", "douglas", "bell", "piper", 
     "airbus", "vickers", "lockheed", "fokker",
-    "embrarer", "bombardier", "pilatus"
+    "embrarer", "bombardier", "pilatus", "robin"
 ]
 
 eraTags = [
@@ -62,7 +62,9 @@ propulsionTags = [
     "supercharged",
     "turboprop",
     "jet", "afterburner", "rocket",
-    "electric"
+    "electric",
+    "twin-engine",
+    "single-engine"
 ]
 
 simFeatureTags = [

download_and_compile.sh

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-VERSION="2.26"
+VERSION="2.28"
 
 #######################################################
 # THANKS TO
@@ -132,8 +132,8 @@ function _make(){
 OSG_STABLE_GIT_BRANCH="OpenSceneGraph-3.2"
 PLIB_STABLE_GIT_BRANCH="master"
 # common stable branch for flightgear, simgear and fgdata
-FGSG_STABLE_GIT_BRANCH="release/3.2.0"
-OPENRTI_STABLE_GIT_BRANCH="release-0.5"
+FGSG_STABLE_GIT_BRANCH="release/3.4.0"
+OPENRTI_STABLE_GIT_BRANCH="release-0.6"
 # unstable branch: next for sg/fg, master for fgdata
 FGSG_UNSTABLE_GIT_BRANCH="next"
 FGDATA_UNSTABLE_GIT_BRANCH="master"
@@ -476,9 +476,6 @@ if [[ "$(declare -p WHATTOBUILD)" =~ '['([0-9]+)']="SIMGEAR"' ]]; then
 	else
 		_gitUpdate $FGSG_UNSTABLE_GIT_BRANCH
 	fi
-
-	# Remove DDS warning message
-	sed -i s/SG_ALERT,\ \"Image/SG_WARN,\ \"Image/g simgear/scene/model/ModelRegistry.cxx
 	
 	if [ ! "$UPDATE" = "UPDATE" ]; then
 		if [ "$RECONFIGURE" = "y" ]; then
@@ -527,7 +524,7 @@ if [[ "$(declare -p WHATTOBUILD)" =~ '['([0-9]+)']="FGFS"' || "$(declare -p WHAT
        				mkdir -p build/flightgear
 	                        cd "$CBD"/build/flightgear
 				rm -f CMakeCache.txt
-				cmake 	-D CMAKE_BUILD_TYPE="Debug" \
+				cmake 	-D CMAKE_BUILD_TYPE="Release" \
 					-D ENABLE_RTI=OFF \
 					-D ENABLE_FLITE=ON \
 					-D CMAKE_INSTALL_PREFIX:PATH="$INSTALL_DIR_FGFS" \

fgdata_checkers.py

@@ -8,15 +8,25 @@ import os.path
 import re
 from collections import defaultdict
 import subprocess
+import multiprocessing
 import math
 import tarfile
+import tempfile
 import gzip
+import shutil
+import time
+try:
+    devnull=subprocess.DEVNULL#hide annoying nvcompress messages
+except (AttributeError,NameError):#pre-3.3 Python
+    devnull=None
 def path_join(*args):
-    """Unlike plain os.path.join, this doesn't add a trailing / if the last component is empty"""
-    return os.path.normpath(os.path.join(*args))
+    """Unlike plain os.path.join, this always uses forward slashes, and doesn't add a trailing / if the last component is empty"""
+    return os.path.normpath(os.path.join(*args)).replace('\\','/')
 def rfilelist(path,exclude_dirs=[]):
     """Dict of files/sizes in path, including those in any subdirectories (as relative paths)"""
     files=defaultdict(int)
+    if not os.path.exists(path):
+        return files
     dirs=[""]
     while dirs:
         cdir=dirs.pop()
@@ -65,16 +75,16 @@ def files_used(pattern,path,exclude_dirs=[],filelist=None,filetypes=None,relativ
             tex=texfind.search(line)
             if tex:
                 if relative_path:
-                    textures.append(os.path.normpath(path_join(os.path.dirname(file),tex.group(1).replace('\\','/'))))
+                    textures.append(os.path.normpath(path_join(os.path.dirname(file),tex.group(1).replace('\\','/'))).replace('\\','/'))
                 else:
-                    textures.append(os.path.normpath(tex.group(1).replace('\\','/')))
+                    textures.append(os.path.normpath(tex.group(1).replace('\\','/')).replace('\\','/'))
     return textures
 def find_unused_textures(basedir,output_lists=True,grep_check=False,output_rsync_rules=False,output_comparison_strips=False, output_removal_commands=False,return_used_noregions=False):
     """Checks if any textures are unused (wasting space), and if any textures are only available as .dds (not recommended in the source repository, as it is a lossy-compressed format)
 
 Set basedir to your fg-root, and enable the kind(s) of output you want:
 output_lists prints lists of unused textures, and of dds-only textures
-grep_check checks for possible use outside the normal directories; requires Unix shell
+grep_check checks for possible use outside the normal directories; requires Unix shell and assumes side-by-side fgdata,flightgear,simgear
 output_rsync_rules prints rsync rules for excluding unused textures from the release flightgear-data.  Warning: if you use this, re-run this script regularly, in case they start being used
 output_comparison_strips creates thumbnail strips, unused_duplicate.png/unused_dds.png/high_low.png, for visually checking whether same-name textures are the same (remove the unused one entirely) or different (move it to Unused); requires imagemagick or graphicsmagick
 output_removal_commands creates another script, delete_unused_textures.sh, which will remove unused textures when run in a Unix shell"""
@@ -117,6 +127,9 @@ output_removal_commands creates another script, delete_unused_textures.sh, which
     low_unneeded_nondup=low_unneeded&set(known_highlow_mismatch)
     def image_check_strip(basedir,index_fname,ilist1,ilist2=None,size=128):
         """Generate two rows of thumbnails, for easy visual comparison (between the two lists given, or if a single list is given, between low and high resolution)"""
+        if not ilist1:
+            print(index_fname," empty, skipping")
+            return
         if ilist2 is None:
             ipairs=[[path_join(basedir,'Textures',f),path_join(basedir,'Textures.high',f)] for f in ilist1]
         else:
@@ -162,6 +175,12 @@ output_removal_commands creates another script, delete_unused_textures.sh, which
     if output_comparison_strips:
         image_check_strip(basedir,"unused_duplicate.png",unused_duplicate,["Terrain"+f[14:] for f in unused_duplicate])
         image_check_strip(basedir,"unused_dds.png",unused_dds,[f[:-4]+".png" for f in unused_dds])
+        dds_skip=set(['Runway/rwy-normalmap.dds','Water/perlin-noise-nm.dds','Water/water_sine_nmap.dds','Water/waves-ver10-nm.dds'])
+        used_dds_withpng=set(f for f in (high_textures|low_textures) if (f[-4:]==".dds" and f[:-4]+".png" in (high_textures|low_textures)))-unused_dds-dds_skip
+        print(".dds omitted from comparison strip (normal maps etc): ",dds_skip)
+        #used_dds_withpng different: p{a,c}_taxiway.dds and possibly more runway markings (darker),sand{4,5,6}.dds(less high-freq noise),water.dds(more high-freq noise),grass_rwy.dds (has stripes),cropwood.dds,irrcrop.dds,shrub.dds,herbtundra.dds,cropgrass.dds(unrelated)
+        #water-reflection.{png,dds} really are plain white (so do match), not an alpha map
+        image_check_strip(basedir,"used_dds.png",used_dds_withpng,[f[:-4]+".png" for f in used_dds_withpng])
         image_check_strip(basedir,"high_low.png",high_textures&low_textures)
         #image_check_strip(basedir,"high_low2.png",[f for f in high_textures&low_textures if (f[0:14]=="Terrain.winter" or "_taxiway." in f or "lava" in f or "sand" in f)],size=512)#closer look at the doubtful cases
     if output_lists:
@@ -171,11 +190,11 @@ output_removal_commands creates another script, delete_unused_textures.sh, which
         unused_f=[os.path.basename(f) for f in unused]
         all_f=[os.path.basename(f) for f in (high_textures|low_textures)]
         print("\n\nPossible use outside main search:")#used to set false_positives
-        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=.git","-e","("+")|(".join(unused)+")","/home/palmer/fs_dev/git/fgdata","/home/palmer/fs_dev/git/flightgear","/home/palmer/fs_dev/git/simgear"])#everywhere using full names
-        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=Textures.high","--exclude-dir=Models","--exclude-dir=Materials","--exclude-dir=Effects","--exclude-dir=.git","-e","("+")|(".join(all_f)+")","/home/palmer/fs_dev/git/fgdata","/home/palmer/fs_dev/git/flightgear","/home/palmer/fs_dev/git/simgear"])#restricted (to avoid false positives from Terrain.winter vs Terrain) using filenames
-        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=Textures.high","--exclude-dir=Models","--exclude-dir=Materials","--exclude-dir=Effects","--exclude-dir=.git","-e",'[."\']dds',"/home/palmer/fs_dev/git/fgdata","/home/palmer/fs_dev/git/flightgear","/home/palmer/fs_dev/git/simgear"])#check for programmatic .png -> .dds swap; none found
+        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=.git","-e","("+")|(".join(unused)+")",basedir,path_join(basedir,"../flightgear"),path_join(basedir,"../simgear")])#everywhere using full names
+        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=Textures.high","--exclude-dir=Models","--exclude-dir=Materials","--exclude-dir=Effects","--exclude-dir=.git","-e","("+")|(".join(all_f)+")",basedir,path_join(basedir,"../flightgear"),path_join(basedir,"../simgear")])#restricted (to avoid false positives from Terrain.winter vs Terrain) using filenames
+        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=Textures.high","--exclude-dir=Models","--exclude-dir=Materials","--exclude-dir=Effects","--exclude-dir=.git","-e",'[."\']dds',basedir,path_join(basedir,"../flightgear"),path_join(basedir,"../simgear")])#check for programmatic .png -> .dds swap; none found
         print("\n\nUse of sourceless textures:")
-        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=.git","-e","("+")|(".join(sourceless)+")","/home/palmer/fs_dev/git/fgdata","/home/palmer/fs_dev/git/flightgear","/home/palmer/fs_dev/git/simgear"])
+        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=.git","-e","("+")|(".join(sourceless)+")",basedir,path_join(basedir,"../flightgear"),path_join(basedir,"../simgear")])
     if output_rsync_rules:
         print("\n\nFull flightgear-data:\n")
         rsync_rules(basedir,unused)
@@ -240,54 +259,171 @@ def find_locally_unused_models(basedir):
     missing=set(f for f in (used_models|used_textures) if ((f.startswith('Models') and f not in models_allfiles.keys()) or not os.path.isfile(path_join(basedir,f))))
     print("used\n",sorted(used_models),"\nsize=",t_size(used_models),"\n\n",sorted(used_textures),"\nsize=",t_size(used_textures),"\n\nunused\n",sorted(unused),"\nsize=",t_size(unused),"\n\nmissing\n",sorted(missing),"\nsize=",t_size(missing))
 
-def size_by_type(path,exclude_dirs=[]):
-    """Dict of total file size by file extension"""
+class FilesetSizes:
+    def __init__(self):
+        self.count=0
+        self.ncsize=0
+        self.csize=0
+    @property
+    def size(self):
+        if self.csize>0:
+            return self.csize
+        else:
+            return self.ncsize
+    def __str__(self):
+        if self.csize>0:
+            return "{0:10,} {1:15,} {2:15,}".format(self.count,self.ncsize,self.csize)
+        else:
+            return "{0:10,} {1:15,}".format(self.count,self.ncsize)
+    def __add__(self,other):
+        result=FilesetSizes()
+        result.count=self.count+other.count
+        result.ncsize=self.ncsize+other.ncsize
+        result.csize=self.csize+other.csize
+        return result
+def size_by_category(path,exclude_dirs,keyfn,compressed_size=False):
+    """Total size of files, in each category returned from keyfn"""
     files=rfilelist(path,exclude_dirs)
-    size_totals=defaultdict(int)
-    for filename,size in files.items():
-        file_ext=os.path.splitext(filename)[1]
-        if file_ext==".gz":
-            file_ext=os.path.splitext(os.path.splitext(filename)[0])[1]+file_ext
-        size_totals[file_ext]=size_totals[file_ext]+size
-    return size_totals
-def size_by_size(path,exclude_dirs=[],exts=[".png",".dds",".rgb"]):
-    """Dict of total file size by individual file size range, of given extensions (empty list for all files)"""
-    files=rfilelist(path,exclude_dirs)
-    size_totals=defaultdict(int)
-    for filename,size in files.items():
-        file_ext=os.path.splitext(filename)[1]
-        if (not exts) or (file_ext in exts):
-            size_totals[2**math.frexp(size)[1]]=size_totals[2**math.frexp(size)[1]]+size
-    return size_totals
-def fgdata_size(path,dirs_to_list=["AI/Aircraft","AI/Traffic","Aircraft","Models","Scenery","Textures","Textures.high"],exclude_dirs=None,compressed_size=False,num_types=3):
-    if dirs_to_list is None:
-        dirs_to_list=[d for d in os.listdir(path) if os.path.isdir(path_join(path,d))]
+    result=defaultdict(FilesetSizes)
+    gzdir=tempfile.TemporaryDirectory()
+    gzpath=gzdir.name
+    gzcount=0
+    for file,size in files.items():
+        cat=keyfn(file,size)
+        if cat is None:
+            continue
+        result[cat].count=result[cat].count+1
+        result[cat].ncsize=result[cat].ncsize+size
+        if compressed_size:
+            try:
+                result[cat].targz.add(path_join(path,file))
+            except AttributeError:
+                result[cat].targzname=path_join(gzpath,str(gzcount)+".tar.gz")
+                result[cat].targz=tarfile.open(result[cat].targzname,mode="w:gz")
+                gzcount=gzcount+1
+                result[cat].targz.add(path_join(path,file))
+    if compressed_size:
+        for cat in result:
+            result[cat].targz.close()
+            result[cat].csize=os.path.getsize(result[cat].targzname)
+    return result
+def fgdata_size(path,by_size=False,exclude_dirs=None,compressed_size=False,min_size=1e6,include_aircraft=['UIUC','777','777-200','b1900d','CitationX','ZLT-NT','dhc2','Cub','sopwithCamel','f-14b','ASK13','bo105','Dragonfly','SenecaII','A6M2'],sections={"Aircraft/c172p":"Aircraft-base","Aircraft/Generic":"Aircraft-base","Aircraft/Instruments":"Aircraft-base","Aircraft/Instruments-3d":"Aircraft-base","Aircraft/ufo":"Aircraft-base","Aircraft/777":"Aircraft-777","Aircraft/777-200":"Aircraft-777","Aircraft":"Aircraft-other","Textures":"Textures","Textures.high":"Textures","AI/Aircraft":"AI-aircraft","AI":"AI-other","Scenery":"scenery","Models":"models"}):
+    if any('\\' in s for s in sections):
+        raise ValueError("sections always uses forward slashes")
+    if "" not in sections:
+        sections[""]="Other"
     if exclude_dirs is None:
         if os.path.exists(path_join(path,".git")):
-            exclude_dirs=[".git","Aircraft"]
+            exclude_dirs=[".git","Textures/Unused"]+[path_join("Aircraft",d) for d in os.listdir(path_join(path,"Aircraft")) if d not in ["c172p","ufo","Generic","Instruments","Instruments-3d"]+include_aircraft]
         else:
             exclude_dirs=[]
-    total_compressed_size=0
-    exclude_list=[[]]*len(dirs_to_list)+[dirs_to_list+exclude_dirs]+[exclude_dirs]
-    names_list=dirs_to_list+["other","all"]
-    for n,dir1 in enumerate(dirs_to_list+["",""]):
-        size_totals=size_by_type(path_join(path,dir1),exclude_list[n])
-        print(names_list[n],sorted(size_totals.items(),key=lambda x:-x[1])[:num_types],"total",sum(size_totals.values()))
-        if compressed_size:
-            if names_list[n]=="all":
-                print("compressed size",total_compressed_size)
-                continue
-            targz=tarfile.open("fgdata_sizetest_temp.tar.gz",mode="w:gz")
-            for file in rfilelist(path_join(path,dir1),exclude_list[n]):
-                targz.add(path_join(path,dir1,file))
-            targz.close()
-            print("compressed size",os.path.getsize("fgdata_sizetest_temp.tar.gz"))
-            total_compressed_size=total_compressed_size+os.path.getsize("fgdata_sizetest_temp.tar.gz")
+    if by_size:
+        keyfn=lambda file,size: ("texture",math.frexp(size)[1]) if os.path.splitext(file)[1] in [".png",".dds",".rgb",".jpg"] else ("nontexture",math.frexp(size)[1])
+    else:
+        def keyfn(file,size):
+            file_ext=os.path.splitext(file)[1]
+            if file_ext==".gz":
+                file_ext=os.path.splitext(os.path.splitext(file)[0])[1]+file_ext
+            file0=file.replace('\\','/')
+            section=sections[max((s for s in sections if file0.startswith(s)),key=len)]
+            return (section,file_ext)
+    result=size_by_category(path,exclude_dirs,keyfn,compressed_size)
+    #print(result)
+    totals1=defaultdict(FilesetSizes)
+    totals2=defaultdict(FilesetSizes)
+    other=defaultdict(FilesetSizes)
+    for cat in result:
+        totals1[cat[0]]=totals1[cat[0]]+result[cat]
+        totals2[cat[1]]=totals2[cat[1]]+result[cat]
+        if (not by_size) and (result[cat].size<min_size):
+            other[cat[0]]=other[cat[0]]+result[cat]
+    print("{0:>10} {1:>15} {2:>15}".format("Count","Size","Compr.size"))
+    for cat0,r0 in sorted(totals1.items(),key=lambda p:-p[1].size):
+        print(cat0)
+        for cat,r in sorted([(c[1],r) for (c,r) in result.items() if (c[0]==cat0 and (by_size or r.size>min_size))],key=(lambda p:p[0]) if by_size else (lambda p:-p[1].size)):
+            print(r,cat)
+        if not by_size:
+            print(other[cat0],"other")
+        print(totals1[cat0],"all")
+    print("All")
+    for cat,r in sorted([(c,r) for (c,r) in totals2.items() if (by_size or r.size>min_size)],key=(lambda p:p[0]) if by_size else (lambda p:-p[1].size)):
+        print(r,cat)
+    if not by_size:
+        print(sum((r for r in totals2.values() if r.size<min_size),FilesetSizes()),"other")
+    print(sum(totals1.values(),FilesetSizes()),"all")
+def create_reduced_file(input_path,output_path,temp_path,cdir,file,downsample_this,compress_this,compress_names_find, compressed_format,texture_filetypes,binary_types,fclass):
+    """Process a single file in create_reduced_fgdata.  (Separate function to allow parallel processing)"""
+    retcode=0
+    if downsample_this or compress_this:
+        image_type=texture_filetypes[os.path.splitext(file)[1]]
+        output_image_type=compressed_format if compress_this else os.path.splitext(file)[1]
+        output_file=os.path.splitext(file)[0]+output_image_type
+        output_image_type=texture_filetypes[output_image_type]
+        if "{0}" in output_path and fclass=="base-textures":#downsampled in base-textures, full resolution in extra-textures
+            shutil.copy(path_join(input_path,cdir,file),path_join(output_path.format("extra-textures"),cdir,file))
+        if output_image_type=="DDS":# in Ubuntu, neither imagemagick nor graphicsmagick can write .dds
+            #doesn't work (for dds -> smaller dds): subprocess.call(["nvzoom","-s","0.5","-f","box",path_join(input_path,cdir,file),path_join(output_path.format(fclass),cdir,file)])
+            if subprocess.call(["convert",image_type+":"+path_join(input_path,cdir,file)]+(["-flip"] if ((image_type=="DDS")!=(output_image_type=="DDS")) else [])+(["-sample","50%"] if downsample_this else [])+[path_join(temp_path,cdir,os.path.splitext(file)[0]+".png")]):#fails on some DDS formats, so just copy them
+                shutil.copy(path_join(input_path,cdir,file),path_join(output_path.format(fclass),cdir,file))
+                print(path_join(cdir,file),"unsupported type, probably normal map")
+                if compress_this:
+                    raise TypeError#copy will have the wrong name
+            else:
+                try:
+                    image_properties=subprocess.check_output(["identify","-verbose",path_join(temp_path,cdir,os.path.splitext(file)[0]+".png")])
+                except subprocess.CalledProcessError as err:
+                    print("identify error on",path_join(cdir,file),"after ",["convert",image_type+":"+path_join(input_path,cdir,file)]+(["-flip"] if ((image_type=="DDS")!=(output_image_type=="DDS")) else [])+(["-sample","50%"] if downsample_this else [])+[path_join(temp_path,cdir,os.path.splitext(file)[0]+".png")],["identify","-verbose",path_join(temp_path,cdir,os.path.splitext(file)[0]+".png")],err)
+                    raise
+                has_alpha=b"Alpha" in image_properties
+                needs_alpha=has_alpha
+                if has_alpha and re.search(rb"Alpha:\s*min: 255 \(1\)",image_properties):
+                    print(path_join(cdir,file),"has always-255 alpha")
+                    needs_alpha=False
+                retcode=subprocess.call(["nvcompress","-bc3" if needs_alpha else "-bc1",path_join(temp_path,cdir,os.path.splitext(file)[0]+".png"),path_join(output_path.format(fclass),cdir,output_file)],stdout=devnull)
+        else:
+            retcode=subprocess.call(["convert",image_type+":"+path_join(input_path,cdir,file)]+(["-sample","50%"] if downsample_this else [])+[output_image_type+":"+path_join(output_path.format(fclass),cdir,output_file)])#we use sample rather than an averaging filter to not break mask/rotation/... maps
+    else:#not to be downsampled/compressed
+        if os.path.splitext(file)[1] in binary_types:#just copy
+            shutil.copy(path_join(input_path,cdir,file),path_join(output_path.format(fclass),cdir,file))
+        else:#texture name replacement
+            file_in=open(path_join(input_path,cdir,file),'rb')
+            file_out=open(path_join(output_path.format(fclass),cdir,file),'wb')
+            file_str=file_in.read(None)
+            file_in.close()
+            (file_strout,num_matches)=compress_names_find.subn(lambda mf: os.path.splitext(mf.group(0))[0]+(compressed_format.encode('utf-8')),file_str)
+            file_out.write(file_strout)
+            file_out.close()
+            #if ((os.path.splitext(file)[1] not in textureuser_types) and num_matches>0):
+                #print("Warning: ",num_matches," unexpected use(s) in ",path_join(cdir,file))
+            #if compress_names_find0.search(file_strout):
+                #print("Warning: unreplaced match(es) in ",path_join(cdir,file),compress_names_find0.search(file_strout).group(0))
+            """Warning: unreplaced match(es) in... correct rejections of match within a filename:
+                Aircraft/Instruments-3d/AN-APS-13.ac b'panel.png'
+                Aircraft/Instruments-3d/magneto-switch/mag_switch.ac b'black.png'
+                Nasal/canvas/map/Images/chart_symbols.svg b'wash.png'
+                Models/Airport/blast-deflector49m.ac b'generic.png'
+                Models/Airport/blast-deflector63m.ac b'generic.png'
+                Models/Industrial/oilrig09.ac b'yellow.png'
+                Models/Industrial/oilrig10.ac b'yellow.png'
+                Models/Industrial/oilrig09.ac.before-color-change b'yellow.png'
+                Models/Industrial/oilrig10.ac.before-color-change b'yellow.png'
+                Models/Maritime/Civilian/Tanker.ac b'black.png'
+                Models/Transport/flatcar.xml b'evergreen.png'
+                Models/Commercial/tower-grey-black.ac b'black.png'
+                Materials/base/materials-base.xml b'yellow.png'
+                
+                Warning: unexpected use(s) in...
+                Docs/README.local_weather.html (the only one that looke like an actual problem; hence, Docs is now skipped)
+                Nasal/canvas/map/Images/chart_symbols.svg (probably inkscape:export-filename, which are creator-specific absolute paths anyway, but now skipped)
+                oilrig09.ac.before-color-change,oilrig10.ac.before-color-change,stbd_coaming_panel.ac.bak (presumably backup files)
+                """
+    if retcode:
+        print("Error ",retcode," in ",path_join(cdir,file))
 
-def create_reduced_fgdata(input_path,output_path,split_textures=True,exclude_parts=[],include_aircraft=['UIUC','777','777-200','b1900d','CitationX','ZLT-NT','dhc2','Cub','sopwithCamel','f-14b','ASK13','bo105','Dragonfly','SenecaII','A6M2'],dirs_to_downsample=(),downsample_min_filesize=30000):
+def create_reduced_fgdata(input_path,output_path,reject_positional_args=None,split_textures=False,exclude_parts=[],include_aircraft=['UIUC','777','777-200','b1900d','CitationX','ZLT-NT','dhc2','Cub','sopwithCamel','f-14b','ASK13','bo105','Dragonfly','SenecaII','A6M2'],dirs_to_downsample=(),dirs_to_compress=(),compressed_format=".dds",downsample_min_filesize=1e5,compress_min_filesize=3e4,use_ready_compressed=True):
     """Create a smaller, reduced-quality flightgear-data package
-Can downsample textures 50% and/or omit sections
-Requires Unix shell; downsampling requires imagemagick or graphicsmagick (for convert) and libnvtt-bin (for nvcompress)
+Can downsample textures 50%, change texture format, and/or omit sections (region-specific textures, aircraft, AI traffic)
+Downsampling and format change require imagemagick or graphicsmagick (for convert) and libnvtt-bin (for nvcompress)
 
 Optional parts, use exclude_parts to omit:
 ai: no background traffic, but tankers etc do still work
@@ -296,13 +432,39 @@ extra-textures (requires split_textures=True): no region-specific textures
 The c172p and ufo are always included; other aircraft are added by include_aircraft
 
 Texture downsampling: textures in dirs_to_downsample and larger than downsample_min_filesize downsampled 50%
-Example: dirs_to_downsample=("Textures.high/Terrain","Textures.high/Trees","Textures.high/Terrain.winter","AI/Aircraft","Models"),downsample_min_filesize=30000
+Texture format conversion: textures in dirs_to_compress and larger than compress_min_filesize converted to compressed_format
+use_ready_compressed determines what happens if a same-basename file in compressed_format already exists: True uses the already-compressed one, False uses the uncompressed one, None keeps both
+Suggested dirs_to_downsample:
+3.2: ('Textures.high/Terrain','Textures.high/Trees','Textures.high/Terrain.winter','AI/Aircraft','Models')
+3.3: ('Textures/Terrain','Textures/Trees','Textures/Terrain.winter','AI/Aircraft','Models')
+To do "everything" (a few are always skipped due to potential breakage), use dirs_to_compress=('',)
 
-To put each section in its own directory use {0} in output_path, e.g.
-python3 -c "import fgdata_checkers; fgdata_checkers.create_reduced_fgdata(input_path='/home/palmer/fs_dev/git/fgdata',output_path='/home/palmer/fs_dev/flightgear/data_split/debian/flightgear-data-{0}/usr/share/games/flightgear',include_aircraft=['UIUC','b1900d','CitationX','ZLT-NT','dhc2','Cub','sopwithCamel','f-14b','ASK13','bo105','Dragonfly','SenecaII','A6M2'])"
+To put each section in its own directory (e.g. for building a Debian-style flightgear-data-* set of packages) use {0} in output_path, e.g.
+python3 -c "import fgdata_checkers; fgdata_checkers.create_reduced_fgdata(input_path='/home/rnpalmer/fs_dev/git/fgdata',output_path='/home/rnpalmer/fs_dev/flightgear/data_split/debian/flightgear-data-{0}/usr/share/games/flightgear',include_aircraft=['UIUC','b1900d','CitationX','ZLT-NT','dhc2','Cub','sopwithCamel','f-14b','ASK13','bo105','Dragonfly','SenecaII','A6M2'])"
 This creates separate preferences-regions.xml and preferences-noregions.xml files for with and without regional textures; you need to handle symlinking preferences.xml to the correct one
 """
-    texture_filetypes={".png":"PNG",".dds":"DDS"}#,".rgb":"SGI" loses cloud transparency
+    start_time=time.time()
+    if reject_positional_args is not None:
+        raise TypeError("Keyword arguments only please: this is not a stable API")
+    if use_ready_compressed not in (True,False,None):
+        raise TypeError("invalid use_ready_compressed setting")
+    texture_filetypes={".png":"PNG",".dds":"DDS",".jpg":"JPEG"}#,".rgb":"SGI" loses cloud transparency
+    if compressed_format not in texture_filetypes:
+        raise ValueError("Invalid compressed_format (include the .)")
+    textureuser_types={".eff",".xml",".ac",".nas"}
+    binary_types={".png",".dds",".rgb",".RGB",".jpg",".wav",".WAV",".btg.gz",".zip",".tar.gz"}#don't search these for texture name replacement
+    """Textures named directly in flightgear/simgear code:
+gui/images/shadow.png,gui/cursor-spin-cw.png (probably safest to treat this as gui/*, they're all small)
+Textures/Globe/world.topo.bathy.200407.3x4096x2048.png
+Textures/buildings.png,Textures/buildings-lightmap.png
+Textures/Sky/*
+Textures/Splash*.png
+unknown.rgb (probably Textures/ or Textures/Terrain/, neither exists)
+Aircraft/Instruments/Textures/nd-symbols.png (doesn't actually exist),Aircraft/Instruments/Textures/compass-ribbon.rgb,Aircraft/Instruments/Textures/od_wxradar.rgb,Aircraft/Instruments/Textures/od_wxradar.rgb,Aircraft/Instruments/Textures/wxecho.rgb,Aircraft/Instruments/Textures/od_groundradar.rgb (doesn't actually exist)
+also, Aircraft/{Instruments,Instruments-3d,Generic} may be used by downloaded aircraft, and Docs images are used in .html
+Nasal (Canvas map) probably wouldn't break anything, but guessing it's a bad idea visually"""
+    no_compress_pattern=re.compile(r'mask|light|relief|nmap|nm\.|normal|dudv|^Splash[0-9].png$|^buildings.png$|^buildings-lightmap.png$|^world.topo.bathy.200407.3x4096x2048.png$|^ocean_depth_1.png$')#edge blurring from lossy compression may break masks (converting ocean_depth_1.png is known to give black water near shore at effects>=4), and this script doesn't know how to create DDS normal maps
+    no_compress_dirs=("gui","Docs","webgui","Nasal","Textures/Sky","Aircraft/Instruments","Aircraft/Instruments-3d","Aircraft/Generic")
     exclude_dirs=[".git","Textures/Unused"]
     exclude_unnamed_subdirs=["Aircraft"]#these are a separate mechanism from subtree_class/exclude_parts mostly to save time (subtree_class still fully scans excluded directories because the class may change again further down the tree, e.g. AI/Aircraft ai -> performancedb.xml base; these don't)
     subtree_class={"Aircraft/c172p":"base","Aircraft/Generic":"base","Aircraft/Instruments":"base","Aircraft/Instruments-3d":"base","Aircraft/ufo":"base","Textures":"textures","Textures.high":"textures","AI/Aircraft":"ai","AI/Traffic":"ai","AI/Aircraft/performancedb.xml":"base","Scenery":"scenery","Models":"models"}
@@ -314,12 +476,41 @@ This creates separate preferences-regions.xml and preferences-noregions.xml file
         base_texture_files=[]
         for t in find_unused_textures(input_path,return_used_noregions=True):
             base_texture_files.extend([path_join("Textures",t),path_join("Textures.high",t)])
-    #if os.path.exists(path_join(input_path,".git")):
-        #print(input_path,"appears to be a git clone; this will work, but the result will be larger than starting from a standard flightgear-data package.\nTo create this use (adjusting paths as necessary) rsync -av --filter=\"merge /home/palmer/fs_dev/git/fgmeta/base-package.rules\" ~/fs_dev/git/fgdata ~/fs_dev/flightgear/data_full")
+    #no longer a significant problem with exclude_dirs: if os.path.exists(path_join(input_path,".git")):
+        #print(input_path,"appears to be a git clone; this will work, but the result will be larger than starting from a standard flightgear-data package.\nTo create this use (adjusting paths as necessary) rsync -av --filter=\"merge /home/rnpalmer/fs_dev/git/fgmeta/base-package.rules\" ~/fs_dev/git/fgdata ~/fs_dev/flightgear/data_full")
     if os.path.exists(output_path.format("base")) and os.listdir(output_path.format("base")):
         print("output path",output_path,"non-empty, aborting to avoid data loss\nIf you did want to lose its previous contents, run:\nrm -r",output_path,"\nthen re-run this script")
         return
+    if compressed_format==".jpg":
+        print("Warning: selected compression format does not support transparency")
+    compress_names=set()
+    if dirs_to_compress:#need this preliminary pass to get names to change in .xml,etc
+        no_compress_names=set()
+        dirs={"":"base"}
+        while dirs:
+            cdir,cclass=dirs.popitem()
+            cdirfiles=os.listdir(path_join(input_path,cdir))
+            for file in cdirfiles:
+                fclass=subtree_class.get(path_join(cdir,file),cclass)
+                if os.path.isdir(path_join(input_path,cdir,file)):
+                    if (path_join(cdir,file) not in exclude_dirs) and (cdir not in exclude_unnamed_subdirs or path_join(cdir,file) in subtree_class):
+                        dirs[path_join(cdir,file)]=fclass
+                else:#file
+                    compress_this=cdir.startswith(dirs_to_compress) and (os.path.splitext(file)[1] in texture_filetypes) and (os.path.splitext(file)[1]!=compressed_format) and (os.path.getsize(path_join(input_path,cdir,file))>compress_min_filesize) and not no_compress_pattern.search(file) and not cdir.startswith(no_compress_dirs) and (file not in no_compress_names) and ((use_ready_compressed is not None) or not os.path.exists(path_join(input_path,cdir,os.path.splitext(file)[0]+compressed_format)))
+                    if compress_this:
+                        compress_names.add(file)
+                    else:
+                        no_compress_names.add(file)
+                        compress_names.discard(file)#if there are two with the same name in different directories, compress both or neither, to simplify name replacement
+    compress_names_find=re.compile(('(?<=["\'>/\\\\ \\n])('+'|'.join(re.escape(f) for f in compress_names)+')($|(?=["\'< \\n]))').encode('utf-8'))
+    compress_names_find0=re.compile(('|'.join(re.escape(f) for f in compress_names)).encode('utf-8'))
+    #print(compress_names,"\n\n",no_compress_names,"\n\n",'(?<=["\'>/\\\\ \\n])('+'|'.join(re.escape(f) for f in compress_names)+')($|(?=["\'< \\n]))',"\n\n",'|'.join(re.escape(f) for f in compress_names),"\n\n")
+    print("Starting conversion...",len(compress_names),"files to change format, runtime so far=",int(time.time()-start_time),"sec")
     dirs={"":"base"}
+    subprocess_pool=multiprocessing.Pool(processes=8)
+    subprocess_list=[]
+    temp_dir=tempfile.TemporaryDirectory()
+    temp_path=temp_dir.name
     while dirs:
         cdir,cclass=dirs.popitem()
         cdirfiles=os.listdir(path_join(input_path,cdir))
@@ -337,25 +528,29 @@ This creates separate preferences-regions.xml and preferences-noregions.xml file
                 if fclass in exclude_parts:
                     continue
                 if not os.path.exists(path_join(output_path.format(fclass),cdir)):
-                    subprocess.call(["mkdir","-p",path_join(output_path.format(fclass),cdir)])
-                if (cdir.startswith(dirs_to_downsample)) and (os.path.splitext(file)[1] in texture_filetypes) and (os.path.getsize(path_join(input_path,cdir,file))>downsample_min_filesize):
-                    image_type=texture_filetypes[os.path.splitext(file)[1]]
-                    if "{0}" in output_path and fclass=="base-textures":#downsampled in base-textures, full resolution in extra-textures
-                        if not os.path.exists(path_join(output_path.format("extra-textures"),cdir)):
-                            subprocess.call(["mkdir","-p",path_join(output_path.format("extra-textures"),cdir)])
-                        subprocess.call(["cp",path_join(input_path,cdir,file),path_join(output_path.format("extra-textures"),cdir,file)])
-                    if image_type=="DDS":# in Ubuntu, neither imagemagick nor graphicsmagick can write .dds
-                        #doesn't work subprocess.call(["nvzoom","-s","0.5","-f","box",path_join(input_path,cdir,file),path_join(output_path.format(fclass),cdir,file)])
-                        if subprocess.call(["convert",image_type+":"+path_join(input_path,cdir,file),"-sample","50%","temp_reduced_size.png"]):#fails on normal maps, so just copy them
-                            subprocess.call(["cp",path_join(input_path,cdir,file),path_join(output_path.format(fclass),cdir,file)])
-                        else:
-                            subprocess.call(["nvcompress","-bc3","temp_reduced_size.png",path_join(output_path.format(fclass),cdir,file)])
-                    else:
-                        subprocess.call(["convert",image_type+":"+path_join(input_path,cdir,file),"-sample","50%",image_type+":"+path_join(output_path.format(fclass),cdir,file)])#we use sample rather than an averaging filter to not break mask/rotation/... maps
-                else:#not to be downsampled
-                    subprocess.call(["cp",path_join(input_path,cdir,file),path_join(output_path.format(fclass),cdir,file)])
+                    os.makedirs(path_join(output_path.format(fclass),cdir))#errors out if the directory does exist, so calling it in the per-file subprocess would be a race condition
+                if "{0}" in output_path and fclass=="base-textures" and not os.path.exists(path_join(output_path.format("extra-textures"),cdir)):#downsampled in base-textures, full resolution in extra-textures
+                    os.makedirs(path_join(output_path.format("extra-textures"),cdir))
+                if not os.path.exists(path_join(temp_path,cdir)):
+                    os.makedirs(path_join(temp_path,cdir))
+                downsample_this=(cdir.startswith(dirs_to_downsample)) and (os.path.splitext(file)[1] in texture_filetypes) and (os.path.getsize(path_join(input_path,cdir,file))>downsample_min_filesize)
+                compress_this=(file in compress_names)
+                if compress_this and use_ready_compressed==True and os.path.exists(path_join(input_path,cdir,os.path.splitext(file)[0]+compressed_format)):
+                    continue
+                if (not compress_this) and use_ready_compressed==False and os.path.splitext(file)[1]==compressed_format and any((os.path.exists(path_join(input_path,cdir,os.path.splitext(file)[0]+f)) and os.path.splitext(file)[0]+f in compress_names) for f in texture_filetypes if f!=os.path.splitext(file)[1]):
+                    continue
+                subprocess_list.append(subprocess_pool.apply_async(create_reduced_file,args=(input_path,output_path,temp_path,cdir,file,downsample_this,compress_this,compress_names_find, compressed_format,texture_filetypes,binary_types,fclass)))
+    print(len(subprocess_list),"file tasks started...runtime so far=",int(time.time()-start_time),"sec\n(Not a linear progress indicator: they are different lengths.)")
+    last_report=time.time()-100#print first report immediately
+    for s0 in subprocess_list:
+        if time.time()>last_report+60:
+            print("Waiting for",len([s for s in subprocess_list if not s.ready()]),"file tasks...runtime so far=",int(time.time()-start_time),"sec")
+            last_report=time.time()
+        s0.get()
+    subprocess_pool.close()
+    subprocess_pool.join()
     if "{0}" in output_path:
-        subprocess.call(["mv",path_join(output_path.format("base"),"preferences.xml"),path_join(output_path.format("base"),"preferences-regions.xml")])
+        os.rename(path_join(output_path.format("base"),"preferences.xml"),path_join(output_path.format("base"),"preferences-regions.xml"))
     if "extra-textures" in exclude_parts or "{0}" in output_path:
         prefs_in=open(path_join(input_path,"preferences.xml"),'r')
         prefs_out=open(path_join(output_path.format("base"),"preferences-noregions.xml" if "{0}" in output_path else "preferences.xml"),'w')
@@ -364,6 +559,7 @@ This creates separate preferences-regions.xml and preferences-noregions.xml file
         prefs_str=prefs_str.replace("Materials/regions/materials.xml","Materials/default/materials.xml")#turn off regional textures
         prefs_out.write(prefs_str)
         prefs_out.close()
+    print("Total runtime=",int(time.time()-start_time),"sec")
 def check_text_encoding(path,filelist=None,binary_types=(".png",".dds",".rgb",".RGB",".jpg",".wav",".WAV",".btg.gz",".xcf.gz",".xcf",".XCF","Thumbs.db",".blend",".bmp",".gif", ".3ds",".3DS",".pdf",".ttf",".txf",".htsvoice",".odt",".ods",".xls",".mp3",".zip",".tar.gz"),exclude_dirs=[".git","Timezone"]):
     """filelist is intended for quick testing: see fgdata_nonascii_filelist.py"""
     def err_context(err):

hudson_build_release.sh

@@ -60,15 +60,25 @@ cp flightgear-*.tar.bz2 ../output/.
 echo "Assembling base package"
 cd $WORKSPACE
 
-echo "Syncing base packages files from sphere.telascience.org"
+rm -rf base_package
 
 # a: archive mode
 # z: compress
 # delete: 'delete extraneous files from dest dirs'; avoid bug 1344
 # filter: use the rules in our rules file
-rsync -az --delete \
+
+echo "Copying FGData files"
+rsync -a --delete \
  --filter 'merge base-package.rules' \
- -e ssh jturner@sphere.telascience.org:/home/jturner/fgdata .
+  fgdata base_package
 
-tar cjf output/FlightGear-$VERSION-data.tar.bz2 fgdata/
+echo "Syncing extended data"
+rsync -a --filter 'merge aircraft.rules' /home/jenkins/fgdata_340/ extended-data
 
+echo "Copying extended data"
+rsync -a extended-data/ base_package/fgdata
+
+echo "Creating tar archive"
+pushd base_package
+tar cjf $WORKSPACE/output/FlightGear-$VERSION-data.tar.bz2 fgdata/
+popd

hudson_mac_build_release.sh

@@ -33,7 +33,7 @@ popd
 ################################################################################
 echo "Starting on FlightGear"
 pushd fgBuild
-cmake -DCMAKE_INSTALL_PREFIX:PATH=$WORKSPACE/dist -G Xcode ../flightgear
+cmake -DCMAKE_INSTALL_PREFIX:PATH=$WORKSPACE/dist -DENABLE_QT=1 -G Xcode ../flightgear
 
 xcodebuild -configuration RelWithDebInfo -target install  build
 
@@ -47,28 +47,6 @@ popd
 chmod +x $WORKSPACE/dist/bin/osgversion
 
 ################################################################################
-echo "Building Macflightgear launcher"
-
-#OSX_TARGET="10.6"
-# JMT - disabling setting the sysroot since it's breaking things on 
-# current build slave. Sinc ethe slave runs 10.6 natively, we don't 
-# actually need to set these. Real solution would be to use a proper
-# build system for the Mac-launcher of course.
-# -mmacosx-version-min=$OSX_TARGET -isysroot $SDK_PATH
-
-pushd maclauncher/FlightGearOSX
-
-# compile the stub executable
-gcc -o FlightGear main.m \
-    -framework Cocoa -framework RubyCocoa -framework Foundation -framework AppKit
-
-popd
-
-################################################################################
-echo "Syncing base packages files from sphere.telascience.org"
-rsync -avz --filter 'merge base-package.rules' \
- -e ssh jturner@sphere.telascience.org:/home/jturner/fgdata .
-
 # run the unlock script now - we need to do this right before code-signing,
 # or the keychain may automatically re-lock after some period of time
 unlock-keychain.sh

hudson_mac_package_release.rb

@@ -22,19 +22,15 @@ puts "Code signing identity is #{$codeSignIdentity}"
 
 puts "osgVersion=#{osgVersion}, so-number=#{$osgSoVersion}"
 
-$svnLibs = ['svn_client', 'svn_wc', 'svn_delta', 'svn_diff', 'svn_ra', 
-  'svn_ra_local', 'svn_repos', 'svn_fs', 'svn_fs_fs', 'svn_fs_util',
-  'svn_ra_svn', 'svn_subr', 'svn_ra_neon']
-
 def fix_install_names(object)
   #puts "fixing install names for #{object}"
-  
+
   $osgLibs.each do |l|
     oldName = "lib#{l}.#{$osgSoVersion}.dylib"
     newName = "@executable_path/../Frameworks/#{oldName}"
     `install_name_tool -change #{oldName} #{newName} #{object}`
   end
-  
+
   oldName = "libOpenThreads.#{$openThreadsSoVersion}.dylib"
   newName= "@executable_path/../Frameworks/#{oldName}"
   `install_name_tool -change #{oldName} #{newName} #{object}`
@@ -44,40 +40,26 @@ $prefixDir=Dir.pwd + "/dist"
 dmgDir=Dir.pwd + "/image"
 srcDir=Dir.pwd + "/flightgear"
 
-def fix_svn_install_names(object)
-  $svnLibs.each do |l|
-    fileName = "lib#{l}-1.0.dylib"
-    newName = "@executable_path/../Frameworks/#{fileName}"
-    `install_name_tool -change #{fileName} #{newName} #{object}`
-  end
-end
-
-def copy_svn_libs()
-  puts "Copying Subversion client libraries"
-  $svnLibs.each do |l|
-    libFile = "lib#{l}-1.0.dylib"
-    path = "#{$frameworksDir}/#{libFile}"
-    `cp #{$prefixDir}/lib/#{libFile} #{$frameworksDir}`
-    fix_svn_install_names(path)
-   # `install_name_tool -id #{libFile}  #{path}`    
-  end
-end
-
-def code_sign(path)
-  puts "Signing #{path}"
-  `codesign -s "#{$codeSignIdentity}" #{path}`
-end
-
-
 puts "Erasing previous image dir"
 `rm -rf #{dmgDir}`
 
+bundle=dmgDir + "/FlightGear.app"
+
+# run macdeployt before we rename the bundle, otherwise it
+# can't find the bundle executable
+puts "Running macdeployqt on the bundle to copy Qt libraries"
+`macdeployqt #{$prefixDir}/fgfs.app`
+
+puts "Moving & renaming app bundle"
+`mkdir -p #{dmgDir}`
+`mv #{$prefixDir}/fgfs.app #{bundle}`
+
 bundle=dmgDir + "/FlightGear.app"
 contents=bundle + "/Contents"
 macosDir=contents + "/MacOS"
 $frameworksDir=contents +"/Frameworks"
 resourcesDir=contents+"/Resources"
-osgPluginsDir=contents+"/PlugIns/osgPlugins-#{osgVersion}"
+osgPluginsDir=contents+"/PlugIns/osgPlugins"
 
 # for writing copyright year to Info.plist
 t = Time.new
@@ -94,17 +76,19 @@ puts "Creating directory structure"
 `mkdir -p #{resourcesDir}`
 `mkdir -p #{osgPluginsDir}`
 
-puts "Copying binaries"
-bins = ['fgfs', 'fgjs', 'fgcom', 'fgviewer']
+# fix install names on the primary executable
+fix_install_names("#{macosDir}/fgfs")
+
+puts "Copying auxilliary binaries"
+bins = ['fgjs', 'fgcom']
 bins.each do |b|
   if !File.exist?("#{$prefixDir}/bin/#{b}")
     next
   end
-  
+
   outPath = "#{macosDir}/#{b}"
   `cp #{$prefixDir}/bin/#{b} #{outPath}`
   fix_install_names(outPath)
-  fix_svn_install_names(outPath)
 end
 
 puts "copying libraries"
@@ -119,21 +103,11 @@ libFile = "libOpenThreads.#{$openThreadsSoVersion}.dylib"
 `cp #{$prefixDir}/lib/#{libFile} #{$frameworksDir}`
 
 $osgPlugins.each do |p|
-  pluginFile = "osgdb_#{p}.so"
-  `cp #{$prefixDir}/lib/osgPlugins-#{osgVersion}/#{pluginFile} #{osgPluginsDir}`
+  pluginFile = "osgdb_#{p}.dylib"
+  `cp #{$prefixDir}/lib/osgPlugins/#{pluginFile} #{osgPluginsDir}`
   fix_install_names("#{osgPluginsDir}/#{pluginFile}")
 end
 
-copy_svn_libs()
-
-# Macflightgear launcher
-puts "Copying Macflightgear launcher files"
-
-Dir.chdir "maclauncher/FlightGearOSX" do
-  `cp FlightGear #{macosDir}`
-  `rsync -a *.rb *.lproj *.sh *.tiff *.html #{resourcesDir}`
-end
-
 if File.exist?("#{$prefixDir}/bin/fgcom-data")
   puts "Copying FGCom data files"
   `ditto #{$prefixDir}/bin/fgcom-data #{resourcesDir}/fgcom-data`
@@ -155,16 +129,14 @@ File.open("#{contents}/Info.plist", 'w') { |f|
 `mv fgdata/Docs/getstart.pdf "#{dmgDir}/Getting Started.pdf"`
 
 puts "Copying base package files into the image"
-`rsync -a fgdata/ #{resourcesDir}/data`
+`rsync -a --filter 'merge base-package.rules' fgdata/ #{resourcesDir}/data`
 
-# code sign all executables in MacOS dir. Do this last since reource
-# changes will invalidate the signature!
-Dir.foreach(macosDir) do |b|
-    if b == '.' or b == '..' then
-        next
-    end
-  code_sign("#{macosDir}/#{b}")
-end
+# work-around for 3.4.1: copy aircraft/scenery files too
+`rsync -a extended-data/ #{resourcesDir}/data`
+
+# code sign the entire bundle once complete - v2 code-signing
+puts "Signing #{bundle}"
+`codesign --deep -s "#{$codeSignIdentity}" #{bundle}`
 
 puts "Creating DMG"
 

hudson_win_build32.bat

@@ -14,7 +14,7 @@ cmake ..\simgear -G "Visual Studio 10" -DMSVC_3RDPARTY_ROOT=%WORKSPACE% -DCMAKE_
 cmake --build . --config Release --target INSTALL
 
 cd ..\build-fg
-cmake ..\flightgear -G "Visual Studio 10" -DMSVC_3RDPARTY_ROOT=%WORKSPACE% -DCMAKE_INSTALL_PREFIX:PATH=%WORKSPACE%/install/msvc100/FlightGear -DFLTK_FLUID_EXECUTABLE=%WORKSPACE%/3rdParty/bin/fluid.exe -DBOOST_ROOT=%WORKSPACE%/Boost
+cmake ..\flightgear -G "Visual Studio 10" -DMSVC_3RDPARTY_ROOT=%WORKSPACE% -DCMAKE_INSTALL_PREFIX:PATH=%WORKSPACE%/install/msvc100/FlightGear -DPNG_LIBRARY=%WORKSPACE%/3rdParty/lib/libpng16.lib -DFLTK_FLUID_EXECUTABLE=%WORKSPACE%/3rdParty/bin/fluid.exe -DBOOST_ROOT=%WORKSPACE%/Boost
 cmake --build . --config Release --target INSTALL
 
 cd ..\build-fgrun

hudson_win_package_release.bat

@@ -29,10 +29,9 @@ ECHO #define OSGVersion "%OSG_VERSION%" >> InstallConfig.iss
 ECHO #define OSGSoNumber "%OSG_SO_NUMBER%" >> InstallConfig.iss
 ECHO #define OTSoNumber "%OT_SO_NUMBER%" >> InstallConfig.iss
 
-set DATA_FILE=FlightGear-%FLIGHTGEAR_VERSION%-data
-
+REM set DATA_FILE=FlightGear-%FLIGHTGEAR_VERSION%-data
 REM extract the data files
-7z e -aoa %DATA_FILE%.tar.bz2 && 7z x -aoa %DATA_FILE%.tar
+REM 7z e -aoa %DATA_FILE%.tar.bz2 && 7z x -aoa %DATA_FILE%.tar
 
 REM run Inno-setup!
 REM use iscc instead of compil32 for better error reporting

sgprops.py

@@ -159,11 +159,12 @@ class PropsHandler(handler.ContentHandler):
         if (name == 'PropertyList'):
             return
             
-        index = 0
         if 'n' in attrs.keys():
             index = int(attrs['n'])
+            self._current = self._current.getChild(name, index, create=True)
+        else:
+            self._current = self._current.addChild(name)
             
-        self._current = self._current.getChild(name, index, create=True)
             
         if 'include' in attrs.keys():
             self.handleInclude(attrs['include'])

webhooks.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0"?>
+<!--
+  The <regex> field is evaluated by preg_match
+  http://php.net/manual/en/function.preg-match.php
+-->
+<PropertyList>
+  <webhook>
+    <regex>/^Merge branch/</regex>
+    <description>Detect merge branch when rebase has been ommited</description>
+  </webhook>
+  <webhook>
+    <regex>/^test$/</regex>
+    <description>Detect for dummy message</description>
+  </webhook>
+  <webhook>
+    <regex>/blabla/</regex>
+    <description>Detect for bad commit description</description>
+  </webhook>
+</PropertyList>