Revert "Linux-release: use local base-package"

This reverts commit 0f1dbad70d,
now fgdata is fixed.

.gitignore

@@ -20,7 +20,3 @@ CMakeFiles
 archivebuild
 osgbuild
 CMakeCache.txt
-aircraft-data
-base_package
-extended-data
-

.gitmodules

@@ -1,16 +1,16 @@
 [submodule "simgear"]
 	path = simgear
-	url = git://git.code.sf.net/p/flightgear/simgear
-	branch = release/3.4.0
+	url = git://gitorious.org/fg/simgear.git
+	branch = next
 [submodule "flightgear"]
 	path = flightgear
-	url = git://git.code.sf.net/p/flightgear/flightgear
-	branch = release/3.4.0
+	url = git://gitorious.org/fg/flightgear.git
+	branch = next
 [submodule "fgrun"]
 	path = fgrun
-	url = git://git.code.sf.net/p/flightgear/fgrun
-	branch = release/3.4.0
-[submodule "fgdata"]
-	path = fgdata
-	url = git://git.code.sf.net/p/flightgear/fgdata
-	branch = release/3.4.0
+	url = git://gitorious.org/fg/fgrun.git
+	branch = next
+[submodule "maclauncher"]
+	path = maclauncher
+	url = git://gitorious.org/fg/maclauncher.git
+	branch = master

FlightGear.iss

@@ -43,16 +43,15 @@ AppPublisherURL=http://www.flightgear.org
 AppSupportURL=http://www.flightgear.org
 AppUpdatesURL=http://www.flightgear.org
 DefaultDirName={pf}\FlightGear {#FGVersion}
-UsePreviousAppDir=no
 DefaultGroupName=FlightGear {#FGVersion}
 LicenseFile=X:\flightgear\COPYING
 Uninstallable=yes
 SetupIconFile=X:\flightgear\package\flightgear.ico
 VersionInfoVersion={#FGVersion}.0
-InfoBeforeFile=X:\flightgear\package\windows\infobefore.txt
-WizardImageFile=X:\flightgear\package\windows\setupimg.bmp
+InfoBeforeFile=X:\flightgear\package\Win32-Inno\infobefore.txt
+WizardImageFile=X:\flightgear\package\Win32-Inno\setupimg.bmp
 WizardImageStretch=No
-WizardSmallImageFile=X:\flightgear\package\windows\setupsmall.bmp
+WizardSmallImageFile=X:\flightgear\package\Win32-Inno\setupsmall.bmp
 VersionInfoCompany=The FlightGear Team
 UninstallDisplayIcon={app}\bin\fgfs.exe
 ArchitecturesInstallIn64BitMode=x64
@@ -116,6 +115,8 @@ Source: "{#VCInstallDir}\redist\x64\Microsoft.VC100.CRT\*.dll"; DestDir: "{app}\
 ; 32/64 bits install
 ;NOTE: FGPanel has no 64 bits equivalent, so we are using the 32 bits binary for 32&64 bits OS
 Source: "X:\install\msvc100\FlightGear\bin\fgpanel.exe"; DestDir: "{app}\bin"; Flags: ignoreversion
+Source: "X:\install\msvc100\FlightGear\share\flightgear\positions.txt"; DestDir: "{app}\share\flightgear"
+Source: "X:\install\msvc100\FlightGear\share\flightgear\special_frequencies.txt"; DestDir: "{app}\share\flightgear"
 Source: "X:\3rdParty\bin\oalinst.exe"; DestDir: "{app}\bin"; Flags: ignoreversion skipifsourcedoesntexist
 
 ; NOTE: tar.gz file uses name 'fgdata', to avoid renaming it, look for both names.
@@ -123,9 +124,6 @@ Source: "X:\3rdParty\bin\oalinst.exe"; DestDir: "{app}\bin"; Flags: ignoreversio
 Source: "X:\data\*.*"; DestDir: "{app}\data"; Flags: ignoreversion recursesubdirs skipifsourcedoesntexist
 Source: "X:\fgdata\*.*"; DestDir: "{app}\data"; Flags: ignoreversion recursesubdirs skipifsourcedoesntexist
 
-; work-around for 3.4.1: look for additional aircraft data
-Source: "X:\extended-data\*.*"; DestDir: "{app}\data"; Flags: ignoreversion recursesubdirs skipifsourcedoesntexist
-
 ; 32 bits install
 Source: "{#OSGInstallDir}\bin\osg{#OSGSoNumber}-osg.dll"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
 Source: "{#OSGInstallDir}\bin\osg{#OSGSoNumber}-osgDB.dll"; DestDir: "{app}\bin"; Check: not Is64BitInstallMode
@@ -226,7 +224,7 @@ Name: "{group}\Tools\metar"; Filename: "cmd"; Parameters: "/k ""{app}\bin\metar.
 Name: "{group}\Tools\yasim"; Filename: "cmd"; Parameters: "/k ""{app}\bin\yasim.exe"" -h"; WorkingDir: "{app}\bin"
 Name: "{group}\Tools\fgpanel"; Filename: "cmd"; Parameters: "/k ""{app}\bin\fgpanel.exe"" -h"; WorkingDir: "{app}\bin"
 Name: "{group}\Tools\FGCom"; Filename: "{app}\bin\fgcom.exe"; WorkingDir: "{app}\bin"
-Name: "{group}\Tools\FGCom-testing"; Filename: "{app}\bin\fgcom.exe"; Parameters: "--frequency=910"; WorkingDir: "{app}\bin"
+Name: "{group}\Tools\FGCom-testing"; Filename: "{app}\bin\fgcom.exe"; Parameters: "-f910"; WorkingDir: "{app}\bin"
 Name: "{group}\Tools\Explore Documentation Folder"; Filename: "{app}\data\Docs"
 
 [Run]
@@ -241,20 +239,6 @@ filename: "{app}\bin\fgrun.exe"; WorkingDir: "{app}\bin"; Parameters: "--silent
 filename: "{app}\bin\fgadmin.exe"; WorkingDir: "{app}\bin"; Parameters: "--silent ""--install-source={src}\..\Scenery"" ""--scenery-dest={userdocs}\FlightGear\Custom Scenery"""
 
 [Code]
-const
-  NET_FW_SCOPE_ALL = 0;
-  NET_FW_IP_VERSION_ANY = 2;
-  NET_FW_ACTION_ALLOW = 1;
-  NET_FW_RULE_DIR_ALL = 0;
-  NET_FW_RULE_DIR_IN = 1;
-  NET_FW_RULE_DIR_OUT = 2;
-  NET_FW_IP_PROTOCOL_ALL = 0;
-  NET_FW_IP_PROTOCOL_TCP = 6;
-  NET_FW_IP_PROTOCOL_UDP = 17;
-  NET_FW_PROFILE2_DOMAIN = 1;
-  NET_FW_PROFILE2_PRIVATE = 2;
-  NET_FW_PROFILE2_PUBLIC = 4;
-
 procedure URLLabelOnClick(Sender: TObject);
 var
   ErrorCode: Integer;
@@ -290,109 +274,3 @@ begin
   Result := S;
 end;
 
-procedure AddBasicFirewallException(AppName, FileName: String);
-var
-  FirewallObject: variant;
-  RuleObject: variant;
-begin
-  try
-    FirewallObject := CreateOleObject('HNetCfg.FwMgr');
-    RuleObject := CreateOleObject('HNetCfg.FwAuthorizedApplication');
-    RuleObject.ProcessImageFileName := FileName;
-    RuleObject.Name := AppName;
-    RuleObject.Scope := NET_FW_SCOPE_ALL;
-    RuleObject.IpVersion := NET_FW_IP_VERSION_ANY;
-    RuleObject.Enabled := true;
-    FirewallObject.LocalPolicy.CurrentProfile.AuthorizedApplications.Add(RuleObject);
-  except
-  end;
-end;
-
-procedure AddAdvancedFirewallException(AppName, AppDescription, FileName: String; Protocol: Integer; LocalPorts, RemotePorts: String; Direction: Integer);
-var
-  FirewallObject: variant;
-  RuleObject: variant;
-begin
-  try
-    FirewallObject := CreateOleObject('HNetCfg.FwPolicy2');
-    RuleObject := CreateOleObject('HNetCfg.FWRule');
-    RuleObject.Name := AppName;
-    RuleObject.Description := AppDescription;
-    RuleObject.ApplicationName := FileName;
-    if (Protocol <> NET_FW_IP_PROTOCOL_ALL) then
-      RuleObject.Protocol := Protocol;
-    if (LocalPorts <> '') then
-      RuleObject.LocalPorts := LocalPorts;
-    if (RemotePorts <> '') then
-      RuleObject.RemotePorts := RemotePorts;
-    if (Direction <> NET_FW_RULE_DIR_ALL) then
-      RuleObject.Direction := Direction;
-    RuleObject.Enabled := true;
-    RuleObject.Grouping := 'FlightGear';
-    RuleObject.Profiles := NET_FW_PROFILE2_DOMAIN + NET_FW_PROFILE2_PRIVATE + NET_FW_PROFILE2_PUBLIC;
-    RuleObject.Action := NET_FW_ACTION_ALLOW;
-    RuleObject.RemoteAddresses := '*';
-    FirewallObject.Rules.Add(RuleObject);
-  except
-  end;
-end;
-
-procedure RemoveFirewallException(AppName, FileName: String);
-var
-  FirewallObject: variant;
-  Version: TWindowsVersion;
-begin
-  GetWindowsVersionEx(Version);
-  try
-    if (Version.Major >= 6) then
-      begin
-        FirewallObject := CreateOleObject('HNetCfg.FwPolicy2');
-        FirewallObject.Rules.Remove(AppName);
-      end
-    else if (Version.Major = 5) and (((Version.Minor = 1) and (Version.ServicePackMajor >= 2)) or ((Version.Minor = 2) and (Version.ServicePackMajor >= 1))) then
-      begin
-        FirewallObject := CreateOleObject('HNetCfg.FwMgr');
-        FirewallObject.LocalPolicy.CurrentProfile.AuthorizedApplications.Remove(FileName);
-      end;
-  except
-  end;
-end;
-
-procedure CurStepChanged(CurStep: TSetupStep);
-var
-  Version: TWindowsVersion;
-begin
-  if CurStep = ssPostInstall then
-    begin
-      GetWindowsVersionEx(Version);
-      if (Version.Major >= 6) then
-        begin
-          { IN and OUT rules must be specified separately, otherwise the firewall will create only the IN rule }
-          AddAdvancedFirewallException('FlightGear', 'Allows FlightGear to send and receive data over the multiplayer network and to get METARs.', ExpandConstant('{app}') + '\bin\fgfs.exe', NET_FW_IP_PROTOCOL_ALL, '', '', NET_FW_RULE_DIR_IN);
-          AddAdvancedFirewallException('FlightGear', 'Allows FlightGear to send and receive data over the multiplayer network and to get METARs.', ExpandConstant('{app}') + '\bin\fgfs.exe', NET_FW_IP_PROTOCOL_ALL, '', '', NET_FW_RULE_DIR_OUT);
-          AddAdvancedFirewallException('FlightGear METAR Utility', 'Allows the FlightGear METAR utility to receive METARs.', ExpandConstant('{app}') + '\bin\metar.exe', NET_FW_IP_PROTOCOL_TCP, '', '80', NET_FW_RULE_DIR_OUT);
-          AddAdvancedFirewallException('FlightGear TerraSync', 'Allows TerraSync to download additional scenery while FlightGear is running.', ExpandConstant('{app}') + '\bin\terrasync.exe', NET_FW_IP_PROTOCOL_ALL, '', '', NET_FW_RULE_DIR_IN);
-          AddAdvancedFirewallException('FlightGear TerraSync', 'Allows TerraSync to download additional scenery while FlightGear is running.', ExpandConstant('{app}') + '\bin\terrasync.exe', NET_FW_IP_PROTOCOL_ALL, '', '', NET_FW_RULE_DIR_OUT);
-          AddAdvancedFirewallException('FlightGear FGCom', 'Allows FGCom to establish a connection to FlightGear and the VoIP server for voice ATC communication.', ExpandConstant('{app}') + '\bin\fgcom.exe', NET_FW_IP_PROTOCOL_ALL, '', '', NET_FW_RULE_DIR_IN);
-          AddAdvancedFirewallException('FlightGear FGCom', 'Allows FGCom to establish a connection to FlightGear and the VoIP server for voice ATC communication.', ExpandConstant('{app}') + '\bin\fgcom.exe', NET_FW_IP_PROTOCOL_ALL, '', '', NET_FW_RULE_DIR_OUT);
-        end
-      else if (Version.Major = 5) and (((Version.Minor = 1) and (Version.ServicePackMajor >= 2)) or ((Version.Minor = 2) and (Version.ServicePackMajor >= 1))) then
-        begin
-          { The Windows XP/Server 2003 firewall does not block outgoing connections at all, so only listening processes should be added }
-          AddBasicFirewallException('FlightGear', ExpandConstant('{app}') + '\bin\fgfs.exe');
-          AddBasicFirewallException('FlightGear TerraSync', ExpandConstant('{app}') + '\bin\terrasync.exe');
-          AddBasicFirewallException('FlightGear FGCom', ExpandConstant('{app}') + '\bin\fgcom.exe');
-        end;
-    end;
-end;
-
-procedure CurUninstallStepChanged(CurUninstallStep: TUninstallStep);
-begin
-  if CurUninstallStep = usPostUninstall then
-    begin
-      RemoveFirewallException('FlightGear', ExpandConstant('{app}') + '\bin\fgfs.exe');
-      RemoveFirewallException('FlightGear METAR Utility', ExpandConstant('{app}') + '\bin\metar.exe');
-      RemoveFirewallException('FlightGear TerraSync', ExpandConstant('{app}') + '\bin\terrasync.exe');
-      RemoveFirewallException('FlightGear FGCom', ExpandConstant('{app}') + '\bin\fgcom.exe');
-    end;
-end;

Info.plist.in

@@ -13,7 +13,7 @@
 	<key>CFBundleSignature</key>
 	<string>????</string>
 	<key>CFBundleExecutable</key>
-	<string>fgfs</string>
+	<string>FlightGear</string>
 	<key>CFBundleIdentifier</key>
 	<string>org.flightgear.FlightGear</string>
 	<key>CFBundleVersion</key>
@@ -21,20 +21,13 @@
 	<key>CFBundleShortVersionString</key>
 	<string><%= fgVersion %></string>
 	<key>LSMinimumSystemVersion</key>
-	<string>10.7.0</string>
+	<string>10.6.0</string>
 	<key>NSMainNibFile</key>
 	<string>MainMenu</string>
 	<key>NSPrincipalClass</key>
 	<string>NSApplication</string>
     <key>NSHumanReadableCopyright</key>
     <string>©1996-<%= fgCurrentYear%>, The FlightGear Project</string>
-
-    <!-- when launched via LaunchServices, run the launcher GUI -->
-    <key>LSEnvironment</key>
-    <dict>
-        <key>FG_LAUNCHER</key>
-        <string>1</string>
-    </dict>
 </dict>
 </plist>
 

aircraft.rules

@@ -1,21 +0,0 @@
-+ /Aircraft
-+ /Aircraft/777
-+ /Aircraft/777-200
-+ /Aircraft/b1900d
-+ /Aircraft/CitationX
-+ /Aircraft/ZLT-NT
-+ /Aircraft/dhc2
-+ /Aircraft/Cub
-+ /Aircraft/sopwithCamel
-+ /Aircraft/f-14b
-+ /Aircraft/ASK13
-+ /Aircraft/bo105
-+ /Aircraft/Dragonfly
-+ /Aircraft/SenecaII
-+ /Aircraft/A6M2
-- /Aircraft/*
-+ /Models
-+ /Scenery
-+ /AI
-- /*
-

base-package.rules

@@ -2,6 +2,33 @@
 - *.xcf
 - *.tex
 
++ /fgdata/Aircraft/Generic
++ /fgdata/Aircraft/Instruments
++ /fgdata/Aircraft/Instruments-3d
+
++ /fgdata/Aircraft/UIUC
++ /fgdata/Aircraft/c172p
++ /fgdata/Aircraft/777
++ /fgdata/Aircraft/777-200
++ /fgdata/Aircraft/b1900d
++ /fgdata/Aircraft/ufo
++ /fgdata/Aircraft/CitationX
++ /fgdata/Aircraft/ZLT-NT
++ /fgdata/Aircraft/dhc2
++ /fgdata/Aircraft/Cub
++ /fgdata/Aircraft/sopwithCamel
++ /fgdata/Aircraft/f-14b
++ /fgdata/Aircraft/ASK13
++ /fgdata/Aircraft/bo105
++ /fgdata/Aircraft/Dragonfly
++ /fgdata/Aircraft/SenecaII
++ /fgdata/Aircraft/A6M2
+
+- /fgdata/Aircraft/*
+
+- /fgdata/Textures.high/*.new
+- /fgdata/Textures.high/*.orig
+- /fgdata/Textures.high/*.save
 - /fgdata/Textures/Unused
 - /fgdata/Textures/*.orig
 - /fgdata/Docs/source

catalogTags.py

@@ -1,12 +1,12 @@
 aircraftTypeTags = [
     "ga", "fighter", "helicopter", "glider", "spaceship", "bomber", "groundvehicle", 
-    "tanker", "cargo", "transport", "bizjet", "trainer", "airship", "balloon"
+    "tanker", "cargo", "passenger", "bizjet", "trainer", "airship", "balloon"
 ]
 
 manufacturerTags = [
     "boeing", "cessna", "diamond", "douglas", "bell", "piper", 
     "airbus", "vickers", "lockheed", "fokker",
-    "embrarer", "bombardier", "pilatus", "robin"
+    "embrarer", "bombardier", "pilatus"
 ]
 
 eraTags = [
@@ -62,9 +62,7 @@ propulsionTags = [
     "supercharged",
     "turboprop",
     "jet", "afterburner", "rocket",
-    "electric",
-    "twin-engine",
-    "single-engine"
+    "electric"
 ]
 
 simFeatureTags = [

download_and_compile.sh

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-VERSION="2.28"
+VERSION="2.26"
 
 #######################################################
 # THANKS TO
@@ -132,8 +132,8 @@ function _make(){
 OSG_STABLE_GIT_BRANCH="OpenSceneGraph-3.2"
 PLIB_STABLE_GIT_BRANCH="master"
 # common stable branch for flightgear, simgear and fgdata
-FGSG_STABLE_GIT_BRANCH="release/3.4.0"
-OPENRTI_STABLE_GIT_BRANCH="release-0.6"
+FGSG_STABLE_GIT_BRANCH="release/3.2.0"
+OPENRTI_STABLE_GIT_BRANCH="release-0.5"
 # unstable branch: next for sg/fg, master for fgdata
 FGSG_UNSTABLE_GIT_BRANCH="next"
 FGDATA_UNSTABLE_GIT_BRANCH="master"
@@ -476,6 +476,9 @@ if [[ "$(declare -p WHATTOBUILD)" =~ '['([0-9]+)']="SIMGEAR"' ]]; then
 	else
 		_gitUpdate $FGSG_UNSTABLE_GIT_BRANCH
 	fi
+
+	# Remove DDS warning message
+	sed -i s/SG_ALERT,\ \"Image/SG_WARN,\ \"Image/g simgear/scene/model/ModelRegistry.cxx
 	
 	if [ ! "$UPDATE" = "UPDATE" ]; then
 		if [ "$RECONFIGURE" = "y" ]; then
@@ -524,7 +527,7 @@ if [[ "$(declare -p WHATTOBUILD)" =~ '['([0-9]+)']="FGFS"' || "$(declare -p WHAT
        				mkdir -p build/flightgear
 	                        cd "$CBD"/build/flightgear
 				rm -f CMakeCache.txt
-				cmake 	-D CMAKE_BUILD_TYPE="Release" \
+				cmake 	-D CMAKE_BUILD_TYPE="Debug" \
 					-D ENABLE_RTI=OFF \
 					-D ENABLE_FLITE=ON \
 					-D CMAKE_INSTALL_PREFIX:PATH="$INSTALL_DIR_FGFS" \

fgdata_checkers.py

@@ -8,25 +8,15 @@ import os.path
 import re
 from collections import defaultdict
 import subprocess
-import multiprocessing
 import math
 import tarfile
-import tempfile
 import gzip
-import shutil
-import time
-try:
-    devnull=subprocess.DEVNULL#hide annoying nvcompress messages
-except (AttributeError,NameError):#pre-3.3 Python
-    devnull=None
 def path_join(*args):
-    """Unlike plain os.path.join, this always uses forward slashes, and doesn't add a trailing / if the last component is empty"""
-    return os.path.normpath(os.path.join(*args)).replace('\\','/')
+    """Unlike plain os.path.join, this doesn't add a trailing / if the last component is empty"""
+    return os.path.normpath(os.path.join(*args))
 def rfilelist(path,exclude_dirs=[]):
     """Dict of files/sizes in path, including those in any subdirectories (as relative paths)"""
     files=defaultdict(int)
-    if not os.path.exists(path):
-        return files
     dirs=[""]
     while dirs:
         cdir=dirs.pop()
@@ -75,16 +65,16 @@ def files_used(pattern,path,exclude_dirs=[],filelist=None,filetypes=None,relativ
             tex=texfind.search(line)
             if tex:
                 if relative_path:
-                    textures.append(os.path.normpath(path_join(os.path.dirname(file),tex.group(1).replace('\\','/'))).replace('\\','/'))
+                    textures.append(os.path.normpath(path_join(os.path.dirname(file),tex.group(1).replace('\\','/'))))
                 else:
-                    textures.append(os.path.normpath(tex.group(1).replace('\\','/')).replace('\\','/'))
+                    textures.append(os.path.normpath(tex.group(1).replace('\\','/')))
     return textures
 def find_unused_textures(basedir,output_lists=True,grep_check=False,output_rsync_rules=False,output_comparison_strips=False, output_removal_commands=False,return_used_noregions=False):
     """Checks if any textures are unused (wasting space), and if any textures are only available as .dds (not recommended in the source repository, as it is a lossy-compressed format)
 
 Set basedir to your fg-root, and enable the kind(s) of output you want:
 output_lists prints lists of unused textures, and of dds-only textures
-grep_check checks for possible use outside the normal directories; requires Unix shell and assumes side-by-side fgdata,flightgear,simgear
+grep_check checks for possible use outside the normal directories; requires Unix shell
 output_rsync_rules prints rsync rules for excluding unused textures from the release flightgear-data.  Warning: if you use this, re-run this script regularly, in case they start being used
 output_comparison_strips creates thumbnail strips, unused_duplicate.png/unused_dds.png/high_low.png, for visually checking whether same-name textures are the same (remove the unused one entirely) or different (move it to Unused); requires imagemagick or graphicsmagick
 output_removal_commands creates another script, delete_unused_textures.sh, which will remove unused textures when run in a Unix shell"""
@@ -127,9 +117,6 @@ output_removal_commands creates another script, delete_unused_textures.sh, which
     low_unneeded_nondup=low_unneeded&set(known_highlow_mismatch)
     def image_check_strip(basedir,index_fname,ilist1,ilist2=None,size=128):
         """Generate two rows of thumbnails, for easy visual comparison (between the two lists given, or if a single list is given, between low and high resolution)"""
-        if not ilist1:
-            print(index_fname," empty, skipping")
-            return
         if ilist2 is None:
             ipairs=[[path_join(basedir,'Textures',f),path_join(basedir,'Textures.high',f)] for f in ilist1]
         else:
@@ -175,12 +162,6 @@ output_removal_commands creates another script, delete_unused_textures.sh, which
     if output_comparison_strips:
         image_check_strip(basedir,"unused_duplicate.png",unused_duplicate,["Terrain"+f[14:] for f in unused_duplicate])
         image_check_strip(basedir,"unused_dds.png",unused_dds,[f[:-4]+".png" for f in unused_dds])
-        dds_skip=set(['Runway/rwy-normalmap.dds','Water/perlin-noise-nm.dds','Water/water_sine_nmap.dds','Water/waves-ver10-nm.dds'])
-        used_dds_withpng=set(f for f in (high_textures|low_textures) if (f[-4:]==".dds" and f[:-4]+".png" in (high_textures|low_textures)))-unused_dds-dds_skip
-        print(".dds omitted from comparison strip (normal maps etc): ",dds_skip)
-        #used_dds_withpng different: p{a,c}_taxiway.dds and possibly more runway markings (darker),sand{4,5,6}.dds(less high-freq noise),water.dds(more high-freq noise),grass_rwy.dds (has stripes),cropwood.dds,irrcrop.dds,shrub.dds,herbtundra.dds,cropgrass.dds(unrelated)
-        #water-reflection.{png,dds} really are plain white (so do match), not an alpha map
-        image_check_strip(basedir,"used_dds.png",used_dds_withpng,[f[:-4]+".png" for f in used_dds_withpng])
         image_check_strip(basedir,"high_low.png",high_textures&low_textures)
         #image_check_strip(basedir,"high_low2.png",[f for f in high_textures&low_textures if (f[0:14]=="Terrain.winter" or "_taxiway." in f or "lava" in f or "sand" in f)],size=512)#closer look at the doubtful cases
     if output_lists:
@@ -190,11 +171,11 @@ output_removal_commands creates another script, delete_unused_textures.sh, which
         unused_f=[os.path.basename(f) for f in unused]
         all_f=[os.path.basename(f) for f in (high_textures|low_textures)]
         print("\n\nPossible use outside main search:")#used to set false_positives
-        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=.git","-e","("+")|(".join(unused)+")",basedir,path_join(basedir,"../flightgear"),path_join(basedir,"../simgear")])#everywhere using full names
-        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=Textures.high","--exclude-dir=Models","--exclude-dir=Materials","--exclude-dir=Effects","--exclude-dir=.git","-e","("+")|(".join(all_f)+")",basedir,path_join(basedir,"../flightgear"),path_join(basedir,"../simgear")])#restricted (to avoid false positives from Terrain.winter vs Terrain) using filenames
-        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=Textures.high","--exclude-dir=Models","--exclude-dir=Materials","--exclude-dir=Effects","--exclude-dir=.git","-e",'[."\']dds',basedir,path_join(basedir,"../flightgear"),path_join(basedir,"../simgear")])#check for programmatic .png -> .dds swap; none found
+        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=.git","-e","("+")|(".join(unused)+")","/home/palmer/fs_dev/git/fgdata","/home/palmer/fs_dev/git/flightgear","/home/palmer/fs_dev/git/simgear"])#everywhere using full names
+        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=Textures.high","--exclude-dir=Models","--exclude-dir=Materials","--exclude-dir=Effects","--exclude-dir=.git","-e","("+")|(".join(all_f)+")","/home/palmer/fs_dev/git/fgdata","/home/palmer/fs_dev/git/flightgear","/home/palmer/fs_dev/git/simgear"])#restricted (to avoid false positives from Terrain.winter vs Terrain) using filenames
+        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=Textures.high","--exclude-dir=Models","--exclude-dir=Materials","--exclude-dir=Effects","--exclude-dir=.git","-e",'[."\']dds',"/home/palmer/fs_dev/git/fgdata","/home/palmer/fs_dev/git/flightgear","/home/palmer/fs_dev/git/simgear"])#check for programmatic .png -> .dds swap; none found
         print("\n\nUse of sourceless textures:")
-        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=.git","-e","("+")|(".join(sourceless)+")",basedir,path_join(basedir,"../flightgear"),path_join(basedir,"../simgear")])
+        subprocess.call(["grep","-r","-E","--exclude-dir=Aircraft","--exclude-dir=.git","-e","("+")|(".join(sourceless)+")","/home/palmer/fs_dev/git/fgdata","/home/palmer/fs_dev/git/flightgear","/home/palmer/fs_dev/git/simgear"])
     if output_rsync_rules:
         print("\n\nFull flightgear-data:\n")
         rsync_rules(basedir,unused)
@@ -259,171 +240,54 @@ def find_locally_unused_models(basedir):
     missing=set(f for f in (used_models|used_textures) if ((f.startswith('Models') and f not in models_allfiles.keys()) or not os.path.isfile(path_join(basedir,f))))
     print("used\n",sorted(used_models),"\nsize=",t_size(used_models),"\n\n",sorted(used_textures),"\nsize=",t_size(used_textures),"\n\nunused\n",sorted(unused),"\nsize=",t_size(unused),"\n\nmissing\n",sorted(missing),"\nsize=",t_size(missing))
 
-class FilesetSizes:
-    def __init__(self):
-        self.count=0
-        self.ncsize=0
-        self.csize=0
-    @property
-    def size(self):
-        if self.csize>0:
-            return self.csize
-        else:
-            return self.ncsize
-    def __str__(self):
-        if self.csize>0:
-            return "{0:10,} {1:15,} {2:15,}".format(self.count,self.ncsize,self.csize)
-        else:
-            return "{0:10,} {1:15,}".format(self.count,self.ncsize)
-    def __add__(self,other):
-        result=FilesetSizes()
-        result.count=self.count+other.count
-        result.ncsize=self.ncsize+other.ncsize
-        result.csize=self.csize+other.csize
-        return result
-def size_by_category(path,exclude_dirs,keyfn,compressed_size=False):
-    """Total size of files, in each category returned from keyfn"""
+def size_by_type(path,exclude_dirs=[]):
+    """Dict of total file size by file extension"""
     files=rfilelist(path,exclude_dirs)
-    result=defaultdict(FilesetSizes)
-    gzdir=tempfile.TemporaryDirectory()
-    gzpath=gzdir.name
-    gzcount=0
-    for file,size in files.items():
-        cat=keyfn(file,size)
-        if cat is None:
-            continue
-        result[cat].count=result[cat].count+1
-        result[cat].ncsize=result[cat].ncsize+size
-        if compressed_size:
-            try:
-                result[cat].targz.add(path_join(path,file))
-            except AttributeError:
-                result[cat].targzname=path_join(gzpath,str(gzcount)+".tar.gz")
-                result[cat].targz=tarfile.open(result[cat].targzname,mode="w:gz")
-                gzcount=gzcount+1
-                result[cat].targz.add(path_join(path,file))
-    if compressed_size:
-        for cat in result:
-            result[cat].targz.close()
-            result[cat].csize=os.path.getsize(result[cat].targzname)
-    return result
-def fgdata_size(path,by_size=False,exclude_dirs=None,compressed_size=False,min_size=1e6,include_aircraft=['UIUC','777','777-200','b1900d','CitationX','ZLT-NT','dhc2','Cub','sopwithCamel','f-14b','ASK13','bo105','Dragonfly','SenecaII','A6M2'],sections={"Aircraft/c172p":"Aircraft-base","Aircraft/Generic":"Aircraft-base","Aircraft/Instruments":"Aircraft-base","Aircraft/Instruments-3d":"Aircraft-base","Aircraft/ufo":"Aircraft-base","Aircraft/777":"Aircraft-777","Aircraft/777-200":"Aircraft-777","Aircraft":"Aircraft-other","Textures":"Textures","Textures.high":"Textures","AI/Aircraft":"AI-aircraft","AI":"AI-other","Scenery":"scenery","Models":"models"}):
-    if any('\\' in s for s in sections):
-        raise ValueError("sections always uses forward slashes")
-    if "" not in sections:
-        sections[""]="Other"
+    size_totals=defaultdict(int)
+    for filename,size in files.items():
+        file_ext=os.path.splitext(filename)[1]
+        if file_ext==".gz":
+            file_ext=os.path.splitext(os.path.splitext(filename)[0])[1]+file_ext
+        size_totals[file_ext]=size_totals[file_ext]+size
+    return size_totals
+def size_by_size(path,exclude_dirs=[],exts=[".png",".dds",".rgb"]):
+    """Dict of total file size by individual file size range, of given extensions (empty list for all files)"""
+    files=rfilelist(path,exclude_dirs)
+    size_totals=defaultdict(int)
+    for filename,size in files.items():
+        file_ext=os.path.splitext(filename)[1]
+        if (not exts) or (file_ext in exts):
+            size_totals[2**math.frexp(size)[1]]=size_totals[2**math.frexp(size)[1]]+size
+    return size_totals
+def fgdata_size(path,dirs_to_list=["AI/Aircraft","AI/Traffic","Aircraft","Models","Scenery","Textures","Textures.high"],exclude_dirs=None,compressed_size=False,num_types=3):
+    if dirs_to_list is None:
+        dirs_to_list=[d for d in os.listdir(path) if os.path.isdir(path_join(path,d))]
     if exclude_dirs is None:
         if os.path.exists(path_join(path,".git")):
-            exclude_dirs=[".git","Textures/Unused"]+[path_join("Aircraft",d) for d in os.listdir(path_join(path,"Aircraft")) if d not in ["c172p","ufo","Generic","Instruments","Instruments-3d"]+include_aircraft]
+            exclude_dirs=[".git","Aircraft"]
         else:
             exclude_dirs=[]
-    if by_size:
-        keyfn=lambda file,size: ("texture",math.frexp(size)[1]) if os.path.splitext(file)[1] in [".png",".dds",".rgb",".jpg"] else ("nontexture",math.frexp(size)[1])
-    else:
-        def keyfn(file,size):
-            file_ext=os.path.splitext(file)[1]
-            if file_ext==".gz":
-                file_ext=os.path.splitext(os.path.splitext(file)[0])[1]+file_ext
-            file0=file.replace('\\','/')
-            section=sections[max((s for s in sections if file0.startswith(s)),key=len)]
-            return (section,file_ext)
-    result=size_by_category(path,exclude_dirs,keyfn,compressed_size)
-    #print(result)
-    totals1=defaultdict(FilesetSizes)
-    totals2=defaultdict(FilesetSizes)
-    other=defaultdict(FilesetSizes)
-    for cat in result:
-        totals1[cat[0]]=totals1[cat[0]]+result[cat]
-        totals2[cat[1]]=totals2[cat[1]]+result[cat]
-        if (not by_size) and (result[cat].size<min_size):
-            other[cat[0]]=other[cat[0]]+result[cat]
-    print("{0:>10} {1:>15} {2:>15}".format("Count","Size","Compr.size"))
-    for cat0,r0 in sorted(totals1.items(),key=lambda p:-p[1].size):
-        print(cat0)
-        for cat,r in sorted([(c[1],r) for (c,r) in result.items() if (c[0]==cat0 and (by_size or r.size>min_size))],key=(lambda p:p[0]) if by_size else (lambda p:-p[1].size)):
-            print(r,cat)
-        if not by_size:
-            print(other[cat0],"other")
-        print(totals1[cat0],"all")
-    print("All")
-    for cat,r in sorted([(c,r) for (c,r) in totals2.items() if (by_size or r.size>min_size)],key=(lambda p:p[0]) if by_size else (lambda p:-p[1].size)):
-        print(r,cat)
-    if not by_size:
-        print(sum((r for r in totals2.values() if r.size<min_size),FilesetSizes()),"other")
-    print(sum(totals1.values(),FilesetSizes()),"all")
-def create_reduced_file(input_path,output_path,temp_path,cdir,file,downsample_this,compress_this,compress_names_find, compressed_format,texture_filetypes,binary_types,fclass):
-    """Process a single file in create_reduced_fgdata.  (Separate function to allow parallel processing)"""
-    retcode=0
-    if downsample_this or compress_this:
-        image_type=texture_filetypes[os.path.splitext(file)[1]]
-        output_image_type=compressed_format if compress_this else os.path.splitext(file)[1]
-        output_file=os.path.splitext(file)[0]+output_image_type
-        output_image_type=texture_filetypes[output_image_type]
-        if "{0}" in output_path and fclass=="base-textures":#downsampled in base-textures, full resolution in extra-textures
-            shutil.copy(path_join(input_path,cdir,file),path_join(output_path.format("extra-textures"),cdir,file))
-        if output_image_type=="DDS":# in Ubuntu, neither imagemagick nor graphicsmagick can write .dds
-            #doesn't work (for dds -> smaller dds): subprocess.call(["nvzoom","-s","0.5","-f","box",path_join(input_path,cdir,file),path_join(output_path.format(fclass),cdir,file)])
-            if subprocess.call(["convert",image_type+":"+path_join(input_path,cdir,file)]+(["-flip"] if ((image_type=="DDS")!=(output_image_type=="DDS")) else [])+(["-sample","50%"] if downsample_this else [])+[path_join(temp_path,cdir,os.path.splitext(file)[0]+".png")]):#fails on some DDS formats, so just copy them
-                shutil.copy(path_join(input_path,cdir,file),path_join(output_path.format(fclass),cdir,file))
-                print(path_join(cdir,file),"unsupported type, probably normal map")
-                if compress_this:
-                    raise TypeError#copy will have the wrong name
-            else:
-                try:
-                    image_properties=subprocess.check_output(["identify","-verbose",path_join(temp_path,cdir,os.path.splitext(file)[0]+".png")])
-                except subprocess.CalledProcessError as err:
-                    print("identify error on",path_join(cdir,file),"after ",["convert",image_type+":"+path_join(input_path,cdir,file)]+(["-flip"] if ((image_type=="DDS")!=(output_image_type=="DDS")) else [])+(["-sample","50%"] if downsample_this else [])+[path_join(temp_path,cdir,os.path.splitext(file)[0]+".png")],["identify","-verbose",path_join(temp_path,cdir,os.path.splitext(file)[0]+".png")],err)
-                    raise
-                has_alpha=b"Alpha" in image_properties
-                needs_alpha=has_alpha
-                if has_alpha and re.search(rb"Alpha:\s*min: 255 \(1\)",image_properties):
-                    print(path_join(cdir,file),"has always-255 alpha")
-                    needs_alpha=False
-                retcode=subprocess.call(["nvcompress","-bc3" if needs_alpha else "-bc1",path_join(temp_path,cdir,os.path.splitext(file)[0]+".png"),path_join(output_path.format(fclass),cdir,output_file)],stdout=devnull)
-        else:
-            retcode=subprocess.call(["convert",image_type+":"+path_join(input_path,cdir,file)]+(["-sample","50%"] if downsample_this else [])+[output_image_type+":"+path_join(output_path.format(fclass),cdir,output_file)])#we use sample rather than an averaging filter to not break mask/rotation/... maps
-    else:#not to be downsampled/compressed
-        if os.path.splitext(file)[1] in binary_types:#just copy
-            shutil.copy(path_join(input_path,cdir,file),path_join(output_path.format(fclass),cdir,file))
-        else:#texture name replacement
-            file_in=open(path_join(input_path,cdir,file),'rb')
-            file_out=open(path_join(output_path.format(fclass),cdir,file),'wb')
-            file_str=file_in.read(None)
-            file_in.close()
-            (file_strout,num_matches)=compress_names_find.subn(lambda mf: os.path.splitext(mf.group(0))[0]+(compressed_format.encode('utf-8')),file_str)
-            file_out.write(file_strout)
-            file_out.close()
-            #if ((os.path.splitext(file)[1] not in textureuser_types) and num_matches>0):
-                #print("Warning: ",num_matches," unexpected use(s) in ",path_join(cdir,file))
-            #if compress_names_find0.search(file_strout):
-                #print("Warning: unreplaced match(es) in ",path_join(cdir,file),compress_names_find0.search(file_strout).group(0))
-            """Warning: unreplaced match(es) in... correct rejections of match within a filename:
-                Aircraft/Instruments-3d/AN-APS-13.ac b'panel.png'
-                Aircraft/Instruments-3d/magneto-switch/mag_switch.ac b'black.png'
-                Nasal/canvas/map/Images/chart_symbols.svg b'wash.png'
-                Models/Airport/blast-deflector49m.ac b'generic.png'
-                Models/Airport/blast-deflector63m.ac b'generic.png'
-                Models/Industrial/oilrig09.ac b'yellow.png'
-                Models/Industrial/oilrig10.ac b'yellow.png'
-                Models/Industrial/oilrig09.ac.before-color-change b'yellow.png'
-                Models/Industrial/oilrig10.ac.before-color-change b'yellow.png'
-                Models/Maritime/Civilian/Tanker.ac b'black.png'
-                Models/Transport/flatcar.xml b'evergreen.png'
-                Models/Commercial/tower-grey-black.ac b'black.png'
-                Materials/base/materials-base.xml b'yellow.png'
-                
-                Warning: unexpected use(s) in...
-                Docs/README.local_weather.html (the only one that looke like an actual problem; hence, Docs is now skipped)
-                Nasal/canvas/map/Images/chart_symbols.svg (probably inkscape:export-filename, which are creator-specific absolute paths anyway, but now skipped)
-                oilrig09.ac.before-color-change,oilrig10.ac.before-color-change,stbd_coaming_panel.ac.bak (presumably backup files)
-                """
-    if retcode:
-        print("Error ",retcode," in ",path_join(cdir,file))
+    total_compressed_size=0
+    exclude_list=[[]]*len(dirs_to_list)+[dirs_to_list+exclude_dirs]+[exclude_dirs]
+    names_list=dirs_to_list+["other","all"]
+    for n,dir1 in enumerate(dirs_to_list+["",""]):
+        size_totals=size_by_type(path_join(path,dir1),exclude_list[n])
+        print(names_list[n],sorted(size_totals.items(),key=lambda x:-x[1])[:num_types],"total",sum(size_totals.values()))
+        if compressed_size:
+            if names_list[n]=="all":
+                print("compressed size",total_compressed_size)
+                continue
+            targz=tarfile.open("fgdata_sizetest_temp.tar.gz",mode="w:gz")
+            for file in rfilelist(path_join(path,dir1),exclude_list[n]):
+                targz.add(path_join(path,dir1,file))
+            targz.close()
+            print("compressed size",os.path.getsize("fgdata_sizetest_temp.tar.gz"))
+            total_compressed_size=total_compressed_size+os.path.getsize("fgdata_sizetest_temp.tar.gz")
 
-def create_reduced_fgdata(input_path,output_path,reject_positional_args=None,split_textures=False,exclude_parts=[],include_aircraft=['UIUC','777','777-200','b1900d','CitationX','ZLT-NT','dhc2','Cub','sopwithCamel','f-14b','ASK13','bo105','Dragonfly','SenecaII','A6M2'],dirs_to_downsample=(),dirs_to_compress=(),compressed_format=".dds",downsample_min_filesize=1e5,compress_min_filesize=3e4,use_ready_compressed=True):
+def create_reduced_fgdata(input_path,output_path,split_textures=True,exclude_parts=[],include_aircraft=['UIUC','777','777-200','b1900d','CitationX','ZLT-NT','dhc2','Cub','sopwithCamel','f-14b','ASK13','bo105','Dragonfly','SenecaII','A6M2'],dirs_to_downsample=(),downsample_min_filesize=30000):
     """Create a smaller, reduced-quality flightgear-data package
-Can downsample textures 50%, change texture format, and/or omit sections (region-specific textures, aircraft, AI traffic)
-Downsampling and format change require imagemagick or graphicsmagick (for convert) and libnvtt-bin (for nvcompress)
+Can downsample textures 50% and/or omit sections
+Requires Unix shell; downsampling requires imagemagick or graphicsmagick (for convert) and libnvtt-bin (for nvcompress)
 
 Optional parts, use exclude_parts to omit:
 ai: no background traffic, but tankers etc do still work
@@ -432,39 +296,13 @@ extra-textures (requires split_textures=True): no region-specific textures
 The c172p and ufo are always included; other aircraft are added by include_aircraft
 
 Texture downsampling: textures in dirs_to_downsample and larger than downsample_min_filesize downsampled 50%
-Texture format conversion: textures in dirs_to_compress and larger than compress_min_filesize converted to compressed_format
-use_ready_compressed determines what happens if a same-basename file in compressed_format already exists: True uses the already-compressed one, False uses the uncompressed one, None keeps both
-Suggested dirs_to_downsample:
-3.2: ('Textures.high/Terrain','Textures.high/Trees','Textures.high/Terrain.winter','AI/Aircraft','Models')
-3.3: ('Textures/Terrain','Textures/Trees','Textures/Terrain.winter','AI/Aircraft','Models')
-To do "everything" (a few are always skipped due to potential breakage), use dirs_to_compress=('',)
+Example: dirs_to_downsample=("Textures.high/Terrain","Textures.high/Trees","Textures.high/Terrain.winter","AI/Aircraft","Models"),downsample_min_filesize=30000
 
-To put each section in its own directory (e.g. for building a Debian-style flightgear-data-* set of packages) use {0} in output_path, e.g.
-python3 -c "import fgdata_checkers; fgdata_checkers.create_reduced_fgdata(input_path='/home/rnpalmer/fs_dev/git/fgdata',output_path='/home/rnpalmer/fs_dev/flightgear/data_split/debian/flightgear-data-{0}/usr/share/games/flightgear',include_aircraft=['UIUC','b1900d','CitationX','ZLT-NT','dhc2','Cub','sopwithCamel','f-14b','ASK13','bo105','Dragonfly','SenecaII','A6M2'])"
+To put each section in its own directory use {0} in output_path, e.g.
+python3 -c "import fgdata_checkers; fgdata_checkers.create_reduced_fgdata(input_path='/home/palmer/fs_dev/git/fgdata',output_path='/home/palmer/fs_dev/flightgear/data_split/debian/flightgear-data-{0}/usr/share/games/flightgear',include_aircraft=['UIUC','b1900d','CitationX','ZLT-NT','dhc2','Cub','sopwithCamel','f-14b','ASK13','bo105','Dragonfly','SenecaII','A6M2'])"
 This creates separate preferences-regions.xml and preferences-noregions.xml files for with and without regional textures; you need to handle symlinking preferences.xml to the correct one
 """
-    start_time=time.time()
-    if reject_positional_args is not None:
-        raise TypeError("Keyword arguments only please: this is not a stable API")
-    if use_ready_compressed not in (True,False,None):
-        raise TypeError("invalid use_ready_compressed setting")
-    texture_filetypes={".png":"PNG",".dds":"DDS",".jpg":"JPEG"}#,".rgb":"SGI" loses cloud transparency
-    if compressed_format not in texture_filetypes:
-        raise ValueError("Invalid compressed_format (include the .)")
-    textureuser_types={".eff",".xml",".ac",".nas"}
-    binary_types={".png",".dds",".rgb",".RGB",".jpg",".wav",".WAV",".btg.gz",".zip",".tar.gz"}#don't search these for texture name replacement
-    """Textures named directly in flightgear/simgear code:
-gui/images/shadow.png,gui/cursor-spin-cw.png (probably safest to treat this as gui/*, they're all small)
-Textures/Globe/world.topo.bathy.200407.3x4096x2048.png
-Textures/buildings.png,Textures/buildings-lightmap.png
-Textures/Sky/*
-Textures/Splash*.png
-unknown.rgb (probably Textures/ or Textures/Terrain/, neither exists)
-Aircraft/Instruments/Textures/nd-symbols.png (doesn't actually exist),Aircraft/Instruments/Textures/compass-ribbon.rgb,Aircraft/Instruments/Textures/od_wxradar.rgb,Aircraft/Instruments/Textures/od_wxradar.rgb,Aircraft/Instruments/Textures/wxecho.rgb,Aircraft/Instruments/Textures/od_groundradar.rgb (doesn't actually exist)
-also, Aircraft/{Instruments,Instruments-3d,Generic} may be used by downloaded aircraft, and Docs images are used in .html
-Nasal (Canvas map) probably wouldn't break anything, but guessing it's a bad idea visually"""
-    no_compress_pattern=re.compile(r'mask|light|relief|nmap|nm\.|normal|dudv|^Splash[0-9].png$|^buildings.png$|^buildings-lightmap.png$|^world.topo.bathy.200407.3x4096x2048.png$|^ocean_depth_1.png$')#edge blurring from lossy compression may break masks (converting ocean_depth_1.png is known to give black water near shore at effects>=4), and this script doesn't know how to create DDS normal maps
-    no_compress_dirs=("gui","Docs","webgui","Nasal","Textures/Sky","Aircraft/Instruments","Aircraft/Instruments-3d","Aircraft/Generic")
+    texture_filetypes={".png":"PNG",".dds":"DDS"}#,".rgb":"SGI" loses cloud transparency
     exclude_dirs=[".git","Textures/Unused"]
     exclude_unnamed_subdirs=["Aircraft"]#these are a separate mechanism from subtree_class/exclude_parts mostly to save time (subtree_class still fully scans excluded directories because the class may change again further down the tree, e.g. AI/Aircraft ai -> performancedb.xml base; these don't)
     subtree_class={"Aircraft/c172p":"base","Aircraft/Generic":"base","Aircraft/Instruments":"base","Aircraft/Instruments-3d":"base","Aircraft/ufo":"base","Textures":"textures","Textures.high":"textures","AI/Aircraft":"ai","AI/Traffic":"ai","AI/Aircraft/performancedb.xml":"base","Scenery":"scenery","Models":"models"}
@@ -476,41 +314,12 @@ Nasal (Canvas map) probably wouldn't break anything, but guessing it's a bad ide
         base_texture_files=[]
         for t in find_unused_textures(input_path,return_used_noregions=True):
             base_texture_files.extend([path_join("Textures",t),path_join("Textures.high",t)])
-    #no longer a significant problem with exclude_dirs: if os.path.exists(path_join(input_path,".git")):
-        #print(input_path,"appears to be a git clone; this will work, but the result will be larger than starting from a standard flightgear-data package.\nTo create this use (adjusting paths as necessary) rsync -av --filter=\"merge /home/rnpalmer/fs_dev/git/fgmeta/base-package.rules\" ~/fs_dev/git/fgdata ~/fs_dev/flightgear/data_full")
+    #if os.path.exists(path_join(input_path,".git")):
+        #print(input_path,"appears to be a git clone; this will work, but the result will be larger than starting from a standard flightgear-data package.\nTo create this use (adjusting paths as necessary) rsync -av --filter=\"merge /home/palmer/fs_dev/git/fgmeta/base-package.rules\" ~/fs_dev/git/fgdata ~/fs_dev/flightgear/data_full")
     if os.path.exists(output_path.format("base")) and os.listdir(output_path.format("base")):
         print("output path",output_path,"non-empty, aborting to avoid data loss\nIf you did want to lose its previous contents, run:\nrm -r",output_path,"\nthen re-run this script")
         return
-    if compressed_format==".jpg":
-        print("Warning: selected compression format does not support transparency")
-    compress_names=set()
-    if dirs_to_compress:#need this preliminary pass to get names to change in .xml,etc
-        no_compress_names=set()
-        dirs={"":"base"}
-        while dirs:
-            cdir,cclass=dirs.popitem()
-            cdirfiles=os.listdir(path_join(input_path,cdir))
-            for file in cdirfiles:
-                fclass=subtree_class.get(path_join(cdir,file),cclass)
-                if os.path.isdir(path_join(input_path,cdir,file)):
-                    if (path_join(cdir,file) not in exclude_dirs) and (cdir not in exclude_unnamed_subdirs or path_join(cdir,file) in subtree_class):
-                        dirs[path_join(cdir,file)]=fclass
-                else:#file
-                    compress_this=cdir.startswith(dirs_to_compress) and (os.path.splitext(file)[1] in texture_filetypes) and (os.path.splitext(file)[1]!=compressed_format) and (os.path.getsize(path_join(input_path,cdir,file))>compress_min_filesize) and not no_compress_pattern.search(file) and not cdir.startswith(no_compress_dirs) and (file not in no_compress_names) and ((use_ready_compressed is not None) or not os.path.exists(path_join(input_path,cdir,os.path.splitext(file)[0]+compressed_format)))
-                    if compress_this:
-                        compress_names.add(file)
-                    else:
-                        no_compress_names.add(file)
-                        compress_names.discard(file)#if there are two with the same name in different directories, compress both or neither, to simplify name replacement
-    compress_names_find=re.compile(('(?<=["\'>/\\\\ \\n])('+'|'.join(re.escape(f) for f in compress_names)+')($|(?=["\'< \\n]))').encode('utf-8'))
-    compress_names_find0=re.compile(('|'.join(re.escape(f) for f in compress_names)).encode('utf-8'))
-    #print(compress_names,"\n\n",no_compress_names,"\n\n",'(?<=["\'>/\\\\ \\n])('+'|'.join(re.escape(f) for f in compress_names)+')($|(?=["\'< \\n]))',"\n\n",'|'.join(re.escape(f) for f in compress_names),"\n\n")
-    print("Starting conversion...",len(compress_names),"files to change format, runtime so far=",int(time.time()-start_time),"sec")
     dirs={"":"base"}
-    subprocess_pool=multiprocessing.Pool(processes=8)
-    subprocess_list=[]
-    temp_dir=tempfile.TemporaryDirectory()
-    temp_path=temp_dir.name
     while dirs:
         cdir,cclass=dirs.popitem()
         cdirfiles=os.listdir(path_join(input_path,cdir))
@@ -528,29 +337,25 @@ Nasal (Canvas map) probably wouldn't break anything, but guessing it's a bad ide
                 if fclass in exclude_parts:
                     continue
                 if not os.path.exists(path_join(output_path.format(fclass),cdir)):
-                    os.makedirs(path_join(output_path.format(fclass),cdir))#errors out if the directory does exist, so calling it in the per-file subprocess would be a race condition
-                if "{0}" in output_path and fclass=="base-textures" and not os.path.exists(path_join(output_path.format("extra-textures"),cdir)):#downsampled in base-textures, full resolution in extra-textures
-                    os.makedirs(path_join(output_path.format("extra-textures"),cdir))
-                if not os.path.exists(path_join(temp_path,cdir)):
-                    os.makedirs(path_join(temp_path,cdir))
-                downsample_this=(cdir.startswith(dirs_to_downsample)) and (os.path.splitext(file)[1] in texture_filetypes) and (os.path.getsize(path_join(input_path,cdir,file))>downsample_min_filesize)
-                compress_this=(file in compress_names)
-                if compress_this and use_ready_compressed==True and os.path.exists(path_join(input_path,cdir,os.path.splitext(file)[0]+compressed_format)):
-                    continue
-                if (not compress_this) and use_ready_compressed==False and os.path.splitext(file)[1]==compressed_format and any((os.path.exists(path_join(input_path,cdir,os.path.splitext(file)[0]+f)) and os.path.splitext(file)[0]+f in compress_names) for f in texture_filetypes if f!=os.path.splitext(file)[1]):
-                    continue
-                subprocess_list.append(subprocess_pool.apply_async(create_reduced_file,args=(input_path,output_path,temp_path,cdir,file,downsample_this,compress_this,compress_names_find, compressed_format,texture_filetypes,binary_types,fclass)))
-    print(len(subprocess_list),"file tasks started...runtime so far=",int(time.time()-start_time),"sec\n(Not a linear progress indicator: they are different lengths.)")
-    last_report=time.time()-100#print first report immediately
-    for s0 in subprocess_list:
-        if time.time()>last_report+60:
-            print("Waiting for",len([s for s in subprocess_list if not s.ready()]),"file tasks...runtime so far=",int(time.time()-start_time),"sec")
-            last_report=time.time()
-        s0.get()
-    subprocess_pool.close()
-    subprocess_pool.join()
+                    subprocess.call(["mkdir","-p",path_join(output_path.format(fclass),cdir)])
+                if (cdir.startswith(dirs_to_downsample)) and (os.path.splitext(file)[1] in texture_filetypes) and (os.path.getsize(path_join(input_path,cdir,file))>downsample_min_filesize):
+                    image_type=texture_filetypes[os.path.splitext(file)[1]]
+                    if "{0}" in output_path and fclass=="base-textures":#downsampled in base-textures, full resolution in extra-textures
+                        if not os.path.exists(path_join(output_path.format("extra-textures"),cdir)):
+                            subprocess.call(["mkdir","-p",path_join(output_path.format("extra-textures"),cdir)])
+                        subprocess.call(["cp",path_join(input_path,cdir,file),path_join(output_path.format("extra-textures"),cdir,file)])
+                    if image_type=="DDS":# in Ubuntu, neither imagemagick nor graphicsmagick can write .dds
+                        #doesn't work subprocess.call(["nvzoom","-s","0.5","-f","box",path_join(input_path,cdir,file),path_join(output_path.format(fclass),cdir,file)])
+                        if subprocess.call(["convert",image_type+":"+path_join(input_path,cdir,file),"-sample","50%","temp_reduced_size.png"]):#fails on normal maps, so just copy them
+                            subprocess.call(["cp",path_join(input_path,cdir,file),path_join(output_path.format(fclass),cdir,file)])
+                        else:
+                            subprocess.call(["nvcompress","-bc3","temp_reduced_size.png",path_join(output_path.format(fclass),cdir,file)])
+                    else:
+                        subprocess.call(["convert",image_type+":"+path_join(input_path,cdir,file),"-sample","50%",image_type+":"+path_join(output_path.format(fclass),cdir,file)])#we use sample rather than an averaging filter to not break mask/rotation/... maps
+                else:#not to be downsampled
+                    subprocess.call(["cp",path_join(input_path,cdir,file),path_join(output_path.format(fclass),cdir,file)])
     if "{0}" in output_path:
-        os.rename(path_join(output_path.format("base"),"preferences.xml"),path_join(output_path.format("base"),"preferences-regions.xml"))
+        subprocess.call(["mv",path_join(output_path.format("base"),"preferences.xml"),path_join(output_path.format("base"),"preferences-regions.xml")])
     if "extra-textures" in exclude_parts or "{0}" in output_path:
         prefs_in=open(path_join(input_path,"preferences.xml"),'r')
         prefs_out=open(path_join(output_path.format("base"),"preferences-noregions.xml" if "{0}" in output_path else "preferences.xml"),'w')
@@ -559,7 +364,6 @@ Nasal (Canvas map) probably wouldn't break anything, but guessing it's a bad ide
         prefs_str=prefs_str.replace("Materials/regions/materials.xml","Materials/default/materials.xml")#turn off regional textures
         prefs_out.write(prefs_str)
         prefs_out.close()
-    print("Total runtime=",int(time.time()-start_time),"sec")
 def check_text_encoding(path,filelist=None,binary_types=(".png",".dds",".rgb",".RGB",".jpg",".wav",".WAV",".btg.gz",".xcf.gz",".xcf",".XCF","Thumbs.db",".blend",".bmp",".gif", ".3ds",".3DS",".pdf",".ttf",".txf",".htsvoice",".odt",".ods",".xls",".mp3",".zip",".tar.gz"),exclude_dirs=[".git","Timezone"]):
     """filelist is intended for quick testing: see fgdata_nonascii_filelist.py"""
     def err_context(err):

hudson_build_release.sh

@@ -60,25 +60,15 @@ cp flightgear-*.tar.bz2 ../output/.
 echo "Assembling base package"
 cd $WORKSPACE
 
-rm -rf base_package
+echo "Syncing base packages files from sphere.telascience.org"
 
 # a: archive mode
 # z: compress
 # delete: 'delete extraneous files from dest dirs'; avoid bug 1344
 # filter: use the rules in our rules file
-
-echo "Copying FGData files"
-rsync -a --delete \
+rsync -az --delete \
  --filter 'merge base-package.rules' \
-  fgdata base_package
+ -e ssh jturner@sphere.telascience.org:/home/jturner/fgdata .
 
-echo "Syncing extended data"
-rsync -a --filter 'merge aircraft.rules' /home/jenkins/fgdata_340/ extended-data
+tar cjf output/FlightGear-$VERSION-data.tar.bz2 fgdata/
 
-echo "Copying extended data"
-rsync -a extended-data/ base_package/fgdata
-
-echo "Creating tar archive"
-pushd base_package
-tar cjf $WORKSPACE/output/FlightGear-$VERSION-data.tar.bz2 fgdata/
-popd

hudson_mac_build_release.sh

@@ -33,7 +33,7 @@ popd
 ################################################################################
 echo "Starting on FlightGear"
 pushd fgBuild
-cmake -DCMAKE_INSTALL_PREFIX:PATH=$WORKSPACE/dist -DENABLE_QT=1 -G Xcode ../flightgear
+cmake -DCMAKE_INSTALL_PREFIX:PATH=$WORKSPACE/dist -G Xcode ../flightgear
 
 xcodebuild -configuration RelWithDebInfo -target install  build
 
@@ -47,6 +47,28 @@ popd
 chmod +x $WORKSPACE/dist/bin/osgversion
 
 ################################################################################
+echo "Building Macflightgear launcher"
+
+#OSX_TARGET="10.6"
+# JMT - disabling setting the sysroot since it's breaking things on 
+# current build slave. Sinc ethe slave runs 10.6 natively, we don't 
+# actually need to set these. Real solution would be to use a proper
+# build system for the Mac-launcher of course.
+# -mmacosx-version-min=$OSX_TARGET -isysroot $SDK_PATH
+
+pushd maclauncher/FlightGearOSX
+
+# compile the stub executable
+gcc -o FlightGear main.m \
+    -framework Cocoa -framework RubyCocoa -framework Foundation -framework AppKit
+
+popd
+
+################################################################################
+echo "Syncing base packages files from sphere.telascience.org"
+rsync -avz --filter 'merge base-package.rules' \
+ -e ssh jturner@sphere.telascience.org:/home/jturner/fgdata .
+
 # run the unlock script now - we need to do this right before code-signing,
 # or the keychain may automatically re-lock after some period of time
 unlock-keychain.sh

hudson_mac_package_release.rb

@@ -24,13 +24,13 @@ puts "osgVersion=#{osgVersion}, so-number=#{$osgSoVersion}"
 
 def fix_install_names(object)
   #puts "fixing install names for #{object}"
-
+  
   $osgLibs.each do |l|
     oldName = "lib#{l}.#{$osgSoVersion}.dylib"
     newName = "@executable_path/../Frameworks/#{oldName}"
     `install_name_tool -change #{oldName} #{newName} #{object}`
   end
-
+  
   oldName = "libOpenThreads.#{$openThreadsSoVersion}.dylib"
   newName= "@executable_path/../Frameworks/#{oldName}"
   `install_name_tool -change #{oldName} #{newName} #{object}`
@@ -40,26 +40,20 @@ $prefixDir=Dir.pwd + "/dist"
 dmgDir=Dir.pwd + "/image"
 srcDir=Dir.pwd + "/flightgear"
 
+def code_sign(path)
+  puts "Signing #{path}"
+  `codesign -s "#{$codeSignIdentity}" #{path}`
+end
+
 puts "Erasing previous image dir"
 `rm -rf #{dmgDir}`
 
-bundle=dmgDir + "/FlightGear.app"
-
-# run macdeployt before we rename the bundle, otherwise it
-# can't find the bundle executable
-puts "Running macdeployqt on the bundle to copy Qt libraries"
-`macdeployqt #{$prefixDir}/fgfs.app`
-
-puts "Moving & renaming app bundle"
-`mkdir -p #{dmgDir}`
-`mv #{$prefixDir}/fgfs.app #{bundle}`
-
 bundle=dmgDir + "/FlightGear.app"
 contents=bundle + "/Contents"
 macosDir=contents + "/MacOS"
 $frameworksDir=contents +"/Frameworks"
 resourcesDir=contents+"/Resources"
-osgPluginsDir=contents+"/PlugIns/osgPlugins"
+osgPluginsDir=contents+"/PlugIns/osgPlugins-#{osgVersion}"
 
 # for writing copyright year to Info.plist
 t = Time.new
@@ -76,16 +70,14 @@ puts "Creating directory structure"
 `mkdir -p #{resourcesDir}`
 `mkdir -p #{osgPluginsDir}`
 
-# fix install names on the primary executable
-fix_install_names("#{macosDir}/fgfs")
-
-puts "Copying auxilliary binaries"
-bins = ['fgjs', 'fgcom']
+puts "Copying binaries"
+`cp #{$prefixDir}/fgfs.app/Contents/MacOS/fgfs #{macosDir}/fgfs`
+bins = ['fgjs', 'fgcom', 'fgviewer']
 bins.each do |b|
   if !File.exist?("#{$prefixDir}/bin/#{b}")
     next
   end
-
+  
   outPath = "#{macosDir}/#{b}"
   `cp #{$prefixDir}/bin/#{b} #{outPath}`
   fix_install_names(outPath)
@@ -103,11 +95,19 @@ libFile = "libOpenThreads.#{$openThreadsSoVersion}.dylib"
 `cp #{$prefixDir}/lib/#{libFile} #{$frameworksDir}`
 
 $osgPlugins.each do |p|
-  pluginFile = "osgdb_#{p}.dylib"
-  `cp #{$prefixDir}/lib/osgPlugins/#{pluginFile} #{osgPluginsDir}`
+  pluginFile = "osgdb_#{p}.so"
+  `cp #{$prefixDir}/lib/osgPlugins-#{osgVersion}/#{pluginFile} #{osgPluginsDir}`
   fix_install_names("#{osgPluginsDir}/#{pluginFile}")
 end
 
+# Macflightgear launcher
+puts "Copying Macflightgear launcher files"
+
+Dir.chdir "maclauncher/FlightGearOSX" do
+  `cp FlightGear #{macosDir}`
+  `rsync -a *.rb *.lproj *.sh *.tiff *.html #{resourcesDir}`
+end
+
 if File.exist?("#{$prefixDir}/bin/fgcom-data")
   puts "Copying FGCom data files"
   `ditto #{$prefixDir}/bin/fgcom-data #{resourcesDir}/fgcom-data`
@@ -129,14 +129,16 @@ File.open("#{contents}/Info.plist", 'w') { |f|
 `mv fgdata/Docs/getstart.pdf "#{dmgDir}/Getting Started.pdf"`
 
 puts "Copying base package files into the image"
-`rsync -a --filter 'merge base-package.rules' fgdata/ #{resourcesDir}/data`
+`rsync -a fgdata/ #{resourcesDir}/data`
 
-# work-around for 3.4.1: copy aircraft/scenery files too
-`rsync -a extended-data/ #{resourcesDir}/data`
-
-# code sign the entire bundle once complete - v2 code-signing
-puts "Signing #{bundle}"
-`codesign --deep -s "#{$codeSignIdentity}" #{bundle}`
+# code sign all executables in MacOS dir. Do this last since reource
+# changes will invalidate the signature!
+Dir.foreach(macosDir) do |b|
+    if b == '.' or b == '..' then
+        next
+    end
+  code_sign("#{macosDir}/#{b}")
+end
 
 puts "Creating DMG"
 

hudson_win_package_release.bat

@@ -29,9 +29,10 @@ ECHO #define OSGVersion "%OSG_VERSION%" >> InstallConfig.iss
 ECHO #define OSGSoNumber "%OSG_SO_NUMBER%" >> InstallConfig.iss
 ECHO #define OTSoNumber "%OT_SO_NUMBER%" >> InstallConfig.iss
 
-REM set DATA_FILE=FlightGear-%FLIGHTGEAR_VERSION%-data
+set DATA_FILE=FlightGear-%FLIGHTGEAR_VERSION%-data
+
 REM extract the data files
-REM 7z e -aoa %DATA_FILE%.tar.bz2 && 7z x -aoa %DATA_FILE%.tar
+7z e -aoa %DATA_FILE%.tar.bz2 && 7z x -aoa %DATA_FILE%.tar
 
 REM run Inno-setup!
 REM use iscc instead of compil32 for better error reporting

sgprops.py

@@ -159,12 +159,11 @@ class PropsHandler(handler.ContentHandler):
         if (name == 'PropertyList'):
             return
             
+        index = 0
         if 'n' in attrs.keys():
             index = int(attrs['n'])
-            self._current = self._current.getChild(name, index, create=True)
-        else:
-            self._current = self._current.addChild(name)
             
+        self._current = self._current.getChild(name, index, create=True)
             
         if 'include' in attrs.keys():
             self.handleInclude(attrs['include'])

webhooks.xml

@@ -1,19 +0,0 @@
-<?xml version="1.0"?>
-<!--
-  The <regex> field is evaluated by preg_match
-  http://php.net/manual/en/function.preg-match.php
--->
-<PropertyList>
-  <webhook>
-    <regex>/^Merge branch/</regex>
-    <description>Detect merge branch when rebase has been ommited</description>
-  </webhook>
-  <webhook>
-    <regex>/^test$/</regex>
-    <description>Detect for dummy message</description>
-  </webhook>
-  <webhook>
-    <regex>/blabla/</regex>
-    <description>Detect for bad commit description</description>
-  </webhook>
-</PropertyList>