Reorganize catalog modules
- Move the support modules inside python3-flightgear/flightgear/ and remove their shebang, if any. - Accordingly adapt the import statements. - Change shebangs from e.g. '#! /usr/bin/python' to '#! /usr/bin/env python3'. - Small changes to make Python 3 happy with all scripts. catalog/check_aircraft.py should be run under Python 3 from now on.
This commit is contained in:
Florent Rougon
@@ -22,6 +22,9 @@ The script can be run directly from this directory, or the script and its
|
||||
modules can be copied together and run from any location. The steps to use
|
||||
these are:
|
||||
|
||||
* Have something like `export PYTHONPATH="/path/to/fgmeta/python3-flightgear"`
|
||||
in your shell setup or use a .pth file (see `python3-flightgear/README.md`
|
||||
for more details).
|
||||
* Create an output directory where the catalog and zip files will be located.
|
||||
* Copy the configuration files `catalog.config.xml`, `template.xml`, and
|
||||
`zip-excludes.lst` from one of the `*catalog*` example directories into the
|
||||
|
||||
@@ -1,430 +0,0 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
from fnmatch import fnmatch, translate
|
||||
import lxml.etree as ET
|
||||
import os
|
||||
from os.path import exists, join, relpath
|
||||
from os import F_OK, access, walk
|
||||
import re
|
||||
import sgprops
|
||||
import sys
|
||||
import catalogTags
|
||||
import zipfile
|
||||
|
||||
CATALOG_VERSION = 4
|
||||
quiet = False
|
||||
verbose = False
|
||||
|
||||
# The Python version.
|
||||
PY_VERSION = sys.version_info[0]
|
||||
|
||||
# Python 2 and 3 compatibility.
|
||||
if PY_VERSION == 3:
|
||||
long = int
|
||||
|
||||
|
||||
def warning(msg):
|
||||
if not quiet:
|
||||
print(msg)
|
||||
|
||||
def log(msg):
|
||||
if verbose:
|
||||
print(msg)
|
||||
|
||||
# xml node (robust) get text helper
|
||||
def get_xml_text(e):
|
||||
if e != None and e.text != None:
|
||||
return e.text
|
||||
else:
|
||||
return ''
|
||||
|
||||
# return all available aircraft information from the set file as a
|
||||
# dict
|
||||
def scan_set_file(aircraft_dir, set_file, includes):
|
||||
base_file = os.path.basename(set_file)
|
||||
base_id = base_file[:-8]
|
||||
set_path = os.path.join(aircraft_dir, set_file)
|
||||
|
||||
includes.append(aircraft_dir)
|
||||
root_node = sgprops.readProps(set_path, includePaths = includes)
|
||||
|
||||
if not root_node.hasChild("sim"):
|
||||
return None
|
||||
|
||||
sim_node = root_node.getChild("sim")
|
||||
if sim_node == None:
|
||||
return None
|
||||
|
||||
# allow -set.xml files to specifcially exclude themselves from
|
||||
# the creation process, by setting <exclude-from-catalog>true</>
|
||||
if (sim_node.getValue("exclude-from-catalog", False) == True):
|
||||
return None
|
||||
|
||||
variant = {}
|
||||
name = sim_node.getValue("description", None)
|
||||
if (name == None or len(name) == 0):
|
||||
warning("Set file " + set_file + " is missing a <description>, skipping")
|
||||
return None
|
||||
|
||||
variant['name'] = name
|
||||
variant['status'] = sim_node.getValue("status", None)
|
||||
|
||||
if sim_node.hasChild('authors'):
|
||||
# aircraft has structured authors data, handle that
|
||||
variant['authors'] = sim_node.getChild('authors')
|
||||
|
||||
# can have legacy author tag alongside new strucutred data for
|
||||
# backwards FG compatability
|
||||
if sim_node.hasChild('author'):
|
||||
variant['author'] = sim_node.getValue("author", None)
|
||||
|
||||
if sim_node.hasChild('maintainers'):
|
||||
variant['maintainers'] = sim_node.getChild('maintainers')
|
||||
|
||||
if sim_node.hasChild('urls'):
|
||||
variant['urls'] = sim_node.getChild('urls')
|
||||
|
||||
if sim_node.hasChild('long-description'):
|
||||
variant['description'] = sim_node.getValue("long-description", None)
|
||||
variant['id'] = base_id
|
||||
|
||||
# allow -set.xml files to declare themselves as primary.
|
||||
# we use this avoid needing a variant-of in every other -set.xml
|
||||
variant['primary-set'] = sim_node.getValue('primary-set', False)
|
||||
|
||||
# extract and record previews for each variant
|
||||
if sim_node.hasChild('previews'):
|
||||
variant['previews'] = extract_previews(sim_node.getChild('previews'), aircraft_dir)
|
||||
|
||||
if sim_node.hasChild('rating'):
|
||||
variant['rating'] = sim_node.getChild("rating")
|
||||
|
||||
if sim_node.hasChild('tags'):
|
||||
variant['tags'] = extract_tags(sim_node.getChild('tags'), set_file)
|
||||
|
||||
if sim_node.hasChild('thumbnail'):
|
||||
variant['thumbnail'] = sim_node.getValue("thumbnail", None)
|
||||
|
||||
variant['variant-of'] = sim_node.getValue("variant-of", None)
|
||||
|
||||
if sim_node.hasChild('minimum-fg-version'):
|
||||
variant['minimum-fg-version'] = sim_node.getValue('minimum-fg-version', None)
|
||||
|
||||
#print(" %s" % variant)
|
||||
return variant
|
||||
|
||||
def extract_previews(previews_node, aircraft_dir):
|
||||
result = []
|
||||
for node in previews_node.getChildren("preview"):
|
||||
previewType = node.getValue("type", None)
|
||||
previewPath = node.getValue("path", None)
|
||||
|
||||
# check path exists in base-name-dir
|
||||
fullPath = os.path.join(aircraft_dir, previewPath)
|
||||
if not os.path.isfile(fullPath):
|
||||
warning("Bad preview path, skipping:" + fullPath)
|
||||
continue
|
||||
result.append({'type':previewType, 'path':previewPath})
|
||||
|
||||
return result
|
||||
|
||||
def extract_tags(tags_node, set_path):
|
||||
result = []
|
||||
for node in tags_node.getChildren("tag"):
|
||||
tag = node.value
|
||||
# check tag is in the allowed list
|
||||
if not catalogTags.isValidTag(tag):
|
||||
warning("Unknown tag value:" + tag + " in " + set_path)
|
||||
result.append(tag)
|
||||
|
||||
return result
|
||||
|
||||
# scan all the -set.xml files in an aircraft directory. Returns a
|
||||
# package dict and a list of variants.
|
||||
def scan_aircraft_dir(aircraft_dir, includes):
|
||||
setDicts = []
|
||||
primaryAircraft = []
|
||||
package = None
|
||||
|
||||
files = os.listdir(aircraft_dir)
|
||||
for file in sorted(files, key=lambda s: s.lower()):
|
||||
if file.endswith('-set.xml'):
|
||||
# print('trying: %s' % file)
|
||||
try:
|
||||
d = scan_set_file(aircraft_dir, file, includes)
|
||||
if d == None:
|
||||
continue
|
||||
except:
|
||||
print("Skipping set file since couldn't be parsed: %s %s" % os.path.join(aircraft_dir, file), sys.exc_info()[0])
|
||||
continue
|
||||
|
||||
setDicts.append(d)
|
||||
if d['primary-set']:
|
||||
# ensure explicit primary-set aircraft goes first
|
||||
primaryAircraft.insert(0, d)
|
||||
elif d['variant-of'] == None:
|
||||
primaryAircraft.append(d)
|
||||
|
||||
# print(setDicts)
|
||||
if len(setDicts) == 0:
|
||||
return None, None
|
||||
|
||||
# use the first one
|
||||
if len(primaryAircraft) == 0:
|
||||
print("Aircraft has no primary aircraft at all: %s" % aircraft_dir)
|
||||
primaryAircraft = [setDicts[0]]
|
||||
|
||||
package = primaryAircraft[0]
|
||||
if not 'thumbnail' in package:
|
||||
if (os.path.exists(os.path.join(aircraft_dir, "thumbnail.jpg"))):
|
||||
package['thumbnail'] = "thumbnail.jpg"
|
||||
|
||||
# variants is just all the set dicts except the first one
|
||||
variants = setDicts
|
||||
variants.remove(package)
|
||||
return (package, variants)
|
||||
|
||||
# create an xml node with text content
|
||||
def make_xml_leaf(name, text):
|
||||
leaf = ET.Element(name)
|
||||
if text != None:
|
||||
if isinstance(text, (int, long)):
|
||||
leaf.text = str(text)
|
||||
else:
|
||||
leaf.text = text
|
||||
else:
|
||||
leaf.text = ''
|
||||
return leaf
|
||||
|
||||
def append_preview_nodes(node, variant, download_base, package_name):
|
||||
if not 'previews' in variant:
|
||||
return
|
||||
|
||||
for preview in variant['previews']:
|
||||
preview_node = ET.Element('preview')
|
||||
preview_url = download_base + 'previews/' + package_name + '_' + preview['path']
|
||||
preview_node.append( make_xml_leaf('type', preview['type']) )
|
||||
preview_node.append( make_xml_leaf('url', preview_url) )
|
||||
preview_node.append( make_xml_leaf('path', preview['path']) )
|
||||
node.append(preview_node)
|
||||
|
||||
def append_tag_nodes(node, variant):
|
||||
if not 'tags' in variant:
|
||||
return
|
||||
|
||||
for tag in variant['tags']:
|
||||
node.append(make_xml_leaf('tag', tag))
|
||||
|
||||
def append_author_nodes(node, info):
|
||||
if 'authors' in info:
|
||||
node.append(info['authors']._createXMLElement())
|
||||
if 'author' in info:
|
||||
# traditional single author string
|
||||
node.append( make_xml_leaf('author', info['author']) )
|
||||
|
||||
def make_aircraft_node(aircraftDirName, package, variants, downloadBase, mirrors):
|
||||
#print("package: %s" % package)
|
||||
#print("variants: %s" % variants)
|
||||
package_node = ET.Element('package')
|
||||
package_node.append( make_xml_leaf('name', package['name']) )
|
||||
package_node.append( make_xml_leaf('status', package['status']) )
|
||||
|
||||
append_author_nodes(package_node, package)
|
||||
|
||||
if 'description' in package:
|
||||
package_node.append( make_xml_leaf('description', package['description']) )
|
||||
|
||||
if 'minimum-fg-version' in package:
|
||||
package_node.append( make_xml_leaf('minimum-fg-version', package['minimum-fg-version']) )
|
||||
|
||||
if 'rating' in package:
|
||||
package_node.append(package['rating']._createXMLElement())
|
||||
|
||||
package_node.append( make_xml_leaf('id', package['id']) )
|
||||
for variant in variants:
|
||||
variant_node = ET.Element('variant')
|
||||
package_node.append(variant_node)
|
||||
variant_node.append( make_xml_leaf('id', variant['id']) )
|
||||
variant_node.append( make_xml_leaf('name', variant['name']) )
|
||||
if 'description' in variant:
|
||||
variant_node.append( make_xml_leaf('description', variant['description']) )
|
||||
|
||||
if 'author' in variant:
|
||||
variant_node.append( make_xml_leaf('author', variant['author']) )
|
||||
|
||||
if 'thumbnail' in variant:
|
||||
# note here we prefix with the package name, since the thumbnail path
|
||||
# is assumed to be unique within the package
|
||||
thumbUrl = downloadBase + "thumbnails/" + aircraftDirName + '_' + variant['thumbnail']
|
||||
variant_node.append(make_xml_leaf('thumbnail', thumbUrl))
|
||||
variant_node.append(make_xml_leaf('thumbnail-path', variant['thumbnail']))
|
||||
|
||||
variantOf = variant['variant-of']
|
||||
if variantOf is None:
|
||||
variant_node.append(make_xml_leaf('variant-of', '_primary_'))
|
||||
else:
|
||||
variant_node.append(make_xml_leaf('variant-of', variantOf))
|
||||
|
||||
append_preview_nodes(variant_node, variant, downloadBase, aircraftDirName)
|
||||
append_tag_nodes(variant_node, variant)
|
||||
append_author_nodes(variant_node, variant)
|
||||
|
||||
package_node.append( make_xml_leaf('dir', aircraftDirName) )
|
||||
|
||||
# primary URL is first
|
||||
download_url = downloadBase + aircraftDirName + '.zip'
|
||||
package_node.append( make_xml_leaf('url', download_url) )
|
||||
|
||||
for m in mirrors:
|
||||
mu = m + aircraftDirName + '.zip'
|
||||
package_node.append( make_xml_leaf('url', mu) )
|
||||
|
||||
|
||||
if 'thumbnail' in package:
|
||||
thumbnail_url = downloadBase + 'thumbnails/' + aircraftDirName + '_' + package['thumbnail']
|
||||
package_node.append( make_xml_leaf('thumbnail', thumbnail_url) )
|
||||
package_node.append( make_xml_leaf('thumbnail-path', package['thumbnail']))
|
||||
|
||||
append_preview_nodes(package_node, package, downloadBase, aircraftDirName)
|
||||
append_tag_nodes(package_node, package)
|
||||
|
||||
if 'maintainers' in package:
|
||||
package_node.append(package['maintainers']._createXMLElement())
|
||||
|
||||
if 'urls' in package:
|
||||
package_node.append(package['urls']._createXMLElement())
|
||||
|
||||
return package_node
|
||||
|
||||
|
||||
def make_aircraft_zip(repo_path, craft_name, zip_file, global_zip_excludes, verbose=True):
|
||||
"""Create a zip archive of the given aircraft."""
|
||||
|
||||
# Printout.
|
||||
if verbose:
|
||||
print("Zip file creation: %s.zip" % craft_name)
|
||||
|
||||
# Go to the directory of crafts to catalog.
|
||||
savedir = os.getcwd()
|
||||
os.chdir(repo_path)
|
||||
|
||||
# Clear out the old file.
|
||||
if exists(zip_file):
|
||||
os.remove(zip_file)
|
||||
|
||||
# Use the Python zipfile module to create the zip file.
|
||||
zip_handle = zipfile.ZipFile(zip_file, 'w', zipfile.ZIP_DEFLATED)
|
||||
|
||||
# Find a per-craft exclude list.
|
||||
craft_path = join(repo_path, craft_name)
|
||||
exclude_file = join(craft_path, 'zip-excludes.lst')
|
||||
if exists(exclude_file):
|
||||
if verbose:
|
||||
print("Found the craft specific exclusion list '%s'" % exclude_file)
|
||||
|
||||
# Otherwise use the catalog default exclusion list.
|
||||
else:
|
||||
exclude_file = global_zip_excludes
|
||||
|
||||
# Process the exclusion list and find all matching file names.
|
||||
blacklist = fetch_zip_exclude_list(craft_name, craft_path, exclude_file)
|
||||
|
||||
# Walk over all craft files.
|
||||
print_format = " %-30s '%s'"
|
||||
for root, dirs, files in walk(craft_path):
|
||||
# Loop over the files.
|
||||
for file in files:
|
||||
# The directory and relative and absolute paths.
|
||||
dir = relpath(root, start=repo_path)
|
||||
full_path = join(root, file)
|
||||
rel_path = relpath(full_path, start=repo_path)
|
||||
|
||||
# Skip blacklist files or directories.
|
||||
skip = False
|
||||
if file == 'zip-excludes.lst':
|
||||
if verbose:
|
||||
print(print_format % ("Skipping the file:", join(dir, 'zip-excludes.lst')))
|
||||
skip = True
|
||||
if dir in blacklist:
|
||||
if verbose:
|
||||
print(print_format % ("Skipping the file:", join(dir, file)))
|
||||
skip = True
|
||||
for name in blacklist:
|
||||
if fnmatch(rel_path, name):
|
||||
if verbose:
|
||||
print(print_format % ("Skipping the file:", rel_path))
|
||||
skip = True
|
||||
break
|
||||
if skip:
|
||||
continue
|
||||
|
||||
# Otherwise add the file.
|
||||
zip_handle.write(rel_path)
|
||||
|
||||
# Clean up.
|
||||
os.chdir(savedir)
|
||||
zip_handle.close()
|
||||
|
||||
|
||||
def fetch_zip_exclude_list(name, path, exclude_path):
|
||||
"""Use Unix style path regular expression to find all files to exclude."""
|
||||
|
||||
# Init.
|
||||
blacklist = []
|
||||
file = open(exclude_path)
|
||||
exclude_list = file.readlines()
|
||||
file.close()
|
||||
old_path = os.getcwd()
|
||||
os.chdir(path)
|
||||
|
||||
# Process each exclusion path or regular expression, converting to Python RE objects.
|
||||
reobj_list = []
|
||||
for i in range(len(exclude_list)):
|
||||
reobj_list.append(re.compile(translate(exclude_list[i].strip())))
|
||||
|
||||
# Recursively loop over all files, finding the ones to exclude.
|
||||
for root, dirs, files in walk(path):
|
||||
for file in files:
|
||||
full_path = join(root, file)
|
||||
rel_path = join(name, relpath(full_path, start=path))
|
||||
|
||||
# Skip Unix shell-style wildcard matches
|
||||
for i in range(len(reobj_list)):
|
||||
if reobj_list[i].match(rel_path):
|
||||
blacklist.append(rel_path)
|
||||
break
|
||||
|
||||
# Return to the original path.
|
||||
os.chdir(old_path)
|
||||
|
||||
# Return the list.
|
||||
return blacklist
|
||||
|
||||
|
||||
def parse_config_file(parser=None, file_name=None):
|
||||
"""Test and parse the catalog configuration file."""
|
||||
|
||||
# Check for the file.
|
||||
if not access(file_name, F_OK):
|
||||
print("CatalogError: The catalog configuration file '%s' cannot be found." % file_name)
|
||||
sys.exit(1)
|
||||
|
||||
# Parse the XML and return the root node.
|
||||
config = ET.parse(file_name, parser)
|
||||
return config.getroot()
|
||||
|
||||
|
||||
def parse_template_file(parser=None, file_name=None):
|
||||
"""Test and parse the catalog configuration file."""
|
||||
|
||||
# Check for the file.
|
||||
if not access(file_name, F_OK):
|
||||
print("CatalogError: The catalog template file '%s' cannot be found." % file_name)
|
||||
sys.exit(1)
|
||||
|
||||
# Parse the XML and return the template node.
|
||||
template = ET.parse(file_name, parser)
|
||||
template_root = template.getroot()
|
||||
return template_root.find('template')
|
||||
@@ -1,182 +0,0 @@
|
||||
aircraftTypeTags = [
|
||||
"aerobatic",
|
||||
"airship",
|
||||
"balloon",
|
||||
"bizjet",
|
||||
"bomber",
|
||||
"cargo",
|
||||
"carrier",
|
||||
"fighter",
|
||||
"ga",
|
||||
"glider",
|
||||
"groundvehicle",
|
||||
"helicopter",
|
||||
"interceptor",
|
||||
"passenger",
|
||||
"racer",
|
||||
"spaceship",
|
||||
"tanker",
|
||||
"trainer",
|
||||
"transport",
|
||||
"ultralight",
|
||||
"unpowered",
|
||||
"uav",
|
||||
"reconnaissance",
|
||||
"seacraft",
|
||||
"crop-duster",
|
||||
"bush-plane"
|
||||
]
|
||||
|
||||
manufacturerTags = [
|
||||
"airbus",
|
||||
"antonov",
|
||||
"arado",
|
||||
"atr",
|
||||
"avro",
|
||||
"bae",
|
||||
"bell",
|
||||
"bleriot",
|
||||
"boeing",
|
||||
"bombardier",
|
||||
"caudron",
|
||||
"cessna",
|
||||
"consolidated",
|
||||
"dassault",
|
||||
"diamond",
|
||||
"dornier",
|
||||
"douglas",
|
||||
"embraer",
|
||||
"eurocopter",
|
||||
"fairchild",
|
||||
"fairey",
|
||||
"focke-wulf",
|
||||
"fokker",
|
||||
"general-dynamics",
|
||||
"gotha",
|
||||
"grumman",
|
||||
"handley-page",
|
||||
"hawker",
|
||||
"heinkel",
|
||||
"ilyushin",
|
||||
"junkers",
|
||||
"kawasaki",
|
||||
"lockheed",
|
||||
"mc-donnell-douglas",
|
||||
"messerschmitt",
|
||||
"mikoyan-gurevich",
|
||||
"mitsubishi",
|
||||
"north-american",
|
||||
"northrop",
|
||||
"pilatus",
|
||||
"piper",
|
||||
"republic",
|
||||
"robin",
|
||||
"rockwell",
|
||||
"saab",
|
||||
"short",
|
||||
"sopwith",
|
||||
"spad",
|
||||
"sukhoi",
|
||||
"supermarine",
|
||||
"tupolev",
|
||||
"vickers",
|
||||
"vought",
|
||||
"wright",
|
||||
"yakovlev"
|
||||
]
|
||||
|
||||
eraTags = [
|
||||
"1910s",
|
||||
"1920s",
|
||||
"1930s",
|
||||
"1940s",
|
||||
"1950s",
|
||||
"1960s",
|
||||
"1970s",
|
||||
"1980s",
|
||||
"1990s",
|
||||
"2000s",
|
||||
"2010s",
|
||||
"coldwar",
|
||||
"early-pioneers",
|
||||
"golden-age",
|
||||
"gulfwar1",
|
||||
"gulfwar2",
|
||||
"vietnam",
|
||||
"ww1",
|
||||
"ww2"
|
||||
]
|
||||
|
||||
featureTags = [
|
||||
"aerobatic",
|
||||
"airship",
|
||||
"amphibious",
|
||||
"biplane",
|
||||
"canard",
|
||||
"castering-wheel",
|
||||
"combat",
|
||||
"delta",
|
||||
"etops",
|
||||
"experimental",
|
||||
"fictional",
|
||||
"fixed-gear",
|
||||
"floats",
|
||||
"glass-cockpit",
|
||||
"low-wing",
|
||||
"mid-wing",
|
||||
"high-wing",
|
||||
"h-tail",
|
||||
"hud",
|
||||
"ifr",
|
||||
"lifting-body",
|
||||
"pressurised",
|
||||
"prototype",
|
||||
"refuel",
|
||||
"retractable-gear",
|
||||
"seaplane",
|
||||
"skis",
|
||||
"stol",
|
||||
"supersonic",
|
||||
"supercharger",
|
||||
"t-tail",
|
||||
"tail-dragger",
|
||||
"tricycle",
|
||||
"tail-hook",
|
||||
"triplane",
|
||||
"twin-boom",
|
||||
"v-tail",
|
||||
"variable-geometry",
|
||||
"vtol",
|
||||
"wing-fold"
|
||||
"water-drop"
|
||||
]
|
||||
|
||||
propulsionTags = [
|
||||
"afterburner",
|
||||
"diesel",
|
||||
"electric",
|
||||
"jet",
|
||||
"propeller",
|
||||
"piston",
|
||||
"radial",
|
||||
"rocket",
|
||||
"single-engine",
|
||||
"supercharged",
|
||||
"turboprop",
|
||||
"twin-engine",
|
||||
"four-engine",
|
||||
"variable-pitch",
|
||||
"fixed-pitch"
|
||||
]
|
||||
|
||||
simFeatureTags = [
|
||||
"dual-controls",
|
||||
"rembrandt",
|
||||
"tow",
|
||||
"wildfire"
|
||||
]
|
||||
|
||||
tags = aircraftTypeTags + manufacturerTags + eraTags + simFeatureTags + propulsionTags + featureTags
|
||||
|
||||
def isValidTag(maybeTag):
|
||||
return maybeTag in tags
|
||||
+19
-16
@@ -1,8 +1,10 @@
|
||||
#!/usr/bin/python
|
||||
#! /usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sgprops
|
||||
|
||||
from flightgear.meta import sgprops
|
||||
|
||||
|
||||
def check_meta_data(aircraft_dir, set_file, includes):
|
||||
base_file = os.path.basename(set_file)
|
||||
@@ -13,44 +15,46 @@ def check_meta_data(aircraft_dir, set_file, includes):
|
||||
root_node = sgprops.readProps(set_path, includePaths = includes)
|
||||
|
||||
if not root_node.hasChild("sim"):
|
||||
print "-set.xml has no <sim> node:", set_path
|
||||
print("-set.xml has no <sim> node:", set_path)
|
||||
return
|
||||
|
||||
sim_node = root_node.getChild("sim")
|
||||
if not sim_node.hasChild('description'):
|
||||
print "-set.xml missing <description>:", set_path
|
||||
print("-set.xml missing <description>:", set_path)
|
||||
|
||||
if not sim_node.hasChild('long-description'):
|
||||
print "-set.xml missing <long-description>:", set_path
|
||||
print("-set.xml missing <long-description>:", set_path)
|
||||
|
||||
if not sim_node.hasChild('authors'):
|
||||
print "-set.xml is missing structured <authors> data:", set_path
|
||||
print("-set.xml is missing structured <authors> data:", set_path)
|
||||
|
||||
if not sim_node.hasChild('tags'):
|
||||
print "-set.xml does not define any tags", set_path
|
||||
print("-set.xml does not define any tags", set_path)
|
||||
|
||||
# check for non-standard tags
|
||||
|
||||
if not sim_node.hasChild('thumbnail'):
|
||||
print "-set.xml does not define a thumbnail", set_path
|
||||
print("-set.xml does not define a thumbnail", set_path)
|
||||
|
||||
# check thumbnail size and format
|
||||
|
||||
if not sim_node.hasChild('rating'):
|
||||
print "-set.xml does not define any ratings", set_path
|
||||
print("-set.xml does not define any ratings", set_path)
|
||||
|
||||
if not sim_node.hasChild('minimum-fg-version'):
|
||||
print "-set.xml does not define a minimum FG version", set_path
|
||||
print("-set.xml does not define a minimum FG version", set_path)
|
||||
|
||||
# check all the -set.xml files in an aircraft directory.
|
||||
|
||||
# check all the -set.xml files in an aircraft directory.
|
||||
def check_aircraft_dir(d, includes):
|
||||
if not os.path.isdir(d):
|
||||
return
|
||||
|
||||
files = os.listdir(d)
|
||||
for file in sorted(files, key=lambda s: s.lower()):
|
||||
if file.endswith('-set.xml'):
|
||||
check_meta_data(d, file, includes)
|
||||
for f in sorted(files, key=lambda s: s.lower()):
|
||||
if f.endswith('-set.xml'):
|
||||
check_meta_data(d, f, includes)
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--include", help="Include directory to validate -set.xml parsing",
|
||||
@@ -60,7 +64,7 @@ args = parser.parse_args()
|
||||
|
||||
for d in args.dir:
|
||||
if not os.path.isdir(d):
|
||||
print "Skipping missing directory:", d
|
||||
print("Skipping missing directory:", d)
|
||||
|
||||
names = os.listdir(d)
|
||||
for name in sorted(names, key=lambda s: s.lower()):
|
||||
@@ -70,4 +74,3 @@ for d in args.dir:
|
||||
|
||||
acftDir = os.path.join(d, name)
|
||||
check_aircraft_dir(acftDir, args.include)
|
||||
|
||||
@@ -1,306 +0,0 @@
|
||||
# SAX for parsing
|
||||
from xml.sax import make_parser, handler, expatreader
|
||||
|
||||
# ElementTree for writing
|
||||
#import xml.etree.cElementTree as ET
|
||||
import lxml.etree as ET
|
||||
|
||||
import re, os
|
||||
|
||||
class Node(object):
|
||||
def __init__(self, name = '', index = 0, parent = None):
|
||||
self._parent = parent
|
||||
self._name = name
|
||||
self._value = None
|
||||
self._index = index
|
||||
self._children = []
|
||||
|
||||
@property
|
||||
def value(self):
|
||||
return self._value
|
||||
|
||||
@value.setter
|
||||
def value(self, v):
|
||||
self._value = v
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return self._name
|
||||
|
||||
@property
|
||||
def index(self):
|
||||
return self._index
|
||||
|
||||
@property
|
||||
def parent(self):
|
||||
return self._parent
|
||||
|
||||
def getChild(self, n, i=None, create = False):
|
||||
|
||||
if i is None:
|
||||
i = 0
|
||||
# parse name as foo[999] if necessary
|
||||
m = re.match(R"(\w+)\[(\d+)\]", n)
|
||||
if m is not None:
|
||||
n = m.group(1)
|
||||
i = int(m.group(2))
|
||||
|
||||
for c in self._children:
|
||||
if (c.name == n) and (c.index == i):
|
||||
return c
|
||||
|
||||
if create:
|
||||
c = Node(n, i, self)
|
||||
self._children.append(c)
|
||||
return c
|
||||
else:
|
||||
raise IndexError("no such child:" + str(n) + " index=" + str(i))
|
||||
|
||||
def addChild(self, n):
|
||||
# adding an existing instance
|
||||
if isinstance(n, Node):
|
||||
n._parent = self
|
||||
n._index = self.firstUnusedIndex(n.name)
|
||||
self._children.append(n)
|
||||
return n
|
||||
|
||||
i = self.firstUnusedIndex(n)
|
||||
# create it via getChild
|
||||
return self.getChild(n, i, create=True)
|
||||
|
||||
def firstUnusedIndex(self, n):
|
||||
usedIndices = frozenset(c.index for c in self.getChildren(n))
|
||||
i = 0
|
||||
while i < 1000:
|
||||
if i not in usedIndices:
|
||||
return i
|
||||
i += 1
|
||||
raise RuntimeException("too many children with name:" + n)
|
||||
|
||||
def hasChild(self, nm):
|
||||
for c in self._children:
|
||||
if (c.name == nm):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def getChildren(self, n = None):
|
||||
if n is None:
|
||||
return self._children
|
||||
|
||||
return [c for c in self._children if c.name == n]
|
||||
|
||||
def getNode(self, path, cr = False):
|
||||
axes = path.split('/')
|
||||
nd = self
|
||||
for ax in axes:
|
||||
nd = nd.getChild(ax, create = cr)
|
||||
|
||||
return nd
|
||||
|
||||
def getValue(self, path, default = None):
|
||||
try:
|
||||
nd = self.getNode(path)
|
||||
return nd.value
|
||||
except:
|
||||
return default
|
||||
|
||||
def write(self, path):
|
||||
root = self._createXMLElement('PropertyList')
|
||||
t = ET.ElementTree(root)
|
||||
t.write(path, 'utf-8', xml_declaration = True)
|
||||
|
||||
def _createXMLElement(self, nm = None):
|
||||
if nm is None:
|
||||
nm = self.name
|
||||
|
||||
n = ET.Element(nm)
|
||||
|
||||
# value and type specification
|
||||
try:
|
||||
if self._value is not None:
|
||||
if isinstance(self._value, str):
|
||||
# don't call str() on strings, breaks the
|
||||
# encoding
|
||||
n.text = self._value
|
||||
else:
|
||||
# use str() to turn non-string types into text
|
||||
n.text = str(self._value)
|
||||
if isinstance(self._value, int):
|
||||
n.set('type', 'int')
|
||||
elif isinstance(self._value, float):
|
||||
n.set('type', 'double')
|
||||
elif isinstance(self._value, bool):
|
||||
n.set('type', "bool")
|
||||
except UnicodeEncodeError:
|
||||
print("Encoding error with %s %s" % (self._value, type(self._value)))
|
||||
except:
|
||||
print("Unexpected exception in sgprops._createXMLElement():", sys.exc_info()[0])
|
||||
|
||||
# index in parent
|
||||
if (self.index != 0):
|
||||
n.set('n', str(self.index))
|
||||
|
||||
# children
|
||||
for c in self._children:
|
||||
n.append(c._createXMLElement())
|
||||
|
||||
return n;
|
||||
|
||||
class ParseState:
|
||||
def __init__(self):
|
||||
self._counters = {}
|
||||
|
||||
def getNextIndex(self, name):
|
||||
if name in self._counters:
|
||||
self._counters[name] += 1
|
||||
else:
|
||||
self._counters[name] = 0
|
||||
return self._counters[name]
|
||||
|
||||
def recordExplicitIndex(self, name, index):
|
||||
if not name in self._counters:
|
||||
self._counters[name] = index
|
||||
else:
|
||||
self._counters[name] = max(self._counters[name], index)
|
||||
|
||||
class PropsHandler(handler.ContentHandler):
|
||||
def __init__(self, root = None, path = None, includePaths = []):
|
||||
self._root = root
|
||||
self._path = path
|
||||
self._basePath = os.path.dirname(path)
|
||||
self._includes = includePaths
|
||||
self._locator = None
|
||||
self._stateStack = [ParseState()]
|
||||
|
||||
if root is None:
|
||||
# make a nameless root node
|
||||
self._root = Node("", 0)
|
||||
self._current = self._root
|
||||
|
||||
def setDocumentLocator(self, loc):
|
||||
self._locator = loc
|
||||
|
||||
def startElement(self, name, attrs):
|
||||
self._content = None
|
||||
if (name == 'PropertyList'):
|
||||
# still need to handle includes on the root element
|
||||
if 'include' in attrs.keys():
|
||||
self.handleInclude(attrs['include'])
|
||||
return
|
||||
|
||||
currentState = self._stateStack[-1]
|
||||
if 'n' in attrs.keys():
|
||||
try:
|
||||
index = int(attrs['n'])
|
||||
except:
|
||||
print("Invalid index at line: %s of %s" % (self._locator.getLineNumber(), self._path))
|
||||
raise IndexError("Invalid index at line:", self._locator.getLineNumber(), "of", self._path)
|
||||
|
||||
currentState.recordExplicitIndex(name, index)
|
||||
self._current = self._current.getChild(name, index, create=True)
|
||||
else:
|
||||
index = currentState.getNextIndex(name)
|
||||
# important we use getChild here, so that includes are resolved
|
||||
# correctly
|
||||
self._current = self._current.getChild(name, index, create=True)
|
||||
|
||||
self._stateStack.append(ParseState())
|
||||
|
||||
if 'include' in attrs.keys():
|
||||
self.handleInclude(attrs['include'])
|
||||
|
||||
self._currentTy = None;
|
||||
if 'type' in attrs.keys():
|
||||
self._currentTy = attrs['type']
|
||||
|
||||
def handleInclude(self, includePath):
|
||||
if includePath.startswith('/'):
|
||||
includePath = includePath[1:]
|
||||
|
||||
p = os.path.join(self._basePath, includePath)
|
||||
if not os.path.exists(p):
|
||||
found = False
|
||||
for i in self._includes:
|
||||
p = os.path.join(i, includePath)
|
||||
if os.path.exists(p):
|
||||
found = True
|
||||
break
|
||||
|
||||
if not found:
|
||||
raise RuntimeError("include file not found", includePath, "at line", self._locator.getLineNumber())
|
||||
|
||||
readProps(p, self._current, self._includes)
|
||||
|
||||
def endElement(self, name):
|
||||
if (name == 'PropertyList'):
|
||||
return
|
||||
|
||||
try:
|
||||
# convert and store value
|
||||
self._current.value = self._content
|
||||
if self._currentTy == "int":
|
||||
self._current.value = int(self._content) if self._content is not None else 0
|
||||
if self._currentTy == "bool":
|
||||
self._current.value = self.parsePropsBool(self._content)
|
||||
if self._currentTy == "double":
|
||||
if self._content is None:
|
||||
self._current.value = 0.0
|
||||
else:
|
||||
if self._content.endswith('f'):
|
||||
self._content = self._content[:-1]
|
||||
self._current.value = float(self._content)
|
||||
except:
|
||||
print("Parse error for value: %s at line: %s of: %s" % (self._content, self._locator.getLineNumber(), self._path))
|
||||
|
||||
self._current = self._current.parent
|
||||
self._content = None
|
||||
self._currentTy = None
|
||||
self._stateStack.pop()
|
||||
|
||||
|
||||
def parsePropsBool(self, content):
|
||||
if content == "True" or content == "true":
|
||||
return True
|
||||
|
||||
if content == "False" or content == "false":
|
||||
return False
|
||||
|
||||
try:
|
||||
icontent = int(content)
|
||||
if icontent is not None:
|
||||
if icontent == 0:
|
||||
return False
|
||||
else:
|
||||
return True;
|
||||
except:
|
||||
return False
|
||||
|
||||
def characters(self, content):
|
||||
if self._content is None:
|
||||
self._content = ''
|
||||
self._content += content
|
||||
|
||||
def endDocument(self):
|
||||
pass
|
||||
|
||||
@property
|
||||
def root(self):
|
||||
return self._root
|
||||
|
||||
def readProps(path, root = None, includePaths = []):
|
||||
parser = make_parser()
|
||||
locator = expatreader.ExpatLocator( parser )
|
||||
h = PropsHandler(root, path, includePaths)
|
||||
h.setDocumentLocator(locator)
|
||||
parser.setContentHandler(h)
|
||||
parser.parse(path)
|
||||
return h.root
|
||||
|
||||
def copy(src, dest):
|
||||
dest.value = src.value
|
||||
|
||||
# recurse over children
|
||||
for c in src.getChildren() :
|
||||
dc = dest.getChild(c.name, i = c.index, create = True)
|
||||
copy(c, dc)
|
||||
@@ -1,15 +1,16 @@
|
||||
#!/usr/bin/python
|
||||
#! /usr/bin/env python3
|
||||
|
||||
import unittest
|
||||
import sgprops
|
||||
import os
|
||||
from os.path import join
|
||||
import catalog
|
||||
import lxml.etree as ET
|
||||
from shutil import rmtree
|
||||
from tempfile import mkdtemp
|
||||
import zipfile
|
||||
|
||||
from flightgear.meta import sgprops
|
||||
from flightgear.meta.aircraft_catalogs import catalog
|
||||
|
||||
|
||||
catalog.quiet = True
|
||||
|
||||
@@ -247,7 +248,7 @@ class ZipTests(unittest.TestCase):
|
||||
"""General checks for the zip file."""
|
||||
|
||||
# Check for file existence.
|
||||
self.assert_(os.access(file_name, os.F_OK))
|
||||
self.assertTrue(os.access(file_name, os.F_OK))
|
||||
|
||||
# Check the contents.
|
||||
file = zipfile.ZipFile(file_name)
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
#! /usr/bin/env python3
|
||||
|
||||
import unittest
|
||||
|
||||
import types
|
||||
import sgprops
|
||||
from flightgear.meta import sgprops
|
||||
|
||||
|
||||
class SGProps(unittest.TestCase):
|
||||
|
||||
@@ -9,14 +11,14 @@ class SGProps(unittest.TestCase):
|
||||
parsed = sgprops.readProps("testData/props1.xml")
|
||||
|
||||
self.assertEqual(parsed.getValue("value"), 42)
|
||||
self.assertEqual(type(parsed.getValue("value")), types.IntType)
|
||||
self.assertEqual(type(parsed.getValue("value")), int)
|
||||
|
||||
valNode = parsed.getChild("value")
|
||||
self.assertEqual(valNode.parent, parsed)
|
||||
self.assertEqual(valNode.name, "value")
|
||||
|
||||
self.assertEqual(valNode.value, 42)
|
||||
self.assertEqual(type(valNode.value), types.IntType)
|
||||
self.assertEqual(type(valNode.value), int)
|
||||
|
||||
with self.assertRaises(IndexError):
|
||||
missingNode = parsed.getChild("missing")
|
||||
|
||||
@@ -8,12 +8,14 @@ import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
import sgprops
|
||||
import sys
|
||||
import catalogTags
|
||||
import catalog
|
||||
from catalog import make_aircraft_node, make_aircraft_zip, parse_config_file, parse_template_file
|
||||
import time
|
||||
|
||||
from flightgear.meta import sgprops
|
||||
from flightgear.meta.aircraft_catalogs import catalogTags
|
||||
from flightgear.meta.aircraft_catalogs import catalog
|
||||
from flightgear.meta.aircraft_catalogs.catalog import make_aircraft_node, \
|
||||
make_aircraft_zip, parse_config_file, parse_template_file
|
||||
|
||||
|
||||
CATALOG_VERSION = 4
|
||||
|
||||
Reference in New Issue
Block a user