From 242c0f9172b98efefe199e71717a63d91c1abd43 Mon Sep 17 00:00:00 2001 From: James Turner Date: Sat, 25 Jul 2015 22:45:01 -0500 Subject: [PATCH] Move PackageData to its own file. --- maintain_catalog.py | 209 +------------------------------------- package.py | 205 +++++++++++++++++++++++++++++++++++++ svn_catalog_repository.py | 14 +-- 3 files changed, 216 insertions(+), 212 deletions(-) create mode 100644 package.py diff --git a/maintain_catalog.py b/maintain_catalog.py index 43ec1a5..8da8f5b 100755 --- a/maintain_catalog.py +++ b/maintain_catalog.py @@ -1,14 +1,11 @@ #!/usr/bin/python import os, sys, re, glob -import hashlib # for MD5 import subprocess -import shutil # for copy2 -import catalogTags import sgprops -#from multiprocessing import Pool import argparse import urllib2 +import package as pkg import svn_catalog_repository import git_catalog_repository @@ -19,208 +16,8 @@ parser.add_argument("--clean", help="Regenerate every package", type=bool) parser.add_argument("dir", help="Catalog directory") args = parser.parse_args() -standardTagSet = frozenset(catalogTags.tags) -def isNonstandardTag(t): - return t not in standardTagSet - -thumbnailNames = ["thumbnail.png", "thumbnail.jpg"] includePaths = [] -class VariantData: - def __init__(self, path, node): - #self._primary = primary - self._path = path - self._name = node.getValue("sim/description") - - # ratings - - # seperate thumbnails - - @property - def catalogNode(self): - n = sgprops.Node("variant") - n.addChild("id").value = self._path - n.addChild("name").value = self._name - -class PackageData: - def __init__(self, path): - self._path = path - self._previousSCMRevision = None - self._previousRevision = 0 - self._thumbnails = [] - self._variants = {} - self._revision = 0 - self._md5 = None - self._fileSize = 0 - - self._node = sgprops.Node("package") - self._node.addChild("id").value = self.id - - def setPreviousData(self, node): - self._previousRevision = node.getValue("revision") - self._previousMD5 = node.getValue("md5") - self._previousSCMRevision = node.getValue("scm-revision") - self._fileSize = int(node.getValue("file-size-bytes")) - - @property - def id(self): - return os.path.basename(self._path) - - @property - def thumbnails(self): - return self._thumbnails - - @property - def path(self): - return self._path - - @property - def variants(self): - return self._variants - - @property - def scmRevision(self): - currentRev = scmRepo.scmRevisionForPath(self._path) - if (currentRev is None): - raise RuntimeError("Unable to query SCM revision of files") - - return currentRev - - def isSourceModified(self, scmRepo): - if (self._previousSCMRevision == None): - return True - - if (self._previousSCMRevision == self.scmRevision): - return False - - return True - - def scanSetXmlFiles(self, includes): - foundPrimary = False - foundMultiple = False - - for f in os.listdir(self._path): - if not f.endswith("-set.xml"): - continue - - p = os.path.join(self._path, f) - node = sgprops.readProps(p, includePaths = includes) - if not node.hasChild("sim"): - continue - - simNode = node.getChild("sim") - if (simNode.getValue("exclude", False)): - continue - - primary = simNode.getValue("variant-of", None) - if primary: - if not primary in self.variants: - self._variants[primary] = [] - self._variants[primary].append(VariantData(self, node)) - continue - - if foundPrimary: - if not foundMultiple: - print "Multiple primary -set.xml files at:" + self._path - foundMultiple = True - continue - else: - foundPrimary = True; - - self.parsePrimarySetNode(simNode) - - for n in thumbnailNames: - if os.path.exists(os.path.join(self._path, n)): - self._thumbnails.append(n) - - if not foundPrimary: - raise RuntimeError("No primary -set.xml found at:" + self._path) - - - - def parsePrimarySetNode(self, sim): - - # basic / mandatory values - self._node.addChild('name').value = sim.getValue('description') - - longDesc = sim.getValue('long-description') - if longDesc is not None: - self._node.addChild('description').value = longDesc - - # copy all the standard values - for p in ['status', 'author', 'license']: - v = sim.getValue(p) - if v is not None: - self._node.addChild(p).value = v - - # ratings - if sim.hasChild('rating'): - pkgRatings = self._node.addChild('rating') - for r in ['FDM', 'systems', 'cockpit', 'model']: - pkgRatings.addChild(r).value = sim.getValue('rating/' + r, 0) - - # copy tags - if sim.hasChild('tags'): - for c in sim.getChild('tags').getChildren('tag'): - if isNonstandardTag(c.value): - print "Skipping non-standard tag:", c.value, self.path - else: - self._node.addChild('tag').value = c.value - - for t in sim.getChildren("thumbnail"): - self._thumbnails.append(t.value) - - def validate(self): - for t in self._thumbnails: - if not os.path.exists(os.path.join(self._path, t)): - raise RuntimeError("missing thumbnail:" + t); - - def generateZip(self, outDir): - self._revision = self._previousRevision + 1 - - zipName = self.id + ".zip" - zipFilePath = os.path.join(outDir, zipName) - - os.chdir(os.path.dirname(self.path)) - - print "Creating zip", zipFilePath - # TODO: exclude certain files - # anything we can do to make this faster? - subprocess.call(['zip', '--quiet', '-r', zipFilePath, self.id]) - - zipFile = open(zipFilePath, 'r') - self._md5 = hashlib.md5(zipFile.read()).hexdigest() - self._fileSize = os.path.getsize(zipFilePath) - - def useExistingCatalogData(self): - self._md5 = self._previousMD5 - - def packageNode(self, mirrorUrls, thumbnailUrl): - self._node.getChild("md5", create = True).value = self._md5 - self._node.getChild("file-size-bytes", create = True).value = self._fileSize - self._node.getChild("revision", create = True).value = int(self._revision) - self._node.getChild("scm-revision", create = True).value = self.scmRevision - - for m in mirrorUrls: - self._node.addChild("url").value = m + "/" + self.id + ".zip" - - for t in self._thumbnails: - self._node.addChild("thumbnail").value = thumbnailUrl + "/" + self.id + "_" + t - - for pr in self._variants: - for vr in self._variants[pr]: - self._node.addChild(vr.catalogNode) - - return self._node - - def extractThumbnails(self, thumbnailDir): - for t in self._thumbnails: - fullName = self.id + "_" + t - shutil.copy2(os.path.join(self._path, t), - os.path.join(thumbnailDir, fullName) - ) - # TODO : verify image format, size and so on - def scanPackages(globPath): result = [] print "Scanning", globPath @@ -231,7 +28,7 @@ def scanPackages(globPath): print "no -set.xml in", d continue - result.append(PackageData(d)) + result.append(pkg.PackageData(d)) return result @@ -385,7 +182,7 @@ for p in packagesToGenerate: print "Creating catalog" for p in packages.values(): - catalogNode.addChild(p.packageNode(mirrorUrls, thumbnailUrl)) + catalogNode.addChild(p.packageNode(scmRepo, mirrorUrls, thumbnailUrl)) catalogNode.write(os.path.join(outPath, "catalog.xml")) diff --git a/package.py b/package.py new file mode 100644 index 0000000..14eb911 --- /dev/null +++ b/package.py @@ -0,0 +1,205 @@ +import os, subprocess +import sgprops +import hashlib # for MD5 +import shutil # for copy2 +import catalogTags + +standardTagSet = frozenset(catalogTags.tags) +def isNonstandardTag(t): + return t not in standardTagSet + +thumbnailNames = ["thumbnail.png", "thumbnail.jpg"] + +class VariantData: + def __init__(self, path, node): + #self._primary = primary + self._path = path + self._name = node.getValue("sim/description") + + # ratings + + # seperate thumbnails + + @property + def catalogNode(self): + n = sgprops.Node("variant") + n.addChild("id").value = self._path + n.addChild("name").value = self._name + +class PackageData: + def __init__(self, path): + self._path = path + self._previousSCMRevision = None + self._previousRevision = 0 + self._thumbnails = [] + self._variants = {} + self._revision = 0 + self._md5 = None + self._fileSize = 0 + + self._node = sgprops.Node("package") + self._node.addChild("id").value = self.id + + def setPreviousData(self, node): + self._previousRevision = node.getValue("revision") + self._previousMD5 = node.getValue("md5") + self._previousSCMRevision = node.getValue("scm-revision") + self._fileSize = int(node.getValue("file-size-bytes")) + + @property + def id(self): + return os.path.basename(self._path) + + @property + def thumbnails(self): + return self._thumbnails + + @property + def path(self): + return self._path + + @property + def variants(self): + return self._variants + + def scmRevision(self, repo): + currentRev = repo.scmRevisionForPath(self._path) + if (currentRev is None): + raise RuntimeError("Unable to query SCM revision of files") + + return currentRev + + def isSourceModified(self, scmRepo): + if (self._previousSCMRevision == None): + return True + + if (self._previousSCMRevision == self.scmRevision(scmRepo)): + return False + + return True + + def scanSetXmlFiles(self, includes): + foundPrimary = False + foundMultiple = False + + for f in os.listdir(self._path): + if not f.endswith("-set.xml"): + continue + + p = os.path.join(self._path, f) + node = sgprops.readProps(p, includePaths = includes) + if not node.hasChild("sim"): + continue + + simNode = node.getChild("sim") + if (simNode.getValue("exclude", False)): + continue + + primary = simNode.getValue("variant-of", None) + if primary: + if not primary in self.variants: + self._variants[primary] = [] + self._variants[primary].append(VariantData(self, node)) + continue + + if foundPrimary: + if not foundMultiple: + print "Multiple primary -set.xml files at:" + self._path + foundMultiple = True + continue + else: + foundPrimary = True; + + self.parsePrimarySetNode(simNode) + + for n in thumbnailNames: + if os.path.exists(os.path.join(self._path, n)): + self._thumbnails.append(n) + + if not foundPrimary: + raise RuntimeError("No primary -set.xml found at:" + self._path) + + + + def parsePrimarySetNode(self, sim): + + # basic / mandatory values + self._node.addChild('name').value = sim.getValue('description') + + longDesc = sim.getValue('long-description') + if longDesc is not None: + self._node.addChild('description').value = longDesc + + # copy all the standard values + for p in ['status', 'author', 'license']: + v = sim.getValue(p) + if v is not None: + self._node.addChild(p).value = v + + # ratings + if sim.hasChild('rating'): + pkgRatings = self._node.addChild('rating') + for r in ['FDM', 'systems', 'cockpit', 'model']: + pkgRatings.addChild(r).value = sim.getValue('rating/' + r, 0) + + # copy tags + if sim.hasChild('tags'): + for c in sim.getChild('tags').getChildren('tag'): + if isNonstandardTag(c.value): + print "Skipping non-standard tag:", c.value, self.path + else: + self._node.addChild('tag').value = c.value + + for t in sim.getChildren("thumbnail"): + self._thumbnails.append(t.value) + + def validate(self): + for t in self._thumbnails: + if not os.path.exists(os.path.join(self._path, t)): + raise RuntimeError("missing thumbnail:" + t); + + def generateZip(self, outDir): + self._revision = self._previousRevision + 1 + + zipName = self.id + ".zip" + zipFilePath = os.path.join(outDir, zipName) + + os.chdir(os.path.dirname(self.path)) + + print "Creating zip", zipFilePath + # TODO: exclude certain files + # anything we can do to make this faster? + subprocess.call(['zip', '--quiet', '-r', zipFilePath, self.id]) + + zipFile = open(zipFilePath, 'r') + self._md5 = hashlib.md5(zipFile.read()).hexdigest() + self._fileSize = os.path.getsize(zipFilePath) + + def useExistingCatalogData(self): + self._md5 = self._previousMD5 + + def packageNode(self, repo, mirrorUrls, thumbnailUrl): + self._node.getChild("md5", create = True).value = self._md5 + self._node.getChild("file-size-bytes", create = True).value = self._fileSize + self._node.getChild("revision", create = True).value = int(self._revision) + self._node.getChild("scm-revision", create = True).value = self.scmRevision(repo) + + for m in mirrorUrls: + self._node.addChild("url").value = m + "/" + self.id + ".zip" + + for t in self._thumbnails: + self._node.addChild("thumbnail").value = thumbnailUrl + "/" + self.id + "_" + t + + for pr in self._variants: + for vr in self._variants[pr]: + self._node.addChild(vr.catalogNode) + + return self._node + + def extractThumbnails(self, thumbnailDir): + for t in self._thumbnails: + fullName = self.id + "_" + t + shutil.copy2(os.path.join(self._path, t), + os.path.join(thumbnailDir, fullName) + ) + # TODO : verify image format, size and so on diff --git a/svn_catalog_repository.py b/svn_catalog_repository.py index 7fc11eb..9db3dda 100644 --- a/svn_catalog_repository.py +++ b/svn_catalog_repository.py @@ -1,25 +1,27 @@ -import subprocess +import subprocess, os import xml.etree.cElementTree as ET class SVNCatalogRepository: def __init__(self, path): + if not os.path.exists(path): + raise RuntimeError("No directory at:" + path) + self._path = path xml = subprocess.check_output(["svn", "info", "--xml", path]) root = ET.fromstring(xml) - + if (root.find(".//repository/root") == None): raise RuntimeError("Not an SVN repository:" + path) - + def hasPathChanged(self, path, oldRevision): return self.scmRevisionForPath(path) != oldRevision - + def scmRevisionForPath(self, path): xml = subprocess.check_output(["svn", "info", "--xml", path]) root = ET.fromstring(xml) commit = root.find(".//entry/commit") return commit.get('revision', 0) - + def update(self): subprocess.call(["svn", "update"]) - \ No newline at end of file