From 9a66671ba352203bef4b755c05a569350e8ce8ac Mon Sep 17 00:00:00 2001 From: TheFGFSEagle Date: Fri, 29 Jul 2022 17:06:24 +0200 Subject: [PATCH] Fix errors --- scenery/process-shapefiles.py | 108 ++++++++++++++++------------------ 1 file changed, 51 insertions(+), 57 deletions(-) diff --git a/scenery/process-shapefiles.py b/scenery/process-shapefiles.py index ae3f245..3b888fa 100755 --- a/scenery/process-shapefiles.py +++ b/scenery/process-shapefiles.py @@ -9,6 +9,9 @@ import glob import re import argparse import subprocess +import tempfile + +from fgtools.utils import constants DESCRIPTION = """ process-shapefiles.py - merges, slices, and decodes OSM shapefiles @@ -21,71 +24,71 @@ For this reason, you may only remove the 'gis_' and '_free_1' parts from the sha Everything else in the name must be conserved in order for your resulting scenery not to have big areas of default landmass terrain !'' Then, for each shapefile in each category the extents will be queried using ogrinfo. To reduce processing time on subsequent runs, the results will be cached. -Then, the script will decide whether to merge or slice the shapefiles for each category based on the coordinates you input. +Then, the script will merge and slice the shapefiles for each category based on the coordinates you input. It will merge / slice the files accordingly with ogr2ogr. -As the final step, the resulting shapefiles will be decoded into files that tg-constrcut can read using ogr-decode. """ +#As the final step, the resulting shapefiles will be decoded into files that tg-constrcut can read using ogr-decode. +#""" +catnames = ["buildings", "landuse", "natural", "places", "pofw", "pois", "railways", "roads", "traffic", "transport", "water", "waterways"] +tmpdir = tempfile.TemporaryDirectory() def find(src): osm_shapefiles = [] - shapefiles = glob.glob(os.path.join(src, "**", "**.shp") + files = glob.glob(os.path.join(src, "**", "**.shp")) for file in files: if "osm" in os.path.split(file): osm_shapefiles.append(file) return sorted(osm_shapefiles) def categorize(shapefiles): - catnames = ["buildings", "landuse", "natural", "places", "pofw", "pois", "railways", "roads", "traffic", "transport", "water", "waterways"] categorized = [] for shapefile in shapefiles: name = os.path.split(shapefile)[-1].split(".")[0] - # Skip if the name is of the form gis_osm_landuse_a_free_1 - # these files are much smaller than the ones without _a_, probably contain less data. - if "_a_" in name: - continue - name = re.sub(r"gis|osm|a|free|_|(1-9)", "", name) # gis_osm_landuse_a_free_1 becomes just landuse # Skip if the name is not a recognized catname if not name in catnames: continue - categorized.append({"path": shapefile, "category": name}) + categorized[name].append({"path": shapefile}) return categorized def get_extents(categorized): - for shapefile in categorized: - cmd = f"ogrinfo -al -so -ro -nocount -nomd {shapefile['path']}" - query = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - output = list(map(lambda s: s.decode(), query.stdout.splitlines())) # subprocess.Popen.stdout is a binary file - we need normal strings - - extents = [s for s in output if "Extent" in s] - feature_count = [s for s in output if "Feature Count" in s] - - if len(extents) != 1 or len(feature_count) != 1: - print("ERROR: Fetching shapefile information using ogrinfo failed.") - print(" Try reinstalling it through your package manager.") - print(" If that doesn't help, please file a bug report at .") - print(" If you do that, please attach the process-shapefiles-bugreport.md file in order for the maintainers to be able to help you.") + for category in categorized: + for shapefile in categorized[category]: + cmd = f"ogrinfo -al -so -ro -nocount -nomd {shapefile['path']}" + query = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + output = list(map(lambda s: s.decode(), query.stdout.splitlines())) # subprocess.Popen.stdout is a binary file - we need normal strings - with open("process-shapefile-bugreport.md", "w") as f: - f.write(f"### Output of `{cmd}`") - f.writelines(output) - f.write(f"### ogrinfo version:") - f.write(subprocess.run("ogrinfo --version", stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=True).stdout.deocde()) - sys.exit(2) - - # convert "Extent: (10.544425, 51.500000) - (11.500000, 52.500000)" to {"xll": 10.544425, "yll": 51.5, "xur": 11.5, "yur": 52,5] - extents = dict(zip(["xll", "yll", "yur", "yur"], map(float, re.sub(r"Extent:\s\(|\)", "", extents[0]).replace(") - (", ", ").split(", ")))) - shapefile["extents"] = extents - + extents = [s for s in output if "Extent" in s] + feature_count = [s for s in output if "Feature Count" in s] + + if len(extents) != 1 or len(feature_count) != 1: + print("ERROR: Fetching shapefile information using ogrinfo failed.") + print(" Try reinstalling it through your package manager.") + print(" If that doesn't help, please file a bug report at .") + print(" If you do that, please attach the process-shapefiles-bugreport.md file in order for the maintainers to be able to help you.") + + with open("process-shapefile-bugreport.md", "w") as f: + f.write(f"### Output of `{cmd}`") + f.writelines(output) + f.write(f"### ogrinfo version:") + f.write(subprocess.run("ogrinfo --version", stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=True).stdout.deocde()) + sys.exit(2) + + # convert "Extent: (10.544425, 51.500000) - (11.500000, 52.500000)" to {"xll": 10.544425, "yll": 51.5, "xur": 11.5, "yur": 52,5] + extents = dict(zip(["xll", "yll", "yur", "yur"], map(float, re.sub(r"Extent:\s\(|\)", "", extents[0]).replace(") - (", ", ").split(", ")))) + shapefile["extents"] = extents -def merge_slice(shapefiles, coords): - pass +def merge_slice(shapefiles, coords, dest): + for category in shapefiles: + files = ['"' + shapefile["path"] + '"' for shapefile in shapefiles[category]] + cmd = f'ogr2ogr -f "ESRI Shapefile" {os.path.join(dest, category + ".shp")} {" ".join(files)} -clipsrc {coords["xll"]} {coords["yll"]} {coords["xur"]} {coords["yur"]} -progress -single -lco ENCODING=UTF-8' + subprocess.run(cmd, shell=True) -def decode(shapefiles, dest): - pass +#def decode(shapefiles, dest): +# pass if __name__ == "__main__": argp = argparse.ArgumentParser(description="process-shapefiles.py - merges, slices, and decodes OSM shapefiles") @@ -93,7 +96,7 @@ if __name__ == "__main__": argp.add_argument( "-v", "--version", action="version", - version=f"TerraGear tools {'.'.join(map(str, constants.__version__))}" + version=f"FGTools {'.'.join(map(str, constants.__version__))}" ) argp.add_argument( @@ -115,12 +118,6 @@ if __name__ == "__main__": metavar="FOLDER" ) - argp.add_argument( - "-c", "--cache-folder", - help="where to put cache folder (default: %(default)s)", - default=os.path.join(constants.HOME, ".cache", "tgtools") - ) - argp.add_argument( "-l", "--lower-left", help="coordinates of the lower left corner of the bounding box of the region that shapefiles should be processed for (default: %(default)s)", @@ -141,23 +138,20 @@ if __name__ == "__main__": src = args.input_folder dest = args.output_folder - cache = args.cache_folder xll, yll = args.lower_left.split(",") xur, yur = args.upper_right.split(",") coords = {"xll": xll, "yll": yll, "xur": xur, "yur": yur} if not os.path.isdir(src): - print(f"ERROR: input folder {args.input_folder} does not exist, exiting") + print(f"ERROR: input folder {src} does not exist, exiting") sys.exit(1) - if not os.path.isdir(dst): - os.mkdirs(dst) + if not os.path.isdir(dest): + os.makedirs(dest) - if not os.path.isdir(cache): - os.mkdirs(cache) - - shapefiles = shapefiles.find(src) - categories = shapefiles.categorize(shapefiles) - extents = shapefiles.get_extents(categorized) - shapefiles = shapefiles.merge_slice(extents, coords) - result = shapefiles.decode(shapefiles, dest) + shapefiles = find(src) + categories = categorize(shapefiles) + #extents = get_extents(categorized) + #shapefiles = merge_slice(extents, coords, dest) + shapefiles = merge_slice(categories, coords, dest) + #result = shapefiles.decode(shapefiles, dest)