FGFS-4.1/fgtools
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix errors

Browse Source
This commit is contained in:
TheFGFSEagle
2022-07-29 17:06:24 +02:00
parent a05f4fff32
commit 9a66671ba3
1 changed files with 51 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scenery/process-shapefiles.py
+51 -57
View File
@@ -9,6 +9,9 @@ import glob
import re import re
import argparse import argparse
import subprocess import subprocess
import tempfile
from fgtools.utils import constants
DESCRIPTION = """ DESCRIPTION = """
process-shapefiles.py - merges, slices, and decodes OSM shapefiles process-shapefiles.py - merges, slices, and decodes OSM shapefiles
@@ -21,71 +24,71 @@ For this reason, you may only remove the 'gis_' and '_free_1' parts from the sha
Everything else in the name must be conserved in order for your resulting scenery not to have big areas of default landmass terrain !'' Everything else in the name must be conserved in order for your resulting scenery not to have big areas of default landmass terrain !''
Then, for each shapefile in each category the extents will be queried using ogrinfo. To reduce processing time on subsequent runs, the results will be cached. Then, for each shapefile in each category the extents will be queried using ogrinfo. To reduce processing time on subsequent runs, the results will be cached.
Then, the script will decide whether to merge or slice the shapefiles for each category based on the coordinates you input. Then, the script will merge and slice the shapefiles for each category based on the coordinates you input.
It will merge / slice the files accordingly with ogr2ogr. It will merge / slice the files accordingly with ogr2ogr.
As the final step, the resulting shapefiles will be decoded into files that tg-constrcut can read using ogr-decode.
""" """
#As the final step, the resulting shapefiles will be decoded into files that tg-constrcut can read using ogr-decode.
#"""
catnames = ["buildings", "landuse", "natural", "places", "pofw", "pois", "railways", "roads", "traffic", "transport", "water", "waterways"]
tmpdir = tempfile.TemporaryDirectory()
def find(src): def find(src):
osm_shapefiles = [] osm_shapefiles = []
shapefiles = glob.glob(os.path.join(src, "**", "**.shp") files = glob.glob(os.path.join(src, "**", "**.shp"))
for file in files: for file in files:
if "osm" in os.path.split(file): if "osm" in os.path.split(file):
osm_shapefiles.append(file) osm_shapefiles.append(file)
return sorted(osm_shapefiles) return sorted(osm_shapefiles)
def categorize(shapefiles): def categorize(shapefiles):
catnames = ["buildings", "landuse", "natural", "places", "pofw", "pois", "railways", "roads", "traffic", "transport", "water", "waterways"]
categorized = [] categorized = []
for shapefile in shapefiles: for shapefile in shapefiles:
name = os.path.split(shapefile)[-1].split(".")[0] name = os.path.split(shapefile)[-1].split(".")[0]
# Skip if the name is of the form gis_osm_landuse_a_free_1
# these files are much smaller than the ones without _a_, probably contain less data.
if "_a_" in name:
continue
name = re.sub(r"gis|osm|a|free|_|(1-9)", "", name) # gis_osm_landuse_a_free_1 becomes just landuse name = re.sub(r"gis|osm|a|free|_|(1-9)", "", name) # gis_osm_landuse_a_free_1 becomes just landuse
# Skip if the name is not a recognized catname # Skip if the name is not a recognized catname
if not name in catnames: if not name in catnames:
continue continue
categorized.append({"path": shapefile, "category": name}) categorized[name].append({"path": shapefile})
return categorized return categorized
def get_extents(categorized): def get_extents(categorized):
for shapefile in categorized: for category in categorized:
cmd = f"ogrinfo -al -so -ro -nocount -nomd {shapefile['path']}" for shapefile in categorized[category]:
query = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) cmd = f"ogrinfo -al -so -ro -nocount -nomd {shapefile['path']}"
output = list(map(lambda s: s.decode(), query.stdout.splitlines())) # subprocess.Popen.stdout is a binary file - we need normal strings query = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = list(map(lambda s: s.decode(), query.stdout.splitlines())) # subprocess.Popen.stdout is a binary file - we need normal strings
extents = [s for s in output if "Extent" in s]
feature_count = [s for s in output if "Feature Count" in s]
if len(extents) != 1 or len(feature_count) != 1:
print("ERROR: Fetching shapefile information using ogrinfo failed.")
print(" Try reinstalling it through your package manager.")
print(" If that doesn't help, please file a bug report at <github.com/TheFGFSEagle/terragear-tools/issues>.")
print(" If you do that, please attach the process-shapefiles-bugreport.md file in order for the maintainers to be able to help you.")
with open("process-shapefile-bugreport.md", "w") as f: extents = [s for s in output if "Extent" in s]
f.write(f"### Output of `{cmd}`") feature_count = [s for s in output if "Feature Count" in s]
f.writelines(output)
f.write(f"### ogrinfo version:") if len(extents) != 1 or len(feature_count) != 1:
f.write(subprocess.run("ogrinfo --version", stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=True).stdout.deocde()) print("ERROR: Fetching shapefile information using ogrinfo failed.")
sys.exit(2) print(" Try reinstalling it through your package manager.")
print(" If that doesn't help, please file a bug report at <github.com/TheFGFSEagle/terragear-tools/issues>.")
# convert "Extent: (10.544425, 51.500000) - (11.500000, 52.500000)" to {"xll": 10.544425, "yll": 51.5, "xur": 11.5, "yur": 52,5] print(" If you do that, please attach the process-shapefiles-bugreport.md file in order for the maintainers to be able to help you.")
extents = dict(zip(["xll", "yll", "yur", "yur"], map(float, re.sub(r"Extent:\s\(|\)", "", extents[0]).replace(") - (", ", ").split(", "))))
shapefile["extents"] = extents with open("process-shapefile-bugreport.md", "w") as f:
f.write(f"### Output of `{cmd}`")
f.writelines(output)
f.write(f"### ogrinfo version:")
f.write(subprocess.run("ogrinfo --version", stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=True).stdout.deocde())
sys.exit(2)
# convert "Extent: (10.544425, 51.500000) - (11.500000, 52.500000)" to {"xll": 10.544425, "yll": 51.5, "xur": 11.5, "yur": 52,5]
extents = dict(zip(["xll", "yll", "yur", "yur"], map(float, re.sub(r"Extent:\s\(|\)", "", extents[0]).replace(") - (", ", ").split(", "))))
shapefile["extents"] = extents
def merge_slice(shapefiles, coords): def merge_slice(shapefiles, coords, dest):
pass for category in shapefiles:
files = ['"' + shapefile["path"] + '"' for shapefile in shapefiles[category]]
cmd = f'ogr2ogr -f "ESRI Shapefile" {os.path.join(dest, category + ".shp")} {" ".join(files)} -clipsrc {coords["xll"]} {coords["yll"]} {coords["xur"]} {coords["yur"]} -progress -single -lco ENCODING=UTF-8'
subprocess.run(cmd, shell=True)
def decode(shapefiles, dest): #def decode(shapefiles, dest):
pass # pass
if __name__ == "__main__": if __name__ == "__main__":
argp = argparse.ArgumentParser(description="process-shapefiles.py - merges, slices, and decodes OSM shapefiles") argp = argparse.ArgumentParser(description="process-shapefiles.py - merges, slices, and decodes OSM shapefiles")
@@ -93,7 +96,7 @@ if __name__ == "__main__":
argp.add_argument( argp.add_argument(
"-v", "--version", "-v", "--version",
action="version", action="version",
version=f"TerraGear tools {'.'.join(map(str, constants.__version__))}" version=f"FGTools {'.'.join(map(str, constants.__version__))}"
) )
argp.add_argument( argp.add_argument(
@@ -115,12 +118,6 @@ if __name__ == "__main__":
metavar="FOLDER" metavar="FOLDER"
) )
argp.add_argument(
"-c", "--cache-folder",
help="where to put cache folder (default: %(default)s)",
default=os.path.join(constants.HOME, ".cache", "tgtools")
)
argp.add_argument( argp.add_argument(
"-l", "--lower-left", "-l", "--lower-left",
help="coordinates of the lower left corner of the bounding box of the region that shapefiles should be processed for (default: %(default)s)", help="coordinates of the lower left corner of the bounding box of the region that shapefiles should be processed for (default: %(default)s)",
@@ -141,23 +138,20 @@ if __name__ == "__main__":
src = args.input_folder src = args.input_folder
dest = args.output_folder dest = args.output_folder
cache = args.cache_folder
xll, yll = args.lower_left.split(",") xll, yll = args.lower_left.split(",")
xur, yur = args.upper_right.split(",") xur, yur = args.upper_right.split(",")
coords = {"xll": xll, "yll": yll, "xur": xur, "yur": yur} coords = {"xll": xll, "yll": yll, "xur": xur, "yur": yur}
if not os.path.isdir(src): if not os.path.isdir(src):
print(f"ERROR: input folder {args.input_folder} does not exist, exiting") print(f"ERROR: input folder {src} does not exist, exiting")
sys.exit(1) sys.exit(1)
if not os.path.isdir(dst): if not os.path.isdir(dest):
os.mkdirs(dst) os.makedirs(dest)
if not os.path.isdir(cache): shapefiles = find(src)
os.mkdirs(cache) categories = categorize(shapefiles)
#extents = get_extents(categorized)
shapefiles = shapefiles.find(src) #shapefiles = merge_slice(extents, coords, dest)
categories = shapefiles.categorize(shapefiles) shapefiles = merge_slice(categories, coords, dest)
extents = shapefiles.get_extents(categorized) #result = shapefiles.decode(shapefiles, dest)
shapefiles = shapefiles.merge_slice(extents, coords)
result = shapefiles.decode(shapefiles, dest)