From e46ff4ee78657343f1bedcae1618e60af27f5c74 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 4 May 2021 14:46:55 -0400 Subject: [PATCH] End-to-end pipeline runs --- opendm/cropper.py | 16 ++++++++++------ opendm/dem/commands.py | 4 ++-- opendm/dem/pdal.py | 4 ++-- opendm/osfm.py | 2 +- opendm/point_cloud.py | 6 +++--- opendm/tiles/tiler.py | 5 +++-- opendm/utils.py | 14 +++++++++++++- run.py | 26 +++++++++++++------------- stages/odm_orthophoto.py | 8 ++++---- stages/openmvs.py | 2 +- stages/splitmerge.py | 4 ++-- 11 files changed, 54 insertions(+), 37 deletions(-) diff --git a/opendm/cropper.py b/opendm/cropper.py index f2a5b4d7..af4b2a0b 100644 --- a/opendm/cropper.py +++ b/opendm/cropper.py @@ -37,7 +37,7 @@ class Cropper: # ext = .tif original_geotiff = os.path.join(path, "{}.original{}".format(basename, ext)) - os.rename(geotiff_path, original_geotiff) + os.replace(geotiff_path, original_geotiff) try: kwargs = { @@ -64,7 +64,7 @@ class Cropper: log.ODM_WARNING('Something went wrong while cropping: {}'.format(e)) # Revert rename - os.rename(original_geotiff, geotiff_path) + os.replace(original_geotiff, geotiff_path) return geotiff_path @@ -159,8 +159,8 @@ class Cropper: if pc_geojson_boundary_feature is None: raise RuntimeError("Could not determine point cloud boundaries") # Write bounds to GeoJSON - bounds_geojson_path = self.path('bounds.geojson') - with open(bounds_geojson_path, "w") as f: + tmp_bounds_geojson_path = self.path('tmp-bounds.geojson') + with open(tmp_bounds_geojson_path, "w") as f: f.write(json.dumps({ "type": "FeatureCollection", "features": [{ @@ -172,7 +172,7 @@ class Cropper: # Create a convex hull around the boundary # as to encompass the entire area (no holes) driver = ogr.GetDriverByName('GeoJSON') - ds = driver.Open(bounds_geojson_path, 0) # ready-only + ds = driver.Open(tmp_bounds_geojson_path, 0) # ready-only layer = ds.GetLayer() # Collect all Geometry @@ -202,7 +202,7 @@ class Cropper: # Save to a new file bounds_geojson_path = self.path('bounds.geojson') if os.path.exists(bounds_geojson_path): - driver.DeleteDataSource(bounds_geojson_path) + os.remove(bounds_geojson_path) out_ds = driver.CreateDataSource(bounds_geojson_path) layer = out_ds.CreateLayer("convexhull", geom_type=ogr.wkbPolygon) @@ -219,6 +219,10 @@ class Cropper: # Remove decimated point cloud if os.path.exists(decimated_pointcloud_path): os.remove(decimated_pointcloud_path) + + # Remove tmp bounds + if os.path.exists(tmp_bounds_geojson_path): + os.remove(tmp_bounds_geojson_path) return bounds_geojson_path diff --git a/opendm/dem/commands.py b/opendm/dem/commands.py index 8fc68441..94c2a004 100755 --- a/opendm/dem/commands.py +++ b/opendm/dem/commands.py @@ -258,13 +258,13 @@ def create_dem(input_point_cloud, dem_type, output_type='max', radiuses=['0.56'] median_smoothing(geotiff_path, output_path) os.remove(geotiff_path) else: - os.rename(geotiff_path, output_path) + os.replace(geotiff_path, output_path) if os.path.exists(geotiff_tmp_path): if not keep_unfilled_copy: os.remove(geotiff_tmp_path) else: - os.rename(geotiff_tmp_path, io.related_file_path(output_path, postfix=".unfilled")) + os.replace(geotiff_tmp_path, io.related_file_path(output_path, postfix=".unfilled")) for cleanup_file in [tiles_vrt_path, merged_vrt_path, geotiff_small_path, geotiff_small_filled_path]: if os.path.exists(cleanup_file): os.remove(cleanup_file) diff --git a/opendm/dem/pdal.py b/opendm/dem/pdal.py index 1b558266..7224d975 100644 --- a/opendm/dem/pdal.py +++ b/opendm/dem/pdal.py @@ -36,9 +36,9 @@ import json as jsonlib import tempfile from opendm import system from opendm import log +from opendm.utils import double_quote from datetime import datetime -from pipes import quote """ JSON Functions """ @@ -191,7 +191,7 @@ def merge_point_clouds(input_files, output_file, verbose=False): cmd = [ 'pdal', 'merge', - ' '.join(map(quote, input_files + [output_file])), + ' '.join(map(double_quote, input_files + [output_file])), ] if verbose: diff --git a/opendm/osfm.py b/opendm/osfm.py index bccedb8b..835e9bca 100644 --- a/opendm/osfm.py +++ b/opendm/osfm.py @@ -349,7 +349,7 @@ class OSFMContext: # (containing only the primary band) if os.path.exists(self.recon_file()): os.remove(self.recon_file()) - os.rename(self.recon_backup_file(), self.recon_file()) + os.replace(self.recon_backup_file(), self.recon_file()) log.ODM_INFO("Restored reconstruction.json") def backup_reconstruction(self): diff --git a/opendm/point_cloud.py b/opendm/point_cloud.py index 6ae92136..163c1822 100644 --- a/opendm/point_cloud.py +++ b/opendm/point_cloud.py @@ -6,7 +6,7 @@ from opendm.system import run from opendm import entwine from opendm import io from opendm.concurrency import parallel_map -from pipes import quote +from opendm.utils import double_quote def ply_info(input_ply): if not os.path.exists(input_ply): @@ -240,7 +240,7 @@ def merge(input_point_cloud_files, output_file, rerun=False): os.remove(output_file) kwargs = { - 'all_inputs': " ".join(map(quote, input_point_cloud_files)), + 'all_inputs': " ".join(map(double_quote, input_point_cloud_files)), 'output': output_file } @@ -312,7 +312,7 @@ def merge_ply(input_point_cloud_files, output_file, dims=None): '--writers.ply.sized_types=false', '--writers.ply.storage_mode="little endian"', ('--writers.ply.dims="%s"' % dims) if dims is not None else '', - ' '.join(map(quote, input_point_cloud_files + [output_file])), + ' '.join(map(double_quote, input_point_cloud_files + [output_file])), ] system.run(' '.join(cmd)) diff --git a/opendm/tiles/tiler.py b/opendm/tiles/tiler.py index 59dc185a..6b36f209 100644 --- a/opendm/tiles/tiler.py +++ b/opendm/tiles/tiler.py @@ -1,11 +1,12 @@ import os +import sys from opendm import log from opendm import system from opendm import io def generate_tiles(geotiff, output_dir, max_concurrency): gdal2tiles = os.path.join(os.path.dirname(__file__), "gdal2tiles.py") - system.run('python3 "%s" --processes %s -z 5-21 -n -w none "%s" "%s"' % (gdal2tiles, max_concurrency, geotiff, output_dir)) + system.run('%s "%s" --processes %s -z 5-21 -n -w none "%s" "%s"' % (sys.executable, gdal2tiles, max_concurrency, geotiff, output_dir)) def generate_orthophoto_tiles(geotiff, output_dir, max_concurrency): try: @@ -29,7 +30,7 @@ def generate_colored_hillshade(geotiff): system.run('gdaldem color-relief "%s" "%s" "%s" -alpha -co ALPHA=YES' % (geotiff, relief_file, colored_dem)) system.run('gdaldem hillshade "%s" "%s" -z 1.0 -s 1.0 -az 315.0 -alt 45.0' % (geotiff, hillshade_dem)) - system.run('python3 "%s" "%s" "%s" "%s"' % (hsv_merge_script, colored_dem, hillshade_dem, colored_hillshade_dem)) + system.run('%s "%s" "%s" "%s" "%s"' % (sys.executable, hsv_merge_script, colored_dem, hillshade_dem, colored_hillshade_dem)) return outputs except Exception as e: diff --git a/opendm/utils.py b/opendm/utils.py index c365839b..16c90363 100644 --- a/opendm/utils.py +++ b/opendm/utils.py @@ -1,6 +1,7 @@ from opendm import log from opendm.photo import find_largest_photo_dim from osgeo import gdal +from shlex import _find_unsafe def get_depthmap_resolution(args, photos): if 'depthmap_resolution_is_set' in args: @@ -39,4 +40,15 @@ def get_raster_stats(geotiff): 'stddev': s[3] }) - return stats \ No newline at end of file + return stats + +def double_quote(s): + """Return a shell-escaped version of the string *s*.""" + if not s: + return '""' + if _find_unsafe(s) is None: + return s + + # use double quotes, and prefix double quotes with a \ + # the string $"b is then quoted as "$\"b" + return '"' + s.replace('"', '\\\"') + '"' \ No newline at end of file diff --git a/run.py b/run.py index 7b87f109..d140adf4 100755 --- a/run.py +++ b/run.py @@ -11,9 +11,9 @@ from opendm import config from opendm import system from opendm import io from opendm.progress import progressbc +from opendm.utils import double_quote import os -from pipes import quote from stages.odm_app import ODMApp @@ -50,18 +50,18 @@ if __name__ == '__main__': log.ODM_INFO("Rerun all -- Removing old data") os.system("rm -rf " + " ".join([ - quote(os.path.join(args.project_path, "odm_georeferencing")), - quote(os.path.join(args.project_path, "odm_meshing")), - quote(os.path.join(args.project_path, "odm_orthophoto")), - quote(os.path.join(args.project_path, "odm_dem")), - quote(os.path.join(args.project_path, "odm_report")), - quote(os.path.join(args.project_path, "odm_texturing")), - quote(os.path.join(args.project_path, "opensfm")), - quote(os.path.join(args.project_path, "odm_filterpoints")), - quote(os.path.join(args.project_path, "odm_texturing_25d")), - quote(os.path.join(args.project_path, "openmvs")), - quote(os.path.join(args.project_path, "entwine_pointcloud")), - quote(os.path.join(args.project_path, "submodels")), + double_quote(os.path.join(args.project_path, "odm_georeferencing")), + double_quote(os.path.join(args.project_path, "odm_meshing")), + double_quote(os.path.join(args.project_path, "odm_orthophoto")), + double_quote(os.path.join(args.project_path, "odm_dem")), + double_quote(os.path.join(args.project_path, "odm_report")), + double_quote(os.path.join(args.project_path, "odm_texturing")), + double_quote(os.path.join(args.project_path, "opensfm")), + double_quote(os.path.join(args.project_path, "odm_filterpoints")), + double_quote(os.path.join(args.project_path, "odm_texturing_25d")), + double_quote(os.path.join(args.project_path, "openmvs")), + double_quote(os.path.join(args.project_path, "entwine_pointcloud")), + double_quote(os.path.join(args.project_path, "submodels")), ])) app = ODMApp(args) diff --git a/stages/odm_orthophoto.py b/stages/odm_orthophoto.py index 9cbad637..f9fac5de 100644 --- a/stages/odm_orthophoto.py +++ b/stages/odm_orthophoto.py @@ -9,7 +9,7 @@ from opendm import gsd from opendm import orthophoto from opendm.concurrency import get_max_memory from opendm.cutline import compute_cutline -from pipes import quote +from opendm.utils import double_quote from opendm import pseudogeo from opendm.multispectral import get_primary_band_name @@ -63,11 +63,11 @@ class ODMOrthoPhotoStage(types.ODM_Stage): if not primary: subdir = band['name'].lower() models.append(os.path.join(base_dir, subdir, model_file)) - kwargs['bands'] = '-bands %s' % (','.join([quote(b['name']) for b in reconstruction.multi_camera])) + kwargs['bands'] = '-bands %s' % (','.join([double_quote(b['name']) for b in reconstruction.multi_camera])) else: models.append(os.path.join(base_dir, model_file)) - kwargs['models'] = ','.join(map(quote, models)) + kwargs['models'] = ','.join(map(double_quote, models)) # run odm_orthophoto system.run('{odm_ortho_bin} -inputFiles {models} ' @@ -148,7 +148,7 @@ class ODMOrthoPhotoStage(types.ODM_Stage): if io.file_exists(tree.odm_orthophoto_render): pseudogeo.add_pseudo_georeferencing(tree.odm_orthophoto_render) log.ODM_INFO("Renaming %s --> %s" % (tree.odm_orthophoto_render, tree.odm_orthophoto_tif)) - os.rename(tree.odm_orthophoto_render, tree.odm_orthophoto_tif) + os.replace(tree.odm_orthophoto_render, tree.odm_orthophoto_tif) else: log.ODM_WARNING("Could not generate an orthophoto (it did not render)") else: diff --git a/stages/openmvs.py b/stages/openmvs.py index 15bd7054..a89f261c 100644 --- a/stages/openmvs.py +++ b/stages/openmvs.py @@ -144,7 +144,7 @@ class ODMOpenMVSStage(types.ODM_Stage): log.ODM_ERROR("Could not compute dense point cloud (no PLY files available).") if len(scene_ply_files) == 1: # Simply rename - os.rename(scene_ply_files[0], tree.openmvs_model) + os.replace(scene_ply_files[0], tree.openmvs_model) log.ODM_INFO("%s --> %s"% (scene_ply_files[0], tree.openmvs_model)) else: # Merge diff --git a/stages/splitmerge.py b/stages/splitmerge.py index 0e243a39..c2bd5d55 100644 --- a/stages/splitmerge.py +++ b/stages/splitmerge.py @@ -17,7 +17,7 @@ from opendm.concurrency import get_max_memory from opendm.remote import LocalRemoteExecutor from opendm.shots import merge_geojson_shots from opendm import point_cloud -from pipes import quote +from opendm.utils import double_quote from opendm.tiles.tiler import generate_dem_tiles class ODMSplitStage(types.ODM_Stage): @@ -219,7 +219,7 @@ class ODMSplitStage(types.ODM_Stage): argv = get_submodel_argv(args, tree.submodels_path, sp_octx.name()) # Re-run the ODM toolchain on the submodel - system.run(" ".join(map(quote, map(str, argv))), env_vars=os.environ.copy()) + system.run(" ".join(map(double_quote, map(str, argv))), env_vars=os.environ.copy()) else: lre.set_projects([os.path.abspath(os.path.join(p, "..")) for p in submodel_paths]) lre.run_toolchain()