diff --git a/opendm/config.py b/opendm/config.py index 645cb23e..b1847d9d 100755 --- a/opendm/config.py +++ b/opendm/config.py @@ -702,6 +702,15 @@ def config(argv=None): 'This flag is useful if you have high precision GPS measurements. ' 'If there are no GCPs, this flag does nothing. Default: %(default)s')) + parser.add_argument('--optimize-disk-space', + action=StoreTrue, + nargs=0, + default=False, + help=('Delete heavy intermediate files to optimize disk space usage. This ' + 'affects the ability to restart the pipeline from an intermediate stage, ' + 'but allows datasets to be processed on machines that don\'t have sufficient ' + 'disk space available. Default: %(default)s')) + parser.add_argument('--pc-rectify', action=StoreTrue, nargs=0, diff --git a/opendm/orthophoto.py b/opendm/orthophoto.py index 7f6b4d11..b4c635fd 100644 --- a/opendm/orthophoto.py +++ b/opendm/orthophoto.py @@ -44,7 +44,7 @@ def generate_png(orthophoto_file): def post_orthophoto_steps(args, bounds_file_path, orthophoto_file): if args.crop > 0: - Cropper.crop(bounds_file_path, orthophoto_file, get_orthophoto_vars(args), warp_options=['-dstalpha']) + Cropper.crop(bounds_file_path, orthophoto_file, get_orthophoto_vars(args), keep_original=not args.optimize_disk_space, warp_options=['-dstalpha']) if args.build_overviews: build_overviews(orthophoto_file) diff --git a/stages/mve.py b/stages/mve.py index da02e3c4..9828d848 100644 --- a/stages/mve.py +++ b/stages/mve.py @@ -93,6 +93,9 @@ class ODMMveStage(types.ODM_Stage): os.rename(mve_filtered_model, tree.mve_model) else: log.ODM_WARNING("Couldn't filter MVE model (%s does not exist)." % mve_filtered_model) + + if args.optimize_disk_space: + shutil.rmtree(tree.mve_views) else: log.ODM_WARNING('Found a valid MVE reconstruction file in: %s' % tree.mve_model) diff --git a/stages/mvstex.py b/stages/mvstex.py index 94c37808..8a90e8a0 100644 --- a/stages/mvstex.py +++ b/stages/mvstex.py @@ -114,9 +114,27 @@ class ODMMvsTexStage(types.ODM_Stage): '{nadirMode} ' '-n {nadirWeight}'.format(**kwargs)) + if args.optimize_disk_space: + cleanup_files = [ + os.path.join(r['out_dir'], "odm_textured_model_data_costs.spt"), + os.path.join(r['out_dir'], "odm_textured_model_labeling.vec"), + ] + for f in cleanup_files: + if io.file_exists(f): + os.remove(f) + progress += progress_per_run self.update_progress(progress) else: log.ODM_WARNING('Found a valid ODM Texture file in: %s' % odm_textured_model_obj) + + if args.optimize_disk_space: + for r in nonloc.runs: + if io.file_exists(r['model']): + os.remove(r['model']) + + undistorted_images_path = os.path.join(tree.opensfm, "undistorted", "images") + if io.dir_exists(undistorted_images_path): + shutil.rmtree(undistorted_images_path) diff --git a/stages/odm_dem.py b/stages/odm_dem.py index d68c7726..5c7c85e3 100755 --- a/stages/odm_dem.py +++ b/stages/odm_dem.py @@ -116,14 +116,14 @@ class ODMDEMStage(types.ODM_Stage): if args.crop > 0: # Crop DEM - Cropper.crop(bounds_file_path, dem_geotiff_path, utils.get_dem_vars(args)) + Cropper.crop(bounds_file_path, dem_geotiff_path, utils.get_dem_vars(args), keep_original=not args.optimize_disk_space) if args.dem_euclidean_map: unfilled_dem_path = io.related_file_path(dem_geotiff_path, postfix=".unfilled") if args.crop > 0: # Crop unfilled DEM - Cropper.crop(bounds_file_path, unfilled_dem_path, utils.get_dem_vars(args)) + Cropper.crop(bounds_file_path, unfilled_dem_path, utils.get_dem_vars(args), keep_original=not args.optimize_disk_space) commands.compute_euclidean_map(unfilled_dem_path, io.related_file_path(dem_geotiff_path, postfix=".euclideand"), diff --git a/stages/odm_filterpoints.py b/stages/odm_filterpoints.py index 5179b755..182fa867 100644 --- a/stages/odm_filterpoints.py +++ b/stages/odm_filterpoints.py @@ -28,6 +28,10 @@ class ODMFilterPoints(types.ODM_Stage): confidence=None, sample_radius=args.pc_sample, verbose=args.verbose) + else: log.ODM_WARNING('Found a valid point cloud file in: %s' % tree.filtered_point_cloud) + + if args.optimize_disk_space: + os.remove(inputPointCloud) diff --git a/stages/odm_georeferencing.py b/stages/odm_georeferencing.py index 3d0d44c5..a889a362 100644 --- a/stages/odm_georeferencing.py +++ b/stages/odm_georeferencing.py @@ -137,6 +137,9 @@ class ODMGeoreferencingStage(types.ODM_Stage): else: log.ODM_WARNING('Found a valid georeferenced model in: %s' % tree.odm_georeferencing_model_laz) + + if args.optimize_disk_space and io.file_exists(tree.odm_georeferencing_model_laz) and io.file_exists(tree.filtered_point_cloud): + os.remove(tree.filtered_point_cloud) progress += progress_per_run self.update_progress(progress) diff --git a/stages/odm_orthophoto.py b/stages/odm_orthophoto.py index 65c5af69..f87c981b 100644 --- a/stages/odm_orthophoto.py +++ b/stages/odm_orthophoto.py @@ -168,3 +168,6 @@ class ODMOrthoPhotoStage(types.ODM_Stage): log.ODM_WARNING("Could not generate an orthophoto (it did not render)") else: log.ODM_WARNING('Found a valid orthophoto in: %s' % tree.odm_orthophoto_tif) + + if args.optimize_disk_space and io.file_exists(tree.odm_orthophoto_render): + os.remove(tree.odm_orthophoto_render) diff --git a/stages/run_opensfm.py b/stages/run_opensfm.py index 1694b42c..e806c655 100644 --- a/stages/run_opensfm.py +++ b/stages/run_opensfm.py @@ -1,5 +1,7 @@ import sys import os +import shutil +import glob from opendm import log from opendm import io @@ -31,6 +33,12 @@ class ODMOpenSfMStage(types.ODM_Stage): octx.extract_cameras(tree.path("cameras.json"), self.rerun()) self.update_progress(70) + if args.optimize_disk_space: + shutil.rmtree(octx.path("features")) + shutil.rmtree(octx.path("matches")) + shutil.rmtree(octx.path("exif")) + shutil.rmtree(octx.path("reports")) + # If we find a special flag file for split/merge we stop right here if os.path.exists(octx.path("split_merge_stop_at_reconstruction.txt")): log.ODM_INFO("Stopping OpenSfM early because we found: %s" % octx.path("split_merge_stop_at_reconstruction.txt")) @@ -117,4 +125,13 @@ class ODMOpenSfMStage(types.ODM_Stage): if reconstruction.is_georeferenced() and (not io.file_exists(tree.opensfm_transformation) or self.rerun()): octx.run('export_geocoords --transformation --proj \'%s\'' % reconstruction.georef.proj4()) else: - log.ODM_WARNING("Will skip exporting %s" % tree.opensfm_transformation) \ No newline at end of file + log.ODM_WARNING("Will skip exporting %s" % tree.opensfm_transformation) + + if args.optimize_disk_space: + os.remove(octx.path("tracks.csv")) + os.remove(octx.path("undistorted", "tracks.csv")) + os.remove(octx.path("undistorted", "reconstruction.json")) + if io.dir_exists(octx.path("undistorted", "depthmaps")): + files = glob.glob(octx.path("undistorted", "depthmaps", "*.npz")) + for f in files: + os.remove(f) \ No newline at end of file diff --git a/stages/splitmerge.py b/stages/splitmerge.py index fd87c00e..67f16cdb 100644 --- a/stages/splitmerge.py +++ b/stages/splitmerge.py @@ -260,7 +260,7 @@ class ODMMergeStage(types.ODM_Stage): if io.file_exists(dem_file): # Crop if args.crop > 0: - Cropper.crop(merged_bounds_file, dem_file, dem_vars) + Cropper.crop(merged_bounds_file, dem_file, dem_vars, keep_original=not args.optimize_disk_space) log.ODM_INFO("Created %s" % dem_file) else: log.ODM_WARNING("Cannot merge %s, %s was not created" % (human_name, dem_file))