Merge pull request #1096 from pierotofy/diskoptimize

Optimize disk space

Former-commit-id: d4ea030d79

opendm/config.py

@@ -702,6 +702,15 @@ def config(argv=None):
                           'This flag is useful if you have high precision GPS measurements. '
                           'If there are no GCPs, this flag does nothing. Default: %(default)s'))
 
+    parser.add_argument('--optimize-disk-space',
+                action=StoreTrue,
+                nargs=0,
+                default=False,
+                help=('Delete heavy intermediate files to optimize disk space usage. This '
+                      'affects the ability to restart the pipeline from an intermediate stage, '
+                      'but allows datasets to be processed on machines that don\'t have sufficient '
+                      'disk space available. Default: %(default)s'))
+
     parser.add_argument('--pc-rectify',
                     action=StoreTrue,
                     nargs=0,

opendm/orthophoto.py

@@ -44,7 +44,7 @@ def generate_png(orthophoto_file):
 
 def post_orthophoto_steps(args, bounds_file_path, orthophoto_file):
     if args.crop > 0:
-        Cropper.crop(bounds_file_path, orthophoto_file, get_orthophoto_vars(args), warp_options=['-dstalpha'])
+        Cropper.crop(bounds_file_path, orthophoto_file, get_orthophoto_vars(args), keep_original=not args.optimize_disk_space, warp_options=['-dstalpha'])
 
     if args.build_overviews:
         build_overviews(orthophoto_file)

stages/mve.py

@@ -93,6 +93,9 @@ class ODMMveStage(types.ODM_Stage):
                     os.rename(mve_filtered_model, tree.mve_model)
                 else:
                     log.ODM_WARNING("Couldn't filter MVE model (%s does not exist)." % mve_filtered_model)
+        
+            if args.optimize_disk_space:
+                shutil.rmtree(tree.mve_views)
         else:
             log.ODM_WARNING('Found a valid MVE reconstruction file in: %s' %
                             tree.mve_model)

stages/mvstex.py

@@ -114,9 +114,27 @@ class ODMMvsTexStage(types.ODM_Stage):
                         '{nadirMode} '
                         '-n {nadirWeight}'.format(**kwargs))
                 
+                if args.optimize_disk_space:
+                    cleanup_files = [
+                        os.path.join(r['out_dir'], "odm_textured_model_data_costs.spt"),
+                        os.path.join(r['out_dir'], "odm_textured_model_labeling.vec"),
+                    ]
+                    for f in cleanup_files:
+                        if io.file_exists(f):
+                            os.remove(f)
+                
                 progress += progress_per_run
                 self.update_progress(progress)
             else:
                 log.ODM_WARNING('Found a valid ODM Texture file in: %s'
                                 % odm_textured_model_obj)
+        
+        if args.optimize_disk_space:
+            for r in nonloc.runs:
+                if io.file_exists(r['model']):
+                    os.remove(r['model'])
+            
+            undistorted_images_path = os.path.join(tree.opensfm, "undistorted", "images")
+            if io.dir_exists(undistorted_images_path):
+                shutil.rmtree(undistorted_images_path)
 

stages/odm_dem.py

@@ -116,14 +116,14 @@ class ODMDEMStage(types.ODM_Stage):
 
                     if args.crop > 0:
                         # Crop DEM
-                        Cropper.crop(bounds_file_path, dem_geotiff_path, utils.get_dem_vars(args))
+                        Cropper.crop(bounds_file_path, dem_geotiff_path, utils.get_dem_vars(args), keep_original=not args.optimize_disk_space)
 
                     if args.dem_euclidean_map:
                         unfilled_dem_path = io.related_file_path(dem_geotiff_path, postfix=".unfilled")
                         
                         if args.crop > 0:
                             # Crop unfilled DEM
-                            Cropper.crop(bounds_file_path, unfilled_dem_path, utils.get_dem_vars(args))
+                            Cropper.crop(bounds_file_path, unfilled_dem_path, utils.get_dem_vars(args), keep_original=not args.optimize_disk_space)
 
                         commands.compute_euclidean_map(unfilled_dem_path, 
                                             io.related_file_path(dem_geotiff_path, postfix=".euclideand"), 

stages/odm_filterpoints.py

@@ -28,6 +28,10 @@ class ODMFilterPoints(types.ODM_Stage):
                                 confidence=None, 
                                 sample_radius=args.pc_sample,
                                 verbose=args.verbose)
+            
         else:
             log.ODM_WARNING('Found a valid point cloud file in: %s' %
                             tree.filtered_point_cloud)
+        
+        if args.optimize_disk_space:
+            os.remove(inputPointCloud)

stages/odm_georeferencing.py

@@ -137,6 +137,9 @@ class ODMGeoreferencingStage(types.ODM_Stage):
             else:
                 log.ODM_WARNING('Found a valid georeferenced model in: %s'
                                 % tree.odm_georeferencing_model_laz)
+
+            if args.optimize_disk_space and io.file_exists(tree.odm_georeferencing_model_laz) and io.file_exists(tree.filtered_point_cloud):
+                os.remove(tree.filtered_point_cloud)
             
             progress += progress_per_run
             self.update_progress(progress)

stages/odm_orthophoto.py

@@ -168,3 +168,6 @@ class ODMOrthoPhotoStage(types.ODM_Stage):
                     log.ODM_WARNING("Could not generate an orthophoto (it did not render)")
         else:
             log.ODM_WARNING('Found a valid orthophoto in: %s' % tree.odm_orthophoto_tif)
+
+        if args.optimize_disk_space and io.file_exists(tree.odm_orthophoto_render):
+            os.remove(tree.odm_orthophoto_render)

stages/run_opensfm.py

@@ -1,5 +1,7 @@
 import sys
 import os
+import shutil
+import glob
 
 from opendm import log
 from opendm import io
@@ -31,6 +33,12 @@ class ODMOpenSfMStage(types.ODM_Stage):
         octx.extract_cameras(tree.path("cameras.json"), self.rerun())
         self.update_progress(70)
 
+        if args.optimize_disk_space:
+            shutil.rmtree(octx.path("features"))
+            shutil.rmtree(octx.path("matches"))
+            shutil.rmtree(octx.path("exif"))
+            shutil.rmtree(octx.path("reports"))
+
         # If we find a special flag file for split/merge we stop right here
         if os.path.exists(octx.path("split_merge_stop_at_reconstruction.txt")):
             log.ODM_INFO("Stopping OpenSfM early because we found: %s" % octx.path("split_merge_stop_at_reconstruction.txt"))
@@ -117,4 +125,13 @@ class ODMOpenSfMStage(types.ODM_Stage):
         if reconstruction.is_georeferenced() and (not io.file_exists(tree.opensfm_transformation) or self.rerun()):
             octx.run('export_geocoords --transformation --proj \'%s\'' % reconstruction.georef.proj4())
         else:
-            log.ODM_WARNING("Will skip exporting %s" % tree.opensfm_transformation)
+            log.ODM_WARNING("Will skip exporting %s" % tree.opensfm_transformation)
+
+        if args.optimize_disk_space:
+            os.remove(octx.path("tracks.csv"))
+            os.remove(octx.path("undistorted", "tracks.csv"))
+            os.remove(octx.path("undistorted", "reconstruction.json"))
+            if io.dir_exists(octx.path("undistorted", "depthmaps")):
+                files = glob.glob(octx.path("undistorted", "depthmaps", "*.npz"))
+                for f in files:
+                    os.remove(f)

stages/splitmerge.py

@@ -260,7 +260,7 @@ class ODMMergeStage(types.ODM_Stage):
                     if io.file_exists(dem_file):
                         # Crop
                         if args.crop > 0:
-                            Cropper.crop(merged_bounds_file, dem_file, dem_vars)
+                            Cropper.crop(merged_bounds_file, dem_file, dem_vars, keep_original=not args.optimize_disk_space)
                         log.ODM_INFO("Created %s" % dem_file)
                     else:
                         log.ODM_WARNING("Cannot merge %s, %s was not created" % (human_name, dem_file))