Minor refactoring, concurrency module, max concurrency for DEM

opendm/concurrency.py

@@ -0,0 +1,26 @@
+from psutil import virtual_memory
+import os
+
+def get_max_memory(minimum = 5, use_at_most = 0.5):
+    """
+    :param minimum minimum value to return (return value will never be lower than this)
+    :param use_at_most use at most this fraction of the available memory. 0.5 = use at most 50% of available memory
+    :return percentage value of memory to use (75 = 75%).
+    """
+    return max(minimum, (100 - virtual_memory().percent) * use_at_most)
+
+
+def get_max_concurrency_for_dem(available_cores, input_file, use_at_most = 0.8):
+    """
+    DEM generation requires ~2x the input file size of memory per available core.
+    :param available_cores number of cores available (return value will never exceed this value)
+    :param input_file path to input file
+    :use_at_most use at most this fraction of the available memory when calculating a concurrency value. 0.9 = assume that we can only use 90% of available memory.
+    :return maximum number of cores recommended to use for DEM processing.
+    """
+    memory_available = virtual_memory().available * use_at_most
+    file_size = os.path.getsize(input_file)
+    memory_required_per_core = max(1, file_size * 2)
+    
+    return min(available_cores, max(1, int(memory_available) / int(memory_required_per_core)))
+

opendm/cropper.py

@@ -3,7 +3,7 @@ from opendm.system import run
 from opendm import log
 from osgeo import ogr
 import json, os
-from psutil import virtual_memory
+from opendm.concurrency import get_max_memory
 
 class Cropper:
     def __init__(self, storage_dir, files_prefix = "crop"):
@@ -42,7 +42,7 @@ class Cropper:
                 'geotiffInput': original_geotiff,
                 'geotiffOutput': geotiff_path,
                 'options': ' '.join(map(lambda k: '-co {}={}'.format(k, gdal_options[k]), gdal_options)),
-                'max_memory': max(5, (100 - virtual_memory().percent) / 2)
+                'max_memory': get_max_memory()
             }
 
             run('gdalwarp -cutline {shapefile_path} '

opendm/mesh.py

@@ -5,10 +5,11 @@ from opendm.dem import commands
 from opendm import system
 from opendm import log
 from opendm import context
+from opendm.concurrency import get_max_concurrency_for_dem
 from scipy import signal, ndimage
 import numpy as np
 
-def create_25dmesh(inPointCloud, outMesh, dsm_radius=0.07, dsm_resolution=0.05, depth=8, samples=1, maxVertexCount=100000, verbose=False, max_workers=None, method='gridded'):
+def create_25dmesh(inPointCloud, outMesh, dsm_radius=0.07, dsm_resolution=0.05, depth=8, samples=1, maxVertexCount=100000, verbose=False, available_cores=None, method='gridded'):
     # Create DSM from point cloud
 
     # Create temporary directory
@@ -32,7 +33,7 @@ def create_25dmesh(inPointCloud, outMesh, dsm_radius=0.07, dsm_resolution=0.05,
             resolution=dsm_resolution,
             products=['max'],
             verbose=verbose,
-            max_workers=max_workers
+            max_workers=get_max_concurrency_for_dem(available_cores, inPointCloud)
         )
 
     if method == 'gridded':
@@ -42,7 +43,7 @@ def create_25dmesh(inPointCloud, outMesh, dsm_radius=0.07, dsm_resolution=0.05,
         mesh = screened_poisson_reconstruction(dsm_points, outMesh, depth=depth, 
                                     samples=samples, 
                                     maxVertexCount=maxVertexCount, 
-                                    threads=max_workers,
+                                    threads=available_cores,
                                     verbose=verbose)
     else:
         raise 'Not a valid method: ' + method

scripts/odm_dem.py

@@ -9,7 +9,7 @@ from opendm import types
 from opendm import gsd
 from opendm.dem import commands
 from opendm.cropper import Cropper
-
+from opendm.concurrency import get_max_concurrency_for_dem
 
 class ODMDEMCell(ecto.Cell):
     def declare_params(self, params):
@@ -109,7 +109,7 @@ class ODMDEMCell(ecto.Cell):
                             maxangle=args.dem_maxangle,
                             decimation=args.dem_decimation,
                             verbose=args.verbose,
-                            max_workers=args.max_concurrency
+                            max_workers=get_max_concurrency_for_dem(args.max_concurrency,tree.odm_georeferencing_model_laz)
                         )
 
                     if args.crop > 0:

scripts/odm_meshing.py

@@ -104,7 +104,7 @@ class ODMeshingCell(ecto.Cell):
                     maxVertexCount=self.params.max_vertex,
                     samples=self.params.samples,
                     verbose=self.params.verbose,
-                    max_workers=args.max_concurrency,
+                    available_cores=args.max_concurrency,
                     method='poisson' if args.fast_orthophoto else 'gridded')
           else:
               log.ODM_WARNING('Found a valid ODM 2.5D Mesh file in: %s' %

scripts/odm_orthophoto.py

@@ -1,5 +1,4 @@
 import ecto, os
-from psutil import virtual_memory
 
 from opendm import io
 from opendm import log
@@ -7,6 +6,7 @@ from opendm import system
 from opendm import context
 from opendm import types
 from opendm import gsd
+from opendm.concurrency import get_max_memory
 from opendm.cropper import Cropper
 
 
@@ -127,7 +127,7 @@ class ODMOrthoPhotoCell(ecto.Cell):
                     'png': tree.odm_orthophoto_file,
                     'tiff': tree.odm_orthophoto_tif,
                     'log': tree.odm_orthophoto_tif_log,
-                    'max_memory': max(5, (100 - virtual_memory().percent) / 2),
+                    'max_memory': get_max_memory(),
                     'threads': self.params.max_concurrency
                 }