diff --git a/opendm/concurrency.py b/opendm/concurrency.py new file mode 100644 index 00000000..5e390528 --- /dev/null +++ b/opendm/concurrency.py @@ -0,0 +1,26 @@ +from psutil import virtual_memory +import os + +def get_max_memory(minimum = 5, use_at_most = 0.5): + """ + :param minimum minimum value to return (return value will never be lower than this) + :param use_at_most use at most this fraction of the available memory. 0.5 = use at most 50% of available memory + :return percentage value of memory to use (75 = 75%). + """ + return max(minimum, (100 - virtual_memory().percent) * use_at_most) + + +def get_max_concurrency_for_dem(available_cores, input_file, use_at_most = 0.8): + """ + DEM generation requires ~2x the input file size of memory per available core. + :param available_cores number of cores available (return value will never exceed this value) + :param input_file path to input file + :use_at_most use at most this fraction of the available memory when calculating a concurrency value. 0.9 = assume that we can only use 90% of available memory. + :return maximum number of cores recommended to use for DEM processing. + """ + memory_available = virtual_memory().available * use_at_most + file_size = os.path.getsize(input_file) + memory_required_per_core = max(1, file_size * 2) + + return min(available_cores, max(1, int(memory_available) / int(memory_required_per_core))) + diff --git a/opendm/cropper.py b/opendm/cropper.py index dac61a91..0762631c 100644 --- a/opendm/cropper.py +++ b/opendm/cropper.py @@ -3,7 +3,7 @@ from opendm.system import run from opendm import log from osgeo import ogr import json, os -from psutil import virtual_memory +from opendm.concurrency import get_max_memory class Cropper: def __init__(self, storage_dir, files_prefix = "crop"): @@ -42,7 +42,7 @@ class Cropper: 'geotiffInput': original_geotiff, 'geotiffOutput': geotiff_path, 'options': ' '.join(map(lambda k: '-co {}={}'.format(k, gdal_options[k]), gdal_options)), - 'max_memory': max(5, (100 - virtual_memory().percent) / 2) + 'max_memory': get_max_memory() } run('gdalwarp -cutline {shapefile_path} ' diff --git a/opendm/mesh.py b/opendm/mesh.py index 155f3934..51542285 100644 --- a/opendm/mesh.py +++ b/opendm/mesh.py @@ -5,10 +5,11 @@ from opendm.dem import commands from opendm import system from opendm import log from opendm import context +from opendm.concurrency import get_max_concurrency_for_dem from scipy import signal, ndimage import numpy as np -def create_25dmesh(inPointCloud, outMesh, dsm_radius=0.07, dsm_resolution=0.05, depth=8, samples=1, maxVertexCount=100000, verbose=False, max_workers=None, method='gridded'): +def create_25dmesh(inPointCloud, outMesh, dsm_radius=0.07, dsm_resolution=0.05, depth=8, samples=1, maxVertexCount=100000, verbose=False, available_cores=None, method='gridded'): # Create DSM from point cloud # Create temporary directory @@ -32,7 +33,7 @@ def create_25dmesh(inPointCloud, outMesh, dsm_radius=0.07, dsm_resolution=0.05, resolution=dsm_resolution, products=['max'], verbose=verbose, - max_workers=max_workers + max_workers=get_max_concurrency_for_dem(available_cores, inPointCloud) ) if method == 'gridded': @@ -42,7 +43,7 @@ def create_25dmesh(inPointCloud, outMesh, dsm_radius=0.07, dsm_resolution=0.05, mesh = screened_poisson_reconstruction(dsm_points, outMesh, depth=depth, samples=samples, maxVertexCount=maxVertexCount, - threads=max_workers, + threads=available_cores, verbose=verbose) else: raise 'Not a valid method: ' + method diff --git a/scripts/odm_dem.py b/scripts/odm_dem.py index 6622b71e..3e692c53 100644 --- a/scripts/odm_dem.py +++ b/scripts/odm_dem.py @@ -9,7 +9,7 @@ from opendm import types from opendm import gsd from opendm.dem import commands from opendm.cropper import Cropper - +from opendm.concurrency import get_max_concurrency_for_dem class ODMDEMCell(ecto.Cell): def declare_params(self, params): @@ -109,7 +109,7 @@ class ODMDEMCell(ecto.Cell): maxangle=args.dem_maxangle, decimation=args.dem_decimation, verbose=args.verbose, - max_workers=args.max_concurrency + max_workers=get_max_concurrency_for_dem(args.max_concurrency,tree.odm_georeferencing_model_laz) ) if args.crop > 0: diff --git a/scripts/odm_meshing.py b/scripts/odm_meshing.py index 0ff6bf38..97875ffb 100644 --- a/scripts/odm_meshing.py +++ b/scripts/odm_meshing.py @@ -104,7 +104,7 @@ class ODMeshingCell(ecto.Cell): maxVertexCount=self.params.max_vertex, samples=self.params.samples, verbose=self.params.verbose, - max_workers=args.max_concurrency, + available_cores=args.max_concurrency, method='poisson' if args.fast_orthophoto else 'gridded') else: log.ODM_WARNING('Found a valid ODM 2.5D Mesh file in: %s' % diff --git a/scripts/odm_orthophoto.py b/scripts/odm_orthophoto.py index 29178407..2b4b76e1 100644 --- a/scripts/odm_orthophoto.py +++ b/scripts/odm_orthophoto.py @@ -1,5 +1,4 @@ import ecto, os -from psutil import virtual_memory from opendm import io from opendm import log @@ -7,6 +6,7 @@ from opendm import system from opendm import context from opendm import types from opendm import gsd +from opendm.concurrency import get_max_memory from opendm.cropper import Cropper @@ -127,7 +127,7 @@ class ODMOrthoPhotoCell(ecto.Cell): 'png': tree.odm_orthophoto_file, 'tiff': tree.odm_orthophoto_tif, 'log': tree.odm_orthophoto_tif_log, - 'max_memory': max(5, (100 - virtual_memory().percent) / 2), + 'max_memory': get_max_memory(), 'threads': self.params.max_concurrency }