Minor refactoring, concurrency module, max concurrency for DEM

pull/925/head
Piero Toffanin 2018-12-12 14:24:27 -05:00
rodzic 47358aacdb
commit 5e03a08f9b
6 zmienionych plików z 37 dodań i 10 usunięć

Wyświetl plik

@ -0,0 +1,26 @@
from psutil import virtual_memory
import os
def get_max_memory(minimum = 5, use_at_most = 0.5):
"""
:param minimum minimum value to return (return value will never be lower than this)
:param use_at_most use at most this fraction of the available memory. 0.5 = use at most 50% of available memory
:return percentage value of memory to use (75 = 75%).
"""
return max(minimum, (100 - virtual_memory().percent) * use_at_most)
def get_max_concurrency_for_dem(available_cores, input_file, use_at_most = 0.8):
"""
DEM generation requires ~2x the input file size of memory per available core.
:param available_cores number of cores available (return value will never exceed this value)
:param input_file path to input file
:use_at_most use at most this fraction of the available memory when calculating a concurrency value. 0.9 = assume that we can only use 90% of available memory.
:return maximum number of cores recommended to use for DEM processing.
"""
memory_available = virtual_memory().available * use_at_most
file_size = os.path.getsize(input_file)
memory_required_per_core = max(1, file_size * 2)
return min(available_cores, max(1, int(memory_available) / int(memory_required_per_core)))

Wyświetl plik

@ -3,7 +3,7 @@ from opendm.system import run
from opendm import log
from osgeo import ogr
import json, os
from psutil import virtual_memory
from opendm.concurrency import get_max_memory
class Cropper:
def __init__(self, storage_dir, files_prefix = "crop"):
@ -42,7 +42,7 @@ class Cropper:
'geotiffInput': original_geotiff,
'geotiffOutput': geotiff_path,
'options': ' '.join(map(lambda k: '-co {}={}'.format(k, gdal_options[k]), gdal_options)),
'max_memory': max(5, (100 - virtual_memory().percent) / 2)
'max_memory': get_max_memory()
}
run('gdalwarp -cutline {shapefile_path} '

Wyświetl plik

@ -5,10 +5,11 @@ from opendm.dem import commands
from opendm import system
from opendm import log
from opendm import context
from opendm.concurrency import get_max_concurrency_for_dem
from scipy import signal, ndimage
import numpy as np
def create_25dmesh(inPointCloud, outMesh, dsm_radius=0.07, dsm_resolution=0.05, depth=8, samples=1, maxVertexCount=100000, verbose=False, max_workers=None, method='gridded'):
def create_25dmesh(inPointCloud, outMesh, dsm_radius=0.07, dsm_resolution=0.05, depth=8, samples=1, maxVertexCount=100000, verbose=False, available_cores=None, method='gridded'):
# Create DSM from point cloud
# Create temporary directory
@ -32,7 +33,7 @@ def create_25dmesh(inPointCloud, outMesh, dsm_radius=0.07, dsm_resolution=0.05,
resolution=dsm_resolution,
products=['max'],
verbose=verbose,
max_workers=max_workers
max_workers=get_max_concurrency_for_dem(available_cores, inPointCloud)
)
if method == 'gridded':
@ -42,7 +43,7 @@ def create_25dmesh(inPointCloud, outMesh, dsm_radius=0.07, dsm_resolution=0.05,
mesh = screened_poisson_reconstruction(dsm_points, outMesh, depth=depth,
samples=samples,
maxVertexCount=maxVertexCount,
threads=max_workers,
threads=available_cores,
verbose=verbose)
else:
raise 'Not a valid method: ' + method

Wyświetl plik

@ -9,7 +9,7 @@ from opendm import types
from opendm import gsd
from opendm.dem import commands
from opendm.cropper import Cropper
from opendm.concurrency import get_max_concurrency_for_dem
class ODMDEMCell(ecto.Cell):
def declare_params(self, params):
@ -109,7 +109,7 @@ class ODMDEMCell(ecto.Cell):
maxangle=args.dem_maxangle,
decimation=args.dem_decimation,
verbose=args.verbose,
max_workers=args.max_concurrency
max_workers=get_max_concurrency_for_dem(args.max_concurrency,tree.odm_georeferencing_model_laz)
)
if args.crop > 0:

Wyświetl plik

@ -104,7 +104,7 @@ class ODMeshingCell(ecto.Cell):
maxVertexCount=self.params.max_vertex,
samples=self.params.samples,
verbose=self.params.verbose,
max_workers=args.max_concurrency,
available_cores=args.max_concurrency,
method='poisson' if args.fast_orthophoto else 'gridded')
else:
log.ODM_WARNING('Found a valid ODM 2.5D Mesh file in: %s' %

Wyświetl plik

@ -1,5 +1,4 @@
import ecto, os
from psutil import virtual_memory
from opendm import io
from opendm import log
@ -7,6 +6,7 @@ from opendm import system
from opendm import context
from opendm import types
from opendm import gsd
from opendm.concurrency import get_max_memory
from opendm.cropper import Cropper
@ -127,7 +127,7 @@ class ODMOrthoPhotoCell(ecto.Cell):
'png': tree.odm_orthophoto_file,
'tiff': tree.odm_orthophoto_tif,
'log': tree.odm_orthophoto_tif_log,
'max_memory': max(5, (100 - virtual_memory().percent) / 2),
'max_memory': get_max_memory(),
'threads': self.params.max_concurrency
}