Merge pull request #786 from giovannicimolin/ImproveGDALPerformance

Make GDAL use 50% of the available memory when cropping

Dockerfile

@@ -21,7 +21,7 @@ libexiv2-dev liblas-bin python-matplotlib libatlas-base-dev swig2.0 python-wheel
 RUN apt-get remove libdc1394-22-dev
 RUN pip install --upgrade pip
 RUN pip install setuptools
-RUN pip install -U PyYAML exifread gpxpy xmltodict catkin-pkg appsettings https://github.com/OpenDroneMap/gippy/archive/v0.3.9.tar.gz loky scipy shapely numpy pyproj
+RUN pip install -U PyYAML exifread gpxpy xmltodict catkin-pkg appsettings https://github.com/OpenDroneMap/gippy/archive/v0.3.9.tar.gz loky scipy shapely numpy pyproj psutil
 
 ENV PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python2.7/dist-packages"
 ENV PYTHONPATH="$PYTHONPATH:/code/SuperBuild/src/opensfm"

configure.sh

@@ -100,7 +100,7 @@ install() {
     echo "Installing split-merge Dependencies"
     pip install -U scipy shapely numpy pyproj
 
-    pip install -U https://github.com/OpenDroneMap/gippy/archive/v0.3.9.tar.gz
+    pip install -U https://github.com/OpenDroneMap/gippy/archive/v0.3.9.tar.gz psutil
 
     echo "Compiling SuperBuild"
     cd ${RUNPATH}/SuperBuild

core2.Dockerfile

@@ -19,7 +19,7 @@ libexiv2-dev liblas-bin python-matplotlib libatlas-base-dev swig2.0 python-wheel
 RUN apt-get remove libdc1394-22-dev
 RUN pip install --upgrade pip
 RUN pip install setuptools
-RUN pip install -U PyYAML exifread gpxpy xmltodict catkin-pkg appsettings https://github.com/OpenDroneMap/gippy/archive/v0.3.9.tar.gz loky scipy shapely numpy pyproj
+RUN pip install -U PyYAML exifread gpxpy xmltodict catkin-pkg appsettings https://github.com/OpenDroneMap/gippy/archive/v0.3.9.tar.gz loky scipy shapely numpy pyproj psutil
 
 ENV PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python2.7/dist-packages"
 ENV PYTHONPATH="$PYTHONPATH:/code/SuperBuild/src/opensfm"

opendm/cropper.py

@@ -3,6 +3,7 @@ from opendm.system import run
 from opendm import log
 from osgeo import ogr
 import json, os
+from psutil import virtual_memory
 
 class Cropper:
     def __init__(self, storage_dir, files_prefix = "crop"):
@@ -40,14 +41,16 @@ class Cropper:
                 'shapefile_path': shapefile_path,
                 'geotiffInput': original_geotiff,
                 'geotiffOutput': geotiff_path,
-                'options': ' '.join(map(lambda k: '-co {}={}'.format(k, gdal_options[k]), gdal_options))
+                'options': ' '.join(map(lambda k: '-co {}={}'.format(k, gdal_options[k]), gdal_options)),
+                'max_memory': max(5, (100 - virtual_memory().percent) / 2)
             }
 
             run('gdalwarp -cutline {shapefile_path} '
                 '-crop_to_cutline '
                 '{options} '
                 '{geotiffInput} '
-                '{geotiffOutput} '.format(**kwargs))
+                '{geotiffOutput} '
+                '--config GDAL_CACHEMAX {max_memory}%'.format(**kwargs))
 
             if not keep_original:
                 os.remove(original_geotiff)

scripts/odm_orthophoto.py

@@ -1,4 +1,5 @@
 import ecto, os
+from psutil import virtual_memory
 
 from opendm import io
 from opendm import log
@@ -109,7 +110,8 @@ class ODMOrthoPhotoCell(ecto.Cell):
                     'bigtiff': self.params.bigtiff,
                     'png': tree.odm_orthophoto_file,
                     'tiff': tree.odm_orthophoto_tif,
-                    'log': tree.odm_orthophoto_tif_log
+                    'log': tree.odm_orthophoto_tif_log,
+                    'max_memory': max(5, (100 - virtual_memory().percent) / 2)
                 }
 
                 system.run('gdal_translate -a_ullr {ulx} {uly} {lrx} {lry} '
@@ -121,6 +123,7 @@ class ODMOrthoPhotoCell(ecto.Cell):
                            '-co BLOCKYSIZE=512 '
                            '-co NUM_THREADS=ALL_CPUS '
                            '-a_srs \"{proj}\" '
+                           '--config GDAL_CACHEMAX {max_memory}%'
                            '{png} {tiff} > {log}'.format(**kwargs))
 
                 if args.crop > 0: