Tiled DEM generation (WIP)

opendm/dem/commands.py

@@ -1,6 +1,12 @@
 import os, glob
 import gippy
 import numpy
+import math
+from opendm.system import run
+from opendm import point_cloud
+from opendm.concurrency import get_max_memory
+import pprint
+
 from scipy import ndimage
 from datetime import datetime
 from opendm import log
@@ -21,96 +27,147 @@ def classify(lasFile, slope=0.15, cellsize=1, maxWindowSize=18, verbose=False):
     return lasFile
 
 
-def create_dems(filenames, demtype, radius=['0.56'], gapfill=False,
+def create_dem(input_point_cloud, dem_type, output_type='max', radiuses=['0.56'], gapfill=True,
-                outdir='', suffix='', resolution=0.1, max_workers=None, **kwargs):
+                outdir='', resolution=0.1, max_workers=None, max_tile_size=4096, 
-    """ Create DEMS for multiple radii, and optionally gapfill """
+                verbose=False, decimation=None):
-    fouts = []
+    """ Create DEM from multiple radii, and optionally gapfill """
-    
+    start = datetime.now()
-    create_dem_for_radius = partial(create_dem, 
+
-        filenames, demtype,
+    if not os.path.exists(outdir):
-        outdir=outdir, suffix=suffix, resolution=resolution, **kwargs)
+        log.ODM_INFO("Creating %s" % outdir)
-    
+        os.mkdir(outdir)
-    with get_reusable_executor(max_workers=max_workers, timeout=None) as e:
+
-         fouts = list(e.map(create_dem_for_radius, radius))
+    extent = point_cloud.get_extent(input_point_cloud)
-    
+    log.ODM_INFO("Point cloud bounds are [minx: %s, maxx: %s] [miny: %s, maxy: %s]" % (extent['minx'], extent['maxx'], extent['miny'], extent['maxy']))
-    fnames = {}
+    # extent = {
-    # convert from list of dicts, to dict of lists
+    #     'maxx': 100,
-    for product in fouts[0].keys():
+    #     'minx': 0,
-        fnames[product] = [f[product] for f in fouts]
+    #     'maxy': 100,
-    fouts = fnames
+    #     'miny': 0
+    # }
+    ext_width = extent['maxx'] - extent['minx']
+    ext_height = extent['maxy'] - extent['miny']
+
+    final_dem_resolution = (int(math.ceil(ext_width / float(resolution))),
+                            int(math.ceil(ext_height / float(resolution))))
+    num_splits = int(math.ceil(max(final_dem_resolution) / float(max_tile_size)))
+    num_tiles = num_splits * num_splits
+    log.ODM_INFO("DEM resolution is %s, max tile size is %s, will split DEM generation into %s tiles" % (final_dem_resolution, max_tile_size, num_tiles))
+
+    tile_bounds_width = ext_width / float(num_splits)
+    tile_bounds_height = ext_height / float(num_splits)
+
+    tiles = []
+
+    for r in radiuses:
+        minx = extent['minx']
+
+        for x in range(num_splits):
+            miny = extent['miny']
+            if x == num_splits - 1:
+                maxx = extent['maxx']
+            else:
+                maxx = minx + tile_bounds_width
+
+            for y in range(num_splits):
+                if y == num_splits - 1:
+                    maxy = extent['maxy']
+                else:
+                    maxy = miny + tile_bounds_height
+
+                filename = os.path.join(os.path.abspath(outdir), '%s_r%s_x%s_y%s.tif' % (dem_type, r, x, y))
+
+                tiles.append({
+                    'radius': r,
+                    'bounds': {
+                        'minx': minx,
+                        'maxx': maxx,
+                        'miny': miny,
+                        'maxy': maxy 
+                    },
+                    'filename': filename
+                })
+
+                miny = maxy
+            minx = maxx
+
+    # Sort tiles by increasing radius
+    tiles.sort(key=lambda t: float(t['radius']), reverse=True)
+
+    # pp = pprint.PrettyPrinter(indent=4)
+    # pp.pprint(queue)
+    # TODO: parallel queue
+    queue = tiles[:]
+
+    for q in queue:
+        log.ODM_INFO("Generating %s (%s, radius: %s, resolution: %s)" % (q['filename'], output_type, q['radius'], resolution))
+        
+        d = pdal.json_gdal_base(q['filename'], output_type, q['radius'], resolution, q['bounds'])
+
+        if dem_type == 'dsm':
+            d = pdal.json_add_classification_filter(d, 2, equality='max')
+        elif dem_type == 'dtm':
+            d = pdal.json_add_classification_filter(d, 2)
+
+        if decimation is not None:
+            d = pdal.json_add_decimation_filter(d, decimation)
+
+        pdal.json_add_readers(d, [input_point_cloud])
+        pdal.run_pipeline(d, verbose=verbose)
+    
+
+    output_file = "%s.tif" % dem_type
+    output_path = os.path.abspath(os.path.join(outdir, output_file))
+
+    # Verify tile results
+    for t in tiles: 
+        if not os.path.exists(t['filename']):
+            raise Exception("Error creating %s, %s failed to be created" % (output_file, t['filename']))
+    
+    # Create virtual raster
+    vrt_path = os.path.abspath(os.path.join(outdir, "merged.vrt"))
+    run('gdalbuildvrt "%s" "%s"' % (vrt_path, '" "'.join(map(lambda t: t['filename'], tiles))))
+
+    geotiff_path = os.path.abspath(os.path.join(outdir, 'merged.tiff'))
+
+    # Build GeoTIFF
+    kwargs = {
+        'max_memory': get_max_memory(),
+        'threads': max_workers if max_workers else 'ALL_CPUS',
+        'vrt': vrt_path,
+        'geotiff': geotiff_path
+    }
+
+    run('gdal_translate '
+            '-co NUM_THREADS={threads} '
+            '--config GDAL_CACHEMAX {max_memory}% '
+            '{vrt} {geotiff}'.format(**kwargs))
 
-    # gapfill all products
-    _fouts = {}
     if gapfill:
-        for product in fouts.keys():
+        gapfill_and_smooth(geotiff_path, output_path)
-            # output filename
+        os.remove(geotiff_path)
-            fout = os.path.join(outdir, '%s%s.tif' % (demtype, suffix))
-            gap_fill(fouts[product], fout)
-            _fouts[product] = fout
     else:
-        # only return single filename (first radius run)
+        log.ODM_INFO("Skipping gap-fill interpolation")
-        for product in fouts.keys():
+        os.rename(geotiff_path, output_path)
-            _fouts[product] = fouts[product][0]
 
-    return _fouts
+    # TODO cleanup
-
-
-def create_dem(filenames, demtype, radius, decimation=None,
-               products=['idw'], outdir='', suffix='', verbose=False, resolution=0.1):
-    """ Create DEM from collection of LAS files """
-    start = datetime.now()
-    # filename based on demtype, radius, and optional suffix
-    bname = os.path.join(os.path.abspath(outdir), '%s_r%s%s' % (demtype, radius, suffix))
-    ext = 'tif'
-
-    fouts = {o: bname + '.%s.%s' % (o, ext) for o in products}
-    prettyname = os.path.relpath(bname) + ' [%s]' % (' '.join(products))
-
-    log.ODM_INFO('Creating %s from %s files' % (prettyname, len(filenames)))
-    # JSON pipeline
-    json = pdal.json_gdal_base(bname, products, radius, resolution)
     
-    if demtype == 'dsm':
+    log.ODM_INFO('Completed %s in %s' % (output_file, datetime.now() - start))
-        json = pdal.json_add_classification_filter(json, 2, equality='max')
-    elif demtype == 'dtm':
-        json = pdal.json_add_classification_filter(json, 2)
-
-    if decimation is not None:
-        json = pdal.json_add_decimation_filter(json, decimation)
-
-    pdal.json_add_readers(json, filenames)
-
-    pdal.run_pipeline(json, verbose=verbose)
-    
-    # verify existence of fout
-    exists = True
-    for f in fouts.values():
-        if not os.path.exists(f):
-            exists = False
-    if not exists:
-        raise Exception("Error creating dems: %s" % ' '.join(fouts))
-
-    log.ODM_INFO('Completed %s in %s' % (prettyname, datetime.now() - start))
-    return fouts
 
 
-def gap_fill(filenames, fout):
+
-    """ Gap fill from higher radius DTMs, then fill remainder with interpolation """
+def gapfill_and_smooth(geotiff_path, output_path):
+    """ Gap fill with nearest neighbor interpolation and apply median smoothing """
     start = datetime.now()
 
-    if len(filenames) == 0:
+    if not os.path.exists(geotiff_path):
-        raise Exception('No filenames provided!')
+        raise Exception('File %s does not exist!' % geotiff_path)
 
     log.ODM_INFO('Starting gap-filling with nearest interpolation...')
-    filenames = sorted(filenames)
 
-    imgs = map(gippy.GeoImage, filenames)
+    img = gippy.GeoImage(geotiff_path)
-    nodata = imgs[0][0].nodata()
+    nodata = img[0].nodata()
-    arr = imgs[0][0].read()
+    arr = img[0].read()
-
-    for i in range(1, len(imgs)):
-        locs = numpy.where(arr == nodata)
-        arr[locs] = imgs[i][0].read()[locs]
 
     # Nearest neighbor interpolation at bad points
     indices = ndimage.distance_transform_edt(arr == nodata, 
@@ -132,12 +189,12 @@ def gap_fill(filenames, fout):
         arr[-1][-2:] = arr[-2][-1] = arr[-2][-2]
 
     # write output
-    imgout = gippy.GeoImage.create_from(imgs[0], fout)
+    imgout = gippy.GeoImage.create_from(img, output_path)
     imgout.set_nodata(nodata)
     imgout[0].write(arr)
-    fout = imgout.filename()
+    output_path = imgout.filename()
     imgout = None
 
-    log.ODM_INFO('Completed gap-filling to create %s in %s' % (os.path.relpath(fout), datetime.now() - start))
+    log.ODM_INFO('Completed gap-filling to create %s in %s' % (os.path.relpath(output_path), datetime.now() - start))
 
-    return fout
+    return output_path

opendm/dem/pdal.py

@@ -48,24 +48,23 @@ def json_base():
     return {'pipeline': []}
 
 
-def json_gdal_base(fout, output, radius, resolution=1):
+def json_gdal_base(filename, output_type, radius, resolution=1, bounds=None):
     """ Create initial JSON for PDAL pipeline containing a Writer element """
     json = json_base()
 
-    if len(output) > 1:
+    d = {
-        # TODO: we might want to create a multiband raster with max/min/idw
-        # in the future
-        print "More than 1 output, will only create {0}".format(output[0])
-        output = [output[0]]
-
-    json['pipeline'].insert(0, {
         'type': 'writers.gdal',
         'resolution': resolution,
         'radius': radius,
-        'filename': '{0}.{1}.tif'.format(fout, output[0]),
+        'filename': filename,
-        'output_type': output[0],
+        'output_type': output_type,
         'data_type': 'float'
-    })
+    }
+
+    if bounds is not None:
+        d['bounds'] = "([%s,%s],[%s,%s])" % (bounds['minx'], bounds['maxx'], bounds['miny'], bounds['maxy'])
+
+    json['pipeline'].insert(0, d)
 
     return json
 
@@ -155,7 +154,6 @@ def run_pipeline(json, verbose=False):
     cmd = [
         'pdal',
         'pipeline',
-        '--nostream',
         '-i %s' % jsonfile
     ]
     if verbose:

opendm/mesh.py

@@ -24,14 +24,14 @@ def create_25dmesh(inPointCloud, outMesh, dsm_radius=0.07, dsm_resolution=0.05,
 
     log.ODM_INFO('Creating DSM for 2.5D mesh')
 
-    commands.create_dems(
+    commands.create_dem(
-            [inPointCloud],
+            inPointCloud,
             'mesh_dsm',
-            radius=map(str, radius_steps),
+            output_type='max',
+            radiuses=map(str, radius_steps),
             gapfill=True,
             outdir=tmp_directory,
             resolution=dsm_resolution,
-            products=['max'],
             verbose=verbose,
             max_workers=get_max_concurrency_for_dem(available_cores, inPointCloud)
         )

opendm/point_cloud.py

@@ -1,9 +1,10 @@
-import os, sys, shutil
+import os, sys, shutil, tempfile, json
 from opendm import system
 from opendm import log
 from opendm import context
+from opendm.system import run
 
-def filter(inputPointCloud, outputPointCloud, standard_deviation=2.5, meank=16, confidence=None, verbose=False):
+def filter(input_point_cloud, output_point_cloud, standard_deviation=2.5, meank=16, confidence=None, verbose=False):
     """
     Filters a point cloud
     """
@@ -15,20 +16,20 @@ def filter(inputPointCloud, outputPointCloud, standard_deviation=2.5, meank=16,
     if confidence:
         log.ODM_INFO("Keeping only points with > %s confidence" % confidence)
 
-    if not os.path.exists(inputPointCloud):
+    if not os.path.exists(input_point_cloud):
-        log.ODM_ERROR("{} does not exist, cannot filter point cloud. The program will now exit.".format(inputPointCloud))
+        log.ODM_ERROR("{} does not exist, cannot filter point cloud. The program will now exit.".format(input_point_cloud))
         sys.exit(1)
 
     filter_program = os.path.join(context.odm_modules_path, 'odm_filterpoints')
     if not os.path.exists(filter_program):
         log.ODM_WARNING("{} program not found. Will skip filtering, but this installation should be fixed.")
-        shutil.copy(inputPointCloud, outputPointCloud)
+        shutil.copy(input_point_cloud, output_point_cloud)
         return
 
     filterArgs = {
       'bin': filter_program,
-      'inputFile': inputPointCloud,
+      'inputFile': input_point_cloud,
-      'outputFile': outputPointCloud,
+      'outputFile': output_point_cloud,
       'sd': standard_deviation,
       'meank': meank,
       'verbose': '-verbose' if verbose else '',
@@ -41,5 +42,56 @@ def filter(inputPointCloud, outputPointCloud, standard_deviation=2.5, meank=16,
          '-meank {meank} {confidence} {verbose} '.format(**filterArgs))
 
     # Remove input file, swap temp file
-    if not os.path.exists(outputPointCloud):
+    if not os.path.exists(output_point_cloud):
-        log.ODM_WARNING("{} not found, filtering has failed.".format(outputPointCloud))
+        log.ODM_WARNING("{} not found, filtering has failed.".format(output_point_cloud))
+
+def get_extent(input_point_cloud):
+    fd, json_file = tempfile.mkstemp(suffix='.json')
+    os.close(fd)
+    
+    # Get point cloud extent
+    fallback = False
+
+    # We know PLY files do not have --summary support
+    if input_point_cloud.lower().endswith(".ply"):
+        fallback = True
+        run('pdal info {0} > {1}'.format(input_point_cloud, json_file))
+
+    try:
+        if not fallback:
+            run('pdal info --summary {0} > {1}'.format(input_point_cloud, json_file))
+    except:
+        fallback = True
+        run('pdal info {0} > {1}'.format(input_point_cloud, json_file))
+
+    bounds = {}
+    with open(json_file, 'r') as f:
+        result = json.loads(f.read())
+        
+        if not fallback:
+            summary = result.get('summary')
+            if summary is None: raise Exception("Cannot compute summary for %s (summary key missing)" % input_point_cloud)
+            bounds = summary.get('bounds')
+        else:
+            stats = result.get('stats')
+            if stats is None: raise Exception("Cannot compute bounds for %s (stats key missing)" % input_point_cloud)
+            bbox = stats.get('bbox')
+            if bbox is None: raise Exception("Cannot compute bounds for %s (bbox key missing)" % input_point_cloud)
+            native = bbox.get('native')
+            if native is None: raise Exception("Cannot compute bounds for %s (native key missing)" % input_point_cloud)
+            bounds = native.get('bbox')
+
+        if bounds is None: raise Exception("Cannot compute bounds for %s (bounds key missing)" % input_point_cloud)
+        
+        if bounds.get('maxx', None) is None or \
+            bounds.get('minx', None) is None or \
+            bounds.get('maxy', None) is None or \
+            bounds.get('miny', None) is None or \
+            bounds.get('maxz', None) is None or \
+            bounds.get('minz', None) is None:
+            raise Exception("Cannot compute bounds for %s (invalid keys) %s" % (input_point_cloud, str(bounds)))
+            
+    os.remove(json_file)
+    return bounds    
+   
+

scripts/odm_dem.py

@@ -86,11 +86,12 @@ class ODMDEMCell(ecto.Cell):
                     radius_steps.append(radius_steps[-1] * 2) # 2 is arbitrary, maybe there's a better value?
 
                 for product in products:
-                    commands.create_dems(
+                    commands.create_dem(
-                            [tree.odm_georeferencing_model_laz],
+                            tree.odm_georeferencing_model_laz,
                             product,
-                            radius=map(str, radius_steps),
+                            output_type='idw' if product == 'dtm' else 'max'
-                            gapfill=True,
+                            radiuses=map(str, radius_steps),
+                            gapfill=args.dem_gapfill_steps > 0,
                             outdir=odm_dem_root,
                             resolution=resolution / 100.0,
                             decimation=args.dem_decimation,