kopia lustrzana https://github.com/OpenDroneMap/ODM
Tiled DEM generation (WIP)
Piero Toffanin
rodzic
a41f0c64ee
commit
4fb27d6987
|
|
@ -1,6 +1,12 @@
|
|||
import os, glob
|
||||
import gippy
|
||||
import numpy
|
||||
import math
|
||||
from opendm.system import run
|
||||
from opendm import point_cloud
|
||||
from opendm.concurrency import get_max_memory
|
||||
import pprint
|
||||
|
||||
from scipy import ndimage
|
||||
from datetime import datetime
|
||||
from opendm import log
|
||||
|
|
@ -21,96 +27,147 @@ def classify(lasFile, slope=0.15, cellsize=1, maxWindowSize=18, verbose=False):
|
|||
return lasFile
|
||||
|
||||
|
||||
def create_dems(filenames, demtype, radius=['0.56'], gapfill=False,
|
||||
outdir='', suffix='', resolution=0.1, max_workers=None, **kwargs):
|
||||
""" Create DEMS for multiple radii, and optionally gapfill """
|
||||
fouts = []
|
||||
|
||||
create_dem_for_radius = partial(create_dem,
|
||||
filenames, demtype,
|
||||
outdir=outdir, suffix=suffix, resolution=resolution, **kwargs)
|
||||
|
||||
with get_reusable_executor(max_workers=max_workers, timeout=None) as e:
|
||||
fouts = list(e.map(create_dem_for_radius, radius))
|
||||
|
||||
fnames = {}
|
||||
# convert from list of dicts, to dict of lists
|
||||
for product in fouts[0].keys():
|
||||
fnames[product] = [f[product] for f in fouts]
|
||||
fouts = fnames
|
||||
def create_dem(input_point_cloud, dem_type, output_type='max', radiuses=['0.56'], gapfill=True,
|
||||
outdir='', resolution=0.1, max_workers=None, max_tile_size=4096,
|
||||
verbose=False, decimation=None):
|
||||
""" Create DEM from multiple radii, and optionally gapfill """
|
||||
start = datetime.now()
|
||||
|
||||
if not os.path.exists(outdir):
|
||||
log.ODM_INFO("Creating %s" % outdir)
|
||||
os.mkdir(outdir)
|
||||
|
||||
extent = point_cloud.get_extent(input_point_cloud)
|
||||
log.ODM_INFO("Point cloud bounds are [minx: %s, maxx: %s] [miny: %s, maxy: %s]" % (extent['minx'], extent['maxx'], extent['miny'], extent['maxy']))
|
||||
# extent = {
|
||||
# 'maxx': 100,
|
||||
# 'minx': 0,
|
||||
# 'maxy': 100,
|
||||
# 'miny': 0
|
||||
# }
|
||||
ext_width = extent['maxx'] - extent['minx']
|
||||
ext_height = extent['maxy'] - extent['miny']
|
||||
|
||||
final_dem_resolution = (int(math.ceil(ext_width / float(resolution))),
|
||||
int(math.ceil(ext_height / float(resolution))))
|
||||
num_splits = int(math.ceil(max(final_dem_resolution) / float(max_tile_size)))
|
||||
num_tiles = num_splits * num_splits
|
||||
log.ODM_INFO("DEM resolution is %s, max tile size is %s, will split DEM generation into %s tiles" % (final_dem_resolution, max_tile_size, num_tiles))
|
||||
|
||||
tile_bounds_width = ext_width / float(num_splits)
|
||||
tile_bounds_height = ext_height / float(num_splits)
|
||||
|
||||
tiles = []
|
||||
|
||||
for r in radiuses:
|
||||
minx = extent['minx']
|
||||
|
||||
for x in range(num_splits):
|
||||
miny = extent['miny']
|
||||
if x == num_splits - 1:
|
||||
maxx = extent['maxx']
|
||||
else:
|
||||
maxx = minx + tile_bounds_width
|
||||
|
||||
for y in range(num_splits):
|
||||
if y == num_splits - 1:
|
||||
maxy = extent['maxy']
|
||||
else:
|
||||
maxy = miny + tile_bounds_height
|
||||
|
||||
filename = os.path.join(os.path.abspath(outdir), '%s_r%s_x%s_y%s.tif' % (dem_type, r, x, y))
|
||||
|
||||
tiles.append({
|
||||
'radius': r,
|
||||
'bounds': {
|
||||
'minx': minx,
|
||||
'maxx': maxx,
|
||||
'miny': miny,
|
||||
'maxy': maxy
|
||||
},
|
||||
'filename': filename
|
||||
})
|
||||
|
||||
miny = maxy
|
||||
minx = maxx
|
||||
|
||||
# Sort tiles by increasing radius
|
||||
tiles.sort(key=lambda t: float(t['radius']), reverse=True)
|
||||
|
||||
# pp = pprint.PrettyPrinter(indent=4)
|
||||
# pp.pprint(queue)
|
||||
# TODO: parallel queue
|
||||
queue = tiles[:]
|
||||
|
||||
for q in queue:
|
||||
log.ODM_INFO("Generating %s (%s, radius: %s, resolution: %s)" % (q['filename'], output_type, q['radius'], resolution))
|
||||
|
||||
d = pdal.json_gdal_base(q['filename'], output_type, q['radius'], resolution, q['bounds'])
|
||||
|
||||
if dem_type == 'dsm':
|
||||
d = pdal.json_add_classification_filter(d, 2, equality='max')
|
||||
elif dem_type == 'dtm':
|
||||
d = pdal.json_add_classification_filter(d, 2)
|
||||
|
||||
if decimation is not None:
|
||||
d = pdal.json_add_decimation_filter(d, decimation)
|
||||
|
||||
pdal.json_add_readers(d, [input_point_cloud])
|
||||
pdal.run_pipeline(d, verbose=verbose)
|
||||
|
||||
|
||||
output_file = "%s.tif" % dem_type
|
||||
output_path = os.path.abspath(os.path.join(outdir, output_file))
|
||||
|
||||
# Verify tile results
|
||||
for t in tiles:
|
||||
if not os.path.exists(t['filename']):
|
||||
raise Exception("Error creating %s, %s failed to be created" % (output_file, t['filename']))
|
||||
|
||||
# Create virtual raster
|
||||
vrt_path = os.path.abspath(os.path.join(outdir, "merged.vrt"))
|
||||
run('gdalbuildvrt "%s" "%s"' % (vrt_path, '" "'.join(map(lambda t: t['filename'], tiles))))
|
||||
|
||||
geotiff_path = os.path.abspath(os.path.join(outdir, 'merged.tiff'))
|
||||
|
||||
# Build GeoTIFF
|
||||
kwargs = {
|
||||
'max_memory': get_max_memory(),
|
||||
'threads': max_workers if max_workers else 'ALL_CPUS',
|
||||
'vrt': vrt_path,
|
||||
'geotiff': geotiff_path
|
||||
}
|
||||
|
||||
run('gdal_translate '
|
||||
'-co NUM_THREADS={threads} '
|
||||
'--config GDAL_CACHEMAX {max_memory}% '
|
||||
'{vrt} {geotiff}'.format(**kwargs))
|
||||
|
||||
# gapfill all products
|
||||
_fouts = {}
|
||||
if gapfill:
|
||||
for product in fouts.keys():
|
||||
# output filename
|
||||
fout = os.path.join(outdir, '%s%s.tif' % (demtype, suffix))
|
||||
gap_fill(fouts[product], fout)
|
||||
_fouts[product] = fout
|
||||
gapfill_and_smooth(geotiff_path, output_path)
|
||||
os.remove(geotiff_path)
|
||||
else:
|
||||
# only return single filename (first radius run)
|
||||
for product in fouts.keys():
|
||||
_fouts[product] = fouts[product][0]
|
||||
log.ODM_INFO("Skipping gap-fill interpolation")
|
||||
os.rename(geotiff_path, output_path)
|
||||
|
||||
return _fouts
|
||||
|
||||
|
||||
def create_dem(filenames, demtype, radius, decimation=None,
|
||||
products=['idw'], outdir='', suffix='', verbose=False, resolution=0.1):
|
||||
""" Create DEM from collection of LAS files """
|
||||
start = datetime.now()
|
||||
# filename based on demtype, radius, and optional suffix
|
||||
bname = os.path.join(os.path.abspath(outdir), '%s_r%s%s' % (demtype, radius, suffix))
|
||||
ext = 'tif'
|
||||
|
||||
fouts = {o: bname + '.%s.%s' % (o, ext) for o in products}
|
||||
prettyname = os.path.relpath(bname) + ' [%s]' % (' '.join(products))
|
||||
|
||||
log.ODM_INFO('Creating %s from %s files' % (prettyname, len(filenames)))
|
||||
# JSON pipeline
|
||||
json = pdal.json_gdal_base(bname, products, radius, resolution)
|
||||
# TODO cleanup
|
||||
|
||||
if demtype == 'dsm':
|
||||
json = pdal.json_add_classification_filter(json, 2, equality='max')
|
||||
elif demtype == 'dtm':
|
||||
json = pdal.json_add_classification_filter(json, 2)
|
||||
|
||||
if decimation is not None:
|
||||
json = pdal.json_add_decimation_filter(json, decimation)
|
||||
|
||||
pdal.json_add_readers(json, filenames)
|
||||
|
||||
pdal.run_pipeline(json, verbose=verbose)
|
||||
|
||||
# verify existence of fout
|
||||
exists = True
|
||||
for f in fouts.values():
|
||||
if not os.path.exists(f):
|
||||
exists = False
|
||||
if not exists:
|
||||
raise Exception("Error creating dems: %s" % ' '.join(fouts))
|
||||
|
||||
log.ODM_INFO('Completed %s in %s' % (prettyname, datetime.now() - start))
|
||||
return fouts
|
||||
log.ODM_INFO('Completed %s in %s' % (output_file, datetime.now() - start))
|
||||
|
||||
|
||||
def gap_fill(filenames, fout):
|
||||
""" Gap fill from higher radius DTMs, then fill remainder with interpolation """
|
||||
|
||||
def gapfill_and_smooth(geotiff_path, output_path):
|
||||
""" Gap fill with nearest neighbor interpolation and apply median smoothing """
|
||||
start = datetime.now()
|
||||
|
||||
if len(filenames) == 0:
|
||||
raise Exception('No filenames provided!')
|
||||
if not os.path.exists(geotiff_path):
|
||||
raise Exception('File %s does not exist!' % geotiff_path)
|
||||
|
||||
log.ODM_INFO('Starting gap-filling with nearest interpolation...')
|
||||
filenames = sorted(filenames)
|
||||
|
||||
imgs = map(gippy.GeoImage, filenames)
|
||||
nodata = imgs[0][0].nodata()
|
||||
arr = imgs[0][0].read()
|
||||
|
||||
for i in range(1, len(imgs)):
|
||||
locs = numpy.where(arr == nodata)
|
||||
arr[locs] = imgs[i][0].read()[locs]
|
||||
img = gippy.GeoImage(geotiff_path)
|
||||
nodata = img[0].nodata()
|
||||
arr = img[0].read()
|
||||
|
||||
# Nearest neighbor interpolation at bad points
|
||||
indices = ndimage.distance_transform_edt(arr == nodata,
|
||||
|
|
@ -132,12 +189,12 @@ def gap_fill(filenames, fout):
|
|||
arr[-1][-2:] = arr[-2][-1] = arr[-2][-2]
|
||||
|
||||
# write output
|
||||
imgout = gippy.GeoImage.create_from(imgs[0], fout)
|
||||
imgout = gippy.GeoImage.create_from(img, output_path)
|
||||
imgout.set_nodata(nodata)
|
||||
imgout[0].write(arr)
|
||||
fout = imgout.filename()
|
||||
output_path = imgout.filename()
|
||||
imgout = None
|
||||
|
||||
log.ODM_INFO('Completed gap-filling to create %s in %s' % (os.path.relpath(fout), datetime.now() - start))
|
||||
log.ODM_INFO('Completed gap-filling to create %s in %s' % (os.path.relpath(output_path), datetime.now() - start))
|
||||
|
||||
return fout
|
||||
return output_path
|
||||
|
|
@ -48,24 +48,23 @@ def json_base():
|
|||
return {'pipeline': []}
|
||||
|
||||
|
||||
def json_gdal_base(fout, output, radius, resolution=1):
|
||||
def json_gdal_base(filename, output_type, radius, resolution=1, bounds=None):
|
||||
""" Create initial JSON for PDAL pipeline containing a Writer element """
|
||||
json = json_base()
|
||||
|
||||
if len(output) > 1:
|
||||
# TODO: we might want to create a multiband raster with max/min/idw
|
||||
# in the future
|
||||
print "More than 1 output, will only create {0}".format(output[0])
|
||||
output = [output[0]]
|
||||
|
||||
json['pipeline'].insert(0, {
|
||||
d = {
|
||||
'type': 'writers.gdal',
|
||||
'resolution': resolution,
|
||||
'radius': radius,
|
||||
'filename': '{0}.{1}.tif'.format(fout, output[0]),
|
||||
'output_type': output[0],
|
||||
'filename': filename,
|
||||
'output_type': output_type,
|
||||
'data_type': 'float'
|
||||
})
|
||||
}
|
||||
|
||||
if bounds is not None:
|
||||
d['bounds'] = "([%s,%s],[%s,%s])" % (bounds['minx'], bounds['maxx'], bounds['miny'], bounds['maxy'])
|
||||
|
||||
json['pipeline'].insert(0, d)
|
||||
|
||||
return json
|
||||
|
||||
|
|
@ -155,7 +154,6 @@ def run_pipeline(json, verbose=False):
|
|||
cmd = [
|
||||
'pdal',
|
||||
'pipeline',
|
||||
'--nostream',
|
||||
'-i %s' % jsonfile
|
||||
]
|
||||
if verbose:
|
||||
|
|
|
|||
|
|
@ -24,14 +24,14 @@ def create_25dmesh(inPointCloud, outMesh, dsm_radius=0.07, dsm_resolution=0.05,
|
|||
|
||||
log.ODM_INFO('Creating DSM for 2.5D mesh')
|
||||
|
||||
commands.create_dems(
|
||||
[inPointCloud],
|
||||
commands.create_dem(
|
||||
inPointCloud,
|
||||
'mesh_dsm',
|
||||
radius=map(str, radius_steps),
|
||||
output_type='max',
|
||||
radiuses=map(str, radius_steps),
|
||||
gapfill=True,
|
||||
outdir=tmp_directory,
|
||||
resolution=dsm_resolution,
|
||||
products=['max'],
|
||||
verbose=verbose,
|
||||
max_workers=get_max_concurrency_for_dem(available_cores, inPointCloud)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
import os, sys, shutil
|
||||
import os, sys, shutil, tempfile, json
|
||||
from opendm import system
|
||||
from opendm import log
|
||||
from opendm import context
|
||||
from opendm.system import run
|
||||
|
||||
def filter(inputPointCloud, outputPointCloud, standard_deviation=2.5, meank=16, confidence=None, verbose=False):
|
||||
def filter(input_point_cloud, output_point_cloud, standard_deviation=2.5, meank=16, confidence=None, verbose=False):
|
||||
"""
|
||||
Filters a point cloud
|
||||
"""
|
||||
|
|
@ -15,20 +16,20 @@ def filter(inputPointCloud, outputPointCloud, standard_deviation=2.5, meank=16,
|
|||
if confidence:
|
||||
log.ODM_INFO("Keeping only points with > %s confidence" % confidence)
|
||||
|
||||
if not os.path.exists(inputPointCloud):
|
||||
log.ODM_ERROR("{} does not exist, cannot filter point cloud. The program will now exit.".format(inputPointCloud))
|
||||
if not os.path.exists(input_point_cloud):
|
||||
log.ODM_ERROR("{} does not exist, cannot filter point cloud. The program will now exit.".format(input_point_cloud))
|
||||
sys.exit(1)
|
||||
|
||||
filter_program = os.path.join(context.odm_modules_path, 'odm_filterpoints')
|
||||
if not os.path.exists(filter_program):
|
||||
log.ODM_WARNING("{} program not found. Will skip filtering, but this installation should be fixed.")
|
||||
shutil.copy(inputPointCloud, outputPointCloud)
|
||||
shutil.copy(input_point_cloud, output_point_cloud)
|
||||
return
|
||||
|
||||
filterArgs = {
|
||||
'bin': filter_program,
|
||||
'inputFile': inputPointCloud,
|
||||
'outputFile': outputPointCloud,
|
||||
'inputFile': input_point_cloud,
|
||||
'outputFile': output_point_cloud,
|
||||
'sd': standard_deviation,
|
||||
'meank': meank,
|
||||
'verbose': '-verbose' if verbose else '',
|
||||
|
|
@ -41,5 +42,56 @@ def filter(inputPointCloud, outputPointCloud, standard_deviation=2.5, meank=16,
|
|||
'-meank {meank} {confidence} {verbose} '.format(**filterArgs))
|
||||
|
||||
# Remove input file, swap temp file
|
||||
if not os.path.exists(outputPointCloud):
|
||||
log.ODM_WARNING("{} not found, filtering has failed.".format(outputPointCloud))
|
||||
if not os.path.exists(output_point_cloud):
|
||||
log.ODM_WARNING("{} not found, filtering has failed.".format(output_point_cloud))
|
||||
|
||||
def get_extent(input_point_cloud):
|
||||
fd, json_file = tempfile.mkstemp(suffix='.json')
|
||||
os.close(fd)
|
||||
|
||||
# Get point cloud extent
|
||||
fallback = False
|
||||
|
||||
# We know PLY files do not have --summary support
|
||||
if input_point_cloud.lower().endswith(".ply"):
|
||||
fallback = True
|
||||
run('pdal info {0} > {1}'.format(input_point_cloud, json_file))
|
||||
|
||||
try:
|
||||
if not fallback:
|
||||
run('pdal info --summary {0} > {1}'.format(input_point_cloud, json_file))
|
||||
except:
|
||||
fallback = True
|
||||
run('pdal info {0} > {1}'.format(input_point_cloud, json_file))
|
||||
|
||||
bounds = {}
|
||||
with open(json_file, 'r') as f:
|
||||
result = json.loads(f.read())
|
||||
|
||||
if not fallback:
|
||||
summary = result.get('summary')
|
||||
if summary is None: raise Exception("Cannot compute summary for %s (summary key missing)" % input_point_cloud)
|
||||
bounds = summary.get('bounds')
|
||||
else:
|
||||
stats = result.get('stats')
|
||||
if stats is None: raise Exception("Cannot compute bounds for %s (stats key missing)" % input_point_cloud)
|
||||
bbox = stats.get('bbox')
|
||||
if bbox is None: raise Exception("Cannot compute bounds for %s (bbox key missing)" % input_point_cloud)
|
||||
native = bbox.get('native')
|
||||
if native is None: raise Exception("Cannot compute bounds for %s (native key missing)" % input_point_cloud)
|
||||
bounds = native.get('bbox')
|
||||
|
||||
if bounds is None: raise Exception("Cannot compute bounds for %s (bounds key missing)" % input_point_cloud)
|
||||
|
||||
if bounds.get('maxx', None) is None or \
|
||||
bounds.get('minx', None) is None or \
|
||||
bounds.get('maxy', None) is None or \
|
||||
bounds.get('miny', None) is None or \
|
||||
bounds.get('maxz', None) is None or \
|
||||
bounds.get('minz', None) is None:
|
||||
raise Exception("Cannot compute bounds for %s (invalid keys) %s" % (input_point_cloud, str(bounds)))
|
||||
|
||||
os.remove(json_file)
|
||||
return bounds
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -86,11 +86,12 @@ class ODMDEMCell(ecto.Cell):
|
|||
radius_steps.append(radius_steps[-1] * 2) # 2 is arbitrary, maybe there's a better value?
|
||||
|
||||
for product in products:
|
||||
commands.create_dems(
|
||||
[tree.odm_georeferencing_model_laz],
|
||||
commands.create_dem(
|
||||
tree.odm_georeferencing_model_laz,
|
||||
product,
|
||||
radius=map(str, radius_steps),
|
||||
gapfill=True,
|
||||
output_type='idw' if product == 'dtm' else 'max'
|
||||
radiuses=map(str, radius_steps),
|
||||
gapfill=args.dem_gapfill_steps > 0,
|
||||
outdir=odm_dem_root,
|
||||
resolution=resolution / 100.0,
|
||||
decimation=args.dem_decimation,
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue