diff --git a/opendm/dem/pdal.py b/opendm/dem/pdal.py index 164832a3..832604be 100644 --- a/opendm/dem/pdal.py +++ b/opendm/dem/pdal.py @@ -34,10 +34,10 @@ import os import json as jsonlib import tempfile from opendm import system +from opendm import log -import glob from datetime import datetime -import uuid +from pipes import quote """ JSON Functions """ @@ -134,7 +134,7 @@ def json_add_readers(json, filenames): def json_print(json): """ Pretty print JSON """ - print jsonlib.dumps(json, indent=4, separators=(',', ': ')) + log.ODM_DEBUG(jsonlib.dumps(json, indent=4, separators=(',', ': '))) """ Run PDAL commands """ @@ -147,7 +147,7 @@ def run_pipeline(json, verbose=False): # write to temp file f, jsonfile = tempfile.mkstemp(suffix='.json') if verbose: - print 'Pipeline file: %s' % jsonfile + log.ODM_INFO('Pipeline file: %s' % jsonfile) os.write(f, jsonlib.dumps(json)) os.close(f) @@ -157,9 +157,9 @@ def run_pipeline(json, verbose=False): '-i %s' % jsonfile ] if verbose: - out = system.run(' '.join(cmd)) + system.run(' '.join(cmd)) else: - out = system.run(' '.join(cmd) + ' > /dev/null 2>&1') + system.run(' '.join(cmd) + ' > /dev/null 2>&1') os.remove(jsonfile) @@ -178,9 +178,23 @@ def run_pdaltranslate_smrf(fin, fout, scalar, slope, threshold, window, verbose= ] if verbose: - print ' '.join(cmd) + log.ODM_DEBUG(' '.join(cmd)) + + system.run(' '.join(cmd)) + +def merge_point_clouds(input_files, output_file, verbose=False): + if len(input_files) == 0: + log.ODM_WARNING("Cannot merge point clouds, no point clouds to merge.") + return + + cmd = [ + 'pdal', + 'merge', + ' '.join(map(quote, input_files + [output_file])), + ] - out = system.run(' '.join(cmd)) if verbose: - print out + log.ODM_DEBUG(' '.join(cmd)) + + system.run(' '.join(cmd)) diff --git a/opendm/osfm.py b/opendm/osfm.py index b4a9c9be..07843605 100644 --- a/opendm/osfm.py +++ b/opendm/osfm.py @@ -141,6 +141,7 @@ def get_submodel_argv(args, submodels_path, submodel_name): """ :return the same as argv, but removing references to --split, setting/replacing --project-path and name + setting/replacing --crop to 0 (never crop on submodels) """ argv = sys.argv @@ -148,6 +149,7 @@ def get_submodel_argv(args, submodels_path, submodel_name): i = 1 project_path_found = False project_name_added = False + crop_found = True # TODO: what about GCP paths? @@ -168,6 +170,11 @@ def get_submodel_argv(args, submodels_path, submodel_name): result.append(submodels_path) project_path_found = True i += 2 + elif arg == '--crop': + result.append(arg) + result.append('0') + crop_found = True + i += 2 elif arg == '--split': i += 2 else: @@ -177,8 +184,28 @@ def get_submodel_argv(args, submodels_path, submodel_name): if not project_path_found: result.append('--project-path') result.append(submodel_project_path) + + if not crop_found: + result.append('--crop') + result.append('0') if not project_name_added: result.append(submodel_name) return result + + +def get_submodel_paths(submodels_path, *paths): + """ + :return Existing paths for all submodels + """ + result = [] + for f in os.listdir(submodels_path): + if f.startswith('submodel'): + p = os.path.join(submodels_path, f, *paths) + if os.path.exists(p): + result.append(p) + else: + log.ODM_WARNING("Missing %s from submodel %s" % (p, f)) + + return result \ No newline at end of file diff --git a/opendm/types.py b/opendm/types.py index 5a1c98ca..7c733228 100644 --- a/opendm/types.py +++ b/opendm/types.py @@ -337,25 +337,6 @@ class ODM_Tree(object): return io.join_paths(self.root_path, *args) -class SplitMerge(object): - """ docstring for SplitMerge""" - def __init__(self, project_name, progress): - self.project_name = project_name - self.progress = 0 - def load_progress(self, sm_file): - with open(sm_file) as f: - prog = int(f.readline()) - if prog in range(0, 6): - self.update_progress(prog) - def save_progress(self, sm_file): - with open(sm_file, 'w') as f: - f.write(str(self.progress)) - def update_progress(self, progress): - if progress in range(0,6): - self.progress = progress - else: - log.ODM_ERROR("Failed to save progress") - class ODM_Stage: def __init__(self, name, args, **params): self.name = name diff --git a/run.py b/run.py index de7f3c26..86326dfc 100755 --- a/run.py +++ b/run.py @@ -42,45 +42,47 @@ if __name__ == '__main__': app = ODMApp(args) app.execute() - log.ODM_INFO('MMMMMMMMMMMNNNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMNNNMMMMMMMMMMM') - log.ODM_INFO('MMMMMMdo:..---../sNMMMMMMMMMMMMMMMMMMMMMMMMMMNs/..---..:odMMMMMM') - log.ODM_INFO('MMMMy-.odNMMMMMNy/`/mMMMMMMMMMMMMMMMMMMMMMMm/`/hNMMMMMNdo.-yMMMM') - log.ODM_INFO('MMN/`sMMMMMMMMMNNMm/`yMMMMMMMMMMMMMMMMMMMMy`/mMNNMMMMMMMMNs`/MMM') - log.ODM_INFO('MM/ hMMMMMMMMNs.+MMM/ dMMMMMMMMMMMMMMMMMMh +MMM+.sNMMMMMMMMh +MM') - log.ODM_INFO('MN /MMMMMMNo/./mMMMMN :MMMMMMMMMMMMMMMMMM: NMMMMm/./oNMMMMMM: NM') - log.ODM_INFO('Mm +MMMMMN+ `/MMMMMMM`-MMMMMMMMMMMMMMMMMM-`MMMMMMM:` oNMMMMM+ mM') - log.ODM_INFO('MM..NMMNs./mNMMMMMMMy sMMMMMMMMMMMMMMMMMMo hMMMMMMMNm/.sNMMN`-MM') - log.ODM_INFO('MMd`:mMNomMMMMMMMMMy`:MMMMMMMNmmmmNMMMMMMN:`hMMMMMMMMMdoNMm-`dMM') - log.ODM_INFO('MMMm:.omMMMMMMMMNh/ sdmmho/.`..`-``-/sddh+ /hNMMMMMMMMdo.:mMMM') - log.ODM_INFO('MMMMMd+--/osss+:-:/` ```:- .ym+ hmo``:-` `+:-:ossso/-:+dMMMMM') - log.ODM_INFO('MMMMMMMNmhysosydmNMo /ds`/NMM+ hMMd..dh. sMNmdysosyhmNMMMMMMM') - log.ODM_INFO('MMMMMMMMMMMMMMMMMMMs .:-:``hmmN+ yNmds -:.:`-NMMMMMMMMMMMMMMMMMM') - log.ODM_INFO('MMMMMMMMMMMMMMMMMMN.-mNm- //:::. -:://: +mMd`-NMMMMMMMMMMMMMMMMM') - log.ODM_INFO('MMMMMMMMMMMMMMMMMM+ dMMN -MMNNN+ yNNNMN :MMMs sMMMMMMMMMMMMMMMMM') - log.ODM_INFO('MMMMMMMMMMMMMMMMMM`.mmmy /mmmmm/ smmmmm``mmmh :MMMMMMMMMMMMMMMMM') - log.ODM_INFO('MMMMMMMMMMMMMMMMMM``:::- ./////. -:::::` :::: -MMMMMMMMMMMMMMMMM') - log.ODM_INFO('MMMMMMMMMMMMMMMMMM:`mNNd /NNNNN+ hNNNNN .NNNy +MMMMMMMMMMMMMMMMM') - log.ODM_INFO('MMMMMMMMMMMMMMMMMMd`/MMM.`ys+//. -/+oso +MMN.`mMMMMMMMMMMMMMMMMM') - log.ODM_INFO('MMMMMMMMMMMMMMMMMMMy /o:- `oyhd/ shys+ `-:s-`hMMMMMMMMMMMMMMMMMM') - log.ODM_INFO('MMMMMMMMNmdhhhdmNMMM` +d+ sMMM+ hMMN:`hh- sMMNmdhhhdmNMMMMMMMM') - log.ODM_INFO('MMMMMms:::/++//::+ho .+- /dM+ hNh- +/` -h+:://++/::/smMMMMM') - log.ODM_INFO('MMMN+./hmMMMMMMNds- ./oso:.``:. :-``.:os+- -sdNMMMMMMmy:.oNMMM') - log.ODM_INFO('MMm-.hMNhNMMMMMMMMNo`/MMMMMNdhyyyyhhdNMMMM+`oNMMMMMMMMNhNMh.-mMM') - log.ODM_INFO('MM:`mMMN/-sNNMMMMMMMo yMMMMMMMMMMMMMMMMMMy sMMMMMMMNNs-/NMMm`:MM') - log.ODM_INFO('Mm /MMMMMd/.-oMMMMMMN :MMMMMMMMMMMMMMMMMM-`MMMMMMMo-./dMMMMM/ NM') - log.ODM_INFO('Mm /MMMMMMm:-`sNMMMMN :MMMMMMMMMMMMMMMMMM-`MMMMMNs`-/NMMMMMM/ NM') - log.ODM_INFO('MM:`mMMMMMMMMd/-sMMMo yMMMMMMMMMMMMMMMMMMy sMMMs-/dMMMMMMMMd`:MM') - log.ODM_INFO('MMm-.hMMMMMMMMMdhMNo`+MMMMMMMMMMMMMMMMMMMM+`oNMhdMMMMMMMMMh.-mMM') - log.ODM_INFO('MMMNo./hmNMMMMMNms--yMMMMMMMMMMMMMMMMMMMMMMy--smNMMMMMNmy/.oNMMM') - log.ODM_INFO('MMMMMms:-:/+++/:-+hMMMMMMMMMMMMMMMMMMMMMMMMMNh+-:/+++/:-:smMMMMM') - log.ODM_INFO('MMMMMMMMNdhhyhdmMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMmdhyhhmNMMMMMMMM') - log.ODM_INFO('MMMMMMMMMMMMMMMNNNNNMMMMMMNNNNNNMMMMMMMMNNMMMMMMMNNMMMMMMMMMMMMM') - log.ODM_INFO('MMMMMMMMMMMMMh/-...-+dMMMm......:+hMMMMs../MMMMMo..sMMMMMMMMMMMM') - log.ODM_INFO('MMMMMMMMMMMM/ /yhy- sMMm -hhy/ :NMM+ oMMMy /MMMMMMMMMMMM') - log.ODM_INFO('MMMMMMMMMMMy /MMMMN` NMm /MMMMo +MM: .` yMd``` :MMMMMMMMMMMM') - log.ODM_INFO('MMMMMMMMMMM+ sMMMMM: hMm /MMMMd -MM- /s `h.`d- -MMMMMMMMMMMM') - log.ODM_INFO('MMMMMMMMMMMs +MMMMM. mMm /MMMMy /MM. +M/ yM: `MMMMMMMMMMMM') - log.ODM_INFO('MMMMMMMMMMMN- smNm/ +MMm :NNdo` .mMM` oMM+/yMM/ MMMMMMMMMMMM') - log.ODM_INFO('MMMMMMMMMMMMNo- `:yMMMm `:sNMMM` sMMMMMMM+ NMMMMMMMMMMM') - log.ODM_INFO('MMMMMMMMMMMMMMMNmmNMMMMMMMNmmmmNMMMMMMMNNMMMMMMMMMNNMMMMMMMMMMMM') + # Do not show ASCII art for local submodels runs + if not "submodels/submodel_" in args.project_path: + log.ODM_INFO('MMMMMMMMMMMNNNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMNNNMMMMMMMMMMM') + log.ODM_INFO('MMMMMMdo:..---../sNMMMMMMMMMMMMMMMMMMMMMMMMMMNs/..---..:odMMMMMM') + log.ODM_INFO('MMMMy-.odNMMMMMNy/`/mMMMMMMMMMMMMMMMMMMMMMMm/`/hNMMMMMNdo.-yMMMM') + log.ODM_INFO('MMN/`sMMMMMMMMMNNMm/`yMMMMMMMMMMMMMMMMMMMMy`/mMNNMMMMMMMMNs`/MMM') + log.ODM_INFO('MM/ hMMMMMMMMNs.+MMM/ dMMMMMMMMMMMMMMMMMMh +MMM+.sNMMMMMMMMh +MM') + log.ODM_INFO('MN /MMMMMMNo/./mMMMMN :MMMMMMMMMMMMMMMMMM: NMMMMm/./oNMMMMMM: NM') + log.ODM_INFO('Mm +MMMMMN+ `/MMMMMMM`-MMMMMMMMMMMMMMMMMM-`MMMMMMM:` oNMMMMM+ mM') + log.ODM_INFO('MM..NMMNs./mNMMMMMMMy sMMMMMMMMMMMMMMMMMMo hMMMMMMMNm/.sNMMN`-MM') + log.ODM_INFO('MMd`:mMNomMMMMMMMMMy`:MMMMMMMNmmmmNMMMMMMN:`hMMMMMMMMMdoNMm-`dMM') + log.ODM_INFO('MMMm:.omMMMMMMMMNh/ sdmmho/.`..`-``-/sddh+ /hNMMMMMMMMdo.:mMMM') + log.ODM_INFO('MMMMMd+--/osss+:-:/` ```:- .ym+ hmo``:-` `+:-:ossso/-:+dMMMMM') + log.ODM_INFO('MMMMMMMNmhysosydmNMo /ds`/NMM+ hMMd..dh. sMNmdysosyhmNMMMMMMM') + log.ODM_INFO('MMMMMMMMMMMMMMMMMMMs .:-:``hmmN+ yNmds -:.:`-NMMMMMMMMMMMMMMMMMM') + log.ODM_INFO('MMMMMMMMMMMMMMMMMMN.-mNm- //:::. -:://: +mMd`-NMMMMMMMMMMMMMMMMM') + log.ODM_INFO('MMMMMMMMMMMMMMMMMM+ dMMN -MMNNN+ yNNNMN :MMMs sMMMMMMMMMMMMMMMMM') + log.ODM_INFO('MMMMMMMMMMMMMMMMMM`.mmmy /mmmmm/ smmmmm``mmmh :MMMMMMMMMMMMMMMMM') + log.ODM_INFO('MMMMMMMMMMMMMMMMMM``:::- ./////. -:::::` :::: -MMMMMMMMMMMMMMMMM') + log.ODM_INFO('MMMMMMMMMMMMMMMMMM:`mNNd /NNNNN+ hNNNNN .NNNy +MMMMMMMMMMMMMMMMM') + log.ODM_INFO('MMMMMMMMMMMMMMMMMMd`/MMM.`ys+//. -/+oso +MMN.`mMMMMMMMMMMMMMMMMM') + log.ODM_INFO('MMMMMMMMMMMMMMMMMMMy /o:- `oyhd/ shys+ `-:s-`hMMMMMMMMMMMMMMMMMM') + log.ODM_INFO('MMMMMMMMNmdhhhdmNMMM` +d+ sMMM+ hMMN:`hh- sMMNmdhhhdmNMMMMMMMM') + log.ODM_INFO('MMMMMms:::/++//::+ho .+- /dM+ hNh- +/` -h+:://++/::/smMMMMM') + log.ODM_INFO('MMMN+./hmMMMMMMNds- ./oso:.``:. :-``.:os+- -sdNMMMMMMmy:.oNMMM') + log.ODM_INFO('MMm-.hMNhNMMMMMMMMNo`/MMMMMNdhyyyyhhdNMMMM+`oNMMMMMMMMNhNMh.-mMM') + log.ODM_INFO('MM:`mMMN/-sNNMMMMMMMo yMMMMMMMMMMMMMMMMMMy sMMMMMMMNNs-/NMMm`:MM') + log.ODM_INFO('Mm /MMMMMd/.-oMMMMMMN :MMMMMMMMMMMMMMMMMM-`MMMMMMMo-./dMMMMM/ NM') + log.ODM_INFO('Mm /MMMMMMm:-`sNMMMMN :MMMMMMMMMMMMMMMMMM-`MMMMMNs`-/NMMMMMM/ NM') + log.ODM_INFO('MM:`mMMMMMMMMd/-sMMMo yMMMMMMMMMMMMMMMMMMy sMMMs-/dMMMMMMMMd`:MM') + log.ODM_INFO('MMm-.hMMMMMMMMMdhMNo`+MMMMMMMMMMMMMMMMMMMM+`oNMhdMMMMMMMMMh.-mMM') + log.ODM_INFO('MMMNo./hmNMMMMMNms--yMMMMMMMMMMMMMMMMMMMMMMy--smNMMMMMNmy/.oNMMM') + log.ODM_INFO('MMMMMms:-:/+++/:-+hMMMMMMMMMMMMMMMMMMMMMMMMMNh+-:/+++/:-:smMMMMM') + log.ODM_INFO('MMMMMMMMNdhhyhdmMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMmdhyhhmNMMMMMMMM') + log.ODM_INFO('MMMMMMMMMMMMMMMNNNNNMMMMMMNNNNNNMMMMMMMMNNMMMMMMMNNMMMMMMMMMMMMM') + log.ODM_INFO('MMMMMMMMMMMMMh/-...-+dMMMm......:+hMMMMs../MMMMMo..sMMMMMMMMMMMM') + log.ODM_INFO('MMMMMMMMMMMM/ /yhy- sMMm -hhy/ :NMM+ oMMMy /MMMMMMMMMMMM') + log.ODM_INFO('MMMMMMMMMMMy /MMMMN` NMm /MMMMo +MM: .` yMd``` :MMMMMMMMMMMM') + log.ODM_INFO('MMMMMMMMMMM+ sMMMMM: hMm /MMMMd -MM- /s `h.`d- -MMMMMMMMMMMM') + log.ODM_INFO('MMMMMMMMMMMs +MMMMM. mMm /MMMMy /MM. +M/ yM: `MMMMMMMMMMMM') + log.ODM_INFO('MMMMMMMMMMMN- smNm/ +MMm :NNdo` .mMM` oMM+/yMM/ MMMMMMMMMMMM') + log.ODM_INFO('MMMMMMMMMMMMNo- `:yMMMm `:sNMMM` sMMMMMMM+ NMMMMMMMMMMM') + log.ODM_INFO('MMMMMMMMMMMMMMMNmmNMMMMMMMNmmmmNMMMMMMMNNMMMMMMMMMNNMMMMMMMMMMMM') log.ODM_INFO('OpenDroneMap app finished - %s' % system.now()) diff --git a/scripts/run_opensfm.py b/scripts/run_opensfm.py index 5f35cc14..9c761b68 100644 --- a/scripts/run_opensfm.py +++ b/scripts/run_opensfm.py @@ -27,13 +27,8 @@ class ODMOpenSfMStage(types.ODM_Stage): else: output_file = tree.opensfm_reconstruction - # check if reconstruction was done before - # TODO: more granularity for each step (setup/featurematch/reconstruction/etc.) - osfm.setup(args, tree.dataset_raw, tree.opensfm, photos, gcp_path=tree.odm_georeferencing_gcp, rerun=self.rerun()) - osfm.feature_matching(tree.opensfm, self.rerun()) - osfm.reconstruct(tree.opensfm, self.rerun()) if not io.file_exists(output_file) or self.rerun(): diff --git a/scripts/splitmerge.py b/scripts/splitmerge.py index 7f00fec1..3e2090ec 100644 --- a/scripts/splitmerge.py +++ b/scripts/splitmerge.py @@ -4,6 +4,7 @@ from opendm import osfm from opendm import types from opendm import io from opendm import system +from opendm.dem import pdal from opensfm.large import metadataset from pipes import quote @@ -52,11 +53,17 @@ class ODMSplitStage(types.ODM_Stage): for sp in submodel_paths: log.ODM_INFO("Reconstructing %s" % sp) - #osfm.reconstruct(sp, self.rerun()) + osfm.reconstruct(sp, self.rerun()) # Align - log.ODM_INFO("Aligning submodels...") - #osfm.run('align_submodels', tree.opensfm) + alignment_file = io.join_paths(tree.opensfm, 'alignment_done.txt') + if not io.file_exists(alignment_file) or self.rerun(): + log.ODM_INFO("Aligning submodels...") + osfm.run('align_submodels', tree.opensfm) + with open(alignment_file, 'w') as fout: + fout.write("Alignment done!\n") + else: + log.ODM_WARNING('Found a alignment matching done progress file in: %s' % alignment_file) # Dense reconstruction for each submodel for sp in submodel_paths: @@ -68,15 +75,14 @@ class ODMSplitStage(types.ODM_Stage): submodel_name = os.path.basename(os.path.abspath(os.path.join(sp, ".."))) - log.ODM_INFO("====================") + log.ODM_INFO("========================") log.ODM_INFO("Processing %s" % submodel_name) - log.ODM_INFO("====================") + log.ODM_INFO("========================") argv = osfm.get_submodel_argv(args, tree.submodels_path, submodel_name) # Re-run the ODM toolchain on the submodel system.run(" ".join(map(quote, argv)), env_vars=os.environ.copy()) - else: log.ODM_INFO("Normal dataset, will process all at once.") @@ -87,6 +93,23 @@ class ODMMergeStage(types.ODM_Stage): reconstruction = outputs['reconstruction'] if outputs['large']: - + # Merge point clouds + all_point_clouds = osfm.get_submodel_paths(tree.submodels_path, "odm_georeferencing", "odm_georeferenced_model.laz") + pdal.merge_point_clouds(all_point_clouds, tree.odm_georeferencing_model_laz, args.verbose) + + # Merge orthophoto + + # TODO: crop ortho if necessary + + # Merge DEM + + # TODO: crop DEM if necessary + + + # Stop the pipeline short! We're done. + self.next_stage = None else: log.ODM_INFO("Normal dataset, nothing to merge.") + + + \ No newline at end of file