From 9e921f44acf80bf941147f46f6845c93a0b1a0c7 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 22 Apr 2019 15:14:39 -0400 Subject: [PATCH] Started refactoring ecto out --- Dockerfile | 4 - configure.sh | 6 - opendm/types.py | 54 ++++++++ portable.Dockerfile | 4 - run.py | 35 ++--- scripts/dataset.py | 55 +++----- scripts/example_ecto_python.py | 35 ----- scripts/mve.py | 44 ++---- scripts/mvstex.py | 67 ++------- scripts/odm_app.py | 244 ++++++++++++--------------------- scripts/odm_dem.py | 43 +----- scripts/odm_filterpoints.py | 40 +----- scripts/odm_georeferencing.py | 53 ++----- scripts/odm_meshing.py | 76 +++------- scripts/odm_orthophoto.py | 73 +++------- scripts/odm_slam.py | 38 ++--- scripts/run_opensfm.py | 77 +++-------- tests/test_odm.py | 108 --------------- 18 files changed, 286 insertions(+), 770 deletions(-) delete mode 100644 scripts/example_ecto_python.py delete mode 100644 tests/test_odm.py diff --git a/Dockerfile b/Dockerfile index 7e9f68fc..82d79e34 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,14 +48,11 @@ RUN apt-get install --no-install-recommends -y \ libvtk6-dev \ libxext-dev \ python-dev \ - python-empy \ python-gdal \ python-matplotlib \ python-networkx \ - python-nose \ python-pip \ python-pyproj \ - python-pyside \ python-software-properties \ python-wheel \ swig2.0 @@ -65,7 +62,6 @@ RUN pip install --upgrade pip RUN pip install setuptools RUN pip install -U \ appsettings \ - catkin-pkg \ exifread \ gpxpy \ loky \ diff --git a/configure.sh b/configure.sh index bd765d2a..807256e1 100755 --- a/configure.sh +++ b/configure.sh @@ -80,12 +80,6 @@ install() { loky \ repoze.lru - echo "Installing Ecto Dependencies" - pip install -U catkin-pkg - apt-get install -y -qq python-empy \ - python-nose \ - python-pyside - echo "Installing OpenDroneMap Dependencies" apt-get install -y -qq python-scipy \ liblas-bin diff --git a/opendm/types.py b/opendm/types.py index b3d31675..b58c72fa 100644 --- a/opendm/types.py +++ b/opendm/types.py @@ -329,3 +329,57 @@ class ODM_Tree(object): def path(self, *args): return io.join_paths(self.root_path, *args) + + +class ODM_Stage: + def __init__(self, name, args, **params): + self.name = name + self.args = args + self.params = params + if self.params is None: + self.params = {} + self.next_stage = None + + def connect(self, stage): + self.next_stage = stage + return stage + + def rerun(self): + """ + Does this stage need to be rerun? + """ + return (self.args.rerun is not None and + self.args.rerun == self.name) or \ + (self.args.rerun_all) or \ + (self.args.rerun_from is not None and + self.name in self.args.rerun_from) + + def run(self, outputs = {}): + start_time = system.now_raw() + log.ODM_INFO('Running %s stage' % self.name) + + self.process(self.args, outputs) + + # The tree variable should always be populated at this point + if outputs.get('tree') is None: + raise Exception("Assert violation: tree variable is missing from outputs dictionary.") + + if self.args.time: + system.benchmark(start_time, outputs['tree'].benchmarking, self.name) + + log.ODM_INFO('Finished %s stage' % self.name) + + # Last stage? + if self.args.end_with == self.name: + exit(0) + + # Run next stage? + elif self.next_stage is not None: + self.next_stage.run(outputs) + + else: + log.ODM_INFO("No more stages to run") + + def process(self, args, outputs): + raise NotImplementedError + diff --git a/portable.Dockerfile b/portable.Dockerfile index e36b46fb..410619c5 100644 --- a/portable.Dockerfile +++ b/portable.Dockerfile @@ -48,14 +48,11 @@ RUN apt-get install --no-install-recommends -y \ libvtk6-dev \ libxext-dev \ python-dev \ - python-empy \ python-gdal \ python-matplotlib \ python-networkx \ - python-nose \ python-pip \ python-pyproj \ - python-pyside \ python-software-properties \ python-wheel \ swig2.0 @@ -65,7 +62,6 @@ RUN pip install --upgrade pip RUN pip install setuptools RUN pip install -U \ appsettings \ - catkin-pkg \ exifread \ gpxpy \ loky \ diff --git a/run.py b/run.py index b9b0089f..731a8ba4 100644 --- a/run.py +++ b/run.py @@ -5,8 +5,8 @@ from opendm import config from opendm import system from opendm import io -import ecto import os +from pipes import quote from scripts.odm_app import ODMApp @@ -23,28 +23,23 @@ if __name__ == '__main__': system.mkdir_p(os.path.abspath(args.project_path)) # If user asks to rerun everything, delete all of the existing progress directories. - # TODO: Move this somewhere it's not hard-coded if args.rerun_all: log.ODM_DEBUG("Rerun all -- Removing old data") - os.system("rm -rf " - + args.project_path + "/images_resize " - + args.project_path + "/odm_georeferencing " - + args.project_path + "/odm_meshing " - + args.project_path + "/odm_orthophoto " - + args.project_path + "/odm_texturing " - + args.project_path + "/opensfm " - + args.project_path + "/mve") + os.system("rm -rf " + + " ".join([ + quote(os.path.join(args.project_path, "odm_georeferencing")), + quote(os.path.join(args.project_path, "odm_meshing")), + quote(os.path.join(args.project_path, "odm_orthophoto")), + quote(os.path.join(args.project_path, "odm_texturing")), + quote(os.path.join(args.project_path, "opensfm")), + quote(os.path.join(args.project_path, "odm_filterpoints")), + quote(os.path.join(args.project_path, "odm_25dmeshing")), + quote(os.path.join(args.project_path, "odm_25dtexturing")), + quote(os.path.join(args.project_path, "mve")), + ]) + "") - # create an instance of my App BlackBox - # internally configure all tasks - app = ODMApp(args=args) - - # create a plasm that only contains the BlackBox - plasm = ecto.Plasm() - plasm.insert(app) - - # execute the plasm - plasm.execute(niter=1) + app = ODMApp(args) + app.execute() log.ODM_INFO('MMMMMMMMMMMNNNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMNNNMMMMMMMMMMM') log.ODM_INFO('MMMMMMdo:..---../sNMMMMMMMMMMMMMMMMMMMMMMMMMMNs/..---..:odMMMMMM') diff --git a/scripts/dataset.py b/scripts/dataset.py index bf620da5..8d3d9f77 100644 --- a/scripts/dataset.py +++ b/scripts/dataset.py @@ -1,5 +1,4 @@ import os -import ecto import json from opendm import context @@ -37,18 +36,18 @@ def load_images_database(database_file): return result -class ODMLoadDatasetCell(ecto.Cell): +class ODMLoadDatasetStage(types.ODM_Stage): + def process(self, args, outputs): + # Load tree + tree = types.ODM_Tree(args.project_path, args.images, args.gcp) + outputs['tree'] = tree - def declare_params(self, params): - params.declare("verbose", 'indicate verbosity', False) - params.declare("proj", 'Geographic projection', None) - - def declare_io(self, params, inputs, outputs): - inputs.declare("tree", "Struct with paths", []) - outputs.declare("reconstruction", "ODMReconstruction", []) - inputs.declare("args", "The application arguments.", {}) - - def process(self, inputs, outputs): + if args.time and io.file_exists(tree.benchmarking): + # Delete the previously made file + os.remove(tree.benchmarking) + with open(tree.benchmarking, 'a') as b: + b.write('ODM Benchmarking file created %s\nNumber of Cores: %s\n\n' % (system.now(), context.num_cores)) + # check if the extension is supported def supported_extension(file_name): (pathfn, ext) = os.path.splitext(file_name) @@ -60,12 +59,6 @@ class ODMLoadDatasetCell(ecto.Cell): log.ODM_DEBUG(in_dir) return [f for f in io.get_files_list(in_dir) if supported_extension(f)] - log.ODM_INFO('Running ODM Load Dataset Cell') - - # get inputs - tree = self.inputs.tree - args = self.inputs.args - # get images directory input_dir = tree.input_images images_dir = tree.dataset_raw @@ -82,14 +75,8 @@ class ODMLoadDatasetCell(ecto.Cell): log.ODM_DEBUG('Loading dataset from: %s' % images_dir) # check if we rerun cell or not - rerun_cell = (args.rerun is not None and - args.rerun == 'dataset') or \ - (args.rerun_all) or \ - (args.rerun_from is not None and - 'dataset' in args.rerun_from) - images_database_file = io.join_paths(tree.root_path, 'images.json') - if not io.file_exists(images_database_file) or rerun_cell: + if not io.file_exists(images_database_file) or self.rerun(): files = get_images(images_dir) if files: # create ODMPhoto list @@ -105,7 +92,7 @@ class ODMLoadDatasetCell(ecto.Cell): save_images_database(photos, images_database_file) else: log.ODM_ERROR('Not enough supported images in %s' % images_dir) - return ecto.QUIT + exit(1) else: # We have an images database, just load it photos = load_images_database(images_database_file) @@ -113,13 +100,13 @@ class ODMLoadDatasetCell(ecto.Cell): log.ODM_INFO('Found %s usable images' % len(photos)) # append photos to cell output - if not self.params.proj: + if not self.params.get('proj'): if tree.odm_georeferencing_gcp: - outputs.reconstruction = types.ODM_Reconstruction(photos, coords_file=tree.odm_georeferencing_gcp) + outputs['reconstruction'] = types.ODM_Reconstruction(photos, coords_file=tree.odm_georeferencing_gcp) else: # Generate UTM from images try: - if not io.file_exists(tree.odm_georeferencing_coords) or rerun_cell: + if not io.file_exists(tree.odm_georeferencing_coords) or self.rerun(): location.extract_utm_coords(photos, tree.dataset_raw, tree.odm_georeferencing_coords) else: log.ODM_INFO("Coordinates file already exist: %s" % tree.odm_georeferencing_coords) @@ -127,15 +114,13 @@ class ODMLoadDatasetCell(ecto.Cell): log.ODM_WARNING('Could not generate coordinates file. ' 'Ignore if there is a GCP file') - outputs.reconstruction = types.ODM_Reconstruction(photos, coords_file=tree.odm_georeferencing_coords) + outputs['reconstruction'] = types.ODM_Reconstruction(photos, coords_file=tree.odm_georeferencing_coords) else: - outputs.reconstruction = types.ODM_Reconstruction(photos, projstring=self.params.proj) + outputs['reconstruction'] = types.ODM_Reconstruction(photos, projstring=self.params.get('proj')) # Save proj to file for future use (unless this # dataset is not georeferenced) - if outputs.reconstruction.projection: + if outputs['reconstruction'].projection: with open(io.join_paths(tree.odm_georeferencing, tree.odm_georeferencing_proj), 'w') as f: - f.write(outputs.reconstruction.projection.srs) + f.write(outputs['reconstruction'].projection.srs) - log.ODM_INFO('Running ODM Load Dataset Cell - Finished') - return ecto.OK if args.end_with != 'dataset' else ecto.QUIT diff --git a/scripts/example_ecto_python.py b/scripts/example_ecto_python.py deleted file mode 100644 index 9d482675..00000000 --- a/scripts/example_ecto_python.py +++ /dev/null @@ -1,35 +0,0 @@ -import ecto -import numpy as np -class ClusterDetector(ecto.Cell): - def declare_params(self, params): - params.declare("n", "Max number of clusters.", 10) - - def declare_io(self, params, inputs, outputs): - outputs.declare("clusters", "Clusters output. list of tuples", []) - - def process(self, inputs, outputs): - clusters = [] - for i in range(int(np.random.uniform(0, self.params.n))): - clusters.append( (i, 'c%d'%i) ) - outputs.clusters = clusters - -class ClusterPrinter(ecto.Cell): - def declare_io(self, params, inputs, outputs): - inputs.declare("clusters", "Clusters input") - - def process(self, inputs, outputs): - print "Clusters: ", - for c in inputs.clusters: - print c, - print "\n" - -def app(): - cd = ClusterDetector(n=20) - cp = ClusterPrinter() - plasm = ecto.Plasm() - plasm.connect(cd['clusters'] >> cp['clusters']) - sched = ecto.Scheduler(plasm) - sched.execute(niter=3) - -if __name__ == "__main__": - app() \ No newline at end of file diff --git a/scripts/mve.py b/scripts/mve.py index a42f25b4..352cdd5e 100644 --- a/scripts/mve.py +++ b/scripts/mve.py @@ -1,45 +1,27 @@ -import ecto, shutil, os, glob, math +import shutil, os, glob, math from opendm import log from opendm import io from opendm import system from opendm import context from opendm import point_cloud +from opendm import types -class ODMMveCell(ecto.Cell): - def declare_io(self, params, inputs, outputs): - inputs.declare("tree", "Struct with paths", []) - inputs.declare("args", "The application arguments.", {}) - inputs.declare("reconstruction", "ODMReconstruction", []) - outputs.declare("reconstruction", "list of ODMReconstructions", []) - - def process(self, inputs, outputs): - # Benchmarking - start_time = system.now_raw() - - log.ODM_INFO('Running MVE Cell') - +class ODMMveStage(types.ODM_Stage): + def process(self, args, outputs): # get inputs - tree = inputs.tree - args = inputs.args - reconstruction = inputs.reconstruction + tree = outputs['tree'] + reconstruction = outputs['reconstruction'] photos = reconstruction.photos if not photos: log.ODM_ERROR('Not enough photos in photos array to start MVE') - return ecto.QUIT - - # check if we rerun cell or not - rerun_cell = (args.rerun is not None and - args.rerun == 'mve') or \ - (args.rerun_all) or \ - (args.rerun_from is not None and - 'mve' in args.rerun_from) + exit(1) # check if reconstruction was done before - if not io.file_exists(tree.mve_model) or rerun_cell: + if not io.file_exists(tree.mve_model) or self.rerun(): # cleanup if a rerun - if io.dir_exists(tree.mve_path) and rerun_cell: + if io.dir_exists(tree.mve_path) and self.rerun(): shutil.rmtree(tree.mve_path) # make bundle directory @@ -134,11 +116,3 @@ class ODMMveCell(ecto.Cell): else: log.ODM_WARNING('Found a valid MVE reconstruction file in: %s' % tree.mve_model) - - outputs.reconstruction = reconstruction - - if args.time: - system.benchmark(start_time, tree.benchmarking, 'MVE') - - log.ODM_INFO('Running ODM MVE Cell - Finished') - return ecto.OK if args.end_with != 'mve' else ecto.QUIT diff --git a/scripts/mvstex.py b/scripts/mvstex.py index 6403d1d9..7197731b 100644 --- a/scripts/mvstex.py +++ b/scripts/mvstex.py @@ -1,50 +1,20 @@ -import ecto, os, shutil +import os, shutil from opendm import log from opendm import io from opendm import system from opendm import context +from opendm import types -class ODMMvsTexCell(ecto.Cell): - def declare_params(self, params): - params.declare("data_term", 'Data term: [area, gmi] default: gmi', "gmi") - params.declare("outlier_rem_type", 'Type of photometric outlier removal method: [none, gauss_damping, gauss_clamping]. default: none', "none") - params.declare("skip_vis_test", 'Skip geometric visibility test based on ray intersection.', False) - params.declare("skip_glob_seam_leveling", 'Skip global seam leveling.', False) - params.declare("skip_loc_seam_leveling", 'Skip local seam leveling (Poisson editing).', False) - params.declare("skip_hole_fill", 'Skip hole filling.', False) - params.declare("keep_unseen_faces", 'Keep unseen faces.', False) - params.declare("tone_mapping", 'Type of tone mapping: [none, gamma]. Default: gamma', "gamma") - - def declare_io(self, params, inputs, outputs): - inputs.declare("tree", "Struct with paths", []) - inputs.declare("args", "The application arguments.", {}) - inputs.declare("reconstruction", "Clusters output. list of ODMReconstructions", []) - outputs.declare("reconstruction", "Clusters output. list of ODMReconstructions", []) - - def process(self, inputs, outputs): - - # Benchmarking - start_time = system.now_raw() - - log.ODM_INFO('Running MVS Texturing Cell') - - # get inputs - args = inputs.args - tree = inputs.tree - reconstruction = inputs.reconstruction +class ODMMvsTexStage(types.ODM_Stage): + def process(self, args, outputs): + tree = outputs['tree'] + reconstruction = outputs['reconstruction'] # define paths and create working directories system.mkdir_p(tree.odm_texturing) if not args.use_3dmesh: system.mkdir_p(tree.odm_25dtexturing) - # check if we rerun cell or not - rerun_cell = (args.rerun is not None and - args.rerun == 'mvs_texturing') or \ - (args.rerun_all) or \ - (args.rerun_from is not None and - 'mvs_texturing' in args.rerun_from) - runs = [{ 'out_dir': tree.odm_texturing, 'model': tree.odm_mesh, @@ -64,7 +34,7 @@ class ODMMvsTexCell(ecto.Cell): for r in runs: odm_textured_model_obj = os.path.join(r['out_dir'], tree.odm_textured_model_obj) - if not io.file_exists(odm_textured_model_obj) or rerun_cell: + if not io.file_exists(odm_textured_model_obj) or self.rerun(): log.ODM_DEBUG('Writing MVS Textured file in: %s' % odm_textured_model_obj) @@ -76,15 +46,15 @@ class ODMMvsTexCell(ecto.Cell): keepUnseenFaces = "" nadir = "" - if (self.params.skip_vis_test): + if (self.params.get('skip_vis_test')): skipGeometricVisibilityTest = "--skip_geometric_visibility_test" - if (self.params.skip_glob_seam_leveling): + if (self.params.get('skip_glob_seam_leveling')): skipGlobalSeamLeveling = "--skip_global_seam_leveling" - if (self.params.skip_loc_seam_leveling): + if (self.params.get('skip_loc_seam_leveling')): skipLocalSeamLeveling = "--skip_local_seam_leveling" - if (self.params.skip_hole_fill): + if (self.params.get('skip_hole_fill')): skipHoleFilling = "--skip_hole_filling" - if (self.params.keep_unseen_faces): + if (self.params.get('keep_unseen_faces')): keepUnseenFaces = "--keep_unseen_faces" if (r['nadir']): nadir = '--nadir_mode' @@ -94,14 +64,14 @@ class ODMMvsTexCell(ecto.Cell): 'bin': context.mvstex_path, 'out_dir': io.join_paths(r['out_dir'], "odm_textured_model"), 'model': r['model'], - 'dataTerm': self.params.data_term, - 'outlierRemovalType': self.params.outlier_rem_type, + 'dataTerm': self.params.get('data_term'), + 'outlierRemovalType': self.params.get('outlier_rem_type'), 'skipGeometricVisibilityTest': skipGeometricVisibilityTest, 'skipGlobalSeamLeveling': skipGlobalSeamLeveling, 'skipLocalSeamLeveling': skipLocalSeamLeveling, 'skipHoleFilling': skipHoleFilling, 'keepUnseenFaces': keepUnseenFaces, - 'toneMapping': self.params.tone_mapping, + 'toneMapping': self.params.get('tone_mapping'), 'nadirMode': nadir, 'nadirWeight': 2 ** args.texturing_nadir_weight - 1, 'nvm_file': io.join_paths(tree.opensfm, "reconstruction.nvm") @@ -128,10 +98,3 @@ class ODMMvsTexCell(ecto.Cell): log.ODM_WARNING('Found a valid ODM Texture file in: %s' % odm_textured_model_obj) - outputs.reconstruction = reconstruction - - if args.time: - system.benchmark(start_time, tree.benchmarking, 'Texturing') - - log.ODM_INFO('Running ODM Texturing Cell - Finished') - return ecto.OK if args.end_with != 'mvs_texturing' else ecto.QUIT diff --git a/scripts/odm_app.py b/scripts/odm_app.py index 85fee8d5..4c8a5f07 100644 --- a/scripts/odm_app.py +++ b/scripts/odm_app.py @@ -1,174 +1,104 @@ -import ecto import os from opendm import context from opendm import types from opendm import io from opendm import system +from opendm import log -from dataset import ODMLoadDatasetCell -from run_opensfm import ODMOpenSfMCell -from mve import ODMMveCell -from odm_slam import ODMSlamCell -from odm_meshing import ODMeshingCell -from mvstex import ODMMvsTexCell -from odm_georeferencing import ODMGeoreferencingCell -from odm_orthophoto import ODMOrthoPhotoCell -from odm_dem import ODMDEMCell +from dataset import ODMLoadDatasetStage +from run_opensfm import ODMOpenSfMStage +from mve import ODMMveStage +from odm_slam import ODMSlamStage +from odm_meshing import ODMeshingStage +from mvstex import ODMMvsTexStage +from odm_georeferencing import ODMGeoreferencingStage +from odm_orthophoto import ODMOrthoPhotoStage +from odm_dem import ODMDEMStage from odm_filterpoints import ODMFilterPoints -class ODMApp(ecto.BlackBox): - """ODMApp - a class for ODM Activities - """ - - def __init__(self, *args, **kwargs): - ecto.BlackBox.__init__(self, *args, **kwargs) - self.tree = None - - @staticmethod - def declare_direct_params(p): - p.declare("args", "The application arguments.", {}) - - @staticmethod - def declare_cells(p): +class ODMApp: + def __init__(self, args): """ - Implement the virtual function from the base class - Only cells from which something is forwarded have to be declared + Initializes the application and defines the ODM application pipeline stages """ - cells = {'args': ecto.Constant(value=p.args), - 'dataset': ODMLoadDatasetCell(verbose=p.args.verbose, - proj=p.args.proj), - 'opensfm': ODMOpenSfMCell(use_exif_size=False, - feature_process_size=p.args.resize_to, - feature_min_frames=p.args.min_num_features, - processes=p.args.max_concurrency, - matching_gps_neighbors=p.args.matcher_neighbors, - matching_gps_distance=p.args.matcher_distance, - fixed_camera_params=p.args.use_fixed_camera_params, - hybrid_bundle_adjustment=p.args.use_hybrid_bundle_adjustment), - 'slam': ODMSlamCell(), - - 'mve': ODMMveCell(), + + dataset = ODMLoadDatasetStage('dataset', args, + verbose=args.verbose, + proj=args.proj) + opensfm = ODMOpenSfMStage('opensfm', args, + use_exif_size=False, + feature_process_size=args.resize_to, + feature_min_frames=args.min_num_features, + processes=args.max_concurrency, + matching_gps_neighbors=args.matcher_neighbors, + matching_gps_distance=args.matcher_distance, + fixed_camera_params=args.use_fixed_camera_params, + hybrid_bundle_adjustment=args.use_hybrid_bundle_adjustment) + slam = ODMSlamStage('slam', args) + mve = ODMMveStage('mve', args) + filterpoints = ODMFilterPoints('odm_filterpoints', args) + meshing = ODMeshingStage('odm_meshing', args, + max_vertex=args.mesh_size, + oct_tree=args.mesh_octree_depth, + samples=args.mesh_samples, + point_weight=args.mesh_point_weight, + max_concurrency=args.max_concurrency, + verbose=args.verbose) + texturing = ODMMvsTexStage('mvs_texturing', args, + data_term=args.texturing_data_term, + outlier_rem_type=args.texturing_outlier_removal_type, + skip_vis_test=args.texturing_skip_visibility_test, + skip_glob_seam_leveling=args.texturing_skip_global_seam_leveling, + skip_loc_seam_leveling=args.texturing_skip_local_seam_leveling, + skip_hole_fill=args.texturing_skip_hole_filling, + keep_unseen_faces=args.texturing_keep_unseen_faces, + tone_mapping=args.texturing_tone_mapping) + georeferencing = ODMGeoreferencingStage('odm_georeferencing', args, + gcp_file=args.gcp, + use_exif=args.use_exif, + verbose=args.verbose) + dem = ODMDEMStage('odm_dem', args, + max_concurrency=args.max_concurrency, + verbose=args.verbose) + orthophoto = ODMOrthoPhotoStage('odm_orthophoto', args, + resolution=args.orthophoto_resolution, + no_tiled=args.orthophoto_no_tiled, + compress=args.orthophoto_compression, + bigtiff=args.orthophoto_bigtiff, + build_overviews=args.build_overviews, + max_concurrency=args.max_concurrency, + verbose=args.verbose) + + if not args.video: + # Normal pipeline + self.first_stage = dataset - 'filterpoints': ODMFilterPoints(), - - 'meshing': ODMeshingCell(max_vertex=p.args.mesh_size, - oct_tree=p.args.mesh_octree_depth, - samples=p.args.mesh_samples, - point_weight=p.args.mesh_point_weight, - max_concurrency=p.args.max_concurrency, - verbose=p.args.verbose), - 'texturing': ODMMvsTexCell(data_term=p.args.texturing_data_term, - outlier_rem_type=p.args.texturing_outlier_removal_type, - skip_vis_test=p.args.texturing_skip_visibility_test, - skip_glob_seam_leveling=p.args.texturing_skip_global_seam_leveling, - skip_loc_seam_leveling=p.args.texturing_skip_local_seam_leveling, - skip_hole_fill=p.args.texturing_skip_hole_filling, - keep_unseen_faces=p.args.texturing_keep_unseen_faces, - tone_mapping=p.args.texturing_tone_mapping), - 'georeferencing': ODMGeoreferencingCell(gcp_file=p.args.gcp, - use_exif=p.args.use_exif, - verbose=p.args.verbose), - 'dem': ODMDEMCell(max_concurrency=p.args.max_concurrency, - verbose=p.args.verbose), - 'orthophoto': ODMOrthoPhotoCell(resolution=p.args.orthophoto_resolution, - no_tiled=p.args.orthophoto_no_tiled, - compress=p.args.orthophoto_compression, - bigtiff=p.args.orthophoto_bigtiff, - build_overviews=p.args.build_overviews, - max_concurrency=p.args.max_concurrency, - verbose=p.args.verbose) - } + dataset.connect(opensfm) - return cells - - def configure(self, p, _i, _o): - tree = types.ODM_Tree(p.args.project_path, p.args.images, p.args.gcp) - self.tree = ecto.Constant(value=tree) - - # TODO(dakota) put this somewhere better maybe - if p.args.time and io.file_exists(tree.benchmarking): - # Delete the previously made file - os.remove(tree.benchmarking) - with open(tree.benchmarking, 'a') as b: - b.write('ODM Benchmarking file created %s\nNumber of Cores: %s\n\n' % (system.now(), context.num_cores)) - - def connections(self, p): - if p.args.video: - return self.slam_connections(p) - - # load the dataset - connections = [self.tree[:] >> self.dataset['tree'], - self.args[:] >> self.dataset['args']] - - # run opensfm with images from load dataset - connections += [self.tree[:] >> self.opensfm['tree'], - self.args[:] >> self.opensfm['args'], - self.dataset['reconstruction'] >> self.opensfm['reconstruction']] - - if p.args.use_opensfm_dense or p.args.fast_orthophoto: - # filter points from opensfm point cloud - connections += [self.tree[:] >> self.filterpoints['tree'], - self.args[:] >> self.filterpoints['args'], - self.opensfm['reconstruction'] >> self.filterpoints['reconstruction']] + if args.use_opensfm_dense or args.fast_orthophoto: + opensfm.connect(filterpoints) + else: + opensfm.connect(mve) \ + .connect(filterpoints) + + filterpoints \ + .connect(meshing) \ + .connect(texturing) \ + .connect(georeferencing) \ + .connect(dem) \ + .connect(orthophoto) + else: - # run mve - connections += [self.tree[:] >> self.mve['tree'], - self.args[:] >> self.mve['args'], - self.opensfm['reconstruction'] >> self.mve['reconstruction']] + # SLAM pipeline + # TODO: this is broken and needs work + log.ODM_WARNING("SLAM module is currently broken. We could use some help fixing this. If you know Python, get in touch at https://community.opendronemap.org.") + self.first_stage = slam - # filter points from mve point cloud - connections += [self.tree[:] >> self.filterpoints['tree'], - self.args[:] >> self.filterpoints['args'], - self.mve['reconstruction'] >> self.filterpoints['reconstruction']] + slam.connect(mve) \ + .connect(meshing) \ + .connect(texturing) - # create mesh - connections += [self.tree[:] >> self.meshing['tree'], - self.args[:] >> self.meshing['args'], - self.filterpoints['reconstruction'] >> self.meshing['reconstruction']] - - # create odm texture - connections += [self.tree[:] >> self.texturing['tree'], - self.args[:] >> self.texturing['args'], - self.meshing['reconstruction'] >> self.texturing['reconstruction']] - - # create odm georeference - connections += [self.tree[:] >> self.georeferencing['tree'], - self.args[:] >> self.georeferencing['args'], - self.texturing['reconstruction'] >> self.georeferencing['reconstruction']] - - # create odm dem - connections += [self.tree[:] >> self.dem['tree'], - self.args[:] >> self.dem['args'], - self.georeferencing['reconstruction'] >> self.dem['reconstruction']] - - # create odm orthophoto - connections += [self.tree[:] >> self.orthophoto['tree'], - self.args[:] >> self.orthophoto['args'], - self.georeferencing['reconstruction'] >> self.orthophoto['reconstruction']] - return connections - - def slam_connections(self, p): - """Get connections used when running from video instead of images.""" - connections = [] - - # run slam cell - connections += [self.tree[:] >> self.slam['tree'], - self.args[:] >> self.slam['args']] - - connections += [self.tree[:] >> self.mve['tree'], - self.args[:] >> self.mve['args'], - self.slam['reconstruction'] >> self.mve['reconstruction']] - - # create odm mesh - connections += [self.tree[:] >> self.meshing['tree'], - self.args[:] >> self.meshing['args'], - self.mve['reconstruction'] >> self.meshing['reconstruction']] - - # create odm texture - connections += [self.tree[:] >> self.texturing['tree'], - self.args[:] >> self.texturing['args'], - self.meshing['reconstruction'] >> self.texturing['reconstruction']] - - return connections + def execute(self): + self.first_stage.run() \ No newline at end of file diff --git a/scripts/odm_dem.py b/scripts/odm_dem.py index f4b02b6f..c01e7617 100644 --- a/scripts/odm_dem.py +++ b/scripts/odm_dem.py @@ -1,4 +1,4 @@ -import ecto, os, json +import os, json from shutil import copyfile from opendm import io @@ -10,34 +10,11 @@ from opendm import gsd from opendm.dem import commands from opendm.cropper import Cropper -class ODMDEMCell(ecto.Cell): - def declare_params(self, params): - params.declare("verbose", 'print additional messages to console', False) - params.declare("max_concurrency", "Number of threads", context.num_cores) - - def declare_io(self, params, inputs, outputs): - inputs.declare("tree", "Struct with paths", []) - inputs.declare("args", "The application arguments.", {}) - inputs.declare("reconstruction", "list of ODMReconstructions", []) - - def process(self, inputs, outputs): - # Benchmarking - start_time = system.now_raw() - - log.ODM_INFO('Running ODM DEM Cell') - - # get inputs - args = self.inputs.args - tree = self.inputs.tree +class ODMDEMStage(types.ODM_Stage): + def process(self, args, outputs): + tree = outputs['tree'] las_model_found = io.file_exists(tree.odm_georeferencing_model_laz) - # check if we rerun cell or not - rerun_cell = (args.rerun is not None and - args.rerun == 'odm_dem') or \ - (args.rerun_all) or \ - (args.rerun_from is not None and - 'odm_dem' in args.rerun_from) - log.ODM_INFO('Classify: ' + str(args.pc_classify)) log.ODM_INFO('Create DSM: ' + str(args.dsm)) log.ODM_INFO('Create DTM: ' + str(args.dtm)) @@ -51,7 +28,7 @@ class ODMDEMCell(ecto.Cell): if args.pc_classify and las_model_found: pc_classify_marker = os.path.join(odm_dem_root, 'pc_classify_done.txt') - if not io.file_exists(pc_classify_marker) or rerun_cell: + if not io.file_exists(pc_classify_marker) or self.rerun(): log.ODM_INFO("Classifying {} using Simple Morphological Filter".format(tree.odm_georeferencing_model_laz)) commands.classify(tree.odm_georeferencing_model_laz, args.smrf_scalar, @@ -75,7 +52,7 @@ class ODMDEMCell(ecto.Cell): if (args.dtm and not io.file_exists(dtm_output_filename)) or \ (args.dsm and not io.file_exists(dsm_output_filename)) or \ - rerun_cell: + self.rerun(): products = [] if args.dsm: products.append('dsm') @@ -108,15 +85,9 @@ class ODMDEMCell(ecto.Cell): 'COMPRESS': 'LZW', 'BLOCKXSIZE': 512, 'BLOCKYSIZE': 512, - 'NUM_THREADS': self.params.max_concurrency + 'NUM_THREADS': self.params.get('max_concurrency') }) else: log.ODM_WARNING('Found existing outputs in: %s' % odm_dem_root) else: log.ODM_WARNING('DEM will not be generated') - - if args.time: - system.benchmark(start_time, tree.benchmarking, 'Dem') - - log.ODM_INFO('Running ODM DEM Cell - Finished') - return ecto.OK if args.end_with != 'odm_dem' else ecto.QUIT diff --git a/scripts/odm_filterpoints.py b/scripts/odm_filterpoints.py index fa6a69e2..d0754cbb 100644 --- a/scripts/odm_filterpoints.py +++ b/scripts/odm_filterpoints.py @@ -1,39 +1,21 @@ -import ecto, os +import os from opendm import log from opendm import io from opendm import system from opendm import context from opendm import point_cloud +from opendm import types -class ODMFilterPoints(ecto.Cell): - def declare_io(self, params, inputs, outputs): - inputs.declare("tree", "Struct with paths", []) - inputs.declare("args", "The application arguments.", {}) - inputs.declare("reconstruction", "ODMReconstruction", []) - outputs.declare("reconstruction", "list of ODMReconstructions", []) +class ODMFilterPoints(types.ODM_Stage): + def process(self, args, outputs): + tree = outputs['tree'] + reconstruction = outputs['reconstruction'] - def process(self, inputs, outputs): - # Benchmarking - start_time = system.now_raw() - - log.ODM_INFO('Running ODM FilterPoints Cell') - - # get inputs - tree = inputs.tree - args = inputs.args - reconstruction = inputs.reconstruction - - # check if we rerun cell or not - rerun_cell = (args.rerun is not None and - args.rerun == 'odm_filterpoints') or \ - (args.rerun_all) or \ - (args.rerun_from is not None and - 'odm_filterpoints' in args.rerun_from) if not os.path.exists(tree.odm_filterpoints): system.mkdir_p(tree.odm_filterpoints) # check if reconstruction was done before - if not io.file_exists(tree.filtered_point_cloud) or rerun_cell: + if not io.file_exists(tree.filtered_point_cloud) or self.rerun(): if args.fast_orthophoto: inputPointCloud = os.path.join(tree.opensfm, 'reconstruction.ply') elif args.use_opensfm_dense: @@ -49,11 +31,3 @@ class ODMFilterPoints(ecto.Cell): else: log.ODM_WARNING('Found a valid point cloud file in: %s' % tree.filtered_point_cloud) - - outputs.reconstruction = reconstruction - - if args.time: - system.benchmark(start_time, tree.benchmarking, 'MVE') - - log.ODM_INFO('Running ODM FilterPoints Cell - Finished') - return ecto.OK if args.end_with != 'odm_filterpoints' else ecto.QUIT diff --git a/scripts/odm_georeferencing.py b/scripts/odm_georeferencing.py index c6461c18..92f9fbe0 100644 --- a/scripts/odm_georeferencing.py +++ b/scripts/odm_georeferencing.py @@ -1,4 +1,3 @@ -import ecto import os import struct import pipes @@ -12,45 +11,17 @@ from opendm.cropper import Cropper from opendm import point_cloud -class ODMGeoreferencingCell(ecto.Cell): - def declare_params(self, params): - params.declare("gcp_file", 'path to the file containing the ground control ' - 'points used for georeferencing.The file needs to ' - 'be on the following line format: \neasting ' - 'northing height pixelrow pixelcol imagename', 'gcp_list.txt') - params.declare("use_exif", 'use exif', False) - params.declare("verbose", 'print additional messages to console', False) +class ODMGeoreferencingStage(types.ODM_Stage): + def process(self, args, outputs): + tree = outputs['tree'] + reconstruction = outputs['reconstruction'] - def declare_io(self, params, inputs, outputs): - inputs.declare("tree", "Struct with paths", []) - inputs.declare("args", "The application arguments.", {}) - inputs.declare("reconstruction", "list of ODMReconstructions", []) - outputs.declare("reconstruction", "list of ODMReconstructions", []) - - def process(self, inputs, outputs): - - # Benchmarking - start_time = system.now_raw() - - log.ODM_INFO('Running ODM Georeferencing Cell') - - # get inputs - args = inputs.args - tree = inputs.tree - reconstruction = inputs.reconstruction gcpfile = tree.odm_georeferencing_gcp doPointCloudGeo = True transformPointCloud = True - verbose = '-verbose' if self.params.verbose else '' + verbose = '-verbose' if self.params.get('verbose') else '' geo_ref = reconstruction.georef - # check if we rerun cell or not - rerun_cell = (args.rerun is not None and - args.rerun == 'odm_georeferencing') or \ - (args.rerun_all) or \ - (args.rerun_from is not None and - 'odm_georeferencing' in args.rerun_from) - runs = [{ 'georeferencing_dir': tree.odm_georeferencing, 'texturing_dir': tree.odm_texturing, @@ -78,7 +49,7 @@ class ODMGeoreferencingCell(ecto.Cell): odm_georeferencing_model_txt_geo_file = os.path.join(r['georeferencing_dir'], tree.odm_georeferencing_model_txt_geo) if not io.file_exists(odm_georeferencing_model_obj_geo) or \ - not io.file_exists(tree.odm_georeferencing_model_laz) or rerun_cell: + not io.file_exists(tree.odm_georeferencing_model_laz) or self.rerun(): # odm_georeference definitions kwargs = { @@ -111,7 +82,7 @@ class ODMGeoreferencingCell(ecto.Cell): # Check to see if the GCP file exists - if not self.params.use_exif and (self.params.gcp_file or tree.odm_georeferencing_gcp): + if not self.params.get('use_exif') and (self.params.get('gcp_file') or tree.odm_georeferencing_gcp): log.ODM_INFO('Found %s' % gcpfile) try: system.run('{bin}/odm_georef -bundleFile {bundle} -imagesPath {imgs} -imagesListPath {imgs_list} ' @@ -121,7 +92,7 @@ class ODMGeoreferencingCell(ecto.Cell): '-outputCoordFile {coords}'.format(**kwargs)) except Exception: log.ODM_EXCEPTION('Georeferencing failed. ') - return ecto.QUIT + exit(1) elif io.file_exists(tree.opensfm_transformation) and io.file_exists(tree.odm_georeferencing_coords): log.ODM_INFO('Running georeferencing with OpenSfM transformation matrix') system.run('{bin}/odm_georef -bundleFile {bundle} -inputTransformFile {input_trans_file} -inputCoordFile {coords} ' @@ -188,11 +159,3 @@ class ODMGeoreferencingCell(ecto.Cell): else: log.ODM_WARNING('Found a valid georeferenced model in: %s' % tree.odm_georeferencing_model_laz) - - outputs.reconstruction = reconstruction - - if args.time: - system.benchmark(start_time, tree.benchmarking, 'Georeferencing') - - log.ODM_INFO('Running ODM Georeferencing Cell - Finished') - return ecto.OK if args.end_with != 'odm_georeferencing' else ecto.QUIT diff --git a/scripts/odm_meshing.py b/scripts/odm_meshing.py index 835c2c37..1409ff6a 100644 --- a/scripts/odm_meshing.py +++ b/scripts/odm_meshing.py @@ -1,4 +1,4 @@ -import ecto, os, math +import os, math from opendm import log from opendm import io @@ -6,62 +6,29 @@ from opendm import system from opendm import context from opendm import mesh from opendm import gsd +from opendm import types - -class ODMeshingCell(ecto.Cell): - def declare_params(self, params): - params.declare("max_vertex", 'The maximum vertex count of the output ' - 'mesh', 100000) - params.declare("oct_tree", 'Oct-tree depth used in the mesh reconstruction, ' - 'increase to get more vertices, recommended ' - 'values are 8-12', 9) - params.declare("samples", 'Number of points per octree node, recommended ' - 'value: 1.0', 1) - params.declare("point_weight", "specifies the importance that interpolation of the point samples is given in the formulation of the screened Poisson equation.", 4) - params.declare("max_concurrency", 'max threads', context.num_cores) - params.declare("verbose", 'print additional messages to console', False) - - def declare_io(self, params, inputs, outputs): - inputs.declare("tree", "Struct with paths", []) - inputs.declare("args", "The application arguments.", {}) - inputs.declare("reconstruction", "Clusters output. list of ODMReconstructions", []) - outputs.declare("reconstruction", "Clusters output. list of ODMReconstructions", []) - - def process(self, inputs, outputs): - - # Benchmarking - start_time = system.now_raw() - - log.ODM_INFO('Running ODM Meshing Cell') - - # get inputs - args = inputs.args - tree = inputs.tree - reconstruction = inputs.reconstruction +class ODMeshingStage(types.ODM_Stage): + def process(self, args, outputs): + tree = outputs['tree'] + reconstruction = outputs['reconstruction'] # define paths and create working directories system.mkdir_p(tree.odm_meshing) - # check if we rerun cell or not - rerun_cell = (args.rerun is not None and - args.rerun == 'odm_meshing') or \ - (args.rerun_all) or \ - (args.rerun_from is not None and - 'odm_meshing' in args.rerun_from) - # Create full 3D model unless --skip-3dmodel is set if not args.skip_3dmodel: - if not io.file_exists(tree.odm_mesh) or rerun_cell: + if not io.file_exists(tree.odm_mesh) or self.rerun(): log.ODM_DEBUG('Writing ODM Mesh file in: %s' % tree.odm_mesh) mesh.screened_poisson_reconstruction(tree.filtered_point_cloud, tree.odm_mesh, - depth=self.params.oct_tree, - samples=self.params.samples, - maxVertexCount=self.params.max_vertex, - pointWeight=self.params.point_weight, - threads=self.params.max_concurrency, - verbose=self.params.verbose) + depth=self.params.get('oct_tree'), + samples=self.params.get('samples'), + maxVertexCount=self.params.get('max_vertex'), + pointWeight=self.params.get('point_weight'), + threads=self.params.get('max_concurrency'), + verbose=self.params.get('verbose')) else: log.ODM_WARNING('Found a valid ODM Mesh file in: %s' % @@ -70,7 +37,7 @@ class ODMeshingCell(ecto.Cell): # Always generate a 2.5D mesh # unless --use-3dmesh is set. if not args.use_3dmesh: - if not io.file_exists(tree.odm_25dmesh) or rerun_cell: + if not io.file_exists(tree.odm_25dmesh) or self.rerun(): log.ODM_DEBUG('Writing ODM 2.5D Mesh file in: %s' % tree.odm_25dmesh) ortho_resolution = gsd.cap_resolution(args.orthophoto_resolution, tree.opensfm_reconstruction, ignore_gsd=args.ignore_gsd) / 100.0 @@ -94,20 +61,13 @@ class ODMeshingCell(ecto.Cell): mesh.create_25dmesh(tree.filtered_point_cloud, tree.odm_25dmesh, dsm_radius=dsm_radius, dsm_resolution=dsm_resolution, - depth=self.params.oct_tree, - maxVertexCount=self.params.max_vertex, - samples=self.params.samples, - verbose=self.params.verbose, + depth=self.params.get('oct_tree'), + maxVertexCount=self.params.get('max_vertex'), + samples=self.params.get('samples'), + verbose=self.params.get('verbose'), available_cores=args.max_concurrency, method='poisson' if args.fast_orthophoto else 'gridded') else: log.ODM_WARNING('Found a valid ODM 2.5D Mesh file in: %s' % tree.odm_25dmesh) - outputs.reconstruction = reconstruction - - if args.time: - system.benchmark(start_time, tree.benchmarking, 'Meshing') - - log.ODM_INFO('Running ODM Meshing Cell - Finished') - return ecto.OK if args.end_with != 'odm_meshing' else ecto.QUIT diff --git a/scripts/odm_orthophoto.py b/scripts/odm_orthophoto.py index 9d176866..f9f16131 100644 --- a/scripts/odm_orthophoto.py +++ b/scripts/odm_orthophoto.py @@ -1,4 +1,4 @@ -import ecto, os +import os from opendm import io from opendm import log @@ -10,45 +10,16 @@ from opendm.concurrency import get_max_memory from opendm.cropper import Cropper -class ODMOrthoPhotoCell(ecto.Cell): - def declare_params(self, params): - params.declare("resolution", 'Orthophoto resolution in cm / pixel', 5) - params.declare("no_tiled", 'Do not tile tiff', False) - params.declare("compress", 'Compression type', 'DEFLATE') - params.declare("bigtiff", 'Make BigTIFF orthophoto', 'IF_SAFER') - params.declare("build_overviews", 'Build overviews', False) - params.declare("verbose", 'print additional messages to console', False) - params.declare("max_concurrency", "number of threads", context.num_cores) - - def declare_io(self, params, inputs, outputs): - inputs.declare("tree", "Struct with paths", []) - inputs.declare("args", "The application arguments.", {}) - inputs.declare("reconstruction", "list of ODMReconstructions", []) - - def process(self, inputs, outputs): - - # Benchmarking - start_time = system.now_raw() - - log.ODM_INFO('Running ODM Orthophoto Cell') - - # get inputs - args = self.inputs.args - tree = self.inputs.tree - reconstruction = inputs.reconstruction - verbose = '-verbose' if self.params.verbose else '' +class ODMOrthoPhotoStage(types.ODM_Stage): + def process(self, args, outputs): + tree = outputs['tree'] + reconstruction = outputs['reconstruction'] + verbose = '-verbose' if self.params.get('verbose') else '' # define paths and create working directories system.mkdir_p(tree.odm_orthophoto) - # check if we rerun cell or not - rerun_cell = (args.rerun is not None and - args.rerun == 'odm_orthophoto') or \ - (args.rerun_all) or \ - (args.rerun_from is not None and - 'odm_orthophoto' in args.rerun_from) - - if not io.file_exists(tree.odm_orthophoto_file) or rerun_cell: + if not io.file_exists(tree.odm_orthophoto_file) or self.rerun(): # odm_orthophoto definitions kwargs = { @@ -56,7 +27,7 @@ class ODMOrthoPhotoCell(ecto.Cell): 'log': tree.odm_orthophoto_log, 'ortho': tree.odm_orthophoto_file, 'corners': tree.odm_orthophoto_corners, - 'res': 1.0 / (gsd.cap_resolution(self.params.resolution, tree.opensfm_reconstruction, ignore_gsd=args.ignore_gsd) / 100.0), + 'res': 1.0 / (gsd.cap_resolution(self.params.get('resolution'), tree.opensfm_reconstruction, ignore_gsd=args.ignore_gsd) / 100.0), 'verbose': verbose } @@ -118,17 +89,17 @@ class ODMOrthoPhotoCell(ecto.Cell): 'uly': uly, 'lrx': lrx, 'lry': lry, - 'tiled': '' if self.params.no_tiled else '-co TILED=yes ', - 'compress': self.params.compress, - 'predictor': '-co PREDICTOR=2 ' if self.params.compress in + 'tiled': '' if self.params.get('no_tiled') else '-co TILED=yes ', + 'compress': self.params.get('compress'), + 'predictor': '-co PREDICTOR=2 ' if self.params.get('compress') in ['LZW', 'DEFLATE'] else '', 'proj': georef.projection.srs, - 'bigtiff': self.params.bigtiff, + 'bigtiff': self.params.get('bigtiff'), 'png': tree.odm_orthophoto_file, 'tiff': tree.odm_orthophoto_tif, 'log': tree.odm_orthophoto_tif_log, 'max_memory': get_max_memory(), - 'threads': self.params.max_concurrency + 'threads': self.params.get('max_concurrency') } system.run('gdal_translate -a_ullr {ulx} {uly} {lrx} {lry} ' @@ -146,16 +117,16 @@ class ODMOrthoPhotoCell(ecto.Cell): if args.crop > 0: shapefile_path = os.path.join(tree.odm_georeferencing, 'odm_georeferenced_model.bounds.shp') Cropper.crop(shapefile_path, tree.odm_orthophoto_tif, { - 'TILED': 'NO' if self.params.no_tiled else 'YES', - 'COMPRESS': self.params.compress, - 'PREDICTOR': '2' if self.params.compress in ['LZW', 'DEFLATE'] else '1', - 'BIGTIFF': self.params.bigtiff, + 'TILED': 'NO' if self.params.get('no_tiled') else 'YES', + 'COMPRESS': self.params.get('compress'), + 'PREDICTOR': '2' if self.params.get('compress') in ['LZW', 'DEFLATE'] else '1', + 'BIGTIFF': self.params.get('bigtiff'), 'BLOCKXSIZE': 512, 'BLOCKYSIZE': 512, - 'NUM_THREADS': self.params.max_concurrency + 'NUM_THREADS': self.params.get('max_concurrency') }) - if self.params.build_overviews: + if self.params.get('build_overviews'): log.ODM_DEBUG("Building Overviews") kwargs = { 'orthophoto': tree.odm_orthophoto_tif, @@ -174,9 +145,3 @@ class ODMOrthoPhotoCell(ecto.Cell): else: log.ODM_WARNING('Found a valid orthophoto in: %s' % tree.odm_orthophoto_file) - - if args.time: - system.benchmark(start_time, tree.benchmarking, 'Orthophoto') - - log.ODM_INFO('Running ODM OrthoPhoto Cell - Finished') - return ecto.OK if args.end_with != 'odm_orthophoto' else ecto.QUIT diff --git a/scripts/odm_slam.py b/scripts/odm_slam.py index 5e532971..9e4fdaa5 100644 --- a/scripts/odm_slam.py +++ b/scripts/odm_slam.py @@ -2,40 +2,23 @@ import os -import ecto - from opendm import log from opendm import io from opendm import system from opendm import context +from opendm import types - -class ODMSlamCell(ecto.Cell): +class ODMSlamStage(types.ODM_Stage): """Run odm_slam on a video and export to opensfm format.""" - def declare_params(self, params): - """Cell parameters.""" - pass - - def declare_io(self, params, inputs, outputs): - """Cell inputs and outputs.""" - inputs.declare("tree", "Struct with paths", []) - inputs.declare("args", "The application arguments.", {}) - outputs.declare("reconstruction", "list of ODMReconstructions", []) - - def process(self, inputs, outputs): - """Run the cell.""" - log.ODM_INFO('Running OMD Slam Cell') - - # get inputs - tree = self.inputs.tree - args = self.inputs.args + def process(self, args, outputs): + tree = outputs['tree'] video = os.path.join(tree.root_path, args.video) slam_config = os.path.join(tree.root_path, args.slam_config) if not video: log.ODM_ERROR('No video provided') - return ecto.QUIT + exit(1) # create working directories system.mkdir_p(tree.opensfm) @@ -46,11 +29,8 @@ class ODMSlamCell(ecto.Cell): trajectory = os.path.join(tree.opensfm, 'KeyFrameTrajectory.txt') map_points = os.path.join(tree.opensfm, 'MapPoints.txt') - # check if we rerun cell or not - rerun_cell = args.rerun == 'slam' - # check if slam was run before - if not io.file_exists(trajectory) or rerun_cell: + if not io.file_exists(trajectory) or self.rerun(): # run slam binary system.run(' '.join([ 'cd {} &&'.format(tree.opensfm), @@ -64,7 +44,7 @@ class ODMSlamCell(ecto.Cell): trajectory)) # check if trajectory was exported to opensfm before - if not io.file_exists(tree.opensfm_reconstruction) or rerun_cell: + if not io.file_exists(tree.opensfm_reconstruction) or self.rerun(): # convert slam to opensfm system.run(' '.join([ 'cd {} &&'.format(tree.opensfm), @@ -85,7 +65,7 @@ class ODMSlamCell(ecto.Cell): tree.opensfm_reconstruction)) # check if reconstruction was exported to bundler before - if not io.file_exists(tree.opensfm_bundle_list) or rerun_cell: + if not io.file_exists(tree.opensfm_bundle_list) or self.rerun(): # convert back to bundler's format system.run( 'PYTHONPATH={} {}/bin/export_bundler {}'.format( @@ -95,5 +75,3 @@ class ODMSlamCell(ecto.Cell): 'Found a valid Bundler file in: {}'.format( tree.opensfm_reconstruction)) - log.ODM_INFO('Running OMD Slam Cell - Finished') - return ecto.OK if args.end_with != 'odm_slam' else ecto.QUIT diff --git a/scripts/run_opensfm.py b/scripts/run_opensfm.py index 1b1491e1..37f8e8ca 100644 --- a/scripts/run_opensfm.py +++ b/scripts/run_opensfm.py @@ -1,4 +1,3 @@ -import ecto import sys import os @@ -8,51 +7,21 @@ from opendm import system from opendm import context from opendm import gsd from opendm import point_cloud +from opendm import types -class ODMOpenSfMCell(ecto.Cell): - def declare_params(self, params): - params.declare("use_exif_size", "The application arguments.", False) - params.declare("feature_process_size", "The application arguments.", 2400) - params.declare("feature_min_frames", "The application arguments.", 4000) - params.declare("processes", "The application arguments.", context.num_cores) - params.declare("matching_gps_neighbors", "The application arguments.", 8) - params.declare("matching_gps_distance", "The application arguments.", 0) - params.declare("fixed_camera_params", "Optimize internal camera parameters", True) - params.declare("hybrid_bundle_adjustment", "Use local + global bundle adjustment", False) - - def declare_io(self, params, inputs, outputs): - inputs.declare("tree", "Struct with paths", []) - inputs.declare("args", "The application arguments.", {}) - inputs.declare("reconstruction", "ODMReconstruction", []) - outputs.declare("reconstruction", "list of ODMReconstructions", []) - - def process(self, inputs, outputs): - - # Benchmarking - start_time = system.now_raw() - - log.ODM_INFO('Running ODM OpenSfM Cell') - - # get inputs - tree = inputs.tree - args = inputs.args - reconstruction = inputs.reconstruction +class ODMOpenSfMStage(types.ODM_Stage): + def process(self, args, outputs): + tree = outputs['tree'] + reconstruction = outputs['reconstruction'] photos = reconstruction.photos if not photos: log.ODM_ERROR('Not enough photos in photos array to start OpenSfM') - return ecto.QUIT + exit(1) # create working directories system.mkdir_p(tree.opensfm) - # check if we rerun cell or not - rerun_cell = (args.rerun is not None and - args.rerun == 'opensfm') or \ - (args.rerun_all) or \ - (args.rerun_from is not None and - 'opensfm' in args.rerun_from) - if args.fast_orthophoto: output_file = io.join_paths(tree.opensfm, 'reconstruction.ply') elif args.use_opensfm_dense: @@ -61,7 +30,7 @@ class ODMOpenSfMCell(ecto.Cell): output_file = tree.opensfm_reconstruction # check if reconstruction was done before - if not io.file_exists(output_file) or rerun_cell: + if not io.file_exists(output_file) or self.rerun(): # create file list list_path = io.join_paths(tree.opensfm, 'image_list.txt') has_alt = True @@ -73,16 +42,16 @@ class ODMOpenSfMCell(ecto.Cell): # create config file for OpenSfM config = [ - "use_exif_size: %s" % ('no' if not self.params.use_exif_size else 'yes'), - "feature_process_size: %s" % self.params.feature_process_size, - "feature_min_frames: %s" % self.params.feature_min_frames, - "processes: %s" % self.params.processes, - "matching_gps_neighbors: %s" % self.params.matching_gps_neighbors, + "use_exif_size: %s" % ('no' if not self.params.get('use_exif_size') else 'yes'), + "feature_process_size: %s" % self.params.get('feature_process_size'), + "feature_min_frames: %s" % self.params.get('feature_min_frames'), + "processes: %s" % self.params.get('processes'), + "matching_gps_neighbors: %s" % self.params.get('matching_gps_neighbors'), "depthmap_method: %s" % args.opensfm_depthmap_method, "depthmap_resolution: %s" % args.depthmap_resolution, "depthmap_min_patch_sd: %s" % args.opensfm_depthmap_min_patch_sd, "depthmap_min_consistent_views: %s" % args.opensfm_depthmap_min_consistent_views, - "optimize_camera_parameters: %s" % ('no' if self.params.fixed_camera_params else 'yes') + "optimize_camera_parameters: %s" % ('no' if self.params.get('fixed_camera_params') else 'yes') ] if has_alt: @@ -100,7 +69,7 @@ class ODMOpenSfMCell(ecto.Cell): config.append("local_bundle_radius: 1") # Max image graph distance for images to be included in local bundle adjustment if args.matcher_distance > 0: - config.append("matching_gps_distance: %s" % self.params.matching_gps_distance) + config.append("matching_gps_distance: %s" % self.params.get('matching_gps_distance')) if tree.odm_georeferencing_gcp: config.append("bundle_use_gcp: yes") @@ -114,7 +83,7 @@ class ODMOpenSfMCell(ecto.Cell): # run OpenSfM reconstruction matched_done_file = io.join_paths(tree.opensfm, 'matching_done.txt') - if not io.file_exists(matched_done_file) or rerun_cell: + if not io.file_exists(matched_done_file) or self.rerun(): system.run('PYTHONPATH=%s %s/bin/opensfm extract_metadata %s' % (context.pyopencv_path, context.opensfm_path, tree.opensfm)) system.run('PYTHONPATH=%s %s/bin/opensfm detect_features %s' % @@ -127,14 +96,14 @@ class ODMOpenSfMCell(ecto.Cell): log.ODM_WARNING('Found a feature matching done progress file in: %s' % matched_done_file) - if not io.file_exists(tree.opensfm_tracks) or rerun_cell: + if not io.file_exists(tree.opensfm_tracks) or self.rerun(): system.run('PYTHONPATH=%s %s/bin/opensfm create_tracks %s' % (context.pyopencv_path, context.opensfm_path, tree.opensfm)) else: log.ODM_WARNING('Found a valid OpenSfM tracks file in: %s' % tree.opensfm_tracks) - if not io.file_exists(tree.opensfm_reconstruction) or rerun_cell: + if not io.file_exists(tree.opensfm_reconstruction) or self.rerun(): system.run('PYTHONPATH=%s %s/bin/opensfm reconstruct %s' % (context.pyopencv_path, context.opensfm_path, tree.opensfm)) else: @@ -159,7 +128,7 @@ class ODMOpenSfMCell(ecto.Cell): else: image_scale = 1.0 - if not io.file_exists(tree.opensfm_reconstruction_nvm) or rerun_cell: + if not io.file_exists(tree.opensfm_reconstruction_nvm) or self.rerun(): system.run('PYTHONPATH=%s %s/bin/opensfm export_visualsfm --image_extension png --scale_focal %s %s' % (context.pyopencv_path, context.opensfm_path, image_scale, tree.opensfm)) else: @@ -188,7 +157,7 @@ class ODMOpenSfMCell(ecto.Cell): tree.opensfm_reconstruction) # check if reconstruction was exported to bundler before - if not io.file_exists(tree.opensfm_bundle_list) or rerun_cell: + if not io.file_exists(tree.opensfm_bundle_list) or self.rerun(): # convert back to bundler's format system.run('PYTHONPATH=%s %s/bin/export_bundler %s' % (context.pyopencv_path, context.opensfm_path, tree.opensfm)) @@ -199,11 +168,3 @@ class ODMOpenSfMCell(ecto.Cell): if reconstruction.georef: system.run('PYTHONPATH=%s %s/bin/opensfm export_geocoords %s --transformation --proj \'%s\'' % (context.pyopencv_path, context.opensfm_path, tree.opensfm, reconstruction.georef.projection.srs)) - - outputs.reconstruction = reconstruction - - if args.time: - system.benchmark(start_time, tree.benchmarking, 'OpenSfM') - - log.ODM_INFO('Running ODM OpenSfM Cell - Finished') - return ecto.OK if args.end_with != 'opensfm' else ecto.QUIT diff --git a/tests/test_odm.py b/tests/test_odm.py deleted file mode 100644 index 851b5565..00000000 --- a/tests/test_odm.py +++ /dev/null @@ -1,108 +0,0 @@ -import unittest -import os -import shutil - -import ecto -from opendm import config -from opendm import context -from scripts.odm_app import ODMApp -from ecto.opts import scheduler_options, run_plasm - -parser = config.parser -scheduler_options(parser) -options = config.config() - - -def appSetup(options): - app = ODMApp(args=options) - plasm = ecto.Plasm() - plasm.insert(app) - return app, plasm - - -def setup_module(): - # Run tests - print '%s' % options - options.project_path = context.tests_data_path - # options.rerun_all = True - app, plasm = appSetup(options) - print 'Run Setup: Initial Run' - run_plasm(options, plasm) - # options.rerun_all = False - - -def teardown_module(): - # Delete generated test directories - dirnames = ['opensfm', 'odm_meshing', - 'odm_texturing', 'odm_georeferencing', 'odm_orthophoto'] - for n in dirnames: - rmpath = os.path.join(context.tests_data_path, n) - if os.path.exists(rmpath): - shutil.rmtree(rmpath) - - -class TestOpenSfM(unittest.TestCase): - """ - Tests the OpenSfM module - """ - def setUp(self): - options.rerun = 'opensfm' - self.app, self.plasm = appSetup(options) - run_plasm(options, self.plasm) - - def test_opensfm(self): - # Test configuration - self.assertTrue(os.path.isfile(self.app.opensfm.inputs.tree.opensfm_reconstruction)) - - -class TestMeshing(unittest.TestCase): - - def setUp(self): - options.rerun = 'odm_meshing' - self.app, self.plasm = appSetup(options) - run_plasm(options, self.plasm) - - def test_meshing(self): - self.assertTrue(os.path.isfile(self.app.meshing.inputs.tree.odm_mesh)) - - -class TestTexturing(unittest.TestCase): - - def setUp(self): - options.rerun = 'odm_texturing' - self.app, self.plasm = appSetup(options) - run_plasm(options, self.plasm) - - def test_texturing(self): - self.assertTrue(os.path.isfile(self.app.texturing.inputs.tree.odm_textured_model_obj)) - - -class TestGeoreferencing(unittest.TestCase): - - def setUp(self): - options.rerun = 'odm_georeferencing' - self.app, self.plasm = appSetup(options) - run_plasm(options, self.plasm) - - def test_georef(self): - self.assertTrue(os.path.isfile(self.app.georeferencing.inputs.tree.odm_georeferencing_coords) & - os.path.isfile(self.app.georeferencing.inputs.tree.odm_georeferencing_model_obj_geo)) - - def test_las_out(self): - self.assertTrue(os.path.isfile(os.path.join(self.app.georeferencing.inputs.tree.odm_georeferencing, - "odm_georeferenced_model.laz"))) - - -class TestOrthophoto(unittest.TestCase): - - def setUp(self): - options.rerun = 'odm_orthophoto' - self.app, self.plasm = appSetup(options) - run_plasm(options, self.plasm) - - def test_orthophoto(self): - self.assertTrue(os.path.isfile(self.app.orthophoto.inputs.tree.odm_orthophoto_file)) - - -if __name__ == '__main__': - unittest.main() \ No newline at end of file