Merge pull request #973 from pierotofy/ectorep

Remove ecto

Dockerfile

@@ -48,14 +48,11 @@ RUN apt-get install --no-install-recommends -y \
   libvtk6-dev \
   libxext-dev \
   python-dev \
-  python-empy \
   python-gdal \
   python-matplotlib \
   python-networkx \
-  python-nose \
   python-pip \
   python-pyproj \
-  python-pyside \
   python-software-properties \
   python-wheel \
   swig2.0
@@ -65,7 +62,6 @@ RUN pip install --upgrade pip
 RUN pip install setuptools
 RUN pip install -U \
   appsettings \
-  catkin-pkg \
   exifread \
   gpxpy \
   loky \

SuperBuild/CMakeLists.txt

@@ -103,14 +103,10 @@ SETUP_EXTERNAL_PROJECT(Hexer 1.4 ON)
 # ---------------------------------------------------------------------------------------------
 # Open Geometric Vision (OpenGV)
 # Open Structure from Motion (OpenSfM)
-# Catkin
-# Ecto
 #
 
 set(custom_libs OpenGV
 				OpenSfM
-				Catkin
-				Ecto
 				LASzip
 				PDAL
 				MvsTexturing

SuperBuild/cmake/External-Catkin.cmake

@@ -1,27 +0,0 @@
-set(_proj_name catkin)
-set(_SB_BINARY_DIR "${SB_BINARY_DIR}/${_proj_name}")
-
-ExternalProject_Add(${_proj_name}
-  PREFIX            ${_SB_BINARY_DIR}
-  TMP_DIR           ${_SB_BINARY_DIR}/tmp
-  STAMP_DIR         ${_SB_BINARY_DIR}/stamp
-  #--Download step--------------
-  DOWNLOAD_DIR      ${SB_DOWNLOAD_DIR}
-  URL               https://github.com/ros/catkin/archive/0.6.16.zip
-  URL_MD5           F5D45AE68709CE6E3346FB8C019416F8
-  #--Update/Patch step----------
-  UPDATE_COMMAND    ""
-  #--Configure step-------------
-  SOURCE_DIR        ${SB_SOURCE_DIR}/${_proj_name}
-  CMAKE_ARGS
-    -DCATKIN_ENABLE_TESTING=OFF
-    -DCMAKE_INSTALL_PREFIX:PATH=${SB_INSTALL_DIR}
-  #--Build step-----------------
-  BINARY_DIR        ${_SB_BINARY_DIR}
-  #--Install step---------------
-  INSTALL_DIR       ${SB_INSTALL_DIR}
-  #--Output logging-------------
-  LOG_DOWNLOAD      OFF
-  LOG_CONFIGURE     OFF
-  LOG_BUILD         OFF
-)

SuperBuild/cmake/External-Ecto.cmake

@@ -1,30 +0,0 @@
-set(_proj_name ecto)
-set(_SB_BINARY_DIR "${SB_BINARY_DIR}/${_proj_name}")
-
-ExternalProject_Add(${_proj_name}
-  DEPENDS           catkin
-  PREFIX            ${_SB_BINARY_DIR}
-  TMP_DIR           ${_SB_BINARY_DIR}/tmp
-  STAMP_DIR         ${_SB_BINARY_DIR}/stamp
-  #--Download step--------------
-  DOWNLOAD_DIR      ${SB_DOWNLOAD_DIR}/${_proj_name}
-  URL               https://github.com/plasmodic/ecto/archive/c6178ed0102a66cebf503a4213c27b0f60cfca69.zip
-  URL_MD5           A5C4757B656D536D3E3CC1DC240EC158
-  #--Update/Patch step----------
-  UPDATE_COMMAND    ""
-  #--Configure step-------------
-  SOURCE_DIR        ${SB_SOURCE_DIR}/${_proj_name}
-  CMAKE_ARGS
-    -DBUILD_DOC=OFF
-    -DBUILD_SAMPLES=OFF
-    -DCATKIN_ENABLE_TESTING=OFF
-    -DCMAKE_INSTALL_PREFIX:PATH=${SB_INSTALL_DIR}
-  #--Build step-----------------
-  BINARY_DIR        ${_SB_BINARY_DIR}
-  #--Install step---------------
-  INSTALL_DIR       ${SB_INSTALL_DIR}
-  #--Output logging-------------
-  LOG_DOWNLOAD      OFF
-  LOG_CONFIGURE     OFF
-  LOG_BUILD         OFF
-)

configure.sh

@@ -80,12 +80,6 @@ install() {
                         loky \
                         repoze.lru
 
-    echo "Installing Ecto Dependencies"
-    pip install -U catkin-pkg
-    apt-get install -y -qq python-empy \
-                         python-nose \
-                         python-pyside
-
     echo "Installing OpenDroneMap Dependencies"
     apt-get install -y -qq python-scipy \
                          liblas-bin

opendm/types.py

@@ -329,3 +329,55 @@ class ODM_Tree(object):
 
     def path(self, *args):
         return io.join_paths(self.root_path, *args)
+
+
+class ODM_Stage:
+    def __init__(self, name, args, **params):
+        self.name = name
+        self.args = args
+        self.params = params
+        if self.params is None:
+            self.params = {}
+        self.next_stage = None
+
+    def connect(self, stage):
+        self.next_stage = stage
+        return stage
+
+    def rerun(self):
+        """
+        Does this stage need to be rerun?
+        """
+        return (self.args.rerun is not None and
+                      self.args.rerun == self.name) or \
+                     (self.args.rerun_all) or \
+                     (self.args.rerun_from is not None and
+                      self.name in self.args.rerun_from)
+    
+    def run(self, outputs = {}):
+        start_time = system.now_raw()
+        log.ODM_INFO('Running %s stage' % self.name)
+
+        self.process(self.args, outputs)
+
+        # The tree variable should always be populated at this point
+        if outputs.get('tree') is None:
+            raise Exception("Assert violation: tree variable is missing from outputs dictionary.")
+
+        if self.args.time:
+            system.benchmark(start_time, outputs['tree'].benchmarking, self.name)
+
+        log.ODM_INFO('Finished %s stage' % self.name)
+
+        # Last stage?
+        if self.args.end_with == self.name:
+            log.ODM_INFO("No more stages to run")
+            return
+
+        # Run next stage?
+        elif self.next_stage is not None:
+            self.next_stage.run(outputs)
+
+    def process(self, args, outputs):
+        raise NotImplementedError
+

portable.Dockerfile

@@ -48,14 +48,11 @@ RUN apt-get install --no-install-recommends -y \
   libvtk6-dev \
   libxext-dev \
   python-dev \
-  python-empy \
   python-gdal \
   python-matplotlib \
   python-networkx \
-  python-nose \
   python-pip \
   python-pyproj \
-  python-pyside \
   python-software-properties \
   python-wheel \
   swig2.0
@@ -65,7 +62,6 @@ RUN pip install --upgrade pip
 RUN pip install setuptools
 RUN pip install -U \
   appsettings \
-  catkin-pkg \
   exifread \
   gpxpy \
   loky \

run.py

@@ -5,8 +5,8 @@ from opendm import config
 from opendm import system
 from opendm import io
 
-import ecto
 import os
+from pipes import quote
 
 from scripts.odm_app import ODMApp
 
@@ -23,28 +23,23 @@ if __name__ == '__main__':
         system.mkdir_p(os.path.abspath(args.project_path))
 
     # If user asks to rerun everything, delete all of the existing progress directories.
-    # TODO: Move this somewhere it's not hard-coded
     if args.rerun_all:
         log.ODM_DEBUG("Rerun all -- Removing old data")
-        os.system("rm -rf "
+        os.system("rm -rf " + 
-                  + args.project_path + "/images_resize "
+                    " ".join([
-                  + args.project_path + "/odm_georeferencing "
+                        quote(os.path.join(args.project_path, "odm_georeferencing")),
-                  + args.project_path + "/odm_meshing "
+                        quote(os.path.join(args.project_path, "odm_meshing")),
-                  + args.project_path + "/odm_orthophoto "
+                        quote(os.path.join(args.project_path, "odm_orthophoto")),
-                  + args.project_path + "/odm_texturing "
+                        quote(os.path.join(args.project_path, "odm_texturing")),
-                  + args.project_path + "/opensfm "
+                        quote(os.path.join(args.project_path, "opensfm")),
-                  + args.project_path + "/mve")
+                        quote(os.path.join(args.project_path, "odm_filterpoints")),
+                        quote(os.path.join(args.project_path, "odm_25dmeshing")),
+                        quote(os.path.join(args.project_path, "odm_25dtexturing")),
+                        quote(os.path.join(args.project_path, "mve")),
+                    ]) + "")
 
-    # create an instance of my App BlackBox
+    app = ODMApp(args)
-    # internally configure all tasks
+    app.execute()
-    app = ODMApp(args=args)
-
-    # create a plasm that only contains the BlackBox
-    plasm = ecto.Plasm()
-    plasm.insert(app)
-
-    # execute the plasm
-    plasm.execute(niter=1)
     
     log.ODM_INFO('MMMMMMMMMMMNNNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMNNNMMMMMMMMMMM')
     log.ODM_INFO('MMMMMMdo:..---../sNMMMMMMMMMMMMMMMMMMMMMMMMMMNs/..---..:odMMMMMM')

scripts/dataset.py

@@ -1,5 +1,4 @@
 import os
-import ecto
 import json
 
 from opendm import context
@@ -37,18 +36,18 @@ def load_images_database(database_file):
 
     return result
 
-class ODMLoadDatasetCell(ecto.Cell):
+class ODMLoadDatasetStage(types.ODM_Stage):
+    def process(self, args, outputs):
+        # Load tree
+        tree = types.ODM_Tree(args.project_path, args.images, args.gcp)
+        outputs['tree'] = tree
 
-    def declare_params(self, params):
+        if args.time and io.file_exists(tree.benchmarking):
-        params.declare("verbose", 'indicate verbosity', False)
+            # Delete the previously made file
-        params.declare("proj", 'Geographic projection', None)
+            os.remove(tree.benchmarking)
+            with open(tree.benchmarking, 'a') as b:
+                b.write('ODM Benchmarking file created %s\nNumber of Cores: %s\n\n' % (system.now(), context.num_cores))
     
-    def declare_io(self, params, inputs, outputs):
-        inputs.declare("tree", "Struct with paths", [])
-        outputs.declare("reconstruction", "ODMReconstruction", [])
-        inputs.declare("args", "The application arguments.", {})
-
-    def process(self, inputs, outputs):
         # check if the extension is supported
         def supported_extension(file_name):
             (pathfn, ext) = os.path.splitext(file_name)
@@ -60,12 +59,6 @@ class ODMLoadDatasetCell(ecto.Cell):
             log.ODM_DEBUG(in_dir)
             return [f for f in io.get_files_list(in_dir) if supported_extension(f)]
 
-        log.ODM_INFO('Running ODM Load Dataset Cell')
-
-        # get inputs
-        tree = self.inputs.tree
-        args = self.inputs.args
-
         # get images directory
         input_dir = tree.input_images
         images_dir = tree.dataset_raw
@@ -82,14 +75,8 @@ class ODMLoadDatasetCell(ecto.Cell):
         log.ODM_DEBUG('Loading dataset from: %s' % images_dir)
 
         # check if we rerun cell or not
-        rerun_cell = (args.rerun is not None and
-                      args.rerun == 'dataset') or \
-                     (args.rerun_all) or \
-                     (args.rerun_from is not None and
-                      'dataset' in args.rerun_from)
-        
         images_database_file = io.join_paths(tree.root_path, 'images.json')
-        if not io.file_exists(images_database_file) or rerun_cell:
+        if not io.file_exists(images_database_file) or self.rerun():
             files = get_images(images_dir)
             if files:
                 # create ODMPhoto list
@@ -105,7 +92,7 @@ class ODMLoadDatasetCell(ecto.Cell):
                 save_images_database(photos, images_database_file)
             else:
                 log.ODM_ERROR('Not enough supported images in %s' % images_dir)
-                return ecto.QUIT
+                exit(1)
         else:
             # We have an images database, just load it
             photos = load_images_database(images_database_file)
@@ -113,13 +100,13 @@ class ODMLoadDatasetCell(ecto.Cell):
         log.ODM_INFO('Found %s usable images' % len(photos))
 
         # append photos to cell output
-        if not self.params.proj:
+        if not self.params.get('proj'):
             if tree.odm_georeferencing_gcp:
-                outputs.reconstruction = types.ODM_Reconstruction(photos, coords_file=tree.odm_georeferencing_gcp)
+                outputs['reconstruction'] = types.ODM_Reconstruction(photos, coords_file=tree.odm_georeferencing_gcp)
             else:
                 # Generate UTM from images
                 try:
-                    if not io.file_exists(tree.odm_georeferencing_coords) or rerun_cell:
+                    if not io.file_exists(tree.odm_georeferencing_coords) or self.rerun():
                         location.extract_utm_coords(photos, tree.dataset_raw, tree.odm_georeferencing_coords)
                     else:
                         log.ODM_INFO("Coordinates file already exist: %s" % tree.odm_georeferencing_coords)
@@ -127,15 +114,13 @@ class ODMLoadDatasetCell(ecto.Cell):
                     log.ODM_WARNING('Could not generate coordinates file. '
                                     'Ignore if there is a GCP file')
 
-                outputs.reconstruction = types.ODM_Reconstruction(photos, coords_file=tree.odm_georeferencing_coords)
+                outputs['reconstruction'] = types.ODM_Reconstruction(photos, coords_file=tree.odm_georeferencing_coords)
         else:
-            outputs.reconstruction = types.ODM_Reconstruction(photos, projstring=self.params.proj)
+            outputs['reconstruction'] = types.ODM_Reconstruction(photos, projstring=self.params.get('proj'))
 
         # Save proj to file for future use (unless this 
         # dataset is not georeferenced)
-        if outputs.reconstruction.projection:
+        if outputs['reconstruction'].projection:
             with open(io.join_paths(tree.odm_georeferencing, tree.odm_georeferencing_proj), 'w') as f:
-                f.write(outputs.reconstruction.projection.srs)
+                f.write(outputs['reconstruction'].projection.srs)
 
-        log.ODM_INFO('Running ODM Load Dataset Cell - Finished')
-        return ecto.OK if args.end_with != 'dataset' else ecto.QUIT

scripts/example_ecto_python.py

@@ -1,35 +0,0 @@
-import ecto
-import numpy as np
-class ClusterDetector(ecto.Cell):
-    def declare_params(self, params):
-        params.declare("n", "Max number of clusters.", 10)
-
-    def declare_io(self, params, inputs, outputs):
-        outputs.declare("clusters", "Clusters output. list of tuples", [])
-
-    def process(self, inputs, outputs):
-        clusters = []
-        for i in range(int(np.random.uniform(0, self.params.n))):
-            clusters.append( (i, 'c%d'%i) )
-        outputs.clusters = clusters
-
-class ClusterPrinter(ecto.Cell):
-    def declare_io(self, params, inputs, outputs):
-        inputs.declare("clusters", "Clusters input")
-
-    def process(self, inputs, outputs):
-        print "Clusters: ",
-        for c in inputs.clusters:
-            print c,
-        print "\n"
-
-def app():
-    cd = ClusterDetector(n=20)
-    cp = ClusterPrinter()
-    plasm = ecto.Plasm()
-    plasm.connect(cd['clusters'] >> cp['clusters'])
-    sched = ecto.Scheduler(plasm)
-    sched.execute(niter=3)
-
-if __name__ == "__main__":
-    app()

scripts/mve.py

@@ -1,45 +1,27 @@
-import ecto, shutil, os, glob, math
+import shutil, os, glob, math
 
 from opendm import log
 from opendm import io
 from opendm import system
 from opendm import context
 from opendm import point_cloud
+from opendm import types
 
-class ODMMveCell(ecto.Cell):
+class ODMMveStage(types.ODM_Stage):
-    def declare_io(self, params, inputs, outputs):
+    def process(self, args, outputs):
-        inputs.declare("tree", "Struct with paths", [])
-        inputs.declare("args", "The application arguments.", {})
-        inputs.declare("reconstruction", "ODMReconstruction", [])
-        outputs.declare("reconstruction", "list of ODMReconstructions", [])
-
-    def process(self, inputs, outputs):
-        # Benchmarking
-        start_time = system.now_raw()
-
-        log.ODM_INFO('Running MVE Cell')
-
         # get inputs
-        tree = inputs.tree
+        tree = outputs['tree']
-        args = inputs.args
+        reconstruction = outputs['reconstruction']
-        reconstruction = inputs.reconstruction
         photos = reconstruction.photos
 
         if not photos:
             log.ODM_ERROR('Not enough photos in photos array to start MVE')
-            return ecto.QUIT
+            exit(1)
-
-        # check if we rerun cell or not
-        rerun_cell = (args.rerun is not None and
-                      args.rerun == 'mve') or \
-                     (args.rerun_all) or \
-                     (args.rerun_from is not None and
-                      'mve' in args.rerun_from)
 
         # check if reconstruction was done before
-        if not io.file_exists(tree.mve_model) or rerun_cell:
+        if not io.file_exists(tree.mve_model) or self.rerun():
             # cleanup if a rerun
-            if io.dir_exists(tree.mve_path) and rerun_cell:
+            if io.dir_exists(tree.mve_path) and self.rerun():
                 shutil.rmtree(tree.mve_path)
 
             # make bundle directory
@@ -134,11 +116,3 @@ class ODMMveCell(ecto.Cell):
         else:
             log.ODM_WARNING('Found a valid MVE reconstruction file in: %s' %
                             tree.mve_model)
-
-        outputs.reconstruction = reconstruction
-
-        if args.time:
-            system.benchmark(start_time, tree.benchmarking, 'MVE')
-
-        log.ODM_INFO('Running ODM MVE Cell - Finished')
-        return ecto.OK if args.end_with != 'mve' else ecto.QUIT

scripts/mvstex.py

@@ -1,50 +1,20 @@
-import ecto, os, shutil
+import os, shutil
 
 from opendm import log
 from opendm import io
 from opendm import system
 from opendm import context
+from opendm import types
 
-class ODMMvsTexCell(ecto.Cell):
+class ODMMvsTexStage(types.ODM_Stage):
-    def declare_params(self, params):
+    def process(self, args, outputs):
-        params.declare("data_term", 'Data term: [area, gmi] default: gmi', "gmi")
+        tree = outputs['tree']
-        params.declare("outlier_rem_type", 'Type of photometric outlier removal method: [none, gauss_damping, gauss_clamping]. default: none', "none")
+        reconstruction = outputs['reconstruction']
-        params.declare("skip_vis_test", 'Skip geometric visibility test based on ray intersection.', False)
-        params.declare("skip_glob_seam_leveling", 'Skip global seam leveling.', False)
-        params.declare("skip_loc_seam_leveling", 'Skip local seam leveling (Poisson editing).', False)
-        params.declare("skip_hole_fill", 'Skip hole filling.', False)
-        params.declare("keep_unseen_faces", 'Keep unseen faces.', False)
-        params.declare("tone_mapping", 'Type of tone mapping: [none, gamma]. Default: gamma', "gamma")
-
-    def declare_io(self, params, inputs, outputs):
-        inputs.declare("tree", "Struct with paths", [])
-        inputs.declare("args", "The application arguments.", {})
-        inputs.declare("reconstruction", "Clusters output. list of ODMReconstructions", [])
-        outputs.declare("reconstruction", "Clusters output. list of ODMReconstructions", [])
-
-    def process(self, inputs, outputs):
-
-        # Benchmarking
-        start_time = system.now_raw()
-
-        log.ODM_INFO('Running MVS Texturing Cell')
-
-        # get inputs
-        args = inputs.args
-        tree = inputs.tree
-        reconstruction = inputs.reconstruction
 
         # define paths and create working directories
         system.mkdir_p(tree.odm_texturing)
         if not args.use_3dmesh: system.mkdir_p(tree.odm_25dtexturing)
 
-        # check if we rerun cell or not
-        rerun_cell = (args.rerun is not None and
-                      args.rerun == 'mvs_texturing') or \
-                     (args.rerun_all) or \
-                     (args.rerun_from is not None and
-                      'mvs_texturing' in args.rerun_from)
-
         runs = [{
             'out_dir': tree.odm_texturing,
             'model': tree.odm_mesh,
@@ -64,7 +34,7 @@ class ODMMvsTexCell(ecto.Cell):
         for r in runs:
             odm_textured_model_obj = os.path.join(r['out_dir'], tree.odm_textured_model_obj)
 
-            if not io.file_exists(odm_textured_model_obj) or rerun_cell:
+            if not io.file_exists(odm_textured_model_obj) or self.rerun():
                 log.ODM_DEBUG('Writing MVS Textured file in: %s'
                               % odm_textured_model_obj)
 
@@ -76,15 +46,15 @@ class ODMMvsTexCell(ecto.Cell):
                 keepUnseenFaces = ""
                 nadir = ""
 
-                if (self.params.skip_vis_test):
+                if (self.params.get('skip_vis_test')):
                     skipGeometricVisibilityTest = "--skip_geometric_visibility_test"
-                if (self.params.skip_glob_seam_leveling):
+                if (self.params.get('skip_glob_seam_leveling')):
                     skipGlobalSeamLeveling = "--skip_global_seam_leveling"
-                if (self.params.skip_loc_seam_leveling):
+                if (self.params.get('skip_loc_seam_leveling')):
                     skipLocalSeamLeveling = "--skip_local_seam_leveling"
-                if (self.params.skip_hole_fill):
+                if (self.params.get('skip_hole_fill')):
                     skipHoleFilling = "--skip_hole_filling"
-                if (self.params.keep_unseen_faces):
+                if (self.params.get('keep_unseen_faces')):
                     keepUnseenFaces = "--keep_unseen_faces"
                 if (r['nadir']):
                     nadir = '--nadir_mode'
@@ -94,14 +64,14 @@ class ODMMvsTexCell(ecto.Cell):
                     'bin': context.mvstex_path,
                     'out_dir': io.join_paths(r['out_dir'], "odm_textured_model"),
                     'model': r['model'],
-                    'dataTerm': self.params.data_term,
+                    'dataTerm': self.params.get('data_term'),
-                    'outlierRemovalType': self.params.outlier_rem_type,
+                    'outlierRemovalType': self.params.get('outlier_rem_type'),
                     'skipGeometricVisibilityTest': skipGeometricVisibilityTest,
                     'skipGlobalSeamLeveling': skipGlobalSeamLeveling,
                     'skipLocalSeamLeveling': skipLocalSeamLeveling,
                     'skipHoleFilling': skipHoleFilling,
                     'keepUnseenFaces': keepUnseenFaces,
-                    'toneMapping': self.params.tone_mapping,
+                    'toneMapping': self.params.get('tone_mapping'),
                     'nadirMode': nadir,
                     'nadirWeight': 2 ** args.texturing_nadir_weight - 1,
                     'nvm_file': io.join_paths(tree.opensfm, "reconstruction.nvm")
@@ -128,10 +98,3 @@ class ODMMvsTexCell(ecto.Cell):
                 log.ODM_WARNING('Found a valid ODM Texture file in: %s'
                                 % odm_textured_model_obj)
 
-        outputs.reconstruction = reconstruction
-
-        if args.time:
-            system.benchmark(start_time, tree.benchmarking, 'Texturing')
-
-        log.ODM_INFO('Running ODM Texturing Cell - Finished')
-        return ecto.OK if args.end_with != 'mvs_texturing' else ecto.QUIT

scripts/odm_app.py

@@ -1,174 +1,104 @@
-import ecto
 import os
 
 from opendm import context
 from opendm import types
 from opendm import io
 from opendm import system
+from opendm import log
 
-from dataset import ODMLoadDatasetCell
+from dataset import ODMLoadDatasetStage
-from run_opensfm import ODMOpenSfMCell
+from run_opensfm import ODMOpenSfMStage
-from mve import ODMMveCell
+from mve import ODMMveStage
-from odm_slam import ODMSlamCell
+from odm_slam import ODMSlamStage
-from odm_meshing import ODMeshingCell
+from odm_meshing import ODMeshingStage
-from mvstex import ODMMvsTexCell
+from mvstex import ODMMvsTexStage
-from odm_georeferencing import ODMGeoreferencingCell
+from odm_georeferencing import ODMGeoreferencingStage
-from odm_orthophoto import ODMOrthoPhotoCell
+from odm_orthophoto import ODMOrthoPhotoStage
-from odm_dem import ODMDEMCell
+from odm_dem import ODMDEMStage
 from odm_filterpoints import ODMFilterPoints
 
 
-class ODMApp(ecto.BlackBox):
+class ODMApp:
-    """ODMApp - a class for ODM Activities
+    def __init__(self, args):
-    """
-
-    def __init__(self, *args, **kwargs):
-        ecto.BlackBox.__init__(self, *args, **kwargs)
-        self.tree = None
-
-    @staticmethod
-    def declare_direct_params(p):
-        p.declare("args", "The application arguments.", {})
-
-    @staticmethod
-    def declare_cells(p):
         """
-        Implement the virtual function from the base class
+        Initializes the application and defines the ODM application pipeline stages
-        Only cells from which something is forwarded have to be declared
         """
-        cells = {'args': ecto.Constant(value=p.args),
-                 'dataset': ODMLoadDatasetCell(verbose=p.args.verbose,
-                                               proj=p.args.proj),
-                 'opensfm': ODMOpenSfMCell(use_exif_size=False,
-                                           feature_process_size=p.args.resize_to,
-                                           feature_min_frames=p.args.min_num_features,
-                                           processes=p.args.max_concurrency,
-                                           matching_gps_neighbors=p.args.matcher_neighbors,
-                                           matching_gps_distance=p.args.matcher_distance,
-                                           fixed_camera_params=p.args.use_fixed_camera_params,
-                                           hybrid_bundle_adjustment=p.args.use_hybrid_bundle_adjustment),
-                 'slam': ODMSlamCell(),
         
-                 'mve': ODMMveCell(),
+        dataset = ODMLoadDatasetStage('dataset', args, 
+                                          verbose=args.verbose,
+                                          proj=args.proj)
+        opensfm = ODMOpenSfMStage('opensfm', args,
+                                    use_exif_size=False,
+                                    feature_process_size=args.resize_to,
+                                    feature_min_frames=args.min_num_features,
+                                    processes=args.max_concurrency,
+                                    matching_gps_neighbors=args.matcher_neighbors,
+                                    matching_gps_distance=args.matcher_distance,
+                                    fixed_camera_params=args.use_fixed_camera_params,
+                                    hybrid_bundle_adjustment=args.use_hybrid_bundle_adjustment)
+        slam = ODMSlamStage('slam', args)
+        mve = ODMMveStage('mve', args)
+        filterpoints = ODMFilterPoints('odm_filterpoints', args)
+        meshing = ODMeshingStage('odm_meshing', args,
+                                    max_vertex=args.mesh_size,
+                                    oct_tree=args.mesh_octree_depth,
+                                    samples=args.mesh_samples,
+                                    point_weight=args.mesh_point_weight,
+                                    max_concurrency=args.max_concurrency,
+                                    verbose=args.verbose)
+        texturing = ODMMvsTexStage('mvs_texturing', args,
+                                    data_term=args.texturing_data_term,
+                                    outlier_rem_type=args.texturing_outlier_removal_type,
+                                    skip_vis_test=args.texturing_skip_visibility_test,
+                                    skip_glob_seam_leveling=args.texturing_skip_global_seam_leveling,
+                                    skip_loc_seam_leveling=args.texturing_skip_local_seam_leveling,
+                                    skip_hole_fill=args.texturing_skip_hole_filling,
+                                    keep_unseen_faces=args.texturing_keep_unseen_faces,
+                                    tone_mapping=args.texturing_tone_mapping)
+        georeferencing = ODMGeoreferencingStage('odm_georeferencing', args,
+                                                    gcp_file=args.gcp,
+                                                    use_exif=args.use_exif,
+                                                    verbose=args.verbose)
+        dem = ODMDEMStage('odm_dem', args,
+                            max_concurrency=args.max_concurrency,
+                            verbose=args.verbose)
+        orthophoto = ODMOrthoPhotoStage('odm_orthophoto', args,
+                                            resolution=args.orthophoto_resolution,
+                                            no_tiled=args.orthophoto_no_tiled,
+                                            compress=args.orthophoto_compression,
+                                            bigtiff=args.orthophoto_bigtiff,
+                                            build_overviews=args.build_overviews,
+                                            max_concurrency=args.max_concurrency,
+                                            verbose=args.verbose)
         
-                 'filterpoints': ODMFilterPoints(),
+        if not args.video:
+            # Normal pipeline
+            self.first_stage = dataset
 
-                 'meshing': ODMeshingCell(max_vertex=p.args.mesh_size,
+            dataset.connect(opensfm)
-                                          oct_tree=p.args.mesh_octree_depth,
-                                          samples=p.args.mesh_samples,
-                                          point_weight=p.args.mesh_point_weight,
-                                          max_concurrency=p.args.max_concurrency,
-                                          verbose=p.args.verbose),
-                 'texturing': ODMMvsTexCell(data_term=p.args.texturing_data_term,
-                                            outlier_rem_type=p.args.texturing_outlier_removal_type,
-                                            skip_vis_test=p.args.texturing_skip_visibility_test,
-                                            skip_glob_seam_leveling=p.args.texturing_skip_global_seam_leveling,
-                                            skip_loc_seam_leveling=p.args.texturing_skip_local_seam_leveling,
-                                            skip_hole_fill=p.args.texturing_skip_hole_filling,
-                                            keep_unseen_faces=p.args.texturing_keep_unseen_faces,
-                                            tone_mapping=p.args.texturing_tone_mapping),
-                 'georeferencing': ODMGeoreferencingCell(gcp_file=p.args.gcp,
-                                                         use_exif=p.args.use_exif,
-                                                         verbose=p.args.verbose),
-                 'dem': ODMDEMCell(max_concurrency=p.args.max_concurrency,
-                                   verbose=p.args.verbose),
-                 'orthophoto': ODMOrthoPhotoCell(resolution=p.args.orthophoto_resolution,
-                                                 no_tiled=p.args.orthophoto_no_tiled,
-                                                 compress=p.args.orthophoto_compression,
-                                                 bigtiff=p.args.orthophoto_bigtiff,
-                                                 build_overviews=p.args.build_overviews,
-                                                 max_concurrency=p.args.max_concurrency,
-                                                 verbose=p.args.verbose)
-                 }
 
-        return cells
+            if args.use_opensfm_dense or args.fast_orthophoto:
+                opensfm.connect(filterpoints)
+            else:
+                opensfm.connect(mve) \
+                       .connect(filterpoints)
             
-    def configure(self, p, _i, _o):
+            filterpoints \
-        tree = types.ODM_Tree(p.args.project_path, p.args.images, p.args.gcp)
+                .connect(meshing) \
-        self.tree = ecto.Constant(value=tree)
+                .connect(texturing) \
+                .connect(georeferencing) \
+                .connect(dem) \
+                .connect(orthophoto)
                 
-        # TODO(dakota) put this somewhere better maybe
-        if p.args.time and io.file_exists(tree.benchmarking):
-            # Delete the previously made file
-            os.remove(tree.benchmarking)
-            with open(tree.benchmarking, 'a') as b:
-                b.write('ODM Benchmarking file created %s\nNumber of Cores: %s\n\n' % (system.now(), context.num_cores))
-
-    def connections(self, p):
-        if p.args.video:
-            return self.slam_connections(p)
-
-        # load the dataset
-        connections = [self.tree[:] >> self.dataset['tree'],
-                       self.args[:] >> self.dataset['args']]
-
-        # run opensfm with images from load dataset
-        connections += [self.tree[:] >> self.opensfm['tree'],
-                        self.args[:] >> self.opensfm['args'],
-                        self.dataset['reconstruction'] >> self.opensfm['reconstruction']]
-
-        if p.args.use_opensfm_dense or p.args.fast_orthophoto:
-            # filter points from opensfm point cloud
-            connections += [self.tree[:] >> self.filterpoints['tree'],
-                            self.args[:] >> self.filterpoints['args'],
-                            self.opensfm['reconstruction'] >> self.filterpoints['reconstruction']]
         else:
-            # run mve
+            # SLAM pipeline
-            connections += [self.tree[:] >> self.mve['tree'],
+            # TODO: this is broken and needs work
-                            self.args[:] >> self.mve['args'],
+            log.ODM_WARNING("SLAM module is currently broken. We could use some help fixing this. If you know Python, get in touch at https://community.opendronemap.org.")
-                            self.opensfm['reconstruction'] >> self.mve['reconstruction']]
+            self.first_stage = slam
 
-            # filter points from mve point cloud
+            slam.connect(mve) \
-            connections += [self.tree[:] >> self.filterpoints['tree'],
+                .connect(meshing) \
-                            self.args[:] >> self.filterpoints['args'],
+                .connect(texturing)
-                            self.mve['reconstruction'] >> self.filterpoints['reconstruction']]
 
-        # create mesh
+    def execute(self):
-        connections += [self.tree[:] >> self.meshing['tree'],
+        self.first_stage.run()
-                        self.args[:] >> self.meshing['args'],
-                        self.filterpoints['reconstruction'] >> self.meshing['reconstruction']]
-
-        # create odm texture
-        connections += [self.tree[:] >> self.texturing['tree'],
-                        self.args[:] >> self.texturing['args'],
-                        self.meshing['reconstruction'] >> self.texturing['reconstruction']]
-
-        # create odm georeference
-        connections += [self.tree[:] >> self.georeferencing['tree'],
-                        self.args[:] >> self.georeferencing['args'],
-                        self.texturing['reconstruction'] >> self.georeferencing['reconstruction']]
-
-        # create odm dem
-        connections += [self.tree[:] >> self.dem['tree'],
-                        self.args[:] >> self.dem['args'],
-                        self.georeferencing['reconstruction'] >> self.dem['reconstruction']]
-
-        # create odm orthophoto
-        connections += [self.tree[:] >> self.orthophoto['tree'],
-                        self.args[:] >> self.orthophoto['args'],
-                        self.georeferencing['reconstruction'] >> self.orthophoto['reconstruction']]
-        return connections
-
-    def slam_connections(self, p):
-        """Get connections used when running from video instead of images."""
-        connections = []
-
-        # run slam cell
-        connections += [self.tree[:] >> self.slam['tree'],
-                        self.args[:] >> self.slam['args']]
-
-        connections += [self.tree[:] >> self.mve['tree'],
-                        self.args[:] >> self.mve['args'],
-                        self.slam['reconstruction'] >> self.mve['reconstruction']]
-
-        # create odm mesh
-        connections += [self.tree[:] >> self.meshing['tree'],
-                        self.args[:] >> self.meshing['args'],
-                        self.mve['reconstruction'] >> self.meshing['reconstruction']]
-
-        # create odm texture
-        connections += [self.tree[:] >> self.texturing['tree'],
-                        self.args[:] >> self.texturing['args'],
-                        self.meshing['reconstruction'] >> self.texturing['reconstruction']]
-
-        return connections

scripts/odm_dem.py

@@ -1,4 +1,4 @@
-import ecto, os, json
+import os, json
 from shutil import copyfile
 
 from opendm import io
@@ -10,34 +10,11 @@ from opendm import gsd
 from opendm.dem import commands
 from opendm.cropper import Cropper
 
-class ODMDEMCell(ecto.Cell):
+class ODMDEMStage(types.ODM_Stage):
-    def declare_params(self, params):
+    def process(self, args, outputs):
-        params.declare("verbose", 'print additional messages to console', False)
+        tree = outputs['tree']
-        params.declare("max_concurrency", "Number of threads", context.num_cores)
-
-    def declare_io(self, params, inputs, outputs):
-        inputs.declare("tree", "Struct with paths", [])
-        inputs.declare("args", "The application arguments.", {})
-        inputs.declare("reconstruction", "list of ODMReconstructions", [])
-
-    def process(self, inputs, outputs):
-        # Benchmarking
-        start_time = system.now_raw()
-
-        log.ODM_INFO('Running ODM DEM Cell')
-
-        # get inputs
-        args = self.inputs.args
-        tree = self.inputs.tree
         las_model_found = io.file_exists(tree.odm_georeferencing_model_laz)
 
-        # check if we rerun cell or not
-        rerun_cell = (args.rerun is not None and
-                      args.rerun == 'odm_dem') or \
-                     (args.rerun_all) or \
-                     (args.rerun_from is not None and
-                      'odm_dem' in args.rerun_from)
-
         log.ODM_INFO('Classify: ' + str(args.pc_classify))
         log.ODM_INFO('Create DSM: ' + str(args.dsm))
         log.ODM_INFO('Create DTM: ' + str(args.dtm))
@@ -51,7 +28,7 @@ class ODMDEMCell(ecto.Cell):
         if args.pc_classify and las_model_found:
             pc_classify_marker = os.path.join(odm_dem_root, 'pc_classify_done.txt')
 
-            if not io.file_exists(pc_classify_marker) or rerun_cell:
+            if not io.file_exists(pc_classify_marker) or self.rerun():
                 log.ODM_INFO("Classifying {} using Simple Morphological Filter".format(tree.odm_georeferencing_model_laz))
                 commands.classify(tree.odm_georeferencing_model_laz,
                                   args.smrf_scalar, 
@@ -75,7 +52,7 @@ class ODMDEMCell(ecto.Cell):
 
             if (args.dtm and not io.file_exists(dtm_output_filename)) or \
                 (args.dsm and not io.file_exists(dsm_output_filename)) or \
-                rerun_cell:
+                self.rerun():
 
                 products = []
                 if args.dsm: products.append('dsm')
@@ -108,15 +85,9 @@ class ODMDEMCell(ecto.Cell):
                                 'COMPRESS': 'LZW',
                                 'BLOCKXSIZE': 512,
                                 'BLOCKYSIZE': 512,
-                                'NUM_THREADS': self.params.max_concurrency
+                                'NUM_THREADS': self.params.get('max_concurrency')
                             })
             else:
                 log.ODM_WARNING('Found existing outputs in: %s' % odm_dem_root)
         else:
             log.ODM_WARNING('DEM will not be generated')
-
-        if args.time:
-            system.benchmark(start_time, tree.benchmarking, 'Dem')
-
-        log.ODM_INFO('Running ODM DEM Cell - Finished')
-        return ecto.OK if args.end_with != 'odm_dem' else ecto.QUIT

scripts/odm_filterpoints.py

@@ -1,39 +1,21 @@
-import ecto, os
+import os
 
 from opendm import log
 from opendm import io
 from opendm import system
 from opendm import context
 from opendm import point_cloud
+from opendm import types
 
-class ODMFilterPoints(ecto.Cell):
+class ODMFilterPoints(types.ODM_Stage):
-    def declare_io(self, params, inputs, outputs):
+    def process(self, args, outputs):
-        inputs.declare("tree", "Struct with paths", [])
+        tree = outputs['tree']
-        inputs.declare("args", "The application arguments.", {})
+        reconstruction = outputs['reconstruction']
-        inputs.declare("reconstruction", "ODMReconstruction", [])
-        outputs.declare("reconstruction", "list of ODMReconstructions", [])
 
-    def process(self, inputs, outputs):
-        # Benchmarking
-        start_time = system.now_raw()
-
-        log.ODM_INFO('Running ODM FilterPoints Cell')
-
-        # get inputs
-        tree = inputs.tree
-        args = inputs.args
-        reconstruction = inputs.reconstruction
-
-        # check if we rerun cell or not
-        rerun_cell = (args.rerun is not None and
-                      args.rerun == 'odm_filterpoints') or \
-                     (args.rerun_all) or \
-                     (args.rerun_from is not None and
-                      'odm_filterpoints' in args.rerun_from)
         if not os.path.exists(tree.odm_filterpoints): system.mkdir_p(tree.odm_filterpoints)
 
         # check if reconstruction was done before
-        if not io.file_exists(tree.filtered_point_cloud) or rerun_cell:
+        if not io.file_exists(tree.filtered_point_cloud) or self.rerun():
             if args.fast_orthophoto:
                 inputPointCloud = os.path.join(tree.opensfm, 'reconstruction.ply')
             elif args.use_opensfm_dense:
@@ -49,11 +31,3 @@ class ODMFilterPoints(ecto.Cell):
         else:
             log.ODM_WARNING('Found a valid point cloud file in: %s' %
                             tree.filtered_point_cloud)
-
-        outputs.reconstruction = reconstruction
-
-        if args.time:
-            system.benchmark(start_time, tree.benchmarking, 'MVE')
-
-        log.ODM_INFO('Running ODM FilterPoints Cell - Finished')
-        return ecto.OK if args.end_with != 'odm_filterpoints' else ecto.QUIT

scripts/odm_georeferencing.py

@@ -1,4 +1,3 @@
-import ecto
 import os
 import struct
 import pipes
@@ -12,45 +11,17 @@ from opendm.cropper import Cropper
 from opendm import point_cloud
 
 
-class ODMGeoreferencingCell(ecto.Cell):
+class ODMGeoreferencingStage(types.ODM_Stage):
-    def declare_params(self, params):
+    def process(self, args, outputs):
-        params.declare("gcp_file", 'path to the file containing the ground control '
+        tree = outputs['tree']
-                            'points used for georeferencing.The file needs to '
+        reconstruction = outputs['reconstruction']
-                            'be on the following line format: \neasting '
-                            'northing height pixelrow pixelcol imagename', 'gcp_list.txt')
-        params.declare("use_exif", 'use exif', False)
-        params.declare("verbose", 'print additional messages to console', False)
 
-    def declare_io(self, params, inputs, outputs):
-        inputs.declare("tree", "Struct with paths", [])
-        inputs.declare("args", "The application arguments.", {})
-        inputs.declare("reconstruction", "list of ODMReconstructions", [])
-        outputs.declare("reconstruction", "list of ODMReconstructions", [])
-
-    def process(self, inputs, outputs):
-
-        # Benchmarking
-        start_time = system.now_raw()
-
-        log.ODM_INFO('Running ODM Georeferencing Cell')
-
-        # get inputs
-        args = inputs.args
-        tree = inputs.tree
-        reconstruction = inputs.reconstruction
         gcpfile = tree.odm_georeferencing_gcp
         doPointCloudGeo = True
         transformPointCloud = True
-        verbose = '-verbose' if self.params.verbose else ''
+        verbose = '-verbose' if self.params.get('verbose') else ''
         geo_ref = reconstruction.georef
 
-        # check if we rerun cell or not
-        rerun_cell = (args.rerun is not None and
-                      args.rerun == 'odm_georeferencing') or \
-                     (args.rerun_all) or \
-                     (args.rerun_from is not None and
-                      'odm_georeferencing' in args.rerun_from)
-
         runs = [{
             'georeferencing_dir': tree.odm_georeferencing,
             'texturing_dir': tree.odm_texturing,
@@ -78,7 +49,7 @@ class ODMGeoreferencingCell(ecto.Cell):
             odm_georeferencing_model_txt_geo_file = os.path.join(r['georeferencing_dir'], tree.odm_georeferencing_model_txt_geo)
 
             if not io.file_exists(odm_georeferencing_model_obj_geo) or \
-               not io.file_exists(tree.odm_georeferencing_model_laz) or rerun_cell:
+               not io.file_exists(tree.odm_georeferencing_model_laz) or self.rerun():
 
                 # odm_georeference definitions
                 kwargs = {
@@ -111,7 +82,7 @@ class ODMGeoreferencingCell(ecto.Cell):
  
                 # Check to see if the GCP file exists
 
-                if not self.params.use_exif and (self.params.gcp_file or tree.odm_georeferencing_gcp):
+                if not self.params.get('use_exif') and (self.params.get('gcp_file') or tree.odm_georeferencing_gcp):
                    log.ODM_INFO('Found %s' % gcpfile)
                    try:
                        system.run('{bin}/odm_georef -bundleFile {bundle} -imagesPath {imgs} -imagesListPath {imgs_list} '
@@ -121,7 +92,7 @@ class ODMGeoreferencingCell(ecto.Cell):
                                   '-outputCoordFile {coords}'.format(**kwargs))
                    except Exception:
                        log.ODM_EXCEPTION('Georeferencing failed. ')
-                       return ecto.QUIT
+                       exit(1)
                 elif io.file_exists(tree.opensfm_transformation) and io.file_exists(tree.odm_georeferencing_coords):
                     log.ODM_INFO('Running georeferencing with OpenSfM transformation matrix')
                     system.run('{bin}/odm_georef -bundleFile {bundle} -inputTransformFile {input_trans_file} -inputCoordFile {coords} '
@@ -188,11 +159,3 @@ class ODMGeoreferencingCell(ecto.Cell):
             else:
                 log.ODM_WARNING('Found a valid georeferenced model in: %s'
                                 % tree.odm_georeferencing_model_laz)
-
-        outputs.reconstruction = reconstruction
-
-        if args.time:
-            system.benchmark(start_time, tree.benchmarking, 'Georeferencing')
-
-        log.ODM_INFO('Running ODM Georeferencing Cell - Finished')
-        return ecto.OK if args.end_with != 'odm_georeferencing' else ecto.QUIT

scripts/odm_meshing.py

@@ -1,4 +1,4 @@
-import ecto, os, math
+import os, math
 
 from opendm import log
 from opendm import io
@@ -6,62 +6,29 @@ from opendm import system
 from opendm import context
 from opendm import mesh
 from opendm import gsd
+from opendm import types
 
-
+class ODMeshingStage(types.ODM_Stage):
-class ODMeshingCell(ecto.Cell):
+    def process(self, args, outputs):
-    def declare_params(self, params):
+        tree = outputs['tree']
-        params.declare("max_vertex", 'The maximum vertex count of the output '
+        reconstruction = outputs['reconstruction']
-                                     'mesh', 100000)
-        params.declare("oct_tree", 'Oct-tree depth used in the mesh reconstruction, '
-                                   'increase to get more vertices, recommended '
-                                   'values are 8-12', 9)
-        params.declare("samples", 'Number of points per octree node, recommended '
-                                  'value: 1.0', 1)
-        params.declare("point_weight", "specifies the importance that interpolation of the point samples is given in the formulation of the screened Poisson equation.", 4)
-        params.declare("max_concurrency", 'max threads', context.num_cores)
-        params.declare("verbose", 'print additional messages to console', False)
-
-    def declare_io(self, params, inputs, outputs):
-        inputs.declare("tree", "Struct with paths", [])
-        inputs.declare("args", "The application arguments.", {})
-        inputs.declare("reconstruction", "Clusters output. list of ODMReconstructions", [])
-        outputs.declare("reconstruction", "Clusters output. list of ODMReconstructions", [])
-
-    def process(self, inputs, outputs):
-
-        # Benchmarking
-        start_time = system.now_raw()
-
-        log.ODM_INFO('Running ODM Meshing Cell')
-
-        # get inputs
-        args = inputs.args
-        tree = inputs.tree
-        reconstruction = inputs.reconstruction
 
         # define paths and create working directories
         system.mkdir_p(tree.odm_meshing)
 
-        # check if we rerun cell or not
-        rerun_cell = (args.rerun is not None and
-                      args.rerun == 'odm_meshing') or \
-                     (args.rerun_all) or \
-                     (args.rerun_from is not None and
-                      'odm_meshing' in args.rerun_from)
-
         # Create full 3D model unless --skip-3dmodel is set
         if not args.skip_3dmodel:
-          if not io.file_exists(tree.odm_mesh) or rerun_cell:
+          if not io.file_exists(tree.odm_mesh) or self.rerun():
               log.ODM_DEBUG('Writing ODM Mesh file in: %s' % tree.odm_mesh)
 
               mesh.screened_poisson_reconstruction(tree.filtered_point_cloud,
                 tree.odm_mesh,
-                depth=self.params.oct_tree,
+                depth=self.params.get('oct_tree'),
-                samples=self.params.samples,
+                samples=self.params.get('samples'),
-                maxVertexCount=self.params.max_vertex,
+                maxVertexCount=self.params.get('max_vertex'),
-                pointWeight=self.params.point_weight,
+                pointWeight=self.params.get('point_weight'),
-                threads=self.params.max_concurrency,
+                threads=self.params.get('max_concurrency'),
-                verbose=self.params.verbose)
+                verbose=self.params.get('verbose'))
 
           else:
               log.ODM_WARNING('Found a valid ODM Mesh file in: %s' %
@@ -70,7 +37,7 @@ class ODMeshingCell(ecto.Cell):
         # Always generate a 2.5D mesh
         # unless --use-3dmesh is set.
         if not args.use_3dmesh:
-          if not io.file_exists(tree.odm_25dmesh) or rerun_cell:
+          if not io.file_exists(tree.odm_25dmesh) or self.rerun():
 
               log.ODM_DEBUG('Writing ODM 2.5D Mesh file in: %s' % tree.odm_25dmesh)
               ortho_resolution = gsd.cap_resolution(args.orthophoto_resolution, tree.opensfm_reconstruction, ignore_gsd=args.ignore_gsd) / 100.0 
@@ -94,20 +61,13 @@ class ODMeshingCell(ecto.Cell):
               mesh.create_25dmesh(tree.filtered_point_cloud, tree.odm_25dmesh,
                     dsm_radius=dsm_radius,
                     dsm_resolution=dsm_resolution, 
-                    depth=self.params.oct_tree,
+                    depth=self.params.get('oct_tree'),
-                    maxVertexCount=self.params.max_vertex,
+                    maxVertexCount=self.params.get('max_vertex'),
-                    samples=self.params.samples,
+                    samples=self.params.get('samples'),
-                    verbose=self.params.verbose,
+                    verbose=self.params.get('verbose'),
                     available_cores=args.max_concurrency,
                     method='poisson' if args.fast_orthophoto else 'gridded')
           else:
               log.ODM_WARNING('Found a valid ODM 2.5D Mesh file in: %s' %
                               tree.odm_25dmesh)
 
-        outputs.reconstruction = reconstruction
-
-        if args.time:
-            system.benchmark(start_time, tree.benchmarking, 'Meshing')
-
-        log.ODM_INFO('Running ODM Meshing Cell - Finished')
-        return ecto.OK if args.end_with != 'odm_meshing' else ecto.QUIT

scripts/odm_orthophoto.py

@@ -1,4 +1,4 @@
-import ecto, os
+import os
 
 from opendm import io
 from opendm import log
@@ -10,45 +10,16 @@ from opendm.concurrency import get_max_memory
 from opendm.cropper import Cropper
 
 
-class ODMOrthoPhotoCell(ecto.Cell):
+class ODMOrthoPhotoStage(types.ODM_Stage):
-    def declare_params(self, params):
+    def process(self, args, outputs):
-        params.declare("resolution", 'Orthophoto resolution in cm / pixel', 5)
+        tree = outputs['tree']
-        params.declare("no_tiled", 'Do not tile tiff', False)
+        reconstruction = outputs['reconstruction']
-        params.declare("compress", 'Compression type', 'DEFLATE')
+        verbose = '-verbose' if self.params.get('verbose') else ''
-        params.declare("bigtiff", 'Make BigTIFF orthophoto', 'IF_SAFER')
-        params.declare("build_overviews", 'Build overviews', False)
-        params.declare("verbose", 'print additional messages to console', False)
-        params.declare("max_concurrency", "number of threads", context.num_cores)
-
-    def declare_io(self, params, inputs, outputs):
-        inputs.declare("tree", "Struct with paths", [])
-        inputs.declare("args", "The application arguments.", {})
-        inputs.declare("reconstruction", "list of ODMReconstructions", [])
-
-    def process(self, inputs, outputs):
-
-        # Benchmarking
-        start_time = system.now_raw()
-
-        log.ODM_INFO('Running ODM Orthophoto Cell')
-
-        # get inputs
-        args = self.inputs.args
-        tree = self.inputs.tree
-        reconstruction = inputs.reconstruction
-        verbose = '-verbose' if self.params.verbose else ''
 
         # define paths and create working directories
         system.mkdir_p(tree.odm_orthophoto)
 
-        # check if we rerun cell or not
+        if not io.file_exists(tree.odm_orthophoto_file) or self.rerun():
-        rerun_cell = (args.rerun is not None and
-                      args.rerun == 'odm_orthophoto') or \
-                     (args.rerun_all) or \
-                     (args.rerun_from is not None and
-                      'odm_orthophoto' in args.rerun_from)
-
-        if not io.file_exists(tree.odm_orthophoto_file) or rerun_cell:
 
             # odm_orthophoto definitions
             kwargs = {
@@ -56,7 +27,7 @@ class ODMOrthoPhotoCell(ecto.Cell):
                 'log': tree.odm_orthophoto_log,
                 'ortho': tree.odm_orthophoto_file,
                 'corners': tree.odm_orthophoto_corners,
-                'res': 1.0 / (gsd.cap_resolution(self.params.resolution, tree.opensfm_reconstruction, ignore_gsd=args.ignore_gsd) / 100.0),
+                'res': 1.0 / (gsd.cap_resolution(self.params.get('resolution'), tree.opensfm_reconstruction, ignore_gsd=args.ignore_gsd) / 100.0),
                 'verbose': verbose
             }
 
@@ -118,17 +89,17 @@ class ODMOrthoPhotoCell(ecto.Cell):
                     'uly': uly,
                     'lrx': lrx,
                     'lry': lry,
-                    'tiled': '' if self.params.no_tiled else '-co TILED=yes ',
+                    'tiled': '' if self.params.get('no_tiled') else '-co TILED=yes ',
-                    'compress': self.params.compress,
+                    'compress': self.params.get('compress'),
-                    'predictor': '-co PREDICTOR=2 ' if self.params.compress in
+                    'predictor': '-co PREDICTOR=2 ' if self.params.get('compress') in
                                                        ['LZW', 'DEFLATE'] else '',
                     'proj': georef.projection.srs,
-                    'bigtiff': self.params.bigtiff,
+                    'bigtiff': self.params.get('bigtiff'),
                     'png': tree.odm_orthophoto_file,
                     'tiff': tree.odm_orthophoto_tif,
                     'log': tree.odm_orthophoto_tif_log,
                     'max_memory': get_max_memory(),
-                    'threads': self.params.max_concurrency
+                    'threads': self.params.get('max_concurrency')
                 }
 
                 system.run('gdal_translate -a_ullr {ulx} {uly} {lrx} {lry} '
@@ -146,16 +117,16 @@ class ODMOrthoPhotoCell(ecto.Cell):
                 if args.crop > 0:
                     shapefile_path = os.path.join(tree.odm_georeferencing, 'odm_georeferenced_model.bounds.shp')
                     Cropper.crop(shapefile_path, tree.odm_orthophoto_tif, {
-                            'TILED': 'NO' if self.params.no_tiled else 'YES',
+                            'TILED': 'NO' if self.params.get('no_tiled') else 'YES',
-                            'COMPRESS': self.params.compress,
+                            'COMPRESS': self.params.get('compress'),
-                            'PREDICTOR': '2' if self.params.compress in ['LZW', 'DEFLATE'] else '1',
+                            'PREDICTOR': '2' if self.params.get('compress') in ['LZW', 'DEFLATE'] else '1',
-                            'BIGTIFF': self.params.bigtiff,
+                            'BIGTIFF': self.params.get('bigtiff'),
                             'BLOCKXSIZE': 512,
                             'BLOCKYSIZE': 512,
-                            'NUM_THREADS': self.params.max_concurrency
+                            'NUM_THREADS': self.params.get('max_concurrency')
                         })
 
-                if self.params.build_overviews:
+                if self.params.get('build_overviews'):
                     log.ODM_DEBUG("Building Overviews")
                     kwargs = {
                         'orthophoto': tree.odm_orthophoto_tif,
@@ -174,9 +145,3 @@ class ODMOrthoPhotoCell(ecto.Cell):
 
         else:
             log.ODM_WARNING('Found a valid orthophoto in: %s' % tree.odm_orthophoto_file)
-
-        if args.time:
-            system.benchmark(start_time, tree.benchmarking, 'Orthophoto')
-
-        log.ODM_INFO('Running ODM OrthoPhoto Cell - Finished')
-        return ecto.OK if args.end_with != 'odm_orthophoto' else ecto.QUIT

scripts/odm_slam.py

@@ -2,40 +2,23 @@
 
 import os
 
-import ecto
-
 from opendm import log
 from opendm import io
 from opendm import system
 from opendm import context
+from opendm import types
 
-
+class ODMSlamStage(types.ODM_Stage):
-class ODMSlamCell(ecto.Cell):
     """Run odm_slam on a video and export to opensfm format."""
 
-    def declare_params(self, params):
+    def process(self, args, outputs):
-        """Cell parameters."""
+        tree = outputs['tree']
-        pass
-
-    def declare_io(self, params, inputs, outputs):
-        """Cell inputs and outputs."""
-        inputs.declare("tree", "Struct with paths", [])
-        inputs.declare("args", "The application arguments.", {})
-        outputs.declare("reconstruction", "list of ODMReconstructions", [])
-
-    def process(self, inputs, outputs):
-        """Run the cell."""
-        log.ODM_INFO('Running OMD Slam Cell')
-
-        # get inputs
-        tree = self.inputs.tree
-        args = self.inputs.args
         video = os.path.join(tree.root_path, args.video)
         slam_config = os.path.join(tree.root_path, args.slam_config)
 
         if not video:
             log.ODM_ERROR('No video provided')
-            return ecto.QUIT
+            exit(1)
 
         # create working directories
         system.mkdir_p(tree.opensfm)
@@ -46,11 +29,8 @@ class ODMSlamCell(ecto.Cell):
         trajectory = os.path.join(tree.opensfm, 'KeyFrameTrajectory.txt')
         map_points = os.path.join(tree.opensfm, 'MapPoints.txt')
 
-        # check if we rerun cell or not
-        rerun_cell = args.rerun == 'slam'
-
         # check if slam was run before
-        if not io.file_exists(trajectory) or rerun_cell:
+        if not io.file_exists(trajectory) or self.rerun():
             # run slam binary
             system.run(' '.join([
                 'cd {} &&'.format(tree.opensfm),
@@ -64,7 +44,7 @@ class ODMSlamCell(ecto.Cell):
                 trajectory))
 
         # check if trajectory was exported to opensfm before
-        if not io.file_exists(tree.opensfm_reconstruction) or rerun_cell:
+        if not io.file_exists(tree.opensfm_reconstruction) or self.rerun():
             # convert slam to opensfm
             system.run(' '.join([
                 'cd {} &&'.format(tree.opensfm),
@@ -85,7 +65,7 @@ class ODMSlamCell(ecto.Cell):
                 tree.opensfm_reconstruction))
 
         # check if reconstruction was exported to bundler before
-        if not io.file_exists(tree.opensfm_bundle_list) or rerun_cell:
+        if not io.file_exists(tree.opensfm_bundle_list) or self.rerun():
             # convert back to bundler's format
             system.run(
                 'PYTHONPATH={} {}/bin/export_bundler {}'.format(
@@ -95,5 +75,3 @@ class ODMSlamCell(ecto.Cell):
                 'Found a valid Bundler file in: {}'.format(
                     tree.opensfm_reconstruction))
 
-        log.ODM_INFO('Running OMD Slam Cell - Finished')
-        return ecto.OK if args.end_with != 'odm_slam' else ecto.QUIT

scripts/run_opensfm.py

@@ -1,4 +1,3 @@
-import ecto
 import sys
 import os
 
@@ -8,51 +7,21 @@ from opendm import system
 from opendm import context
 from opendm import gsd
 from opendm import point_cloud
+from opendm import types
 
-class ODMOpenSfMCell(ecto.Cell):
+class ODMOpenSfMStage(types.ODM_Stage):
-    def declare_params(self, params):
+    def process(self, args, outputs):
-        params.declare("use_exif_size", "The application arguments.", False)
+        tree = outputs['tree']
-        params.declare("feature_process_size", "The application arguments.", 2400)
+        reconstruction = outputs['reconstruction']
-        params.declare("feature_min_frames", "The application arguments.", 4000)
-        params.declare("processes", "The application arguments.", context.num_cores)
-        params.declare("matching_gps_neighbors", "The application arguments.", 8)
-        params.declare("matching_gps_distance", "The application arguments.", 0)
-        params.declare("fixed_camera_params", "Optimize internal camera parameters", True)
-        params.declare("hybrid_bundle_adjustment", "Use local + global bundle adjustment", False)
-
-    def declare_io(self, params, inputs, outputs):
-        inputs.declare("tree", "Struct with paths", [])
-        inputs.declare("args", "The application arguments.", {})
-        inputs.declare("reconstruction", "ODMReconstruction", [])
-        outputs.declare("reconstruction", "list of ODMReconstructions", [])
-
-    def process(self, inputs, outputs):
-
-        # Benchmarking
-        start_time = system.now_raw()
-
-        log.ODM_INFO('Running ODM OpenSfM Cell')
-
-        # get inputs
-        tree = inputs.tree
-        args = inputs.args
-        reconstruction = inputs.reconstruction
         photos = reconstruction.photos
 
         if not photos:
             log.ODM_ERROR('Not enough photos in photos array to start OpenSfM')
-            return ecto.QUIT
+            exit(1)
 
         # create working directories
         system.mkdir_p(tree.opensfm)
 
-        # check if we rerun cell or not
-        rerun_cell = (args.rerun is not None and
-                      args.rerun == 'opensfm') or \
-                     (args.rerun_all) or \
-                     (args.rerun_from is not None and
-                      'opensfm' in args.rerun_from)
-
         if args.fast_orthophoto:
             output_file = io.join_paths(tree.opensfm, 'reconstruction.ply')
         elif args.use_opensfm_dense:
@@ -61,7 +30,7 @@ class ODMOpenSfMCell(ecto.Cell):
             output_file = tree.opensfm_reconstruction
 
         # check if reconstruction was done before
-        if not io.file_exists(output_file) or rerun_cell:
+        if not io.file_exists(output_file) or self.rerun():
             # create file list
             list_path = io.join_paths(tree.opensfm, 'image_list.txt')
             has_alt = True
@@ -73,16 +42,16 @@ class ODMOpenSfMCell(ecto.Cell):
 
             # create config file for OpenSfM
             config = [
-                "use_exif_size: %s" % ('no' if not self.params.use_exif_size else 'yes'),
+                "use_exif_size: %s" % ('no' if not self.params.get('use_exif_size') else 'yes'),
-                "feature_process_size: %s" % self.params.feature_process_size,
+                "feature_process_size: %s" % self.params.get('feature_process_size'),
-                "feature_min_frames: %s" % self.params.feature_min_frames,
+                "feature_min_frames: %s" % self.params.get('feature_min_frames'),
-                "processes: %s" % self.params.processes,
+                "processes: %s" % self.params.get('processes'),
-                "matching_gps_neighbors: %s" % self.params.matching_gps_neighbors,
+                "matching_gps_neighbors: %s" % self.params.get('matching_gps_neighbors'),
                 "depthmap_method: %s" % args.opensfm_depthmap_method,
                 "depthmap_resolution: %s" % args.depthmap_resolution,
                 "depthmap_min_patch_sd: %s" % args.opensfm_depthmap_min_patch_sd,
                 "depthmap_min_consistent_views: %s" % args.opensfm_depthmap_min_consistent_views,
-                "optimize_camera_parameters: %s" % ('no' if self.params.fixed_camera_params else 'yes')
+                "optimize_camera_parameters: %s" % ('no' if self.params.get('fixed_camera_params') else 'yes')
             ]
 
             if has_alt:
@@ -100,7 +69,7 @@ class ODMOpenSfMCell(ecto.Cell):
                 config.append("local_bundle_radius: 1")        # Max image graph distance for images to be included in local bundle adjustment
 
             if args.matcher_distance > 0:
-                config.append("matching_gps_distance: %s" % self.params.matching_gps_distance)
+                config.append("matching_gps_distance: %s" % self.params.get('matching_gps_distance'))
 
             if tree.odm_georeferencing_gcp:
                 config.append("bundle_use_gcp: yes")
@@ -114,7 +83,7 @@ class ODMOpenSfMCell(ecto.Cell):
 
             # run OpenSfM reconstruction
             matched_done_file = io.join_paths(tree.opensfm, 'matching_done.txt')
-            if not io.file_exists(matched_done_file) or rerun_cell:
+            if not io.file_exists(matched_done_file) or self.rerun():
                 system.run('PYTHONPATH=%s %s/bin/opensfm extract_metadata %s' %
                            (context.pyopencv_path, context.opensfm_path, tree.opensfm))
                 system.run('PYTHONPATH=%s %s/bin/opensfm detect_features %s' %
@@ -127,14 +96,14 @@ class ODMOpenSfMCell(ecto.Cell):
                 log.ODM_WARNING('Found a feature matching done progress file in: %s' %
                                 matched_done_file)
 
-            if not io.file_exists(tree.opensfm_tracks) or rerun_cell:
+            if not io.file_exists(tree.opensfm_tracks) or self.rerun():
                 system.run('PYTHONPATH=%s %s/bin/opensfm create_tracks %s' %
                            (context.pyopencv_path, context.opensfm_path, tree.opensfm))
             else:
                 log.ODM_WARNING('Found a valid OpenSfM tracks file in: %s' %
                                 tree.opensfm_tracks)
 
-            if not io.file_exists(tree.opensfm_reconstruction) or rerun_cell:
+            if not io.file_exists(tree.opensfm_reconstruction) or self.rerun():
                 system.run('PYTHONPATH=%s %s/bin/opensfm reconstruct %s' %
                            (context.pyopencv_path, context.opensfm_path, tree.opensfm))
             else:
@@ -159,7 +128,7 @@ class ODMOpenSfMCell(ecto.Cell):
             else:
                 image_scale = 1.0
 
-            if not io.file_exists(tree.opensfm_reconstruction_nvm) or rerun_cell:
+            if not io.file_exists(tree.opensfm_reconstruction_nvm) or self.rerun():
                 system.run('PYTHONPATH=%s %s/bin/opensfm export_visualsfm --image_extension png --scale_focal %s %s' %
                             (context.pyopencv_path, context.opensfm_path, image_scale, tree.opensfm))
             else:
@@ -188,7 +157,7 @@ class ODMOpenSfMCell(ecto.Cell):
                             tree.opensfm_reconstruction)
 
         # check if reconstruction was exported to bundler before
-        if not io.file_exists(tree.opensfm_bundle_list) or rerun_cell:
+        if not io.file_exists(tree.opensfm_bundle_list) or self.rerun():
             # convert back to bundler's format
             system.run('PYTHONPATH=%s %s/bin/export_bundler %s' %
                        (context.pyopencv_path, context.opensfm_path, tree.opensfm))
@@ -199,11 +168,3 @@ class ODMOpenSfMCell(ecto.Cell):
         if reconstruction.georef:
             system.run('PYTHONPATH=%s %s/bin/opensfm export_geocoords %s --transformation --proj \'%s\'' %
                        (context.pyopencv_path, context.opensfm_path, tree.opensfm, reconstruction.georef.projection.srs))
-
-        outputs.reconstruction = reconstruction
-
-        if args.time:
-            system.benchmark(start_time, tree.benchmarking, 'OpenSfM')
-
-        log.ODM_INFO('Running ODM OpenSfM Cell - Finished')
-        return ecto.OK if args.end_with != 'opensfm' else ecto.QUIT

tests/test_odm.py

@@ -1,108 +0,0 @@
-import unittest
-import os
-import shutil
-
-import ecto
-from opendm import config
-from opendm import context
-from scripts.odm_app import ODMApp
-from ecto.opts import scheduler_options, run_plasm
-
-parser = config.parser
-scheduler_options(parser)
-options = config.config()
-
-
-def appSetup(options):
-    app = ODMApp(args=options)
-    plasm = ecto.Plasm()
-    plasm.insert(app)
-    return app, plasm
-
-
-def setup_module():
-    # Run tests
-    print '%s' % options
-    options.project_path = context.tests_data_path
-    # options.rerun_all = True
-    app, plasm = appSetup(options)
-    print 'Run Setup: Initial Run'
-    run_plasm(options, plasm)
-    # options.rerun_all = False
-
-
-def teardown_module():
-    # Delete generated test directories
-    dirnames = ['opensfm', 'odm_meshing',
-                'odm_texturing', 'odm_georeferencing', 'odm_orthophoto']
-    for n in dirnames:
-        rmpath = os.path.join(context.tests_data_path, n)
-        if os.path.exists(rmpath):
-            shutil.rmtree(rmpath)
-
-
-class TestOpenSfM(unittest.TestCase):
-    """
-    Tests the OpenSfM module
-    """
-    def setUp(self):
-        options.rerun = 'opensfm'
-        self.app, self.plasm = appSetup(options)
-        run_plasm(options, self.plasm)
-
-    def test_opensfm(self):
-        # Test configuration
-        self.assertTrue(os.path.isfile(self.app.opensfm.inputs.tree.opensfm_reconstruction))
-
-
-class TestMeshing(unittest.TestCase):
-
-    def setUp(self):
-        options.rerun = 'odm_meshing'
-        self.app, self.plasm = appSetup(options)
-        run_plasm(options, self.plasm)
-
-    def test_meshing(self):
-        self.assertTrue(os.path.isfile(self.app.meshing.inputs.tree.odm_mesh))
-
-
-class TestTexturing(unittest.TestCase):
-
-    def setUp(self):
-        options.rerun = 'odm_texturing'
-        self.app, self.plasm = appSetup(options)
-        run_plasm(options, self.plasm)
-
-    def test_texturing(self):
-        self.assertTrue(os.path.isfile(self.app.texturing.inputs.tree.odm_textured_model_obj))
-
-
-class TestGeoreferencing(unittest.TestCase):
-
-    def setUp(self):
-        options.rerun = 'odm_georeferencing'
-        self.app, self.plasm = appSetup(options)
-        run_plasm(options, self.plasm)
-
-    def test_georef(self):
-        self.assertTrue(os.path.isfile(self.app.georeferencing.inputs.tree.odm_georeferencing_coords) &
-                        os.path.isfile(self.app.georeferencing.inputs.tree.odm_georeferencing_model_obj_geo))
-
-    def test_las_out(self):
-        self.assertTrue(os.path.isfile(os.path.join(self.app.georeferencing.inputs.tree.odm_georeferencing,
-                                                    "odm_georeferenced_model.laz")))
-
-
-class TestOrthophoto(unittest.TestCase):
-
-    def setUp(self):
-        options.rerun = 'odm_orthophoto'
-        self.app, self.plasm = appSetup(options)
-        run_plasm(options, self.plasm)
-
-    def test_orthophoto(self):
-        self.assertTrue(os.path.isfile(self.app.orthophoto.inputs.tree.odm_orthophoto_file))
-
-
-if __name__ == '__main__':
-    unittest.main()