Progress reporting IPC

Former-commit-id: d46a25155f

opendm/progress.py

@@ -0,0 +1,39 @@
+import socket
+import os
+from opendm import log
+
+PROGRESS_BROADCAST_PORT = 6367 #ODMR
+try:
+    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+except:
+    log.ODM_WARNING("Cannot create UDP socket, progress reporting will be disabled.")
+    sock = None
+
+class Broadcaster:
+    def __init__(self, port):
+        self.port = port
+        self.project_name = "<unnamed>"
+        self.pid = os.getpid()
+
+    def set_project_name(self, project_name):
+        self.project_name = project_name
+
+    def send_update(self, global_progress, stage_progress, stage):
+        """
+        Update any listener on the pipeline progress (in percentage terms)
+        """
+        if not sock:
+            return
+
+        UDP_IP = "127.0.0.1"
+
+        if global_progress > 100:
+            log.ODM_WARNING("Global progress is > 100, please contact the developers.")
+            global_progress = 100
+
+        try:
+            sock.sendto("PGUP/{}/{}/{}/{}/{}".format(self.pid, self.project_name, float(global_progress), float(stage_progress), stage).encode('utf-8'), (UDP_IP, self.port))
+        except:
+            log.ODM_WARNING("Failed to broadcast progress update on UDP port %s" % str(self.port))
+
+progressbc = Broadcaster(PROGRESS_BROADCAST_PORT)

opendm/types.py

@@ -12,6 +12,7 @@ import io
 import system
 import context
 import logging
+from opendm.progress import progressbc
 
 class ODM_Photo:
     """   ODMPhoto - a class for ODMPhotos
@@ -325,16 +326,19 @@ class ODM_Tree(object):
 
 
 class ODM_Stage:
-    def __init__(self, name, args, **params):
+    def __init__(self, name, args, progress=0.0, **params):
         self.name = name
         self.args = args
+        self.progress = progress
         self.params = params
         if self.params is None:
             self.params = {}
         self.next_stage = None
+        self.prev_stage = None
 
     def connect(self, stage):
         self.next_stage = stage
+        stage.prev_stage = self
         return stage
 
     def rerun(self):
@@ -359,6 +363,7 @@ class ODM_Stage:
             system.benchmark(start_time, outputs['tree'].benchmarking, self.name)
 
         log.ODM_INFO('Finished %s stage' % self.name)
+        self.update_progress_end()
 
         # Last stage?
         if self.args.end_with == self.name or self.args.rerun == self.name:
@@ -369,6 +374,27 @@ class ODM_Stage:
         elif self.next_stage is not None:
             self.next_stage.run(outputs)
 
+    def delta_progress(self):
+        if self.prev_stage:
+            return max(0.0, self.progress - self.prev_stage.progress)
+        else:
+            return max(0.0, self.progress)
+    
+    def previous_stages_progress(self):
+        sum = 0
+        stage = self.prev_stage
+        while stage:
+            sum += stage.delta_progress()
+            stage = stage.prev_stage
+        return sum
+
+    def update_progress_end(self):
+        self.update_progress(100.0)
+
+    def update_progress(self, progress):
+        progressbc.send_update(self.previous_stages_progress() + 
+                                (self.delta_progress() / 100.0) * float(progress), progress, self.name)
+
     def process(self, args, outputs):
         raise NotImplementedError
 

run.py

@@ -4,6 +4,7 @@ from opendm import log
 from opendm import config
 from opendm import system
 from opendm import io
+from opendm.progress import progressbc
 
 import os
 from pipes import quote
@@ -15,6 +16,8 @@ if __name__ == '__main__':
 
     log.ODM_INFO('Initializing OpenDroneMap app - %s' % system.now())
 
+    progressbc.set_project_name(args.name)
+
     # Add project dir if doesn't exist
     args.project_path = io.join_paths(args.project_path, args.name)
     if not io.dir_exists(args.project_path):

stages/dataset.py

@@ -8,6 +8,7 @@ from opendm import log
 from opendm import system
 from opendm import location
 from shutil import copyfile
+from opendm import progress
 
 def save_images_database(photos, database_file):
     with open(database_file, 'w') as f:
@@ -123,4 +124,3 @@ class ODMLoadDatasetStage(types.ODM_Stage):
         if outputs['reconstruction'].projection:
             with open(io.join_paths(tree.odm_georeferencing, tree.odm_georeferencing_proj), 'w') as f:
                 f.write(outputs['reconstruction'].projection.srs)
-

stages/odm_app.py

@@ -25,23 +25,23 @@ class ODMApp:
         Initializes the application and defines the ODM application pipeline stages
         """
         
-        dataset = ODMLoadDatasetStage('dataset', args, 
+        dataset = ODMLoadDatasetStage('dataset', args, progress=5.0,
                                           verbose=args.verbose,
                                           proj=args.proj)
-        split = ODMSplitStage('split', args)
+        split = ODMSplitStage('split', args, progress=75.0)
-        merge = ODMMergeStage('merge', args)
+        merge = ODMMergeStage('merge', args, progress=100.0)
-        opensfm = ODMOpenSfMStage('opensfm', args)
+        opensfm = ODMOpenSfMStage('opensfm', args, progress=25.0)
         slam = ODMSlamStage('slam', args)
-        mve = ODMMveStage('mve', args)
+        mve = ODMMveStage('mve', args, progress=50.0)
-        filterpoints = ODMFilterPoints('odm_filterpoints', args)
+        filterpoints = ODMFilterPoints('odm_filterpoints', args, progress=52.0)
-        meshing = ODMeshingStage('odm_meshing', args,
+        meshing = ODMeshingStage('odm_meshing', args, progress=60.0,
                                     max_vertex=args.mesh_size,
                                     oct_tree=args.mesh_octree_depth,
                                     samples=args.mesh_samples,
                                     point_weight=args.mesh_point_weight,
                                     max_concurrency=args.max_concurrency,
                                     verbose=args.verbose)
-        texturing = ODMMvsTexStage('mvs_texturing', args,
+        texturing = ODMMvsTexStage('mvs_texturing', args, progress=70.0,
                                     data_term=args.texturing_data_term,
                                     outlier_rem_type=args.texturing_outlier_removal_type,
                                     skip_vis_test=args.texturing_skip_visibility_test,
@@ -50,14 +50,14 @@ class ODMApp:
                                     skip_hole_fill=args.texturing_skip_hole_filling,
                                     keep_unseen_faces=args.texturing_keep_unseen_faces,
                                     tone_mapping=args.texturing_tone_mapping)
-        georeferencing = ODMGeoreferencingStage('odm_georeferencing', args,
+        georeferencing = ODMGeoreferencingStage('odm_georeferencing', args, progress=80.0,
                                                     gcp_file=args.gcp,
                                                     use_exif=args.use_exif,
                                                     verbose=args.verbose)
-        dem = ODMDEMStage('odm_dem', args,
+        dem = ODMDEMStage('odm_dem', args, progress=90.0,
                             max_concurrency=args.max_concurrency,
                             verbose=args.verbose)
-        orthophoto = ODMOrthoPhotoStage('odm_orthophoto', args)
+        orthophoto = ODMOrthoPhotoStage('odm_orthophoto', args, progress=100.0)
 
         if not args.video:
             # Normal pipeline

stages/splitmerge.py

@@ -148,6 +148,7 @@ class ODMSplitStage(types.ODM_Stage):
                 log.ODM_WARNING('Found a split done file in: %s' % split_done_file)
         else:
             log.ODM_INFO("Normal dataset, will process all at once.")
+            self.progress = 0.0
 
 
 class ODMMergeStage(types.ODM_Stage):
@@ -311,6 +312,6 @@ class ODMMergeStage(types.ODM_Stage):
             self.next_stage = None
         else:
             log.ODM_INFO("Normal dataset, nothing to merge.")
-
+            self.progress = 0.0