From d46a25155f561d19f42827c4f57cfe1988409371 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 15 May 2019 17:04:09 -0400 Subject: [PATCH] Progress reporting IPC --- opendm/progress.py | 39 +++++++++++++++++++++++++++++++++++++++ opendm/types.py | 28 +++++++++++++++++++++++++++- run.py | 3 +++ stages/dataset.py | 2 +- stages/odm_app.py | 22 +++++++++++----------- stages/splitmerge.py | 3 ++- 6 files changed, 83 insertions(+), 14 deletions(-) create mode 100644 opendm/progress.py diff --git a/opendm/progress.py b/opendm/progress.py new file mode 100644 index 00000000..3dd02523 --- /dev/null +++ b/opendm/progress.py @@ -0,0 +1,39 @@ +import socket +import os +from opendm import log + +PROGRESS_BROADCAST_PORT = 6367 #ODMR +try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +except: + log.ODM_WARNING("Cannot create UDP socket, progress reporting will be disabled.") + sock = None + +class Broadcaster: + def __init__(self, port): + self.port = port + self.project_name = "" + self.pid = os.getpid() + + def set_project_name(self, project_name): + self.project_name = project_name + + def send_update(self, global_progress, stage_progress, stage): + """ + Update any listener on the pipeline progress (in percentage terms) + """ + if not sock: + return + + UDP_IP = "127.0.0.1" + + if global_progress > 100: + log.ODM_WARNING("Global progress is > 100, please contact the developers.") + global_progress = 100 + + try: + sock.sendto("PGUP/{}/{}/{}/{}/{}".format(self.pid, self.project_name, float(global_progress), float(stage_progress), stage).encode('utf-8'), (UDP_IP, self.port)) + except: + log.ODM_WARNING("Failed to broadcast progress update on UDP port %s" % str(self.port)) + +progressbc = Broadcaster(PROGRESS_BROADCAST_PORT) \ No newline at end of file diff --git a/opendm/types.py b/opendm/types.py index ad10ab0e..5dcf00b6 100644 --- a/opendm/types.py +++ b/opendm/types.py @@ -12,6 +12,7 @@ import io import system import context import logging +from opendm.progress import progressbc class ODM_Photo: """ ODMPhoto - a class for ODMPhotos @@ -325,16 +326,19 @@ class ODM_Tree(object): class ODM_Stage: - def __init__(self, name, args, **params): + def __init__(self, name, args, progress=0.0, **params): self.name = name self.args = args + self.progress = progress self.params = params if self.params is None: self.params = {} self.next_stage = None + self.prev_stage = None def connect(self, stage): self.next_stage = stage + stage.prev_stage = self return stage def rerun(self): @@ -359,6 +363,7 @@ class ODM_Stage: system.benchmark(start_time, outputs['tree'].benchmarking, self.name) log.ODM_INFO('Finished %s stage' % self.name) + self.update_progress_end() # Last stage? if self.args.end_with == self.name or self.args.rerun == self.name: @@ -369,6 +374,27 @@ class ODM_Stage: elif self.next_stage is not None: self.next_stage.run(outputs) + def delta_progress(self): + if self.prev_stage: + return max(0.0, self.progress - self.prev_stage.progress) + else: + return max(0.0, self.progress) + + def previous_stages_progress(self): + sum = 0 + stage = self.prev_stage + while stage: + sum += stage.delta_progress() + stage = stage.prev_stage + return sum + + def update_progress_end(self): + self.update_progress(100.0) + + def update_progress(self, progress): + progressbc.send_update(self.previous_stages_progress() + + (self.delta_progress() / 100.0) * float(progress), progress, self.name) + def process(self, args, outputs): raise NotImplementedError diff --git a/run.py b/run.py index 17aea0d4..1c53bb4b 100755 --- a/run.py +++ b/run.py @@ -4,6 +4,7 @@ from opendm import log from opendm import config from opendm import system from opendm import io +from opendm.progress import progressbc import os from pipes import quote @@ -15,6 +16,8 @@ if __name__ == '__main__': log.ODM_INFO('Initializing OpenDroneMap app - %s' % system.now()) + progressbc.set_project_name(args.name) + # Add project dir if doesn't exist args.project_path = io.join_paths(args.project_path, args.name) if not io.dir_exists(args.project_path): diff --git a/stages/dataset.py b/stages/dataset.py index 8d3d9f77..daae9021 100644 --- a/stages/dataset.py +++ b/stages/dataset.py @@ -8,6 +8,7 @@ from opendm import log from opendm import system from opendm import location from shutil import copyfile +from opendm import progress def save_images_database(photos, database_file): with open(database_file, 'w') as f: @@ -123,4 +124,3 @@ class ODMLoadDatasetStage(types.ODM_Stage): if outputs['reconstruction'].projection: with open(io.join_paths(tree.odm_georeferencing, tree.odm_georeferencing_proj), 'w') as f: f.write(outputs['reconstruction'].projection.srs) - diff --git a/stages/odm_app.py b/stages/odm_app.py index 5a735737..25bd7e74 100644 --- a/stages/odm_app.py +++ b/stages/odm_app.py @@ -25,23 +25,23 @@ class ODMApp: Initializes the application and defines the ODM application pipeline stages """ - dataset = ODMLoadDatasetStage('dataset', args, + dataset = ODMLoadDatasetStage('dataset', args, progress=5.0, verbose=args.verbose, proj=args.proj) - split = ODMSplitStage('split', args) - merge = ODMMergeStage('merge', args) - opensfm = ODMOpenSfMStage('opensfm', args) + split = ODMSplitStage('split', args, progress=75.0) + merge = ODMMergeStage('merge', args, progress=100.0) + opensfm = ODMOpenSfMStage('opensfm', args, progress=25.0) slam = ODMSlamStage('slam', args) - mve = ODMMveStage('mve', args) - filterpoints = ODMFilterPoints('odm_filterpoints', args) - meshing = ODMeshingStage('odm_meshing', args, + mve = ODMMveStage('mve', args, progress=50.0) + filterpoints = ODMFilterPoints('odm_filterpoints', args, progress=52.0) + meshing = ODMeshingStage('odm_meshing', args, progress=60.0, max_vertex=args.mesh_size, oct_tree=args.mesh_octree_depth, samples=args.mesh_samples, point_weight=args.mesh_point_weight, max_concurrency=args.max_concurrency, verbose=args.verbose) - texturing = ODMMvsTexStage('mvs_texturing', args, + texturing = ODMMvsTexStage('mvs_texturing', args, progress=70.0, data_term=args.texturing_data_term, outlier_rem_type=args.texturing_outlier_removal_type, skip_vis_test=args.texturing_skip_visibility_test, @@ -50,14 +50,14 @@ class ODMApp: skip_hole_fill=args.texturing_skip_hole_filling, keep_unseen_faces=args.texturing_keep_unseen_faces, tone_mapping=args.texturing_tone_mapping) - georeferencing = ODMGeoreferencingStage('odm_georeferencing', args, + georeferencing = ODMGeoreferencingStage('odm_georeferencing', args, progress=80.0, gcp_file=args.gcp, use_exif=args.use_exif, verbose=args.verbose) - dem = ODMDEMStage('odm_dem', args, + dem = ODMDEMStage('odm_dem', args, progress=90.0, max_concurrency=args.max_concurrency, verbose=args.verbose) - orthophoto = ODMOrthoPhotoStage('odm_orthophoto', args) + orthophoto = ODMOrthoPhotoStage('odm_orthophoto', args, progress=100.0) if not args.video: # Normal pipeline diff --git a/stages/splitmerge.py b/stages/splitmerge.py index 15df5505..38ed191f 100644 --- a/stages/splitmerge.py +++ b/stages/splitmerge.py @@ -148,6 +148,7 @@ class ODMSplitStage(types.ODM_Stage): log.ODM_WARNING('Found a split done file in: %s' % split_done_file) else: log.ODM_INFO("Normal dataset, will process all at once.") + self.progress = 0.0 class ODMMergeStage(types.ODM_Stage): @@ -311,6 +312,6 @@ class ODMMergeStage(types.ODM_Stage): self.next_stage = None else: log.ODM_INFO("Normal dataset, nothing to merge.") - + self.progress = 0.0 \ No newline at end of file