Retry mve on error code 134, progress fix

opendm/progress.py

@@ -28,7 +28,7 @@ class Broadcaster:
         UDP_IP = "127.0.0.1"
 
         if global_progress > 100:
-            log.ODM_WARNING("Global progress is > 100, please contact the developers.")
+            log.ODM_WARNING("Global progress is > 100 (%s), please contact the developers." % global_progress)
             global_progress = 100
 
         try:

opendm/types.py

@@ -381,12 +381,10 @@ class ODM_Stage:
             return max(0.0, self.progress)
     
     def previous_stages_progress(self):
-        sum = 0
-        stage = self.prev_stage
-        while stage:
-            sum += stage.delta_progress()
-            stage = stage.prev_stage
-        return sum
+        if self.prev_stage:
+            return max(0.0, self.prev_stage.progress)
+        else:
+            return 0.0
 
     def update_progress_end(self):
         self.update_progress(100.0)

run.py

@@ -22,7 +22,7 @@ if __name__ == '__main__':
     for k in sorted(args_dict.keys()):
         # Don't leak token
         if k == 'sm_cluster' and args_dict[k] is not None:
-            log.ODM_DEBUG('%s: True')
+            log.ODM_DEBUG('%s: True' % k)
         else:
             log.ODM_DEBUG('%s: %s' % (k, args_dict[k]))
     log.ODM_DEBUG('==============')

stages/mve.py

@@ -110,8 +110,23 @@ class ODMMveStage(types.ODM_Stage):
             log.ODM_INFO("Running dense reconstruction. This might take a while. Please be patient, the process is not dead or hung.")
             log.ODM_INFO("                              Process is running")
             
-            system.run('%s %s %s' % (context.dmrecon_path, ' '.join(dmrecon_config), tree.mve), env_vars={'OMP_NUM_THREADS': args.max_concurrency})
-            
+            # TODO: find out why MVE is crashing at random
+            # MVE *seems* to have a race condition, triggered randomly, regardless of dataset
+            # https://gist.github.com/pierotofy/6c9ce93194ba510b61e42e3698cfbb89
+            # Temporary workaround is to retry the reconstruction until we get it right
+            # (up to a certain number of retries).
+            retry_count = 1
+            while retry_count < 10:
+                try:
+                    system.run('%s %s %s' % (context.dmrecon_path, ' '.join(dmrecon_config), tree.mve), env_vars={'OMP_NUM_THREADS': args.max_concurrency})
+                    break
+                except Exception as e:
+                    if str(e) == "Child returned 134":
+                        retry_count += 1
+                        log.ODM_WARNING("Caught error code, retrying attempt #%s" % retry_count)
+                    else:
+                        raise e
+
             self.update_progress(90)
 
             scene2pset_config = [