Better subprocess termination handling

Former-commit-id: 1a369bd0d7

opendm/config.py

@@ -7,7 +7,7 @@ from appsettings import SettingsParser
 import sys
 
 # parse arguments
-processopts = ['dataset', 'split', 'opensfm', 'slam', 'mve', 'odm_filterpoints',
+processopts = ['dataset', 'split', 'merge', 'opensfm', 'slam', 'mve', 'odm_filterpoints',
                'odm_meshing', 'odm_25dmeshing', 'mvs_texturing', 'odm_georeferencing',
                'odm_dem', 'odm_orthophoto']
 

opendm/osfm.py

@@ -149,6 +149,8 @@ def get_submodel_argv(args, submodels_path, submodel_name):
     project_path_found = False
     project_name_added = False
 
+    # TODO: what about GCP paths?
+
     while i < len(argv):
         arg = argv[i]
         

opendm/system.py

@@ -5,6 +5,7 @@ import datetime
 import sys
 import subprocess
 import string
+import signal
 
 from opendm import context
 from opendm import log
@@ -16,9 +17,23 @@ def get_ccd_widths():
         sensor_data = json.loads(f.read())
     return dict(zip(map(string.lower, sensor_data.keys()), sensor_data.values()))
 
+running_subprocesses = []
+def exit_gracefully(signum, frame):
+    global running_subprocesses
+
+    log.ODM_WARNING("Caught TERM/INT signal, attempting to exit gracefully...")
+
+    for sp in running_subprocesses:
+        log.ODM_WARNING("Sending TERM signal to PID %s..." % sp.pid)
+        os.killpg(os.getpgid(sp.pid), signal.SIGTERM)
+
+signal.signal(signal.SIGINT, exit_gracefully)
+signal.signal(signal.SIGTERM, exit_gracefully)
 
 def run(cmd, env_paths=[context.superbuild_bin_path], env_vars={}):
     """Run a system command"""
+    global running_subprocesses
+
     log.ODM_DEBUG('running %s' % cmd)
 
     env = os.environ.copy()
@@ -28,8 +43,10 @@ def run(cmd, env_paths=[context.superbuild_bin_path], env_vars={}):
     for k in env_vars:
         env[k] = str(env_vars[k])
 
-    retcode = subprocess.call(cmd, shell=True, env=env)
-
+    p = subprocess.Popen(cmd, shell=True, env=env, preexec_fn=os.setsid)
+    running_subprocesses.append(p)
+    retcode = p.wait()
+    running_subprocesses.remove(p)
     if retcode < 0:
         raise Exception("Child was terminated by signal {}".format(-retcode))
     elif retcode > 0:

scripts/odm_app.py

@@ -16,7 +16,7 @@ from odm_georeferencing import ODMGeoreferencingStage
 from odm_orthophoto import ODMOrthoPhotoStage
 from odm_dem import ODMDEMStage
 from odm_filterpoints import ODMFilterPoints
-from splitmerge import ODMSplitStage
+from splitmerge import ODMSplitStage, ODMMergeStage
 
 
 class ODMApp:
@@ -29,6 +29,7 @@ class ODMApp:
                                           verbose=args.verbose,
                                           proj=args.proj)
         split = ODMSplitStage('split', args)
+        merge = ODMMergeStage('merge', args)
         opensfm = ODMOpenSfMStage('opensfm', args)
         slam = ODMSlamStage('slam', args)
         mve = ODMMveStage('mve', args)
@@ -69,7 +70,9 @@ class ODMApp:
             # Normal pipeline
             self.first_stage = dataset
 
-            dataset.connect(split).connect(opensfm)
+            dataset.connect(split) \
+                   .connect(merge) \
+                   .connect(opensfm)
 
             if args.use_opensfm_dense or args.fast_orthophoto:
                 opensfm.connect(filterpoints)

scripts/splitmerge.py

@@ -65,7 +65,7 @@ class ODMSplitStage(types.ODM_Stage):
                 
                 # We have already done matching
                 osfm.mark_feature_matching_done(sp)
-                
+
                 submodel_name = os.path.basename(os.path.abspath(os.path.join(sp, "..")))
 
                 log.ODM_INFO("====================")
@@ -73,11 +73,20 @@ class ODMSplitStage(types.ODM_Stage):
                 log.ODM_INFO("====================")
 
                 argv = osfm.get_submodel_argv(args, tree.submodels_path, submodel_name)
+
+                # Re-run the ODM toolchain on the submodel
                 system.run(" ".join(map(quote, argv)), env_vars=os.environ.copy())
 
-            exit(1)
         else:
             log.ODM_INFO("Normal dataset, will process all at once.")
 
 
+class ODMMergeStage(types.ODM_Stage):
+    def process(self, args, outputs):
+        tree = outputs['tree']
+        reconstruction = outputs['reconstruction']
 
+        if outputs['large']:
+            
+        else:
+            log.ODM_INFO("Normal dataset, nothing to merge.")