PoC JSON log

opendm/log.py

@@ -1,4 +1,10 @@
 import sys
+import threading
+import os
+import json
+import datetime
+
+from opendm.loghelpers import double_quote, args_to_dict
 
 if sys.platform == 'win32':
     # No colors on Windows, sorry!
@@ -18,16 +24,33 @@ else:
     FAIL = '\033[91m'
     ENDC = '\033[0m'
 
-# logging has too many quirks...
+lock = threading.Lock()
+
+def odm_version():
+    with open(os.path.join(os.path.dirname(__file__), "..", "VERSION")) as f:
+        return f.read().split("\n")[0].strip()
+
 class ODMLogger:
     def __init__(self):
         self.show_debug = False
+        self.json = None
+        self.json_output_file = None
+        self.start_time = datetime.datetime.now()
 
     def log(self, startc, msg, level_name):
         level = ("[" + level_name + "]").ljust(9)
-        print("%s%s %s%s" % (startc, level, msg, ENDC))
-        sys.stdout.flush()
-
+        with lock:
+            print("%s%s %s%s" % (startc, level, msg, ENDC))
+            sys.stdout.flush()
+    
+    def init_json_output(self, output_file, args):
+        self.json_output_file = output_file
+        self.json = {}
+        self.json['odmVersion'] = odm_version()
+        self.json['options'] = args_to_dict(args)
+        self.json["startTime"] = self.start_time.isoformat()
+        self.json["stages"] = []
+            
     def info(self, msg):
         self.log(DEFAULT, msg, "INFO")
 
@@ -44,6 +67,11 @@ class ODMLogger:
         if self.show_debug:
             self.log(OKGREEN, msg, "DEBUG")
 
+    def close(self):
+        if self.json is not None and self.json_output_file is not None:
+            with open(self.json_output_file, 'w') as f:
+                f.write(json.dumps(self.json, indent=4))
+
 logger = ODMLogger()
 
 ODM_INFO = logger.info

opendm/loghelpers.py

@@ -0,0 +1,28 @@
+from shlex import _find_unsafe
+
+def double_quote(s):
+    """Return a shell-escaped version of the string *s*."""
+    if not s:
+        return '""'
+    if _find_unsafe(s) is None:
+        return s
+
+    # use double quotes, and prefix double quotes with a \
+    # the string $"b is then quoted as "$\"b"
+    return '"' + s.replace('"', '\\\"') + '"'
+
+def args_to_dict(args):
+    args_dict = vars(args)
+    result = {}
+    for k in sorted(args_dict.keys()):
+        # Skip _is_set keys
+        if k.endswith("_is_set"):
+            continue
+
+        # Don't leak token
+        if k == 'sm_cluster' and args_dict[k] is not None:
+            result[k] = True
+        else:
+            result[k] = args_dict[k]
+    
+    return result

opendm/utils.py

@@ -2,7 +2,7 @@ import os, shutil
 from opendm import log
 from opendm.photo import find_largest_photo_dim
 from osgeo import gdal
-from shlex import _find_unsafe
+from opendm.loghelpers import double_quote
 
 def get_depthmap_resolution(args, photos):
     if 'depthmap_resolution_is_set' in args:
@@ -43,17 +43,6 @@ def get_raster_stats(geotiff):
             
     return stats
 
-def double_quote(s):
-    """Return a shell-escaped version of the string *s*."""
-    if not s:
-        return '""'
-    if _find_unsafe(s) is None:
-        return s
-
-    # use double quotes, and prefix double quotes with a \
-    # the string $"b is then quoted as "$\"b"
-    return '"' + s.replace('"', '\\\"') + '"'
-
 def get_processing_results_paths():
     return [
         "odm_georeferencing",
@@ -67,6 +56,7 @@ def get_processing_results_paths():
         "orthophoto_tiles",
         "images.json",
         "cameras.json",
+        "log.json",
     ]
 
 def copy_paths(paths, destination, rerun):

run.py

@@ -12,6 +12,7 @@ from opendm import system
 from opendm import io
 from opendm.progress import progressbc
 from opendm.utils import double_quote, get_processing_results_paths
+from opendm.loghelpers import args_to_dict
 
 import os
 
@@ -23,18 +24,10 @@ if __name__ == '__main__':
     log.ODM_INFO('Initializing ODM - %s' % system.now())
 
     # Print args
-    args_dict = vars(args)
+    args_dict = args_to_dict(args)
     log.ODM_INFO('==============')
-    for k in sorted(args_dict.keys()):
-        # Skip _is_set keys
-        if k.endswith("_is_set"):
-            continue
-
-        # Don't leak token
-        if k == 'sm_cluster' and args_dict[k] is not None:
-            log.ODM_INFO('%s: True' % k)
-        else:
-            log.ODM_INFO('%s: %s' % (k, args_dict[k]))
+    for k in args_dict.keys():
+        log.ODM_INFO('%s: %s' % (k, args_dict[k]))
     log.ODM_INFO('==============')
 
     progressbc.set_project_name(args.name)

stages/odm_app.py

@@ -27,6 +27,8 @@ class ODMApp:
         if args.debug:
             log.logger.show_debug = True
         
+        log.logger.init_json_output(os.path.join(args.project_path, "log.json"), args)
+        
         dataset = ODMLoadDatasetStage('dataset', args, progress=5.0,
                                           verbose=args.verbose)
         split = ODMSplitStage('split', args, progress=75.0)
@@ -107,3 +109,5 @@ class ODMApp:
             # TODO: more?
 
             return code
+        finally:
+            log.logger.close()

stages/odm_report.py

@@ -198,6 +198,7 @@ class ODMReport(types.ODM_Stage):
         octx.export_report(os.path.join(tree.odm_report, "report.pdf"), odm_stats, self.rerun())
 
         # TODO: does this warrant a new stage?
+        # TODO: move to pipeline end ?
         if args.copy_to:
             try:
                 copy_paths([os.path.join(args.project_path, p) for p in get_processing_results_paths()], args.copy_to, self.rerun())