From 6df5e0b711765818af90f6d00ff14dea1af04675 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 8 Nov 2023 11:07:20 -0500 Subject: [PATCH] Feat: Auto rerun-from --- VERSION | 2 +- opendm/arghelpers.py | 74 ++++++++++++++++++++++++++++++++++ opendm/config.py | 95 ++++++++++++++++++++++++++++++++++++++++++++ opendm/log.py | 2 +- opendm/loghelpers.py | 28 ------------- opendm/utils.py | 2 +- run.py | 29 +++++++++----- stages/odm_app.py | 1 + 8 files changed, 192 insertions(+), 41 deletions(-) create mode 100644 opendm/arghelpers.py delete mode 100644 opendm/loghelpers.py diff --git a/VERSION b/VERSION index bea438e9..47725433 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.3.1 +3.3.2 diff --git a/opendm/arghelpers.py b/opendm/arghelpers.py new file mode 100644 index 00000000..d2dd9d53 --- /dev/null +++ b/opendm/arghelpers.py @@ -0,0 +1,74 @@ +from opendm import log +from shlex import _find_unsafe +import json +import os + +def double_quote(s): + """Return a shell-escaped version of the string *s*.""" + if not s: + return '""' + if _find_unsafe(s) is None: + return s + + # use double quotes, and prefix double quotes with a \ + # the string $"b is then quoted as "$\"b" + return '"' + s.replace('"', '\\\"') + '"' + +def args_to_dict(args): + args_dict = vars(args) + result = {} + for k in sorted(args_dict.keys()): + # Skip _is_set keys + if k.endswith("_is_set"): + continue + + # Don't leak token + if k == 'sm_cluster' and args_dict[k] is not None: + result[k] = True + else: + result[k] = args_dict[k] + + return result + +def save_opts(opts_json, args): + try: + with open(opts_json, "w", encoding='utf-8') as f: + f.write(json.dumps(args_to_dict(args))) + except Exception as e: + log.ODM_WARNING("Cannot save options to %s: %s" % (opts_json, str(e))) + +def compare_args(opts_json, args, rerun_stages): + if not os.path.isfile(opts_json): + return {} + + try: + diff = {} + + with open(opts_json, "r", encoding="utf-8") as f: + prev_args = json.loads(f.read()) + cur_args = args_to_dict(args) + + for opt in cur_args: + cur_value = cur_args[opt] + prev_value = prev_args.get(opt, None) + stage = rerun_stages.get(opt, None) + + if stage is not None and cur_value != prev_value: + diff[opt] = prev_value + + return diff + except: + return {} + +def find_rerun_stage(opts_json, args, rerun_stages, processopts): + # Find the proper rerun stage if one is not explicitly set + if not ('rerun_is_set' in args or 'rerun_from_is_set' in args or 'rerun_all_is_set' in args): + args_diff = compare_args(opts_json, args, rerun_stages) + if args_diff: + try: + stage_idxs = [processopts.index(rerun_stages[opt]) for opt in args_diff.keys() if rerun_stages[opt] is not None] + return processopts[min(stage_idxs)], args_diff + except ValueError as e: + print(str(e)) + return None, {} + return None, {} \ No newline at end of file diff --git a/opendm/config.py b/opendm/config.py index a8b4a9e6..849b59ce 100755 --- a/opendm/config.py +++ b/opendm/config.py @@ -13,6 +13,101 @@ processopts = ['dataset', 'split', 'merge', 'opensfm', 'openmvs', 'odm_filterpoi 'odm_meshing', 'mvs_texturing', 'odm_georeferencing', 'odm_dem', 'odm_orthophoto', 'odm_report', 'odm_postprocess'] +rerun_stages = { + '3d_tiles': 'odm_postprocess', + 'align': 'odm_georeferencing', + 'auto_boundary': 'odm_filterpoints', + 'auto_boundary_distance': 'odm_filterpoints', + 'bg_removal': 'dataset', + 'boundary': 'odm_filterpoints', + 'build_overviews': 'odm_orthophoto', + 'camera_lens': 'dataset', + 'cameras': 'dataset', + 'cog': 'odm_dem', + 'copy_to': 'odm_postprocess', + 'crop': 'odm_georeferencing', + 'dem_decimation': 'odm_dem', + 'dem_euclidean_map': 'odm_dem', + 'dem_gapfill_steps': 'odm_dem', + 'dem_resolution': 'odm_dem', + 'dsm': 'odm_dem', + 'dtm': 'odm_dem', + 'end_with': None, + 'fast_orthophoto': 'odm_filterpoints', + 'feature_quality': 'opensfm', + 'feature_type': 'opensfm', + 'force_gps': 'opensfm', + 'gcp': 'dataset', + 'geo': 'dataset', + 'gltf': 'mvs_texturing', + 'gps_accuracy': 'dataset', + 'help': None, + 'ignore_gsd': 'opensfm', + 'matcher_neighbors': 'opensfm', + 'matcher_order': 'opensfm', + 'matcher_type': 'opensfm', + 'max_concurrency': None, + 'merge': 'Merge', + 'mesh_octree_depth': 'odm_meshing', + 'mesh_size': 'odm_meshing', + 'min_num_features': 'opensfm', + 'name': None, + 'no_gpu': None, + 'optimize_disk_space': None, + 'orthophoto_compression': 'odm_orthophoto', + 'orthophoto_cutline': 'odm_orthophoto', + 'orthophoto_kmz': 'odm_orthophoto', + 'orthophoto_no_tiled': 'odm_orthophoto', + 'orthophoto_png': 'odm_orthophoto', + 'orthophoto_resolution': 'odm_orthophoto', + 'pc_classify': 'odm_dem', + 'pc_copc': 'odm_georeferencing', + 'pc_csv': 'odm_georeferencing', + 'pc_ept': 'odm_georeferencing', + 'pc_filter': 'openmvs', + 'pc_las': 'odm_georeferencing', + 'pc_quality': 'opensfm', + 'pc_rectify': 'odm_dem', + 'pc_sample': 'odm_filterpoints', + 'pc_skip_geometric': 'openmvs', + 'primary_band': 'dataset', + 'project_path': None, + 'radiometric_calibration': 'opensfm', + 'rerun': None, + 'rerun_all': None, + 'rerun_from': None, + 'rolling_shutter': 'opensfm', + 'rolling_shutter_readout': 'opensfm', + 'sfm_algorithm': 'opensfm', + 'sfm_no_partial': 'opensfm', + 'skip_3dmodel': 'odm_meshing', + 'skip_band_alignment': 'opensfm', + 'skip_orthophoto': 'odm_orthophoto', + 'skip_report': 'odm_report', + 'sky_removal': 'dataset', + 'sm_cluster': 'split', + 'sm_no_align': 'split', + 'smrf_scalar': 'odm_dem', + 'smrf_slope': 'odm_dem', + 'smrf_threshold': 'odm_dem', + 'smrf_window': 'odm_dem', + 'split': 'split', + 'split_image_groups': 'split', + 'split_overlap': 'split', + 'texturing_keep_unseen_faces': 'mvs_texturing', + 'texturing_single_material': 'mvs_texturing', + 'texturing_skip_global_seam_leveling': 'mvs_texturing', + 'texturing_skip_local_seam_leveling': 'mvs_texturing', + 'tiles': 'odm_dem', + 'use_3dmesh': 'mvs_texturing', + 'use_exif': 'dataset', + 'use_fixed_camera_params': 'opensfm', + 'use_hybrid_bundle_adjustment': 'opensfm', + 'version': None, + 'video_limit': 'dataset', + 'video_resolution': 'dataset', +} + with open(os.path.join(context.root_path, 'VERSION')) as version_file: __version__ = version_file.read().strip() diff --git a/opendm/log.py b/opendm/log.py index 18b7b9cb..8e04d779 100644 --- a/opendm/log.py +++ b/opendm/log.py @@ -7,7 +7,7 @@ import dateutil.parser import shutil import multiprocessing -from opendm.loghelpers import double_quote, args_to_dict +from opendm.arghelpers import double_quote, args_to_dict from vmem import virtual_memory if sys.platform == 'win32' or os.getenv('no_ansiesc'): diff --git a/opendm/loghelpers.py b/opendm/loghelpers.py deleted file mode 100644 index 283816fa..00000000 --- a/opendm/loghelpers.py +++ /dev/null @@ -1,28 +0,0 @@ -from shlex import _find_unsafe - -def double_quote(s): - """Return a shell-escaped version of the string *s*.""" - if not s: - return '""' - if _find_unsafe(s) is None: - return s - - # use double quotes, and prefix double quotes with a \ - # the string $"b is then quoted as "$\"b" - return '"' + s.replace('"', '\\\"') + '"' - -def args_to_dict(args): - args_dict = vars(args) - result = {} - for k in sorted(args_dict.keys()): - # Skip _is_set keys - if k.endswith("_is_set"): - continue - - # Don't leak token - if k == 'sm_cluster' and args_dict[k] is not None: - result[k] = True - else: - result[k] = args_dict[k] - - return result \ No newline at end of file diff --git a/opendm/utils.py b/opendm/utils.py index 85a32003..71d7ca3d 100644 --- a/opendm/utils.py +++ b/opendm/utils.py @@ -4,7 +4,7 @@ import json from opendm import log from opendm.photo import find_largest_photo_dims from osgeo import gdal -from opendm.loghelpers import double_quote +from opendm.arghelpers import double_quote class NumpyEncoder(json.JSONEncoder): def default(self, obj): diff --git a/run.py b/run.py index e921f931..91e9f1c7 100755 --- a/run.py +++ b/run.py @@ -13,7 +13,7 @@ from opendm import system from opendm import io from opendm.progress import progressbc from opendm.utils import get_processing_results_paths, rm_r -from opendm.loghelpers import args_to_dict +from opendm.arghelpers import args_to_dict, save_opts, compare_args, find_rerun_stage from stages.odm_app import ODMApp @@ -29,20 +29,26 @@ if __name__ == '__main__': log.ODM_INFO('Initializing ODM %s - %s' % (odm_version(), system.now())) + progressbc.set_project_name(args.name) + args.project_path = os.path.join(args.project_path, args.name) + + if not io.dir_exists(args.project_path): + log.ODM_ERROR('Directory %s does not exist.' % args.name) + exit(1) + + opts_json = os.path.join(args.project_path, "options.json") + auto_rerun_stage, opts_diff = find_rerun_stage(opts_json, args, config.rerun_stages, config.processopts) + if auto_rerun_stage is not None: + log.ODM_INFO("Rerunning from: %s" % auto_rerun_stage) + args.rerun_from = auto_rerun_stage + # Print args args_dict = args_to_dict(args) log.ODM_INFO('==============') for k in args_dict.keys(): - log.ODM_INFO('%s: %s' % (k, args_dict[k])) + log.ODM_INFO('%s: %s%s' % (k, args_dict[k], ' [changed]' if k in opts_diff else '')) log.ODM_INFO('==============') - - progressbc.set_project_name(args.name) - - # Add project dir if doesn't exist - args.project_path = os.path.join(args.project_path, args.name) - if not io.dir_exists(args.project_path): - log.ODM_WARNING('Directory %s does not exist. Creating it now.' % args.name) - system.mkdir_p(os.path.abspath(args.project_path)) + # If user asks to rerun everything, delete all of the existing progress directories. if args.rerun_all: @@ -57,6 +63,9 @@ if __name__ == '__main__': app = ODMApp(args) retcode = app.execute() + + if retcode == 0: + save_opts(opts_json, args) # Do not show ASCII art for local submodels runs if retcode == 0 and not "submodels" in args.project_path: diff --git a/stages/odm_app.py b/stages/odm_app.py index 79a09c09..aee04277 100644 --- a/stages/odm_app.py +++ b/stages/odm_app.py @@ -27,6 +27,7 @@ class ODMApp: Initializes the application and defines the ODM application pipeline stages """ json_log_paths = [os.path.join(args.project_path, "log.json")] + if args.copy_to: json_log_paths.append(args.copy_to)