From c424769cd70c6b4a35310bc7ee569a822b34d786 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 19 Jan 2024 15:37:25 -0500 Subject: [PATCH] PoC Mavic 3M RGB unwarping --- opendm/multispectral.py | 61 +++++++++++++++++++++++++++++++++++++++-- opendm/photo.py | 9 ++++++ opendm/types.py | 28 +++++++++++++------ stages/dataset.py | 8 ++++-- 4 files changed, 94 insertions(+), 12 deletions(-) diff --git a/opendm/multispectral.py b/opendm/multispectral.py index 3d2373d3..b7228109 100644 --- a/opendm/multispectral.py +++ b/opendm/multispectral.py @@ -6,7 +6,8 @@ from opendm import dls import numpy as np from opendm import log from opendm.concurrency import parallel_map -from opensfm.io import imread +from opendm.io import related_file_path +from opensfm.io import imread, imwrite from skimage import exposure from skimage.morphology import disk @@ -635,4 +636,60 @@ def resize_match(image, dimension): fy=fx, interpolation=(cv2.INTER_AREA if (fx < 1.0 and fy < 1.0) else cv2.INTER_LANCZOS4)) - return image \ No newline at end of file + return image + +def dewarp_photos(photos, images_path, scale_factor=1.0): + # Caution! This will make changes to the photo objects' filename + + for p in photos: + # This should never happen + if os.path.splitext(p.filename)[0].endswith("_dewarped"): + log.ODM_WARNING("Cannot dewarp %s, already dewarped?" % (p.filename)) + continue + + if p.dewarp_data is None: + log.ODM_WARNING("Cannot dewarp %s, dewarp data is missing" % p.filename) + continue + + if p.dewarp_data.count(";") != 1: + log.ODM_WARNING("Cannot dewarp %s, cannot parse dewarp data" % p.filename) + continue + + datestamp, params = p.dewarp_data.split(";") + try: + params = [float(p) for p in params.split(",")] + except ValueError as e: + log.ODM_WARNING("Cannot dewarp %s, failed to parse dewarp data" % p.filename) + continue + + if len(params) != 9: + log.ODM_WARNING("Cannot dewarp %s, invalid dewarp data parameters (expected 9, got: %s)" % (p.filename, len(params))) + continue + + dewarped_filename = related_file_path(p.filename, postfix="_dewarped") + dewarped_path = os.path.join(images_path, dewarped_filename) + if os.path.isfile(dewarped_path): + # Already dewarped + p.filename = dewarped_filename + else: + image = imread(os.path.join(images_path, p.filename), unchanged=True, anydepth=True) + w, h = image.shape[1], image.shape[0] + fx, fy, cx, cy, k1, k2, p1, p2, k3 = params + cam_m = np.array([[fx, 0, w / 2 - cx], [0, fy, h / 2 + cy], [0, 0, 1]]) + dist_c = np.array([k1, k2, p1, p2, k3]) + map1, map2 = cv2.initUndistortRectifyMap(cam_m, dist_c, None, cam_m, (w, h), cv2.CV_32FC1) + dewarped_image = cv2.remap(image, map1, map2, cv2.INTER_LINEAR) + + if scale_factor > 1.0: + new_w = int(w * (1.0 / scale_factor)) + new_h = int(h * (1.0 / scale_factor)) + center_x, center_y = w // 2, h // 2 + crop_x1 = max(0, center_x - new_w // 2) + crop_y1 = max(0, center_y - new_h // 2) + crop_x2 = min(w, center_x + new_w // 2) + crop_y2 = min(h, center_y + new_h // 2) + cropped = dewarped_image[crop_y1:crop_y2, crop_x1:crop_x2] + dewarped_image = cv2.resize(cropped, (w, h), interpolation=cv2.INTER_LANCZOS4) + imwrite(dewarped_path, dewarped_image) + log.ODM_INFO("Dewarped %s --> %s" % (p.filename, dewarped_filename)) + p.filename = dewarped_filename \ No newline at end of file diff --git a/opendm/photo.py b/opendm/photo.py index a7e638be..bf148cb0 100644 --- a/opendm/photo.py +++ b/opendm/photo.py @@ -159,6 +159,9 @@ class ODM_Photo: self.gps_xy_stddev = None # Dilution of Precision X/Y self.gps_z_stddev = None # Dilution of Precision Z + # DJI + self.dewarp_data = None + # Misc SFM self.camera_projection = 'brown' self.focal_ratio = 0.85 @@ -413,6 +416,12 @@ class ODM_Photo: self.set_attr_from_xmp_tag('speed_z', xtags, [ '@drone-dji:FlightZSpeed', ], float) + + # DJI dewarp data + if '@drone-dji:DewarpData' in xtags: + self.set_attr_from_xmp_tag('dewarp_data', xtags, [ + '@drone-dji:DewarpData' + ]) # Account for over-estimation if self.gps_xy_stddev is not None: diff --git a/opendm/types.py b/opendm/types.py index 3e437c0f..82501ec5 100644 --- a/opendm/types.py +++ b/opendm/types.py @@ -22,12 +22,13 @@ from opendm.photo import ODM_Photo warnings.filterwarnings("ignore") class ODM_Reconstruction(object): - def __init__(self, photos): + def __init__(self, photos, images_path): self.photos = photos self.georef = None self.gcp = None self.multi_camera = self.detect_multi_camera() self.filter_photos() + self.dewarp_photos(images_path) def detect_multi_camera(self): """ @@ -148,13 +149,7 @@ class ODM_Reconstruction(object): if 'rgb' in bands or 'redgreenblue' in bands: if 'red' in bands and 'green' in bands and 'blue' in bands: - bands_to_remove.append(bands['rgb'] if 'rgb' in bands else bands['redgreenblue']) - - # Mavic 3M's RGB camera lens are too different than the multispectral ones - # so we drop the RGB channel instead - elif self.photos[0].is_make_model("DJI", "M3M") and 'red' in bands and 'green' in bands: - bands_to_remove.append(bands['rgb'] if 'rgb' in bands else bands['redgreenblue']) - + bands_to_remove.append(bands['rgb'] if 'rgb' in bands else bands['redgreenblue']) else: for b in ['red', 'green', 'blue']: if b in bands: @@ -285,6 +280,23 @@ class ODM_Reconstruction(object): if p.filename == filename: return p + def dewarp_photos(self, images_path): + if not self.multi_camera: + return # Nothing to do + else: + bands = {} + for b in self.multi_camera: + bands[b['name'].lower()] = b['photos'] + + # Mavic 3M's RGB camera lens are too different than the multispectral ones + # so we unwarp them before reconstruction when needed + if self.photos[0].is_make_model("DJI", "M3M") and 'nir' in bands and ('rgb' in bands or 'redgreenblue' in bands): + log.ODM_INFO("Dewarping RGB photos before processing") + rgb = 'rgb' if 'rgb' in bands else 'redgreenblue' + upscale = max(1.0, bands['nir'][0].focal_ratio / bands[rgb][0].focal_ratio) + if upscale != 1.0: + log.ODM_INFO("Adjusting focal distance of RGB images by %s" % upscale) + multispectral.dewarp_photos(bands['rgb'] if 'rgb' in bands else bands['redgreenblue'], images_path, upscale) class ODM_GeoRef(object): @staticmethod diff --git a/stages/dataset.py b/stages/dataset.py index 4e5eb360..ebcfb95f 100644 --- a/stages/dataset.py +++ b/stages/dataset.py @@ -61,7 +61,7 @@ class ODMLoadDatasetStage(types.ODM_Stage): def valid_filename(filename, supported_extensions): (pathfn, ext) = os.path.splitext(filename) - return ext.lower() in supported_extensions and pathfn[-5:] != "_mask" + return ext.lower() in supported_extensions and pathfn[-5:] != "_mask" and pathfn[-9:] != "_dewarped" # Get supported images from dir def get_images(in_dir): @@ -162,6 +162,10 @@ class ODMLoadDatasetStage(types.ODM_Stage): (p, ext) = os.path.splitext(r) if p[-5:] == "_mask" and ext.lower() in context.supported_extensions: masks[p] = r + + # Remove dewarped images on re-run + if p[-9:] == "_dewarped" and self.rerun(): + os.unlink(os.path.join(images_dir, p + ext)) photos = [] with open(tree.dataset_list, 'w') as dataset_list: @@ -298,7 +302,7 @@ class ODMLoadDatasetStage(types.ODM_Stage): log.logger.log_json_images(len(photos)) # Create reconstruction object - reconstruction = types.ODM_Reconstruction(photos) + reconstruction = types.ODM_Reconstruction(photos, tree.dataset_raw) if tree.odm_georeferencing_gcp and not args.use_exif: reconstruction.georeference_with_gcp(tree.odm_georeferencing_gcp,