From e819dab16b3ea07e7575a1fbd4602e49b1f6a6bd Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 2 Dec 2020 17:17:51 -0500 Subject: [PATCH] Band alignment PoC working --- opendm/config.py | 2 +- opendm/geo.py | 2 +- opendm/multispectral.py | 176 ++++++++++++++++++++++++++++++++++++++-- opendm/osfm.py | 10 +-- stages/run_opensfm.py | 31 ++++++- 5 files changed, 204 insertions(+), 17 deletions(-) diff --git a/opendm/config.py b/opendm/config.py index d78ed23e..031b3a62 100755 --- a/opendm/config.py +++ b/opendm/config.py @@ -157,7 +157,7 @@ def config(argv=None, parser=None): 'Can be one of: %(choices)s. Default: ' '%(default)s')) - parser.add_argument('--feature-matcher', + parser.add_argument('--matcher-type', metavar='', action=StoreValue, default='flann', diff --git a/opendm/geo.py b/opendm/geo.py index a1dd50b6..85c007b5 100644 --- a/opendm/geo.py +++ b/opendm/geo.py @@ -50,7 +50,7 @@ class GeoFile: horizontal_accuracy, vertical_accuracy, extras) else: - logger.warning("Malformed geo line: %s" % line) + log.ODM_WARNING("Malformed geo line: %s" % line) def get_entry(self, filename): return self.entries.get(filename) diff --git a/opendm/multispectral.py b/opendm/multispectral.py index 8110a3a7..9a5f97cf 100644 --- a/opendm/multispectral.py +++ b/opendm/multispectral.py @@ -1,7 +1,10 @@ -from opendm import dls import math +import re +import cv2 +from opendm import dls import numpy as np from opendm import log +from opensfm.io import imread # Loosely based on https://github.com/micasense/imageprocessing/blob/master/micasense/utils.py @@ -152,17 +155,174 @@ def compute_irradiance(photo, use_sun_sensor=True): return 1.0 -def get_photos_by_band(multi_camera, band_name): +def get_photos_by_band(multi_camera, user_band_name): + band_name = get_primary_band_name(multi_camera, user_band_name) + + for band in multi_camera: + if band['name'] == band_name: + return band['photos'] + + +def get_primary_band_name(multi_camera, user_band_name): if len(multi_camera) < 1: raise Exception("Invalid multi_camera list") # multi_camera is already sorted by band_index - if band_name == "auto": - return multi_camera[0]['photos'] + if user_band_name == "auto": + return multi_camera[0]['name'] for band in multi_camera: - if band['name'].lower() == band_name.lower(): - return band['photos'] + if band['name'].lower() == user_band_name.lower(): + return band['name'] + + band_name_fallback = multi_camera[0]['name'] - logger.ODM_WARNING("Cannot find band name \"%s\", will use \"auto\" instead" % band_name) - return multi_camera[0]['photos'] + log.ODM_WARNING("Cannot find band name \"%s\", will use \"%s\" instead" % (user_band_name, band_name_fallback)) + return band_name_fallback + + +def compute_primary_band_map(multi_camera, primary_band): + """ + Computes a map of { photo filename --> associated primary band photo } + by looking at capture time or filenames as a fallback + """ + band_name = get_primary_band_name(multi_camera, primary_band) + primary_band_photos = None + for band in multi_camera: + if band['name'] == band_name: + primary_band_photos = band['photos'] + break + + # Try using capture time as the grouping factor + try: + capture_time_map = {} + result = {} + + for p in primary_band_photos: + t = p.get_utc_time() + if t is None: + raise Exception("Cannot use capture time (no information in %s)" % p.filename) + + # Should be unique across primary band + if capture_time_map.get(t) is not None: + raise Exception("Unreliable capture time detected (duplicate)") + + capture_time_map[t] = p + + for band in multi_camera: + photos = band['photos'] + + for p in photos: + t = p.get_utc_time() + if t is None: + raise Exception("Cannot use capture time (no information in %s)" % p.filename) + + # Should match the primary band + if capture_time_map.get(t) is None: + raise Exception("Unreliable capture time detected (no primary band match)") + + result[p.filename] = capture_time_map[t] + + return result + except Exception as e: + # Fallback on filename conventions + log.ODM_WARNING("%s, will use filenames instead" % str(e)) + + filename_map = {} + result = {} + file_regex = re.compile(r"^(.+)[-_]\w+(\.[A-Za-z]{3,4})$") + + for p in primary_band_photos: + filename_without_band = re.sub(file_regex, "\\1\\2", p.filename) + + # Quick check + if filename_without_band == p.filename: + raise Exception("Cannot match bands by filename on %s, make sure to name your files [filename]_band[.ext] uniformly." % p.filename) + + filename_map[filename_without_band] = p + + for band in multi_camera: + photos = band['photos'] + + for p in photos: + filename_without_band = re.sub(file_regex, "\\1\\2", p.filename) + + # Quick check + if filename_without_band == p.filename: + raise Exception("Cannot match bands by filename on %s, make sure to name your files [filename]_band[.ext] uniformly." % p.filename) + + result[p.filename] = filename_map[filename_without_band] + + return result + +def compute_aligned_image(image, align_image_filename, feature_retention=0.15): + if len(image.shape) != 3: + raise ValueError("Image should have shape length of 3 (got: %s)" % len(image.shape)) + + # Convert images to grayscale if needed + if image.shape[2] == 3: + image_gray = to_8bit(cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)) + else: + image_gray = to_8bit(image[:,:,0]) + + align_image = imread(align_image_filename, unchanged=True, anydepth=True) + if align_image.shape[2] == 3: + align_image_gray = to_8bit(cv2.cvtColor(align_image, cv2.COLOR_BGR2GRAY)) + else: + align_image_gray = to_8bit(align_image[:,:,0]) + + # Detect SIFT features and compute descriptors. + detector = cv2.SIFT_create(edgeThreshold=10, contrastThreshold=0.1) + kp_image, desc_image = detector.detectAndCompute(image_gray, None) + kp_align_image, desc_align_image = detector.detectAndCompute(align_image_gray, None) + + # Match + bf = cv2.BFMatcher(cv2.NORM_L1,crossCheck=True) + matches = bf.match(desc_image, desc_align_image) + + # Sort by score + matches.sort(key=lambda x: x.distance, reverse=False) + + # Remove bad matches + num_good_matches = int(len(matches) * feature_retention) + matches = matches[:num_good_matches] + + # Debug + # imMatches = cv2.drawMatches(im1, kp_image, im2, kp_align_image, matches, None) + # cv2.imwrite("matches.jpg", imMatches) + + # Extract location of good matches + points_image = np.zeros((len(matches), 2), dtype=np.float32) + points_align_image = np.zeros((len(matches), 2), dtype=np.float32) + + for i, match in enumerate(matches): + points_image[i, :] = kp_image[match.queryIdx].pt + points_align_image[i, :] = kp_align_image[match.trainIdx].pt + + # Find homography + h, _ = cv2.findHomography(points_image, points_align_image, cv2.RANSAC) + + # Use homography + height, width = align_image_gray.shape + aligned_image = cv2.warpPerspective(image, h, (width, height)) + + return aligned_image + +def to_8bit(image): + if image.dtype == np.uint8: + return image + + # Convert to 8bit + try: + data_range = np.iinfo(image.dtype) + value_range = float(data_range.max) - float(data_range.min) + except ValueError: + # For floats use the actual range of the image values + value_range = float(image.max()) - float(image.min()) + + image = image.astype(np.float32) + image *= 255.0 / value_range + np.around(image, out=image) + image = image.astype(np.uint8) + + return image \ No newline at end of file diff --git a/opendm/osfm.py b/opendm/osfm.py index 55e45d94..e451ee9b 100644 --- a/opendm/osfm.py +++ b/opendm/osfm.py @@ -56,7 +56,7 @@ class OSFMContext: exit(1) - def setup(self, args, images_path, photos, reconstruction, append_config = [], rerun=False): + def setup(self, args, images_path, reconstruction, append_config = [], rerun=False): """ Setup a OpenSfM project """ @@ -68,12 +68,12 @@ class OSFMContext: list_path = os.path.join(self.opensfm_project_path, 'image_list.txt') if not io.file_exists(list_path) or rerun: - + if reconstruction.multi_camera: photos = get_photos_by_band(reconstruction.multi_camera, args.primary_band) if len(photos) < 1: raise Exception("Not enough images in selected band %s" % args.primary_band.lower()) - logger.ODM_INFO("Reconstruction will use %s images from %s band" % (len(photos), args.primary_band.lower())) + log.ODM_INFO("Reconstruction will use %s images from %s band" % (len(photos), args.primary_band.lower())) else: photos = reconstruction.photos @@ -105,7 +105,7 @@ class OSFMContext: except Exception as e: log.ODM_WARNING("Cannot set camera_models_overrides.json: %s" % str(e)) - use_bow = args.feature_matcher == "bow" + use_bow = args.matcher_type == "bow" feature_type = "SIFT" # GPSDOP override if we have GPS accuracy information (such as RTK) @@ -181,7 +181,7 @@ class OSFMContext: "feature_process_size: %s" % feature_process_size, "feature_min_frames: %s" % args.min_num_features, "processes: %s" % args.max_concurrency, - "matching_gps_neighbors: %s" % matcher_neighbors, + "matching_gps_neighbors: %s" % args.matcher_neighbors, "matching_gps_distance: %s" % args.matcher_distance, "depthmap_method: %s" % args.opensfm_depthmap_method, "depthmap_resolution: %s" % depthmap_resolution, diff --git a/stages/run_opensfm.py b/stages/run_opensfm.py index a47feba7..227f05a9 100644 --- a/stages/run_opensfm.py +++ b/stages/run_opensfm.py @@ -62,6 +62,8 @@ class ODMOpenSfMStage(types.ODM_Stage): octx.touch(updated_config_flag_file) # Undistorted images will be used for texturing / MVS + + primary_band_map = None undistort_pipeline = [] def undistort_callback(shot_id, image): @@ -73,14 +75,37 @@ class ODMOpenSfMStage(types.ODM_Stage): photo = reconstruction.get_photo(shot_id) return multispectral.dn_to_reflectance(photo, image, use_sun_sensor=args.radiometric_calibration=="camera+sun") + def align_to_primary_band(shot_id, image): - # TODO + primary_band_photo = primary_band_map.get(shot_id) + + try: + if primary_band_photo is None: + raise Exception("Cannot find primary band file for %s" % shot_id) + + if primary_band_photo.filename == shot_id: + # This is a photo from the primary band, skip + return image + + align_image_filename = os.path.join(tree.dataset_raw, primary_band_map[shot_id].filename) + image = multispectral.compute_aligned_image(image, align_image_filename) + except Exception as e: + log.ODM_WARNING("Cannot align %s: %s. Output quality might be affected." % (shot_id, str(e))) + return image - if args.radiometric_calibration != "none" + if args.radiometric_calibration != "none": undistort_pipeline.append(radiometric_calibrate) if reconstruction.multi_camera: + primary_band_map = multispectral.compute_primary_band_map(reconstruction.multi_camera, args.primary_band) + + # TODO: + # if (not flag_file exists or rerun) + # Change reconstruction.json and tracks.csv by adding copies of the primary + # band camera to the reconstruction + + undistort_pipeline.append(align_to_primary_band) octx.convert_and_undistort(self.rerun(), undistort_callback) @@ -102,6 +127,8 @@ class ODMOpenSfMStage(types.ODM_Stage): log.ODM_WARNING("Found a valid image list in %s for %s band" % (image_list_file, band['name'])) # TODO!! CHANGE + print("TODO!") + exit(1) nvm_file = octx.path("undistorted", "reconstruction_%s.nvm" % band['name'].lower()) if not io.file_exists(nvm_file) or self.rerun(): octx.run('export_visualsfm --points --image_list "%s"' % image_list_file)