From 1a0711671ffde47d46f49ac1eaafd06ac3585e45 Mon Sep 17 00:00:00 2001 From: usplm Date: Tue, 14 Jun 2022 13:03:39 -0400 Subject: [PATCH 01/16] Use ECC algorithm only for low resolution images MC-909 --- opendm/multispectral.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/opendm/multispectral.py b/opendm/multispectral.py index 6a14c3b5..5398579a 100644 --- a/opendm/multispectral.py +++ b/opendm/multispectral.py @@ -381,12 +381,24 @@ def compute_homography(image_filename, align_image_filename): return h, (align_image_gray.shape[1], align_image_gray.shape[0]) - algo = 'feat' - result = compute_using(find_features_homography) + warp_matrix = None + dimension = None + algo = None - if result[0] is None: + if max_dim > 320: + algo = 'feat' + result = compute_using(find_features_homography) + + if result[0] is None: + algo = 'ecc' + log.ODM_INFO("Can't use features matching, will use ECC (this might take a bit)") + result = compute_using(find_ecc_homography) + if result[0] is None: + algo = None + + else: # ECC only for low resolution images algo = 'ecc' - log.ODM_INFO("Can't use features matching, will use ECC (this might take a bit)") + log.ODM_INFO("Using ECC (this might take a bit)") result = compute_using(find_ecc_homography) if result[0] is None: algo = None @@ -396,7 +408,7 @@ def compute_homography(image_filename, align_image_filename): except Exception as e: log.ODM_WARNING("Compute homography: %s" % str(e)) - return None, None, (None, None) + return None, (None, None), None def find_ecc_homography(image_gray, align_image_gray, number_of_iterations=1000, termination_eps=1e-8, start_eps=1e-4): pyramid_levels = 0 From d640e0dfd93e33790c18c0bfc6c4ecdeb93ff870 Mon Sep 17 00:00:00 2001 From: usplm Date: Wed, 15 Jun 2022 03:31:42 -0400 Subject: [PATCH 02/16] Removed thermal temperature conversion --- opendm/multispectral.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/opendm/multispectral.py b/opendm/multispectral.py index 5398579a..6f0b1c39 100644 --- a/opendm/multispectral.py +++ b/opendm/multispectral.py @@ -25,12 +25,6 @@ def dn_to_radiance(photo, image): image = image.astype("float32") if len(image.shape) != 3: raise ValueError("Image should have shape length of 3 (got: %s)" % len(image.shape)) - - # Handle thermal bands (experimental) - if photo.band_name == 'LWIR': - image -= (273.15 * 100.0) # Convert Kelvin to Celsius - image *= 0.01 - return image # All others a1, a2, a3 = photo.get_radiometric_calibration() From 7ec2434072b1e9c9a5f2c28a1d1674670624f6d1 Mon Sep 17 00:00:00 2001 From: usplm Date: Wed, 15 Jun 2022 03:34:50 -0400 Subject: [PATCH 03/16] Return thermal band without radiance calibration --- opendm/multispectral.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/opendm/multispectral.py b/opendm/multispectral.py index 6f0b1c39..a2ee0ce5 100644 --- a/opendm/multispectral.py +++ b/opendm/multispectral.py @@ -26,6 +26,10 @@ def dn_to_radiance(photo, image): if len(image.shape) != 3: raise ValueError("Image should have shape length of 3 (got: %s)" % len(image.shape)) + # Thermal (this should never happen, but just in case..) + if photo.is_thermal(): + return image + # All others a1, a2, a3 = photo.get_radiometric_calibration() dark_level = photo.get_dark_level() From cca10a82ac4edf1cfbc2d75733e9707f8de07037 Mon Sep 17 00:00:00 2001 From: usplm Date: Fri, 17 Jun 2022 13:39:16 -0400 Subject: [PATCH 04/16] Read Micasense capture id metadata --- opendm/photo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/opendm/photo.py b/opendm/photo.py index 333e3d6d..3f8d31e2 100644 --- a/opendm/photo.py +++ b/opendm/photo.py @@ -339,6 +339,7 @@ class ODM_Photo: self.set_attr_from_xmp_tag('capture_uuid', xtags, [ '@drone-dji:CaptureUUID', # DJI + 'MicaSense:CaptureId', # MicaSense Altum '@Camera:ImageUniqueID', # sentera 6x ]) From d7ae81095879f967e06fa9361bf6e2d11faa51c6 Mon Sep 17 00:00:00 2001 From: usplm Date: Sun, 26 Jun 2022 16:42:13 -0400 Subject: [PATCH 05/16] Use Lowe's ration test to filter good matches --- opendm/multispectral.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/opendm/multispectral.py b/opendm/multispectral.py index a2ee0ce5..8b5e074a 100644 --- a/opendm/multispectral.py +++ b/opendm/multispectral.py @@ -472,7 +472,9 @@ def find_ecc_homography(image_gray, align_image_gray, number_of_iterations=1000, return warp_matrix -def find_features_homography(image_gray, align_image_gray, feature_retention=0.25): +def find_features_homography(image_gray, align_image_gray, feature_retention=0.7): + min_match_count = 10 + # Detect SIFT features and compute descriptors. detector = cv2.SIFT_create(edgeThreshold=10, contrastThreshold=0.1) kp_image, desc_image = detector.detectAndCompute(image_gray, None) @@ -487,13 +489,21 @@ def find_features_homography(image_gray, align_image_gray, feature_retention=0.2 return None # Sort by score - matches.sort(key=lambda x: x.distance, reverse=False) + # matches.sort(key=lambda x: x.distance, reverse=False) # Remove bad matches - num_good_matches = int(len(matches) * feature_retention) - matches = matches[:num_good_matches] + # num_good_matches = int(len(matches) * feature_retention) + # matches = matches[:num_good_matches] - if len(matches) < 4: + # Filter good matches following Lowe's ratio test + good_matches = [] + for m, n in matches: + if m.distance < feature_retention * n.distance: + good_matches.append(m) + + matches = good_matches + + if len(matches) < min_match_count: log.ODM_INFO("Insufficient features: %s" % len(matches)) return None From c35ab4480be4929da81a69d54413ab2a014bc1cf Mon Sep 17 00:00:00 2001 From: usplm Date: Sun, 26 Jun 2022 22:15:24 -0400 Subject: [PATCH 06/16] Use FLANN matcher instead of brute force --- opendm/multispectral.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/opendm/multispectral.py b/opendm/multispectral.py index 8b5e074a..ea33f624 100644 --- a/opendm/multispectral.py +++ b/opendm/multispectral.py @@ -481,12 +481,12 @@ def find_features_homography(image_gray, align_image_gray, feature_retention=0.7 kp_align_image, desc_align_image = detector.detectAndCompute(align_image_gray, None) # Match - bf = cv2.BFMatcher(cv2.NORM_L1,crossCheck=True) - try: - matches = bf.match(desc_image, desc_align_image) - except Exception as e: - log.ODM_INFO("Cannot match features") - return None + # bf = cv2.BFMatcher(cv2.NORM_L1,crossCheck=True) + # try: + # matches = bf.match(desc_image, desc_align_image) + # except Exception as e: + # log.ODM_INFO("Cannot match features") + # return None # Sort by score # matches.sort(key=lambda x: x.distance, reverse=False) @@ -495,6 +495,18 @@ def find_features_homography(image_gray, align_image_gray, feature_retention=0.7 # num_good_matches = int(len(matches) * feature_retention) # matches = matches[:num_good_matches] + # Use FLANN based method to match keypoints + FLANN_INDEX_KDTREE = 1 + index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=5) + search_params = dict(checks=50) + + flann = cv2.FlannBasedMatcher(index_params, search_params) + try: + matches = flann.knnMatch(desc_image, desc_align_image, k=2) + except Exception as e: + log.ODM_INFO("Cannot match features") + return None + # Filter good matches following Lowe's ratio test good_matches = [] for m, n in matches: @@ -510,6 +522,7 @@ def find_features_homography(image_gray, align_image_gray, feature_retention=0.7 # Debug # imMatches = cv2.drawMatches(im1, kp_image, im2, kp_align_image, matches, None) # cv2.imwrite("matches.jpg", imMatches) + log.ODM_INFO("Good feature matches: %s" % len(matches)) # Extract location of good matches points_image = np.zeros((len(matches), 2), dtype=np.float32) From 760238b9cde6a19f86f29de87154b02a2750be16 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 7 Jul 2022 02:57:45 -0400 Subject: [PATCH 07/16] Remove comments --- opendm/multispectral.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/opendm/multispectral.py b/opendm/multispectral.py index ea33f624..f948afd8 100644 --- a/opendm/multispectral.py +++ b/opendm/multispectral.py @@ -481,21 +481,6 @@ def find_features_homography(image_gray, align_image_gray, feature_retention=0.7 kp_align_image, desc_align_image = detector.detectAndCompute(align_image_gray, None) # Match - # bf = cv2.BFMatcher(cv2.NORM_L1,crossCheck=True) - # try: - # matches = bf.match(desc_image, desc_align_image) - # except Exception as e: - # log.ODM_INFO("Cannot match features") - # return None - - # Sort by score - # matches.sort(key=lambda x: x.distance, reverse=False) - - # Remove bad matches - # num_good_matches = int(len(matches) * feature_retention) - # matches = matches[:num_good_matches] - - # Use FLANN based method to match keypoints FLANN_INDEX_KDTREE = 1 index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=5) search_params = dict(checks=50) @@ -505,7 +490,7 @@ def find_features_homography(image_gray, align_image_gray, feature_retention=0.7 matches = flann.knnMatch(desc_image, desc_align_image, k=2) except Exception as e: log.ODM_INFO("Cannot match features") - return None + return None # Filter good matches following Lowe's ratio test good_matches = [] From c1b9ff4c8c8940d03fcdaad3a8e0fb521815ce05 Mon Sep 17 00:00:00 2001 From: usplm Date: Mon, 27 Jun 2022 08:08:30 -0400 Subject: [PATCH 08/16] Reduced minimal good matches threshold --- opendm/multispectral.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/opendm/multispectral.py b/opendm/multispectral.py index f948afd8..aed193ff 100644 --- a/opendm/multispectral.py +++ b/opendm/multispectral.py @@ -472,8 +472,7 @@ def find_ecc_homography(image_gray, align_image_gray, number_of_iterations=1000, return warp_matrix -def find_features_homography(image_gray, align_image_gray, feature_retention=0.7): - min_match_count = 10 +def find_features_homography(image_gray, align_image_gray, feature_retention=0.7, min_match_count = 4): # Detect SIFT features and compute descriptors. detector = cv2.SIFT_create(edgeThreshold=10, contrastThreshold=0.1) From da73ada89ba8e9bb772bd534b321b67b29aad3f5 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 7 Jul 2022 02:59:57 -0400 Subject: [PATCH 09/16] Change min_match_count --- opendm/multispectral.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opendm/multispectral.py b/opendm/multispectral.py index aed193ff..e6521cf4 100644 --- a/opendm/multispectral.py +++ b/opendm/multispectral.py @@ -472,7 +472,7 @@ def find_ecc_homography(image_gray, align_image_gray, number_of_iterations=1000, return warp_matrix -def find_features_homography(image_gray, align_image_gray, feature_retention=0.7, min_match_count = 4): +def find_features_homography(image_gray, align_image_gray, feature_retention=0.7, min_match_count=10): # Detect SIFT features and compute descriptors. detector = cv2.SIFT_create(edgeThreshold=10, contrastThreshold=0.1) From ea5a4b4053da4e641b98fcaf794131462f6eff55 Mon Sep 17 00:00:00 2001 From: usplm Date: Wed, 29 Jun 2022 14:15:56 -0400 Subject: [PATCH 10/16] Parse RGB bands to generate PNG image --- opendm/orthophoto.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/opendm/orthophoto.py b/opendm/orthophoto.py index f99197c0..e2d5cdb1 100644 --- a/opendm/orthophoto.py +++ b/opendm/orthophoto.py @@ -46,11 +46,28 @@ def generate_png(orthophoto_file, output_file=None, outsize=None): bandparam = "" gtif = gdal.Open(orthophoto_file) if gtif.RasterCount > 4: - bandparam = "-b 1 -b 2 -b 3 -a_nodata 0" + try: + bands = [] + for idx in range(1, gtif.RasterCount+1): + bands.append(gtif.GetRasterBand(idx).GetColorInterpretation()) + + bands = dict(zip(bands, range(1, len(bands)+1))) + red = bands.get(3) + green = bands.get(4) + blue = bands.get(5) + alpha = bands.get(6) + if alpha is not None: + bandparam = "-b %s -b %s -b %s -b %s -a_nodata 0" % (red, green, blue, alpha) + else: + bandparam = "-b %s -b %s -b %s -a_nodata 0" % (red, green, blue) + except Exception as e: + bandparam = "-b 1 -b 2 -b 3 -a_nodata 0" osparam = "" if outsize is not None: - osparam = "-outsize %s 0" % outsize + osparam = "-outsize %s 0 -ot Byte -scale 0 0.5 0 255" % outsize + else: + osparam = "-outsize %s%% %s%% ot Byte -scale 0 0.5 0 255" % (10, 10) system.run('gdal_translate -of png "%s" "%s" %s %s ' '--config GDAL_CACHEMAX %s%% ' % (orthophoto_file, output_file, osparam, bandparam, get_max_memory())) From 5c82b6578874420066fad917fdfc1b8e7b7105a7 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 7 Jul 2022 03:08:39 -0400 Subject: [PATCH 11/16] Less logging --- opendm/multispectral.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/opendm/multispectral.py b/opendm/multispectral.py index e6521cf4..14f5fe25 100644 --- a/opendm/multispectral.py +++ b/opendm/multispectral.py @@ -488,7 +488,6 @@ def find_features_homography(image_gray, align_image_gray, feature_retention=0.7 try: matches = flann.knnMatch(desc_image, desc_align_image, k=2) except Exception as e: - log.ODM_INFO("Cannot match features") return None # Filter good matches following Lowe's ratio test @@ -500,13 +499,11 @@ def find_features_homography(image_gray, align_image_gray, feature_retention=0.7 matches = good_matches if len(matches) < min_match_count: - log.ODM_INFO("Insufficient features: %s" % len(matches)) return None # Debug # imMatches = cv2.drawMatches(im1, kp_image, im2, kp_align_image, matches, None) # cv2.imwrite("matches.jpg", imMatches) - log.ODM_INFO("Good feature matches: %s" % len(matches)) # Extract location of good matches points_image = np.zeros((len(matches), 2), dtype=np.float32) From d99043ca6b8433759d8c55ad2f2a950e2fe16f05 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 7 Jul 2022 03:21:12 -0400 Subject: [PATCH 12/16] Better resize interpolation --- opendm/multispectral.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/opendm/multispectral.py b/opendm/multispectral.py index 14f5fe25..bc541780 100644 --- a/opendm/multispectral.py +++ b/opendm/multispectral.py @@ -423,10 +423,15 @@ def find_ecc_homography(image_gray, align_image_gray, number_of_iterations=1000, if align_image_gray.shape[0] != image_gray.shape[0]: align_image_gray = to_8bit(align_image_gray) image_gray = to_8bit(image_gray) + + interpolation_mode = cv2.INTER_CUBIC + if image_gray.shape[0] < align_image_gray.shape[0] and image_gray.shape[1] < align_image_gray.shape[1]: + interpolation_mode = cv2.INTER_AREA + image_gray = cv2.resize(image_gray, None, fx=align_image_gray.shape[1]/image_gray.shape[1], fy=align_image_gray.shape[0]/image_gray.shape[0], - interpolation=cv2.INTER_AREA) + interpolation=interpolation_mode) # Build pyramids image_gray_pyr = [image_gray] From 812281a6aeadf7707b7b7bd5985a6c532768245b Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 7 Jul 2022 03:55:53 -0400 Subject: [PATCH 13/16] Fix interpolation --- opendm/multispectral.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/opendm/multispectral.py b/opendm/multispectral.py index bc541780..7d51e882 100644 --- a/opendm/multispectral.py +++ b/opendm/multispectral.py @@ -424,14 +424,13 @@ def find_ecc_homography(image_gray, align_image_gray, number_of_iterations=1000, align_image_gray = to_8bit(align_image_gray) image_gray = to_8bit(image_gray) - interpolation_mode = cv2.INTER_CUBIC - if image_gray.shape[0] < align_image_gray.shape[0] and image_gray.shape[1] < align_image_gray.shape[1]: - interpolation_mode = cv2.INTER_AREA + fx = align_image_gray.shape[1]/image_gray.shape[1] + fy = align_image_gray.shape[0]/image_gray.shape[0] image_gray = cv2.resize(image_gray, None, - fx=align_image_gray.shape[1]/image_gray.shape[1], - fy=align_image_gray.shape[0]/image_gray.shape[0], - interpolation=interpolation_mode) + fx=fx, + fy=fy, + interpolation=(cv2.INTER_AREA if (fx < 1.0 and fy < 1.0) else cv2.INTER_CUBIC)) # Build pyramids image_gray_pyr = [image_gray] From 1789f09387309ff687a44b21ac8ddbd3b844facd Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 7 Jul 2022 04:06:44 -0400 Subject: [PATCH 14/16] Use INTER_LANCZOS4 --- opendm/multispectral.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opendm/multispectral.py b/opendm/multispectral.py index 7d51e882..80e48f05 100644 --- a/opendm/multispectral.py +++ b/opendm/multispectral.py @@ -430,7 +430,7 @@ def find_ecc_homography(image_gray, align_image_gray, number_of_iterations=1000, image_gray = cv2.resize(image_gray, None, fx=fx, fy=fy, - interpolation=(cv2.INTER_AREA if (fx < 1.0 and fy < 1.0) else cv2.INTER_CUBIC)) + interpolation=(cv2.INTER_AREA if (fx < 1.0 and fy < 1.0) else cv2.INTER_LANCZOS4)) # Build pyramids image_gray_pyr = [image_gray] From ba4fa0d555bc4531d5db883bf3c750f5455a9f8d Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 7 Jul 2022 11:15:29 -0400 Subject: [PATCH 15/16] scale the initial warp matrix to smallest level --- opendm/multispectral.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/opendm/multispectral.py b/opendm/multispectral.py index 80e48f05..c10f0264 100644 --- a/opendm/multispectral.py +++ b/opendm/multispectral.py @@ -444,8 +444,9 @@ def find_ecc_homography(image_gray, align_image_gray, number_of_iterations=1000, align_image_pyr.insert(0, cv2.resize(align_image_pyr[0], None, fx=1/2, fy=1/2, interpolation=cv2.INTER_AREA)) - # Define the motion model + # Define the motion model, scale the initial warp matrix to smallest level warp_matrix = np.eye(3, 3, dtype=np.float32) + warp_matrix = warp_matrix * np.array([[1,1,2],[1,1,2],[0.5,0.5,1]], dtype=np.float32)**(1-(pyramid_levels+1)) for level in range(pyramid_levels+1): ig = gradient(gaussian(image_gray_pyr[level])) @@ -467,6 +468,7 @@ def find_ecc_homography(image_gray, align_image_gray, number_of_iterations=1000, if level != pyramid_levels: log.ODM_INFO("Could not compute ECC warp_matrix at pyramid level %s, resetting matrix" % level) warp_matrix = np.eye(3, 3, dtype=np.float32) + warp_matrix = warp_matrix * np.array([[1,1,2],[1,1,2],[0.5,0.5,1]], dtype=np.float32)**(1-(pyramid_levels+1)) else: raise e From c7b6fe52f3f648b57135f8dcbe34d8eeeacb2ea9 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 7 Jul 2022 12:14:59 -0400 Subject: [PATCH 16/16] Revert some PNG generation logic --- opendm/orthophoto.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/opendm/orthophoto.py b/opendm/orthophoto.py index e2d5cdb1..62b103d9 100644 --- a/opendm/orthophoto.py +++ b/opendm/orthophoto.py @@ -44,30 +44,26 @@ def generate_png(orthophoto_file, output_file=None, outsize=None): # See if we need to select top three bands bandparam = "" + gtif = gdal.Open(orthophoto_file) if gtif.RasterCount > 4: - try: - bands = [] - for idx in range(1, gtif.RasterCount+1): - bands.append(gtif.GetRasterBand(idx).GetColorInterpretation()) + bands = [] + for idx in range(1, gtif.RasterCount+1): + bands.append(gtif.GetRasterBand(idx).GetColorInterpretation()) + bands = dict(zip(bands, range(1, len(bands)+1))) - bands = dict(zip(bands, range(1, len(bands)+1))) - red = bands.get(3) - green = bands.get(4) - blue = bands.get(5) - alpha = bands.get(6) - if alpha is not None: - bandparam = "-b %s -b %s -b %s -b %s -a_nodata 0" % (red, green, blue, alpha) - else: - bandparam = "-b %s -b %s -b %s -a_nodata 0" % (red, green, blue) - except Exception as e: + try: + red = bands.get(gdal.GCI_RedBand) + green = bands.get(gdal.GCI_GreenBand) + blue = bands.get(gdal.GCI_BlueBand) + bandparam = "-b %s -b %s -b %s -a_nodata 0" % (red, green, blue) + except: bandparam = "-b 1 -b 2 -b 3 -a_nodata 0" + gtif = None osparam = "" if outsize is not None: - osparam = "-outsize %s 0 -ot Byte -scale 0 0.5 0 255" % outsize - else: - osparam = "-outsize %s%% %s%% ot Byte -scale 0 0.5 0 255" % (10, 10) + osparam = "-outsize %s 0" % outsize system.run('gdal_translate -of png "%s" "%s" %s %s ' '--config GDAL_CACHEMAX %s%% ' % (orthophoto_file, output_file, osparam, bandparam, get_max_memory()))