GCP EPSG code support, fixes

opendm/gcp.py

@@ -18,7 +18,7 @@ class GCPFile:
     def read(self):
         if self.exists():
             with open(self.gcp_path, 'r') as f:
-                contents = f.read().strip()
+                contents = f.read().decode('utf-8-sig').encode('utf-8').strip()
     
             lines = map(str.strip, contents.split('\n'))
             if lines:
@@ -33,12 +33,9 @@ class GCPFile:
                         else:
                             log.ODM_WARNING("Malformed GCP line: %s" % line)
 
-    def iter_entries(self, allowed_filenames=None):
+    def iter_entries(self):
         for entry in self.entries:
-            pe = self.parse_entry(entry)
+            yield self.parse_entry(entry)
-
-            if allowed_filenames is None or pe.filename in allowed_filenames:
-                yield pe
     
     def parse_entry(self, entry):
         if entry:
@@ -69,11 +66,12 @@ class GCPFile:
             utm_zone, hemisphere = location.get_utm_zone_and_hemisphere_from(lon, lat)
             return "WGS84 UTM %s%s" % (utm_zone, hemisphere)
 
-    def create_utm_copy(self, gcp_file_output, filenames=None):
+    def create_utm_copy(self, gcp_file_output, filenames=None, rejected_entries=None):
         """
         Creates a new GCP file from an existing GCP file
         by optionally including only filenames and reprojecting each point to 
-        a UTM CRS
+        a UTM CRS. Rejected entries can recorded by passing a list object to 
+        rejected_entries.
         """
         if os.path.exists(gcp_file_output):
             os.remove(gcp_file_output)
@@ -82,9 +80,12 @@ class GCPFile:
         target_srs = location.parse_srs_header(output[0])
         transformer = location.transformer(self.srs, target_srs)
 
-        for entry in self.iter_entries(filenames):
+        for entry in self.iter_entries():
-            entry.x, entry.y, entry.z = transformer.TransformPoint(entry.x, entry.y, entry.z)
+            if filenames is None or entry.filename in filenames:
-            output.append(str(entry))
+                entry.x, entry.y, entry.z = transformer.TransformPoint(entry.x, entry.y, entry.z)
+                output.append(str(entry))
+            elif isinstance(rejected_entries, list):
+                rejected_entries.append(entry)
 
         with open(gcp_file_output, 'w') as f:
             f.write('\n'.join(output) + '\n')

opendm/osfm.py

@@ -58,7 +58,7 @@ class OSFMContext:
             raise Exception("Reconstruction could not be generated")
 
 
-    def setup(self, args, images_path, photos, gcp_path=None, append_config = [], rerun=False): #georeferenced=True, 
+    def setup(self, args, images_path, photos, gcp_path=None, append_config = [], rerun=False):
         """
         Setup a OpenSfM project
         """
@@ -109,6 +109,8 @@ class OSFMContext:
                 "bundle_outlier_filtering_type: AUTO",
             ]
 
+            # TODO: add BOW matching when dataset is not georeferenced (no gps)
+
             if has_alt:
                 log.ODM_DEBUG("Altitude data detected, enabling it for GPS alignment")
                 config.append("use_altitude_tag: yes")
@@ -117,10 +119,6 @@ class OSFMContext:
                 config.append("align_method: orientation_prior")
                 config.append("align_orientation_prior: vertical")
             
-            # if not georeferenced:
-            #     config.append("")
-
-
             if args.use_hybrid_bundle_adjustment:
                 log.ODM_DEBUG("Enabling hybrid bundle adjustment")
                 config.append("bundle_interval: 100")          # Bundle after adding 'bundle_interval' cameras

opendm/types.py

@@ -92,31 +92,53 @@ class ODM_Reconstruction(object):
     def __init__(self, photos):
         self.photos = photos
         self.georef = None
+        self.gcp = None
 
     def is_georeferenced(self):
         return self.georef is not None
 
-    def georeference_with_gcp(self, gcp_file, output_coords_file, reload_coords=False):
+    def georeference_with_gcp(self, gcp_file, output_coords_file, output_gcp_file, rerun=False):
-        if not io.file_exists(output_coords_file) or reload_coords:
+        if not io.file_exists(output_coords_file) or not io.file_exists(output_gcp_file) or rerun:
             gcp = GCPFile(gcp_file)
             if gcp.exists():
-                # Create coords file
+                # Create coords file, we'll be using this later
+                # during georeferencing
                 with open(output_coords_file, 'w') as f:
                     coords_header = gcp.wgs84_utm_zone()
                     f.write(coords_header + "\n")
                     log.ODM_DEBUG("Generated coords file from GCP: %s" % coords_header)
+
+                # Convert GCP file to a UTM projection since the rest of the pipeline
+                # does not handle other SRS well.
+                rejected_entries = []
+                utm_gcp = GCPFile(gcp.create_utm_copy(output_gcp_file, filenames=[p.filename for p in self.photos], rejected_entries=rejected_entries))
+                
+                if not utm_gcp.exists():
+                    raise RuntimeError("Could not project GCP file to UTM. Please double check your GCP file for mistakes.")
+                
+                for re in rejected_entries:
+                    log.ODM_WARNING("GCP line ignored (image not found): %s" % str(re))
+                
+                if utm_gcp.entries_count() > 0:
+                    log.ODM_INFO("%s GCP points will be used for georeferencing" % utm_gcp.entries_count())
+                else:
+                    raise RuntimeError("A GCP file was provided, but no valid GCP entries could be used. Note that the GCP file is case sensitive (\".JPG\" is not the same as \".jpg\").")
+
+                self.gcp = utm_gcp
             else:
                 log.ODM_WARNING("GCP file does not exist: %s" % gcp_file)
                 return
         else:
             log.ODM_INFO("Coordinates file already exist: %s" % output_coords_file)
+            log.ODM_INFO("GCP file already exist: %s" % output_gcp_file)
+            self.gcp = GCPFile(output_gcp_file)
         
         self.georef = ODM_GeoRef.FromCoordsFile(output_coords_file)
         return self.georef
 
-    def georeference_with_gps(self, images_path, output_coords_file, reload_coords=False):
+    def georeference_with_gps(self, images_path, output_coords_file, rerun=False):
         try:
-            if not io.file_exists(output_coords_file) or reload_coords:
+            if not io.file_exists(output_coords_file) or rerun:
                 location.extract_utm_coords(photos, tree.dataset_raw, output_coords_file)
             else:
                 log.ODM_INFO("Coordinates file already exist: %s" % output_coords_file)
@@ -267,6 +289,7 @@ class ODM_Tree(object):
         self.odm_georeferencing_coords = io.join_paths(
             self.odm_georeferencing, 'coords.txt')
         self.odm_georeferencing_gcp = gcp_file or io.find('gcp_list.txt', self.root_path)
+        self.odm_georeferencing_gcp_utm = io.join_paths(self.odm_georeferencing, 'gcp_list_utm.txt')
         self.odm_georeferencing_utm_log = io.join_paths(
             self.odm_georeferencing, 'odm_georeferencing_utm_log.txt')
         self.odm_georeferencing_log = 'odm_georeferencing_log.txt'

stages/dataset.py

@@ -103,9 +103,14 @@ class ODMLoadDatasetStage(types.ODM_Stage):
         reconstruction = types.ODM_Reconstruction(photos)
         
         if tree.odm_georeferencing_gcp:
-            reconstruction.georeference_with_gcp(tree.odm_georeferencing_gcp, tree.odm_georeferencing_coords, reload_coords=self.rerun())
+            reconstruction.georeference_with_gcp(tree.odm_georeferencing_gcp,
+                                                 tree.odm_georeferencing_coords,
+                                                 tree.odm_georeferencing_gcp_utm,
+                                                 rerun=self.rerun())
         else:
-            reconstruction.georeference_with_gps(tree.dataset_raw, tree.odm_georeferencing_coords, reload_coords=self.rerun())
+            reconstruction.georeference_with_gps(tree.dataset_raw, 
+                                                 tree.odm_georeferencing_coords, 
+                                                 rerun=self.rerun())
 
         reconstruction.save_proj_srs(io.join_paths(tree.odm_georeferencing, tree.odm_georeferencing_proj))
         outputs['reconstruction'] = reconstruction

stages/odm_app.py

@@ -26,8 +26,7 @@ class ODMApp:
         """
         
         dataset = ODMLoadDatasetStage('dataset', args, progress=5.0,
-                                          verbose=args.verbose,
+                                          verbose=args.verbose)
-                                          proj=args.proj)
         split = ODMSplitStage('split', args, progress=75.0)
         merge = ODMMergeStage('merge', args, progress=100.0)
         opensfm = ODMOpenSfMStage('opensfm', args, progress=25.0)
@@ -59,36 +58,34 @@ class ODMApp:
                             verbose=args.verbose)
         orthophoto = ODMOrthoPhotoStage('odm_orthophoto', args, progress=100.0)
 
-        if not args.video:
+        # Normal pipeline
-            # Normal pipeline
+        self.first_stage = dataset
-            self.first_stage = dataset
 
-            dataset.connect(split) \
+        dataset.connect(split) \
-                   .connect(merge) \
+                .connect(merge) \
-                   .connect(opensfm)
+                .connect(opensfm)
-
-            if args.use_opensfm_dense or args.fast_orthophoto:
-                opensfm.connect(filterpoints)
-            else:
-                opensfm.connect(mve) \
-                       .connect(filterpoints)
-            
-            filterpoints \
-                .connect(meshing) \
-                .connect(texturing) \
-                .connect(georeferencing) \
-                .connect(dem) \
-                .connect(orthophoto)
 
+        if args.use_opensfm_dense or args.fast_orthophoto:
+            opensfm.connect(filterpoints)
         else:
-            # SLAM pipeline
+            opensfm.connect(mve) \
-            # TODO: this is broken and needs work
+                    .connect(filterpoints)
-            log.ODM_WARNING("SLAM module is currently broken. We could use some help fixing this. If you know Python, get in touch at https://community.opendronemap.org.")
-            self.first_stage = slam
         
-            slam.connect(mve) \
+        filterpoints \
-                .connect(meshing) \
+            .connect(meshing) \
-                .connect(texturing)
+            .connect(texturing) \
+            .connect(georeferencing) \
+            .connect(dem) \
+            .connect(orthophoto)
+                
+        # # SLAM pipeline
+        # # TODO: this is broken and needs work
+        # log.ODM_WARNING("SLAM module is currently broken. We could use some help fixing this. If you know Python, get in touch at https://community.opendronemap.org.")
+        # self.first_stage = slam
+
+        # slam.connect(mve) \
+        #     .connect(meshing) \
+        #     .connect(texturing)
 
     def execute(self):
         self.first_stage.run()

stages/odm_georeferencing.py

@@ -16,7 +16,6 @@ class ODMGeoreferencingStage(types.ODM_Stage):
         tree = outputs['tree']
         reconstruction = outputs['reconstruction']
 
-        gcpfile = tree.odm_georeferencing_gcp
         doPointCloudGeo = True
         transformPointCloud = True
         verbose = '-verbose' if self.params.get('verbose') else ''
@@ -65,7 +64,6 @@ class ODMGeoreferencingStage(types.ODM_Stage):
                     'output_pc_file': tree.odm_georeferencing_model_laz,
                     'geo_sys': odm_georeferencing_model_txt_geo_file,
                     'model_geo': odm_georeferencing_model_obj_geo,
-                    'gcp': gcpfile,
                     'verbose': verbose
                 }
 

stages/run_opensfm.py

@@ -21,7 +21,7 @@ class ODMOpenSfMStage(types.ODM_Stage):
             exit(1)
 
         octx = OSFMContext(tree.opensfm)
-        octx.setup(args, tree.dataset_raw, photos, gcp_path=tree.odm_georeferencing_gcp, rerun=self.rerun())
+        octx.setup(args, tree.dataset_raw, photos, gcp_path=reconstruction.gcp.gcp_path, rerun=self.rerun())
         octx.extract_metadata(self.rerun())
         self.update_progress(20)
         octx.feature_matching(self.rerun())

stages/splitmerge.py

@@ -50,7 +50,7 @@ class ODMSplitStage(types.ODM_Stage):
                     "submodel_overlap: %s" % args.split_overlap,
                 ]
 
-                octx.setup(args, tree.dataset_raw, photos, gcp_path=tree.odm_georeferencing_gcp, append_config=config, rerun=self.rerun())
+                octx.setup(args, tree.dataset_raw, photos, gcp_path=reconstruction.gcp.gcp_path, append_config=config, rerun=self.rerun())
                 octx.extract_metadata(self.rerun())
 
                 self.update_progress(5)
@@ -74,18 +74,16 @@ class ODMSplitStage(types.ODM_Stage):
                 mds = metadataset.MetaDataSet(tree.opensfm)
                 submodel_paths = [os.path.abspath(p) for p in mds.get_submodel_paths()]
 
-                gcp_file = GCPFile(tree.odm_georeferencing_gcp)
-
                 for sp in submodel_paths:
                     sp_octx = OSFMContext(sp)
 
                     # Copy filtered GCP file if needed
                     # One in OpenSfM's directory, one in the submodel project directory
-                    if gcp_file.exists():
+                    if reconstruction.gcp and reconstruction.gcp.exists():
                         submodel_gcp_file = os.path.abspath(sp_octx.path("..", "gcp_list.txt"))
                         submodel_images_dir = os.path.abspath(sp_octx.path("..", "images"))
 
-                        if gcp_file.make_filtered_copy(submodel_gcp_file, submodel_images_dir):
+                        if reconstruction.gcp.make_filtered_copy(submodel_gcp_file, submodel_images_dir):
                             log.ODM_DEBUG("Copied filtered GCP file to %s" % submodel_gcp_file)
                             io.copy(submodel_gcp_file, os.path.abspath(sp_octx.path("gcp_list.txt")))
                         else: