diff --git a/ccd_defs_check.py b/ccd_defs_check.py deleted file mode 100644 index efacfee6..00000000 --- a/ccd_defs_check.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/python -import sys -import os -import json - -BIN_PATH_ABS = os.path.abspath(os.path.dirname(os.path.abspath(__file__))) - -def get_ccd_widths(): - """Return the CCD Width of the camera listed in the JSON defs file.""" - with open(BIN_PATH_ABS + '/data/ccd_defs.json') as jsonFile: - return json.load(jsonFile) - -try: - ccd_defs = get_ccd_widths() - print "CCD_DEFS compiles OK" - print "Definitions in file: {0}".format(len(ccd_defs)) - exit_code=0 -except IOError as e: - print "I/O error with CCD_DEFS file: {0}".format(e.strerror) - exit_code=255 -except: - print "Error with CCD_DEFS file: {0}".format(sys.exc_info()[1]) - exit_code=255 - -sys.exit(exit_code) diff --git a/opendm/config.py b/opendm/config.py index fe22e650..bc92912f 100644 --- a/opendm/config.py +++ b/opendm/config.py @@ -91,21 +91,10 @@ def config(): metavar='', help='Path to config file for orb-slam') - parser.add_argument('--force-focal', - metavar='', - type=float, - help=('Override the focal length information for the ' - 'images')) - parser.add_argument('--proj', metavar='', help='Projection used to transform the model into geographic coordinates') - parser.add_argument('--force-ccd', - metavar='', - type=float, - help='Override the ccd width information for the images') - parser.add_argument('--min-num-features', metavar='', default=8000, diff --git a/opendm/context.py b/opendm/context.py index 53edb9d8..cdf09784 100644 --- a/opendm/context.py +++ b/opendm/context.py @@ -18,7 +18,6 @@ sys.path.append(pyopencv_path) # define opensfm path opensfm_path = os.path.join(superbuild_path, "src/opensfm") -ccd_widths_path = os.path.join(opensfm_path, 'opensfm/data/sensor_data.json') # define orb_slam2 path orb_slam2_path = os.path.join(superbuild_path, "src/orb_slam2") diff --git a/opendm/get_image_size.py b/opendm/get_image_size.py new file mode 100644 index 00000000..8483b1d2 --- /dev/null +++ b/opendm/get_image_size.py @@ -0,0 +1,440 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from __future__ import print_function +from __future__ import absolute_import +""" + +get_image_size.py +==================== + + :Name: get_image_size + :Purpose: extract image dimensions given a file path + + :Author: Paulo Scardine (based on code from Emmanuel VAÏSSE) + + :Created: 26/09/2013 + :Copyright: (c) Paulo Scardine 2013 + :Licence: MIT + +""" +import collections +import json +import os +import io +import struct + +FILE_UNKNOWN = "Sorry, don't know how to get size for this file." + + +class UnknownImageFormat(Exception): + pass + + +types = collections.OrderedDict() +BMP = types['BMP'] = 'BMP' +GIF = types['GIF'] = 'GIF' +ICO = types['ICO'] = 'ICO' +JPEG = types['JPEG'] = 'JPEG' +PNG = types['PNG'] = 'PNG' +TIFF = types['TIFF'] = 'TIFF' + +image_fields = ['path', 'type', 'file_size', 'width', 'height'] + + +class Image(collections.namedtuple('Image', image_fields)): + + def to_str_row(self): + return ("%d\t%d\t%d\t%s\t%s" % ( + self.width, + self.height, + self.file_size, + self.type, + self.path.replace('\t', '\\t'), + )) + + def to_str_row_verbose(self): + return ("%d\t%d\t%d\t%s\t%s\t##%s" % ( + self.width, + self.height, + self.file_size, + self.type, + self.path.replace('\t', '\\t'), + self)) + + def to_str_json(self, indent=None): + return json.dumps(self._asdict(), indent=indent) + + +def get_image_size(file_path): + """ + Return (width, height) for a given img file content - no external + dependencies except the os and struct builtin modules + """ + img = get_image_metadata(file_path) + return (img.width, img.height) + + +def get_image_size_from_bytesio(input, size): + """ + Return (width, height) for a given img file content - no external + dependencies except the os and struct builtin modules + + Args: + input (io.IOBase): io object support read & seek + size (int): size of buffer in byte + """ + img = get_image_metadata_from_bytesio(input, size) + return (img.width, img.height) + + +def get_image_metadata(file_path): + """ + Return an `Image` object for a given img file content - no external + dependencies except the os and struct builtin modules + + Args: + file_path (str): path to an image file + + Returns: + Image: (path, type, file_size, width, height) + """ + size = os.path.getsize(file_path) + + # be explicit with open arguments - we need binary mode + with io.open(file_path, "rb") as input: + return get_image_metadata_from_bytesio(input, size, file_path) + + +def get_image_metadata_from_bytesio(input, size, file_path=None): + """ + Return an `Image` object for a given img file content - no external + dependencies except the os and struct builtin modules + + Args: + input (io.IOBase): io object support read & seek + size (int): size of buffer in byte + file_path (str): path to an image file + + Returns: + Image: (path, type, file_size, width, height) + """ + height = -1 + width = -1 + data = input.read(26) + msg = " raised while trying to decode as JPEG." + + if (size >= 10) and data[:6] in (b'GIF87a', b'GIF89a'): + # GIFs + imgtype = GIF + w, h = struct.unpack("= 24) and data.startswith(b'\211PNG\r\n\032\n') + and (data[12:16] == b'IHDR')): + # PNGs + imgtype = PNG + w, h = struct.unpack(">LL", data[16:24]) + width = int(w) + height = int(h) + elif (size >= 16) and data.startswith(b'\211PNG\r\n\032\n'): + # older PNGs + imgtype = PNG + w, h = struct.unpack(">LL", data[8:16]) + width = int(w) + height = int(h) + elif (size >= 2) and data.startswith(b'\377\330'): + # JPEG + imgtype = JPEG + input.seek(0) + input.read(2) + b = input.read(1) + try: + while (b and ord(b) != 0xDA): + while (ord(b) != 0xFF): + b = input.read(1) + while (ord(b) == 0xFF): + b = input.read(1) + if (ord(b) >= 0xC0 and ord(b) <= 0xC3): + input.read(3) + h, w = struct.unpack(">HH", input.read(4)) + break + else: + input.read( + int(struct.unpack(">H", input.read(2))[0]) - 2) + b = input.read(1) + width = int(w) + height = int(h) + except struct.error: + raise UnknownImageFormat("StructError" + msg) + except ValueError: + raise UnknownImageFormat("ValueError" + msg) + except Exception as e: + raise UnknownImageFormat(e.__class__.__name__ + msg) + elif (size >= 26) and data.startswith(b'BM'): + # BMP + imgtype = 'BMP' + headersize = struct.unpack("= 40: + w, h = struct.unpack("= 8) and data[:4] in (b"II\052\000", b"MM\000\052"): + # Standard TIFF, big- or little-endian + # BigTIFF and other different but TIFF-like formats are not + # supported currently + imgtype = TIFF + byteOrder = data[:2] + boChar = ">" if byteOrder == "MM" else "<" + # maps TIFF type id to size (in bytes) + # and python format char for struct + tiffTypes = { + 1: (1, boChar + "B"), # BYTE + 2: (1, boChar + "c"), # ASCII + 3: (2, boChar + "H"), # SHORT + 4: (4, boChar + "L"), # LONG + 5: (8, boChar + "LL"), # RATIONAL + 6: (1, boChar + "b"), # SBYTE + 7: (1, boChar + "c"), # UNDEFINED + 8: (2, boChar + "h"), # SSHORT + 9: (4, boChar + "l"), # SLONG + 10: (8, boChar + "ll"), # SRATIONAL + 11: (4, boChar + "f"), # FLOAT + 12: (8, boChar + "d") # DOUBLE + } + ifdOffset = struct.unpack(boChar + "L", data[4:8])[0] + try: + countSize = 2 + input.seek(ifdOffset) + ec = input.read(countSize) + ifdEntryCount = struct.unpack(boChar + "H", ec)[0] + # 2 bytes: TagId + 2 bytes: type + 4 bytes: count of values + 4 + # bytes: value offset + ifdEntrySize = 12 + for i in range(ifdEntryCount): + entryOffset = ifdOffset + countSize + i * ifdEntrySize + input.seek(entryOffset) + tag = input.read(2) + tag = struct.unpack(boChar + "H", tag)[0] + if(tag == 256 or tag == 257): + # if type indicates that value fits into 4 bytes, value + # offset is not an offset but value itself + type = input.read(2) + type = struct.unpack(boChar + "H", type)[0] + if type not in tiffTypes: + raise UnknownImageFormat( + "Unkown TIFF field type:" + + str(type)) + typeSize = tiffTypes[type][0] + typeChar = tiffTypes[type][1] + input.seek(entryOffset + 8) + value = input.read(typeSize) + value = int(struct.unpack(typeChar, value)[0]) + if tag == 256: + width = value + else: + height = value + if width > -1 and height > -1: + break + except Exception as e: + raise UnknownImageFormat(str(e)) + elif size >= 2: + # see http://en.wikipedia.org/wiki/ICO_(file_format) + imgtype = 'ICO' + input.seek(0) + reserved = input.read(2) + if 0 != struct.unpack(" 1: + import warnings + warnings.warn("ICO File contains more than one image") + # http://msdn.microsoft.com/en-us/library/ms997538.aspx + w = input.read(1) + h = input.read(1) + width = ord(w) + height = ord(h) + else: + raise UnknownImageFormat(FILE_UNKNOWN) + + return Image(path=file_path, + type=imgtype, + file_size=size, + width=width, + height=height) + + +import unittest + + +class Test_get_image_size(unittest.TestCase): + data = [{ + 'path': 'lookmanodeps.png', + 'width': 251, + 'height': 208, + 'file_size': 22228, + 'type': 'PNG'}] + + def setUp(self): + pass + + def test_get_image_size_from_bytesio(self): + img = self.data[0] + p = img['path'] + with io.open(p, 'rb') as fp: + b = fp.read() + fp = io.BytesIO(b) + sz = len(b) + output = get_image_size_from_bytesio(fp, sz) + self.assertTrue(output) + self.assertEqual(output, + (img['width'], + img['height'])) + + def test_get_image_metadata_from_bytesio(self): + img = self.data[0] + p = img['path'] + with io.open(p, 'rb') as fp: + b = fp.read() + fp = io.BytesIO(b) + sz = len(b) + output = get_image_metadata_from_bytesio(fp, sz) + self.assertTrue(output) + for field in image_fields: + self.assertEqual(getattr(output, field), None if field == 'path' else img[field]) + + def test_get_image_metadata(self): + img = self.data[0] + output = get_image_metadata(img['path']) + self.assertTrue(output) + for field in image_fields: + self.assertEqual(getattr(output, field), img[field]) + + def test_get_image_metadata__ENOENT_OSError(self): + with self.assertRaises(OSError): + get_image_metadata('THIS_DOES_NOT_EXIST') + + def test_get_image_metadata__not_an_image_UnknownImageFormat(self): + with self.assertRaises(UnknownImageFormat): + get_image_metadata('README.rst') + + def test_get_image_size(self): + img = self.data[0] + output = get_image_size(img['path']) + self.assertTrue(output) + self.assertEqual(output, + (img['width'], + img['height'])) + + def tearDown(self): + pass + + +def main(argv=None): + """ + Print image metadata fields for the given file path. + + Keyword Arguments: + argv (list): commandline arguments (e.g. sys.argv[1:]) + Returns: + int: zero for OK + """ + import logging + import optparse + import sys + + prs = optparse.OptionParser( + usage="%prog [-v|--verbose] [--json|--json-indent] []", + description="Print metadata for the given image paths " + "(without image library bindings).") + + prs.add_option('--json', + dest='json', + action='store_true') + prs.add_option('--json-indent', + dest='json_indent', + action='store_true') + + prs.add_option('-v', '--verbose', + dest='verbose', + action='store_true',) + prs.add_option('-q', '--quiet', + dest='quiet', + action='store_true',) + prs.add_option('-t', '--test', + dest='run_tests', + action='store_true',) + + argv = list(argv) if argv is not None else sys.argv[1:] + (opts, args) = prs.parse_args(args=argv) + loglevel = logging.INFO + if opts.verbose: + loglevel = logging.DEBUG + elif opts.quiet: + loglevel = logging.ERROR + logging.basicConfig(level=loglevel) + log = logging.getLogger() + log.debug('argv: %r', argv) + log.debug('opts: %r', opts) + log.debug('args: %r', args) + + if opts.run_tests: + import sys + sys.argv = [sys.argv[0]] + args + import unittest + return unittest.main() + + output_func = Image.to_str_row + if opts.json_indent: + import functools + output_func = functools.partial(Image.to_str_json, indent=2) + elif opts.json: + output_func = Image.to_str_json + elif opts.verbose: + output_func = Image.to_str_row_verbose + + EX_OK = 0 + EX_NOT_OK = 2 + + if len(args) < 1: + prs.print_help() + print('') + prs.error("You must specify one or more paths to image files") + + errors = [] + for path_arg in args: + try: + img = get_image_metadata(path_arg) + print(output_func(img)) + except KeyboardInterrupt: + raise + except OSError as e: + log.error((path_arg, e)) + errors.append((path_arg, e)) + except Exception as e: + log.exception(e) + errors.append((path_arg, e)) + pass + if len(errors): + import pprint + print("ERRORS", file=sys.stderr) + print("======", file=sys.stderr) + print(pprint.pformat(errors, indent=2), file=sys.stderr) + return EX_NOT_OK + return EX_OK + + +if __name__ == "__main__": + import sys + sys.exit(main(argv=sys.argv[1:])) \ No newline at end of file diff --git a/opendm/types.py b/opendm/types.py index 269241a9..3fbc1f2e 100644 --- a/opendm/types.py +++ b/opendm/types.py @@ -3,6 +3,7 @@ import exifread import re from fractions import Fraction from opensfm.exif import sensor_string +from opendm import get_image_size from pyproj import Proj import log @@ -15,15 +16,11 @@ class ODM_Photo: """ ODMPhoto - a class for ODMPhotos """ - def __init__(self, path_file, force_focal, force_ccd): + def __init__(self, path_file): # general purpose self.filename = io.extract_file_from_path_file(path_file) - # useful attibutes self.width = None self.height = None - self.ccd_width = None - self.focal_length = None - self.focal_length_px = None # other attributes self.camera_make = '' self.camera_model = '' @@ -32,33 +29,17 @@ class ODM_Photo: self.longitude = None self.altitude = None # parse values from metadata - self.parse_exif_values(path_file, force_focal, force_ccd) - # compute focal length into pixels - self.update_focal() + self.parse_exif_values(path_file) # print log message log.ODM_DEBUG('Loaded {}'.format(self)) def __str__(self): - return '{} | camera: {} | dimensions: {} x {} | focal: {} | ccd: {} | lat: {} | lon: {} | alt: {}'.format( - self.filename, self.make_model, self.width, self.height, self.focal_length, - self.ccd_width, self.latitude, self.longitude, self.altitude) + return '{} | camera: {} | dimensions: {} x {} | lat: {} | lon: {} | alt: {}'.format( + self.filename, self.make_model, self.width, self.height, self.latitude, self.longitude, self.altitude) - def update_focal(self): - # compute focal length in pixels - if self.focal_length and self.ccd_width: - # take width or height as reference - if self.width > self.height: - # f(px) = w(px) * f(mm) / ccd(mm) - self.focal_length_px = \ - self.width * (self.focal_length / self.ccd_width) - else: - # f(px) = h(px) * f(mm) / ccd(mm) - self.focal_length_px = \ - self.height * (self.focal_length / self.ccd_width) - - def parse_exif_values(self, _path_file, _force_focal, _force_ccd): + def parse_exif_values(self, _path_file): # Disable exifread log logging.getLogger('exifread').setLevel(logging.CRITICAL) @@ -70,8 +51,6 @@ class ODM_Photo: self.camera_make = tags['Image Make'].values.encode('utf8') if 'Image Model' in tags: self.camera_model = tags['Image Model'].values.encode('utf8') - if 'EXIF FocalLength' in tags: - self.focal_length = self.float_values(tags['EXIF FocalLength'])[0] if 'GPS GPSAltitude' in tags: self.altitude = self.float_values(tags['GPS GPSAltitude'])[0] if 'GPS GPSAltitudeRef' in tags and self.int_values(tags['GPS GPSAltitudeRef'])[0] > 0: @@ -87,28 +66,13 @@ class ODM_Photo: self.make_model = sensor_string(self.camera_make, self.camera_model) # needed to do that since sometimes metadata contains wrong data - img = cv2.imread(_path_file) - self.width = img.shape[1] - self.height = img.shape[0] - - # force focal and ccd_width with user parameter - if _force_focal: - self.focal_length = _force_focal - if _force_ccd: - self.ccd_width = _force_ccd - - # find ccd_width from file if needed - if self.ccd_width is None and self.camera_model is not None: - # load ccd_widths from file - ccd_widths = system.get_ccd_widths() - # search ccd by camera model - key = [x for x in ccd_widths.keys() if self.make_model in x] - # convert to float if found - if key: - self.ccd_width = float(ccd_widths[key[0]]) - else: - log.ODM_WARNING('Could not find ccd_width in file. Use --force-ccd or edit the sensor_data.json ' - 'file to manually input ccd width') + try: + self.width, self.height = get_image_size.get_image_size(_path_file) + except get_image_size.UnknownImageFormat: + # Fallback to slower cv2 + img = cv2.imread(_path_file) + self.width = img.shape[1] + self.height = img.shape[0] def dms_to_decimal(self, dms, sign): """Converts dms coords to decimal degrees""" @@ -126,7 +90,7 @@ class ODM_Photo: def int_values(self, tag): return map(int, tag.values) -# TODO: finish this class + class ODM_Reconstruction(object): """docstring for ODMReconstruction""" @@ -197,15 +161,6 @@ class ODM_Reconstruction(object): log.ODM_EXCEPTION('Could not set projection. Please use a proj4 string') -class ODM_GCPoint(object): - """docstring for ODMPoint""" - - def __init__(self, x, y, z): - self.x = x - self.y = y - self.z = z - - class ODM_GeoRef(object): """docstring for ODMUtmZone""" @@ -258,24 +213,6 @@ class ODM_GeoRef(object): self.utm_east_offset = float(offsets[0]) self.utm_north_offset = float(offsets[1]) - def create_gcps(self, _file): - if not io.file_exists(_file): - log.ODM_ERROR('Could not find file %s' % _file) - return - - with open(_file) as f: - # parse coordinates - lines = f.readlines()[2:] - for l in lines: - xyz = l.split(' ') - if len(xyz) == 3: - x, y, z = xyz[:3] - elif len(xyz) == 2: - x, y = xyz[:2] - z = 0 - self.gcps.append(ODM_GCPoint(float(x), float(y), float(z))) - # Write to json file - def parse_transformation_matrix(self, _file): if not io.file_exists(_file): log.ODM_ERROR('Could not find file %s' % _file) diff --git a/scripts/dataset.py b/scripts/dataset.py index aabca6eb..274e880f 100644 --- a/scripts/dataset.py +++ b/scripts/dataset.py @@ -9,12 +9,6 @@ from opendm import log from opendm import system from shutil import copyfile - -def make_odm_photo(force_focal, force_ccd, path_file): - return types.ODM_Photo(path_file, - force_focal, - force_ccd) - def save_images_database(photos, database_file): with open(database_file, 'w') as f: f.write(json.dumps(map(lambda p: p.__dict__, photos))) @@ -45,10 +39,6 @@ def load_images_database(database_file): class ODMLoadDatasetCell(ecto.Cell): def declare_params(self, params): - params.declare("force_focal", 'Override the focal length information for the ' - 'images', None) - params.declare("force_ccd", 'Override the ccd width information for the ' - 'images', None) params.declare("verbose", 'indicate verbosity', False) params.declare("proj", 'Geographic projection', None) @@ -106,8 +96,8 @@ class ODMLoadDatasetCell(ecto.Cell): photos = [] with open(tree.dataset_list, 'w') as dataset_list: - for files in path_files: - photos += [make_odm_photo(self.params.force_focal, self.params.force_ccd, files)] + for f in path_files: + photos += [types.ODM_Photo(f)] dataset_list.write(photos[-1].filename + '\n') # Save image database for faster restart diff --git a/scripts/odm_app.py b/scripts/odm_app.py index 6d9794b6..5fce963a 100644 --- a/scripts/odm_app.py +++ b/scripts/odm_app.py @@ -36,9 +36,7 @@ class ODMApp(ecto.BlackBox): Only cells from which something is forwarded have to be declared """ cells = {'args': ecto.Constant(value=p.args), - 'dataset': ODMLoadDatasetCell(force_focal=p.args.force_focal, - force_ccd=p.args.force_ccd, - verbose=p.args.verbose, + 'dataset': ODMLoadDatasetCell(verbose=p.args.verbose, proj=p.args.proj), 'opensfm': ODMOpenSfMCell(use_exif_size=False, feature_process_size=p.args.resize_to, diff --git a/tests/test_odm.py b/tests/test_odm.py index 2604c315..851b5565 100644 --- a/tests/test_odm.py +++ b/tests/test_odm.py @@ -33,7 +33,7 @@ def setup_module(): def teardown_module(): # Delete generated test directories - dirnames = ['images_resize', 'opensfm', 'pmvs', 'odm_meshing', + dirnames = ['opensfm', 'odm_meshing', 'odm_texturing', 'odm_georeferencing', 'odm_orthophoto'] for n in dirnames: rmpath = os.path.join(context.tests_data_path, n) @@ -41,30 +41,6 @@ def teardown_module(): shutil.rmtree(rmpath) -class TestResize(unittest.TestCase): - """ - Tests the resize function - """ - - def setUp(self): - # rerun resize cell and set params - options.rerun = 'resize' - options.resize_to = 1600 - # rebuild app - self.app, self.plasm = appSetup(options) - run_plasm(options, self.plasm) - - - def test_resize(self): - # assert each image is sized to the option.resize_to - self.assertEquals(max(self.app.resize.outputs.photos[0].height, self.app.resize.outputs.photos[0].width), - options.resize_to) - - def test_all_resized(self): - # assert the number of images in images == number of images in resize - self.assertEquals(len(self.app.resize.outputs.photos), len(self.app.dataset.outputs.photos)) - - class TestOpenSfM(unittest.TestCase): """ Tests the OpenSfM module @@ -79,28 +55,6 @@ class TestOpenSfM(unittest.TestCase): self.assertTrue(os.path.isfile(self.app.opensfm.inputs.tree.opensfm_reconstruction)) -class TestCMVS(unittest.TestCase): - - def setUp(self): - options.rerun = 'cmvs' - self.app, self.plasm = appSetup(options) - run_plasm(options, self.plasm) - - def test_cmvs(self): - self.assertTrue(os.path.isfile(self.app.cmvs.inputs.tree.pmvs_bundle)) - - -class TestPMVS(unittest.TestCase): - - def setUp(self): - options.rerun = 'pmvs' - self.app, self.plasm = appSetup(options) - run_plasm(options, self.plasm) - - def test_pmvs(self): - self.assertTrue(os.path.isfile(self.app.pmvs.inputs.tree.pmvs_model)) - - class TestMeshing(unittest.TestCase): def setUp(self):