From 60367ee97b42145a708704c3c8a22121e3b7b5d7 Mon Sep 17 00:00:00 2001 From: Pau Gargallo Date: Sun, 18 Jun 2017 17:59:10 +0200 Subject: [PATCH 01/35] Fix x,y dimensions in exif after resize. In numpy arrays, shape[0] corresponds to y and shape[1] corresponds to x. --- scripts/resize.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/resize.py b/scripts/resize.py index 584fe2bc..7c50b0bb 100644 --- a/scripts/resize.py +++ b/scripts/resize.py @@ -37,13 +37,13 @@ def resize(src_dir, target_dir, resize_to, rerun_cell, photo): # copy metadata old_meta.copy(new_meta) # update metadata size - new_meta['Exif.Photo.PixelXDimension'] = img_r.shape[0] - new_meta['Exif.Photo.PixelYDimension'] = img_r.shape[1] + new_meta['Exif.Photo.PixelXDimension'] = img_r.shape[1] + new_meta['Exif.Photo.PixelYDimension'] = img_r.shape[0] new_meta.write() # update photos array with new values photo.path_file = new_path_file - photo.width = img_r.shape[0] - photo.height = img_r.shape[1] + photo.width = img_r.shape[1] + photo.height = img_r.shape[0] photo.update_focal() # log message @@ -69,8 +69,8 @@ def no_resize(src_dir,target_dir,rerun_cell,photo): img = cv2.imread(path_file) io.copy(path_file, new_path_file) photo.path_file = new_path_file - photo.width = img.shape[0] - photo.height = img.shape[1] + photo.width = img.shape[1] + photo.height = img.shape[0] photo.update_focal() # log message From 5fb99673687854de20d0db44850d81b0752e1cc6 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 20 Jun 2017 10:35:33 -0400 Subject: [PATCH 02/35] Modified dockerfile, configure.sh script to install lidar2dems module --- .gitmodules | 6 +++--- Dockerfile | 11 +++++++++-- configure.sh | 17 +++++++++++++++++ modules/lidar2dems | 1 + 4 files changed, 30 insertions(+), 5 deletions(-) create mode 160000 modules/lidar2dems diff --git a/.gitmodules b/.gitmodules index b01146d3..3a5f4fa0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "src/bundler"] - path = src/bundler - url = https://github.com/chris-cooper/bundler_sfm +[submodule "modules/lidar2dems"] + path = modules/lidar2dems + url = https://github.com/pierotofy/lidar2dems diff --git a/Dockerfile b/Dockerfile index 73278c3d..1ea62125 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,12 +14,12 @@ RUN apt-get install --no-install-recommends -y git cmake python-pip build-essent libgtk2.0-dev libavcodec-dev libavformat-dev libswscale-dev python-dev python-numpy libtbb2 libtbb-dev libjpeg-dev libpng-dev libtiff-dev libjasper-dev libflann-dev \ libproj-dev libxext-dev liblapack-dev libeigen3-dev libvtk5-dev python-networkx libgoogle-glog-dev libsuitesparse-dev libboost-filesystem-dev libboost-iostreams-dev \ libboost-regex-dev libboost-python-dev libboost-date-time-dev libboost-thread-dev python-pyproj python-empy python-nose python-pyside python-pyexiv2 python-scipy \ -jhead liblas-bin python-matplotlib libatlas-base-dev libgmp-dev libmpfr-dev +jhead liblas-bin python-matplotlib libatlas-base-dev libgmp-dev libmpfr-dev swig2.0 python-wheel libboost-log-dev libxslt-dev RUN apt-get remove libdc1394-22-dev RUN pip install --upgrade pip RUN pip install setuptools -RUN pip install -U PyYAML exifread gpxpy xmltodict catkin-pkg appsettings +RUN pip install -U PyYAML exifread gpxpy xmltodict catkin-pkg appsettings https://github.com/Applied-GeoSolutions/gippy/archive/v0.3.9.tar.gz ENV PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python2.7/dist-packages" ENV PYTHONPATH="$PYTHONPATH:/code/SuperBuild/src/opensfm" @@ -44,10 +44,17 @@ COPY /SuperBuild/CMakeLists.txt /code/SuperBuild/CMakeLists.txt COPY docker.settings.yaml /code/settings.yaml COPY VERSION /code/VERSION + #Compile code in SuperBuild and root directories RUN cd SuperBuild && mkdir build && cd build && cmake .. && make -j$(nproc) && cd ../.. && mkdir build && cd build && cmake .. && make -j$(nproc) +# Update and initialize git submodules +RUN git submodule update --init + +# Setup lidar2dems +RUN CPLUS_INCLUDE_PATH=/usr/include/gdal C_INCLUDE_PATH=/usr/include/gdal /code/modules/lidar2dems/setup.py install && ln -s /code/SuperBuild/build/pdal/bin/pdal /usr/bin/pdal + RUN apt-get -y remove libgl1-mesa-dri git cmake python-pip build-essential RUN apt-get install -y libvtk5-dev diff --git a/configure.sh b/configure.sh index 9593e849..199fb378 100755 --- a/configure.sh +++ b/configure.sh @@ -86,6 +86,23 @@ install() { jhead \ liblas-bin + echo "Installing lidar2dems Dependencies" + sudo apt-get install -y -qq swig2.0 \ + python-wheel \ + libboost-log-dev \ + libxslt-dev + + echo "Updating and initializing git submodules" + git submodule update --init + + echo "Installing lidar2dems" + cd ${RUNPATH}/modules/lidar2dems + CPLUS_INCLUDE_PATH=/usr/include/gdal C_INCLUDE_PATH=/usr/include/gdal ./setup.py install + + if [ ! -f /usr/bin/pdal ]; then + ln -s ${RUNPATH}/SuperBuild/build/pdal/bin/pdal /usr/bin/pdal + fi + echo "Compiling SuperBuild" cd ${RUNPATH}/SuperBuild mkdir -p build && cd build diff --git a/modules/lidar2dems b/modules/lidar2dems new file mode 160000 index 00000000..fe0710bd --- /dev/null +++ b/modules/lidar2dems @@ -0,0 +1 @@ +Subproject commit fe0710bd68e6add3be5621e94f20f56c1e99fd00 From fed4cb95dc2ead79430ea071805e30a9197bdbd9 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 20 Jun 2017 14:53:17 -0400 Subject: [PATCH 03/35] Added lidar2dems superbuild module, modified dockerfile, configure.sh to install lidar2dems deps --- .gitmodules | 3 -- Dockerfile | 8 +----- SuperBuild/CMakeLists.txt | 1 + SuperBuild/cmake/External-Lidar2dems.cmake | 25 ++++++++++++++++ configure.sh | 13 ++------- modules/lidar2dems | 1 - opendm/types.py | 33 ++-------------------- scripts/odm_georeferencing.py | 1 - 8 files changed, 31 insertions(+), 54 deletions(-) create mode 100644 SuperBuild/cmake/External-Lidar2dems.cmake delete mode 160000 modules/lidar2dems diff --git a/.gitmodules b/.gitmodules index 3a5f4fa0..e69de29b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "modules/lidar2dems"] - path = modules/lidar2dems - url = https://github.com/pierotofy/lidar2dems diff --git a/Dockerfile b/Dockerfile index 1ea62125..038363e1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ jhead liblas-bin python-matplotlib libatlas-base-dev libgmp-dev libmpfr-dev swig RUN apt-get remove libdc1394-22-dev RUN pip install --upgrade pip RUN pip install setuptools -RUN pip install -U PyYAML exifread gpxpy xmltodict catkin-pkg appsettings https://github.com/Applied-GeoSolutions/gippy/archive/v0.3.9.tar.gz +RUN pip install -U PyYAML exifread gpxpy xmltodict catkin-pkg appsettings https://github.com/pierotofy/gippy/archive/v0.3.9.tar.gz ENV PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python2.7/dist-packages" ENV PYTHONPATH="$PYTHONPATH:/code/SuperBuild/src/opensfm" @@ -49,12 +49,6 @@ COPY VERSION /code/VERSION RUN cd SuperBuild && mkdir build && cd build && cmake .. && make -j$(nproc) && cd ../.. && mkdir build && cd build && cmake .. && make -j$(nproc) -# Update and initialize git submodules -RUN git submodule update --init - -# Setup lidar2dems -RUN CPLUS_INCLUDE_PATH=/usr/include/gdal C_INCLUDE_PATH=/usr/include/gdal /code/modules/lidar2dems/setup.py install && ln -s /code/SuperBuild/build/pdal/bin/pdal /usr/bin/pdal - RUN apt-get -y remove libgl1-mesa-dri git cmake python-pip build-essential RUN apt-get install -y libvtk5-dev diff --git a/SuperBuild/CMakeLists.txt b/SuperBuild/CMakeLists.txt index f6b61d75..b6892d06 100644 --- a/SuperBuild/CMakeLists.txt +++ b/SuperBuild/CMakeLists.txt @@ -114,6 +114,7 @@ set(custom_libs OpenGV Ecto PDAL MvsTexturing + Lidar2dems ) # Dependencies of the SLAM module diff --git a/SuperBuild/cmake/External-Lidar2dems.cmake b/SuperBuild/cmake/External-Lidar2dems.cmake new file mode 100644 index 00000000..14d30e99 --- /dev/null +++ b/SuperBuild/cmake/External-Lidar2dems.cmake @@ -0,0 +1,25 @@ +set(_proj_name lidar2dems) +set(_SB_BINARY_DIR "${SB_BINARY_DIR}/${_proj_name}") + +ExternalProject_Add(${_proj_name} + PREFIX ${_SB_BINARY_DIR} + TMP_DIR ${_SB_BINARY_DIR}/tmp + STAMP_DIR ${_SB_BINARY_DIR}/stamp + #--Download step-------------- + DOWNLOAD_DIR ${SB_DOWNLOAD_DIR}/${_proj_name} + URL https://github.com/pierotofy/lidar2dems/archive/master.zip + URL_MD5 6cfe2d081cda1680e6e98a9925651b03 + #--Update/Patch step---------- + UPDATE_COMMAND "" + #--Configure step------------- + SOURCE_DIR ${SB_SOURCE_DIR}/${_proj_name} + CONFIGURE_COMMAND "" + #--Build step----------------- + BUILD_COMMAND "" + #--Install step--------------- + INSTALL_COMMAND "${SB_SOURCE_DIR}/${_proj_name}/install.sh" + #--Output logging------------- + LOG_DOWNLOAD OFF + LOG_CONFIGURE OFF + LOG_BUILD OFF +) diff --git a/configure.sh b/configure.sh index 199fb378..658c046a 100755 --- a/configure.sh +++ b/configure.sh @@ -72,7 +72,7 @@ install() { appsettings echo "Installing CGAL dependencies" - sudo apt-get install libgmp-dev libmpfr-dev + sudo apt-get install -y -qq libgmp-dev libmpfr-dev echo "Installing Ecto Dependencies" sudo pip install -U catkin-pkg @@ -92,17 +92,8 @@ install() { libboost-log-dev \ libxslt-dev - echo "Updating and initializing git submodules" - git submodule update --init + sudo pip install -U https://github.com/pierotofy/gippy/archive/v0.3.9.tar.gz - echo "Installing lidar2dems" - cd ${RUNPATH}/modules/lidar2dems - CPLUS_INCLUDE_PATH=/usr/include/gdal C_INCLUDE_PATH=/usr/include/gdal ./setup.py install - - if [ ! -f /usr/bin/pdal ]; then - ln -s ${RUNPATH}/SuperBuild/build/pdal/bin/pdal /usr/bin/pdal - fi - echo "Compiling SuperBuild" cd ${RUNPATH}/SuperBuild mkdir -p build && cd build diff --git a/modules/lidar2dems b/modules/lidar2dems deleted file mode 160000 index fe0710bd..00000000 --- a/modules/lidar2dems +++ /dev/null @@ -1 +0,0 @@ -Subproject commit fe0710bd68e6add3be5621e94f20f56c1e99fd00 diff --git a/opendm/types.py b/opendm/types.py index 8c0dfd57..27751d60 100644 --- a/opendm/types.py +++ b/opendm/types.py @@ -196,7 +196,7 @@ class ODM_GeoRef(object): system.run('{bin}/pdal pipeline -i {json} --readers.ply.filename={f_in} ' '--writers.las.filename={f_out}'.format(**kwargs)) - def convert_to_dem(self, _file, _file_out, pdalJSON, sample_radius, gdal_res, gdal_radius): + def convert_to_dem(self, _file, _file_out, sample_radius, gdal_res, gdal_radius): # Check if exists f_in if not io.file_exists(_file): log.ODM_ERROR('LAS file does not exist') @@ -208,36 +208,9 @@ class ODM_GeoRef(object): 'sample_radius': sample_radius, 'gdal_res': gdal_res, 'gdal_radius': gdal_radius, - 'f_out': _file_out, - 'json': pdalJSON + 'f_out': _file_out } - pipelineJSON = '{{' \ - ' "pipeline":[' \ - ' "input.las",' \ - ' {{' \ - ' "type":"filters.sample",' \ - ' "radius":"{sample_radius}"' \ - ' }},' \ - ' {{' \ - ' "type":"filters.pmf"' \ - ' }},' \ - ' {{' \ - ' "type":"filters.range",' \ - ' "limits":"Classification[2:2]"' \ - ' }},' \ - ' {{' \ - ' "resolution": {gdal_res},' \ - ' "radius": {gdal_radius},' \ - ' "output_type":"idw",' \ - ' "filename":"outputfile.tif"' \ - ' }}' \ - ' ]' \ - '}}'.format(**kwargs) - - with open(pdalJSON, 'w') as f: - f.write(pipelineJSON) - system.run('{bin}/pdal pipeline {json} --readers.las.filename={f_in} ' '--writers.gdal.filename={f_out}'.format(**kwargs)) @@ -454,8 +427,6 @@ class ODM_Tree(object): self.odm_georeferencing, 'odm_georeferenced_model.las') self.odm_georeferencing_dem = io.join_paths( self.odm_georeferencing, 'odm_georeferencing_model_dem.tif') - self.odm_georeferencing_dem_json = io.join_paths( - self.odm_georeferencing, 'dem.json') # odm_orthophoto self.odm_orthophoto_file = io.join_paths(self.odm_orthophoto, 'odm_orthophoto.png') diff --git a/scripts/odm_georeferencing.py b/scripts/odm_georeferencing.py index acecedba..b5d03cc4 100644 --- a/scripts/odm_georeferencing.py +++ b/scripts/odm_georeferencing.py @@ -179,7 +179,6 @@ class ODMGeoreferencingCell(ecto.Cell): if args.dem: demcreated = geo_ref.convert_to_dem(tree.odm_georeferencing_model_las, tree.odm_georeferencing_dem, - tree.odm_georeferencing_dem_json, self.params.sample_radius, self.params.gdal_res, self.params.gdal_radius) From af0c2c986f956a8aeec1d01f38d4af90e388c636 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 23 Jun 2017 11:20:46 -0400 Subject: [PATCH 04/35] Started adding odm_dem module --- Dockerfile | 2 +- configure.sh | 3 +- opendm/config.py | 73 +++++++++++++++++----- opendm/tasks.py | 5 +- opendm/types.py | 3 + scripts/odm_app.py | 6 +- scripts/odm_dem.py | 148 +++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 216 insertions(+), 24 deletions(-) create mode 100644 scripts/odm_dem.py diff --git a/Dockerfile b/Dockerfile index 038363e1..e683f3e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ RUN apt-get install --no-install-recommends -y git cmake python-pip build-essent libgtk2.0-dev libavcodec-dev libavformat-dev libswscale-dev python-dev python-numpy libtbb2 libtbb-dev libjpeg-dev libpng-dev libtiff-dev libjasper-dev libflann-dev \ libproj-dev libxext-dev liblapack-dev libeigen3-dev libvtk5-dev python-networkx libgoogle-glog-dev libsuitesparse-dev libboost-filesystem-dev libboost-iostreams-dev \ libboost-regex-dev libboost-python-dev libboost-date-time-dev libboost-thread-dev python-pyproj python-empy python-nose python-pyside python-pyexiv2 python-scipy \ -jhead liblas-bin python-matplotlib libatlas-base-dev libgmp-dev libmpfr-dev swig2.0 python-wheel libboost-log-dev libxslt-dev +jhead liblas-bin python-matplotlib libatlas-base-dev libgmp-dev libmpfr-dev swig2.0 python-wheel libboost-log-dev RUN apt-get remove libdc1394-22-dev RUN pip install --upgrade pip diff --git a/configure.sh b/configure.sh index 658c046a..f5321c05 100755 --- a/configure.sh +++ b/configure.sh @@ -89,8 +89,7 @@ install() { echo "Installing lidar2dems Dependencies" sudo apt-get install -y -qq swig2.0 \ python-wheel \ - libboost-log-dev \ - libxslt-dev + libboost-log-dev sudo pip install -U https://github.com/pierotofy/gippy/archive/v0.3.9.tar.gz diff --git a/opendm/config.py b/opendm/config.py index ea0f91ac..38f4a7fd 100644 --- a/opendm/config.py +++ b/opendm/config.py @@ -10,7 +10,7 @@ import sys # parse arguments processopts = ['resize', 'opensfm', 'slam', 'cmvs', 'pmvs', 'odm_meshing', 'odm_25dmeshing', 'mvs_texturing', 'odm_georeferencing', - 'odm_orthophoto'] + 'odm_dem', 'odm_orthophoto'] with open(io.join_paths(context.root_path, 'VERSION')) as version_file: __version__ = version_file.read().strip() @@ -341,34 +341,77 @@ def config(): help=('Use this tag if you have a gcp_list.txt but ' 'want to use the exif geotags instead')) - parser.add_argument('--dem', + parser.add_argument('--dtm', action='store_true', default=False, - help='Use this tag to build a DEM using a progressive ' - 'morphological filter in PDAL.') + help='Use this tag to build a DTM (Digital Terrain Model, ground only) using a progressive ' + 'morphological filter in PDAL. Check the --dem* parameters for fine tuning.') + + parser.add_argument('--dsm', + action='store_true', + default=False, + help='Use this tag to build a DSM (Digital Surface Model, ground + objects) using a progressive ' + 'morphological filter in PDAL. Check the --dem* parameters for fine tuning.') - parser.add_argument('--dem-sample-radius', - metavar='', - default=1.0, - type=float, - help='Minimum distance between samples for DEM ' + parser.add_argument('--dem-gapfill-steps', + metavar='', + default=4, + type=int, + help='Number of steps used to fill areas with gaps. Set to 0 to disable gap filling. ' + 'Starting with a radius equal to the output resolution, N different DEMs are generated with ' + 'progressively bigger radius using the inverse distance weighted (IDW) algorithm ' + 'and merged together. Remaining gaps are then merged using linear interpolation. ' 'generation.\nDefault=%(default)s') parser.add_argument('--dem-resolution', metavar='', type=float, - default=2, + default=0.1, help='Length of raster cell edges in X/Y units.' '\nDefault: %(default)s') - parser.add_argument('--dem-radius', - metavar='', + parser.add_argument('--dem-maxangle', + metavar='', type=float, - default=0.5, - help='Radius about cell center bounding points to ' - 'use to calculate a cell value.\nDefault: ' + default=20, + help='Points that are more than maxangle degrees off-nadir are discarded. ' + '\nDefault: ' '%(default)s') + parser.add_argument('--dem-maxsd', + metavar='', + type=float, + default=2.5, + help='Points that deviate more than maxsd standard deviations from the local mean ' + 'are discarded. \nDefault: ' + '%(default)s') + + parser.add_argument('--dem-approximate', + action='store_true', + default=False, + help='Use this tag use the approximate progressive ' + 'morphological filter in PDAL, which computes DEMs faster ' + 'but is not as accurate.') + + parser.add_argument('--dem-decimation', + metavar='', + default=1, + type=int, + help='Decimate the points before generating the DEM. 1 is no decimation (full quality). ' + '100 decimates ~99% of the points. Useful for speeding up ' + 'generation.\nDefault=%(default)s') + + parser.add_argument('--dem-terrain-type', + metavar='', + choices=['FlatNonForest', 'FlatForest', 'ComplexNonForest', 'ComplexForest'], + default='ComplexForest', + help='Specifies the type of terrain. This mainly helps reduce processing time. ' + 'FlatNonForest: Relatively flat region with little to no vegetation (slope 1, cell size 3)' + 'FlatForest: Relatively flat region that is forested (slope 1, cell size 2)' + 'ComplexNonForest: Varied terrain with little to no vegetation (slope 5, cell size 2)' + 'ComplexForest: Varied terrain that is forested (slope 10, cell size 2)' + '\nDefault=%(default)s') + parser.add_argument('--orthophoto-resolution', metavar=' 0.0>', default=20.0, diff --git a/opendm/tasks.py b/opendm/tasks.py index 0305049c..77d229e7 100644 --- a/opendm/tasks.py +++ b/opendm/tasks.py @@ -15,8 +15,9 @@ tasks_dict = {'1': 'resize', '5': 'odm_meshing', '6': 'mvs_texturing', '7': 'odm_georeferencing', - '8': 'odm_orthophoto', - '9': 'zip_results'} + '8': 'odm_dem', + '9': 'odm_orthophoto', + '10': 'zip_results'} class ODMTaskManager(object): diff --git a/opendm/types.py b/opendm/types.py index 27751d60..ea1d9777 100644 --- a/opendm/types.py +++ b/opendm/types.py @@ -435,3 +435,6 @@ class ODM_Tree(object): self.odm_orthophoto_log = io.join_paths(self.odm_orthophoto, 'odm_orthophoto_log.txt') self.odm_orthophoto_tif_log = io.join_paths(self.odm_orthophoto, 'gdal_translate_log.txt') self.odm_orthophoto_gdaladdo_log = io.join_paths(self.odm_orthophoto, 'gdaladdo_log.txt') + + def path(self, **args): + return io.join_paths(self.root_path, **args) \ No newline at end of file diff --git a/scripts/odm_app.py b/scripts/odm_app.py index 060b49b7..bd5c8298 100644 --- a/scripts/odm_app.py +++ b/scripts/odm_app.py @@ -16,6 +16,7 @@ from odm_meshing import ODMeshingCell from mvstex import ODMMvsTexCell from odm_georeferencing import ODMGeoreferencingCell from odm_orthophoto import ODMOrthoPhotoCell +from odm_dem import ODMDEMCell class ODMApp(ecto.BlackBox): @@ -73,11 +74,8 @@ class ODMApp(ecto.BlackBox): 'georeferencing': ODMGeoreferencingCell(img_size=p.args.resize_to, gcp_file=p.args.gcp, use_exif=p.args.use_exif, - dem=p.args.dem, - sample_radius=p.args.dem_sample_radius, - gdal_res=p.args.dem_resolution, - gdal_radius=p.args.dem_radius, verbose=p.args.verbose), + 'dem': ODMDEMCell(verbose=p.args.verbose), 'orthophoto': ODMOrthoPhotoCell(resolution=p.args.orthophoto_resolution, t_srs=p.args.orthophoto_target_srs, no_tiled=p.args.orthophoto_no_tiled, diff --git a/scripts/odm_dem.py b/scripts/odm_dem.py new file mode 100644 index 00000000..070b2d1a --- /dev/null +++ b/scripts/odm_dem.py @@ -0,0 +1,148 @@ +import ecto, os + +from opendm import io +from opendm import log +from opendm import system +from opendm import context +from opendm import types + + +class ODMDemCell(ecto.Cell): + def declare_params(self, params): + params.declare("verbose", 'print additional messages to console', False) + + def declare_io(self, params, inputs, outputs): + inputs.declare("args", "The application arguments.", {}) + + def process(self, inputs, outputs): + # Benchmarking + start_time = system.now_raw() + + log.ODM_INFO('Running ODM DEM Cell') + + # get inputs + args = self.inputs.args + verbose = '-v' if self.params.verbose else '' + + # define paths and create working directories + odm_dem_root = tree.path('odm_dem') + system.mkdir_p(odm_dem_root) + + dsm_output_filename = os.path.join(odm_dem_root, ) + + # check if we rerun cell or not + rerun_cell = (args.rerun is not None and + args.rerun == 'odm_dem') or \ + (args.rerun_all) or \ + (args.rerun_from is not None and + 'odm_dem' in args.rerun_from) + + if not io.file_exists(tree.odm_orthophoto_file) or rerun_cell: + + # odm_orthophoto definitions + kwargs = { + 'bin': context.odm_modules_path, + 'log': tree.odm_orthophoto_log, + 'ortho': tree.odm_orthophoto_file, + 'corners': tree.odm_orthophoto_corners, + 'res': self.params.resolution, + 'verbose': verbose + } + + # Have geo coordinates? + if io.file_exists(tree.odm_georeferencing_coords): + if args.use_25dmesh: + kwargs['model_geo'] = os.path.join(tree.odm_25dtexturing, tree.odm_georeferencing_model_obj_geo) + else: + kwargs['model_geo'] = os.path.join(tree.odm_texturing, tree.odm_georeferencing_model_obj_geo) + else: + if args.use_25dmesh: + kwargs['model_geo'] = os.path.join(tree.odm_25dtexturing, tree.odm_textured_model_obj) + else: + kwargs['model_geo'] = os.path.join(tree.odm_texturing, tree.odm_textured_model_obj) + + # run odm_orthophoto + system.run('{bin}/odm_orthophoto -inputFile {model_geo} ' + '-logFile {log} -outputFile {ortho} -resolution {res} {verbose} ' + '-outputCornerFile {corners}'.format(**kwargs)) + + if not io.file_exists(tree.odm_georeferencing_coords): + log.ODM_WARNING('No coordinates file. A georeferenced raster ' + 'will not be created') + else: + # Create georeferenced GeoTiff + geotiffcreated = False + georef = types.ODM_GeoRef() + # creates the coord refs # TODO I don't want to have to do this twice- after odm_georef + georef.parse_coordinate_system(tree.odm_georeferencing_coords) + + if georef.epsg and georef.utm_east_offset and georef.utm_north_offset: + ulx = uly = lrx = lry = 0.0 + with open(tree.odm_orthophoto_corners) as f: + for lineNumber, line in enumerate(f): + if lineNumber == 0: + tokens = line.split(' ') + if len(tokens) == 4: + ulx = float(tokens[0]) + \ + float(georef.utm_east_offset) + lry = float(tokens[1]) + \ + float(georef.utm_north_offset) + lrx = float(tokens[2]) + \ + float(georef.utm_east_offset) + uly = float(tokens[3]) + \ + float(georef.utm_north_offset) + log.ODM_INFO('Creating GeoTIFF') + + kwargs = { + 'ulx': ulx, + 'uly': uly, + 'lrx': lrx, + 'lry': lry, + 'tiled': '' if self.params.no_tiled else '-co TILED=yes ', + 'compress': self.params.compress, + 'predictor': '-co PREDICTOR=2 ' if self.params.compress in + ['LZW', 'DEFLATE'] else '', + 'epsg': georef.epsg, + 't_srs': self.params.t_srs or "EPSG:{0}".format(georef.epsg), + 'bigtiff': self.params.bigtiff, + 'png': tree.odm_orthophoto_file, + 'tiff': tree.odm_orthophoto_tif, + 'log': tree.odm_orthophoto_tif_log + } + + system.run('gdal_translate -a_ullr {ulx} {uly} {lrx} {lry} ' + '{tiled} ' + '-co BIGTIFF={bigtiff} ' + '-co COMPRESS={compress} ' + '{predictor} ' + '-co BLOCKXSIZE=512 ' + '-co BLOCKYSIZE=512 ' + '-co NUM_THREADS=ALL_CPUS ' + '-a_srs \"EPSG:{epsg}\" ' + '{png} {tiff} > {log}'.format(**kwargs)) + + if self.params.build_overviews: + log.ODM_DEBUG("Building Overviews") + kwargs = { + 'orthophoto': tree.odm_orthophoto_tif, + 'log': tree.odm_orthophoto_gdaladdo_log + } + # Run gdaladdo + system.run('gdaladdo -ro -r average ' + '--config BIGTIFF_OVERVIEW IF_SAFER ' + '--config COMPRESS_OVERVIEW JPEG ' + '{orthophoto} 2 4 8 16 > {log}'.format(**kwargs)) + + geotiffcreated = True + if not geotiffcreated: + log.ODM_WARNING('No geo-referenced orthophoto created due ' + 'to missing geo-referencing or corner coordinates.') + + else: + log.ODM_WARNING('Found a valid orthophoto in: %s' % tree.odm_orthophoto_file) + + if args.time: + system.benchmark(start_time, tree.benchmarking, 'Orthophoto') + + log.ODM_INFO('Running ODM OrthoPhoto Cell - Finished') + return ecto.OK if args.end_with != 'odm_orthophoto' else ecto.QUIT From bb8d21ab3b671e782e1ec68b9802cf7c39ae02ca Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 23 Jun 2017 15:28:46 -0400 Subject: [PATCH 05/35] Wrote ODM_DEM cell --- opendm/config.py | 3 +- scripts/odm_app.py | 6 +- scripts/odm_dem.py | 186 +++++++++++++++------------------- scripts/odm_georeferencing.py | 16 --- 4 files changed, 86 insertions(+), 125 deletions(-) diff --git a/opendm/config.py b/opendm/config.py index 38f4a7fd..d1894ed7 100644 --- a/opendm/config.py +++ b/opendm/config.py @@ -360,7 +360,7 @@ def config(): help='Number of steps used to fill areas with gaps. Set to 0 to disable gap filling. ' 'Starting with a radius equal to the output resolution, N different DEMs are generated with ' 'progressively bigger radius using the inverse distance weighted (IDW) algorithm ' - 'and merged together. Remaining gaps are then merged using linear interpolation. ' + 'and merged together. Remaining gaps are then merged using nearest neighbor interpolation. ' 'generation.\nDefault=%(default)s') parser.add_argument('--dem-resolution', @@ -403,6 +403,7 @@ def config(): parser.add_argument('--dem-terrain-type', metavar='', + type=str.lower, # make case insensitive choices=['FlatNonForest', 'FlatForest', 'ComplexNonForest', 'ComplexForest'], default='ComplexForest', help='Specifies the type of terrain. This mainly helps reduce processing time. ' diff --git a/scripts/odm_app.py b/scripts/odm_app.py index bd5c8298..554d4ed4 100644 --- a/scripts/odm_app.py +++ b/scripts/odm_app.py @@ -153,7 +153,11 @@ class ODMApp(ecto.BlackBox): self.args[:] >> self.georeferencing['args'], self.resize['photos'] >> self.georeferencing['photos'], self.texturing['reconstruction'] >> self.georeferencing['reconstruction']] - + + # create odm dem + connections += [self.args[:] >> self.dem['args'], + self.georeferencing['reconstruction'] >> self.dem['reconstruction']] + # create odm orthophoto connections += [self.tree[:] >> self.orthophoto['tree'], self.args[:] >> self.orthophoto['args'], diff --git a/scripts/odm_dem.py b/scripts/odm_dem.py index 070b2d1a..7c141215 100644 --- a/scripts/odm_dem.py +++ b/scripts/odm_dem.py @@ -1,4 +1,4 @@ -import ecto, os +import ecto, os, math from opendm import io from opendm import log @@ -13,6 +13,7 @@ class ODMDemCell(ecto.Cell): def declare_io(self, params, inputs, outputs): inputs.declare("args", "The application arguments.", {}) + inputs.declare("reconstruction", "list of ODMReconstructions", []) def process(self, inputs, outputs): # Benchmarking @@ -22,127 +23,98 @@ class ODMDemCell(ecto.Cell): # get inputs args = self.inputs.args - verbose = '-v' if self.params.verbose else '' + las_model_exists = io.file_exists(tree.odm_georeferencing_model_las) - # define paths and create working directories - odm_dem_root = tree.path('odm_dem') - system.mkdir_p(odm_dem_root) + # Just to make sure + l2d_module_installed = True + try: + system.run('l2d_classify --help') + except: + log.ODM_WARNING('lidar2dems is not installed properly') + l2d_module_installed = False - dsm_output_filename = os.path.join(odm_dem_root, ) + log.ODM_INFO('Create DSM: ' + str(args.dsm)) + log.ODM_INFO('Create DTM: ' + str(args.dtm)) + log.ODM_INFO('{0} exists: {1}'.format(tree.odm_georeferencing_model_las, str(las_model_exists))) - # check if we rerun cell or not - rerun_cell = (args.rerun is not None and - args.rerun == 'odm_dem') or \ - (args.rerun_all) or \ - (args.rerun_from is not None and - 'odm_dem' in args.rerun_from) + # Do we need to process anything here? + if (args.dsm or args.dtm) and las_model_exists and l2d_module_installed: - if not io.file_exists(tree.odm_orthophoto_file) or rerun_cell: + # define paths and create working directories + odm_dem_root = tree.path('odm_dem') + system.mkdir_p(odm_dem_root) - # odm_orthophoto definitions - kwargs = { - 'bin': context.odm_modules_path, - 'log': tree.odm_orthophoto_log, - 'ortho': tree.odm_orthophoto_file, - 'corners': tree.odm_orthophoto_corners, - 'res': self.params.resolution, - 'verbose': verbose - } + dsm_output_filename = os.path.join(odm_dem_root, 'dsm.idw.tif') + dtm_output_filename = os.path.join(odm_dem_root, 'dtm.idw.tif') - # Have geo coordinates? - if io.file_exists(tree.odm_georeferencing_coords): - if args.use_25dmesh: - kwargs['model_geo'] = os.path.join(tree.odm_25dtexturing, tree.odm_georeferencing_model_obj_geo) - else: - kwargs['model_geo'] = os.path.join(tree.odm_texturing, tree.odm_georeferencing_model_obj_geo) - else: - if args.use_25dmesh: - kwargs['model_geo'] = os.path.join(tree.odm_25dtexturing, tree.odm_textured_model_obj) - else: - kwargs['model_geo'] = os.path.join(tree.odm_texturing, tree.odm_textured_model_obj) + # check if we rerun cell or not + rerun_cell = (args.rerun is not None and + args.rerun == 'odm_dem') or \ + (args.rerun_all) or \ + (args.rerun_from is not None and + 'odm_dem' in args.rerun_from) - # run odm_orthophoto - system.run('{bin}/odm_orthophoto -inputFile {model_geo} ' - '-logFile {log} -outputFile {ortho} -resolution {res} {verbose} ' - '-outputCornerFile {corners}'.format(**kwargs)) + if (args.dtm and not io.file_exists(dtm_output_filename)) or \ + (args.dsm and not io.file_exists(dsm_output_filename)) or \ + rerun_cell: - if not io.file_exists(tree.odm_georeferencing_coords): - log.ODM_WARNING('No coordinates file. A georeferenced raster ' - 'will not be created') - else: - # Create georeferenced GeoTiff - geotiffcreated = False - georef = types.ODM_GeoRef() - # creates the coord refs # TODO I don't want to have to do this twice- after odm_georef - georef.parse_coordinate_system(tree.odm_georeferencing_coords) + terrain_params_map = { + 'flatnonforest': (1, 3), + 'flatforest': (1, 2), + 'complexnonforest': (5, 2), + 'complexforest': (10, 2) + } + terrain_params = terrain_params_map[args.dem_terrain_type.lower()] - if georef.epsg and georef.utm_east_offset and georef.utm_north_offset: - ulx = uly = lrx = lry = 0.0 - with open(tree.odm_orthophoto_corners) as f: - for lineNumber, line in enumerate(f): - if lineNumber == 0: - tokens = line.split(' ') - if len(tokens) == 4: - ulx = float(tokens[0]) + \ - float(georef.utm_east_offset) - lry = float(tokens[1]) + \ - float(georef.utm_north_offset) - lrx = float(tokens[2]) + \ - float(georef.utm_east_offset) - uly = float(tokens[3]) + \ - float(georef.utm_north_offset) - log.ODM_INFO('Creating GeoTIFF') + kwargs = { + 'verbose': '-v' if self.params.verbose else '', + 'slope': terrain_params[0], + 'cellsize': terrain_params[1], + 'outdir': odm_dem_root, + 'approximate': '-a' if args.dem_approximate else '', + 'decimation': args.dem_decimation, + } - kwargs = { - 'ulx': ulx, - 'uly': uly, - 'lrx': lrx, - 'lry': lry, - 'tiled': '' if self.params.no_tiled else '-co TILED=yes ', - 'compress': self.params.compress, - 'predictor': '-co PREDICTOR=2 ' if self.params.compress in - ['LZW', 'DEFLATE'] else '', - 'epsg': georef.epsg, - 't_srs': self.params.t_srs or "EPSG:{0}".format(georef.epsg), - 'bigtiff': self.params.bigtiff, - 'png': tree.odm_orthophoto_file, - 'tiff': tree.odm_orthophoto_tif, - 'log': tree.odm_orthophoto_tif_log + l2d_params = '--slope {slope} --cellsize {cellsize} ' \ + '{verbose} {approximate} --decimation {decimation} ' \ + '-o ' \ + '--outdir {outdir}'.format(**kwargs) + + system.run('l2d_classify {0} {1}'.format(l2d_params, tree.odm_georeferencing)) + + products = [] + if args.dsm: products.append('dsm') + if args.dtm: products.append('dtm') + + radius_steps = [args.dem_resolution] + for _ in range(args.dem_gapfill_steps - 1): + radius_steps.append(radius_steps[-1] * math.sqrt(2)) + + for product in products: + demargs = { + 'product': product, + 'indir': odm_dem_root, + 'l2d_params': l2d_params, + 'maxsd': args.dem_maxsd, + 'maxangle': args.dem_maxangle, + 'resolution': args.dem_resolution, + 'radius_steps': ' '.join(map(str, radius_steps)) + 'gapfill': '--gapfill' if len(args.dem_gapfill_steps) > 0 else '' } - system.run('gdal_translate -a_ullr {ulx} {uly} {lrx} {lry} ' - '{tiled} ' - '-co BIGTIFF={bigtiff} ' - '-co COMPRESS={compress} ' - '{predictor} ' - '-co BLOCKXSIZE=512 ' - '-co BLOCKYSIZE=512 ' - '-co NUM_THREADS=ALL_CPUS ' - '-a_srs \"EPSG:{epsg}\" ' - '{png} {tiff} > {log}'.format(**kwargs)) + system.run('l2d_dems {product} {indir} {l2d_params} ' + '--maxsd {maxsd} --maxangle {maxangle} ' + '--resolution {resolution} --radius {radius_steps} ' + '{gapfill} '.format(**demargs)) - if self.params.build_overviews: - log.ODM_DEBUG("Building Overviews") - kwargs = { - 'orthophoto': tree.odm_orthophoto_tif, - 'log': tree.odm_orthophoto_gdaladdo_log - } - # Run gdaladdo - system.run('gdaladdo -ro -r average ' - '--config BIGTIFF_OVERVIEW IF_SAFER ' - '--config COMPRESS_OVERVIEW JPEG ' - '{orthophoto} 2 4 8 16 > {log}'.format(**kwargs)) - - geotiffcreated = True - if not geotiffcreated: - log.ODM_WARNING('No geo-referenced orthophoto created due ' - 'to missing geo-referencing or corner coordinates.') + else: + log.ODM_WARNING('Found existing outputs in: %s' % odm_dem_root) else: - log.ODM_WARNING('Found a valid orthophoto in: %s' % tree.odm_orthophoto_file) + log.ODM_WARNING('DEM will not be generated') if args.time: - system.benchmark(start_time, tree.benchmarking, 'Orthophoto') + system.benchmark(start_time, tree.benchmarking, 'Dem') - log.ODM_INFO('Running ODM OrthoPhoto Cell - Finished') - return ecto.OK if args.end_with != 'odm_orthophoto' else ecto.QUIT + log.ODM_INFO('Running ODM DEM Cell - Finished') + return ecto.OK if args.end_with != 'odm_dem' else ecto.QUIT diff --git a/scripts/odm_georeferencing.py b/scripts/odm_georeferencing.py index b5d03cc4..99f376c1 100644 --- a/scripts/odm_georeferencing.py +++ b/scripts/odm_georeferencing.py @@ -17,10 +17,6 @@ class ODMGeoreferencingCell(ecto.Cell): 'northing height pixelrow pixelcol imagename', 'gcp_list.txt') params.declare("img_size", 'image size used in calibration', 2400) params.declare("use_exif", 'use exif', False) - params.declare("dem", 'Generate a dem', False) - params.declare("sample_radius", "Minimum distance between samples for DEM gen", 3) - params.declare("gdal_res", "Length of raster cell edges in X/Y units ", 2) - params.declare("gdal_radius", "Radius about cell center bounding points to use to calculate a cell value", 0.5) params.declare("verbose", 'print additional messages to console', False) def declare_io(self, params, inputs, outputs): @@ -175,18 +171,6 @@ class ODMGeoreferencingCell(ecto.Cell): tree.odm_georeferencing_model_las, tree.odm_georeferencing_las_json) - # If --dem, create a DEM - if args.dem: - demcreated = geo_ref.convert_to_dem(tree.odm_georeferencing_model_las, - tree.odm_georeferencing_dem, - self.params.sample_radius, - self.params.gdal_res, - self.params.gdal_radius) - if not demcreated: - log.ODM_WARNING('Something went wrong. Check the logs in odm_georeferencing.') - else: - log.ODM_INFO('DEM created at {0}'.format(tree.odm_georeferencing_dem)) - # XYZ point cloud output log.ODM_INFO("Creating geo-referenced CSV file (XYZ format)") with open(tree.odm_georeferencing_xyz_file, "wb") as csvfile: From c41f9121e9d73b4fd5dfc9a3e46d9749f3014937 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 23 Jun 2017 16:15:13 -0400 Subject: [PATCH 06/35] Syntax fix, decimation parameter fix, tweaks --- opendm/config.py | 19 +++++++++---------- opendm/types.py | 4 ++-- scripts/odm_app.py | 3 ++- scripts/odm_dem.py | 33 +++++++++++++++++++-------------- 4 files changed, 32 insertions(+), 27 deletions(-) diff --git a/opendm/config.py b/opendm/config.py index d1894ed7..dfbfe928 100644 --- a/opendm/config.py +++ b/opendm/config.py @@ -345,13 +345,13 @@ def config(): action='store_true', default=False, help='Use this tag to build a DTM (Digital Terrain Model, ground only) using a progressive ' - 'morphological filter in PDAL. Check the --dem* parameters for fine tuning.') + 'morphological filter. Check the --dem* parameters for fine tuning.') parser.add_argument('--dsm', action='store_true', default=False, help='Use this tag to build a DSM (Digital Surface Model, ground + objects) using a progressive ' - 'morphological filter in PDAL. Check the --dem* parameters for fine tuning.') + 'morphological filter. Check the --dem* parameters for fine tuning.') parser.add_argument('--dem-gapfill-steps', metavar='', @@ -390,7 +390,7 @@ def config(): action='store_true', default=False, help='Use this tag use the approximate progressive ' - 'morphological filter in PDAL, which computes DEMs faster ' + 'morphological filter, which computes DEMs faster ' 'but is not as accurate.') parser.add_argument('--dem-decimation', @@ -398,19 +398,18 @@ def config(): default=1, type=int, help='Decimate the points before generating the DEM. 1 is no decimation (full quality). ' - '100 decimates ~99% of the points. Useful for speeding up ' + '100 decimates ~99%% of the points. Useful for speeding up ' 'generation.\nDefault=%(default)s') parser.add_argument('--dem-terrain-type', metavar='', - type=str.lower, # make case insensitive choices=['FlatNonForest', 'FlatForest', 'ComplexNonForest', 'ComplexForest'], default='ComplexForest', - help='Specifies the type of terrain. This mainly helps reduce processing time. ' - 'FlatNonForest: Relatively flat region with little to no vegetation (slope 1, cell size 3)' - 'FlatForest: Relatively flat region that is forested (slope 1, cell size 2)' - 'ComplexNonForest: Varied terrain with little to no vegetation (slope 5, cell size 2)' - 'ComplexForest: Varied terrain that is forested (slope 10, cell size 2)' + help='One of: %(choices)s. Specifies the type of terrain. This mainly helps reduce processing time. ' + '\nFlatNonForest: Relatively flat region with little to no vegetation' + '\nFlatForest: Relatively flat region that is forested' + '\nComplexNonForest: Varied terrain with little to no vegetation' + '\nComplexForest: Varied terrain that is forested' '\nDefault=%(default)s') parser.add_argument('--orthophoto-resolution', diff --git a/opendm/types.py b/opendm/types.py index ea1d9777..3153f777 100644 --- a/opendm/types.py +++ b/opendm/types.py @@ -436,5 +436,5 @@ class ODM_Tree(object): self.odm_orthophoto_tif_log = io.join_paths(self.odm_orthophoto, 'gdal_translate_log.txt') self.odm_orthophoto_gdaladdo_log = io.join_paths(self.odm_orthophoto, 'gdaladdo_log.txt') - def path(self, **args): - return io.join_paths(self.root_path, **args) \ No newline at end of file + def path(self, *args): + return io.join_paths(self.root_path, *args) \ No newline at end of file diff --git a/scripts/odm_app.py b/scripts/odm_app.py index 554d4ed4..aac4a085 100644 --- a/scripts/odm_app.py +++ b/scripts/odm_app.py @@ -155,7 +155,8 @@ class ODMApp(ecto.BlackBox): self.texturing['reconstruction'] >> self.georeferencing['reconstruction']] # create odm dem - connections += [self.args[:] >> self.dem['args'], + connections += [self.tree[:] >> self.dem['tree'], + self.args[:] >> self.dem['args'], self.georeferencing['reconstruction'] >> self.dem['reconstruction']] # create odm orthophoto diff --git a/scripts/odm_dem.py b/scripts/odm_dem.py index 7c141215..9d4fe9c8 100644 --- a/scripts/odm_dem.py +++ b/scripts/odm_dem.py @@ -1,4 +1,4 @@ -import ecto, os, math +import ecto, os from opendm import io from opendm import log @@ -7,11 +7,12 @@ from opendm import context from opendm import types -class ODMDemCell(ecto.Cell): +class ODMDEMCell(ecto.Cell): def declare_params(self, params): params.declare("verbose", 'print additional messages to console', False) def declare_io(self, params, inputs, outputs): + inputs.declare("tree", "Struct with paths", []) inputs.declare("args", "The application arguments.", {}) inputs.declare("reconstruction", "list of ODMReconstructions", []) @@ -23,22 +24,23 @@ class ODMDemCell(ecto.Cell): # get inputs args = self.inputs.args - las_model_exists = io.file_exists(tree.odm_georeferencing_model_las) + tree = self.inputs.tree + las_model_found = io.file_exists(tree.odm_georeferencing_model_las) # Just to make sure l2d_module_installed = True try: - system.run('l2d_classify --help') + system.run('l2d_classify --help > /dev/null') except: log.ODM_WARNING('lidar2dems is not installed properly') l2d_module_installed = False log.ODM_INFO('Create DSM: ' + str(args.dsm)) log.ODM_INFO('Create DTM: ' + str(args.dtm)) - log.ODM_INFO('{0} exists: {1}'.format(tree.odm_georeferencing_model_las, str(las_model_exists))) + log.ODM_INFO('DEM input file {0} found: {1}'.format(tree.odm_georeferencing_model_las, str(las_model_found))) # Do we need to process anything here? - if (args.dsm or args.dtm) and las_model_exists and l2d_module_installed: + if (args.dsm or args.dtm) and las_model_found and l2d_module_installed: # define paths and create working directories odm_dem_root = tree.path('odm_dem') @@ -70,17 +72,20 @@ class ODMDemCell(ecto.Cell): 'verbose': '-v' if self.params.verbose else '', 'slope': terrain_params[0], 'cellsize': terrain_params[1], - 'outdir': odm_dem_root, - 'approximate': '-a' if args.dem_approximate else '', - 'decimation': args.dem_decimation, + 'outdir': odm_dem_root } l2d_params = '--slope {slope} --cellsize {cellsize} ' \ - '{verbose} {approximate} --decimation {decimation} ' \ + '{verbose} ' \ '-o ' \ '--outdir {outdir}'.format(**kwargs) - system.run('l2d_classify {0} {1}'.format(l2d_params, tree.odm_georeferencing)) + approximate = '--approximate' if args.dem_approximate else '' + + system.run('l2d_classify {0} --decimation {1} ' + '{2} {3}'.format( + l2d_params, args.dem_decimation, + approximate, tree.odm_georeferencing)) products = [] if args.dsm: products.append('dsm') @@ -88,7 +93,7 @@ class ODMDemCell(ecto.Cell): radius_steps = [args.dem_resolution] for _ in range(args.dem_gapfill_steps - 1): - radius_steps.append(radius_steps[-1] * math.sqrt(2)) + radius_steps.append(radius_steps[-1] * 3) # 3 is arbitrary, maybe there's a better value? for product in products: demargs = { @@ -98,8 +103,8 @@ class ODMDemCell(ecto.Cell): 'maxsd': args.dem_maxsd, 'maxangle': args.dem_maxangle, 'resolution': args.dem_resolution, - 'radius_steps': ' '.join(map(str, radius_steps)) - 'gapfill': '--gapfill' if len(args.dem_gapfill_steps) > 0 else '' + 'radius_steps': ' '.join(map(str, radius_steps)), + 'gapfill': '--gapfill' if args.dem_gapfill_steps > 0 else '' } system.run('l2d_dems {product} {indir} {l2d_params} ' From 9ddfb18ca46c019fe9a39adaa9bdfaff391ec757 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 23 Jun 2017 16:17:40 -0400 Subject: [PATCH 07/35] Updated checksum --- SuperBuild/cmake/External-Lidar2dems.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SuperBuild/cmake/External-Lidar2dems.cmake b/SuperBuild/cmake/External-Lidar2dems.cmake index 14d30e99..0df4f8bb 100644 --- a/SuperBuild/cmake/External-Lidar2dems.cmake +++ b/SuperBuild/cmake/External-Lidar2dems.cmake @@ -8,7 +8,7 @@ ExternalProject_Add(${_proj_name} #--Download step-------------- DOWNLOAD_DIR ${SB_DOWNLOAD_DIR}/${_proj_name} URL https://github.com/pierotofy/lidar2dems/archive/master.zip - URL_MD5 6cfe2d081cda1680e6e98a9925651b03 + URL_MD5 cece43dcf578ad47f169ee9078e17e8c #--Update/Patch step---------- UPDATE_COMMAND "" #--Configure step------------- From 87b4f5ef568f7e3fe8da067778f49997da54a9c7 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 23 Jun 2017 16:35:21 -0400 Subject: [PATCH 08/35] Removed convert_to_dem --- opendm/types.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/opendm/types.py b/opendm/types.py index 3153f777..71f01503 100644 --- a/opendm/types.py +++ b/opendm/types.py @@ -196,29 +196,6 @@ class ODM_GeoRef(object): system.run('{bin}/pdal pipeline -i {json} --readers.ply.filename={f_in} ' '--writers.las.filename={f_out}'.format(**kwargs)) - def convert_to_dem(self, _file, _file_out, sample_radius, gdal_res, gdal_radius): - # Check if exists f_in - if not io.file_exists(_file): - log.ODM_ERROR('LAS file does not exist') - return False - - kwargs = { - 'bin': context.pdal_path, - 'f_in': _file, - 'sample_radius': sample_radius, - 'gdal_res': gdal_res, - 'gdal_radius': gdal_radius, - 'f_out': _file_out - } - - system.run('{bin}/pdal pipeline {json} --readers.las.filename={f_in} ' - '--writers.gdal.filename={f_out}'.format(**kwargs)) - - if io.file_exists(kwargs['f_out']): - return True - else: - return False - def utm_to_latlon(self, _file, _photo, idx): gcp = self.gcps[idx] From af7050619ae8e4e66b2e4d3c8f1050de273ef65e Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 23 Jun 2017 16:38:01 -0400 Subject: [PATCH 09/35] Typo --- opendm/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opendm/config.py b/opendm/config.py index dfbfe928..3586f4c0 100644 --- a/opendm/config.py +++ b/opendm/config.py @@ -361,7 +361,7 @@ def config(): 'Starting with a radius equal to the output resolution, N different DEMs are generated with ' 'progressively bigger radius using the inverse distance weighted (IDW) algorithm ' 'and merged together. Remaining gaps are then merged using nearest neighbor interpolation. ' - 'generation.\nDefault=%(default)s') + '\nDefault=%(default)s') parser.add_argument('--dem-resolution', metavar='', From 388f1e816a1ae554f267602386e244f1d1f03a2d Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sat, 24 Jun 2017 15:18:39 -0400 Subject: [PATCH 10/35] Labeled units --- opendm/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opendm/config.py b/opendm/config.py index 3586f4c0..64d36dc0 100644 --- a/opendm/config.py +++ b/opendm/config.py @@ -367,7 +367,7 @@ def config(): metavar='', type=float, default=0.1, - help='Length of raster cell edges in X/Y units.' + help='Length of raster cell edges in meters.' '\nDefault: %(default)s') parser.add_argument('--dem-maxangle', From ac37aba605df6bbcdc7323a3288b263dfb37694b Mon Sep 17 00:00:00 2001 From: Kalpaj Chaudhari Date: Mon, 26 Jun 2017 10:27:57 +0530 Subject: [PATCH 11/35] configure.sh: Yes to GAL deps Change-Id: If31faf28039f771238173490152d039a39e682ee --- configure.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.sh b/configure.sh index 9593e849..7fcab7f1 100755 --- a/configure.sh +++ b/configure.sh @@ -72,7 +72,7 @@ install() { appsettings echo "Installing CGAL dependencies" - sudo apt-get install libgmp-dev libmpfr-dev + sudo apt-get install -y libgmp-dev libmpfr-dev echo "Installing Ecto Dependencies" sudo pip install -U catkin-pkg @@ -134,4 +134,4 @@ else echo "Invalid instructions." >&2 usage exit 1 -fi \ No newline at end of file +fi From af5e231f2b62243c0525a4f0c7e98e5bb0ac6271 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 26 Jun 2017 12:09:14 -0400 Subject: [PATCH 12/35] Upgraded PDAL to include ground kernel patch commit --- SuperBuild/cmake/External-PDAL.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SuperBuild/cmake/External-PDAL.cmake b/SuperBuild/cmake/External-PDAL.cmake index 7faae6e2..bfd77488 100644 --- a/SuperBuild/cmake/External-PDAL.cmake +++ b/SuperBuild/cmake/External-PDAL.cmake @@ -7,8 +7,8 @@ ExternalProject_Add(${_proj_name} STAMP_DIR ${_SB_BINARY_DIR}/stamp #--Download step-------------- DOWNLOAD_DIR ${SB_DOWNLOAD_DIR} - URL https://github.com/PDAL/PDAL/archive/aea5bb0cacc64b91d626eff491fbdbb5668c06d7.tar.gz - URL_MD5 726933f63f661e11e13775d6ce4e5ed0 + URL https://github.com/PDAL/PDAL/archive/e881b581e3b91a928105d67db44c567f3b6d1afe.tar.gz + URL_MD5 438acbb736ba01fbe8f9ca7cdbf113bf #--Update/Patch step---------- UPDATE_COMMAND "" #--Configure step------------- From 98a92905d97ecaaafeb792316e435dea6a6999d8 Mon Sep 17 00:00:00 2001 From: Tomasz Nycz Date: Mon, 26 Jun 2017 19:38:36 +0200 Subject: [PATCH 13/35] Initial commit for vegind.py Vegetation Indexes for Visible RGB imagery calculations, Usage: ./vegind.py index Available indexes: ngrdi (Normalized green red difference index), vari (Visible Atmospheric Resistant Index), tgi (Triangular Greenness Index). Work inspired by DroneDeploy Plant Health app, that uses VARI index too. --- contrib/visveg/vegind.py | 93 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 contrib/visveg/vegind.py diff --git a/contrib/visveg/vegind.py b/contrib/visveg/vegind.py new file mode 100644 index 00000000..9abb58dc --- /dev/null +++ b/contrib/visveg/vegind.py @@ -0,0 +1,93 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +import rasterio, os, sys +import numpy as np + +class bcolors: + OKBLUE = '\033[94m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + +try: + file = sys.argv[1] + typ = sys.argv[2] + (fileRoot, fileExt) = os.path.splitext(file) + outFileName = fileRoot + "_" + typ + fileExt +except (TypeError, IndexError, NameError): + print bcolors.FAIL + 'Arguments messed up. Check arguments order and index name' + bcolors.ENDC + print 'Usage: ./vegind.py orto index' + print ' orto - filepath to RGB orthophoto' + print ' index - Vegetation Index' + print bcolors.OKGREEN + 'Available indexes: vari, ngrdi, tgi' + bcolors.ENDC + sys.exit() + + +def calcNgrdi(red, green): + """ + Normalized green red difference index + Tucker,C.J.,1979. + Red and photographic infrared linear combinations for monitoring vegetation. + Remote Sensing of Environment 8, 127–150 + :param red: red visible channel + :param green: green visible channel + :return: ngrdi index array + """ + mask = np.not_equal(np.add(red,green), 0.0) + return np.choose(mask, (-9999.0, np.true_divide( + np.subtract(green,red), + np.add(red,green)))) + +def calcVari(red,green,blue): + """ + Calculates Visible Atmospheric Resistant Index + Gitelson, A.A., Kaufman, Y.J., Stark, R., Rundquist, D., 2002. + Novel algorithms for remote estimation of vegetation fraction. + Remote Sensing of Environment 80, 76–87. + :param red: red visible channel + :param green: green visible channel + :param blue: blue visible channel + :return: vari index array, that will be saved to tiff + """ + mask = np.not_equal(np.subtract(np.add(green,red),blue), 0.0) + return np.choose(mask, (-9999.0, np.true_divide(np.subtract(green,red),np.subtract(np.add(green,red),blue)))) + +def calcTgi(red,green,blue): + """ + Calculates Triangular Greenness Index + Hunt, E. Raymond Jr.; Doraiswamy, Paul C.; McMurtrey, James E.; Daughtry, Craig S.T.; Perry, Eileen M.; and Akhmedov, Bakhyt, + A visible band index for remote sensing leaf chlorophyll content at the canopy scale (2013). + Publications from USDA-ARS / UNL Faculty. Paper 1156. + http://digitalcommons.unl.edu/usdaarsfacpub/1156 + :param red: red channel + :param green: green channel + :param blue: blue channel + :return: tgi index array, that will be saved to tiff + """ + mask = np.not_equal(green-red+blue-255.0, 0.0) + return np.choose(mask, (-9999.0, np.subtract(green, np.multiply(0.39,red), np.multiply(0.61, blue)))) + +try: + with rasterio.Env(): + ds = rasterio.open(file) + profile = ds.profile + profile.update(dtype=rasterio.float32, count=1, nodata=-9999) + red = np.float32(ds.read(1)) + green = np.float32(ds.read(2)) + blue = np.float32(ds.read(3)) + np.seterr(divide='ignore', invalid='ignore') + if typ == 'ngrdi': + indeks = calcNgrdi(red,green) + elif typ == 'vari': + indeks = calcVari(red, green, blue) + elif typ == 'tgi': + indeks = calcTgi(red, green, blue) + + with rasterio.open(outFileName, 'w', **profile) as dst: + dst.write(indeks.astype(rasterio.float32), 1) +except rasterio.errors.RasterioIOError: + print bcolors.FAIL + 'Orthophoto file not found or access denied' + bcolors.ENDC + sys.exit() From b07ce7baf6f98c2307b037ce4ee381dffcecbcee Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 27 Jun 2017 10:39:39 -0400 Subject: [PATCH 14/35] Updated l2d md5 --- SuperBuild/cmake/External-Lidar2dems.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SuperBuild/cmake/External-Lidar2dems.cmake b/SuperBuild/cmake/External-Lidar2dems.cmake index 0df4f8bb..e0dea256 100644 --- a/SuperBuild/cmake/External-Lidar2dems.cmake +++ b/SuperBuild/cmake/External-Lidar2dems.cmake @@ -8,7 +8,7 @@ ExternalProject_Add(${_proj_name} #--Download step-------------- DOWNLOAD_DIR ${SB_DOWNLOAD_DIR}/${_proj_name} URL https://github.com/pierotofy/lidar2dems/archive/master.zip - URL_MD5 cece43dcf578ad47f169ee9078e17e8c + URL_MD5 dc1ac68bd05d1a5dde45ee079e0765ac #--Update/Patch step---------- UPDATE_COMMAND "" #--Configure step------------- From 1950788412925ce54fc2e46ef00178ff65639d18 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 27 Jun 2017 12:43:36 -0400 Subject: [PATCH 15/35] Added automated DEM clipping --- Dockerfile | 2 +- SuperBuild/CMakeLists.txt | 4 ++ SuperBuild/cmake/External-Hexer.cmake | 27 +++++++++++++ SuperBuild/cmake/External-PDAL.cmake | 3 +- configure.sh | 3 +- scripts/odm_dem.py | 56 ++++++++++++++++++++++++--- 6 files changed, 87 insertions(+), 8 deletions(-) create mode 100644 SuperBuild/cmake/External-Hexer.cmake diff --git a/Dockerfile b/Dockerfile index e683f3e0..b35d7b44 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ RUN apt-get install --no-install-recommends -y git cmake python-pip build-essent libgtk2.0-dev libavcodec-dev libavformat-dev libswscale-dev python-dev python-numpy libtbb2 libtbb-dev libjpeg-dev libpng-dev libtiff-dev libjasper-dev libflann-dev \ libproj-dev libxext-dev liblapack-dev libeigen3-dev libvtk5-dev python-networkx libgoogle-glog-dev libsuitesparse-dev libboost-filesystem-dev libboost-iostreams-dev \ libboost-regex-dev libboost-python-dev libboost-date-time-dev libboost-thread-dev python-pyproj python-empy python-nose python-pyside python-pyexiv2 python-scipy \ -jhead liblas-bin python-matplotlib libatlas-base-dev libgmp-dev libmpfr-dev swig2.0 python-wheel libboost-log-dev +jhead liblas-bin python-matplotlib libatlas-base-dev libgmp-dev libmpfr-dev swig2.0 python-wheel libboost-log-dev libjsoncpp-dev RUN apt-get remove libdc1394-22-dev RUN pip install --upgrade pip diff --git a/SuperBuild/CMakeLists.txt b/SuperBuild/CMakeLists.txt index b6892d06..453c3fb4 100644 --- a/SuperBuild/CMakeLists.txt +++ b/SuperBuild/CMakeLists.txt @@ -98,6 +98,10 @@ option(ODM_BUILD_CGAL "Force to build CGAL library" OFF) SETUP_EXTERNAL_PROJECT(CGAL ${ODM_CGAL_Version} ${ODM_BUILD_CGAL}) +# --------------------------------------------------------------------------------------------- +# Hexer +# +SETUP_EXTERNAL_PROJECT(Hexer 1.4 ON) # --------------------------------------------------------------------------------------------- # Open Geometric Vision (OpenGV) diff --git a/SuperBuild/cmake/External-Hexer.cmake b/SuperBuild/cmake/External-Hexer.cmake new file mode 100644 index 00000000..64de1ae2 --- /dev/null +++ b/SuperBuild/cmake/External-Hexer.cmake @@ -0,0 +1,27 @@ +set(_proj_name hexer) +set(_SB_BINARY_DIR "${SB_BINARY_DIR}/${_proj_name}") + +ExternalProject_Add(${_proj_name} + DEPENDS + PREFIX ${_SB_BINARY_DIR} + TMP_DIR ${_SB_BINARY_DIR}/tmp + STAMP_DIR ${_SB_BINARY_DIR}/stamp + #--Download step-------------- + DOWNLOAD_DIR ${SB_DOWNLOAD_DIR} + URL https://github.com/hobu/hexer/archive/2898b96b1105991e151696391b9111610276258f.tar.gz + URL_MD5 e8f2788332ad212cf78efa81a82e95dd + #--Update/Patch step---------- + UPDATE_COMMAND "" + #--Configure step------------- + SOURCE_DIR ${SB_SOURCE_DIR}/${_proj_name} + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX:PATH=${SB_INSTALL_DIR} + #--Build step----------------- + BINARY_DIR ${_SB_BINARY_DIR} + #--Install step--------------- + INSTALL_DIR ${SB_INSTALL_DIR} + #--Output logging------------- + LOG_DOWNLOAD OFF + LOG_CONFIGURE OFF + LOG_BUILD OFF +) diff --git a/SuperBuild/cmake/External-PDAL.cmake b/SuperBuild/cmake/External-PDAL.cmake index bfd77488..7ff10867 100644 --- a/SuperBuild/cmake/External-PDAL.cmake +++ b/SuperBuild/cmake/External-PDAL.cmake @@ -2,6 +2,7 @@ set(_proj_name pdal) set(_SB_BINARY_DIR "${SB_BINARY_DIR}/${_proj_name}") ExternalProject_Add(${_proj_name} + DEPENDS hexer PREFIX ${_SB_BINARY_DIR} TMP_DIR ${_SB_BINARY_DIR}/tmp STAMP_DIR ${_SB_BINARY_DIR}/stamp @@ -19,7 +20,7 @@ ExternalProject_Add(${_proj_name} -BUILD_PLUGIN_PGPOINTCLOUD=ON -DBUILD_PLUGIN_CPD=OFF -DBUILD_PLUGIN_GREYHOUND=OFF - -DBUILD_PLUGIN_HEXBIN=OFF + -DBUILD_PLUGIN_HEXBIN=ON -DBUILD_PLUGIN_ICEBRIDGE=OFF -DBUILD_PLUGIN_MRSID=OFF -DBUILD_PLUGIN_NITF=OFF diff --git a/configure.sh b/configure.sh index f5321c05..de9522f1 100755 --- a/configure.sh +++ b/configure.sh @@ -21,7 +21,8 @@ install() { libgdal-dev \ gdal-bin \ libgeotiff-dev \ - pkg-config + pkg-config \ + libjsoncpp-dev echo "Getting CMake 3.1 for MVS-Texturing" sudo apt-get install -y software-properties-common python-software-properties diff --git a/scripts/odm_dem.py b/scripts/odm_dem.py index 9d4fe9c8..aceade15 100644 --- a/scripts/odm_dem.py +++ b/scripts/odm_dem.py @@ -1,4 +1,4 @@ -import ecto, os +import ecto, os, json from opendm import io from opendm import log @@ -46,8 +46,8 @@ class ODMDEMCell(ecto.Cell): odm_dem_root = tree.path('odm_dem') system.mkdir_p(odm_dem_root) - dsm_output_filename = os.path.join(odm_dem_root, 'dsm.idw.tif') - dtm_output_filename = os.path.join(odm_dem_root, 'dtm.idw.tif') + dsm_output_filename = os.path.join(odm_dem_root, 'dsm.tif') + dtm_output_filename = os.path.join(odm_dem_root, 'dtm.tif') # check if we rerun cell or not rerun_cell = (args.rerun is not None and @@ -60,6 +60,46 @@ class ODMDEMCell(ecto.Cell): (args.dsm and not io.file_exists(dsm_output_filename)) or \ rerun_cell: + # Extract boundaries and srs of point cloud + summary_file_path = os.path.join(odm_dem_root, 'odm_georeferenced_model.summary.json') + boundary_file_path = os.path.join(odm_dem_root, 'odm_georeferenced_model.boundary.json') + + system.run('pdal info --summary {0} > {1}'.format(tree.odm_georeferencing_model_las, summary_file_path)) + system.run('pdal info --boundary {0} > {1}'.format(tree.odm_georeferencing_model_las, boundary_file_path)) + + pc_proj4 = "" + pc_geojson_bounds_feature = None + + with open(summary_file_path, 'r') as f: + json_f = json.loads(f.read()) + pc_proj4 = json_f['summary']['srs']['proj4'] + + with open(boundary_file_path, 'r') as f: + json_f = json.loads(f.read()) + pc_geojson_boundary_feature = json_f['boundary']['boundary_json'] + + # Write bounds to GeoJSON + bounds_geojson_path = os.path.join(odm_dem_root, 'odm_georeferenced_model.bounds.geojson') + with open(bounds_geojson_path, "w") as f: + f.write(json.dumps({ + "type": "FeatureCollection", + "features": [{ + "type": "Feature", + "geometry": pc_geojson_boundary_feature + }] + })) + + bounds_shapefile_path = os.path.join(odm_dem_root, 'bounds.shp') + + # Convert bounds to Shapefile + kwargs = { + 'input': bounds_geojson_path, + 'output': bounds_shapefile_path, + 'proj4': pc_proj4 + } + system.run('ogr2ogr -overwrite -a_srs "{proj4}" {output} {input}'.format(**kwargs)) + + # Process with lidar2dems terrain_params_map = { 'flatnonforest': (1, 3), 'flatforest': (1, 2), @@ -72,12 +112,13 @@ class ODMDEMCell(ecto.Cell): 'verbose': '-v' if self.params.verbose else '', 'slope': terrain_params[0], 'cellsize': terrain_params[1], - 'outdir': odm_dem_root + 'outdir': odm_dem_root, + 'site': bounds_shapefile_path } l2d_params = '--slope {slope} --cellsize {cellsize} ' \ '{verbose} ' \ - '-o ' \ + '-o -s {site} ' \ '--outdir {outdir}'.format(**kwargs) approximate = '--approximate' if args.dem_approximate else '' @@ -112,6 +153,11 @@ class ODMDEMCell(ecto.Cell): '--resolution {resolution} --radius {radius_steps} ' '{gapfill} '.format(**demargs)) + # Rename final output + if product == 'dsm': + os.rename(os.path.join(odm_dem_root, 'bounds-0_dsm.idw.tif'), dsm_output_filename) + elif product == 'dtm': + os.rename(os.path.join(odm_dem_root, 'bounds-0_dtm.idw.tif'), dtm_output_filename) else: log.ODM_WARNING('Found existing outputs in: %s' % odm_dem_root) From 9eaa0f16c4f252b818a73c3aadaf133dfdb8b6a0 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 27 Jun 2017 13:14:09 -0400 Subject: [PATCH 16/35] Skip classification if only DSM is requested, decimation parameter passing --- SuperBuild/cmake/External-Lidar2dems.cmake | 2 +- scripts/odm_dem.py | 22 +++++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/SuperBuild/cmake/External-Lidar2dems.cmake b/SuperBuild/cmake/External-Lidar2dems.cmake index e0dea256..5e66d05f 100644 --- a/SuperBuild/cmake/External-Lidar2dems.cmake +++ b/SuperBuild/cmake/External-Lidar2dems.cmake @@ -8,7 +8,7 @@ ExternalProject_Add(${_proj_name} #--Download step-------------- DOWNLOAD_DIR ${SB_DOWNLOAD_DIR}/${_proj_name} URL https://github.com/pierotofy/lidar2dems/archive/master.zip - URL_MD5 dc1ac68bd05d1a5dde45ee079e0765ac + URL_MD5 ba3f7da85ae569fe2610646f540d2828 #--Update/Patch step---------- UPDATE_COMMAND "" #--Configure step------------- diff --git a/scripts/odm_dem.py b/scripts/odm_dem.py index aceade15..0e927b32 100644 --- a/scripts/odm_dem.py +++ b/scripts/odm_dem.py @@ -1,4 +1,5 @@ import ecto, os, json +from shutil import copyfile from opendm import io from opendm import log @@ -123,10 +124,17 @@ class ODMDEMCell(ecto.Cell): approximate = '--approximate' if args.dem_approximate else '' - system.run('l2d_classify {0} --decimation {1} ' - '{2} {3}'.format( - l2d_params, args.dem_decimation, - approximate, tree.odm_georeferencing)) + # Classify only if we need a DTM + run_classification = args.dtm + + if run_classification: + system.run('l2d_classify {0} --decimation {1} ' + '{2} {3}'.format( + l2d_params, args.dem_decimation, + approximate, tree.odm_georeferencing)) + else: + log.ODM_INFO("Will skip classification, only DSM is needed") + copyfile(tree.odm_georeferencing_model_las, os.path.join(odm_dem_root, 'bounds-0_l2d_s{slope}c{cellsize}.las'.format(**kwargs))) products = [] if args.dsm: products.append('dsm') @@ -145,12 +153,16 @@ class ODMDEMCell(ecto.Cell): 'maxangle': args.dem_maxangle, 'resolution': args.dem_resolution, 'radius_steps': ' '.join(map(str, radius_steps)), - 'gapfill': '--gapfill' if args.dem_gapfill_steps > 0 else '' + 'gapfill': '--gapfill' if args.dem_gapfill_steps > 0 else '', + + # If we didn't run a classification, we should pass the decimate parameter here + 'decimation': '--decimation {0}'.format(args.dem_decimation) if not run_classification else '' } system.run('l2d_dems {product} {indir} {l2d_params} ' '--maxsd {maxsd} --maxangle {maxangle} ' '--resolution {resolution} --radius {radius_steps} ' + '{decimation} ' '{gapfill} '.format(**demargs)) # Rename final output From a414259e6fa63bbe07231bc2e0c2f4126b0a2248 Mon Sep 17 00:00:00 2001 From: Tomasz Nycz Date: Wed, 28 Jun 2017 20:39:20 +0200 Subject: [PATCH 17/35] Create readme.md Visible/RGB Vegetation Indices contrib module description, usage, and bibliography --- contrib/visveg/readme.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 contrib/visveg/readme.md diff --git a/contrib/visveg/readme.md b/contrib/visveg/readme.md new file mode 100644 index 00000000..31ba0711 --- /dev/null +++ b/contrib/visveg/readme.md @@ -0,0 +1,31 @@ +# Visible Vegetation Indexes + +This script produces a Vegetation Index raster from a RGB orthophoto (odm_orthophoto.tif in your project) + +## Requirements +* rasterio (pip install rasterio) +* numpy python package (included in ODM build) + +## Usage +``` +vegind.py index + +positional arguments: + The RGB orthophoto. Must be a GeoTiff. + index Index identifier. Allowed values: ngrdi, tgi, vari + +Output will be generated with index suffix in the same directory as input. + +## Examples + +`python vegind.py /path/to/odm_orthophoto.tif tgi` + +Orthophoto photo of Koniaków grass field and forest in QGIS: ![](http://imgur.com/K6x3nB2.jpg) +The Triangular Greenness Index output in QGIS (with a spectral pseudocolor): ![](http://i.imgur.com/f9TzISU.jpg) +Visible Atmospheric Resistant Index: ![](http://imgur.com/Y7BHzLs.jpg) +Normalized green-red difference index: ![](http://imgur.com/v8cmaPS.jpg) + +## Bibliography + +1. Hunt, E. Raymond, et al. "A Visible Band Index for Remote Sensing Leaf Chlorophyll Content At the Canopy Scale." ITC journal 21(2013): 103-112. doi: 10.1016/j.jag.2012.07.020 +(https://doi.org/10.1016/j.jag.2012.07.020) From cc1c9a06c47453ca6668dde9de1d9543b275eaf7 Mon Sep 17 00:00:00 2001 From: Tomasz Nycz Date: Wed, 28 Jun 2017 20:42:48 +0200 Subject: [PATCH 18/35] update usage section update section apostrophes. --- contrib/visveg/readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/visveg/readme.md b/contrib/visveg/readme.md index 31ba0711..1d883be4 100644 --- a/contrib/visveg/readme.md +++ b/contrib/visveg/readme.md @@ -13,7 +13,7 @@ vegind.py index positional arguments: The RGB orthophoto. Must be a GeoTiff. index Index identifier. Allowed values: ngrdi, tgi, vari - +``` Output will be generated with index suffix in the same directory as input. ## Examples From 3391b09d09b3f306f418d6a1368c7c6565da71ad Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 28 Jun 2017 15:00:29 -0400 Subject: [PATCH 19/35] Updated repo URLs, modified core2.dockerfile --- Dockerfile | 2 +- SuperBuild/cmake/External-Lidar2dems.cmake | 2 +- configure.sh | 2 +- core2.Dockerfile | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index b35d7b44..850e53ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ jhead liblas-bin python-matplotlib libatlas-base-dev libgmp-dev libmpfr-dev swig RUN apt-get remove libdc1394-22-dev RUN pip install --upgrade pip RUN pip install setuptools -RUN pip install -U PyYAML exifread gpxpy xmltodict catkin-pkg appsettings https://github.com/pierotofy/gippy/archive/v0.3.9.tar.gz +RUN pip install -U PyYAML exifread gpxpy xmltodict catkin-pkg appsettings https://github.com/OpenDroneMap/gippy/archive/v0.3.9.tar.gz ENV PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python2.7/dist-packages" ENV PYTHONPATH="$PYTHONPATH:/code/SuperBuild/src/opensfm" diff --git a/SuperBuild/cmake/External-Lidar2dems.cmake b/SuperBuild/cmake/External-Lidar2dems.cmake index 5e66d05f..6b888f68 100644 --- a/SuperBuild/cmake/External-Lidar2dems.cmake +++ b/SuperBuild/cmake/External-Lidar2dems.cmake @@ -7,7 +7,7 @@ ExternalProject_Add(${_proj_name} STAMP_DIR ${_SB_BINARY_DIR}/stamp #--Download step-------------- DOWNLOAD_DIR ${SB_DOWNLOAD_DIR}/${_proj_name} - URL https://github.com/pierotofy/lidar2dems/archive/master.zip + URL https://github.com/OpenDroneMap/lidar2dems/archive/master.zip URL_MD5 ba3f7da85ae569fe2610646f540d2828 #--Update/Patch step---------- UPDATE_COMMAND "" diff --git a/configure.sh b/configure.sh index 20584a67..dd3ca8db 100755 --- a/configure.sh +++ b/configure.sh @@ -92,7 +92,7 @@ install() { python-wheel \ libboost-log-dev - sudo pip install -U https://github.com/pierotofy/gippy/archive/v0.3.9.tar.gz + sudo pip install -U https://github.com/OpenDroneMap/gippy/archive/v0.3.9.tar.gz echo "Compiling SuperBuild" cd ${RUNPATH}/SuperBuild diff --git a/core2.Dockerfile b/core2.Dockerfile index 52a06c07..f8294c65 100644 --- a/core2.Dockerfile +++ b/core2.Dockerfile @@ -14,12 +14,12 @@ RUN apt-get install --no-install-recommends -y git cmake python-pip build-essent libgtk2.0-dev libavcodec-dev libavformat-dev libswscale-dev python-dev python-numpy libtbb2 libtbb-dev libjpeg-dev libpng-dev libtiff-dev libjasper-dev libflann-dev \ libproj-dev libxext-dev liblapack-dev libeigen3-dev libvtk5-dev python-networkx libgoogle-glog-dev libsuitesparse-dev libboost-filesystem-dev libboost-iostreams-dev \ libboost-regex-dev libboost-python-dev libboost-date-time-dev libboost-thread-dev python-pyproj python-empy python-nose python-pyside python-pyexiv2 python-scipy \ -jhead liblas-bin python-matplotlib libatlas-base-dev libgmp-dev libmpfr-dev +jhead liblas-bin python-matplotlib libatlas-base-dev libgmp-dev libmpfr-dev swig2.0 python-wheel libboost-log-dev libjsoncpp-dev RUN apt-get remove libdc1394-22-dev RUN pip install --upgrade pip RUN pip install setuptools -RUN pip install -U PyYAML exifread gpxpy xmltodict catkin-pkg appsettings +RUN pip install -U PyYAML exifread gpxpy xmltodict catkin-pkg appsettings https://github.com/OpenDroneMap/gippy/archive/v0.3.9.tar.gz ENV PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python2.7/dist-packages" ENV PYTHONPATH="$PYTHONPATH:/code/SuperBuild/src/opensfm" From ad47dd9fd4d217dc1d18a629f4a6dbce513d654c Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 29 Jun 2017 10:55:43 -0400 Subject: [PATCH 20/35] Added env_paths option in system.run, added path to superbuild install directory for odm_dem module, fixed lidar2dems install --- SuperBuild/cmake/External-Lidar2dems.cmake | 4 ++-- opendm/context.py | 1 + opendm/system.py | 10 ++++++++-- scripts/odm_dem.py | 11 ++++++----- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/SuperBuild/cmake/External-Lidar2dems.cmake b/SuperBuild/cmake/External-Lidar2dems.cmake index 6b888f68..7920148a 100644 --- a/SuperBuild/cmake/External-Lidar2dems.cmake +++ b/SuperBuild/cmake/External-Lidar2dems.cmake @@ -8,7 +8,7 @@ ExternalProject_Add(${_proj_name} #--Download step-------------- DOWNLOAD_DIR ${SB_DOWNLOAD_DIR}/${_proj_name} URL https://github.com/OpenDroneMap/lidar2dems/archive/master.zip - URL_MD5 ba3f7da85ae569fe2610646f540d2828 + URL_MD5 76ae5d0182f280bec890bdfd156aa538 #--Update/Patch step---------- UPDATE_COMMAND "" #--Configure step------------- @@ -17,7 +17,7 @@ ExternalProject_Add(${_proj_name} #--Build step----------------- BUILD_COMMAND "" #--Install step--------------- - INSTALL_COMMAND "${SB_SOURCE_DIR}/${_proj_name}/install.sh" + INSTALL_COMMAND "${SB_SOURCE_DIR}/${_proj_name}/install.sh" "${SB_INSTALL_DIR}" #--Output logging------------- LOG_DOWNLOAD OFF LOG_CONFIGURE OFF diff --git a/opendm/context.py b/opendm/context.py index 9e1b7e12..198b07a8 100644 --- a/opendm/context.py +++ b/opendm/context.py @@ -8,6 +8,7 @@ scripts_path = os.path.abspath(os.path.dirname(__file__)) root_path, _ = os.path.split(scripts_path) superbuild_path = os.path.join(root_path, 'SuperBuild') +superbuild_bin_path = os.path.join(superbuild_path, 'install', 'bin') tests_path = os.path.join(root_path, 'tests') tests_data_path = os.path.join(root_path, 'tests/test_data') diff --git a/opendm/system.py b/opendm/system.py index d221ce17..740246d0 100644 --- a/opendm/system.py +++ b/opendm/system.py @@ -17,10 +17,16 @@ def get_ccd_widths(): return dict(zip(map(string.lower, sensor_data.keys()), sensor_data.values())) -def run(cmd): +def run(cmd, env_paths=[]): """Run a system command""" log.ODM_DEBUG('running %s' % cmd) - retcode = subprocess.call(cmd, shell=True) + + env = None + if len(env_paths) > 0: + env = os.environ.copy() + env["PATH"] = env["PATH"] + ":" + ":".join(env_paths) + + retcode = subprocess.call(cmd, shell=True, env=env) if retcode < 0: raise Exception("Child was terminated by signal {}".format(-retcode)) diff --git a/scripts/odm_dem.py b/scripts/odm_dem.py index 0e927b32..7ec4f5b0 100644 --- a/scripts/odm_dem.py +++ b/scripts/odm_dem.py @@ -27,11 +27,12 @@ class ODMDEMCell(ecto.Cell): args = self.inputs.args tree = self.inputs.tree las_model_found = io.file_exists(tree.odm_georeferencing_model_las) + env_paths = [tree.superbuild_bin_path] # Just to make sure l2d_module_installed = True try: - system.run('l2d_classify --help > /dev/null') + system.run('l2d_classify --help > /dev/null', env_paths) except: log.ODM_WARNING('lidar2dems is not installed properly') l2d_module_installed = False @@ -65,8 +66,8 @@ class ODMDEMCell(ecto.Cell): summary_file_path = os.path.join(odm_dem_root, 'odm_georeferenced_model.summary.json') boundary_file_path = os.path.join(odm_dem_root, 'odm_georeferenced_model.boundary.json') - system.run('pdal info --summary {0} > {1}'.format(tree.odm_georeferencing_model_las, summary_file_path)) - system.run('pdal info --boundary {0} > {1}'.format(tree.odm_georeferencing_model_las, boundary_file_path)) + system.run('pdal info --summary {0} > {1}'.format(tree.odm_georeferencing_model_las, summary_file_path), env_paths) + system.run('pdal info --boundary {0} > {1}'.format(tree.odm_georeferencing_model_las, boundary_file_path), env_paths) pc_proj4 = "" pc_geojson_bounds_feature = None @@ -131,7 +132,7 @@ class ODMDEMCell(ecto.Cell): system.run('l2d_classify {0} --decimation {1} ' '{2} {3}'.format( l2d_params, args.dem_decimation, - approximate, tree.odm_georeferencing)) + approximate, tree.odm_georeferencing), env_paths) else: log.ODM_INFO("Will skip classification, only DSM is needed") copyfile(tree.odm_georeferencing_model_las, os.path.join(odm_dem_root, 'bounds-0_l2d_s{slope}c{cellsize}.las'.format(**kwargs))) @@ -163,7 +164,7 @@ class ODMDEMCell(ecto.Cell): '--maxsd {maxsd} --maxangle {maxangle} ' '--resolution {resolution} --radius {radius_steps} ' '{decimation} ' - '{gapfill} '.format(**demargs)) + '{gapfill} '.format(**demargs), env_paths) # Rename final output if product == 'dsm': From 750a4492b12b9265aeac3e79c3400be177170101 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 29 Jun 2017 11:00:52 -0400 Subject: [PATCH 21/35] Fixed path error --- scripts/odm_dem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/odm_dem.py b/scripts/odm_dem.py index 7ec4f5b0..8bbf05f0 100644 --- a/scripts/odm_dem.py +++ b/scripts/odm_dem.py @@ -27,7 +27,7 @@ class ODMDEMCell(ecto.Cell): args = self.inputs.args tree = self.inputs.tree las_model_found = io.file_exists(tree.odm_georeferencing_model_las) - env_paths = [tree.superbuild_bin_path] + env_paths = [context.superbuild_bin_path] # Just to make sure l2d_module_installed = True From c374f7ae12adead3edfe4a49816d5f5bcde2a2e7 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 6 Jul 2017 14:21:42 -0400 Subject: [PATCH 22/35] Added dependency on opensfm --- SuperBuild/cmake/External-Lidar2dems.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/SuperBuild/cmake/External-Lidar2dems.cmake b/SuperBuild/cmake/External-Lidar2dems.cmake index 7920148a..02695426 100644 --- a/SuperBuild/cmake/External-Lidar2dems.cmake +++ b/SuperBuild/cmake/External-Lidar2dems.cmake @@ -2,6 +2,7 @@ set(_proj_name lidar2dems) set(_SB_BINARY_DIR "${SB_BINARY_DIR}/${_proj_name}") ExternalProject_Add(${_proj_name} + DEPENDS opensfm PREFIX ${_SB_BINARY_DIR} TMP_DIR ${_SB_BINARY_DIR}/tmp STAMP_DIR ${_SB_BINARY_DIR}/stamp From 009b58be34209dbc0f138c97b9d503500228d9c6 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 6 Jul 2017 14:39:58 -0400 Subject: [PATCH 23/35] Removed MD5 check for lidar2dems --- SuperBuild/cmake/External-Lidar2dems.cmake | 2 -- 1 file changed, 2 deletions(-) diff --git a/SuperBuild/cmake/External-Lidar2dems.cmake b/SuperBuild/cmake/External-Lidar2dems.cmake index 02695426..4772a246 100644 --- a/SuperBuild/cmake/External-Lidar2dems.cmake +++ b/SuperBuild/cmake/External-Lidar2dems.cmake @@ -2,14 +2,12 @@ set(_proj_name lidar2dems) set(_SB_BINARY_DIR "${SB_BINARY_DIR}/${_proj_name}") ExternalProject_Add(${_proj_name} - DEPENDS opensfm PREFIX ${_SB_BINARY_DIR} TMP_DIR ${_SB_BINARY_DIR}/tmp STAMP_DIR ${_SB_BINARY_DIR}/stamp #--Download step-------------- DOWNLOAD_DIR ${SB_DOWNLOAD_DIR}/${_proj_name} URL https://github.com/OpenDroneMap/lidar2dems/archive/master.zip - URL_MD5 76ae5d0182f280bec890bdfd156aa538 #--Update/Patch step---------- UPDATE_COMMAND "" #--Configure step------------- From a32e385d9f2fcf15e359ca5e8872267b1f8b0bd6 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 6 Jul 2017 19:40:03 -0400 Subject: [PATCH 24/35] Added dem-initial-distance parameter --- opendm/config.py | 10 ++++++++++ scripts/odm_dem.py | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/opendm/config.py b/opendm/config.py index 64d36dc0..8a48968b 100644 --- a/opendm/config.py +++ b/opendm/config.py @@ -386,6 +386,16 @@ def config(): 'are discarded. \nDefault: ' '%(default)s') + parser.add_argument('--dem-initial-distance', + metavar='', + type=float, + default=0.5, + help='Used to classify ground vs non-ground points. Set this value to account for Z noise in meters. ' + 'If you have an uncertainty of around 50 cm, set this value large enough to not exclude these points. ' + 'Too small of a value will exclude valid ground points, while too large of a value will misclassify non-ground points for ground ones. ' + '\nDefault: ' + '%(default)s') + parser.add_argument('--dem-approximate', action='store_true', default=False, diff --git a/scripts/odm_dem.py b/scripts/odm_dem.py index 8bbf05f0..bbe78b77 100644 --- a/scripts/odm_dem.py +++ b/scripts/odm_dem.py @@ -130,8 +130,8 @@ class ODMDEMCell(ecto.Cell): if run_classification: system.run('l2d_classify {0} --decimation {1} ' - '{2} {3}'.format( - l2d_params, args.dem_decimation, + '{2} --initialDistance {3} {4}'.format( + l2d_params, args.dem_decimation, args.dem_initial_distance, approximate, tree.odm_georeferencing), env_paths) else: log.ODM_INFO("Will skip classification, only DSM is needed") From 7e9a9ac2b44c8780f1145b182183bb1b49455d8e Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 6 Jul 2017 20:38:21 -0400 Subject: [PATCH 25/35] Parameter order fix --- scripts/odm_dem.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/odm_dem.py b/scripts/odm_dem.py index bbe78b77..57d8870f 100644 --- a/scripts/odm_dem.py +++ b/scripts/odm_dem.py @@ -131,8 +131,8 @@ class ODMDEMCell(ecto.Cell): if run_classification: system.run('l2d_classify {0} --decimation {1} ' '{2} --initialDistance {3} {4}'.format( - l2d_params, args.dem_decimation, args.dem_initial_distance, - approximate, tree.odm_georeferencing), env_paths) + l2d_params, args.dem_decimation, approximate, + args.dem_initial_distance, tree.odm_georeferencing), env_paths) else: log.ODM_INFO("Will skip classification, only DSM is needed") copyfile(tree.odm_georeferencing_model_las, os.path.join(odm_dem_root, 'bounds-0_l2d_s{slope}c{cellsize}.las'.format(**kwargs))) From f808f6aa950747bcee4e2a328a2c22eb44dcf745 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 7 Jul 2017 08:35:12 -0400 Subject: [PATCH 26/35] Changed dem-initial-distance default value --- opendm/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opendm/config.py b/opendm/config.py index 8a48968b..e8ef1f71 100644 --- a/opendm/config.py +++ b/opendm/config.py @@ -389,9 +389,9 @@ def config(): parser.add_argument('--dem-initial-distance', metavar='', type=float, - default=0.5, + default=0.15, help='Used to classify ground vs non-ground points. Set this value to account for Z noise in meters. ' - 'If you have an uncertainty of around 50 cm, set this value large enough to not exclude these points. ' + 'If you have an uncertainty of around 15 cm, set this value large enough to not exclude these points. ' 'Too small of a value will exclude valid ground points, while too large of a value will misclassify non-ground points for ground ones. ' '\nDefault: ' '%(default)s') From 23a0993649bfcc77ade398641c5489b419191a4d Mon Sep 17 00:00:00 2001 From: Dakota Benjamin Date: Tue, 11 Jul 2017 15:03:05 -0400 Subject: [PATCH 27/35] Update settings.yaml --- settings.yaml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/settings.yaml b/settings.yaml index 7eac080f..2be76e97 100644 --- a/settings.yaml +++ b/settings.yaml @@ -43,10 +43,15 @@ project_path: '' # Example: '/home/user/ODMProjects #texturing_tone_mapping: 'none' #gcp: !!null # YAML tag for None #use_exif: False # Set to True if you have a GCP file (it auto-detects) and want to use EXIF -#dem: False -#dem_sample_radius: 1.0 -#dem_resolution: 2 -#dem_radius: 0.5 +#dtm: False # Use this tag to build a DTM (Digital Terrain Model +#dsm: False # Use this tag to build a DSM (Digital Surface Model +#dem-gapfill-steps: 4 +#dem-resolution: 0.1 +#dem-maxangle:20 +#dem-maxsd: 2.5 +#dem-approximate: False +#dem-decimation: 1 +#dem-terrain-type: ComplexForest #orthophoto_resolution: 20.0 # Pixels/meter #orthophoto_target_srs: !!null # Currently does nothing #orthophoto_no_tiled: False From 3ee576f5df6a43467e0ba8e83520c067d4540bb0 Mon Sep 17 00:00:00 2001 From: Dakota Benjamin Date: Tue, 11 Jul 2017 15:03:40 -0400 Subject: [PATCH 28/35] Add new dem settings --- docker.settings.yaml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docker.settings.yaml b/docker.settings.yaml index 8109ea52..a1860236 100644 --- a/docker.settings.yaml +++ b/docker.settings.yaml @@ -42,10 +42,15 @@ project_path: '/' #DO NOT CHANGE THIS OR DOCKER WILL NOT WORK. It should be '/' #texturing_keep_unseen_faces: False #texturing_tone_mapping: 'none' #gcp: !!null # YAML tag for None -#dem: False -#dem_sample_radius: 1.0 -#dem_resolution: 2 -#dem_radius: 0.5 +#dtm: False # Use this tag to build a DTM (Digital Terrain Model +#dsm: False # Use this tag to build a DSM (Digital Surface Model +#dem-gapfill-steps: 4 +#dem-resolution: 0.1 +#dem-maxangle:20 +#dem-maxsd: 2.5 +#dem-approximate: False +#dem-decimation: 1 +#dem-terrain-type: ComplexForest #use_exif: False # Set to True if you have a GCP file (it auto-detects) and want to use EXIF #orthophoto_resolution: 20.0 # Pixels/meter #orthophoto_target_srs: !!null # Currently does nothing From aed4cfc3478ebca3c45690c37a4bc1bd54ef8558 Mon Sep 17 00:00:00 2001 From: Dakota Benjamin Date: Tue, 18 Jul 2017 11:19:07 -0400 Subject: [PATCH 29/35] Make Docker more prominent for installation --- README.md | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f1f2786d..9f24ef76 100644 --- a/README.md +++ b/README.md @@ -31,17 +31,30 @@ OpenDroneMap can run natively on Ubuntu 14.04 or later, see [Build and Run Using Current version: 0.3.1 (this software is in beta) +### Docker +The easiest way to run ODM is through Docker. If you don't have it installed, +see the [Docker Ubuntu installation tutorial](https://docs.docker.com/engine/installation/linux/ubuntulinux/) and follow the +instructions through "Create a Docker group". The Docker image workflow +has equivalent procedures for Mac OS X and Windows found at [docs.docker.com](docs.docker.com). Then run the following command which will build a pre-built image and run on images found in `$(pwd)/images` (you can change this if you need to, see the [wiki](https://github.com/OpenDroneMap/OpenDroneMap/wiki/Docker) for more detailed instructions. +``` +docker run -it --rm -v $(pwd)/images:/code/images -v $(pwd)/odm_orthophoto:/code/odm_orthophoto -v $(pwd)/odm_texturing:/code/odm_texturing opendronemap/opendronemap +``` + +### Native Install + 1. Extract and enter the OpenDroneMap directory 2. Run `bash configure.sh install` 4. Edit the `settings.yaml` file in your favorite text editor. Set the `project-path` value to an empty directory (you will place sub-directories containing individual projects inside). You can add many options to this file, [see here](https://github.com/OpenDroneMap/OpenDroneMap/wiki/Run-Time-Parameters) 3. Download a sample dataset from [here](https://github.com/OpenDroneMap/odm_data_aukerman/archive/master.zip) (about 550MB) and extract it as a subdirectory in your project directory. 4. Run `./run.sh odm_data_aukerman` 5. Enter dataset directory to view results: - - orthophoto: odm_orthophoto/odm_orthophoto.tif - - textured mesh model: odm_texturing/odm_textured_model_geo.obj - - point cloud (georeferenced): odm_georeferencing/odm_georeferenced_model.ply + - orthophoto: odm_orthophoto/odm_orthophoto.tif + - textured mesh model: odm_texturing/odm_textured_model_geo.obj + - point cloud (georeferenced): odm_georeferencing/odm_georeferenced_model.ply -See [here](https://github.com/OpenDroneMap/OpenDroneMap/blob/3964f21377e27c261c305b30537f699853ac2004/README.md#installation) for more detailed installation instructions. +See below for more detailed installation instructions. + +## Diving Deeper ### Installation From e413b1fd7b3c5b06313e7075154216ab0441dcb0 Mon Sep 17 00:00:00 2001 From: Dakota Benjamin Date: Tue, 18 Jul 2017 11:36:52 -0400 Subject: [PATCH 30/35] Remove awkward wording and rearrange --- README.md | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 9f24ef76..a3182d01 100644 --- a/README.md +++ b/README.md @@ -20,27 +20,23 @@ In a word, OpenDroneMap is a toolchain for processing raw civilian UAS imagery t Open Drone Map now includes state-of-the-art 3D reconstruction work by Michael Waechter, Nils Moehrle, and Michael Goesele. See their publication at http://www.gcc.tu-darmstadt.de/media/gcc/papers/Waechter-2014-LTB.pdf. - ## QUICKSTART -OpenDroneMap can run natively on Ubuntu 14.04 or later, see [Build and Run Using Docker](#build-and-run-using-docker) for running on Windows / MacOS. A [Vagrant VM](https://github.com/OpenDroneMap/odm_vagrant) is also available. +### Docker (All platforms) -*Support for Ubuntu 12.04 is currently BROKEN with the addition of OpenSfM and Ceres-Solver. It is likely to remain broken unless a champion is found to fix it.* - -**[Download the latest release here](https://github.com/OpenDroneMap/OpenDroneMap/releases)** - -Current version: 0.3.1 (this software is in beta) - -### Docker The easiest way to run ODM is through Docker. If you don't have it installed, see the [Docker Ubuntu installation tutorial](https://docs.docker.com/engine/installation/linux/ubuntulinux/) and follow the instructions through "Create a Docker group". The Docker image workflow has equivalent procedures for Mac OS X and Windows found at [docs.docker.com](docs.docker.com). Then run the following command which will build a pre-built image and run on images found in `$(pwd)/images` (you can change this if you need to, see the [wiki](https://github.com/OpenDroneMap/OpenDroneMap/wiki/Docker) for more detailed instructions. + ``` docker run -it --rm -v $(pwd)/images:/code/images -v $(pwd)/odm_orthophoto:/code/odm_orthophoto -v $(pwd)/odm_texturing:/code/odm_texturing opendronemap/opendronemap ``` -### Native Install +### Native Install (Ubuntu 14.04 or later) + +**[Download the latest release here](https://github.com/OpenDroneMap/OpenDroneMap/releases)** +Current version: 0.3.1 (this software is in beta) 1. Extract and enter the OpenDroneMap directory 2. Run `bash configure.sh install` From 5d998a3ac65f81a537df8a0e05d58b887d8dc6b6 Mon Sep 17 00:00:00 2001 From: Dakota Benjamin Date: Tue, 18 Jul 2017 14:54:35 -0400 Subject: [PATCH 31/35] Add check for bad index name --- contrib/visveg/vegind.py | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/visveg/vegind.py b/contrib/visveg/vegind.py index 9abb58dc..e5966fb3 100644 --- a/contrib/visveg/vegind.py +++ b/contrib/visveg/vegind.py @@ -17,6 +17,7 @@ try: typ = sys.argv[2] (fileRoot, fileExt) = os.path.splitext(file) outFileName = fileRoot + "_" + typ + fileExt + isinstance(typ, ['vari', 'tgi', 'ngrdi']) except (TypeError, IndexError, NameError): print bcolors.FAIL + 'Arguments messed up. Check arguments order and index name' + bcolors.ENDC print 'Usage: ./vegind.py orto index' From 37e49b132cc005fcdcfce6baba873becce8dfbf9 Mon Sep 17 00:00:00 2001 From: Tomasz Nycz Date: Wed, 26 Jul 2017 15:21:05 +0200 Subject: [PATCH 32/35] Repair bad index checking isinstance(object, classinfo) checks only if object has correct python type, not value. Let's try this with more traditional method ;) --- contrib/visveg/vegind.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/visveg/vegind.py b/contrib/visveg/vegind.py index e5966fb3..62554657 100644 --- a/contrib/visveg/vegind.py +++ b/contrib/visveg/vegind.py @@ -17,7 +17,8 @@ try: typ = sys.argv[2] (fileRoot, fileExt) = os.path.splitext(file) outFileName = fileRoot + "_" + typ + fileExt - isinstance(typ, ['vari', 'tgi', 'ngrdi']) + if typ not in ['vari', 'tgi', 'ngrdi']: + raise IndexError except (TypeError, IndexError, NameError): print bcolors.FAIL + 'Arguments messed up. Check arguments order and index name' + bcolors.ENDC print 'Usage: ./vegind.py orto index' From 899e0986af838e81b892e9e2ed8f30a22d498154 Mon Sep 17 00:00:00 2001 From: Dakota Benjamin Date: Mon, 31 Jul 2017 11:29:25 -0400 Subject: [PATCH 33/35] Create CNAME --- CNAME | 1 + 1 file changed, 1 insertion(+) create mode 100644 CNAME diff --git a/CNAME b/CNAME new file mode 100644 index 00000000..ed563e1e --- /dev/null +++ b/CNAME @@ -0,0 +1 @@ +opendronemap.org \ No newline at end of file From a56635d3b5ca3add2db7e6cd9d5e309ca2bfb46c Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 31 Jul 2017 14:03:01 -0400 Subject: [PATCH 34/35] Removed unused matcher_threshold and matcher_ratio --- opendm/config.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/opendm/config.py b/opendm/config.py index e8ef1f71..5bc17002 100644 --- a/opendm/config.py +++ b/opendm/config.py @@ -115,21 +115,6 @@ def config(): 'More features leads to better results but slower ' 'execution. Default: %(default)s')) - parser.add_argument('--matcher-threshold', - metavar='', - default=2.0, - type=float, - help=('Ignore matched keypoints if the two images share ' - 'less than percent of keypoints. Default:' - ' %(default)s')) - - parser.add_argument('--matcher-ratio', - metavar='', - default=0.6, - type=float, - help=('Ratio of the distance to the next best matched ' - 'keypoint. Default: %(default)s')) - parser.add_argument('--matcher-neighbors', type=int, metavar='', From b3903a848cd68c0fc2bc106491ae7913e4cc3128 Mon Sep 17 00:00:00 2001 From: Dakota Benjamin Date: Thu, 3 Aug 2017 13:49:02 -0400 Subject: [PATCH 35/35] Add email --- code_of_conduct.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code_of_conduct.md b/code_of_conduct.md index d05f4bc0..9dd416e9 100644 --- a/code_of_conduct.md +++ b/code_of_conduct.md @@ -55,7 +55,7 @@ further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at [INSERT EMAIL ADDRESS]. All +reported by contacting the project team at `svm at clevelandmetroparks dot com`. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident.