diff --git a/.gitmodules b/.gitmodules index b01146d3..e69de29b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "src/bundler"] - path = src/bundler - url = https://github.com/chris-cooper/bundler_sfm diff --git a/CNAME b/CNAME new file mode 100644 index 00000000..ed563e1e --- /dev/null +++ b/CNAME @@ -0,0 +1 @@ +opendronemap.org \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 73278c3d..850e53ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,12 +14,12 @@ RUN apt-get install --no-install-recommends -y git cmake python-pip build-essent libgtk2.0-dev libavcodec-dev libavformat-dev libswscale-dev python-dev python-numpy libtbb2 libtbb-dev libjpeg-dev libpng-dev libtiff-dev libjasper-dev libflann-dev \ libproj-dev libxext-dev liblapack-dev libeigen3-dev libvtk5-dev python-networkx libgoogle-glog-dev libsuitesparse-dev libboost-filesystem-dev libboost-iostreams-dev \ libboost-regex-dev libboost-python-dev libboost-date-time-dev libboost-thread-dev python-pyproj python-empy python-nose python-pyside python-pyexiv2 python-scipy \ -jhead liblas-bin python-matplotlib libatlas-base-dev libgmp-dev libmpfr-dev +jhead liblas-bin python-matplotlib libatlas-base-dev libgmp-dev libmpfr-dev swig2.0 python-wheel libboost-log-dev libjsoncpp-dev RUN apt-get remove libdc1394-22-dev RUN pip install --upgrade pip RUN pip install setuptools -RUN pip install -U PyYAML exifread gpxpy xmltodict catkin-pkg appsettings +RUN pip install -U PyYAML exifread gpxpy xmltodict catkin-pkg appsettings https://github.com/OpenDroneMap/gippy/archive/v0.3.9.tar.gz ENV PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python2.7/dist-packages" ENV PYTHONPATH="$PYTHONPATH:/code/SuperBuild/src/opensfm" @@ -44,6 +44,7 @@ COPY /SuperBuild/CMakeLists.txt /code/SuperBuild/CMakeLists.txt COPY docker.settings.yaml /code/settings.yaml COPY VERSION /code/VERSION + #Compile code in SuperBuild and root directories RUN cd SuperBuild && mkdir build && cd build && cmake .. && make -j$(nproc) && cd ../.. && mkdir build && cd build && cmake .. && make -j$(nproc) diff --git a/README.md b/README.md index f1f2786d..a3182d01 100644 --- a/README.md +++ b/README.md @@ -20,15 +20,22 @@ In a word, OpenDroneMap is a toolchain for processing raw civilian UAS imagery t Open Drone Map now includes state-of-the-art 3D reconstruction work by Michael Waechter, Nils Moehrle, and Michael Goesele. See their publication at http://www.gcc.tu-darmstadt.de/media/gcc/papers/Waechter-2014-LTB.pdf. - ## QUICKSTART -OpenDroneMap can run natively on Ubuntu 14.04 or later, see [Build and Run Using Docker](#build-and-run-using-docker) for running on Windows / MacOS. A [Vagrant VM](https://github.com/OpenDroneMap/odm_vagrant) is also available. +### Docker (All platforms) -*Support for Ubuntu 12.04 is currently BROKEN with the addition of OpenSfM and Ceres-Solver. It is likely to remain broken unless a champion is found to fix it.* +The easiest way to run ODM is through Docker. If you don't have it installed, +see the [Docker Ubuntu installation tutorial](https://docs.docker.com/engine/installation/linux/ubuntulinux/) and follow the +instructions through "Create a Docker group". The Docker image workflow +has equivalent procedures for Mac OS X and Windows found at [docs.docker.com](docs.docker.com). Then run the following command which will build a pre-built image and run on images found in `$(pwd)/images` (you can change this if you need to, see the [wiki](https://github.com/OpenDroneMap/OpenDroneMap/wiki/Docker) for more detailed instructions. -**[Download the latest release here](https://github.com/OpenDroneMap/OpenDroneMap/releases)** +``` +docker run -it --rm -v $(pwd)/images:/code/images -v $(pwd)/odm_orthophoto:/code/odm_orthophoto -v $(pwd)/odm_texturing:/code/odm_texturing opendronemap/opendronemap +``` +### Native Install (Ubuntu 14.04 or later) + +**[Download the latest release here](https://github.com/OpenDroneMap/OpenDroneMap/releases)** Current version: 0.3.1 (this software is in beta) 1. Extract and enter the OpenDroneMap directory @@ -37,11 +44,13 @@ Current version: 0.3.1 (this software is in beta) 3. Download a sample dataset from [here](https://github.com/OpenDroneMap/odm_data_aukerman/archive/master.zip) (about 550MB) and extract it as a subdirectory in your project directory. 4. Run `./run.sh odm_data_aukerman` 5. Enter dataset directory to view results: - - orthophoto: odm_orthophoto/odm_orthophoto.tif - - textured mesh model: odm_texturing/odm_textured_model_geo.obj - - point cloud (georeferenced): odm_georeferencing/odm_georeferenced_model.ply + - orthophoto: odm_orthophoto/odm_orthophoto.tif + - textured mesh model: odm_texturing/odm_textured_model_geo.obj + - point cloud (georeferenced): odm_georeferencing/odm_georeferenced_model.ply -See [here](https://github.com/OpenDroneMap/OpenDroneMap/blob/3964f21377e27c261c305b30537f699853ac2004/README.md#installation) for more detailed installation instructions. +See below for more detailed installation instructions. + +## Diving Deeper ### Installation diff --git a/SuperBuild/CMakeLists.txt b/SuperBuild/CMakeLists.txt index f6b61d75..453c3fb4 100644 --- a/SuperBuild/CMakeLists.txt +++ b/SuperBuild/CMakeLists.txt @@ -98,6 +98,10 @@ option(ODM_BUILD_CGAL "Force to build CGAL library" OFF) SETUP_EXTERNAL_PROJECT(CGAL ${ODM_CGAL_Version} ${ODM_BUILD_CGAL}) +# --------------------------------------------------------------------------------------------- +# Hexer +# +SETUP_EXTERNAL_PROJECT(Hexer 1.4 ON) # --------------------------------------------------------------------------------------------- # Open Geometric Vision (OpenGV) @@ -114,6 +118,7 @@ set(custom_libs OpenGV Ecto PDAL MvsTexturing + Lidar2dems ) # Dependencies of the SLAM module diff --git a/SuperBuild/cmake/External-Hexer.cmake b/SuperBuild/cmake/External-Hexer.cmake new file mode 100644 index 00000000..64de1ae2 --- /dev/null +++ b/SuperBuild/cmake/External-Hexer.cmake @@ -0,0 +1,27 @@ +set(_proj_name hexer) +set(_SB_BINARY_DIR "${SB_BINARY_DIR}/${_proj_name}") + +ExternalProject_Add(${_proj_name} + DEPENDS + PREFIX ${_SB_BINARY_DIR} + TMP_DIR ${_SB_BINARY_DIR}/tmp + STAMP_DIR ${_SB_BINARY_DIR}/stamp + #--Download step-------------- + DOWNLOAD_DIR ${SB_DOWNLOAD_DIR} + URL https://github.com/hobu/hexer/archive/2898b96b1105991e151696391b9111610276258f.tar.gz + URL_MD5 e8f2788332ad212cf78efa81a82e95dd + #--Update/Patch step---------- + UPDATE_COMMAND "" + #--Configure step------------- + SOURCE_DIR ${SB_SOURCE_DIR}/${_proj_name} + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX:PATH=${SB_INSTALL_DIR} + #--Build step----------------- + BINARY_DIR ${_SB_BINARY_DIR} + #--Install step--------------- + INSTALL_DIR ${SB_INSTALL_DIR} + #--Output logging------------- + LOG_DOWNLOAD OFF + LOG_CONFIGURE OFF + LOG_BUILD OFF +) diff --git a/SuperBuild/cmake/External-Lidar2dems.cmake b/SuperBuild/cmake/External-Lidar2dems.cmake new file mode 100644 index 00000000..4772a246 --- /dev/null +++ b/SuperBuild/cmake/External-Lidar2dems.cmake @@ -0,0 +1,24 @@ +set(_proj_name lidar2dems) +set(_SB_BINARY_DIR "${SB_BINARY_DIR}/${_proj_name}") + +ExternalProject_Add(${_proj_name} + PREFIX ${_SB_BINARY_DIR} + TMP_DIR ${_SB_BINARY_DIR}/tmp + STAMP_DIR ${_SB_BINARY_DIR}/stamp + #--Download step-------------- + DOWNLOAD_DIR ${SB_DOWNLOAD_DIR}/${_proj_name} + URL https://github.com/OpenDroneMap/lidar2dems/archive/master.zip + #--Update/Patch step---------- + UPDATE_COMMAND "" + #--Configure step------------- + SOURCE_DIR ${SB_SOURCE_DIR}/${_proj_name} + CONFIGURE_COMMAND "" + #--Build step----------------- + BUILD_COMMAND "" + #--Install step--------------- + INSTALL_COMMAND "${SB_SOURCE_DIR}/${_proj_name}/install.sh" "${SB_INSTALL_DIR}" + #--Output logging------------- + LOG_DOWNLOAD OFF + LOG_CONFIGURE OFF + LOG_BUILD OFF +) diff --git a/SuperBuild/cmake/External-PDAL.cmake b/SuperBuild/cmake/External-PDAL.cmake index 7faae6e2..7ff10867 100644 --- a/SuperBuild/cmake/External-PDAL.cmake +++ b/SuperBuild/cmake/External-PDAL.cmake @@ -2,13 +2,14 @@ set(_proj_name pdal) set(_SB_BINARY_DIR "${SB_BINARY_DIR}/${_proj_name}") ExternalProject_Add(${_proj_name} + DEPENDS hexer PREFIX ${_SB_BINARY_DIR} TMP_DIR ${_SB_BINARY_DIR}/tmp STAMP_DIR ${_SB_BINARY_DIR}/stamp #--Download step-------------- DOWNLOAD_DIR ${SB_DOWNLOAD_DIR} - URL https://github.com/PDAL/PDAL/archive/aea5bb0cacc64b91d626eff491fbdbb5668c06d7.tar.gz - URL_MD5 726933f63f661e11e13775d6ce4e5ed0 + URL https://github.com/PDAL/PDAL/archive/e881b581e3b91a928105d67db44c567f3b6d1afe.tar.gz + URL_MD5 438acbb736ba01fbe8f9ca7cdbf113bf #--Update/Patch step---------- UPDATE_COMMAND "" #--Configure step------------- @@ -19,7 +20,7 @@ ExternalProject_Add(${_proj_name} -BUILD_PLUGIN_PGPOINTCLOUD=ON -DBUILD_PLUGIN_CPD=OFF -DBUILD_PLUGIN_GREYHOUND=OFF - -DBUILD_PLUGIN_HEXBIN=OFF + -DBUILD_PLUGIN_HEXBIN=ON -DBUILD_PLUGIN_ICEBRIDGE=OFF -DBUILD_PLUGIN_MRSID=OFF -DBUILD_PLUGIN_NITF=OFF diff --git a/code_of_conduct.md b/code_of_conduct.md index d05f4bc0..9dd416e9 100644 --- a/code_of_conduct.md +++ b/code_of_conduct.md @@ -55,7 +55,7 @@ further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at [INSERT EMAIL ADDRESS]. All +reported by contacting the project team at `svm at clevelandmetroparks dot com`. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. diff --git a/configure.sh b/configure.sh index 066fa567..11b833fd 100755 --- a/configure.sh +++ b/configure.sh @@ -21,7 +21,8 @@ install() { libgdal-dev \ gdal-bin \ libgeotiff-dev \ - pkg-config + pkg-config \ + libjsoncpp-dev echo "Getting CMake 3.1 for MVS-Texturing" sudo apt-get install -y software-properties-common python-software-properties @@ -72,7 +73,7 @@ install() { appsettings echo "Installing CGAL dependencies" - sudo apt-get install libgmp-dev libmpfr-dev + sudo apt-get install -y -qq libgmp-dev libmpfr-dev echo "Installing Ecto Dependencies" sudo pip install -U catkin-pkg @@ -86,6 +87,13 @@ install() { jhead \ liblas-bin + echo "Installing lidar2dems Dependencies" + sudo apt-get install -y -qq swig2.0 \ + python-wheel \ + libboost-log-dev + + sudo pip install -U https://github.com/OpenDroneMap/gippy/archive/v0.3.9.tar.gz + echo "Compiling SuperBuild" cd ${RUNPATH}/SuperBuild mkdir -p build && cd build @@ -134,4 +142,4 @@ else echo "Invalid instructions." >&2 usage exit 1 -fi \ No newline at end of file +fi diff --git a/contrib/visveg/readme.md b/contrib/visveg/readme.md new file mode 100644 index 00000000..1d883be4 --- /dev/null +++ b/contrib/visveg/readme.md @@ -0,0 +1,31 @@ +# Visible Vegetation Indexes + +This script produces a Vegetation Index raster from a RGB orthophoto (odm_orthophoto.tif in your project) + +## Requirements +* rasterio (pip install rasterio) +* numpy python package (included in ODM build) + +## Usage +``` +vegind.py index + +positional arguments: + The RGB orthophoto. Must be a GeoTiff. + index Index identifier. Allowed values: ngrdi, tgi, vari +``` +Output will be generated with index suffix in the same directory as input. + +## Examples + +`python vegind.py /path/to/odm_orthophoto.tif tgi` + +Orthophoto photo of Koniaków grass field and forest in QGIS: ![](http://imgur.com/K6x3nB2.jpg) +The Triangular Greenness Index output in QGIS (with a spectral pseudocolor): ![](http://i.imgur.com/f9TzISU.jpg) +Visible Atmospheric Resistant Index: ![](http://imgur.com/Y7BHzLs.jpg) +Normalized green-red difference index: ![](http://imgur.com/v8cmaPS.jpg) + +## Bibliography + +1. Hunt, E. Raymond, et al. "A Visible Band Index for Remote Sensing Leaf Chlorophyll Content At the Canopy Scale." ITC journal 21(2013): 103-112. doi: 10.1016/j.jag.2012.07.020 +(https://doi.org/10.1016/j.jag.2012.07.020) diff --git a/contrib/visveg/vegind.py b/contrib/visveg/vegind.py new file mode 100644 index 00000000..62554657 --- /dev/null +++ b/contrib/visveg/vegind.py @@ -0,0 +1,95 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +import rasterio, os, sys +import numpy as np + +class bcolors: + OKBLUE = '\033[94m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + +try: + file = sys.argv[1] + typ = sys.argv[2] + (fileRoot, fileExt) = os.path.splitext(file) + outFileName = fileRoot + "_" + typ + fileExt + if typ not in ['vari', 'tgi', 'ngrdi']: + raise IndexError +except (TypeError, IndexError, NameError): + print bcolors.FAIL + 'Arguments messed up. Check arguments order and index name' + bcolors.ENDC + print 'Usage: ./vegind.py orto index' + print ' orto - filepath to RGB orthophoto' + print ' index - Vegetation Index' + print bcolors.OKGREEN + 'Available indexes: vari, ngrdi, tgi' + bcolors.ENDC + sys.exit() + + +def calcNgrdi(red, green): + """ + Normalized green red difference index + Tucker,C.J.,1979. + Red and photographic infrared linear combinations for monitoring vegetation. + Remote Sensing of Environment 8, 127–150 + :param red: red visible channel + :param green: green visible channel + :return: ngrdi index array + """ + mask = np.not_equal(np.add(red,green), 0.0) + return np.choose(mask, (-9999.0, np.true_divide( + np.subtract(green,red), + np.add(red,green)))) + +def calcVari(red,green,blue): + """ + Calculates Visible Atmospheric Resistant Index + Gitelson, A.A., Kaufman, Y.J., Stark, R., Rundquist, D., 2002. + Novel algorithms for remote estimation of vegetation fraction. + Remote Sensing of Environment 80, 76–87. + :param red: red visible channel + :param green: green visible channel + :param blue: blue visible channel + :return: vari index array, that will be saved to tiff + """ + mask = np.not_equal(np.subtract(np.add(green,red),blue), 0.0) + return np.choose(mask, (-9999.0, np.true_divide(np.subtract(green,red),np.subtract(np.add(green,red),blue)))) + +def calcTgi(red,green,blue): + """ + Calculates Triangular Greenness Index + Hunt, E. Raymond Jr.; Doraiswamy, Paul C.; McMurtrey, James E.; Daughtry, Craig S.T.; Perry, Eileen M.; and Akhmedov, Bakhyt, + A visible band index for remote sensing leaf chlorophyll content at the canopy scale (2013). + Publications from USDA-ARS / UNL Faculty. Paper 1156. + http://digitalcommons.unl.edu/usdaarsfacpub/1156 + :param red: red channel + :param green: green channel + :param blue: blue channel + :return: tgi index array, that will be saved to tiff + """ + mask = np.not_equal(green-red+blue-255.0, 0.0) + return np.choose(mask, (-9999.0, np.subtract(green, np.multiply(0.39,red), np.multiply(0.61, blue)))) + +try: + with rasterio.Env(): + ds = rasterio.open(file) + profile = ds.profile + profile.update(dtype=rasterio.float32, count=1, nodata=-9999) + red = np.float32(ds.read(1)) + green = np.float32(ds.read(2)) + blue = np.float32(ds.read(3)) + np.seterr(divide='ignore', invalid='ignore') + if typ == 'ngrdi': + indeks = calcNgrdi(red,green) + elif typ == 'vari': + indeks = calcVari(red, green, blue) + elif typ == 'tgi': + indeks = calcTgi(red, green, blue) + + with rasterio.open(outFileName, 'w', **profile) as dst: + dst.write(indeks.astype(rasterio.float32), 1) +except rasterio.errors.RasterioIOError: + print bcolors.FAIL + 'Orthophoto file not found or access denied' + bcolors.ENDC + sys.exit() diff --git a/core2.Dockerfile b/core2.Dockerfile index 52a06c07..f8294c65 100644 --- a/core2.Dockerfile +++ b/core2.Dockerfile @@ -14,12 +14,12 @@ RUN apt-get install --no-install-recommends -y git cmake python-pip build-essent libgtk2.0-dev libavcodec-dev libavformat-dev libswscale-dev python-dev python-numpy libtbb2 libtbb-dev libjpeg-dev libpng-dev libtiff-dev libjasper-dev libflann-dev \ libproj-dev libxext-dev liblapack-dev libeigen3-dev libvtk5-dev python-networkx libgoogle-glog-dev libsuitesparse-dev libboost-filesystem-dev libboost-iostreams-dev \ libboost-regex-dev libboost-python-dev libboost-date-time-dev libboost-thread-dev python-pyproj python-empy python-nose python-pyside python-pyexiv2 python-scipy \ -jhead liblas-bin python-matplotlib libatlas-base-dev libgmp-dev libmpfr-dev +jhead liblas-bin python-matplotlib libatlas-base-dev libgmp-dev libmpfr-dev swig2.0 python-wheel libboost-log-dev libjsoncpp-dev RUN apt-get remove libdc1394-22-dev RUN pip install --upgrade pip RUN pip install setuptools -RUN pip install -U PyYAML exifread gpxpy xmltodict catkin-pkg appsettings +RUN pip install -U PyYAML exifread gpxpy xmltodict catkin-pkg appsettings https://github.com/OpenDroneMap/gippy/archive/v0.3.9.tar.gz ENV PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python2.7/dist-packages" ENV PYTHONPATH="$PYTHONPATH:/code/SuperBuild/src/opensfm" diff --git a/docker.settings.yaml b/docker.settings.yaml index 2a252984..ce55db73 100644 --- a/docker.settings.yaml +++ b/docker.settings.yaml @@ -43,10 +43,15 @@ project_path: '/' #DO NOT CHANGE THIS OR DOCKER WILL NOT WORK. It should be '/' #texturing_keep_unseen_faces: False #texturing_tone_mapping: 'none' #gcp: !!null # YAML tag for None -#dem: False -#dem_sample_radius: 1.0 -#dem_resolution: 2 -#dem_radius: 0.5 +#dtm: False # Use this tag to build a DTM (Digital Terrain Model +#dsm: False # Use this tag to build a DSM (Digital Surface Model +#dem-gapfill-steps: 4 +#dem-resolution: 0.1 +#dem-maxangle:20 +#dem-maxsd: 2.5 +#dem-approximate: False +#dem-decimation: 1 +#dem-terrain-type: ComplexForest #use_exif: False # Set to True if you have a GCP file (it auto-detects) and want to use EXIF #orthophoto_resolution: 20.0 # Pixels/meter #orthophoto_target_srs: !!null # Currently does nothing diff --git a/opendm/config.py b/opendm/config.py index 5bbaf802..b4da1205 100644 --- a/opendm/config.py +++ b/opendm/config.py @@ -10,7 +10,7 @@ import sys # parse arguments processopts = ['resize', 'opensfm', 'slam', 'cmvs', 'pmvs', 'odm_meshing', 'odm_25dmeshing', 'mvs_texturing', 'odm_georeferencing', - 'odm_orthophoto'] + 'odm_dem', 'odm_orthophoto'] with open(io.join_paths(context.root_path, 'VERSION')) as version_file: __version__ = version_file.read().strip() @@ -111,21 +111,6 @@ def config(): 'More features leads to better results but slower ' 'execution. Default: %(default)s')) - parser.add_argument('--matcher-threshold', - metavar='', - default=2.0, - type=float, - help=('Ignore matched keypoints if the two images share ' - 'less than percent of keypoints. Default:' - ' %(default)s')) - - parser.add_argument('--matcher-ratio', - metavar='', - default=0.6, - type=float, - help=('Ratio of the distance to the next best matched ' - 'keypoint. Default: %(default)s')) - parser.add_argument('--matcher-neighbors', type=int, metavar='', @@ -342,34 +327,87 @@ def config(): help=('Use this tag if you have a gcp_list.txt but ' 'want to use the exif geotags instead')) - parser.add_argument('--dem', + parser.add_argument('--dtm', action='store_true', default=False, - help='Use this tag to build a DEM using a progressive ' - 'morphological filter in PDAL.') + help='Use this tag to build a DTM (Digital Terrain Model, ground only) using a progressive ' + 'morphological filter. Check the --dem* parameters for fine tuning.') + + parser.add_argument('--dsm', + action='store_true', + default=False, + help='Use this tag to build a DSM (Digital Surface Model, ground + objects) using a progressive ' + 'morphological filter. Check the --dem* parameters for fine tuning.') - parser.add_argument('--dem-sample-radius', - metavar='', - default=1.0, - type=float, - help='Minimum distance between samples for DEM ' - 'generation.\nDefault=%(default)s') + parser.add_argument('--dem-gapfill-steps', + metavar='', + default=4, + type=int, + help='Number of steps used to fill areas with gaps. Set to 0 to disable gap filling. ' + 'Starting with a radius equal to the output resolution, N different DEMs are generated with ' + 'progressively bigger radius using the inverse distance weighted (IDW) algorithm ' + 'and merged together. Remaining gaps are then merged using nearest neighbor interpolation. ' + '\nDefault=%(default)s') parser.add_argument('--dem-resolution', metavar='', type=float, - default=2, - help='Length of raster cell edges in X/Y units.' + default=0.1, + help='Length of raster cell edges in meters.' '\nDefault: %(default)s') - parser.add_argument('--dem-radius', - metavar='', + parser.add_argument('--dem-maxangle', + metavar='', type=float, - default=0.5, - help='Radius about cell center bounding points to ' - 'use to calculate a cell value.\nDefault: ' + default=20, + help='Points that are more than maxangle degrees off-nadir are discarded. ' + '\nDefault: ' '%(default)s') + parser.add_argument('--dem-maxsd', + metavar='', + type=float, + default=2.5, + help='Points that deviate more than maxsd standard deviations from the local mean ' + 'are discarded. \nDefault: ' + '%(default)s') + + parser.add_argument('--dem-initial-distance', + metavar='', + type=float, + default=0.15, + help='Used to classify ground vs non-ground points. Set this value to account for Z noise in meters. ' + 'If you have an uncertainty of around 15 cm, set this value large enough to not exclude these points. ' + 'Too small of a value will exclude valid ground points, while too large of a value will misclassify non-ground points for ground ones. ' + '\nDefault: ' + '%(default)s') + + parser.add_argument('--dem-approximate', + action='store_true', + default=False, + help='Use this tag use the approximate progressive ' + 'morphological filter, which computes DEMs faster ' + 'but is not as accurate.') + + parser.add_argument('--dem-decimation', + metavar='', + default=1, + type=int, + help='Decimate the points before generating the DEM. 1 is no decimation (full quality). ' + '100 decimates ~99%% of the points. Useful for speeding up ' + 'generation.\nDefault=%(default)s') + + parser.add_argument('--dem-terrain-type', + metavar='', + choices=['FlatNonForest', 'FlatForest', 'ComplexNonForest', 'ComplexForest'], + default='ComplexForest', + help='One of: %(choices)s. Specifies the type of terrain. This mainly helps reduce processing time. ' + '\nFlatNonForest: Relatively flat region with little to no vegetation' + '\nFlatForest: Relatively flat region that is forested' + '\nComplexNonForest: Varied terrain with little to no vegetation' + '\nComplexForest: Varied terrain that is forested' + '\nDefault=%(default)s') + parser.add_argument('--orthophoto-resolution', metavar=' 0.0>', default=20.0, diff --git a/opendm/context.py b/opendm/context.py index 9e1b7e12..198b07a8 100644 --- a/opendm/context.py +++ b/opendm/context.py @@ -8,6 +8,7 @@ scripts_path = os.path.abspath(os.path.dirname(__file__)) root_path, _ = os.path.split(scripts_path) superbuild_path = os.path.join(root_path, 'SuperBuild') +superbuild_bin_path = os.path.join(superbuild_path, 'install', 'bin') tests_path = os.path.join(root_path, 'tests') tests_data_path = os.path.join(root_path, 'tests/test_data') diff --git a/opendm/system.py b/opendm/system.py index d221ce17..740246d0 100644 --- a/opendm/system.py +++ b/opendm/system.py @@ -17,10 +17,16 @@ def get_ccd_widths(): return dict(zip(map(string.lower, sensor_data.keys()), sensor_data.values())) -def run(cmd): +def run(cmd, env_paths=[]): """Run a system command""" log.ODM_DEBUG('running %s' % cmd) - retcode = subprocess.call(cmd, shell=True) + + env = None + if len(env_paths) > 0: + env = os.environ.copy() + env["PATH"] = env["PATH"] + ":" + ":".join(env_paths) + + retcode = subprocess.call(cmd, shell=True, env=env) if retcode < 0: raise Exception("Child was terminated by signal {}".format(-retcode)) diff --git a/opendm/tasks.py b/opendm/tasks.py index 883926da..91135ce7 100644 --- a/opendm/tasks.py +++ b/opendm/tasks.py @@ -13,8 +13,9 @@ tasks_dict = {'1': 'opensfm', '4': 'odm_meshing', '5': 'mvs_texturing', '6': 'odm_georeferencing', - '7': 'odm_orthophoto', - '8': 'zip_results'} + '7': 'odm_dem', + '8': 'odm_orthophoto', + '9': 'zip_results'} class ODMTaskManager(object): diff --git a/opendm/types.py b/opendm/types.py index e1299734..c1cb8958 100644 --- a/opendm/types.py +++ b/opendm/types.py @@ -221,56 +221,6 @@ class ODM_GeoRef(object): system.run('{bin}/pdal pipeline -i {json} --readers.ply.filename={f_in} ' '--writers.las.filename={f_out}'.format(**kwargs)) - def convert_to_dem(self, _file, _file_out, pdalJSON, sample_radius, gdal_res, gdal_radius): - # Check if exists f_in - if not io.file_exists(_file): - log.ODM_ERROR('LAS file does not exist') - return False - - kwargs = { - 'bin': context.pdal_path, - 'f_in': _file, - 'sample_radius': sample_radius, - 'gdal_res': gdal_res, - 'gdal_radius': gdal_radius, - 'f_out': _file_out, - 'json': pdalJSON - } - - pipelineJSON = '{{' \ - ' "pipeline":[' \ - ' "input.las",' \ - ' {{' \ - ' "type":"filters.sample",' \ - ' "radius":"{sample_radius}"' \ - ' }},' \ - ' {{' \ - ' "type":"filters.pmf"' \ - ' }},' \ - ' {{' \ - ' "type":"filters.range",' \ - ' "limits":"Classification[2:2]"' \ - ' }},' \ - ' {{' \ - ' "resolution": {gdal_res},' \ - ' "radius": {gdal_radius},' \ - ' "output_type":"idw",' \ - ' "filename":"outputfile.tif"' \ - ' }}' \ - ' ]' \ - '}}'.format(**kwargs) - - with open(pdalJSON, 'w') as f: - f.write(pipelineJSON) - - system.run('{bin}/pdal pipeline {json} --readers.las.filename={f_in} ' - '--writers.gdal.filename={f_out}'.format(**kwargs)) - - if io.file_exists(kwargs['f_out']): - return True - else: - return False - def utm_to_latlon(self, _file, _photo, idx): gcp = self.gcps[idx] @@ -478,8 +428,6 @@ class ODM_Tree(object): self.odm_georeferencing, 'odm_georeferenced_model.las') self.odm_georeferencing_dem = io.join_paths( self.odm_georeferencing, 'odm_georeferencing_model_dem.tif') - self.odm_georeferencing_dem_json = io.join_paths( - self.odm_georeferencing, 'dem.json') # odm_orthophoto self.odm_orthophoto_file = io.join_paths(self.odm_orthophoto, 'odm_orthophoto.png') @@ -488,3 +436,6 @@ class ODM_Tree(object): self.odm_orthophoto_log = io.join_paths(self.odm_orthophoto, 'odm_orthophoto_log.txt') self.odm_orthophoto_tif_log = io.join_paths(self.odm_orthophoto, 'gdal_translate_log.txt') self.odm_orthophoto_gdaladdo_log = io.join_paths(self.odm_orthophoto, 'gdaladdo_log.txt') + + def path(self, *args): + return io.join_paths(self.root_path, *args) \ No newline at end of file diff --git a/scripts/odm_app.py b/scripts/odm_app.py index 36158282..ba3c24f4 100644 --- a/scripts/odm_app.py +++ b/scripts/odm_app.py @@ -15,6 +15,7 @@ from odm_meshing import ODMeshingCell from mvstex import ODMMvsTexCell from odm_georeferencing import ODMGeoreferencingCell from odm_orthophoto import ODMOrthoPhotoCell +from odm_dem import ODMDEMCell class ODMApp(ecto.BlackBox): @@ -71,11 +72,8 @@ class ODMApp(ecto.BlackBox): 'georeferencing': ODMGeoreferencingCell(img_size=p.args.resize_to, gcp_file=p.args.gcp, use_exif=p.args.use_exif, - dem=p.args.dem, - sample_radius=p.args.dem_sample_radius, - gdal_res=p.args.dem_resolution, - gdal_radius=p.args.dem_radius, verbose=p.args.verbose), + 'dem': ODMDEMCell(verbose=p.args.verbose), 'orthophoto': ODMOrthoPhotoCell(resolution=p.args.orthophoto_resolution, t_srs=p.args.orthophoto_target_srs, no_tiled=p.args.orthophoto_no_tiled, @@ -148,7 +146,12 @@ class ODMApp(ecto.BlackBox): self.args[:] >> self.georeferencing['args'], self.dataset['photos'] >> self.georeferencing['photos'], self.texturing['reconstruction'] >> self.georeferencing['reconstruction']] - + + # create odm dem + connections += [self.tree[:] >> self.dem['tree'], + self.args[:] >> self.dem['args'], + self.georeferencing['reconstruction'] >> self.dem['reconstruction']] + # create odm orthophoto connections += [self.tree[:] >> self.orthophoto['tree'], self.args[:] >> self.orthophoto['args'], diff --git a/scripts/odm_dem.py b/scripts/odm_dem.py new file mode 100644 index 00000000..57d8870f --- /dev/null +++ b/scripts/odm_dem.py @@ -0,0 +1,184 @@ +import ecto, os, json +from shutil import copyfile + +from opendm import io +from opendm import log +from opendm import system +from opendm import context +from opendm import types + + +class ODMDEMCell(ecto.Cell): + def declare_params(self, params): + params.declare("verbose", 'print additional messages to console', False) + + def declare_io(self, params, inputs, outputs): + inputs.declare("tree", "Struct with paths", []) + inputs.declare("args", "The application arguments.", {}) + inputs.declare("reconstruction", "list of ODMReconstructions", []) + + def process(self, inputs, outputs): + # Benchmarking + start_time = system.now_raw() + + log.ODM_INFO('Running ODM DEM Cell') + + # get inputs + args = self.inputs.args + tree = self.inputs.tree + las_model_found = io.file_exists(tree.odm_georeferencing_model_las) + env_paths = [context.superbuild_bin_path] + + # Just to make sure + l2d_module_installed = True + try: + system.run('l2d_classify --help > /dev/null', env_paths) + except: + log.ODM_WARNING('lidar2dems is not installed properly') + l2d_module_installed = False + + log.ODM_INFO('Create DSM: ' + str(args.dsm)) + log.ODM_INFO('Create DTM: ' + str(args.dtm)) + log.ODM_INFO('DEM input file {0} found: {1}'.format(tree.odm_georeferencing_model_las, str(las_model_found))) + + # Do we need to process anything here? + if (args.dsm or args.dtm) and las_model_found and l2d_module_installed: + + # define paths and create working directories + odm_dem_root = tree.path('odm_dem') + system.mkdir_p(odm_dem_root) + + dsm_output_filename = os.path.join(odm_dem_root, 'dsm.tif') + dtm_output_filename = os.path.join(odm_dem_root, 'dtm.tif') + + # check if we rerun cell or not + rerun_cell = (args.rerun is not None and + args.rerun == 'odm_dem') or \ + (args.rerun_all) or \ + (args.rerun_from is not None and + 'odm_dem' in args.rerun_from) + + if (args.dtm and not io.file_exists(dtm_output_filename)) or \ + (args.dsm and not io.file_exists(dsm_output_filename)) or \ + rerun_cell: + + # Extract boundaries and srs of point cloud + summary_file_path = os.path.join(odm_dem_root, 'odm_georeferenced_model.summary.json') + boundary_file_path = os.path.join(odm_dem_root, 'odm_georeferenced_model.boundary.json') + + system.run('pdal info --summary {0} > {1}'.format(tree.odm_georeferencing_model_las, summary_file_path), env_paths) + system.run('pdal info --boundary {0} > {1}'.format(tree.odm_georeferencing_model_las, boundary_file_path), env_paths) + + pc_proj4 = "" + pc_geojson_bounds_feature = None + + with open(summary_file_path, 'r') as f: + json_f = json.loads(f.read()) + pc_proj4 = json_f['summary']['srs']['proj4'] + + with open(boundary_file_path, 'r') as f: + json_f = json.loads(f.read()) + pc_geojson_boundary_feature = json_f['boundary']['boundary_json'] + + # Write bounds to GeoJSON + bounds_geojson_path = os.path.join(odm_dem_root, 'odm_georeferenced_model.bounds.geojson') + with open(bounds_geojson_path, "w") as f: + f.write(json.dumps({ + "type": "FeatureCollection", + "features": [{ + "type": "Feature", + "geometry": pc_geojson_boundary_feature + }] + })) + + bounds_shapefile_path = os.path.join(odm_dem_root, 'bounds.shp') + + # Convert bounds to Shapefile + kwargs = { + 'input': bounds_geojson_path, + 'output': bounds_shapefile_path, + 'proj4': pc_proj4 + } + system.run('ogr2ogr -overwrite -a_srs "{proj4}" {output} {input}'.format(**kwargs)) + + # Process with lidar2dems + terrain_params_map = { + 'flatnonforest': (1, 3), + 'flatforest': (1, 2), + 'complexnonforest': (5, 2), + 'complexforest': (10, 2) + } + terrain_params = terrain_params_map[args.dem_terrain_type.lower()] + + kwargs = { + 'verbose': '-v' if self.params.verbose else '', + 'slope': terrain_params[0], + 'cellsize': terrain_params[1], + 'outdir': odm_dem_root, + 'site': bounds_shapefile_path + } + + l2d_params = '--slope {slope} --cellsize {cellsize} ' \ + '{verbose} ' \ + '-o -s {site} ' \ + '--outdir {outdir}'.format(**kwargs) + + approximate = '--approximate' if args.dem_approximate else '' + + # Classify only if we need a DTM + run_classification = args.dtm + + if run_classification: + system.run('l2d_classify {0} --decimation {1} ' + '{2} --initialDistance {3} {4}'.format( + l2d_params, args.dem_decimation, approximate, + args.dem_initial_distance, tree.odm_georeferencing), env_paths) + else: + log.ODM_INFO("Will skip classification, only DSM is needed") + copyfile(tree.odm_georeferencing_model_las, os.path.join(odm_dem_root, 'bounds-0_l2d_s{slope}c{cellsize}.las'.format(**kwargs))) + + products = [] + if args.dsm: products.append('dsm') + if args.dtm: products.append('dtm') + + radius_steps = [args.dem_resolution] + for _ in range(args.dem_gapfill_steps - 1): + radius_steps.append(radius_steps[-1] * 3) # 3 is arbitrary, maybe there's a better value? + + for product in products: + demargs = { + 'product': product, + 'indir': odm_dem_root, + 'l2d_params': l2d_params, + 'maxsd': args.dem_maxsd, + 'maxangle': args.dem_maxangle, + 'resolution': args.dem_resolution, + 'radius_steps': ' '.join(map(str, radius_steps)), + 'gapfill': '--gapfill' if args.dem_gapfill_steps > 0 else '', + + # If we didn't run a classification, we should pass the decimate parameter here + 'decimation': '--decimation {0}'.format(args.dem_decimation) if not run_classification else '' + } + + system.run('l2d_dems {product} {indir} {l2d_params} ' + '--maxsd {maxsd} --maxangle {maxangle} ' + '--resolution {resolution} --radius {radius_steps} ' + '{decimation} ' + '{gapfill} '.format(**demargs), env_paths) + + # Rename final output + if product == 'dsm': + os.rename(os.path.join(odm_dem_root, 'bounds-0_dsm.idw.tif'), dsm_output_filename) + elif product == 'dtm': + os.rename(os.path.join(odm_dem_root, 'bounds-0_dtm.idw.tif'), dtm_output_filename) + + else: + log.ODM_WARNING('Found existing outputs in: %s' % odm_dem_root) + else: + log.ODM_WARNING('DEM will not be generated') + + if args.time: + system.benchmark(start_time, tree.benchmarking, 'Dem') + + log.ODM_INFO('Running ODM DEM Cell - Finished') + return ecto.OK if args.end_with != 'odm_dem' else ecto.QUIT diff --git a/scripts/odm_georeferencing.py b/scripts/odm_georeferencing.py index 896acf20..9db1a069 100644 --- a/scripts/odm_georeferencing.py +++ b/scripts/odm_georeferencing.py @@ -17,10 +17,6 @@ class ODMGeoreferencingCell(ecto.Cell): 'northing height pixelrow pixelcol imagename', 'gcp_list.txt') params.declare("img_size", 'image size used in calibration', 2400) params.declare("use_exif", 'use exif', False) - params.declare("dem", 'Generate a dem', False) - params.declare("sample_radius", "Minimum distance between samples for DEM gen", 3) - params.declare("gdal_res", "Length of raster cell edges in X/Y units ", 2) - params.declare("gdal_radius", "Radius about cell center bounding points to use to calculate a cell value", 0.5) params.declare("verbose", 'print additional messages to console', False) def declare_io(self, params, inputs, outputs): @@ -175,19 +171,6 @@ class ODMGeoreferencingCell(ecto.Cell): tree.odm_georeferencing_model_las, tree.odm_georeferencing_las_json) - # If --dem, create a DEM - if args.dem: - demcreated = geo_ref.convert_to_dem(tree.odm_georeferencing_model_las, - tree.odm_georeferencing_dem, - tree.odm_georeferencing_dem_json, - self.params.sample_radius, - self.params.gdal_res, - self.params.gdal_radius) - if not demcreated: - log.ODM_WARNING('Something went wrong. Check the logs in odm_georeferencing.') - else: - log.ODM_INFO('DEM created at {0}'.format(tree.odm_georeferencing_dem)) - # XYZ point cloud output log.ODM_INFO("Creating geo-referenced CSV file (XYZ format)") with open(tree.odm_georeferencing_xyz_file, "wb") as csvfile: diff --git a/settings.yaml b/settings.yaml index 587a8af3..9d85393c 100644 --- a/settings.yaml +++ b/settings.yaml @@ -44,10 +44,15 @@ project_path: '' # Example: '/home/user/ODMProjects #texturing_tone_mapping: 'none' #gcp: !!null # YAML tag for None #use_exif: False # Set to True if you have a GCP file (it auto-detects) and want to use EXIF -#dem: False -#dem_sample_radius: 1.0 -#dem_resolution: 2 -#dem_radius: 0.5 +#dtm: False # Use this tag to build a DTM (Digital Terrain Model +#dsm: False # Use this tag to build a DSM (Digital Surface Model +#dem-gapfill-steps: 4 +#dem-resolution: 0.1 +#dem-maxangle:20 +#dem-maxsd: 2.5 +#dem-approximate: False +#dem-decimation: 1 +#dem-terrain-type: ComplexForest #orthophoto_resolution: 20.0 # Pixels/meter #orthophoto_target_srs: !!null # Currently does nothing #orthophoto_no_tiled: False