diff --git a/config-default.json b/config-default.json index e7fad31..e1536fc 100644 --- a/config-default.json +++ b/config-default.json @@ -14,5 +14,6 @@ "parallelQueueProcessing": 2, "cleanupTasksAfter": 3, "test": false, - "testSkipOrthophotos": false + "testSkipOrthophotos": false, + "testSkipDems": false } \ No newline at end of file diff --git a/config.js b/config.js index 703cc3b..27416c7 100644 --- a/config.js +++ b/config.js @@ -35,6 +35,7 @@ Options: --cleanup_tasks_after Number of days that elapse before deleting finished and canceled tasks (default: 3) --test Enable test mode. In test mode, no commands are sent to OpenDroneMap. This can be useful during development or testing (default: false) --test_skip_orthophotos If test mode is enabled, skip orthophoto results when generating assets. (default: false) + --test_skip_dems If test mode is enabled, skip dems results when generating assets. (default: false) --powercycle When set, the application exits immediately after powering up. Useful for testing launch and compilation issues. Log Levels: error | debug | info | verbose | debug | silly @@ -81,6 +82,7 @@ config.parallelQueueProcessing = argv.parallel_queue_processing || fromConfigFil config.cleanupTasksAfter = argv.cleanup_tasks_after || fromConfigFile("cleanupTasksAfter", 3); config.test = argv.test || fromConfigFile("test", false); config.testSkipOrthophotos = argv.test_skip_orthophotos || fromConfigFile("testSkipOrthophotos", false); +config.testSkipDems = argv.test_skip_dems || fromConfigFile("testSkipDems", false); config.powercycle = argv.powercycle || fromConfigFile("powercycle", false); module.exports = config; diff --git a/libs/Task.js b/libs/Task.js index faf797f..6e723a4 100644 --- a/libs/Task.js +++ b/libs/Task.js @@ -305,20 +305,33 @@ module.exports = class Task{ }; // All paths are relative to the project directory (./data//) - let allFolders = ['odm_orthophoto', 'odm_georeferencing', 'odm_texturing', 'odm_meshing', 'orthophoto_tiles', 'potree_pointcloud']; + let allPaths = ['odm_orthophoto', 'odm_georeferencing', 'odm_texturing', + 'odm_dem/dsm.tif', 'odm_dem/dtm.tif', 'dsm_tiles', 'dtm_tiles', + 'odm_meshing', 'orthophoto_tiles', 'potree_pointcloud']; - if (config.test && config.testSkipOrthophotos){ - logger.info("Test mode will skip orthophoto generation"); + if (config.test){ + if (config.testSkipOrthophotos){ + logger.info("Test mode will skip orthophoto generation"); - // Exclude these folders from the all.zip archive - ['odm_orthophoto', 'orthophoto_tiles'].forEach(dir => { - allFolders.splice(allFolders.indexOf(dir), 1); - }); + // Exclude these folders from the all.zip archive + ['odm_orthophoto', 'orthophoto_tiles'].forEach(dir => { + allPaths.splice(allPaths.indexOf(dir), 1); + }); + } + + if (config.testSkipDems){ + logger.info("Test mode will skip DEMs generation"); + + // Exclude these folders from the all.zip archive + ['odm_dem/dsm.tif', 'odm_dem/dtm.tif', 'dsm_tiles', 'dtm_tiles'].forEach(p => { + allPaths.splice(allPaths.indexOf(p), 1); + }); + } } async.series([ runPostProcessingScript(), - createZipArchive('all.zip', allFolders) + createZipArchive('all.zip', allPaths) ], (err) => { if (!err){ this.setStatus(statusCodes.COMPLETED); diff --git a/scripts/color_relief.txt b/scripts/color_relief.txt new file mode 100644 index 0000000..86d03ae --- /dev/null +++ b/scripts/color_relief.txt @@ -0,0 +1,12 @@ +0% 255 0 255 +10% 128 0 255 +20% 0 0 255 +30% 0 128 255 +40% 0 255 255 +50% 0 255 128 +60% 0 255 0 +70% 128 255 0 +80% 255 255 0 +90% 255 128 0 +100% 255 0 0 +nv 0 0 0 0 \ No newline at end of file diff --git a/scripts/hsv_merge.py b/scripts/hsv_merge.py new file mode 100644 index 0000000..0d2e0c2 --- /dev/null +++ b/scripts/hsv_merge.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python +#****************************************************************************** +# $Id$ +# +# Project: GDAL Python Interface +# Purpose: Script to merge greyscale as intensity into an RGB(A) image, for +# instance to apply hillshading to a dem colour relief. +# Author: Frank Warmerdam, warmerdam@pobox.com +# Trent Hare (USGS) +# +#****************************************************************************** +# Copyright (c) 2009, Frank Warmerdam +# Copyright (c) 2010, Even Rouault +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +#****************************************************************************** + +import sys + +import numpy +from osgeo import gdal + +# ============================================================================= +# rgb_to_hsv() +# +# rgb comes in as [r,g,b] with values in the range [0,255]. The returned +# hsv values will be with hue and saturation in the range [0,1] and value +# in the range [0,255] +# +def rgb_to_hsv( r,g,b ): + + maxc = numpy.maximum(r,numpy.maximum(g,b)) + minc = numpy.minimum(r,numpy.minimum(g,b)) + + v = maxc + + minc_eq_maxc = numpy.equal(minc,maxc) + + # compute the difference, but reset zeros to ones to avoid divide by zeros later. + ones = numpy.ones((r.shape[0],r.shape[1])) + maxc_minus_minc = numpy.choose( minc_eq_maxc, (maxc-minc,ones) ) + + s = (maxc-minc) / numpy.maximum(ones,maxc) + rc = (maxc-r) / maxc_minus_minc + gc = (maxc-g) / maxc_minus_minc + bc = (maxc-b) / maxc_minus_minc + + maxc_is_r = numpy.equal(maxc,r) + maxc_is_g = numpy.equal(maxc,g) + maxc_is_b = numpy.equal(maxc,b) + + h = numpy.zeros((r.shape[0],r.shape[1])) + h = numpy.choose( maxc_is_b, (h,4.0+gc-rc) ) + h = numpy.choose( maxc_is_g, (h,2.0+rc-bc) ) + h = numpy.choose( maxc_is_r, (h,bc-gc) ) + + h = numpy.mod(h/6.0,1.0) + + hsv = numpy.asarray([h,s,v]) + + return hsv + +# ============================================================================= +# hsv_to_rgb() +# +# hsv comes in as [h,s,v] with hue and saturation in the range [0,1], +# but value in the range [0,255]. + +def hsv_to_rgb( hsv ): + + h = hsv[0] + s = hsv[1] + v = hsv[2] + + #if s == 0.0: return v, v, v + i = (h*6.0).astype(int) + f = (h*6.0) - i + p = v*(1.0 - s) + q = v*(1.0 - s*f) + t = v*(1.0 - s*(1.0-f)) + + r = i.choose( v, q, p, p, t, v ) + g = i.choose( t, v, v, q, p, p ) + b = i.choose( p, p, t, v, v, q ) + + rgb = numpy.asarray([r,g,b]).astype(numpy.uint8) + + return rgb + +# ============================================================================= +# Usage() + +def Usage(): + print("""Usage: hsv_merge.py [-q] [-of format] src_color src_greyscale dst_color + +where src_color is a RGB or RGBA dataset, + src_greyscale is a greyscale dataset (e.g. the result of gdaldem hillshade) + dst_color will be a RGB or RGBA dataset using the greyscale as the + intensity for the color dataset. +""") + sys.exit(1) + +# ============================================================================= +# Mainline +# ============================================================================= + +argv = gdal.GeneralCmdLineProcessor( sys.argv ) +if argv is None: + sys.exit( 0 ) + +format = 'GTiff' +src_color_filename = None +src_greyscale_filename = None +dst_color_filename = None +quiet = False + +# Parse command line arguments. +i = 1 +while i < len(argv): + arg = argv[i] + + if arg == '-of': + i = i + 1 + format = argv[i] + + elif arg == '-q' or arg == '-quiet': + quiet = True + + elif src_color_filename is None: + src_color_filename = argv[i] + + elif src_greyscale_filename is None: + src_greyscale_filename = argv[i] + + elif dst_color_filename is None: + dst_color_filename = argv[i] + else: + Usage() + + i = i + 1 + +if dst_color_filename is None: + Usage() + +datatype = gdal.GDT_Byte + +hilldataset = gdal.Open( src_greyscale_filename, gdal.GA_ReadOnly ) +colordataset = gdal.Open( src_color_filename, gdal.GA_ReadOnly ) + +#check for 3 or 4 bands in the color file +if (colordataset.RasterCount != 3 and colordataset.RasterCount != 4): + print('Source image does not appear to have three or four bands as required.') + sys.exit(1) + +#define output format, name, size, type and set projection +out_driver = gdal.GetDriverByName(format) +outdataset = out_driver.Create(dst_color_filename, colordataset.RasterXSize, \ + colordataset.RasterYSize, colordataset.RasterCount, datatype) +outdataset.SetProjection(hilldataset.GetProjection()) +outdataset.SetGeoTransform(hilldataset.GetGeoTransform()) + +#assign RGB and hillshade bands +rBand = colordataset.GetRasterBand(1) +gBand = colordataset.GetRasterBand(2) +bBand = colordataset.GetRasterBand(3) +if colordataset.RasterCount == 4: + aBand = colordataset.GetRasterBand(4) +else: + aBand = None + +hillband = hilldataset.GetRasterBand(1) +hillbandnodatavalue = hillband.GetNoDataValue() + +#check for same file size +if ((rBand.YSize != hillband.YSize) or (rBand.XSize != hillband.XSize)): + print('Color and hillshade must be the same size in pixels.') + sys.exit(1) + +#loop over lines to apply hillshade +for i in range(hillband.YSize): + #load RGB and Hillshade arrays + rScanline = rBand.ReadAsArray(0, i, hillband.XSize, 1, hillband.XSize, 1) + gScanline = gBand.ReadAsArray(0, i, hillband.XSize, 1, hillband.XSize, 1) + bScanline = bBand.ReadAsArray(0, i, hillband.XSize, 1, hillband.XSize, 1) + hillScanline = hillband.ReadAsArray(0, i, hillband.XSize, 1, hillband.XSize, 1) + + #convert to HSV + hsv = rgb_to_hsv( rScanline, gScanline, bScanline ) + + # if there's nodata on the hillband, use the v value from the color + # dataset instead of the hillshade value. + if hillbandnodatavalue is not None: + equal_to_nodata = numpy.equal(hillScanline, hillbandnodatavalue) + v = numpy.choose(equal_to_nodata,(hillScanline,hsv[2])) + else: + v = hillScanline + + #replace v with hillshade + hsv_adjusted = numpy.asarray( [hsv[0], hsv[1], v] ) + + #convert back to RGB + dst_color = hsv_to_rgb( hsv_adjusted ) + + #write out new RGB bands to output one band at a time + outband = outdataset.GetRasterBand(1) + outband.WriteArray(dst_color[0], 0, i) + outband = outdataset.GetRasterBand(2) + outband.WriteArray(dst_color[1], 0, i) + outband = outdataset.GetRasterBand(3) + outband.WriteArray(dst_color[2], 0, i) + if aBand is not None: + aScanline = aBand.ReadAsArray(0, i, hillband.XSize, 1, hillband.XSize, 1) + outband = outdataset.GetRasterBand(4) + outband.WriteArray(aScanline, 0, i) + + #update progress line + if not quiet: + gdal.TermProgress_nocb( (float(i+1) / hillband.YSize) ) diff --git a/scripts/postprocess.sh b/scripts/postprocess.sh index dc37b2b..3607fc2 100755 --- a/scripts/postprocess.sh +++ b/scripts/postprocess.sh @@ -15,18 +15,46 @@ if [ -z "$1" ]; then fi # Switch to project path folder (data//) -cd "$(dirname "$0")/../$1" +script_path=$(realpath $(dirname "$0")) +cd "$script_path/../$1" echo "Postprocessing: $(pwd)" +# Generate colored shaded relief for DTM/DSM files if available +dem_products=() +if [ -e "odm_dem/dsm.tif" ]; then dem_products=(${dem_products[@]} dsm); fi +if [ -e "odm_dem/dtm.tif" ]; then dem_products=(${dem_products[@]} dtm); fi + +if hash gdaldem 2>/dev/null; then + for dem_product in ${dem_products[@]}; do + dem_path="odm_dem/""$dem_product"".tif" + + gdaldem color-relief $dem_path $script_path/color_relief.txt "odm_dem/""$dem_product""_colored.tif" -alpha -co ALPHA=YES + gdaldem hillshade $dem_path "odm_dem/""$dem_product""_hillshade.tif" -z 1.0 -s 1.0 -az 315.0 -alt 45.0 + python "$script_path/hsv_merge.py" "odm_dem/""$dem_product""_colored.tif" "odm_dem/""$dem_product""_hillshade.tif" "odm_dem/""$dem_product""_colored_hillshade.tif" + done +else + echo "gdaldem is not installed, will skip colored hillshade generation" +fi + # Generate Tiles if hash gdal2tiles.py 2>/dev/null; then + g2t_options="-z 12-21 -n -w none" orthophoto_path="odm_orthophoto/odm_orthophoto.tif" - + if [ -e "$orthophoto_path" ]; then - gdal2tiles.py -z 12-21 -n -w none $orthophoto_path orthophoto_tiles + gdal2tiles.py $g2t_options $orthophoto_path orthophoto_tiles else echo "No orthophoto found at $orthophoto_path: will skip tiling" fi + + for dem_product in ${dem_products[@]}; do + colored_dem_path="odm_dem/""$dem_product""_colored_hillshade.tif" + if [ -e "$colored_dem_path" ]; then + gdal2tiles.py $g2t_options $colored_dem_path "$dem_product""_tiles" + else + echo "No $dem_product found at $colored_dem_path: will skip tiling" + fi + done else echo "gdal2tiles.py is not installed, will skip tiling" fi diff --git a/tests/processing_results/dsm_tiles/.gitignore b/tests/processing_results/dsm_tiles/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/tests/processing_results/dsm_tiles/16/16020/42443.png b/tests/processing_results/dsm_tiles/16/16020/42443.png new file mode 100644 index 0000000..2cc44a8 Binary files /dev/null and b/tests/processing_results/dsm_tiles/16/16020/42443.png differ diff --git a/tests/processing_results/dsm_tiles/16/16021/42443.png b/tests/processing_results/dsm_tiles/16/16021/42443.png new file mode 100644 index 0000000..935041f Binary files /dev/null and b/tests/processing_results/dsm_tiles/16/16021/42443.png differ diff --git a/tests/processing_results/dsm_tiles/17/32041/84886.png b/tests/processing_results/dsm_tiles/17/32041/84886.png new file mode 100644 index 0000000..a7edb24 Binary files /dev/null and b/tests/processing_results/dsm_tiles/17/32041/84886.png differ diff --git a/tests/processing_results/dsm_tiles/17/32042/84886.png b/tests/processing_results/dsm_tiles/17/32042/84886.png new file mode 100644 index 0000000..a622700 Binary files /dev/null and b/tests/processing_results/dsm_tiles/17/32042/84886.png differ diff --git a/tests/processing_results/dsm_tiles/18/64083/169772.png b/tests/processing_results/dsm_tiles/18/64083/169772.png new file mode 100644 index 0000000..ef2a248 Binary files /dev/null and b/tests/processing_results/dsm_tiles/18/64083/169772.png differ diff --git a/tests/processing_results/dsm_tiles/18/64083/169773.png b/tests/processing_results/dsm_tiles/18/64083/169773.png new file mode 100644 index 0000000..39d499c Binary files /dev/null and b/tests/processing_results/dsm_tiles/18/64083/169773.png differ diff --git a/tests/processing_results/dsm_tiles/18/64084/169772.png b/tests/processing_results/dsm_tiles/18/64084/169772.png new file mode 100644 index 0000000..c5bfcb3 Binary files /dev/null and b/tests/processing_results/dsm_tiles/18/64084/169772.png differ diff --git a/tests/processing_results/dsm_tiles/18/64084/169773.png b/tests/processing_results/dsm_tiles/18/64084/169773.png new file mode 100644 index 0000000..fa22d5e Binary files /dev/null and b/tests/processing_results/dsm_tiles/18/64084/169773.png differ diff --git a/tests/processing_results/dsm_tiles/tilemapresource.xml b/tests/processing_results/dsm_tiles/tilemapresource.xml new file mode 100644 index 0000000..8ba263b --- /dev/null +++ b/tests/processing_results/dsm_tiles/tilemapresource.xml @@ -0,0 +1,15 @@ + + + odm_orthophoto.tif + + EPSG:900913 + + + + + + + + + + \ No newline at end of file diff --git a/tests/processing_results/dtm_tiles/.gitignore b/tests/processing_results/dtm_tiles/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/tests/processing_results/dtm_tiles/16/16020/42443.png b/tests/processing_results/dtm_tiles/16/16020/42443.png new file mode 100644 index 0000000..2cc44a8 Binary files /dev/null and b/tests/processing_results/dtm_tiles/16/16020/42443.png differ diff --git a/tests/processing_results/dtm_tiles/16/16021/42443.png b/tests/processing_results/dtm_tiles/16/16021/42443.png new file mode 100644 index 0000000..935041f Binary files /dev/null and b/tests/processing_results/dtm_tiles/16/16021/42443.png differ diff --git a/tests/processing_results/dtm_tiles/17/32041/84886.png b/tests/processing_results/dtm_tiles/17/32041/84886.png new file mode 100644 index 0000000..a7edb24 Binary files /dev/null and b/tests/processing_results/dtm_tiles/17/32041/84886.png differ diff --git a/tests/processing_results/dtm_tiles/17/32042/84886.png b/tests/processing_results/dtm_tiles/17/32042/84886.png new file mode 100644 index 0000000..a622700 Binary files /dev/null and b/tests/processing_results/dtm_tiles/17/32042/84886.png differ diff --git a/tests/processing_results/dtm_tiles/18/64083/169772.png b/tests/processing_results/dtm_tiles/18/64083/169772.png new file mode 100644 index 0000000..ef2a248 Binary files /dev/null and b/tests/processing_results/dtm_tiles/18/64083/169772.png differ diff --git a/tests/processing_results/dtm_tiles/18/64083/169773.png b/tests/processing_results/dtm_tiles/18/64083/169773.png new file mode 100644 index 0000000..39d499c Binary files /dev/null and b/tests/processing_results/dtm_tiles/18/64083/169773.png differ diff --git a/tests/processing_results/dtm_tiles/18/64084/169772.png b/tests/processing_results/dtm_tiles/18/64084/169772.png new file mode 100644 index 0000000..c5bfcb3 Binary files /dev/null and b/tests/processing_results/dtm_tiles/18/64084/169772.png differ diff --git a/tests/processing_results/dtm_tiles/18/64084/169773.png b/tests/processing_results/dtm_tiles/18/64084/169773.png new file mode 100644 index 0000000..fa22d5e Binary files /dev/null and b/tests/processing_results/dtm_tiles/18/64084/169773.png differ diff --git a/tests/processing_results/dtm_tiles/tilemapresource.xml b/tests/processing_results/dtm_tiles/tilemapresource.xml new file mode 100644 index 0000000..8ba263b --- /dev/null +++ b/tests/processing_results/dtm_tiles/tilemapresource.xml @@ -0,0 +1,15 @@ + + + odm_orthophoto.tif + + EPSG:900913 + + + + + + + + + + \ No newline at end of file diff --git a/tests/processing_results/odm_dem/.gitignore b/tests/processing_results/odm_dem/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/tests/processing_results/odm_dem/dsm.tif b/tests/processing_results/odm_dem/dsm.tif new file mode 100644 index 0000000..33b5b18 Binary files /dev/null and b/tests/processing_results/odm_dem/dsm.tif differ diff --git a/tests/processing_results/odm_dem/dtm.tif b/tests/processing_results/odm_dem/dtm.tif new file mode 100644 index 0000000..33b5b18 Binary files /dev/null and b/tests/processing_results/odm_dem/dtm.tif differ