From d7302d557317008a8a008dc5e9ea586b376b84eb Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 21 Jan 2020 14:59:00 -0500 Subject: [PATCH 01/11] Add multispectral preset --- app/boot.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/boot.py b/app/boot.py index d1e9dbba..52f19c27 100644 --- a/app/boot.py +++ b/app/boot.py @@ -128,6 +128,10 @@ def add_default_presets(): {'name': 'dem-resolution', 'value': "2.0"}, {'name': 'orthophoto-resolution', 'value': "2.0"}, ]}) + Preset.objects.update_or_create(name='Multispectral', system=True, + defaults={'options': [{'name': 'texturing-skip-global-seam-leveling', 'value': True}, + {'name': 'texturing-data-term', 'value': 'area'}, + ]}) Preset.objects.update_or_create(name='Default', system=True, defaults={'options': [{'name': 'dsm', 'value': True}]}) From ec6b74b45b2abbb2865ff1b9b8522f7bd937244d Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 21 Jan 2020 15:00:32 -0500 Subject: [PATCH 02/11] Fixed tests --- app/tests/test_api_preset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/tests/test_api_preset.py b/app/tests/test_api_preset.py index 7c05303e..56131232 100644 --- a/app/tests/test_api_preset.py +++ b/app/tests/test_api_preset.py @@ -28,6 +28,7 @@ class TestApiPreset(BootTestCase): self.assertTrue(Preset.objects.filter(name="Buildings", system=True).exists()) self.assertTrue(Preset.objects.filter(name="3D Model", system=True).exists()) self.assertTrue(Preset.objects.filter(name="Point of Interest", system=True).exists()) + self.assertTrue(Preset.objects.filter(name="Multispectral", system=True).exists()) def test_preset(self): client = APIClient() @@ -57,7 +58,7 @@ class TestApiPreset(BootTestCase): self.assertTrue(res.status_code == status.HTTP_200_OK) # Only ours and global presets are available - self.assertTrue(len(res.data) == 12) + self.assertTrue(len(res.data) == 13) self.assertTrue('My Local Preset' in [preset['name'] for preset in res.data]) self.assertTrue('High Resolution' in [preset['name'] for preset in res.data]) self.assertTrue('Global Preset #1' in [preset['name'] for preset in res.data]) From be4851713d24a3dc4fdadcc618201c1d03566821 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 21 Jan 2020 17:39:30 -0500 Subject: [PATCH 03/11] Updated rio-cogeo --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 28103cb1..93b22aee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -56,4 +56,4 @@ webcolors==1.5 rasterio==1.1.0 -e git://github.com/OpenDroneMap/rio-tiler.git#egg=rio-tiler rio-color==1.0.0 -rio-cogeo==1.1.7 \ No newline at end of file +rio-cogeo==1.1.8 \ No newline at end of file From f56451234b63735ccaeb7644ecc371dff08e70ed Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 21 Jan 2020 17:41:17 -0500 Subject: [PATCH 04/11] Change order of multispectral preset --- app/boot.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/boot.py b/app/boot.py index 52f19c27..b91bc669 100644 --- a/app/boot.py +++ b/app/boot.py @@ -96,6 +96,10 @@ def boot(): def add_default_presets(): try: + Preset.objects.update_or_create(name='Multispectral', system=True, + defaults={'options': [{'name': 'texturing-skip-global-seam-leveling', 'value': True}, + {'name': 'texturing-data-term', 'value': 'area'}, + ]}) Preset.objects.update_or_create(name='Volume Analysis', system=True, defaults={'options': [{'name': 'dsm', 'value': True}, {'name': 'dem-resolution', 'value': '2'}, @@ -128,10 +132,6 @@ def add_default_presets(): {'name': 'dem-resolution', 'value': "2.0"}, {'name': 'orthophoto-resolution', 'value': "2.0"}, ]}) - Preset.objects.update_or_create(name='Multispectral', system=True, - defaults={'options': [{'name': 'texturing-skip-global-seam-leveling', 'value': True}, - {'name': 'texturing-data-term', 'value': 'area'}, - ]}) Preset.objects.update_or_create(name='Default', system=True, defaults={'options': [{'name': 'dsm', 'value': True}]}) From 07a3ca3f2a3d95276cb3fc9a4c455ecc82f227be Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 21 Jan 2020 22:08:40 -0500 Subject: [PATCH 05/11] Scale fix for 16bit orthophoto TIFFs --- app/api/tiler.py | 15 ++++++++------- app/static/app/js/classes/Utils.js | 4 +++- app/static/app/js/components/Map.jsx | 6 +++--- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/app/api/tiler.py b/app/api/tiler.py index 80aacd86..cf8790e0 100644 --- a/app/api/tiler.py +++ b/app/api/tiler.py @@ -6,7 +6,8 @@ from django.http import HttpResponse from rio_tiler.errors import TileOutsideBounds from rio_tiler.mercator import get_zooms from rio_tiler import main -from rio_tiler.utils import array_to_image, get_colormap, expression, linear_rescale, _chunks, _apply_discrete_colormap, has_alpha_band +from rio_tiler.utils import array_to_image, get_colormap, expression, linear_rescale, _chunks, _apply_discrete_colormap, has_alpha_band, \ + non_alpha_indexes from rio_tiler.profiles import img_profiles import numpy as np @@ -73,6 +74,7 @@ def rescale_tile(tile, mask, rescale = None): rescale_arr = list(_chunks(rescale_arr, 2)) if len(rescale_arr) != tile.shape[0]: rescale_arr = ((rescale_arr[0]),) * tile.shape[0] + for bdx in range(tile.shape[0]): if mask is not None: tile[bdx] = np.where( @@ -320,8 +322,8 @@ class Tiles(TaskNestedView): if z < minzoom - ZOOM_EXTRA_LEVELS or z > maxzoom + ZOOM_EXTRA_LEVELS: raise exceptions.NotFound() - # Handle N-bands datasets - if tile_type == 'orthophoto': + # Handle N-bands datasets for orthophotos (not plant health) + if tile_type == 'orthophoto' and expr is None: ci = src.colorinterp # More than 4 bands? @@ -335,9 +337,12 @@ class Tiles(TaskNestedView): ci.index(ColorInterp.blue) + 1,) # TODO: adding alpha band should fix black backgrounds # but the tiles disappear. Probable bug in rasterio/GDAL + else: # Fallback to first four indexes = (1, 2, 3, ) # , 4, ) + elif has_alpha: + indexes = non_alpha_indexes(src) resampling="nearest" padding=0 @@ -357,10 +362,6 @@ class Tiles(TaskNestedView): except TileOutsideBounds: raise exceptions.NotFound("Outside of bounds") - # Use alpha channel for transparency, don't use the mask if one is provided (redundant) - if has_alpha and expr is None: - mask = None - if color_map: try: color_map = get_colormap(color_map, format="gdal") diff --git a/app/static/app/js/classes/Utils.js b/app/static/app/js/classes/Utils.js index b252f88a..989da530 100644 --- a/app/static/app/js/classes/Utils.js +++ b/app/static/app/js/classes/Utils.js @@ -33,7 +33,9 @@ export default { let paramsRaw = (location.search.replace("?", "").match(/([^&=]+)=?([^&]*)/g) || []); for (let i in paramsRaw){ let parts = paramsRaw[i].split("="); - params[parts[0]] = parts[1]; + if (parts[1] !== undefined){ + params[parts[0]] = parts[1]; + } } return params; }, diff --git a/app/static/app/js/components/Map.jsx b/app/static/app/js/components/Map.jsx index 60e8630b..ddb60a0d 100644 --- a/app/static/app/js/components/Map.jsx +++ b/app/static/app/js/components/Map.jsx @@ -109,9 +109,9 @@ class Map extends React.Component { // Build URL let tileUrl = mres.tiles[0]; - - // Certain types need the rescale parameter - if (["plant", "dsm", "dtm"].indexOf(type) !== -1 && statistics){ + + // Set rescale + if (statistics){ const params = Utils.queryParams({search: tileUrl.slice(tileUrl.indexOf("?"))}); if (statistics["1"]){ // Add rescale From f36571978c60567c75da7978e1047d4cf302c82e Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 21 Jan 2020 22:09:46 -0500 Subject: [PATCH 06/11] Removed old comment --- app/api/tiler.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/app/api/tiler.py b/app/api/tiler.py index cf8790e0..549f8066 100644 --- a/app/api/tiler.py +++ b/app/api/tiler.py @@ -331,16 +331,14 @@ class Tiles(TaskNestedView): # Try to find RGBA band order if ColorInterp.red in ci and \ ColorInterp.green in ci and \ - ColorInterp.blue in ci: # and ColorInterp.alpha in ci: + ColorInterp.blue in ci: indexes = (ci.index(ColorInterp.red) + 1, ci.index(ColorInterp.green) + 1, ci.index(ColorInterp.blue) + 1,) - # TODO: adding alpha band should fix black backgrounds - # but the tiles disappear. Probable bug in rasterio/GDAL - else: - # Fallback to first four - indexes = (1, 2, 3, ) # , 4, ) + # Fallback to first three + indexes = (1, 2, 3, ) + elif has_alpha: indexes = non_alpha_indexes(src) From b6db62c0a82f6819fcc0b325c18dce0ba303bebd Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 22 Jan 2020 10:15:55 -0500 Subject: [PATCH 07/11] Increase travis sleep --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 604e554d..feace2dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,6 @@ services: install: - docker-compose -f docker-compose.yml -f docker-compose.build.yml up --build -d - - sleep 50 + - sleep 60 script: docker-compose exec webapp /bin/bash -c "/webodm/webodm.sh test" From 13d180f3f88bf18833740c593f650703fca83721 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 22 Jan 2020 10:30:08 -0500 Subject: [PATCH 08/11] Travis wait for container instead of sleep --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index feace2dd..f646b9a4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,6 @@ services: install: - docker-compose -f docker-compose.yml -f docker-compose.build.yml up --build -d - - sleep 60 + - while sleep 5; do docker ps -a; if [[ -n "`docker ps | grep webapp | grep ' Up '`" ]]; then break; fi; done script: docker-compose exec webapp /bin/bash -c "/webodm/webodm.sh test" From acf37086ef2ca8de3601258b9b4a90a5d9f80d48 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 22 Jan 2020 11:26:04 -0500 Subject: [PATCH 09/11] Update NodeJS to 10 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a68d193a..c90874a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ ENV PROJ_LIB=/usr/share/proj RUN mkdir /webodm WORKDIR /webodm -RUN curl --silent --location https://deb.nodesource.com/setup_8.x | bash - +RUN curl --silent --location https://deb.nodesource.com/setup_10.x | bash - RUN apt-get -qq install -y nodejs # Configure use of testing branch of Debian From 0003416cc70251ad6e90380a67e6fa800f757463 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 22 Jan 2020 12:07:33 -0500 Subject: [PATCH 10/11] Added circleCI config --- config.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 config.yml diff --git a/config.yml b/config.yml new file mode 100644 index 00000000..fd432a78 --- /dev/null +++ b/config.yml @@ -0,0 +1,33 @@ +# CircleCI 2.0 configuration file +# +# Check https://circleci.com/docs/2.0/ for more details +# +version: 2 +jobs: + build: + working_directory: /app + docker: + - image: docker:stable-git + steps: + - checkout + - setup_remote_docker + - run: + name: Install dependencies + command: | + apk add --no-cache \ + py-pip=9.0.0-r1 + pip install \ + docker-compose==1.25.0 + - run: + name: Build application Docker image + command: | + docker-compose -f docker-compose.yml -f docker-compose.build.yml up --build -d + - run: + name: Wait for Docker Image + command: | + while sleep 5; do docker ps -a; if [[ -n "`docker ps | grep webapp | grep ' Up '`" ]]; then break; fi; done + - run: + name: Run tests + command: | + docker-compose exec webapp /bin/bash -c "/webodm/webodm.sh test" + From 19e89ce7bd2cc8be32157ed11c84b3a7bf7e855d Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 22 Jan 2020 13:39:22 -0500 Subject: [PATCH 11/11] Fix GDAL detection, remove circleCI config --- config.yml | 33 --------------------------------- start.sh | 2 +- 2 files changed, 1 insertion(+), 34 deletions(-) delete mode 100644 config.yml diff --git a/config.yml b/config.yml deleted file mode 100644 index fd432a78..00000000 --- a/config.yml +++ /dev/null @@ -1,33 +0,0 @@ -# CircleCI 2.0 configuration file -# -# Check https://circleci.com/docs/2.0/ for more details -# -version: 2 -jobs: - build: - working_directory: /app - docker: - - image: docker:stable-git - steps: - - checkout - - setup_remote_docker - - run: - name: Install dependencies - command: | - apk add --no-cache \ - py-pip=9.0.0-r1 - pip install \ - docker-compose==1.25.0 - - run: - name: Build application Docker image - command: | - docker-compose -f docker-compose.yml -f docker-compose.build.yml up --build -d - - run: - name: Wait for Docker Image - command: | - while sleep 5; do docker ps -a; if [[ -n "`docker ps | grep webapp | grep ' Up '`" ]]; then break; fi; done - - run: - name: Run tests - command: | - docker-compose exec webapp /bin/bash -c "/webodm/webodm.sh test" - diff --git a/start.sh b/start.sh index 1ee08830..8bb75acb 100755 --- a/start.sh +++ b/start.sh @@ -28,7 +28,7 @@ if [ $? -ne 0 ]; then fi # Check GDAL version -python -c "import sys;import re;import subprocess;version = subprocess.Popen([\"gdalinfo\", \"--version\"], stdout=subprocess.PIPE).communicate()[0].decode().rstrip();ret = 0 if re.compile('^GDAL [2-9]\.[1-9]+').match(version) else 1; print('Checking GDAL version... ' + ('{}, excellent!'.format(version) if ret == 0 else version));sys.exit(ret);" +python -c "import sys;import re;import subprocess;version = subprocess.Popen([\"gdalinfo\", \"--version\"], stdout=subprocess.PIPE).communicate()[0].decode().rstrip();ret = 0 if re.compile('^GDAL [2-9]\.[0-9]+').match(version) else 1; print('Checking GDAL version... ' + ('{}, excellent!'.format(version) if ret == 0 else version));sys.exit(ret);" if [ $? -ne 0 ]; then almost_there echo -e "\033[33mYour system is currently using a version of GDAL that is too old, or GDAL is not installed. You need to install or configure your system to use GDAL 2.1 or higher. If you have installed multiple versions of GDAL, make sure the newer one takes priority in your PATH environment variable.\033[39m"