From ae0a0cbf6787bb8d6e17bd5aacd26565295893f6 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 29 Mar 2018 17:28:15 -0400 Subject: [PATCH] Volume measurements! Processing done in celery, updated README --- README.md | 2 +- app/boot.py | 4 +++ app/plugins/grass_engine.py | 48 +++++++++++++++++-------- app/tests/test_plugins.py | 3 +- plugins/measure/api.py | 37 +++++++++++++------ plugins/measure/calc_volume.grass | 8 ++++- plugins/measure/public/MeasurePopup.jsx | 7 ++-- plugins/measure/public/app.jsx | 4 +-- plugins/measure/public/dist | 1 - plugins/measure/public/package.json | 4 ++- requirements.txt | 1 + webodm/settings.py | 1 + worker/tasks.py | 7 ++++ 13 files changed, 89 insertions(+), 38 deletions(-) delete mode 120000 plugins/measure/public/dist diff --git a/README.md b/README.md index 4bd8be2f..ddd63486 100644 --- a/README.md +++ b/README.md @@ -189,7 +189,7 @@ Developer, I'm looking to build an app that will stay behind a firewall and just - [X] 2D Map Display - [X] 3D Model Display - [ ] NDVI display -- [ ] Volumetric Measurements +- [X] Volumetric Measurements - [X] Cluster management and setup. - [ ] Mission Planner - [X] Plugins/Webhooks System diff --git a/app/boot.py b/app/boot.py index 9a929df3..ea898715 100644 --- a/app/boot.py +++ b/app/boot.py @@ -35,6 +35,10 @@ def boot(): if settings.DEBUG: logger.warning("Debug mode is ON (for development this is OK)") + # Make sure our app/media/tmp folder exists + if not os.path.exists(settings.MEDIA_TMP): + os.mkdir(settings.MEDIA_TMP) + # Check default group try: default_group, created = Group.objects.get_or_create(name='Default') diff --git a/app/plugins/grass_engine.py b/app/plugins/grass_engine.py index 94e6b6c8..885780ec 100644 --- a/app/plugins/grass_engine.py +++ b/app/plugins/grass_engine.py @@ -3,8 +3,12 @@ import shutil import tempfile import subprocess import os +import geojson + from string import Template +from webodm import settings + logger = logging.getLogger('app.logger') class GrassEngine: @@ -18,24 +22,28 @@ class GrassEngine: else: logger.info("Initializing GRASS engine using {}".format(self.grass_binary)) - def create_context(self): + def create_context(self, serialized_context = {}): if self.grass_binary is None: raise GrassEngineException("GRASS engine is unavailable") - return GrassContext(self.grass_binary) - + return GrassContext(self.grass_binary, **serialized_context) class GrassContext: - def __init__(self, grass_binary): + def __init__(self, grass_binary, tmpdir = None, template_args = {}, location = None): self.grass_binary = grass_binary - self.cwd = tempfile.mkdtemp('_webodm_grass') - self.template_args = {} - self.location = None + if tmpdir is None: + tmpdir = os.path.basename(tempfile.mkdtemp('_grass_engine', dir=settings.MEDIA_TMP)) + self.tmpdir = tmpdir + self.template_args = template_args + self.location = location + + def get_cwd(self): + return os.path.join(settings.MEDIA_TMP, self.tmpdir) def add_file(self, filename, source, use_as_location=False): param = os.path.splitext(filename)[0] # filename without extension - dst_path = os.path.abspath(os.path.join(self.cwd, filename)) - with open(dst_path) as f: + dst_path = os.path.abspath(os.path.join(self.get_cwd(), filename)) + with open(dst_path, 'w') as f: f.write(source) self.template_args[param] = dst_path @@ -51,7 +59,7 @@ class GrassContext: """ :param location: either a "epsg:XXXXX" string or a path to a geospatial file defining the location """ - if not location.startsWith('epsg:'): + if not location.startswith('epsg:'): location = os.path.abspath(location) self.location = location @@ -74,23 +82,33 @@ class GrassContext: tmpl = Template(script_content) # Write script to disk - with open(os.path.join(self.cwd, 'script.sh')) as f: + with open(os.path.join(self.get_cwd(), 'script.sh'), 'w') as f: f.write(tmpl.substitute(self.template_args)) # Execute it p = subprocess.Popen([self.grass_binary, '-c', self.location, 'location', '--exec', 'sh', 'script.sh'], - cwd=self.cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + cwd=self.get_cwd(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() + out = out.decode('utf-8').strip() + err = err.decode('utf-8').strip() + if p.returncode == 0: return out else: - raise GrassEngineException("Could not execute GRASS script {} from {}: {}".format(script, self.cwd, err)) + raise GrassEngineException("Could not execute GRASS script {} from {}: {}".format(script, self.get_cwd(), err)) + + def serialize(self): + return { + 'tmpdir': self.tmpdir, + 'template_args': self.template_args, + 'location': self.location + } def __del__(self): # Cleanup - if os.path.exists(self.cwd): - shutil.rmtree(self.cwd) + if os.path.exists(self.get_cwd()): + shutil.rmtree(self.get_cwd()) class GrassEngineException(Exception): pass diff --git a/app/tests/test_plugins.py b/app/tests/test_plugins.py index becb52c8..004d6e9c 100644 --- a/app/tests/test_plugins.py +++ b/app/tests/test_plugins.py @@ -46,5 +46,4 @@ class TestPlugins(BootTestCase): self.assertTrue(os.path.exists(test_plugin.get_path("public/node_modules"))) # TODO: - # test API endpoints - # test python hooks + # test GRASS engine diff --git a/plugins/measure/api.py b/plugins/measure/api.py index f0c5234b..1a2ee9b2 100644 --- a/plugins/measure/api.py +++ b/plugins/measure/api.py @@ -1,13 +1,16 @@ import os +import json from rest_framework import serializers from rest_framework import status from rest_framework.response import Response from app.api.tasks import TaskNestedView -from app.plugins.grass_engine import grass +from worker.tasks import execute_grass_script +from app.plugins.grass_engine import grass, GrassEngineException +from geojson import Feature, Point, FeatureCollection class GeoJSONSerializer(serializers.Serializer): area = serializers.JSONField(help_text="Polygon contour defining the volume area to compute") @@ -22,18 +25,30 @@ class TaskVolume(TaskNestedView): serializer = GeoJSONSerializer(data=request.data) serializer.is_valid(raise_exception=True) + area = serializer['area'].value + points = FeatureCollection([Feature(geometry=Point(coords)) for coords in area['geometry']['coordinates'][0]]) + dsm = os.path.abspath(task.get_asset_download_path("dsm.tif")) - # context = grass.create_context() - # context.add_file('area_file.geojson', serializer['area']) - # context.add_file('points_file.geojson', 'aaa') - # context.add_param('dsm_file', os.path.abspath(task.get_asset_download_path("dsm.tif"))) - # context.execute(os.path.join( - # os.path.dirname(os.path.abspath(__file__), - # "calc_volume.grass" - # ))) + try: + context = grass.create_context() + context.add_file('area_file.geojson', json.dumps(area)) + context.add_file('points_file.geojson', str(points)) + context.add_param('dsm_file', dsm) + context.set_location(dsm) + + output = execute_grass_script.delay(os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "calc_volume.grass" + ), context.serialize()).get() + + cols = output.split(':') + if len(cols) == 7: + return Response({'volume': str(abs(float(cols[6])))}, status=status.HTTP_200_OK) + else: + raise GrassEngineException("Invalid GRASS output: {}".format(output)) + except GrassEngineException as e: + return Response({'error': str(e)}, status=status.HTTP_200_OK) - print(serializer['area']) - return Response(30, status=status.HTTP_200_OK) diff --git a/plugins/measure/calc_volume.grass b/plugins/measure/calc_volume.grass index 30427b2f..b990287e 100755 --- a/plugins/measure/calc_volume.grass +++ b/plugins/measure/calc_volume.grass @@ -7,9 +7,15 @@ v.import input=${area_file} output=polygon_area --overwrite v.import input=${points_file} output=polygon_points --overwrite v.buffer -s --overwrite input=polygon_area type=area output=region distance=3 minordistance=3 +r.external input=${dsm_file} output=dsm --overwrite + +g.region rast=dsm g.region vector=region -r.import input=${dsm_file} output=dsm --overwrite +# prevent : removing eventual existing mask +r.mask -r +r.mask vect=region + v.what.rast map=polygon_points raster=dsm column=height v.to.rast input=polygon_area output=r_polygon_area use=val value=255 --overwrite diff --git a/plugins/measure/public/MeasurePopup.jsx b/plugins/measure/public/MeasurePopup.jsx index 5b2a3e02..366570fc 100644 --- a/plugins/measure/public/MeasurePopup.jsx +++ b/plugins/measure/public/MeasurePopup.jsx @@ -18,10 +18,9 @@ module.exports = class MeasurePopup extends React.Component { constructor(props){ super(props); - console.log(props); this.state = { - volume: null, // to be calculated, + volume: null, // to be calculated error: "" }; } @@ -51,7 +50,7 @@ module.exports = class MeasurePopup extends React.Component { contentType: "application/json" }).done(result => { if (result.volume){ - this.setState({volume}); + this.setState({volume: parseFloat(result.volume)}); }else if (result.error){ this.setState({error: result.error}); }else{ @@ -95,7 +94,7 @@ module.exports = class MeasurePopup extends React.Component {

Area: {this.props.model.areaDisplay}

Perimeter: {this.props.model.lengthDisplay}

{volume === null && !error &&

Volume: computing...

} - {typeof volume === "number" &&

Volume: {volume.toFixed("3")} Cubic Meters

} + {typeof volume === "number" &&

Volume: {volume.toFixed("2")} Cubic Meters ({(volume * 35.3147).toFixed(2)} Cubic Feet)

} {error &&

Volume: {error}

} ); } diff --git a/plugins/measure/public/app.jsx b/plugins/measure/public/app.jsx index 64414a0a..338fdeba 100644 --- a/plugins/measure/public/app.jsx +++ b/plugins/measure/public/app.jsx @@ -1,7 +1,7 @@ import L from 'leaflet'; import './app.scss'; -import './dist/leaflet-measure'; -import './dist/leaflet-measure.css'; +import 'leaflet-measure-ex/dist/leaflet-measure'; +import 'leaflet-measure-ex/dist/leaflet-measure.css'; import MeasurePopup from './MeasurePopup'; import ReactDOM from 'ReactDOM'; import React from 'react'; diff --git a/plugins/measure/public/dist b/plugins/measure/public/dist deleted file mode 120000 index 2296f3a0..00000000 --- a/plugins/measure/public/dist +++ /dev/null @@ -1 +0,0 @@ -../../../../leaflet-measure-ex/dist/ \ No newline at end of file diff --git a/plugins/measure/public/package.json b/plugins/measure/public/package.json index 6373d65c..5066d7c5 100644 --- a/plugins/measure/public/package.json +++ b/plugins/measure/public/package.json @@ -8,5 +8,7 @@ }, "author": "", "license": "ISC", - "dependencies": {} + "dependencies": { + "leaflet-measure-ex": "^3.0.4" + } } diff --git a/requirements.txt b/requirements.txt index 923ddd50..b81604fc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,7 @@ djangorestframework-jwt==1.9.0 drf-nested-routers==0.11.1 funcsigs==1.0.2 futures==3.0.5 +geojson==2.3.0 gunicorn==19.7.1 itypes==1.1.0 kombu==4.1.0 diff --git a/webodm/settings.py b/webodm/settings.py index 4682fea9..7adfbe7f 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -249,6 +249,7 @@ CORS_ORIGIN_ALLOW_ALL = True # File uploads MEDIA_ROOT = os.path.join(BASE_DIR, 'app', 'media') +MEDIA_TMP = os.path.join(MEDIA_ROOT, 'tmp') # Store flash messages in cookies MESSAGE_STORAGE = 'django.contrib.messages.storage.cookie.CookieStorage' diff --git a/worker/tasks.py b/worker/tasks.py index 672993e7..c5603309 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -8,6 +8,7 @@ from django.db.models import Q from app.models import Project from app.models import Task +from app.plugins.grass_engine import grass from nodeodm import status_codes from nodeodm.models import ProcessingNode from webodm import settings @@ -77,3 +78,9 @@ def process_pending_tasks(): for task in tasks: process_task.delay(task.id) + + +@app.task +def execute_grass_script(script, serialized_context = {}): + ctx = grass.create_context(serialized_context) + return ctx.execute(script)