From 1b327fb56edfcfb1d2d0de21722ff4047a1ccb4b Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 26 Sep 2023 12:04:04 -0400 Subject: [PATCH 1/4] GDAL based contours --- coreplugins/contours/api.py | 84 ++++++++++++++++++++++++++++--------- worker/tasks.py | 2 - 2 files changed, 65 insertions(+), 21 deletions(-) diff --git a/coreplugins/contours/api.py b/coreplugins/contours/api.py index 7d07b14e..d6c88eae 100644 --- a/coreplugins/contours/api.py +++ b/coreplugins/contours/api.py @@ -3,10 +3,68 @@ import os from rest_framework import status from rest_framework.response import Response from app.plugins.views import TaskView, CheckTask, GetTaskResult -from worker.tasks import execute_grass_script -from app.plugins.grass_engine import grass, GrassEngineException, cleanup_grass_context +from app.plugins.worker import run_function_async from django.utils.translation import gettext_lazy as _ +class ContoursException(Exception): + pass + +def calc_contours(dem, epsg, interval, output_format, simplify): + import os + import subprocess + import tempfile + import shutil + from webodm import settings + + ext = "" + if output_format == "GeoJSON": + ext = "json" + elif output_format == "GPKG": + ext = "gpkg" + elif output_format == "DXF": + ext = "dxf" + elif output_format == "ESRI Shapefile": + ext = "shp" + MIN_CONTOUR_LENGTH = 10 + + tmpdir = os.path.join(settings.MEDIA_TMP, os.path.basename(tempfile.mkdtemp('_contours', dir=settings.MEDIA_TMP))) + gdal_contour_bin = shutil.which("gdal_contour") + ogr2ogr_bin = shutil.which("ogr2ogr") + + if gdal_contour_bin is None: + return {'error': 'Cannot find gdal_contour'} + if ogr2ogr_bin is None: + return {'error': 'Cannot find ogr2ogr'} + + contours_file = f"contours.gpkg" + p = subprocess.Popen([gdal_contour_bin, "-q", "-a", "level", "-3d", "-f", "GPKG", "-i", str(interval), dem, contours_file], cwd=tmpdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + + out = out.decode('utf-8').strip() + err = err.decode('utf-8').strip() + success = p.returncode == 0 + + if not success: + return {'error', f'Error calling gdal_contour: {str(err)}'} + + outfile = os.path.join(tmpdir, f"output.{ext}") + p = subprocess.Popen([ogr2ogr_bin, outfile, contours_file, "-simplify", str(simplify), "-f", output_format, "-t_srs", f"EPSG:{epsg}", + "-dialect", "sqlite", "-sql", f"SELECT * FROM contour WHERE ST_Length(GEOM) >= {MIN_CONTOUR_LENGTH}"], cwd=tmpdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + + out = out.decode('utf-8').strip() + err = err.decode('utf-8').strip() + success = p.returncode == 0 + + if not success: + return {'error', f'Error calling ogr2ogr: {str(err)}'} + + if not os.path.isfile(outfile): + return {'error': f'Cannot find output file: {outfile}'} + + return {'file': outfile} + + class TaskContoursGenerate(TaskView): def post(self, request, pk=None): task = self.get_and_check_task(request, pk) @@ -23,36 +81,24 @@ class TaskContoursGenerate(TaskView): elif layer == 'DTM': dem = os.path.abspath(task.get_asset_download_path("dtm.tif")) else: - raise GrassEngineException('{} is not a valid layer.'.format(layer)) + raise ContoursException('{} is not a valid layer.'.format(layer)) - context = grass.create_context({'auto_cleanup' : False}) epsg = int(request.data.get('epsg', '3857')) interval = float(request.data.get('interval', 1)) format = request.data.get('format', 'GPKG') supported_formats = ['GPKG', 'ESRI Shapefile', 'DXF', 'GeoJSON'] if not format in supported_formats: - raise GrassEngineException("Invalid format {} (must be one of: {})".format(format, ",".join(supported_formats))) + raise ContoursException("Invalid format {} (must be one of: {})".format(format, ",".join(supported_formats))) simplify = float(request.data.get('simplify', 0.01)) - context.add_param('dem_file', dem) - context.add_param('interval', interval) - context.add_param('format', format) - context.add_param('simplify', simplify) - context.add_param('epsg', epsg) - context.set_location(dem) - - celery_task_id = execute_grass_script.delay(os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "calc_contours.py" - ), context.serialize(), 'file').task_id - + celery_task_id = run_function_async(calc_contours, dem, epsg, interval, format, simplify).task_id return Response({'celery_task_id': celery_task_id}, status=status.HTTP_200_OK) - except GrassEngineException as e: + except ContoursException as e: return Response({'error': str(e)}, status=status.HTTP_200_OK) class TaskContoursCheck(CheckTask): def on_error(self, result): - cleanup_grass_context(result['context']) + pass def error_check(self, result): contours_file = result.get('file') diff --git a/worker/tasks.py b/worker/tasks.py index 12ff4f1e..392582b7 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -175,7 +175,6 @@ def execute_grass_script(script, serialized_context = {}, out_key='output'): logger.error(str(e)) return {'error': str(e), 'context': ctx.serialize()} - @app.task(bind=True) def export_raster(self, input, **opts): try: @@ -238,4 +237,3 @@ def check_quotas(): break else: p.clear_quota_deadline() - From c8c0f518052e587c3d7303a7d6c250161271b418 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 26 Sep 2023 12:08:11 -0400 Subject: [PATCH 2/4] Assign layer name --- coreplugins/contours/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coreplugins/contours/api.py b/coreplugins/contours/api.py index d6c88eae..2c4a0d1e 100644 --- a/coreplugins/contours/api.py +++ b/coreplugins/contours/api.py @@ -48,7 +48,7 @@ def calc_contours(dem, epsg, interval, output_format, simplify): return {'error', f'Error calling gdal_contour: {str(err)}'} outfile = os.path.join(tmpdir, f"output.{ext}") - p = subprocess.Popen([ogr2ogr_bin, outfile, contours_file, "-simplify", str(simplify), "-f", output_format, "-t_srs", f"EPSG:{epsg}", + p = subprocess.Popen([ogr2ogr_bin, outfile, contours_file, "-simplify", str(simplify), "-f", output_format, "-t_srs", f"EPSG:{epsg}", "-nln", "contours", "-dialect", "sqlite", "-sql", f"SELECT * FROM contour WHERE ST_Length(GEOM) >= {MIN_CONTOUR_LENGTH}"], cwd=tmpdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() From 0e7d9ee6f2d38c9471e8ad03de03c9b8d492bbac Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 26 Sep 2023 12:20:27 -0400 Subject: [PATCH 3/4] Shapefile support --- coreplugins/contours/api.py | 12 ++++ coreplugins/contours/calc_contours.py | 93 --------------------------- 2 files changed, 12 insertions(+), 93 deletions(-) delete mode 100755 coreplugins/contours/calc_contours.py diff --git a/coreplugins/contours/api.py b/coreplugins/contours/api.py index 2c4a0d1e..603bc217 100644 --- a/coreplugins/contours/api.py +++ b/coreplugins/contours/api.py @@ -14,6 +14,7 @@ def calc_contours(dem, epsg, interval, output_format, simplify): import subprocess import tempfile import shutil + import glob from webodm import settings ext = "" @@ -61,6 +62,17 @@ def calc_contours(dem, epsg, interval, output_format, simplify): if not os.path.isfile(outfile): return {'error': f'Cannot find output file: {outfile}'} + + if output_format == "ESRI Shapefile": + ext="zip" + shp_dir = os.path.join(tmpdir, "contours") + os.makedirs(shp_dir) + contour_files = glob.glob(os.path.join(tmpdir, "output.*")) + for cf in contour_files: + shutil.move(cf, shp_dir) + + shutil.make_archive(os.path.join(tmpdir, 'output'), 'zip', shp_dir) + outfile = os.path.join(tmpdir, f"output.{ext}") return {'file': outfile} diff --git a/coreplugins/contours/calc_contours.py b/coreplugins/contours/calc_contours.py deleted file mode 100755 index 987ab5bb..00000000 --- a/coreplugins/contours/calc_contours.py +++ /dev/null @@ -1,93 +0,0 @@ -#%module -#% description: Calculate contours -#%end -#%option -#% key: dem_file -#% type: string -#% required: yes -#% multiple: no -#% description: GeoTIFF DEM containing the surface to calculate contours -#%end -#%option -#% key: interval -#% type: double -#% required: yes -#% multiple: no -#% description: Contours interval -#%end -#%option -#% key: format -#% type: string -#% required: yes -#% multiple: no -#% description: OGR output format -#%end -#%option -#% key: simplify -#% type: double -#% required: yes -#% multiple: no -#% description: OGR output format -#%end -#%option -#% key: epsg -#% type: string -#% required: yes -#% multiple: no -#% description: target EPSG code -#%end - -# output: If successful, prints the full path to the contours file. Otherwise it prints "error" - -import sys -import glob -import os -import shutil -from grass.pygrass.modules import Module -import grass.script as grass -import subprocess - -def main(): - ext = "" - if opts['format'] == "GeoJSON": - ext = "json" - elif opts['format'] == "GPKG": - ext = "gpkg" - elif opts['format'] == "DXF": - ext = "dxf" - elif opts['format'] == "ESRI Shapefile": - ext = "shp" - - MIN_CONTOUR_LENGTH = 5 - Module("r.external", input=opts['dem_file'], output="dem", overwrite=True) - Module("g.region", raster="dem") - Module("r.contour", input="dem", output="contours", step=opts["interval"], overwrite=True) - Module("v.generalize", input="contours", output="contours_smooth", method="douglas", threshold=opts["simplify"], overwrite=True) - Module("v.generalize", input="contours_smooth", output="contours_simplified", method="chaiken", threshold=1, overwrite=True) - Module("v.generalize", input="contours_simplified", output="contours_final", method="douglas", threshold=opts["simplify"], overwrite=True) - Module("v.edit", map="contours_final", tool="delete", threshold=[-1,0,-MIN_CONTOUR_LENGTH], query="length") - Module("v.out.ogr", input="contours_final", output="temp.gpkg", format="GPKG") - - subprocess.check_call(["ogr2ogr", "-t_srs", "EPSG:%s" % opts['epsg'], - '-overwrite', '-f', opts["format"], "output.%s" % ext, "temp.gpkg"], stdout=subprocess.DEVNULL) - - if os.path.isfile("output.%s" % ext): - if opts["format"] == "ESRI Shapefile": - ext="zip" - os.makedirs("contours") - contour_files = glob.glob("output.*") - for cf in contour_files: - shutil.move(cf, os.path.join("contours", os.path.basename(cf))) - - shutil.make_archive('output', 'zip', 'contours/') - - print(os.path.join(os.getcwd(), "output.%s" % ext)) - else: - print("error") - - return 0 - -if __name__ == "__main__": - opts, _ = grass.parser() - sys.exit(main()) - From a852dfb04e714e00918f3a32876e0ca11ffaab2c Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 26 Sep 2023 12:37:10 -0400 Subject: [PATCH 4/4] Bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 70411580..8c812941 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "WebODM", - "version": "2.1.2", + "version": "2.1.3", "description": "User-friendly, extendable application and API for processing aerial imagery.", "main": "index.js", "scripts": {