From b6c3a004c52d5161885c5f4a637fe27236a1fbd4 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 1 Apr 2019 16:49:56 -0400 Subject: [PATCH] Contours preview working --- app/plugins/grass_engine.py | 4 + app/static/app/js/components/Map.jsx | 9 +- app/tests/test_worker.py | 3 + plugins/contours/api.py | 58 +++++++++--- plugins/contours/calc_contours.grass | 4 +- plugins/contours/plugin.py | 2 + plugins/contours/public/Contours.jsx | 7 +- plugins/contours/public/ContoursPanel.jsx | 104 +++++++++++++++++++--- plugins/contours/public/main.js | 2 +- worker/celery.py | 8 ++ worker/tasks.py | 23 +++++ 11 files changed, 189 insertions(+), 35 deletions(-) diff --git a/app/plugins/grass_engine.py b/app/plugins/grass_engine.py index 94aa080d..8fa979b5 100644 --- a/app/plugins/grass_engine.py +++ b/app/plugins/grass_engine.py @@ -125,4 +125,8 @@ class GrassContext: class GrassEngineException(Exception): pass +def cleanup_grass_context(serialized_context): + ctx = grass.create_context(serialized_context) + ctx.cleanup() + grass = GrassEngine() \ No newline at end of file diff --git a/app/static/app/js/components/Map.jsx b/app/static/app/js/components/Map.jsx index d13a4b8b..8be4d394 100644 --- a/app/static/app/js/components/Map.jsx +++ b/app/static/app/js/components/Map.jsx @@ -120,8 +120,12 @@ class Map extends React.Component { // For some reason, getLatLng is not defined for tileLayer? // We need this function if other code calls layer.openPopup() + let self = this; layer.getLatLng = function(){ - return this.options.bounds.getCenter(); + let latlng = self.lastClickedLatLng ? + self.lastClickedLatLng : + this.options.bounds.getCenter(); + return latlng; }; var popup = L.DomUtil.create('div', 'infoWindow'); @@ -270,6 +274,7 @@ https://a.tile.openstreetmap.org/{z}/{x}/{y}.png // Find first tile layer at the selected coordinates for (let layer of this.imageryLayers){ if (layer._map && layer.options.bounds.contains(e.latlng)){ + this.lastClickedLatLng = this.map.mouseEventToLatLng(e.originalEvent); this.updatePopupFor(layer); layer.openPopup(); break; @@ -279,6 +284,8 @@ https://a.tile.openstreetmap.org/{z}/{x}/{y}.png // Load task assets links in popup if (e.popup && e.popup._source && e.popup._content){ const infoWindow = e.popup._content; + if (typeof infoWindow === 'string') return; + const $assetLinks = $("ul.asset-links", infoWindow); if ($assetLinks.length > 0 && $assetLinks.hasClass('loading')){ diff --git a/app/tests/test_worker.py b/app/tests/test_worker.py index 11a3e666..18ee1bbe 100644 --- a/app/tests/test_worker.py +++ b/app/tests/test_worker.py @@ -55,3 +55,6 @@ class TestWorker(BootTestCase): self.assertFalse(Project.objects.filter(pk=project.id).exists()) pnserver.terminate() + + # TODO: check tmp directory cleanup + diff --git a/plugins/contours/api.py b/plugins/contours/api.py index 82eefd7b..b542e205 100644 --- a/plugins/contours/api.py +++ b/plugins/contours/api.py @@ -1,10 +1,14 @@ +import mimetypes import os +from django.http import FileResponse +from django.http import HttpResponse +from wsgiref.util import FileWrapper from rest_framework import status from rest_framework.response import Response from app.plugins.views import TaskView from worker.tasks import execute_grass_script -from app.plugins.grass_engine import grass, GrassEngineException +from app.plugins.grass_engine import grass, GrassEngineException, cleanup_grass_context from worker.celery import app as celery class TaskContoursGenerate(TaskView): @@ -52,21 +56,47 @@ class TaskContoursGenerate(TaskView): class TaskContoursCheck(TaskView): def get(self, request, pk=None, celery_task_id=None): - task = self.get_and_check_task(request, pk) + res = celery.AsyncResult(celery_task_id) + if not res.ready(): + return Response({'ready': False}, status=status.HTTP_200_OK) + else: + result = res.get() + if result.get('error', None) is not None: + cleanup_grass_context(result['context']) + return Response({'ready': True, 'error': result['error']}) - # res = celery.AsyncResult(celery_task_id) - # res.wait() - # print(res.get()) + contours_file = result.get('output') + if not contours_file: + cleanup_grass_context(result['context']) + return Response({'ready': True, 'error': 'No contours file was generated. This could be a bug.'}) - #while not res.ready(): + request.session['contours_' + celery_task_id] = contours_file + return Response({'ready': True}) - #if isinstance(output, dict) and 'error' in output: raise GrassEngineException(output['error']) - # if isinstance(output, dict) and 'error' in output: raise GrassEngineException(output['error']) - # - # cols = output.split(':') - # if len(cols) == 7: - # return Response({'volume': str(abs(float(cols[6])))}, status=status.HTTP_200_OK) - # else: - # raise GrassEngineException(output) +class TaskContoursDownload(TaskView): + def get(self, request, pk=None, celery_task_id=None): + contours_file = request.session.get('contours_' + celery_task_id, None) + if contours_file is not None: + filename = os.path.basename(contours_file) + filesize = os.stat(contours_file).st_size + + f = open(contours_file, "rb") + + # More than 100mb, normal http response, otherwise stream + # Django docs say to avoid streaming when possible + stream = filesize > 1e8 + if stream: + response = FileResponse(f) + else: + response = HttpResponse(FileWrapper(f), + content_type=(mimetypes.guess_type(filename)[0] or "application/zip")) + + response['Content-Type'] = mimetypes.guess_type(filename)[0] or "application/zip" + response['Content-Disposition'] = "inline; filename={}".format(filename) + response['Content-Length'] = filesize + + return response + else: + return Response({'error': 'Invalid contours download id'}) diff --git a/plugins/contours/calc_contours.grass b/plugins/contours/calc_contours.grass index 21c0a655..e674aea5 100755 --- a/plugins/contours/calc_contours.grass +++ b/plugins/contours/calc_contours.grass @@ -19,8 +19,8 @@ elif [ "${format}" = "ESRI Shapefile" ]; then ext="shp" fi -gdal_contour -i ${interval} -f GPKG "${dem_file}" contours.gpkg > /dev/null -ogr2ogr -simplify ${simplify} -t_srs EPSG:${epsg} -overwrite -f "${format}" output.$$ext contours.gpkg > /dev/null +gdal_contour -a elevation -i ${interval} -f GPKG "${dem_file}" contours.gpkg > /dev/null +ogr2ogr -dialect SQLite -where "ST_Length(geom) > 4" -simplify ${simplify} -t_srs EPSG:${epsg} -overwrite -f "${format}" output.$$ext contours.gpkg > /dev/null if [ -e "output.$$ext" ]; then # ESRI ShapeFile extra steps to compress into a zip archive diff --git a/plugins/contours/plugin.py b/plugins/contours/plugin.py index 7d839d8e..fa908c35 100644 --- a/plugins/contours/plugin.py +++ b/plugins/contours/plugin.py @@ -2,6 +2,7 @@ from app.plugins import PluginBase from app.plugins import MountPoint from .api import TaskContoursGenerate from .api import TaskContoursCheck +from .api import TaskContoursDownload class Plugin(PluginBase): @@ -15,4 +16,5 @@ class Plugin(PluginBase): return [ MountPoint('task/(?P[^/.]+)/contours/generate', TaskContoursGenerate.as_view()), MountPoint('task/(?P[^/.]+)/contours/check/(?P.+)', TaskContoursCheck.as_view()), + MountPoint('task/(?P[^/.]+)/contours/download/(?P.+)', TaskContoursDownload.as_view()), ] \ No newline at end of file diff --git a/plugins/contours/public/Contours.jsx b/plugins/contours/public/Contours.jsx index 8e88e3f7..b0094751 100644 --- a/plugins/contours/public/Contours.jsx +++ b/plugins/contours/public/Contours.jsx @@ -7,7 +7,8 @@ import ContoursPanel from './ContoursPanel'; class ContoursButton extends React.Component { static propTypes = { - tasks: PropTypes.object.isRequired + tasks: PropTypes.object.isRequired, + map: PropTypes.object.isRequired } constructor(props){ @@ -33,7 +34,7 @@ class ContoursButton extends React.Component { - + ); } } @@ -46,7 +47,7 @@ export default L.Control.extend({ onAdd: function (map) { var container = L.DomUtil.create('div', 'leaflet-control-contours leaflet-bar leaflet-control'); L.DomEvent.disableClickPropagation(container); - ReactDOM.render(, container); + ReactDOM.render(, container); return container; } diff --git a/plugins/contours/public/ContoursPanel.jsx b/plugins/contours/public/ContoursPanel.jsx index 810a4e6c..a9fbafcc 100644 --- a/plugins/contours/public/ContoursPanel.jsx +++ b/plugins/contours/public/ContoursPanel.jsx @@ -11,7 +11,8 @@ export default class ContoursPanel extends React.Component { static propTypes = { onClose: PropTypes.func.isRequired, tasks: PropTypes.object.isRequired, - isShowed: PropTypes.bool.isRequired + isShowed: PropTypes.bool.isRequired, + map: PropTypes.object.isRequired } constructor(props){ @@ -65,6 +66,10 @@ export default class ContoursPanel extends React.Component { this.loadingReq.abort(); this.loadingReq = null; } + if (this.generateReq){ + this.generateReq.abort(); + this.generateReq = null; + } } handleSelectInterval = e => { @@ -80,7 +85,7 @@ export default class ContoursPanel extends React.Component { } handleSelectEpsg = e => { - this.setState({Epsg: e.target.value}); + this.setState({epsg: e.target.value}); } handleChangeCustomEpsg = e => { @@ -96,29 +101,100 @@ export default class ContoursPanel extends React.Component { }; } + waitForCompletion = (taskId, celery_task_id, cb) => { + let errorCount = 0; + + const check = () => { + $.ajax({ + type: 'GET', + url: `/api/plugins/contours/task/${taskId}/contours/check/${celery_task_id}` + }).done(result => { + if (result.error){ + cb(result.error); + }else if (result.ready){ + cb(); + }else{ + // Retry + setTimeout(() => check(), 2000); + } + }).fail(error => { + console.warn(error); + if (errorCount++ < 10) setTimeout(() => check(), 2000); + else cb(JSON.stringify(error)); + }); + }; + + check(); + } + + addGeoJSONFromURL = (url, cb) => { + const { map } = this.props; + + $.getJSON(url) + .done((geojson) => { + try{ + if (this.previewLayer){ + map.removeLayer(this.previewLayer); + this.previewLayer = null; + } + + this.previewLayer = L.geoJSON(geojson, { + onEachFeature: (feature, layer) => { + if (feature.properties && feature.properties.elevation !== undefined) { + layer.bindPopup(`Elevation: ${feature.properties.elevation} meters`); + } + }, + style: feature => { + // TODO: different colors for different elevations? + return {color: "yellow"}; + } + }); + this.previewLayer.addTo(map); + + cb(); + }catch(e){ + cb(e.message); + } + }) + .fail(cb); + } + handleShowPreview = () => { this.setState({previewLoading: true}); const data = this.getFormValues(); - data.interval = 1; - data.epsg = 3857; + data.epsg = 4326; data.format = "GeoJSON"; data.simplify = 0.05; + const taskId = this.state.task.id; - $.ajax({ + this.generateReq = $.ajax({ type: 'POST', - url: `/api/plugins/contours/task/${this.state.task.id}/contours/generate`, + url: `/api/plugins/contours/task/${taskId}/contours/generate`, data: data }).done(result => { if (result.celery_task_id){ - console.log(result); - }else if (result.error){ - this.setState({error: result.error}); - }else{ - this.setState({error: "Invalid response: " + result}); - } + this.waitForCompletion(taskId, result.celery_task_id, error => { + if (error) this.setState({previewLoading: false, error}); + else{ + const fileUrl = `/api/plugins/contours/task/${taskId}/contours/download/${result.celery_task_id}`; - this.setState({previewLoading: false}); + // Preview + this.addGeoJSONFromURL(fileUrl, e => { + if (e) this.setState({error: JSON.stringify(e)}); + this.setState({previewLoading: false}); + }); + + // Download + // location.href = ; + // this.setState({previewLoading: false}); + } + }); + }else if (result.error){ + this.setState({previewLoading: false, error: result.error}); + }else{ + this.setState({previewLoading: false, error: "Invalid response: " + result}); + } }).fail(error => { this.setState({previewLoading: false, error: JSON.stringify(error)}); }); @@ -167,7 +243,7 @@ export default class ContoursPanel extends React.Component {
- +