Contours preview working

app/plugins/grass_engine.py

@@ -125,4 +125,8 @@ class GrassContext:
 class GrassEngineException(Exception):
     pass
 
+def cleanup_grass_context(serialized_context):
+    ctx = grass.create_context(serialized_context)
+    ctx.cleanup()
+
 grass = GrassEngine()

app/static/app/js/components/Map.jsx

@@ -120,8 +120,12 @@ class Map extends React.Component {
 
             // For some reason, getLatLng is not defined for tileLayer?
             // We need this function if other code calls layer.openPopup()
+            let self = this;
             layer.getLatLng = function(){
-              return this.options.bounds.getCenter();
+              let latlng = self.lastClickedLatLng ? 
+                            self.lastClickedLatLng : 
+                            this.options.bounds.getCenter();
+              return latlng;
             };
 
             var popup = L.DomUtil.create('div', 'infoWindow');
@@ -270,6 +274,7 @@ https://a.tile.openstreetmap.org/{z}/{x}/{y}.png
           // Find first tile layer at the selected coordinates 
           for (let layer of this.imageryLayers){
             if (layer._map && layer.options.bounds.contains(e.latlng)){
+              this.lastClickedLatLng = this.map.mouseEventToLatLng(e.originalEvent);
               this.updatePopupFor(layer);
               layer.openPopup();
               break;
@@ -279,6 +284,8 @@ https://a.tile.openstreetmap.org/{z}/{x}/{y}.png
             // Load task assets links in popup
             if (e.popup && e.popup._source && e.popup._content){
                 const infoWindow = e.popup._content;
+                if (typeof infoWindow === 'string') return;
+
                 const $assetLinks = $("ul.asset-links", infoWindow);
                 
                 if ($assetLinks.length > 0 && $assetLinks.hasClass('loading')){

app/tests/test_worker.py

@@ -55,3 +55,6 @@ class TestWorker(BootTestCase):
         self.assertFalse(Project.objects.filter(pk=project.id).exists())
 
         pnserver.terminate()
+
+        # TODO: check tmp directory cleanup
+

plugins/contours/api.py

@@ -1,10 +1,14 @@
+import mimetypes
 import os
 
+from django.http import FileResponse
+from django.http import HttpResponse
+from wsgiref.util import FileWrapper
 from rest_framework import status
 from rest_framework.response import Response
 from app.plugins.views import TaskView
 from worker.tasks import execute_grass_script
-from app.plugins.grass_engine import grass, GrassEngineException
+from app.plugins.grass_engine import grass, GrassEngineException, cleanup_grass_context
 from worker.celery import app as celery
 
 class TaskContoursGenerate(TaskView):
@@ -52,21 +56,47 @@ class TaskContoursGenerate(TaskView):
 
 class TaskContoursCheck(TaskView):
     def get(self, request, pk=None, celery_task_id=None):
-        task = self.get_and_check_task(request, pk)
+        res = celery.AsyncResult(celery_task_id)
+        if not res.ready():
+            return Response({'ready': False}, status=status.HTTP_200_OK)
+        else:
+            result = res.get()
+            if result.get('error', None) is not None:
+                cleanup_grass_context(result['context'])
+                return Response({'ready': True, 'error': result['error']})
 
-        # res = celery.AsyncResult(celery_task_id)
+            contours_file = result.get('output')
-        # res.wait()
+            if not contours_file:
-        # print(res.get())
+                cleanup_grass_context(result['context'])
+                return Response({'ready': True, 'error': 'No contours file was generated. This could be a bug.'})
 
-        #while not res.ready():
+            request.session['contours_' + celery_task_id] = contours_file
+            return Response({'ready': True})
 
-        #if isinstance(output, dict) and 'error' in output: raise GrassEngineException(output['error'])
 
-        # if isinstance(output, dict) and 'error' in output: raise GrassEngineException(output['error'])
+class TaskContoursDownload(TaskView):
-        #
+    def get(self, request, pk=None, celery_task_id=None):
-        # cols = output.split(':')
+        contours_file = request.session.get('contours_' + celery_task_id, None)
-        # if len(cols) == 7:
-        #     return Response({'volume': str(abs(float(cols[6])))}, status=status.HTTP_200_OK)
-        # else:
-        #     raise GrassEngineException(output)
 
+        if contours_file is not None:
+            filename = os.path.basename(contours_file)
+            filesize = os.stat(contours_file).st_size
+
+            f = open(contours_file, "rb")
+
+            # More than 100mb, normal http response, otherwise stream
+            # Django docs say to avoid streaming when possible
+            stream = filesize > 1e8
+            if stream:
+                response = FileResponse(f)
+            else:
+                response = HttpResponse(FileWrapper(f),
+                                        content_type=(mimetypes.guess_type(filename)[0] or "application/zip"))
+
+            response['Content-Type'] = mimetypes.guess_type(filename)[0] or "application/zip"
+            response['Content-Disposition'] = "inline; filename={}".format(filename)
+            response['Content-Length'] = filesize
+
+            return response
+        else:
+            return Response({'error': 'Invalid contours download id'})

plugins/contours/calc_contours.grass

@@ -19,8 +19,8 @@ elif [ "${format}" = "ESRI Shapefile" ]; then
     ext="shp"
 fi
 
-gdal_contour -i ${interval} -f GPKG "${dem_file}" contours.gpkg > /dev/null
+gdal_contour -a elevation -i ${interval} -f GPKG "${dem_file}" contours.gpkg > /dev/null
-ogr2ogr -simplify ${simplify} -t_srs EPSG:${epsg} -overwrite -f "${format}" output.$$ext contours.gpkg > /dev/null
+ogr2ogr -dialect SQLite -where "ST_Length(geom) > 4" -simplify ${simplify} -t_srs EPSG:${epsg} -overwrite -f "${format}" output.$$ext contours.gpkg > /dev/null
 
 if [ -e "output.$$ext" ]; then
     # ESRI ShapeFile extra steps to compress into a zip archive

plugins/contours/plugin.py

@@ -2,6 +2,7 @@ from app.plugins import PluginBase
 from app.plugins import MountPoint
 from .api import TaskContoursGenerate
 from .api import TaskContoursCheck
+from .api import TaskContoursDownload
 
 
 class Plugin(PluginBase):
@@ -15,4 +16,5 @@ class Plugin(PluginBase):
         return [
             MountPoint('task/(?P<pk>[^/.]+)/contours/generate', TaskContoursGenerate.as_view()),
             MountPoint('task/(?P<pk>[^/.]+)/contours/check/(?P<celery_task_id>.+)', TaskContoursCheck.as_view()),
+            MountPoint('task/(?P<pk>[^/.]+)/contours/download/(?P<celery_task_id>.+)', TaskContoursDownload.as_view()),
         ]

plugins/contours/public/Contours.jsx

@@ -7,7 +7,8 @@ import ContoursPanel from './ContoursPanel';
 
 class ContoursButton extends React.Component {
   static propTypes = {
-    tasks: PropTypes.object.isRequired
+    tasks: PropTypes.object.isRequired,
+    map: PropTypes.object.isRequired
   }
 
   constructor(props){
@@ -33,7 +34,7 @@ class ContoursButton extends React.Component {
         <a href="javascript:void(0);" 
             onClick={this.handleOpen} 
             className="leaflet-control-contours-button leaflet-bar-part theme-secondary"></a>
-        <ContoursPanel isShowed={showPanel} tasks={this.props.tasks} onClose={this.handleClose} />
+        <ContoursPanel map={this.props.map} isShowed={showPanel} tasks={this.props.tasks} onClose={this.handleClose} />
     </div>);
   }
 }
@@ -46,7 +47,7 @@ export default L.Control.extend({
     onAdd: function (map) {
         var container = L.DomUtil.create('div', 'leaflet-control-contours leaflet-bar leaflet-control');
         L.DomEvent.disableClickPropagation(container);
-        ReactDOM.render(<ContoursButton tasks={this.options.tasks} />, container);
+        ReactDOM.render(<ContoursButton map={this.options.map} tasks={this.options.tasks} />, container);
 
         return container;
     }

plugins/contours/public/ContoursPanel.jsx

@@ -11,7 +11,8 @@ export default class ContoursPanel extends React.Component {
   static propTypes = {
     onClose: PropTypes.func.isRequired,
     tasks: PropTypes.object.isRequired,
-    isShowed: PropTypes.bool.isRequired
+    isShowed: PropTypes.bool.isRequired,
+    map: PropTypes.object.isRequired
   }
 
   constructor(props){
@@ -65,6 +66,10 @@ export default class ContoursPanel extends React.Component {
       this.loadingReq.abort();
       this.loadingReq = null;
     }
+    if (this.generateReq){
+      this.generateReq.abort();
+      this.generateReq = null;
+    }
   }
 
   handleSelectInterval = e => {
@@ -80,7 +85,7 @@ export default class ContoursPanel extends React.Component {
   }
 
   handleSelectEpsg = e => {
-    this.setState({Epsg: e.target.value});
+    this.setState({epsg: e.target.value});
   }
 
   handleChangeCustomEpsg = e => {
@@ -96,29 +101,100 @@ export default class ContoursPanel extends React.Component {
     };
   }
 
+  waitForCompletion = (taskId, celery_task_id, cb) => {
+    let errorCount = 0;
+
+    const check = () => {
+      $.ajax({
+          type: 'GET',
+          url: `/api/plugins/contours/task/${taskId}/contours/check/${celery_task_id}`
+      }).done(result => {
+          if (result.error){
+            cb(result.error);
+          }else if (result.ready){
+            cb();
+          }else{
+            // Retry
+            setTimeout(() => check(), 2000);
+          }
+      }).fail(error => {
+          console.warn(error);
+          if (errorCount++ < 10) setTimeout(() => check(), 2000);
+          else cb(JSON.stringify(error));
+      });
+    };
+
+    check();
+  }
+
+  addGeoJSONFromURL = (url, cb) => {
+    const { map } = this.props;
+
+    $.getJSON(url)
+     .done((geojson) => {
+      try{
+        if (this.previewLayer){
+          map.removeLayer(this.previewLayer);
+          this.previewLayer = null;
+        }
+
+        this.previewLayer = L.geoJSON(geojson, {
+          onEachFeature: (feature, layer) => {
+              if (feature.properties && feature.properties.elevation !== undefined) {
+                  layer.bindPopup(`<b>Elevation:</b> ${feature.properties.elevation} meters`);
+              }
+          },
+          style: feature => {
+              // TODO: different colors for different elevations?
+              return {color: "yellow"};
+          }
+        });
+        this.previewLayer.addTo(map);
+
+        cb();
+      }catch(e){
+        cb(e.message);
+      }
+     })
+     .fail(cb);
+  }
+
   handleShowPreview = () => {
     this.setState({previewLoading: true});
 
     const data = this.getFormValues();
-    data.interval = 1;
+    data.epsg = 4326;
-    data.epsg = 3857;
     data.format = "GeoJSON";
     data.simplify = 0.05;
+    const taskId = this.state.task.id;
 
-    $.ajax({
+    this.generateReq = $.ajax({
         type: 'POST',
-        url: `/api/plugins/contours/task/${this.state.task.id}/contours/generate`,
+        url: `/api/plugins/contours/task/${taskId}/contours/generate`,
         data: data
     }).done(result => {
         if (result.celery_task_id){
-          console.log(result);
+          this.waitForCompletion(taskId, result.celery_task_id, error => {
-        }else if (result.error){
+            if (error) this.setState({previewLoading: false, error});
-            this.setState({error: result.error});
+            else{
-        }else{
+              const fileUrl = `/api/plugins/contours/task/${taskId}/contours/download/${result.celery_task_id}`;
-            this.setState({error: "Invalid response: " + result});
-        }
 
-        this.setState({previewLoading: false});
+              // Preview
+              this.addGeoJSONFromURL(fileUrl, e => {
+                if (e) this.setState({error: JSON.stringify(e)});
+                this.setState({previewLoading: false});
+              });
+
+              // Download
+              // location.href = ;
+              // this.setState({previewLoading: false});
+            }
+          });
+        }else if (result.error){
+            this.setState({previewLoading: false, error: result.error});
+        }else{
+            this.setState({previewLoading: false, error: "Invalid response: " + result});
+        }
     }).fail(error => {
         this.setState({previewLoading: false, error: JSON.stringify(error)});
     });
@@ -167,7 +243,7 @@ export default class ContoursPanel extends React.Component {
         </div>
 
         <div className="row form-group form-inline">
-          <label className="col-sm-3 control-label">Epsg:</label>
+          <label className="col-sm-3 control-label">Projection:</label>
           <div className="col-sm-9 ">
             <select className="form-control" value={epsg} onChange={this.handleSelectEpsg}>
               <option value="4326">WGS84 (EPSG:4326)</option>

plugins/contours/public/main.js

@@ -9,6 +9,6 @@ PluginsAPI.Map.willAddControls([
 
 	// TODO: add support for map view where multiple tasks are available?
 	if (tasks.length === 1){
-		args.map.addControl(new Contours({tasks: tasks}));
+		args.map.addControl(new Contours({map: args.map, tasks: tasks}));
 	}
 });

worker/celery.py

@@ -23,6 +23,14 @@ app.conf.beat_schedule = {
         	'retry': False
         }
     },
+    'cleanup-tmp-directory': {
+        'task': 'worker.tasks.cleanup_tmp_directory',
+        'schedule': 3600,
+        'options': {
+            'expires': 1799,
+            'retry': False
+        }
+    },
     'process-pending-tasks': {
         'task': 'worker.tasks.process_pending_tasks',
         'schedule': 5,

worker/tasks.py

@@ -1,5 +1,8 @@
+import os
+import shutil
 import traceback
 
+import time
 from celery.utils.log import get_task_logger
 from django.core.exceptions import ObjectDoesNotExist
 from django.db.models import Count
@@ -35,6 +38,26 @@ def cleanup_projects():
         logger.info("Deleted {} projects".format(count_dict['app.Project']))
 
 
+@app.task
+def cleanup_tmp_directory():
+    # Delete files and folder in the tmp directory that are
+    # older than 24 hours
+    tmpdir = settings.MEDIA_TMP
+    time_limit = 60 * 60 * 24
+
+    for f in os.listdir(tmpdir):
+        now = time.time()
+        filepath = os.path.join(tmpdir, f)
+        modified = os.stat(filepath).st_mtime
+        if modified < now - time_limit:
+            if os.path.isfile(filepath):
+                os.remove(filepath)
+            else:
+                shutil.rmtree(filepath, ignore_errors=True)
+
+            logger.info('Cleaned up: %s (%s)' % (f, modified))
+
+
 @app.task
 def process_task(taskId):
     try: