Export point clouds

Dockerfile

@@ -14,8 +14,8 @@ WORKDIR /webodm
 RUN apt-get -qq update && apt-get -qq install -y --no-install-recommends wget curl && \
     wget --no-check-certificate https://deb.nodesource.com/setup_12.x -O /tmp/node.sh && bash /tmp/node.sh && \
     apt-get -qq update && apt-get -qq install -y nodejs && \
-    # Install Python3, GDAL, nginx, letsencrypt, psql
+    # Install Python3, GDAL, PDAL, nginx, letsencrypt, psql
-    apt-get -qq update && apt-get -qq install -y --no-install-recommends python3 python3-pip python3-setuptools python3-wheel git g++ python3-dev python2.7-dev libpq-dev binutils libproj-dev gdal-bin libgdal-dev python3-gdal nginx certbot grass-core gettext-base cron postgresql-client-13 gettext tzdata && \
+    apt-get -qq update && apt-get -qq install -y --no-install-recommends python3 python3-pip python3-setuptools python3-wheel git g++ python3-dev python2.7-dev libpq-dev binutils libproj-dev gdal-bin pdal libgdal-dev python3-gdal nginx certbot grass-core gettext-base cron postgresql-client-13 gettext tzdata && \
     update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1 && update-alternatives --install /usr/bin/python python /usr/bin/python3.9 2 && \
     # Install pip reqs
     pip install -U pip && pip install -r requirements.txt "boto3==1.14.14" && \

app/api/tiler.py

@@ -20,14 +20,14 @@ from rio_tiler.io import COGReader
 from rio_tiler.errors import InvalidColorMapName
 import numpy as np
 from .custom_colormaps_helper import custom_colormaps
-from app.raster_utils import export_raster, extension_for_export_format, ZOOM_EXTRA_LEVELS
+from app.raster_utils import extension_for_export_format, ZOOM_EXTRA_LEVELS
 from .hsvblend import hsv_blend
 from .hillshade import LightSource
 from .formulas import lookup_formula, get_algorithm_list
 from .tasks import TaskNestedView
 from rest_framework import exceptions
 from rest_framework.response import Response
-from worker.tasks import export_raster
+from worker.tasks import export_raster, export_pointcloud
 from django.utils.translation import gettext as _
 
 
@@ -76,6 +76,9 @@ def get_extent(task, tile_type):
 def get_raster_path(task, tile_type):
     return task.get_asset_download_path(tile_type + ".tif")
 
+def get_pointcloud_path(task):
+    return task.get_asset_download_path("georeferenced_model.laz")
+
 
 class TileJson(TaskNestedView):
     def get(self, request, pk=None, project_pk=None, tile_type=""):
@@ -491,7 +494,9 @@ class Export(TaskNestedView):
 
         expr = None
 
-        if not export_format in ['gtiff', 'gtiff-rgb', 'jpg', 'png', 'kmz']:
+        if asset_type in ['orthophoto', 'dsm', 'dtm'] and not export_format in ['gtiff', 'gtiff-rgb', 'jpg', 'png', 'kmz']:
+            raise exceptions.ValidationError(_("Unsupported format: %(value)s") % {'value': export_format})
+        if asset_type == 'georeferenced_model' and not export_format in ['laz', 'las', 'ply', 'csv']:
             raise exceptions.ValidationError(_("Unsupported format: %(value)s") % {'value': export_format})
         
         if epsg is not None:
@@ -528,7 +533,10 @@ class Export(TaskNestedView):
             except:
                 raise exceptions.ValidationError(_("Invalid hillshade value: %(value)s") % {'value': hillshade})
         
-        url = get_raster_path(task, asset_type)
+        if asset_type == 'georeferenced_model':
+            url = get_pointcloud_path(task)
+        else:
+            url = get_raster_path(task, asset_type)
 
         if not os.path.isfile(url):
             raise exceptions.NotFound()
@@ -541,16 +549,25 @@ class Export(TaskNestedView):
                         extension
                     )
 
-        # Shortcut the process if no processing is required
+        if asset_type in ['orthophoto', 'dsm', 'dtm']:
-        if export_format == 'gtiff' and (epsg == task.epsg or epsg is None) and expr is None:
+            # Shortcut the process if no processing is required
-            return Response({'url': '/api/projects/{}/tasks/{}/download/{}.tif'.format(task.project.id, task.id, asset_type), 'filename': filename})
+            if export_format == 'gtiff' and (epsg == task.epsg or epsg is None) and expr is None:
-        else:
+                return Response({'url': '/api/projects/{}/tasks/{}/download/{}.tif'.format(task.project.id, task.id, asset_type), 'filename': filename})
-            celery_task_id = export_raster.delay(url, epsg=epsg, 
+            else:
-                                                    expression=expr, 
+                celery_task_id = export_raster.delay(url, epsg=epsg, 
-                                                    format=export_format, 
+                                                        expression=expr, 
-                                                    rescale=rescale, 
+                                                        format=export_format, 
-                                                    color_map=color_map,
+                                                        rescale=rescale, 
-                                                    hillshade=hillshade,
+                                                        color_map=color_map,
-                                                    asset_type=asset_type,
+                                                        hillshade=hillshade,
-                                                    name=task.name).task_id
+                                                        asset_type=asset_type,
-            return Response({'celery_task_id': celery_task_id, 'filename': filename})
+                                                        name=task.name).task_id
+                return Response({'celery_task_id': celery_task_id, 'filename': filename})
+        elif asset_type == 'georeferenced_model':
+            # Shortcut the process if no processing is required
+            if export_format == 'laz' and (epsg == task.epsg or epsg is None):
+                return Response({'url': '/api/projects/{}/tasks/{}/download/{}.laz'.format(task.project.id, task.id, asset_type), 'filename': filename})
+            else:
+                celery_task_id = export_pointcloud.delay(url, epsg=epsg, 
+                                                            format=export_format).task_id
+                return Response({'celery_task_id': celery_task_id, 'filename': filename})

app/api/urls.py

@@ -39,7 +39,7 @@ urlpatterns = [
     url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/(?P<tile_type>orthophoto|dsm|dtm)/metadata$', Metadata.as_view()),
     url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/(?P<tile_type>orthophoto|dsm|dtm)/tiles/(?P<z>[\d]+)/(?P<x>[\d]+)/(?P<y>[\d]+)\.png$', Tiles.as_view()),
     url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/(?P<tile_type>orthophoto|dsm|dtm)/tiles/(?P<z>[\d]+)/(?P<x>[\d]+)/(?P<y>[\d]+)@(?P<scale>[\d]+)x\.png$', Tiles.as_view()),
-    url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/(?P<asset_type>orthophoto|dsm|dtm)/export$', Export.as_view()),
+    url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/(?P<asset_type>orthophoto|dsm|dtm|georeferenced_model)/export$', Export.as_view()),
 
     url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/download/(?P<asset>.+)$', TaskDownloads.as_view()),
     url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/assets/(?P<unsafe_asset_path>.+)$', TaskAssets.as_view()),

app/pointcloud_utils.py

@@ -0,0 +1,23 @@
+import logging
+import os
+import subprocess
+from app.security import double_quote
+
+logger = logging.getLogger('app.logger')
+
+def export_pointcloud(input, output, **opts):
+    epsg = opts.get('epsg')
+    export_format = opts.get('format')
+
+    reprojection_args = []
+    extra_args = []
+
+    if epsg:
+        reprojection_args = ["reprojection",
+                            "--filters.reprojection.out_srs=%s" % double_quote("EPSG:" + str(epsg))]
+    
+    if export_format == "ply":
+        extra_args = ['--writers.ply.sized_types', 'false',
+                      '--writers.ply.storage_mode', 'little endian']
+
+    subprocess.check_output(["pdal", "translate", input, output] + reprojection_args + extra_args)

app/raster_utils.py

@@ -1,4 +1,3 @@
-# Export a raster index after applying a band expression
 import rasterio
 import re
 import logging
@@ -23,11 +22,8 @@ def extension_for_export_format(export_format):
     extensions = {
         'gtiff': 'tif',
         'gtiff-rgb': 'tif',
-        'jpg': 'jpg',
-        'png': 'png',
-        'kmz': 'kmz'
     }
-    return extensions.get(export_format, 'tif')
+    return extensions.get(export_format, export_format)
 
 def export_raster(input, output, **opts):
     epsg = opts.get('epsg')

app/static/app/js/classes/AssetDownloads.js

@@ -1,11 +1,12 @@
 import { _ } from './gettext';
 
 class AssetDownload{
-  constructor(label, asset, icon, exportFormats = null){
+  constructor(label, asset, icon, exportFormats = null, exportParams = {}){
     this.label = label;
     this.asset = asset;
     this.icon = icon;
     this.exportFormats = exportFormats;
+    this.exportParams = exportParams;
   }
 
   downloadUrl(project_id, task_id){
@@ -36,24 +37,24 @@ class AssetDownloadSeparator extends AssetDownload{
   }
 }
 
+const tiffExportFormats = ["gtiff", "gtiff-rgb", "jpg", "png", "kmz"];
+const elevationExportParams = {'hillshade': 6, "color_map": "viridis"};
+
 const api = {
   all: function() {
     return [
-      new AssetDownload(_("Orthophoto"),"orthophoto.tif","far fa-image", ["gtiff", "gtiff-rgb", "jpg", "png", "kmz"]),
+      new AssetDownload(_("Orthophoto"),"orthophoto.tif","far fa-image", tiffExportFormats),
       new AssetDownload(_("Orthophoto (MBTiles)"),"orthophoto.mbtiles","far fa-image"),
       new AssetDownload(_("Orthophoto (Tiles)"),"orthophoto_tiles.zip","fa fa-table"),
-      new AssetDownload(_("Terrain Model"),"dtm.tif","fa fa-chart-area"),
+      new AssetDownload(_("Terrain Model"),"dtm.tif","fa fa-chart-area", tiffExportFormats, elevationExportParams),
       new AssetDownload(_("Terrain Model (Tiles)"),"dtm_tiles.zip","fa fa-table"),
-      new AssetDownload(_("Surface Model"),"dsm.tif","fa fa-chart-area"),
+      new AssetDownload(_("Surface Model"),"dsm.tif","fa fa-chart-area", tiffExportFormats, elevationExportParams),
       new AssetDownload(_("Surface Model (Tiles)"),"dsm_tiles.zip","fa fa-table"),
-      new AssetDownload(_("Point Cloud (LAS)"),"georeferenced_model.las","fa fa-cube"),
+      new AssetDownload(_("Point Cloud"),"georeferenced_model.laz","fa fa-cube", ["laz", "las", "ply", "csv"]),
-      new AssetDownload(_("Point Cloud (LAZ)"),"georeferenced_model.laz","fa fa-cube"),
-      new AssetDownload(_("Point Cloud (PLY)"),"georeferenced_model.ply","fa fa-cube"),
-      new AssetDownload(_("Point Cloud (CSV)"),"georeferenced_model.csv","fa fa-cube"),
       new AssetDownload(_("Textured Model"),"textured_model.zip","fab fa-connectdevelop"),
       new AssetDownload(_("Camera Parameters"),"cameras.json","fa fa-camera"),
-      new AssetDownload(_("Camera Shots (GeoJSON)"),"shots.geojson","fa fa-camera"),
+      new AssetDownload(_("Camera Shots"),"shots.geojson","fa fa-camera"),
-      new AssetDownload(_("Ground Control Points (GeoJSON)"),"ground_control_points.geojson","far fa-dot-circle"),
+      new AssetDownload(_("Ground Control Points"),"ground_control_points.geojson","far fa-dot-circle"),
       new AssetDownload(_("Quality Report"),"report.pdf","far fa-file-pdf"),
       
       

app/static/app/js/components/AssetDownloadButtons.jsx

@@ -43,6 +43,7 @@ class AssetDownloadButtons extends React.Component {
             <ExportAssetDialog task={this.props.task}
                                asset={this.state.exportDialogProps.asset}
                                exportFormats={this.state.exportDialogProps.exportFormats}  
+                               exportParams={this.state.exportDialogProps.exportParams}
                                onHide={this.onHide}
                                assetLabel={this.state.exportDialogProps.assetLabel}
             /> 
@@ -67,6 +68,7 @@ class AssetDownloadButtons extends React.Component {
                             this.setState({exportDialogProps: {
                                 asset: asset.exportId(),
                                 exportFormats: asset.exportFormats,
+                                exportParams: asset.exportParams,
                                 assetLabel: asset.label
                             }});
                         }

app/static/app/js/components/ExportAssetDialog.jsx

@@ -14,6 +14,7 @@ class ExportAssetDialog extends React.Component {
         asset: PropTypes.string.isRequired,
         task: PropTypes.object.isRequired,
         exportFormats: PropTypes.arrayOf(PropTypes.string),
+        exportParams: PropTypes.object,
         assetLabel: PropTypes.string
     };
 
@@ -43,7 +44,8 @@ class ExportAssetDialog extends React.Component {
                                     task={this.props.task}
                                     ref={(domNode) => { this.exportAssetPanel = domNode; }}
                                     selectorOnly
-                                    exportFormats={this.props.exportFormats} />
+                                    exportFormats={this.props.exportFormats}
+                                    exportParams={this.props.exportParams} />
                 </FormDialog>
             </div>
         );

app/static/app/js/components/ExportAssetPanel.jsx

@@ -33,24 +33,40 @@ export default class ExportAssetPanel extends React.Component {
 
     this.efInfo = {
         'gtiff-rgb': {
-            label: _("GeoTIFF (RGB)"),
+            label: "GeoTIFF (RGB)",
             icon: "fas fa-palette"
         },
         'gtiff': {
-            label: _("GeoTIFF (Raw)"),
+            label: "GeoTIFF (Raw)",
             icon: "far fa-image"
         },
         'jpg': {
-            label: _("JPEG (RGB)"),
+            label: "JPEG (RGB)",
             icon: "fas fa-palette"
         },
         'png': {
-            label: _("PNG (RGB)"),
+            label: "PNG (RGB)",
             icon: "fas fa-palette"
         },
         'kmz': {
-            label: _("KMZ (RGB)"),
+            label: "KMZ (RGB)",
             icon: "fa fa-globe"
+        },
+        'laz': {
+            label: "LAZ",
+            icon: "fa fa-cube"
+        },
+        'las': {
+            label: "LAS",
+            icon: "fa fa-cube"
+        },
+        'ply': {
+            label: "PLY",
+            icon: "fa fa-cube"
+        },
+        'csv': {
+            label: "CSV",
+            icon: "fa fa-file-text"
         }        
     };
 
@@ -92,6 +108,7 @@ export default class ExportAssetPanel extends React.Component {
       
       params.format = format;
       params.epsg = this.getEpsg();
+      console.log(params);
       return params;
   }
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "WebODM",
-  "version": "1.9.10",
+  "version": "1.9.11",
   "description": "User-friendly, extendable application and API for processing aerial imagery.",
   "main": "index.js",
   "scripts": {

worker/tasks.py

@@ -20,6 +20,7 @@ from webodm import settings
 import worker
 from .celery import app
 from app.raster_utils import export_raster as export_raster_sync, extension_for_export_format
+from app.pointcloud_utils import export_pointcloud as export_pointcloud_sync
 import redis
 
 logger = get_task_logger("app.logger")
@@ -169,3 +170,19 @@ def export_raster(self, input, **opts):
     except Exception as e:
         logger.error(str(e))
         return {'error': str(e)}
+
+@app.task(bind=True)
+def export_pointcloud(self, input, **opts):
+    try:
+        logger.info("Exporting point cloud {} with options: {}".format(input, json.dumps(opts)))
+        tmpfile = tempfile.mktemp('_pointcloud.{}'.format(opts.get('format', 'laz')), dir=settings.MEDIA_TMP)
+        export_pointcloud_sync(input, tmpfile, **opts)
+        result = {'file': tmpfile}
+
+        if settings.TESTING:
+            TestSafeAsyncResult.set(self.request.id, result)
+
+        return result
+    except Exception as e:
+        logger.error(str(e))
+        return {'error': str(e)}