Export point clouds

2021-11-04 13:27:36 -04:00 · 2021-11-04 13:27:36 -04:00 · b6ce0ae3cd
commit b6ce0ae3cd
--- a/4
+++ b/4
@ -14,8 +14,8 @@ WORKDIR /webodm
 RUN apt-get -qq update && apt-get -qq install -y --no-install-recommends wget curl && \
    wget --no-check-certificate https://deb.nodesource.com/setup_12.x -O /tmp/node.sh && bash /tmp/node.sh && \
    apt-get -qq update && apt-get -qq install -y nodejs && \
-    # Install Python3, GDAL, nginx, letsencrypt, psql
-    apt-get -qq update && apt-get -qq install -y --no-install-recommends python3 python3-pip python3-setuptools python3-wheel git g++ python3-dev python2.7-dev libpq-dev binutils libproj-dev gdal-bin libgdal-dev python3-gdal nginx certbot grass-core gettext-base cron postgresql-client-13 gettext tzdata && \
+    # Install Python3, GDAL, PDAL, nginx, letsencrypt, psql
+    apt-get -qq update && apt-get -qq install -y --no-install-recommends python3 python3-pip python3-setuptools python3-wheel git g++ python3-dev python2.7-dev libpq-dev binutils libproj-dev gdal-bin pdal libgdal-dev python3-gdal nginx certbot grass-core gettext-base cron postgresql-client-13 gettext tzdata && \
    update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1 && update-alternatives --install /usr/bin/python python /usr/bin/python3.9 2 && \
    # Install pip reqs
    pip install -U pip && pip install -r requirements.txt "boto3==1.14.14" && \
--- a/app/api/tiler.py
+++ b/app/api/tiler.py
@ -20,14 +20,14 @@ from rio_tiler.io import COGReader
 from rio_tiler.errors import InvalidColorMapName
 import numpy as np
 from .custom_colormaps_helper import custom_colormaps
-from app.raster_utils import export_raster, extension_for_export_format, ZOOM_EXTRA_LEVELS
+from app.raster_utils import extension_for_export_format, ZOOM_EXTRA_LEVELS
 from .hsvblend import hsv_blend
 from .hillshade import LightSource
 from .formulas import lookup_formula, get_algorithm_list
 from .tasks import TaskNestedView
 from rest_framework import exceptions
 from rest_framework.response import Response
-from worker.tasks import export_raster
+from worker.tasks import export_raster, export_pointcloud
 from django.utils.translation import gettext as _


@ -76,6 +76,9 @@ def get_extent(task, tile_type):
 def get_raster_path(task, tile_type):
    return task.get_asset_download_path(tile_type + ".tif")

+def get_pointcloud_path(task):
+    return task.get_asset_download_path("georeferenced_model.laz")
+

 class TileJson(TaskNestedView):
    def get(self, request, pk=None, project_pk=None, tile_type=""):
@ -491,7 +494,9 @@ class Export(TaskNestedView):

        expr = None

-        if not export_format in ['gtiff', 'gtiff-rgb', 'jpg', 'png', 'kmz']:
+        if asset_type in ['orthophoto', 'dsm', 'dtm'] and not export_format in ['gtiff', 'gtiff-rgb', 'jpg', 'png', 'kmz']:
+            raise exceptions.ValidationError(_("Unsupported format: %(value)s") % {'value': export_format})
+        if asset_type == 'georeferenced_model' and not export_format in ['laz', 'las', 'ply', 'csv']:
            raise exceptions.ValidationError(_("Unsupported format: %(value)s") % {'value': export_format})
        
        if epsg is not None:
@ -527,8 +532,11 @@ class Export(TaskNestedView):
                    raise Exception("Hillshade must be > 0")
            except:
                raise exceptions.ValidationError(_("Invalid hillshade value: %(value)s") % {'value': hillshade})
-
-        url = get_raster_path(task, asset_type)
+        
+        if asset_type == 'georeferenced_model':
+            url = get_pointcloud_path(task)
+        else:
+            url = get_raster_path(task, asset_type)

        if not os.path.isfile(url):
            raise exceptions.NotFound()
@ -541,16 +549,25 @@ class Export(TaskNestedView):
                        extension
                    )

-        # Shortcut the process if no processing is required
-        if export_format == 'gtiff' and (epsg == task.epsg or epsg is None) and expr is None:
-            return Response({'url': '/api/projects/{}/tasks/{}/download/{}.tif'.format(task.project.id, task.id, asset_type), 'filename': filename})
-        else:
-            celery_task_id = export_raster.delay(url, epsg=epsg, 
-                                                    expression=expr, 
-                                                    format=export_format, 
-                                                    rescale=rescale, 
-                                                    color_map=color_map,
-                                                    hillshade=hillshade,
-                                                    asset_type=asset_type,
-                                                    name=task.name).task_id
-            return Response({'celery_task_id': celery_task_id, 'filename': filename})
+        if asset_type in ['orthophoto', 'dsm', 'dtm']:
+            # Shortcut the process if no processing is required
+            if export_format == 'gtiff' and (epsg == task.epsg or epsg is None) and expr is None:
+                return Response({'url': '/api/projects/{}/tasks/{}/download/{}.tif'.format(task.project.id, task.id, asset_type), 'filename': filename})
+            else:
+                celery_task_id = export_raster.delay(url, epsg=epsg, 
+                                                        expression=expr, 
+                                                        format=export_format, 
+                                                        rescale=rescale, 
+                                                        color_map=color_map,
+                                                        hillshade=hillshade,
+                                                        asset_type=asset_type,
+                                                        name=task.name).task_id
+                return Response({'celery_task_id': celery_task_id, 'filename': filename})
+        elif asset_type == 'georeferenced_model':
+            # Shortcut the process if no processing is required
+            if export_format == 'laz' and (epsg == task.epsg or epsg is None):
+                return Response({'url': '/api/projects/{}/tasks/{}/download/{}.laz'.format(task.project.id, task.id, asset_type), 'filename': filename})
+            else:
+                celery_task_id = export_pointcloud.delay(url, epsg=epsg, 
+                                                            format=export_format).task_id
+                return Response({'celery_task_id': celery_task_id, 'filename': filename})
--- a/app/api/urls.py
+++ b/app/api/urls.py
@ -39,7 +39,7 @@ urlpatterns = [
    url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/(?P<tile_type>orthophoto|dsm|dtm)/metadata$', Metadata.as_view()),
    url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/(?P<tile_type>orthophoto|dsm|dtm)/tiles/(?P<z>[\d]+)/(?P<x>[\d]+)/(?P<y>[\d]+)\.png$', Tiles.as_view()),
    url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/(?P<tile_type>orthophoto|dsm|dtm)/tiles/(?P<z>[\d]+)/(?P<x>[\d]+)/(?P<y>[\d]+)@(?P<scale>[\d]+)x\.png$', Tiles.as_view()),
-    url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/(?P<asset_type>orthophoto|dsm|dtm)/export$', Export.as_view()),
+    url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/(?P<asset_type>orthophoto|dsm|dtm|georeferenced_model)/export$', Export.as_view()),

    url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/download/(?P<asset>.+)$', TaskDownloads.as_view()),
    url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/assets/(?P<unsafe_asset_path>.+)$', TaskAssets.as_view()),
--- a/app/pointcloud_utils.py
+++ b/app/pointcloud_utils.py
@ -0,0 +1,23 @@
+import logging
+import os
+import subprocess
+from app.security import double_quote
+
+logger = logging.getLogger('app.logger')
+
+def export_pointcloud(input, output, **opts):
+    epsg = opts.get('epsg')
+    export_format = opts.get('format')
+
+    reprojection_args = []
+    extra_args = []
+
+    if epsg:
+        reprojection_args = ["reprojection",
+                            "--filters.reprojection.out_srs=%s" % double_quote("EPSG:" + str(epsg))]
+    
+    if export_format == "ply":
+        extra_args = ['--writers.ply.sized_types', 'false',
+                      '--writers.ply.storage_mode', 'little endian']
+
+    subprocess.check_output(["pdal", "translate", input, output] + reprojection_args + extra_args)
--- a/app/raster_utils.py
+++ b/app/raster_utils.py
@ -1,4 +1,3 @@
-# Export a raster index after applying a band expression
 import rasterio
 import re
 import logging
@ -23,11 +22,8 @@ def extension_for_export_format(export_format):
    extensions = {
        'gtiff': 'tif',
        'gtiff-rgb': 'tif',
-        'jpg': 'jpg',
-        'png': 'png',
-        'kmz': 'kmz'
    }
-    return extensions.get(export_format, 'tif')
+    return extensions.get(export_format, export_format)

 def export_raster(input, output, **opts):
    epsg = opts.get('epsg')
--- a/app/static/app/js/classes/AssetDownloads.js
+++ b/app/static/app/js/classes/AssetDownloads.js
@ -1,11 +1,12 @@
 import { _ } from './gettext';

 class AssetDownload{
-  constructor(label, asset, icon, exportFormats = null){
+  constructor(label, asset, icon, exportFormats = null, exportParams = {}){
    this.label = label;
    this.asset = asset;
    this.icon = icon;
    this.exportFormats = exportFormats;
+    this.exportParams = exportParams;
  }

  downloadUrl(project_id, task_id){
@ -36,24 +37,24 @@ class AssetDownloadSeparator extends AssetDownload{
  }
 }

+const tiffExportFormats = ["gtiff", "gtiff-rgb", "jpg", "png", "kmz"];
+const elevationExportParams = {'hillshade': 6, "color_map": "viridis"};
+
 const api = {
  all: function() {
    return [
-      new AssetDownload(_("Orthophoto"),"orthophoto.tif","far fa-image", ["gtiff", "gtiff-rgb", "jpg", "png", "kmz"]),
+      new AssetDownload(_("Orthophoto"),"orthophoto.tif","far fa-image", tiffExportFormats),
      new AssetDownload(_("Orthophoto (MBTiles)"),"orthophoto.mbtiles","far fa-image"),
      new AssetDownload(_("Orthophoto (Tiles)"),"orthophoto_tiles.zip","fa fa-table"),
-      new AssetDownload(_("Terrain Model"),"dtm.tif","fa fa-chart-area"),
+      new AssetDownload(_("Terrain Model"),"dtm.tif","fa fa-chart-area", tiffExportFormats, elevationExportParams),
      new AssetDownload(_("Terrain Model (Tiles)"),"dtm_tiles.zip","fa fa-table"),
-      new AssetDownload(_("Surface Model"),"dsm.tif","fa fa-chart-area"),
+      new AssetDownload(_("Surface Model"),"dsm.tif","fa fa-chart-area", tiffExportFormats, elevationExportParams),
      new AssetDownload(_("Surface Model (Tiles)"),"dsm_tiles.zip","fa fa-table"),
-      new AssetDownload(_("Point Cloud (LAS)"),"georeferenced_model.las","fa fa-cube"),
-      new AssetDownload(_("Point Cloud (LAZ)"),"georeferenced_model.laz","fa fa-cube"),
-      new AssetDownload(_("Point Cloud (PLY)"),"georeferenced_model.ply","fa fa-cube"),
-      new AssetDownload(_("Point Cloud (CSV)"),"georeferenced_model.csv","fa fa-cube"),
+      new AssetDownload(_("Point Cloud"),"georeferenced_model.laz","fa fa-cube", ["laz", "las", "ply", "csv"]),
      new AssetDownload(_("Textured Model"),"textured_model.zip","fab fa-connectdevelop"),
      new AssetDownload(_("Camera Parameters"),"cameras.json","fa fa-camera"),
-      new AssetDownload(_("Camera Shots (GeoJSON)"),"shots.geojson","fa fa-camera"),
-      new AssetDownload(_("Ground Control Points (GeoJSON)"),"ground_control_points.geojson","far fa-dot-circle"),
+      new AssetDownload(_("Camera Shots"),"shots.geojson","fa fa-camera"),
+      new AssetDownload(_("Ground Control Points"),"ground_control_points.geojson","far fa-dot-circle"),
      new AssetDownload(_("Quality Report"),"report.pdf","far fa-file-pdf"),
      
      
--- a/app/static/app/js/components/AssetDownloadButtons.jsx
+++ b/app/static/app/js/components/AssetDownloadButtons.jsx
@ -43,6 +43,7 @@ class AssetDownloadButtons extends React.Component {
            <ExportAssetDialog task={this.props.task}
                               asset={this.state.exportDialogProps.asset}
                               exportFormats={this.state.exportDialogProps.exportFormats}  
+                               exportParams={this.state.exportDialogProps.exportParams}
                               onHide={this.onHide}
                               assetLabel={this.state.exportDialogProps.assetLabel}
            /> 
@ -67,6 +68,7 @@ class AssetDownloadButtons extends React.Component {
                            this.setState({exportDialogProps: {
                                asset: asset.exportId(),
                                exportFormats: asset.exportFormats,
+                                exportParams: asset.exportParams,
                                assetLabel: asset.label
                            }});
                        }
--- a/app/static/app/js/components/ExportAssetDialog.jsx
+++ b/app/static/app/js/components/ExportAssetDialog.jsx
@ -14,6 +14,7 @@ class ExportAssetDialog extends React.Component {
        asset: PropTypes.string.isRequired,
        task: PropTypes.object.isRequired,
        exportFormats: PropTypes.arrayOf(PropTypes.string),
+        exportParams: PropTypes.object,
        assetLabel: PropTypes.string
    };

@ -43,7 +44,8 @@ class ExportAssetDialog extends React.Component {
                                    task={this.props.task}
                                    ref={(domNode) => { this.exportAssetPanel = domNode; }}
                                    selectorOnly
-                                    exportFormats={this.props.exportFormats} />
+                                    exportFormats={this.props.exportFormats}
+                                    exportParams={this.props.exportParams} />
                </FormDialog>
            </div>
        );
--- a/app/static/app/js/components/ExportAssetPanel.jsx
+++ b/app/static/app/js/components/ExportAssetPanel.jsx
@ -33,25 +33,41 @@ export default class ExportAssetPanel extends React.Component {

    this.efInfo = {
        'gtiff-rgb': {
-            label: _("GeoTIFF (RGB)"),
+            label: "GeoTIFF (RGB)",
            icon: "fas fa-palette"
        },
        'gtiff': {
-            label: _("GeoTIFF (Raw)"),
+            label: "GeoTIFF (Raw)",
            icon: "far fa-image"
        },
        'jpg': {
-            label: _("JPEG (RGB)"),
+            label: "JPEG (RGB)",
            icon: "fas fa-palette"
        },
        'png': {
-            label: _("PNG (RGB)"),
+            label: "PNG (RGB)",
            icon: "fas fa-palette"
        },
        'kmz': {
-            label: _("KMZ (RGB)"),
+            label: "KMZ (RGB)",
            icon: "fa fa-globe"
-        }
+        },
+        'laz': {
+            label: "LAZ",
+            icon: "fa fa-cube"
+        },
+        'las': {
+            label: "LAS",
+            icon: "fa fa-cube"
+        },
+        'ply': {
+            label: "PLY",
+            icon: "fa fa-cube"
+        },
+        'csv': {
+            label: "CSV",
+            icon: "fa fa-file-text"
+        }        
    };

    this.state = {
@ -92,6 +108,7 @@ export default class ExportAssetPanel extends React.Component {
      
      params.format = format;
      params.epsg = this.getEpsg();
+      console.log(params);
      return params;
  }

--- a/package.json
+++ b/package.json
@ -1,6 +1,6 @@
 {
  "name": "WebODM",
-  "version": "1.9.10",
+  "version": "1.9.11",
  "description": "User-friendly, extendable application and API for processing aerial imagery.",
  "main": "index.js",
  "scripts": {
--- a/worker/tasks.py
+++ b/worker/tasks.py
@ -20,6 +20,7 @@ from webodm import settings
 import worker
 from .celery import app
 from app.raster_utils import export_raster as export_raster_sync, extension_for_export_format
+from app.pointcloud_utils import export_pointcloud as export_pointcloud_sync
 import redis

 logger = get_task_logger("app.logger")
@ -162,6 +163,22 @@ def export_raster(self, input, **opts):
        export_raster_sync(input, tmpfile, **opts)
        result = {'file': tmpfile}

+        if settings.TESTING:
+            TestSafeAsyncResult.set(self.request.id, result)
+
+        return result
+    except Exception as e:
+        logger.error(str(e))
+        return {'error': str(e)}
+
+@app.task(bind=True)
+def export_pointcloud(self, input, **opts):
+    try:
+        logger.info("Exporting point cloud {} with options: {}".format(input, json.dumps(opts)))
+        tmpfile = tempfile.mktemp('_pointcloud.{}'.format(opts.get('format', 'laz')), dir=settings.MEDIA_TMP)
+        export_pointcloud_sync(input, tmpfile, **opts)
+        result = {'file': tmpfile}
+
        if settings.TESTING:
            TestSafeAsyncResult.set(self.request.id, result)