Merge pull request #1510 from pierotofy/backup

Add task backup import/export

Dockerfile

@@ -15,15 +15,15 @@ WORKDIR /webodm
 RUN printf "deb http://old-releases.ubuntu.com/ubuntu/ hirsute main restricted\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-updates main restricted\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute universe\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-updates universe\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute multiverse\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-updates multiverse\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-backports main restricted universe multiverse" > /etc/apt/sources.list
 
 # Install Node.js using new Node install method
-RUN apt-get -qq update && apt-get -qq install -y --no-install-recommends wget curl && \
-    apt-get install -y ca-certificates gnupg && \
+RUN apt-get -qq update && apt-get -o Acquire::Retries=3 -qq install -y --no-install-recommends wget curl && \
+    apt-get -o Acquire::Retries=3 install -y ca-certificates gnupg && \
     mkdir -p /etc/apt/keyrings && \
     curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
     NODE_MAJOR=20 && \
     echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \
-    apt-get -qq update && apt-get -qq install -y nodejs && \
+    apt-get -o Acquire::Retries=3 -qq update && apt-get -o Acquire::Retries=3 -qq install -y nodejs && \
     # Install Python3, GDAL, PDAL, nginx, letsencrypt, psql
-    apt-get -qq update && apt-get -qq install -y --no-install-recommends python3 python3-pip python3-setuptools python3-wheel git g++ python3-dev python2.7-dev libpq-dev binutils libproj-dev gdal-bin pdal libgdal-dev python3-gdal nginx certbot gettext-base cron postgresql-client-13 gettext tzdata && \
+    apt-get -o Acquire::Retries=3 -qq update && apt-get -o Acquire::Retries=3 -qq install -y --no-install-recommends python3 python3-pip python3-setuptools python3-wheel git g++ python3-dev python2.7-dev libpq-dev binutils libproj-dev gdal-bin pdal libgdal-dev python3-gdal nginx certbot gettext-base cron postgresql-client-13 gettext tzdata && \
     update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1 && update-alternatives --install /usr/bin/python python /usr/bin/python3.9 2 && \
     # Install pip reqs
     pip install -U pip && pip install -r requirements.txt "boto3==1.14.14" && \

app/api/tasks.py

@@ -402,6 +402,26 @@ class TaskAssets(TaskNestedView):
 
         return download_file_response(request, asset_path, 'inline')
 
+"""
+Task backup endpoint
+"""
+class TaskBackup(TaskNestedView):
+    def get(self, request, pk=None, project_pk=None):
+        """
+        Downloads a task's backup
+        """
+        task = self.get_and_check_task(request, pk)
+
+        # Check and download
+        try:
+            asset_fs = task.get_task_backup_stream()
+        except FileNotFoundError:
+            raise exceptions.NotFound(_("Asset does not exist"))
+
+        download_filename = request.GET.get('filename', get_asset_download_filename(task, "backup.zip"))
+
+        return download_file_stream(request, asset_fs, 'attachment', download_filename=download_filename)
+
 """
 Task assets import
 """

app/api/urls.py

@@ -3,7 +3,7 @@ from django.conf.urls import url, include
 from app.api.presets import PresetViewSet
 from app.plugins.views import api_view_handler
 from .projects import ProjectViewSet
-from .tasks import TaskViewSet, TaskDownloads, TaskAssets, TaskAssetsImport
+from .tasks import TaskViewSet, TaskDownloads, TaskAssets, TaskBackup, TaskAssetsImport
 from .imageuploads import Thumbnail, ImageDownload
 from .processingnodes import ProcessingNodeViewSet, ProcessingNodeOptionsView
 from .admin import AdminUserViewSet, AdminGroupViewSet, AdminProfileViewSet
@@ -46,6 +46,7 @@ urlpatterns = [
     url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/download/(?P<asset>.+)$', TaskDownloads.as_view()),
     url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/assets/(?P<unsafe_asset_path>.+)$', TaskAssets.as_view()),
     url(r'projects/(?P<project_pk>[^/.]+)/tasks/import$', TaskAssetsImport.as_view()),
+    url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/backup$', TaskBackup.as_view()),
     url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/images/thumbnail/(?P<image_filename>.+)$', Thumbnail.as_view()),
     url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/images/download/(?P<image_filename>.+)$', ImageDownload.as_view()),
 

app/models/task.py

@@ -3,6 +3,7 @@ import os
 import shutil
 import time
 import struct
+from datetime import datetime
 import uuid as uuid_module
 from app.vendor import zipfly
 
@@ -453,6 +454,48 @@ class Task(models.Model):
             logger.warning("Cannot duplicate task: {}".format(str(e)))
         
         return False
+
+    def write_backup_file(self):
+        """Dump this tasks's fields to a backup file"""
+        with open(self.data_path("backup.json"), "w") as f:
+            f.write(json.dumps({
+                'name': self.name,
+                'processing_time': self.processing_time,
+                'options': self.options,
+                'created_at': self.created_at.astimezone(timezone.utc).timestamp(),
+                'public': self.public,
+                'resize_to': self.resize_to,
+                'potree_scene': self.potree_scene,
+                'tags': self.tags
+            }))
+    
+    def read_backup_file(self):
+        """Set this tasks fields based on the backup file (but don't save)"""
+        backup_file = self.data_path("backup.json")
+        if os.path.isfile(backup_file):
+            try:
+                with open(backup_file, "r") as f:
+                    backup = json.loads(f.read())
+
+                    self.name = backup.get('name', self.name)
+                    self.processing_time = backup.get('processing_time', self.processing_time)
+                    self.options = backup.get('options', self.options)
+                    self.created_at = datetime.fromtimestamp(backup.get('created_at', self.created_at.astimezone(timezone.utc).timestamp()), tz=timezone.utc)
+                    self.public = backup.get('public', self.public)
+                    self.resize_to = backup.get('resize_to', self.resize_to)
+                    self.potree_scene = backup.get('potree_scene', self.potree_scene)
+                    self.tags = backup.get('tags', self.tags)
+
+            except Exception as e:
+                logger.warning("Cannot read backup file: %s" % str(e))
+
+    def get_task_backup_stream(self):
+        self.write_backup_file()
+        zip_dir = self.task_path("")
+        paths = [{'n': os.path.relpath(os.path.join(dp, f), zip_dir), 'fs': os.path.join(dp, f)} for dp, dn, filenames in os.walk(zip_dir) for f in filenames]
+        if len(paths) == 0:
+            raise FileNotFoundError("No files available for export")
+        return zipfly.ZipStream(paths)
     
     def get_asset_file_or_stream(self, asset):
         """
@@ -568,7 +611,6 @@ class Task(models.Model):
                 pass
 
         self.pending_action = None
-        self.processing_time = 0
         self.save()
 
     def process(self):
@@ -859,10 +901,24 @@ class Task(models.Model):
             zip_h.extractall(assets_dir)
 
         logger.info("Extracted all.zip for {}".format(self))
-
-        # Remove zip
+        
         os.remove(zip_path)
 
+        # Check if this looks like a backup file, in which case we need to move the files
+        # a directory level higher
+        is_backup = os.path.isfile(self.assets_path("data", "backup.json")) and os.path.isdir(self.assets_path("assets"))
+        if is_backup:
+            logger.info("Restoring from backup")
+            try:
+                tmp_dir = os.path.join(settings.FILE_UPLOAD_TEMP_DIR, f"{self.id}.backup")
+                
+                shutil.move(assets_dir, tmp_dir)
+                shutil.rmtree(self.task_path(""))
+                shutil.move(tmp_dir, self.task_path(""))
+            except shutil.Error as e:
+                logger.warning("Cannot restore from backup: %s" % str(e))
+                raise NodeServerError("Cannot restore from backup")
+
         # Populate *_extent fields
         extent_fields = [
             (os.path.realpath(self.assets_path("odm_orthophoto", "odm_orthophoto.tif")),
@@ -904,8 +960,13 @@ class Task(models.Model):
         self.update_size()
         self.potree_scene = {}
         self.running_progress = 1.0
-        self.console += gettext("Done!") + "\n"
         self.status = status_codes.COMPLETED
+
+        if is_backup:
+            self.read_backup_file()
+        else:
+            self.console += gettext("Done!") + "\n"
+        
         self.save()
 
         from app.plugins import signals as plugin_signals

app/static/app/js/components/AssetDownloadButtons.jsx

@@ -82,6 +82,9 @@ class AssetDownloadButtons extends React.Component {
                         </li>);
                 }
             })}
+            <li>
+                <a href={`/api/projects/${this.props.task.project}/tasks/${this.props.task.id}/backup`}><i className="fa fa-file-download fa-fw"></i> {_("Backup")}</a>
+            </li>
           </ul>
         </div>);
     }

app/static/app/js/components/ImportTaskPanel.jsx

@@ -158,8 +158,8 @@ class ImportTaskPanel extends React.Component {
           <ErrorMessage bind={[this, 'error']} />
 
           <button type="button" className="close theme-color-primary" title="Close" onClick={this.cancel}><span aria-hidden="true">&times;</span></button>
-          <h4>{_("Import Existing Assets")}</h4>
-          <p><Trans params={{arrow: '<i class="glyphicon glyphicon-arrow-right"></i>'}}>{_("You can import .zip files that have been exported from existing tasks via Download Assets %(arrow)s All Assets.")}</Trans></p>
+          <h4>{_("Import Assets or Backups")}</h4>
+          <p><Trans params={{arrow: '<i class="glyphicon glyphicon-arrow-right"></i>'}}>{_("You can import .zip files that have been exported from existing tasks via Download Assets %(arrow)s All Assets | Backup.")}</Trans></p>
           
           <button disabled={this.state.uploading}
                   type="button" 

app/static/app/js/translations/odm_autogenerated.js

@@ -1,93 +1,93 @@
 // Auto-generated with extract_odm_strings.py, do not edit!
 
+_("Set a camera projection type. Manually setting a value can help improve geometric undistortion. By default the application tries to determine a lens type from the images metadata. Can be one of: %(choices)s. Default: %(default)s");
+_("Skip generation of a full 3D model. This can save time if you only need 2D results such as orthophotos and DEMs. Default: %(default)s");
 _("Use this tag to build a DSM (Digital Surface Model, ground + objects) using a progressive morphological filter. Check the --dem* parameters for finer tuning. Default: %(default)s");
-_("Use a full 3D mesh to compute the orthophoto instead of a 2.5D mesh. This option is a bit faster and provides similar results in planar areas. Default: %(default)s");
+_("Minimum number of features to extract per image. More features can be useful for finding more matches between images, potentially allowing the reconstruction of areas with little overlap or insufficient features. More features also slow down processing. Default: %(default)s");
+_("The maximum output resolution of extracted video frames in pixels. Default: %(default)s");
+_("Radius of the overlap between submodels. After grouping images into clusters, images that are closer than this radius to a cluster are added to the cluster. This is done to ensure that neighboring submodels overlap. Default: %(default)s");
+_("Set the radiometric calibration to perform on images. When processing multispectral and thermal images you should set this option to obtain reflectance/temperature values (otherwise you will get digital number values). [camera] applies black level, vignetting, row gradient gain/exposure compensation (if appropriate EXIF tags are found) and computes absolute temperature values. [camera+sun] is experimental, applies all the corrections of [camera], plus compensates for spectral radiance registered via a downwelling light sensor (DLS) taking in consideration the angle of the sun. Can be one of: %(choices)s. Default: %(default)s");
+_("Maximum number of frames to extract from video files for processing. Set to 0 for no limit. Default: %(default)s");
+_("Automatically compute image masks using AI to remove the sky. Experimental. Default: %(default)s");
 _("Path to the image geolocation file containing the camera center coordinates used for georeferencing. If you don't have values for yaw/pitch/roll you can set them to 0. The file needs to use the following format: EPSG:<code> or <+proj definition>image_name geo_x geo_y geo_z [yaw (degrees)] [pitch (degrees)] [roll (degrees)] [horz accuracy (meters)] [vert accuracy (meters)]Default: %(default)s");
-_("DSM/DTM resolution in cm / pixel. Note that this value is capped by a ground sampling distance (GSD) estimate. Default: %(default)s");
-_("Automatically compute image masks using AI to remove the background. Experimental. Default: %(default)s");
+_("Path to a GeoTIFF DEM or a LAS/LAZ point cloud that the reconstruction outputs should be automatically aligned to. Experimental. Default: %(default)s");
 _("Rerun this stage only and stop. Can be one of: %(choices)s. Default: %(default)s");
+_("Set a value in meters for the GPS Dilution of Precision (DOP) information for all images. If your images are tagged with high precision GPS information (RTK), this value will be automatically set accordingly. You can use this option to manually set it in case the reconstruction fails. Lowering this option can sometimes help control bowling-effects over large areas. Default: %(default)s");
+_("Geometric estimates improve the accuracy of the point cloud by computing geometrically consistent depthmaps but may not be usable in larger datasets. This flag disables geometric estimates. Default: %(default)s");
+_("Orthophoto resolution in cm / pixel. Note that this value is capped by a ground sampling distance (GSD) estimate.Default: %(default)s");
+_("Rerun processing from this stage. Can be one of: %(choices)s. Default: %(default)s");
+_("Save the georeferenced point cloud in Cloud Optimized Point Cloud (COPC) format. Default: %(default)s");
+_("Use a full 3D mesh to compute the orthophoto instead of a 2.5D mesh. This option is a bit faster and provides similar results in planar areas. Default: %(default)s");
+_("Decimate the points before generating the DEM. 1 is no decimation (full quality). 100 decimates ~99%% of the points. Useful for speeding up generation of DEM results in very large datasets. Default: %(default)s");
+_("Displays version number and exits. ");
+_("Export the georeferenced point cloud in Entwine Point Tile (EPT) format. Default: %(default)s");
+_("Octree depth used in the mesh reconstruction, increase to get more vertices, recommended values are 8-12. Default: %(default)s");
+_("Do not attempt to merge partial reconstructions. This can happen when images do not have sufficient overlap or are isolated. Default: %(default)s");
+_("Set this parameter if you want to generate a Google Earth (KMZ) rendering of the orthophoto. Default: %(default)s");
+_("Automatically crop image outputs by creating a smooth buffer around the dataset boundaries, shrunk by N meters. Use 0 to disable cropping. Default: %(default)s");
+_("Permanently delete all previous results and rerun the processing pipeline.");
+_("Path to the image groups file that controls how images should be split into groups. The file needs to use the following format: image_name group_nameDefault: %(default)s");
+_("End processing at this stage. Can be one of: %(choices)s. Default: %(default)s");
+_("Copy output results to this folder after processing.");
+_("Generate OBJs that have a single material and a single texture file instead of multiple ones. Default:  %(default)s");
+_("Choose the algorithm for extracting keypoints and computing descriptors. Can be one of: %(choices)s. Default: %(default)s");
+_("Average number of images per submodel. When splitting a large dataset into smaller submodels, images are grouped into clusters. This value regulates the number of images that each cluster should have on average. Default: %(default)s");
+_("When processing multispectral datasets, ODM will automatically align the images for each band. If the images have been postprocessed and are already aligned, use this option. Default: %(default)s");
+_("Number of steps used to fill areas with gaps. Set to 0 to disable gap filling. Starting with a radius equal to the output resolution, N different DEMs are generated with progressively bigger radius using the inverse distance weighted (IDW) algorithm and merged together. Remaining gaps are then merged using nearest neighbor interpolation. Default: %(default)s");
+_("Specify the distance between camera shot locations and the outer edge of the boundary when computing the boundary with --auto-boundary. Set to 0 to automatically choose a value. Default: %(default)s");
+_("DSM/DTM resolution in cm / pixel. Note that this value is capped by a ground sampling distance (GSD) estimate. Default: %(default)s");
+_("Turn on rolling shutter correction. If the camera has a rolling shutter and the images were taken in motion, you can turn on this option to improve the accuracy of the results. See also --rolling-shutter-readout. Default: %(default)s");
+_("Set point cloud quality. Higher quality generates better, denser point clouds, but requires more memory and takes longer. Each step up in quality increases processing time roughly by a factor of 4x.Can be one of: %(choices)s. Default: %(default)s");
+_("Skip generation of PDF report. This can save time if you don't need a report. Default: %(default)s");
+_("Use this tag if you have a GCP File but want to use the EXIF information for georeferencing instead. Default: %(default)s");
+_("Perform ground rectification on the point cloud. This means that wrongly classified ground points will be re-classified and gaps will be filled. Useful for generating DTMs. Default: %(default)s");
+_("Export the georeferenced point cloud in CSV format. Default: %(default)s");
+_("Matcher algorithm, Fast Library for Approximate Nearest Neighbors or Bag of Words. FLANN is slower, but more stable. BOW is faster, but can sometimes miss valid matches. BRUTEFORCE is very slow but robust.Can be one of: %(choices)s. Default: %(default)s");
+_("Set feature extraction quality. Higher quality generates better features, but requires more memory and takes longer. Can be one of: %(choices)s. Default: %(default)s");
+_("show this help message and exit");
+_("URL to a ClusterODM instance for distributing a split-merge workflow on multiple nodes in parallel. Default: %(default)s");
+_("Simple Morphological Filter window radius parameter (meters). Default: %(default)s");
+_("Use images' GPS exif data for reconstruction, even if there are GCPs present.This flag is useful if you have high precision GPS measurements. If there are no GCPs, this flag does nothing. Default: %(default)s");
+_("Delete heavy intermediate files to optimize disk space usage. This affects the ability to restart the pipeline from an intermediate stage, but allows datasets to be processed on machines that don't have sufficient disk space available. Default: %(default)s");
+_("Keep faces in the mesh that are not seen in any camera. Default:  %(default)s");
 _("Export the georeferenced point cloud in LAS format. Default: %(default)s");
+_("Skips dense reconstruction and 3D model generation. It generates an orthophoto directly from the sparse reconstruction. If you just need an orthophoto and do not need a full 3D model, turn on this option. Default: %(default)s");
+_("Name of dataset (i.e subfolder name within project folder). Default: %(default)s");
+_("Ignore Ground Sampling Distance (GSD).A memory and processor hungry change relative to the default behavior if set to true. Ordinarily, GSD estimates are used to cap the maximum resolution of image outputs and resizes images when necessary, resulting in faster processing and lower memory usage. Since GSD is an estimate, sometimes ignoring it can result in slightly better image output quality. Never set --ignore-gsd to true unless you are positive you need it, and even then: do not use it. Default: %(default)s");
+_("Classify the point cloud outputs. You can control the behavior of this option by tweaking the --dem-* parameters. Default: %(default)s");
+_("Perform image matching with the nearest N images based on image filename order. Can speed up processing of sequential images, such as those extracted from video. It is applied only on non-georeferenced datasets. Set to 0 to disable. Default: %(default)s");
+_("Do not use GPU acceleration, even if it's available. Default: %(default)s");
+_("Simple Morphological Filter elevation threshold parameter (meters). Default: %(default)s");
+_("Choose the structure from motion algorithm. For aerial datasets, if camera GPS positions and angles are available, triangulation can generate better results. For planar scenes captured at fixed altitude with nadir-only images, planar can be much faster. Can be one of: %(choices)s. Default: %(default)s");
+_("Turn off camera parameter optimization during bundle adjustment. This can be sometimes useful for improving results that exhibit doming/bowling or when images are taken with a rolling shutter camera. Default: %(default)s");
+_("Filters the point cloud by removing points that deviate more than N standard deviations from the local mean. Set to 0 to disable filtering. Default: %(default)s");
+_("Choose what to merge in the merge step in a split dataset. By default all available outputs are merged. Options: %(choices)s. Default: %(default)s");
+_("Automatically compute image masks using AI to remove the background. Experimental. Default: %(default)s");
+_("Automatically set a boundary using camera shot locations to limit the area of the reconstruction. This can help remove far away background artifacts (sky, background landscapes, etc.). See also --boundary. Default: %(default)s");
+_("Path to the file containing the ground control points used for georeferencing. The file needs to use the following format: EPSG:<code> or <+proj definition>geo_x geo_y geo_z im_x im_y image_name [gcp_name] [extra1] [extra2]Default: %(default)s");
+_("Create Cloud-Optimized GeoTIFFs instead of normal GeoTIFFs. Default: %(default)s");
+_("Set the compression to use for orthophotos. Can be one of: %(choices)s. Default: %(default)s");
+_("Use the camera parameters computed from another dataset instead of calculating them. Can be specified either as path to a cameras.json file or as a JSON string representing the contents of a cameras.json file. Default: %(default)s");
+_("Skip alignment of submodels in split-merge. Useful if GPS is good enough on very large datasets. Default: %(default)s");
+_("Perform image matching with the nearest images based on GPS exif data. Set to 0 to match by triangulation. Default: %(default)s");
+_("Run local bundle adjustment for every image added to the reconstruction and a global adjustment every 100 images. Speeds up reconstruction for very large datasets. Default: %(default)s");
+_("Simple Morphological Filter elevation scalar parameter. Default: %(default)s");
+_("Generate static tiles for orthophotos and DEMs that are suitable for viewers like Leaflet or OpenLayers. Default: %(default)s");
+_("Simple Morphological Filter slope parameter (rise over run). Default: %(default)s");
+_("When processing multispectral datasets, you can specify the name of the primary band that will be used for reconstruction. It's recommended to choose a band which has sharp details and is in focus. Default: %(default)s");
 _("Generate OGC 3D Tiles outputs. Default: %(default)s");
 _("Computes an euclidean raster map for each DEM. The map reports the distance from each cell to the nearest NODATA value (before any hole filling takes place). This can be useful to isolate the areas that have been filled. Default: %(default)s");
-_("Override the rolling shutter readout time for your camera sensor (in milliseconds), instead of using the rolling shutter readout database. Note that not all cameras are present in the database. Set to 0 to use the database value. Default: %(default)s");
-_("Specify the distance between camera shot locations and the outer edge of the boundary when computing the boundary with --auto-boundary. Set to 0 to automatically choose a value. Default: %(default)s");
-_("End processing at this stage. Can be one of: %(choices)s. Default: %(default)s");
-_("Generates a polygon around the cropping area that cuts the orthophoto around the edges of features. This polygon can be useful for stitching seamless mosaics with multiple overlapping orthophotos. Default: %(default)s");
-_("GeoJSON polygon limiting the area of the reconstruction. Can be specified either as path to a GeoJSON file or as a JSON string representing the contents of a GeoJSON file. Default: %(default)s");
-_("Simple Morphological Filter elevation threshold parameter (meters). Default: %(default)s");
-_("Rerun processing from this stage. Can be one of: %(choices)s. Default: %(default)s");
-_("Ignore Ground Sampling Distance (GSD).A memory and processor hungry change relative to the default behavior if set to true. Ordinarily, GSD estimates are used to cap the maximum resolution of image outputs and resizes images when necessary, resulting in faster processing and lower memory usage. Since GSD is an estimate, sometimes ignoring it can result in slightly better image output quality. Never set --ignore-gsd to true unless you are positive you need it, and even then: do not use it. Default: %(default)s");
-_("Skip alignment of submodels in split-merge. Useful if GPS is good enough on very large datasets. Default: %(default)s");
-_("The maximum number of processes to use in various processes. Peak memory requirement is ~1GB per thread and 2 megapixel image resolution. Default: %(default)s");
-_("Keep faces in the mesh that are not seen in any camera. Default:  %(default)s");
-_("Generate OBJs that have a single material and a single texture file instead of multiple ones. Default:  %(default)s");
-_("Export the georeferenced point cloud in Entwine Point Tile (EPT) format. Default: %(default)s");
-_("Radius of the overlap between submodels. After grouping images into clusters, images that are closer than this radius to a cluster are added to the cluster. This is done to ensure that neighboring submodels overlap. Default: %(default)s");
-_("Displays version number and exits. ");
-_("Simple Morphological Filter window radius parameter (meters). Default: %(default)s");
-_("Automatically compute image masks using AI to remove the sky. Experimental. Default: %(default)s");
 _("Use this tag to build a DTM (Digital Terrain Model, ground only) using a simple morphological filter. Check the --dem* and --smrf* parameters for finer tuning. Default: %(default)s");
-_("Path to the image groups file that controls how images should be split into groups. The file needs to use the following format: image_name group_nameDefault: %(default)s");
-_("Use the camera parameters computed from another dataset instead of calculating them. Can be specified either as path to a cameras.json file or as a JSON string representing the contents of a cameras.json file. Default: %(default)s");
 _("Build orthophoto overviews for faster display in programs such as QGIS. Default: %(default)s");
-_("Set the compression to use for orthophotos. Can be one of: %(choices)s. Default: %(default)s");
-_("Create Cloud-Optimized GeoTIFFs instead of normal GeoTIFFs. Default: %(default)s");
-_("Classify the point cloud outputs. You can control the behavior of this option by tweaking the --dem-* parameters. Default: %(default)s");
-_("Turn on rolling shutter correction. If the camera has a rolling shutter and the images were taken in motion, you can turn on this option to improve the accuracy of the results. See also --rolling-shutter-readout. Default: %(default)s");
-_("Do not attempt to merge partial reconstructions. This can happen when images do not have sufficient overlap or are isolated. Default: %(default)s");
-_("Skip generation of the orthophoto. This can save time if you only need 3D results or DEMs. Default: %(default)s");
-_("Perform image matching with the nearest images based on GPS exif data. Set to 0 to match by triangulation. Default: %(default)s");
-_("Save the georeferenced point cloud in Cloud Optimized Point Cloud (COPC) format. Default: %(default)s");
-_("Filters the point cloud by keeping only a single point around a radius N (in meters). This can be useful to limit the output resolution of the point cloud and remove duplicate points. Set to 0 to disable sampling. Default: %(default)s");
-_("Perform ground rectification on the point cloud. This means that wrongly classified ground points will be re-classified and gaps will be filled. Useful for generating DTMs. Default: %(default)s");
-_("Delete heavy intermediate files to optimize disk space usage. This affects the ability to restart the pipeline from an intermediate stage, but allows datasets to be processed on machines that don't have sufficient disk space available. Default: %(default)s");
-_("Name of dataset (i.e subfolder name within project folder). Default: %(default)s");
-_("Automatically crop image outputs by creating a smooth buffer around the dataset boundaries, shrunk by N meters. Use 0 to disable cropping. Default: %(default)s");
-_("Turn off camera parameter optimization during bundle adjustment. This can be sometimes useful for improving results that exhibit doming/bowling or when images are taken with a rolling shutter camera. Default: %(default)s");
-_("Skip generation of a full 3D model. This can save time if you only need 2D results such as orthophotos and DEMs. Default: %(default)s");
-_("When processing multispectral datasets, you can specify the name of the primary band that will be used for reconstruction. It's recommended to choose a band which has sharp details and is in focus. Default: %(default)s");
-_("show this help message and exit");
-_("Copy output results to this folder after processing.");
-_("Path to a GeoTIFF DEM or a LAS/LAZ point cloud that the reconstruction outputs should be automatically aligned to. Experimental. Default: %(default)s");
-_("Skip generation of PDF report. This can save time if you don't need a report. Default: %(default)s");
-_("Octree depth used in the mesh reconstruction, increase to get more vertices, recommended values are 8-12. Default: %(default)s");
-_("Path to the file containing the ground control points used for georeferencing. The file needs to use the following format: EPSG:<code> or <+proj definition>geo_x geo_y geo_z im_x im_y image_name [gcp_name] [extra1] [extra2]Default: %(default)s");
-_("URL to a ClusterODM instance for distributing a split-merge workflow on multiple nodes in parallel. Default: %(default)s");
-_("Simple Morphological Filter elevation scalar parameter. Default: %(default)s");
-_("Permanently delete all previous results and rerun the processing pipeline.");
-_("Matcher algorithm, Fast Library for Approximate Nearest Neighbors or Bag of Words. FLANN is slower, but more stable. BOW is faster, but can sometimes miss valid matches. BRUTEFORCE is very slow but robust.Can be one of: %(choices)s. Default: %(default)s");
-_("Decimate the points before generating the DEM. 1 is no decimation (full quality). 100 decimates ~99%% of the points. Useful for speeding up generation of DEM results in very large datasets. Default: %(default)s");
-_("Maximum number of frames to extract from video files for processing. Set to 0 for no limit. Default: %(default)s");
-_("Use this tag if you have a GCP File but want to use the EXIF information for georeferencing instead. Default: %(default)s");
-_("Choose what to merge in the merge step in a split dataset. By default all available outputs are merged. Options: %(choices)s. Default: %(default)s");
-_("Filters the point cloud by removing points that deviate more than N standard deviations from the local mean. Set to 0 to disable filtering. Default: %(default)s");
-_("Simple Morphological Filter slope parameter (rise over run). Default: %(default)s");
+_("Override the rolling shutter readout time for your camera sensor (in milliseconds), instead of using the rolling shutter readout database. Note that not all cameras are present in the database. Set to 0 to use the database value. Default: %(default)s");
 _("The maximum vertex count of the output mesh. Default: %(default)s");
 _("Path to the project folder. Your project folder should contain subfolders for each dataset. Each dataset should have an \"images\" folder.");
-_("Minimum number of features to extract per image. More features can be useful for finding more matches between images, potentially allowing the reconstruction of areas with little overlap or insufficient features. More features also slow down processing. Default: %(default)s");
-_("Run local bundle adjustment for every image added to the reconstruction and a global adjustment every 100 images. Speeds up reconstruction for very large datasets. Default: %(default)s");
-_("Set point cloud quality. Higher quality generates better, denser point clouds, but requires more memory and takes longer. Each step up in quality increases processing time roughly by a factor of 4x.Can be one of: %(choices)s. Default: %(default)s");
-_("Do not use GPU acceleration, even if it's available. Default: %(default)s");
-_("Set feature extraction quality. Higher quality generates better features, but requires more memory and takes longer. Can be one of: %(choices)s. Default: %(default)s");
-_("The maximum output resolution of extracted video frames in pixels. Default: %(default)s");
-_("Average number of images per submodel. When splitting a large dataset into smaller submodels, images are grouped into clusters. This value regulates the number of images that each cluster should have on average. Default: %(default)s");
-_("Choose the structure from motion algorithm. For aerial datasets, if camera GPS positions and angles are available, triangulation can generate better results. For planar scenes captured at fixed altitude with nadir-only images, planar can be much faster. Can be one of: %(choices)s. Default: %(default)s");
 _("Skip normalization of colors across all images. Useful when processing radiometric data. Default: %(default)s");
-_("Perform image matching with the nearest N images based on image filename order. Can speed up processing of sequential images, such as those extracted from video. It is applied only on non-georeferenced datasets. Set to 0 to disable. Default: %(default)s");
-_("Set a camera projection type. Manually setting a value can help improve geometric undistortion. By default the application tries to determine a lens type from the images metadata. Can be one of: %(choices)s. Default: %(default)s");
-_("Set this parameter if you want to generate a PNG rendering of the orthophoto. Default: %(default)s");
-_("Use images' GPS exif data for reconstruction, even if there are GCPs present.This flag is useful if you have high precision GPS measurements. If there are no GCPs, this flag does nothing. Default: %(default)s");
-_("Set the radiometric calibration to perform on images. When processing multispectral and thermal images you should set this option to obtain reflectance/temperature values (otherwise you will get digital number values). [camera] applies black level, vignetting, row gradient gain/exposure compensation (if appropriate EXIF tags are found) and computes absolute temperature values. [camera+sun] is experimental, applies all the corrections of [camera], plus compensates for spectral radiance registered via a downwelling light sensor (DLS) taking in consideration the angle of the sun. Can be one of: %(choices)s. Default: %(default)s");
-_("Export the georeferenced point cloud in CSV format. Default: %(default)s");
-_("Set this parameter if you want a striped GeoTIFF. Default: %(default)s");
-_("Generate static tiles for orthophotos and DEMs that are suitable for viewers like Leaflet or OpenLayers. Default: %(default)s");
-_("Automatically set a boundary using camera shot locations to limit the area of the reconstruction. This can help remove far away background artifacts (sky, background landscapes, etc.). See also --boundary. Default: %(default)s");
-_("Number of steps used to fill areas with gaps. Set to 0 to disable gap filling. Starting with a radius equal to the output resolution, N different DEMs are generated with progressively bigger radius using the inverse distance weighted (IDW) algorithm and merged together. Remaining gaps are then merged using nearest neighbor interpolation. Default: %(default)s");
-_("Set this parameter if you want to generate a Google Earth (KMZ) rendering of the orthophoto. Default: %(default)s");
-_("Geometric estimates improve the accuracy of the point cloud by computing geometrically consistent depthmaps but may not be usable in larger datasets. This flag disables geometric estimates. Default: %(default)s");
-_("Set a value in meters for the GPS Dilution of Precision (DOP) information for all images. If your images are tagged with high precision GPS information (RTK), this value will be automatically set accordingly. You can use this option to manually set it in case the reconstruction fails. Lowering this option can sometimes help control bowling-effects over large areas. Default: %(default)s");
-_("When processing multispectral datasets, ODM will automatically align the images for each band. If the images have been postprocessed and are already aligned, use this option. Default: %(default)s");
-_("Choose the algorithm for extracting keypoints and computing descriptors. Can be one of: %(choices)s. Default: %(default)s");
 _("Generate single file Binary glTF (GLB) textured models. Default:  %(default)s");
-_("Orthophoto resolution in cm / pixel. Note that this value is capped by a ground sampling distance (GSD) estimate.Default: %(default)s");
-_("Skips dense reconstruction and 3D model generation. It generates an orthophoto directly from the sparse reconstruction. If you just need an orthophoto and do not need a full 3D model, turn on this option. Default: %(default)s");
+_("Set this parameter if you want to generate a PNG rendering of the orthophoto. Default: %(default)s");
+_("Generates a polygon around the cropping area that cuts the orthophoto around the edges of features. This polygon can be useful for stitching seamless mosaics with multiple overlapping orthophotos. Default: %(default)s");
+_("Set this parameter if you want a striped GeoTIFF. Default: %(default)s");
+_("Skip generation of the orthophoto. This can save time if you only need 3D results or DEMs. Default: %(default)s");
+_("The maximum number of processes to use in various processes. Peak memory requirement is ~1GB per thread and 2 megapixel image resolution. Default: %(default)s");
+_("Filters the point cloud by keeping only a single point around a radius N (in meters). This can be useful to limit the output resolution of the point cloud and remove duplicate points. Set to 0 to disable sampling. Default: %(default)s");
+_("GeoJSON polygon limiting the area of the reconstruction. Can be specified either as path to a GeoJSON file or as a JSON string representing the contents of a GeoJSON file. Default: %(default)s");

app/tests/test_api_task_import.py

@@ -217,3 +217,95 @@ class TestApiTask(BootTransactionTestCase):
             self.assertEqual(file_import_task.import_url, "file://all.zip")
             self.assertEqual(file_import_task.images_count, 1)
 
+    def test_backup(self):
+        client = APIClient()
+
+        with start_processing_node():
+            user = User.objects.get(username="testuser")
+            self.assertFalse(user.is_superuser)
+            project = Project.objects.create(
+                owner=user,
+                name="test backup"
+            )
+
+            image1 = open("app/fixtures/tiny_drone_image.jpg", 'rb')
+            image2 = open("app/fixtures/tiny_drone_image_2.jpg", 'rb')
+
+            # Create processing node
+            pnode = ProcessingNode.objects.create(hostname="localhost", port=11223)
+            client.login(username="testuser", password="test1234")
+
+            # Create task
+            res = client.post("/api/projects/{}/tasks/".format(project.id), {
+                'images': [image1, image2]
+            }, format="multipart")
+            image1.close()
+            image2.close()
+            task = Task.objects.get(id=res.data['id'])
+
+            # Wait for completion
+            c = 0
+            while c < 10:
+                worker.tasks.process_pending_tasks()
+                task.refresh_from_db()
+                if task.status == status_codes.COMPLETED:
+                    break
+                c += 1
+                time.sleep(1)
+
+            # Assign some fields
+            task.name = "Backup test"
+            task.potree_scene = {'saved': True}
+            task.public = True
+            task.save()
+
+            self.assertEqual(task.status, status_codes.COMPLETED)
+
+            # Download task backup
+            task_uuid = task.uuid
+            res = client.get("/api/projects/{}/tasks/{}/backup".format(project.id, task.id))
+            self.assertEqual(res.status_code, status.HTTP_200_OK)
+
+            if not os.path.exists(settings.MEDIA_TMP):
+                os.mkdir(settings.MEDIA_TMP)
+
+            assets_path = os.path.join(settings.MEDIA_TMP, "backup.zip")
+
+            with open(assets_path, 'wb') as f:
+                f.write(res.content)
+
+            assets_file = open(assets_path, 'rb')
+
+            # Import with file upload method
+            res = client.post("/api/projects/{}/tasks/import".format(project.id), {
+                'file': [assets_file]
+            }, format="multipart")
+            self.assertEqual(res.status_code, status.HTTP_201_CREATED)
+            assets_file.close()
+
+            file_import_task = Task.objects.get(id=res.data['id'])
+            # Wait for completion
+            c = 0
+            while c < 10:
+                worker.tasks.process_pending_tasks()
+                file_import_task.refresh_from_db()
+                if file_import_task.status == status_codes.COMPLETED:
+                    break
+                c += 1
+                time.sleep(1)
+
+            self.assertEqual(file_import_task.import_url, "file://all.zip")
+            self.assertEqual(file_import_task.images_count, 1)
+            self.assertEqual(file_import_task.processing_node, None)
+            self.assertEqual(file_import_task.auto_processing_node, False)
+            self.assertEqual(file_import_task.name, "Backup test")
+            self.assertTrue('saved' in file_import_task.potree_scene)
+            self.assertEqual(file_import_task.public, True)
+            
+            # Can access assets
+            res = client.get("/api/projects/{}/tasks/{}/assets/odm_orthophoto/odm_orthophoto.tif".format(project.id, file_import_task.id))
+            self.assertEqual(res.status_code, status.HTTP_200_OK)
+
+            self.assertTrue(valid_cogeo(file_import_task.assets_path(task.ASSETS_MAP["orthophoto.tif"])))
+            self.assertTrue(valid_cogeo(file_import_task.assets_path(task.ASSETS_MAP["dsm.tif"])))
+            self.assertTrue(valid_cogeo(file_import_task.assets_path(task.ASSETS_MAP["dtm.tif"])))

locale

@@ -1 +1 @@
-Subproject commit 79a4f1a68b90368a5ec944abec1ec2c18d37f169
+Subproject commit 599f937f7e98064f6c201d01259ec14293f885a5

package.json

@@ -1,6 +1,6 @@
 {
   "name": "WebODM",
-  "version": "2.5.1",
+  "version": "2.5.2",
   "description": "User-friendly, extendable application and API for processing aerial imagery.",
   "main": "index.js",
   "scripts": {