diff --git a/app/api/tasks.py b/app/api/tasks.py index 302e7524..23ac7c64 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -29,18 +29,14 @@ class TaskSerializer(serializers.ModelSerializer): project = serializers.PrimaryKeyRelatedField(queryset=models.Project.objects.all()) processing_node = serializers.PrimaryKeyRelatedField(queryset=ProcessingNode.objects.all()) images_count = serializers.SerializerMethodField() - available_assets = serializers.SerializerMethodField() def get_images_count(self, obj): return obj.imageupload_set.count() - def get_available_assets(self, obj): - return obj.get_available_assets() - class Meta: model = models.Task exclude = ('processing_lock', 'console_output', 'orthophoto_extent', ) - read_only_fields = ('processing_time', 'status', 'last_error', 'created_at', 'pending_action', ) + read_only_fields = ('processing_time', 'status', 'last_error', 'created_at', 'pending_action', 'available_assets', ) class TaskViewSet(viewsets.ViewSet): """ @@ -240,7 +236,7 @@ class TaskDownloads(TaskNestedView): file = open(asset_path, "rb") response = HttpResponse(FileWrapper(file), content_type=(mimetypes.guess_type(asset_filename)[0] or "application/zip")) - response['Content-Disposition'] = "attachment; filename={}".format(asset_filename) + response['Content-Disposition'] = "attachment; filename={}".format(asset) return response """ diff --git a/app/api/urls.py b/app/api/urls.py index 7d3fb7f8..8f1b26a4 100644 --- a/app/api/urls.py +++ b/app/api/urls.py @@ -20,7 +20,8 @@ urlpatterns = [ url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/tiles/(?P[\d]+)/(?P[\d]+)/(?P[\d]+)\.png$', TaskTiles.as_view()), url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/tiles\.json$', TaskTilesJson.as_view()), - url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/download/(?P[^/.]+)/$', TaskDownloads.as_view()), + url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/download/(?P.+)$', TaskDownloads.as_view()), + url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/assets/(?P.+)$', TaskAssets.as_view()), url(r'^auth/', include('rest_framework.urls')), diff --git a/app/migrations/0006_task_available_assets.py b/app/migrations/0006_task_available_assets.py new file mode 100644 index 00000000..c1c44b10 --- /dev/null +++ b/app/migrations/0006_task_available_assets.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.1 on 2017-07-07 18:05 +from __future__ import unicode_literals + +import django.contrib.postgres.fields +from django.db import migrations, models +from app.models import Task + + +def detect_available_assets(apps, schema_editor): + for t in Task.objects.all(): + print("Updating {}".format(t)) + t.update_available_assets_field(True) + + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0005_auto_20170707_1014'), + ] + + operations = [ + migrations.AddField( + model_name='task', + name='available_assets', + field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=80), blank=True, default=[], help_text='List of available assets to download', size=None), + ), + migrations.RunPython(detect_available_assets), + ] diff --git a/app/models.py b/app/models.py index 4555f461..fbf60c28 100644 --- a/app/models.py +++ b/app/models.py @@ -120,7 +120,18 @@ def validate_task_options(value): class Task(models.Model): - ASSET_DOWNLOADS = ("all", "geotiff", "texturedmodel", "las", "csv", "ply",) + ASSETS_MAP = { + 'all.zip': 'all.zip', + 'orthophoto.tif': os.path.join('odm_orthophoto', 'odm_orthophoto.tif'), + 'orthophoto.png': os.path.join('odm_orthophoto', 'odm_orthophoto.png'), + 'georeferenced_model.las': os.path.join('odm_georeferencing', 'odm_georeferenced_model.las'), + 'georeferenced_model.ply': os.path.join('odm_georeferencing', 'odm_georeferenced_model.ply'), + 'georeferenced_model.csv': os.path.join('odm_georeferencing', 'odm_georeferenced_model.csv'), + 'textured_model.zip': { + 'deferred_path': 'textured_model.zip', + 'deferred_compress_dir': 'odm_texturing' + } + } STATUS_CODES = ( (status_codes.QUEUED, 'QUEUED'), @@ -146,6 +157,7 @@ class Task(models.Model): status = models.IntegerField(choices=STATUS_CODES, db_index=True, null=True, blank=True, help_text="Current status of the task") last_error = models.TextField(null=True, blank=True, help_text="The last processing error received") options = fields.JSONField(default=dict(), blank=True, help_text="Options that are being used to process this task", validators=[validate_task_options]) + available_assets = fields.ArrayField(models.CharField(max_length=80), default=list(), blank=True, help_text="List of available assets to download") console_output = models.TextField(null=False, default="", blank=True, help_text="Console output of the OpenDroneMap's process") ground_control_points = models.FileField(null=True, blank=True, upload_to=gcp_directory_path, help_text="Optional Ground Control Points file to use for processing") @@ -196,7 +208,7 @@ class Task(models.Model): logger.warning("Project changed for task {}, but either {} doesn't exist, or {} already exists. This doesn't look right, so we will not move any files.".format(self, old_task_folder, new_task_folder)) - except (shutil.Error, GDALException) as e: + except shutil.Error as e: logger.warning("Could not move assets folder for task {}. We're going to proceed anyway, but you might experience issues: {}".format(self, e)) def save(self, *args, **kwargs): @@ -218,35 +230,46 @@ class Task(models.Model): "assets", *args) + def is_asset_available_slow(self, asset): + """ + Checks whether a particular asset is available in the file system + Generally this should never be used directly, as it's slow. Use the available_assets field + in the database instead. + :param asset: one of ASSETS_MAP keys + :return: boolean + """ + if asset in self.ASSETS_MAP: + value = self.ASSETS_MAP[asset] + if isinstance(value, str): + return os.path.exists(self.assets_path(value)) + elif isinstance(value, dict): + if 'deferred_compress_dir' in value: + return os.path.exists(self.assets_path(value['deferred_compress_dir'])) + + return False + def get_asset_download_path(self, asset): """ Get the path to an asset download - :param asset: one of ASSET_DOWNLOADS + :param asset: one of ASSETS_MAP keys :return: path """ - if asset == 'texturedmodel': - return self.assets_path(os.path.basename(self.get_textured_model_archive())) - else: - map = { - 'all': 'all.zip', - 'geotiff': os.path.join('odm_orthophoto', 'odm_orthophoto.tif'), - 'las': os.path.join('odm_georeferencing', 'odm_georeferenced_model.las'), - 'ply': os.path.join('odm_georeferencing', 'odm_georeferenced_model.ply'), - 'csv': os.path.join('odm_georeferencing', 'odm_georeferenced_model.csv') - } - # BEGIN MIGRATION - # Temporary check for naming migration from *model.ply.las to *model.las - # This can be deleted at some point in the future - if asset == 'las' and not os.path.exists(self.assets_path(map['las'])): - logger.info("migration: using odm_georeferenced_model.ply.las instead of odm_georeferenced_model.las") - map['las'] = os.path.join('odm_georeferencing', 'odm_georeferenced_model.ply.las') - # END MIGRATION + if asset in self.ASSETS_MAP: + value = self.ASSETS_MAP[asset] + if isinstance(value, str): + return self.assets_path(value) + + elif isinstance(value, dict): + if 'deferred_path' in value and 'deferred_compress_dir' in value: + return self.generate_deferred_asset(value['deferred_path'], value['deferred_compress_dir']) + else: + raise FileNotFoundError("{} is not a valid asset (invalid dict values)".format(asset)) - if asset in map: - return self.assets_path(map[asset]) else: - raise FileNotFoundError("{} is not a valid asset".format(asset)) + raise FileNotFoundError("{} is not a valid asset (invalid map)".format(asset)) + else: + raise FileNotFoundError("{} is not a valid asset".format(asset)) def process(self): """ @@ -390,8 +413,13 @@ class Task(models.Model): if self.status == status_codes.COMPLETED: assets_dir = self.assets_path("") - if not os.path.exists(assets_dir): - os.makedirs(assets_dir) + + # Remove previous assets directory + if os.path.exists(assets_dir): + logger.info("Removing old assets directory: {} for {}".format(assets_dir, self)) + shutil.rmtree(assets_dir) + + os.makedirs(assets_dir) logger.info("Downloading all.zip for {}".format(self)) @@ -422,11 +450,6 @@ class Task(models.Model): logger.info("Populated orthophoto_extent for {}".format(self)) - # Remove old odm_texturing.zip archive (if any) - textured_model_archive = self.assets_path(self.get_textured_model_filename()) - if os.path.exists(textured_model_archive): - os.remove(textured_model_archive) - self.save() else: # FAILED, CANCELED @@ -458,35 +481,32 @@ class Task(models.Model): } } - def get_textured_model_filename(self): - return "odm_texturing.zip" + def generate_deferred_asset(self, archive, directory): + """ + :param archive: path of the destination .zip file (relative to /assets/ directory) + :param directory: path of the source directory to compress (relative to /assets/ directory) + :return: full path of the generated archive + """ + archive_path = self.assets_path(archive) + directory_path = self.assets_path(directory) - def get_textured_model_archive(self): - archive_path = self.assets_path(self.get_textured_model_filename()) - textured_model_directory = self.assets_path("odm_texturing") - - if not os.path.exists(textured_model_directory): - raise FileNotFoundError("{} does not exist".format(textured_model_directory)) + if not os.path.exists(directory_path): + raise FileNotFoundError("{} does not exist".format(directory_path)) if not os.path.exists(archive_path): - shutil.make_archive(os.path.splitext(archive_path)[0], 'zip', textured_model_directory) + shutil.make_archive(os.path.splitext(archive_path)[0], 'zip', directory_path) return archive_path - def get_available_assets(self): - # We make some assumptions for the sake of speed - # as checking the filesystem would be slow - if self.status == status_codes.COMPLETED: - assets = list(self.ASSET_DOWNLOADS) + def update_available_assets_field(self, commit=False): + """ + Updates the available_assets field with the actual types of assets available + :param commit: when True also saves the model, otherwise the user should manually call save() + """ + all_assets = list(self.ASSETS_MAP.keys()) + self.available_assets = [asset for asset in all_assets if self.is_asset_available_slow(asset)] + if commit: self.save() - if self.orthophoto_extent is None: - assets.remove('geotiff') - assets.remove('las') - assets.remove('csv') - - return assets - else: - return [] def delete(self, using=None, keep_parents=False): directory_to_delete = os.path.join(settings.MEDIA_ROOT, diff --git a/app/static/app/js/ModelView.jsx b/app/static/app/js/ModelView.jsx index ddd4a347..bbeb7dc4 100644 --- a/app/static/app/js/ModelView.jsx +++ b/app/static/app/js/ModelView.jsx @@ -49,7 +49,7 @@ class ModelView extends React.Component { } hasGeoreferencedAssets(){ - return this.props.task.available_assets.indexOf('geotiff') !== -1; + return this.props.task.available_assets.indexOf('orthophoto.tif') !== -1; } objFilePath(){ @@ -169,7 +169,7 @@ class ModelView extends React.Component { // React render render(){ - const showSwitchModeButton = this.props.task.available_assets.indexOf('geotiff') !== -1; + const showSwitchModeButton = this.hasGeoreferencedAssets(); const hideWithTexturedModel = {display: this.state.showTexturedModel ? "none" : "block"}; return (
diff --git a/app/static/app/js/classes/AssetDownloads.js b/app/static/app/js/classes/AssetDownloads.js index 9189e2aa..bea0187b 100644 --- a/app/static/app/js/classes/AssetDownloads.js +++ b/app/static/app/js/classes/AssetDownloads.js @@ -6,7 +6,7 @@ class AssetDownload{ } downloadUrl(project_id, task_id){ - return `/api/projects/${project_id}/tasks/${task_id}/download/${this.asset}/`; + return `/api/projects/${project_id}/tasks/${task_id}/download/${this.asset}`; } get separator(){ @@ -31,13 +31,14 @@ class AssetDownloadSeparator extends AssetDownload{ const api = { all: function() { return [ - new AssetDownload("GeoTIFF","geotiff","fa fa-map-o"), - new AssetDownload("Textured Model","texturedmodel","fa fa-connectdevelop"), - new AssetDownload("LAS","las","fa fa-cube"), - new AssetDownload("PLY","ply","fa fa-cube"), - new AssetDownload("CSV","csv","fa fa-cube"), + new AssetDownload("Orthophoto (GeoTIFF)","orthophoto.tif","fa fa-map-o"), + new AssetDownload("Orthophoto (PNG)","orthophoto.png","fa fa-picture-o"), + new AssetDownload("Point Cloud (LAS)","georeferenced_model.las","fa fa-cube"), + new AssetDownload("Point Cloud (PLY)","georeferenced_model.ply","fa fa-cube"), + new AssetDownload("Point Cloud (CSV)","georeferenced_model.csv","fa fa-cube"), + new AssetDownload("Textured Model","textured_model.zip","fa fa-connectdevelop"), new AssetDownloadSeparator(), - new AssetDownload("All Assets","all","fa fa-file-archive-o") + new AssetDownload("All Assets","all.zip","fa fa-file-archive-o") ]; }, diff --git a/app/static/app/js/components/TaskListItem.jsx b/app/static/app/js/components/TaskListItem.jsx index e8ed59b6..41171ef2 100644 --- a/app/static/app/js/components/TaskListItem.jsx +++ b/app/static/app/js/components/TaskListItem.jsx @@ -257,7 +257,7 @@ class TaskListItem extends React.Component { let expanded = ""; if (this.state.expanded){ - let showGeotiffMissingWarning = false, + let showOrthophotoMissingWarning = false, showMemoryErrorWarning = this.state.memoryError && task.status == statusCodes.FAILED, showExitedWithCodeOneHints = task.last_error === "Process exited with code 1" && !showMemoryErrorWarning && task.status == statusCodes.FAILED, memoryErrorLink = this.isMacOS() ? "http://stackoverflow.com/a/39720010" : "https://docs.docker.com/docker-for-windows/#advanced"; @@ -270,12 +270,12 @@ class TaskListItem extends React.Component { }; if (task.status === statusCodes.COMPLETED){ - if (task.available_assets.indexOf("geotiff") !== -1){ + if (task.available_assets.indexOf("orthophoto.tif") !== -1){ addActionButton(" View Orthophoto", "btn-primary", "fa fa-globe", () => { location.href = `/map/project/${task.project}/task/${task.id}/`; }); }else{ - showGeotiffMissingWarning = true; + showOrthophotoMissingWarning = true; } addActionButton(" View 3D Model", "btn-primary", "fa fa-cube", () => { @@ -347,8 +347,8 @@ class TaskListItem extends React.Component { : ""} {/* TODO: List of images? */} - {showGeotiffMissingWarning ? -
An orthophoto could not be generated. To generate one, make sure GPS information is embedded in the EXIF tags of your images.
: ""} + {showOrthophotoMissingWarning ? +
An orthophoto could not be generated. To generate one, make sure GPS information is embedded in the EXIF tags of your images, or use a Ground Control Points (GCP) file.
: ""}
diff --git a/app/views.py b/app/views.py index 26f834e5..9a04bfd4 100644 --- a/app/views.py +++ b/app/views.py @@ -85,7 +85,7 @@ def model_display(request, project_pk=None, task_pk=None): 'task': json.dumps({ 'id': task.id, 'project': project.id, - 'available_assets': task.get_available_assets() + 'available_assets': task.available_assets }) }.items() })