From 55712f0d582926bf3f3cb3913b94b80da7a9d29f Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 20 Feb 2019 16:42:20 -0500 Subject: [PATCH] Import task functionality poc --- app/api/tasks.py | 51 +++++++- app/api/urls.py | 5 +- app/migrations/0025_auto_20190220_1854.py | 52 +++++++++ app/models/task.py | 110 ++++++++++++------ app/pending_actions.py | 1 + app/static/app/js/classes/PendingActions.js | 7 +- .../app/js/components/ImportTaskPanel.jsx | 96 +++++++++++++++ .../app/js/components/ProjectListItem.jsx | 29 ++++- app/static/app/js/components/TaskListItem.jsx | 6 +- .../components/tests/ImportTaskPanel.test.jsx | 10 ++ app/static/app/js/css/ImportTaskPanel.scss | 20 ++++ nodeodm/migrations/0006_auto_20190220_1842.py | 24 ++++ nodeodm/models.py | 2 +- 13 files changed, 361 insertions(+), 52 deletions(-) create mode 100644 app/migrations/0025_auto_20190220_1854.py create mode 100644 app/static/app/js/components/ImportTaskPanel.jsx create mode 100644 app/static/app/js/components/tests/ImportTaskPanel.test.jsx create mode 100644 app/static/app/js/css/ImportTaskPanel.scss create mode 100644 nodeodm/migrations/0006_auto_20190220_1842.py diff --git a/app/api/tasks.py b/app/api/tasks.py index 4afd6c96..32645751 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -2,6 +2,8 @@ import os from wsgiref.util import FileWrapper import mimetypes + +import datetime from django.core.exceptions import ObjectDoesNotExist, SuspiciousFileOperation, ValidationError from django.db import transaction from django.http import FileResponse @@ -13,11 +15,19 @@ from rest_framework.response import Response from rest_framework.views import APIView from app import models, pending_actions +from nodeodm import status_codes from nodeodm.models import ProcessingNode from worker import tasks as worker_tasks from .common import get_and_check_project, get_tile_json, path_traversal_check +def flatten_files(request_files): + # MultiValueDict in, flat array of files out + return [file for filesList in map( + lambda key: request_files.getlist(key), + [keys for keys in request_files]) + for file in filesList] + class TaskIDsSerializer(serializers.BaseSerializer): def to_representation(self, obj): return obj.id @@ -36,6 +46,7 @@ class TaskSerializer(serializers.ModelSerializer): return None def get_images_count(self, obj): + # TODO: create a field in the model for this return obj.imageupload_set.count() def get_can_rerun_from(self, obj): @@ -142,11 +153,7 @@ class TaskViewSet(viewsets.ViewSet): def create(self, request, project_pk=None): project = get_and_check_project(request, project_pk, ('change_project', )) - # MultiValueDict in, flat array of files out - files = [file for filesList in map( - lambda key: request.FILES.getlist(key), - [keys for keys in request.FILES]) - for file in filesList] + files = flatten_files(request.FILES) if len(files) <= 1: raise exceptions.ValidationError(detail="Cannot create task, you need at least 2 images") @@ -322,3 +329,37 @@ class TaskAssets(TaskNestedView): raise exceptions.NotFound("Asset does not exist") return download_file_response(request, asset_path, 'inline') + +""" +Task assets import +""" +class TaskAssetsImport(APIView): + permission_classes = (permissions.AllowAny,) + parser_classes = (parsers.MultiPartParser, parsers.JSONParser, parsers.FormParser,) + + def post(self, request, project_pk=None): + project = get_and_check_project(request, project_pk, ('change_project',)) + + files = flatten_files(request.FILES) + + if len(files) != 1: + raise exceptions.ValidationError(detail="Cannot create task, you need to upload 1 file") + + with transaction.atomic(): + task = models.Task.objects.create(project=project, + auto_processing_node=False, + name="Imported Task", + import_url="file://all.zip", + status=status_codes.RUNNING, + pending_action=pending_actions.IMPORT) + task.create_task_directories() + + destination_file = task.assets_path("all.zip") + with open(destination_file, 'wb+') as fd: + for chunk in files[0].chunks(): + fd.write(chunk) + + worker_tasks.process_task.delay(task.id) + + serializer = TaskSerializer(task) + return Response(serializer.data, status=status.HTTP_201_CREATED) diff --git a/app/api/urls.py b/app/api/urls.py index f9e1b1d6..13f94d6c 100644 --- a/app/api/urls.py +++ b/app/api/urls.py @@ -3,7 +3,7 @@ from django.conf.urls import url, include from app.api.presets import PresetViewSet from app.plugins import get_api_url_patterns from .projects import ProjectViewSet -from .tasks import TaskViewSet, TaskTiles, TaskTilesJson, TaskDownloads, TaskAssets +from .tasks import TaskViewSet, TaskTiles, TaskTilesJson, TaskDownloads, TaskAssets, TaskAssetsImport from .processingnodes import ProcessingNodeViewSet, ProcessingNodeOptionsView from rest_framework_nested import routers from rest_framework_jwt.views import obtain_jwt_token @@ -25,10 +25,9 @@ urlpatterns = [ url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/(?Porthophoto|dsm|dtm)/tiles/(?P[\d]+)/(?P[\d]+)/(?P[\d]+)\.png$', TaskTiles.as_view()), url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/(?Porthophoto|dsm|dtm)/tiles\.json$', TaskTilesJson.as_view()), - url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/download/(?P.+)$', TaskDownloads.as_view()), - url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/assets/(?P.+)$', TaskAssets.as_view()), + url(r'projects/(?P[^/.]+)/tasks/import$', TaskAssetsImport.as_view()), url(r'^auth/', include('rest_framework.urls')), url(r'^token-auth/', obtain_jwt_token), diff --git a/app/migrations/0025_auto_20190220_1854.py b/app/migrations/0025_auto_20190220_1854.py new file mode 100644 index 00000000..2bbf86ce --- /dev/null +++ b/app/migrations/0025_auto_20190220_1854.py @@ -0,0 +1,52 @@ +# Generated by Django 2.1.5 on 2019-02-20 18:54 + +import app.models.task +import colorfield.fields +import django.contrib.postgres.fields +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0024_update_task_assets'), + ] + + operations = [ + migrations.AddField( + model_name='task', + name='import_url', + field=models.TextField(blank=True, default='', help_text='URL this task is imported from (only for imported tasks)'), + ), + migrations.AlterField( + model_name='preset', + name='options', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=list, help_text="Options that define this preset (same format as in a Task's options).", validators=[app.models.task.validate_task_options]), + ), + migrations.AlterField( + model_name='task', + name='available_assets', + field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=80), blank=True, default=list, help_text='List of available assets to download', size=None), + ), + migrations.AlterField( + model_name='task', + name='options', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict, help_text='Options that are being used to process this task', validators=[app.models.task.validate_task_options]), + ), + migrations.AlterField( + model_name='task', + name='pending_action', + field=models.IntegerField(blank=True, choices=[(1, 'CANCEL'), (2, 'REMOVE'), (3, 'RESTART'), (4, 'RESIZE'), (5, 'IMPORT')], db_index=True, help_text='A requested action to be performed on the task. The selected action will be performed by the worker at the next iteration.', null=True), + ), + migrations.AlterField( + model_name='theme', + name='header_background', + field=colorfield.fields.ColorField(default='#3498db', help_text="Background color of the site's header.", max_length=18), + ), + migrations.AlterField( + model_name='theme', + name='tertiary', + field=colorfield.fields.ColorField(default='#3498db', help_text='Navigation links.', max_length=18), + ), + ] diff --git a/app/models/task.py b/app/models/task.py index 759f4a16..a654d5b0 100644 --- a/app/models/task.py +++ b/app/models/task.py @@ -8,6 +8,7 @@ import uuid as uuid_module import json from shlex import quote +import errno import piexif import re @@ -170,6 +171,7 @@ class Task(models.Model): (pending_actions.REMOVE, 'REMOVE'), (pending_actions.RESTART, 'RESTART'), (pending_actions.RESIZE, 'RESIZE'), + (pending_actions.IMPORT, 'IMPORT'), ) # Not an exact science @@ -223,6 +225,8 @@ class Task(models.Model): running_progress = models.FloatField(default=0.0, help_text="Value between 0 and 1 indicating the running progress (estimated) of this task", blank=True) + import_url = models.TextField(null=False, default="", blank=True, help_text="URL this task is imported from (only for imported tasks)") + def __init__(self, *args, **kwargs): super(Task, self).__init__(*args, **kwargs) @@ -333,6 +337,14 @@ class Task(models.Model): else: raise FileNotFoundError("{} is not a valid asset".format(asset)) + def handle_import(self): + self.console_output += "Importing assets...\n" + self.save() + self.extract_assets_and_complete() + self.pending_action = None + self.processing_time = 0 + self.save() + def process(self): """ This method contains the logic for processing tasks asynchronously @@ -342,6 +354,9 @@ class Task(models.Model): """ try: + if self.pending_action == pending_actions.IMPORT: + self.handle_import() + if self.pending_action == pending_actions.RESIZE: resized_images = self.resize_images() self.refresh_from_db() @@ -571,42 +586,7 @@ class Task(models.Model): self.console_output += "Extracting results. This could take a few minutes...\n"; self.save() - # Extract from zip - with zipfile.ZipFile(zip_path, "r") as zip_h: - zip_h.extractall(assets_dir) - - logger.info("Extracted all.zip for {}".format(self)) - - # Populate *_extent fields - extent_fields = [ - (os.path.realpath(self.assets_path("odm_orthophoto", "odm_orthophoto.tif")), - 'orthophoto_extent'), - (os.path.realpath(self.assets_path("odm_dem", "dsm.tif")), - 'dsm_extent'), - (os.path.realpath(self.assets_path("odm_dem", "dtm.tif")), - 'dtm_extent'), - ] - - for raster_path, field in extent_fields: - if os.path.exists(raster_path): - # Read extent and SRID - raster = GDALRaster(raster_path) - extent = OGRGeometry.from_bbox(raster.extent) - - # It will be implicitly transformed into the SRID of the model’s field - # self.field = GEOSGeometry(...) - setattr(self, field, GEOSGeometry(extent.wkt, srid=raster.srid)) - - logger.info("Populated extent field with {} for {}".format(raster_path, self)) - - self.update_available_assets_field() - self.running_progress = 1.0 - self.console_output += "Done!\n" - self.status = status_codes.COMPLETED - self.save() - - from app.plugins import signals as plugin_signals - plugin_signals.task_completed.send_robust(sender=self.__class__, task_id=self.id) + self.extract_assets_and_complete() else: # FAILED, CANCELED self.save() @@ -624,6 +604,51 @@ class Task(models.Model): # Task was interrupted during image resize / upload logger.warning("{} interrupted".format(self, str(e))) + def extract_assets_and_complete(self): + """ + Extracts assets/all.zip and populates task fields where required. + :return: + """ + assets_dir = self.assets_path("") + zip_path = self.assets_path("all.zip") + + # Extract from zip + with zipfile.ZipFile(zip_path, "r") as zip_h: + zip_h.extractall(assets_dir) + + logger.info("Extracted all.zip for {}".format(self)) + + # Populate *_extent fields + extent_fields = [ + (os.path.realpath(self.assets_path("odm_orthophoto", "odm_orthophoto.tif")), + 'orthophoto_extent'), + (os.path.realpath(self.assets_path("odm_dem", "dsm.tif")), + 'dsm_extent'), + (os.path.realpath(self.assets_path("odm_dem", "dtm.tif")), + 'dtm_extent'), + ] + + for raster_path, field in extent_fields: + if os.path.exists(raster_path): + # Read extent and SRID + raster = GDALRaster(raster_path) + extent = OGRGeometry.from_bbox(raster.extent) + + # It will be implicitly transformed into the SRID of the model’s field + # self.field = GEOSGeometry(...) + setattr(self, field, GEOSGeometry(extent.wkt, srid=raster.srid)) + + logger.info("Populated extent field with {} for {}".format(raster_path, self)) + + self.update_available_assets_field() + self.running_progress = 1.0 + self.console_output += "Done!\n" + self.status = status_codes.COMPLETED + self.save() + + from app.plugins import signals as plugin_signals + plugin_signals.task_completed.send_robust(sender=self.__class__, task_id=self.id) + def get_tile_path(self, tile_type, z, x, y): return self.assets_path("{}_tiles".format(tile_type), z, x, "{}.png".format(y)) @@ -783,3 +808,16 @@ class Task(models.Model): except subprocess.CalledProcessError as e: logger.warning("Could not resize GCP file {}: {}".format(gcp_path, str(e))) return None + + def create_task_directories(self): + """ + Create directories for this task (if they don't exist already) + """ + assets_dir = self.assets_path("") + try: + os.makedirs(assets_dir) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST and os.path.isdir(assets_dir): + pass + else: + raise diff --git a/app/pending_actions.py b/app/pending_actions.py index 79c0cc6b..c7f02532 100644 --- a/app/pending_actions.py +++ b/app/pending_actions.py @@ -2,3 +2,4 @@ CANCEL = 1 REMOVE = 2 RESTART = 3 RESIZE = 4 +IMPORT = 5 diff --git a/app/static/app/js/classes/PendingActions.js b/app/static/app/js/classes/PendingActions.js index b69f1728..69f3c9a8 100644 --- a/app/static/app/js/classes/PendingActions.js +++ b/app/static/app/js/classes/PendingActions.js @@ -1,7 +1,8 @@ const CANCEL = 1, REMOVE = 2, RESTART = 3, - RESIZE = 4; + RESIZE = 4, + IMPORT = 5; let pendingActions = { [CANCEL]: { @@ -15,6 +16,9 @@ let pendingActions = { }, [RESIZE]: { descr: "Resizing images..." + }, + [IMPORT]: { + descr: "Importing..." } }; @@ -23,6 +27,7 @@ export default { REMOVE: REMOVE, RESTART: RESTART, RESIZE: RESIZE, + IMPORT: IMPORT, description: function(pendingAction) { if (pendingActions[pendingAction]) return pendingActions[pendingAction].descr; diff --git a/app/static/app/js/components/ImportTaskPanel.jsx b/app/static/app/js/components/ImportTaskPanel.jsx new file mode 100644 index 00000000..d39094e6 --- /dev/null +++ b/app/static/app/js/components/ImportTaskPanel.jsx @@ -0,0 +1,96 @@ +import '../css/ImportTaskPanel.scss'; +import React from 'react'; +import PropTypes from 'prop-types'; +import Dropzone from '../vendor/dropzone'; +import csrf from '../django/csrf'; + +class ImportTaskPanel extends React.Component { + static defaultProps = { + }; + + static propTypes = { + // onSave: PropTypes.func.isRequired, + onCancel: PropTypes.func, + projectId: PropTypes.number.isRequired + }; + + constructor(props){ + super(props); + + this.state = { + }; + } + + componentDidMount(){ + Dropzone.autoDiscover = false; + + this.dz = new Dropzone(this.dropzone, { + paramName: "file", + url : `/api/projects/${this.props.projectId}/tasks/import`, + parallelUploads: 1, + uploadMultiple: false, + acceptedFiles: "application/zip", + autoProcessQueue: true, + createImageThumbnails: false, + previewTemplate: '
', + clickable: this.uploadButton, + chunkSize: 2147483647, + timeout: 2147483647, + + headers: { + [csrf.header]: csrf.token + } + }); + + this.dz.on("error", function(file){ + // Show + }) + .on("uploadprogress", function(file, progress){ + console.log(progress); + }) + .on("complete", function(file){ + if (file.status === "success"){ + }else{ + // error + } + }); + } + + cancel = (e) => { + this.props.onCancel(); + } + + setRef = (prop) => { + return (domNode) => { + if (domNode != null) this[prop] = domNode; + } + } + + render() { + return ( +
+
+ +

Import Existing Assets

+

You can import .zip files that have been exported from existing tasks via Download Assets All Assets.

+ + +
+
+ ); + } +} + +export default ImportTaskPanel; diff --git a/app/static/app/js/components/ProjectListItem.jsx b/app/static/app/js/components/ProjectListItem.jsx index 2ee1a5a6..15fece74 100644 --- a/app/static/app/js/components/ProjectListItem.jsx +++ b/app/static/app/js/components/ProjectListItem.jsx @@ -3,6 +3,7 @@ import React from 'react'; import update from 'immutability-helper'; import TaskList from './TaskList'; import NewTaskPanel from './NewTaskPanel'; +import ImportTaskPanel from './ImportTaskPanel'; import UploadProgressBar from './UploadProgressBar'; import ProgressBar from './ProgressBar'; import ErrorMessage from './ErrorMessage'; @@ -32,7 +33,8 @@ class ProjectListItem extends React.Component { upload: this.getDefaultUploadState(), error: "", data: props.data, - refreshing: false + refreshing: false, + importing: false }; this.toggleTaskList = this.toggleTaskList.bind(this); @@ -335,6 +337,14 @@ class ProjectListItem extends React.Component { location.href = `/map/project/${this.state.data.id}/`; } + handleImportTask = () => { + this.setState({importing: true}); + } + + handleCancelImportTask = () => { + this.setState({importing: false}); + } + render() { const { refreshing, data } = this.state; const numTasks = data.tasks.length; @@ -361,13 +371,17 @@ class ProjectListItem extends React.Component {
{this.hasPermission("add") ? - + +
: ""}