Import task functionality poc

app/api/tasks.py

@@ -2,6 +2,8 @@ import os
 from wsgiref.util import FileWrapper
 
 import mimetypes
+
+import datetime
 from django.core.exceptions import ObjectDoesNotExist, SuspiciousFileOperation, ValidationError
 from django.db import transaction
 from django.http import FileResponse
@@ -13,11 +15,19 @@ from rest_framework.response import Response
 from rest_framework.views import APIView
 
 from app import models, pending_actions
+from nodeodm import status_codes
 from nodeodm.models import ProcessingNode
 from worker import tasks as worker_tasks
 from .common import get_and_check_project, get_tile_json, path_traversal_check
 
 
+def flatten_files(request_files):
+    # MultiValueDict in, flat array of files out
+    return [file for filesList in map(
+        lambda key: request_files.getlist(key),
+        [keys for keys in request_files])
+     for file in filesList]
+
 class TaskIDsSerializer(serializers.BaseSerializer):
     def to_representation(self, obj):
         return obj.id
@@ -36,6 +46,7 @@ class TaskSerializer(serializers.ModelSerializer):
             return None
 
     def get_images_count(self, obj):
+        # TODO: create a field in the model for this
         return obj.imageupload_set.count()
 
     def get_can_rerun_from(self, obj):
@@ -142,11 +153,7 @@ class TaskViewSet(viewsets.ViewSet):
     def create(self, request, project_pk=None):
         project = get_and_check_project(request, project_pk, ('change_project', ))
         
-        # MultiValueDict in, flat array of files out
+        files = flatten_files(request.FILES)
-        files = [file for filesList in map(
-                        lambda key: request.FILES.getlist(key), 
-                        [keys for keys in request.FILES])
-                    for file in filesList]
 
         if len(files) <= 1:
             raise exceptions.ValidationError(detail="Cannot create task, you need at least 2 images")
@@ -322,3 +329,37 @@ class TaskAssets(TaskNestedView):
             raise exceptions.NotFound("Asset does not exist")
 
         return download_file_response(request, asset_path, 'inline')
+
+"""
+Task assets import
+"""
+class TaskAssetsImport(APIView):
+    permission_classes = (permissions.AllowAny,)
+    parser_classes = (parsers.MultiPartParser, parsers.JSONParser, parsers.FormParser,)
+
+    def post(self, request, project_pk=None):
+        project = get_and_check_project(request, project_pk, ('change_project',))
+
+        files = flatten_files(request.FILES)
+
+        if len(files) != 1:
+            raise exceptions.ValidationError(detail="Cannot create task, you need to upload 1 file")
+
+        with transaction.atomic():
+            task = models.Task.objects.create(project=project,
+                                              auto_processing_node=False,
+                                              name="Imported Task",
+                                              import_url="file://all.zip",
+                                              status=status_codes.RUNNING,
+                                              pending_action=pending_actions.IMPORT)
+            task.create_task_directories()
+
+            destination_file = task.assets_path("all.zip")
+            with open(destination_file, 'wb+') as fd:
+                for chunk in files[0].chunks():
+                    fd.write(chunk)
+
+            worker_tasks.process_task.delay(task.id)
+
+        serializer = TaskSerializer(task)
+        return Response(serializer.data, status=status.HTTP_201_CREATED)

app/api/urls.py

@@ -3,7 +3,7 @@ from django.conf.urls import url, include
 from app.api.presets import PresetViewSet
 from app.plugins import get_api_url_patterns
 from .projects import ProjectViewSet
-from .tasks import TaskViewSet, TaskTiles, TaskTilesJson, TaskDownloads, TaskAssets
+from .tasks import TaskViewSet, TaskTiles, TaskTilesJson, TaskDownloads, TaskAssets, TaskAssetsImport
 from .processingnodes import ProcessingNodeViewSet, ProcessingNodeOptionsView
 from rest_framework_nested import routers
 from rest_framework_jwt.views import obtain_jwt_token
@@ -25,10 +25,9 @@ urlpatterns = [
 
     url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/(?P<tile_type>orthophoto|dsm|dtm)/tiles/(?P<z>[\d]+)/(?P<x>[\d]+)/(?P<y>[\d]+)\.png$', TaskTiles.as_view()),
     url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/(?P<tile_type>orthophoto|dsm|dtm)/tiles\.json$', TaskTilesJson.as_view()),
-
     url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/download/(?P<asset>.+)$', TaskDownloads.as_view()),
-
     url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/assets/(?P<unsafe_asset_path>.+)$', TaskAssets.as_view()),
+    url(r'projects/(?P<project_pk>[^/.]+)/tasks/import$', TaskAssetsImport.as_view()),
 
     url(r'^auth/', include('rest_framework.urls')),
     url(r'^token-auth/', obtain_jwt_token),

app/migrations/0025_auto_20190220_1854.py

@@ -0,0 +1,52 @@
+# Generated by Django 2.1.5 on 2019-02-20 18:54
+
+import app.models.task
+import colorfield.fields
+import django.contrib.postgres.fields
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('app', '0024_update_task_assets'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='task',
+            name='import_url',
+            field=models.TextField(blank=True, default='', help_text='URL this task is imported from (only for imported tasks)'),
+        ),
+        migrations.AlterField(
+            model_name='preset',
+            name='options',
+            field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=list, help_text="Options that define this preset (same format as in a Task's options).", validators=[app.models.task.validate_task_options]),
+        ),
+        migrations.AlterField(
+            model_name='task',
+            name='available_assets',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=80), blank=True, default=list, help_text='List of available assets to download', size=None),
+        ),
+        migrations.AlterField(
+            model_name='task',
+            name='options',
+            field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict, help_text='Options that are being used to process this task', validators=[app.models.task.validate_task_options]),
+        ),
+        migrations.AlterField(
+            model_name='task',
+            name='pending_action',
+            field=models.IntegerField(blank=True, choices=[(1, 'CANCEL'), (2, 'REMOVE'), (3, 'RESTART'), (4, 'RESIZE'), (5, 'IMPORT')], db_index=True, help_text='A requested action to be performed on the task. The selected action will be performed by the worker at the next iteration.', null=True),
+        ),
+        migrations.AlterField(
+            model_name='theme',
+            name='header_background',
+            field=colorfield.fields.ColorField(default='#3498db', help_text="Background color of the site's header.", max_length=18),
+        ),
+        migrations.AlterField(
+            model_name='theme',
+            name='tertiary',
+            field=colorfield.fields.ColorField(default='#3498db', help_text='Navigation links.', max_length=18),
+        ),
+    ]

app/models/task.py

@@ -8,6 +8,7 @@ import uuid as uuid_module
 import json
 from shlex import quote
 
+import errno
 import piexif
 import re
 
@@ -170,6 +171,7 @@ class Task(models.Model):
         (pending_actions.REMOVE, 'REMOVE'),
         (pending_actions.RESTART, 'RESTART'),
         (pending_actions.RESIZE, 'RESIZE'),
+        (pending_actions.IMPORT, 'IMPORT'),
     )
 
     # Not an exact science
@@ -223,6 +225,8 @@ class Task(models.Model):
     running_progress = models.FloatField(default=0.0,
                                         help_text="Value between 0 and 1 indicating the running progress (estimated) of this task",
                                         blank=True)
+    import_url = models.TextField(null=False, default="", blank=True, help_text="URL this task is imported from (only for imported tasks)")
+
 
     def __init__(self, *args, **kwargs):
         super(Task, self).__init__(*args, **kwargs)
@@ -333,6 +337,14 @@ class Task(models.Model):
         else:
             raise FileNotFoundError("{} is not a valid asset".format(asset))
 
+    def handle_import(self):
+        self.console_output += "Importing assets...\n"
+        self.save()
+        self.extract_assets_and_complete()
+        self.pending_action = None
+        self.processing_time = 0
+        self.save()
+
     def process(self):
         """
         This method contains the logic for processing tasks asynchronously
@@ -342,6 +354,9 @@ class Task(models.Model):
         """
 
         try:
+            if self.pending_action == pending_actions.IMPORT:
+                self.handle_import()
+
             if self.pending_action == pending_actions.RESIZE:
                 resized_images = self.resize_images()
                 self.refresh_from_db()
@@ -571,42 +586,7 @@ class Task(models.Model):
                             self.console_output += "Extracting results. This could take a few minutes...\n";
                             self.save()
 
-                            # Extract from zip
+                            self.extract_assets_and_complete()
-                            with zipfile.ZipFile(zip_path, "r") as zip_h:
-                                zip_h.extractall(assets_dir)
-
-                            logger.info("Extracted all.zip for {}".format(self))
-
-                            # Populate *_extent fields
-                            extent_fields = [
-                                (os.path.realpath(self.assets_path("odm_orthophoto", "odm_orthophoto.tif")),
-                                 'orthophoto_extent'),
-                                (os.path.realpath(self.assets_path("odm_dem", "dsm.tif")),
-                                 'dsm_extent'),
-                                (os.path.realpath(self.assets_path("odm_dem", "dtm.tif")),
-                                 'dtm_extent'),
-                            ]
-
-                            for raster_path, field in extent_fields:
-                                if os.path.exists(raster_path):
-                                    # Read extent and SRID
-                                    raster = GDALRaster(raster_path)
-                                    extent = OGRGeometry.from_bbox(raster.extent)
-
-                                    # It will be implicitly transformed into the SRID of the model’s field
-                                    # self.field = GEOSGeometry(...)
-                                    setattr(self, field, GEOSGeometry(extent.wkt, srid=raster.srid))
-
-                                    logger.info("Populated extent field with {} for {}".format(raster_path, self))
-
-                            self.update_available_assets_field()
-                            self.running_progress = 1.0
-                            self.console_output += "Done!\n"
-                            self.status = status_codes.COMPLETED
-                            self.save()
-
-                            from app.plugins import signals as plugin_signals
-                            plugin_signals.task_completed.send_robust(sender=self.__class__, task_id=self.id)
                         else:
                             # FAILED, CANCELED
                             self.save()
@@ -624,6 +604,51 @@ class Task(models.Model):
             # Task was interrupted during image resize / upload
             logger.warning("{} interrupted".format(self, str(e)))
 
+    def extract_assets_and_complete(self):
+        """
+        Extracts assets/all.zip and populates task fields where required.
+        :return:
+        """
+        assets_dir = self.assets_path("")
+        zip_path = self.assets_path("all.zip")
+
+        # Extract from zip
+        with zipfile.ZipFile(zip_path, "r") as zip_h:
+            zip_h.extractall(assets_dir)
+
+        logger.info("Extracted all.zip for {}".format(self))
+
+        # Populate *_extent fields
+        extent_fields = [
+            (os.path.realpath(self.assets_path("odm_orthophoto", "odm_orthophoto.tif")),
+             'orthophoto_extent'),
+            (os.path.realpath(self.assets_path("odm_dem", "dsm.tif")),
+             'dsm_extent'),
+            (os.path.realpath(self.assets_path("odm_dem", "dtm.tif")),
+             'dtm_extent'),
+        ]
+
+        for raster_path, field in extent_fields:
+            if os.path.exists(raster_path):
+                # Read extent and SRID
+                raster = GDALRaster(raster_path)
+                extent = OGRGeometry.from_bbox(raster.extent)
+
+                # It will be implicitly transformed into the SRID of the model’s field
+                # self.field = GEOSGeometry(...)
+                setattr(self, field, GEOSGeometry(extent.wkt, srid=raster.srid))
+
+                logger.info("Populated extent field with {} for {}".format(raster_path, self))
+
+        self.update_available_assets_field()
+        self.running_progress = 1.0
+        self.console_output += "Done!\n"
+        self.status = status_codes.COMPLETED
+        self.save()
+
+        from app.plugins import signals as plugin_signals
+        plugin_signals.task_completed.send_robust(sender=self.__class__, task_id=self.id)
+
     def get_tile_path(self, tile_type, z, x, y):
         return self.assets_path("{}_tiles".format(tile_type), z, x, "{}.png".format(y))
 
@@ -783,3 +808,16 @@ class Task(models.Model):
         except subprocess.CalledProcessError as e:
             logger.warning("Could not resize GCP file {}: {}".format(gcp_path, str(e)))
             return None
+
+    def create_task_directories(self):
+        """
+        Create directories for this task (if they don't exist already)
+        """
+        assets_dir = self.assets_path("")
+        try:
+            os.makedirs(assets_dir)
+        except OSError as exc:  # Python >2.5
+            if exc.errno == errno.EEXIST and os.path.isdir(assets_dir):
+                pass
+            else:
+                raise

app/pending_actions.py

@@ -2,3 +2,4 @@ CANCEL = 1
 REMOVE = 2
 RESTART = 3
 RESIZE = 4
+IMPORT = 5

app/static/app/js/classes/PendingActions.js

@@ -1,7 +1,8 @@
 const CANCEL = 1,
       REMOVE = 2,
       RESTART = 3,
-      RESIZE = 4;
+      RESIZE = 4,
+      IMPORT = 5;
 
 let pendingActions = {
     [CANCEL]: {
@@ -15,6 +16,9 @@ let pendingActions = {
     },
     [RESIZE]: {
       descr: "Resizing images..."
+    },
+    [IMPORT]: {
+      descr: "Importing..."
     }
 };
 
@@ -23,6 +27,7 @@ export default {
     REMOVE: REMOVE,
     RESTART: RESTART,
     RESIZE: RESIZE,
+    IMPORT: IMPORT,
 
     description: function(pendingAction) {
       if (pendingActions[pendingAction]) return pendingActions[pendingAction].descr;

app/static/app/js/components/ImportTaskPanel.jsx

@@ -0,0 +1,96 @@
+import '../css/ImportTaskPanel.scss';
+import React from 'react';
+import PropTypes from 'prop-types';
+import Dropzone from '../vendor/dropzone';
+import csrf from '../django/csrf';
+
+class ImportTaskPanel extends React.Component {
+  static defaultProps = {
+  };
+
+  static propTypes = {
+      // onSave: PropTypes.func.isRequired,
+      onCancel: PropTypes.func,
+      projectId: PropTypes.number.isRequired
+  };
+
+  constructor(props){
+    super(props);
+
+    this.state = {
+    };
+  }
+
+  componentDidMount(){
+    Dropzone.autoDiscover = false;
+
+    this.dz = new Dropzone(this.dropzone, {
+        paramName: "file",
+        url : `/api/projects/${this.props.projectId}/tasks/import`,
+        parallelUploads: 1,
+        uploadMultiple: false,
+        acceptedFiles: "application/zip",
+        autoProcessQueue: true,
+        createImageThumbnails: false,
+        previewTemplate: '<div style="display:none"></div>',
+        clickable: this.uploadButton,
+        chunkSize: 2147483647,
+        timeout: 2147483647,
+        
+        headers: {
+          [csrf.header]: csrf.token
+        }
+    });
+
+    this.dz.on("error", function(file){
+        // Show 
+      })
+      .on("uploadprogress", function(file, progress){
+          console.log(progress);
+      })
+      .on("complete", function(file){
+          if (file.status === "success"){
+          }else{
+            // error
+          }
+      });
+  }
+
+  cancel = (e) => {
+    this.props.onCancel();
+  }
+
+  setRef = (prop) => {
+    return (domNode) => {
+      if (domNode != null) this[prop] = domNode;
+    }
+  }
+
+  render() {
+    return (
+      <div ref={this.setRef("dropzone")} className="import-task-panel theme-background-highlight">
+        <div className="form-horizontal">
+          <button type="button" className="close theme-color-primary" aria-label="Close" onClick={this.cancel}><span aria-hidden="true">&times;</span></button>
+          <h4>Import Existing Assets</h4>
+          <p>You can import .zip files that have been exported from existing tasks via Download Assets <i className="glyphicon glyphicon-arrow-right"></i> All Assets.</p>
+          <button type="button" 
+                  className="btn btn-primary"
+                  onClick={this.handleUpload}
+                  ref={this.setRef("uploadButton")}>
+            <i className="glyphicon glyphicon-upload"></i>
+            Upload a File
+          </button>
+          <button type="button" 
+                  className="btn btn-primary"
+                  onClick={this.handleImportFromUrl}
+                  ref={this.setRef("importFromUrlButton")}>
+            <i className="glyphicon glyphicon-cloud-download"></i>
+            Import From URL
+          </button>
+        </div>
+      </div>
+    );
+  }
+}
+
+export default ImportTaskPanel;

app/static/app/js/components/ProjectListItem.jsx

@@ -3,6 +3,7 @@ import React from 'react';
 import update from 'immutability-helper';
 import TaskList from './TaskList';
 import NewTaskPanel from './NewTaskPanel';
+import ImportTaskPanel from './ImportTaskPanel';
 import UploadProgressBar from './UploadProgressBar';
 import ProgressBar from './ProgressBar';
 import ErrorMessage from './ErrorMessage';
@@ -32,7 +33,8 @@ class ProjectListItem extends React.Component {
       upload: this.getDefaultUploadState(),
       error: "",
       data: props.data,
-      refreshing: false
+      refreshing: false,
+      importing: false
     };
 
     this.toggleTaskList = this.toggleTaskList.bind(this);
@@ -335,6 +337,14 @@ class ProjectListItem extends React.Component {
     location.href = `/map/project/${this.state.data.id}/`;
   }
 
+  handleImportTask = () => {
+    this.setState({importing: true});
+  }
+
+  handleCancelImportTask = () => {
+    this.setState({importing: false});
+  }
+
   render() {
     const { refreshing, data } = this.state;
     const numTasks = data.tasks.length;
@@ -361,13 +371,17 @@ class ProjectListItem extends React.Component {
           <ErrorMessage bind={[this, 'error']} />
           <div className="btn-group pull-right">
             {this.hasPermission("add") ? 
-              <button type="button" 
+              <div className={"asset-download-buttons btn-group " + (this.state.upload.uploading ? "hide" : "")}>
-                      className={"btn btn-primary btn-sm " + (this.state.upload.uploading ? "hide" : "")} 
+                <button type="button" 
+                      className="btn btn-primary btn-sm"
                       onClick={this.handleUpload}
                       ref={this.setRef("uploadButton")}>
                 <i className="glyphicon glyphicon-upload"></i>
                 Select Images and GCP
-              </button>
+              </button><button type="button" className="btn btn-sm dropdown-toggle btn-primary" data-toggle="dropdown"><span className="caret"></span></button>
+              <ul className="dropdown-menu">
+                <li><a href="javascript:void(0);" onClick={this.handleImportTask}><i className="glyphicon glyphicon-import"></i> Import Existing Assets</a></li>
+             </ul></div>
             : ""}
 
             <button disabled={this.state.upload.error !== ""} 
@@ -432,6 +446,13 @@ class ProjectListItem extends React.Component {
             />
           : ""}
 
+          {this.state.importing ? 
+            <ImportTaskPanel
+              onCancel={this.handleCancelImportTask}
+              projectId={this.state.data.id}
+            />
+          : ""}
+
           {this.state.showTaskList ? 
             <TaskList 
                 ref={this.setRef("taskList")} 

app/static/app/js/components/TaskListItem.jsx

@@ -354,13 +354,15 @@ class TaskListItem extends React.Component {
   render() {
     const task = this.state.task;
     const name = task.name !== null ? task.name : `Task #${task.id}`;
+    const imported = task.import_url !== "";
 
     let status = statusCodes.description(task.status);
     if (status === "") status = "Uploading images to processing node";
 
-    if (!task.processing_node) status = "Waiting for a node...";
+    if (!task.processing_node && !imported) status = "Waiting for a node...";
     if (task.pending_action !== null) status = pendingActions.description(task.pending_action);
 
+
     let expanded = "";
     if (this.state.expanded){
       let showOrthophotoMissingWarning = false,
@@ -572,7 +574,7 @@ class TaskListItem extends React.Component {
 
     if (task.last_error){
       statusLabel = getStatusLabel(task.last_error, 'error');
-    }else if (!task.processing_node){
+    }else if (!task.processing_node && !imported){
       statusLabel = getStatusLabel("Set a processing node");
       statusIcon = "fa fa-hourglass-3";
       showEditLink = true;

app/static/app/js/components/tests/ImportTaskPanel.test.jsx

@@ -0,0 +1,10 @@
+import React from 'react';
+import { shallow } from 'enzyme';
+import ImportTaskPanel from '../ImportTaskPanel';
+
+describe('<ImportTaskPanel />', () => {
+  it('renders without exploding', () => {
+    const wrapper = shallow(<ImportTaskPanel projectId={0} />);
+    expect(wrapper.exists()).toBe(true);
+  })
+});

app/static/app/js/css/ImportTaskPanel.scss

@@ -0,0 +1,20 @@
+.import-task-panel{
+    padding: 10px;
+    margin-top: 10px;
+    margin-bottom: 10px;
+
+    text-align: center;
+
+    button{
+    	margin-right: 12px;
+    	margin-bottom: 8px;
+    }
+
+    .glyphicon-arrow-right{
+    	font-size: 80%;
+    }
+
+    .close:hover, .close:focus{
+    	color: inherit;
+    }
+}

nodeodm/migrations/0006_auto_20190220_1842.py

@@ -0,0 +1,24 @@
+# Generated by Django 2.1.5 on 2019-02-20 18:42
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('nodeodm', '0005_auto_20190115_1346'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='processingnode',
+            name='available_options',
+            field=django.contrib.postgres.fields.jsonb.JSONField(default=dict, help_text='Description of the options that can be used for processing'),
+        ),
+        migrations.AlterField(
+            model_name='processingnode',
+            name='label',
+            field=models.CharField(blank=True, default='', help_text='Optional label for this node. When set, this label will be shown instead of the hostname:port name.', max_length=255),
+        ),
+    ]

nodeodm/models.py

@@ -43,7 +43,7 @@ class ProcessingNode(models.Model):
     available_options = fields.JSONField(default=dict, help_text="Description of the options that can be used for processing")
     token = models.CharField(max_length=1024, blank=True, default="", help_text="Token to use for authentication. If the node doesn't have authentication, you can leave this field blank.")
     max_images = models.PositiveIntegerField(help_text="Maximum number of images accepted by this node.", blank=True, null=True)
-    odm_version = models.CharField(max_length=32, null=True, help_text="OpenDroneMap version used by the node")
+    odm_version = models.CharField(max_length=32, null=True, help_text="ODM version used by the node.")
     label = models.CharField(max_length=255, default="", blank=True, help_text="Optional label for this node. When set, this label will be shown instead of the hostname:port name.")
 
     def __str__(self):