Import tasks working

app/api/tasks.py

@@ -36,7 +36,6 @@ class TaskSerializer(serializers.ModelSerializer):
     project = serializers.PrimaryKeyRelatedField(queryset=models.Project.objects.all())
     processing_node = serializers.PrimaryKeyRelatedField(queryset=ProcessingNode.objects.all()) 
     processing_node_name = serializers.SerializerMethodField()
-    images_count = serializers.SerializerMethodField()
     can_rerun_from = serializers.SerializerMethodField()
 
     def get_processing_node_name(self, obj):
@@ -45,10 +44,6 @@ class TaskSerializer(serializers.ModelSerializer):
         else:
             return None
 
-    def get_images_count(self, obj):
-        # TODO: create a field in the model for this
-        return obj.imageupload_set.count()
-
     def get_can_rerun_from(self, obj):
         """
         When a task has been associated with a processing node
@@ -164,6 +159,7 @@ class TaskViewSet(viewsets.ViewSet):
 
             for image in files:
                 models.ImageUpload.objects.create(task=task, image=image)
+            task.images_count = len(files)
 
             # Update other parameters such as processing node, task name, etc.
             serializer = TaskSerializer(task, data=request.data, partial=True)
@@ -341,23 +337,29 @@ class TaskAssetsImport(APIView):
         project = get_and_check_project(request, project_pk, ('change_project',))
 
         files = flatten_files(request.FILES)
+        import_url = request.data.get('url', None)
+        task_name = request.data.get('name', 'Imported Task')
 
-        if len(files) != 1:
+        if not import_url and len(files) != 1:
             raise exceptions.ValidationError(detail="Cannot create task, you need to upload 1 file")
 
+        if import_url and len(files) > 0:
+            raise exceptions.ValidationError(detail="Cannot create task, either specify a URL or upload 1 file.")
+
         with transaction.atomic():
             task = models.Task.objects.create(project=project,
                                               auto_processing_node=False,
-                                              name="Imported Task",
+                                              name=task_name,
-                                              import_url="file://all.zip",
+                                              import_url=import_url if import_url else "file://all.zip",
                                               status=status_codes.RUNNING,
                                               pending_action=pending_actions.IMPORT)
             task.create_task_directories()
 
-            destination_file = task.assets_path("all.zip")
+            if len(files) > 0:
-            with open(destination_file, 'wb+') as fd:
+                destination_file = task.assets_path("all.zip")
-                for chunk in files[0].chunks():
+                with open(destination_file, 'wb+') as fd:
-                    fd.write(chunk)
+                    for chunk in files[0].chunks():
+                        fd.write(chunk)
 
             worker_tasks.process_task.delay(task.id)
 

app/migrations/0025_auto_20190220_1854.py

@@ -17,36 +17,53 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='task',
             name='import_url',
-            field=models.TextField(blank=True, default='', help_text='URL this task is imported from (only for imported tasks)'),
+            field=models.TextField(blank=True, default='',
+                                   help_text='URL this task is imported from (only for imported tasks)'),
         ),
         migrations.AlterField(
             model_name='preset',
             name='options',
-            field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=list, help_text="Options that define this preset (same format as in a Task's options).", validators=[app.models.task.validate_task_options]),
+            field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=list,
+                                                                 help_text="Options that define this preset (same format as in a Task's options).",
+                                                                 validators=[app.models.task.validate_task_options]),
         ),
         migrations.AlterField(
             model_name='task',
             name='available_assets',
-            field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=80), blank=True, default=list, help_text='List of available assets to download', size=None),
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=80), blank=True,
+                                                            default=list,
+                                                            help_text='List of available assets to download',
+                                                            size=None),
         ),
         migrations.AlterField(
             model_name='task',
             name='options',
-            field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict, help_text='Options that are being used to process this task', validators=[app.models.task.validate_task_options]),
+            field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict,
+                                                                 help_text='Options that are being used to process this task',
+                                                                 validators=[app.models.task.validate_task_options]),
         ),
         migrations.AlterField(
             model_name='task',
             name='pending_action',
-            field=models.IntegerField(blank=True, choices=[(1, 'CANCEL'), (2, 'REMOVE'), (3, 'RESTART'), (4, 'RESIZE'), (5, 'IMPORT')], db_index=True, help_text='A requested action to be performed on the task. The selected action will be performed by the worker at the next iteration.', null=True),
+            field=models.IntegerField(blank=True, choices=[(1, 'CANCEL'), (2, 'REMOVE'), (3, 'RESTART'), (4, 'RESIZE'),
+                                                           (5, 'IMPORT')], db_index=True,
+                                      help_text='A requested action to be performed on the task. The selected action will be performed by the worker at the next iteration.',
+                                      null=True),
         ),
         migrations.AlterField(
             model_name='theme',
             name='header_background',
-            field=colorfield.fields.ColorField(default='#3498db', help_text="Background color of the site's header.", max_length=18),
+            field=colorfield.fields.ColorField(default='#3498db', help_text="Background color of the site's header.",
+                                               max_length=18),
         ),
         migrations.AlterField(
             model_name='theme',
             name='tertiary',
             field=colorfield.fields.ColorField(default='#3498db', help_text='Navigation links.', max_length=18),
         ),
+        migrations.AddField(
+            model_name='task',
+            name='images_count',
+            field=models.IntegerField(blank=True, default=0, help_text='Number of images associated with this task'),
+        ),
     ]

app/models/task.py

@@ -226,7 +226,7 @@ class Task(models.Model):
                                         help_text="Value between 0 and 1 indicating the running progress (estimated) of this task",
                                         blank=True)
     import_url = models.TextField(null=False, default="", blank=True, help_text="URL this task is imported from (only for imported tasks)")
-
+    images_count = models.IntegerField(null=False, blank=True, default=0, help_text="Number of images associated with this task")
 
     def __init__(self, *args, **kwargs):
         super(Task, self).__init__(*args, **kwargs)
@@ -340,7 +340,50 @@ class Task(models.Model):
     def handle_import(self):
         self.console_output += "Importing assets...\n"
         self.save()
+
+        zip_path = self.assets_path("all.zip")
+
+        if self.import_url and not os.path.exists(zip_path):
+            try:
+                # TODO: this is potentially vulnerable to a zip bomb attack
+                #       mitigated by the fact that a valid account is needed to
+                #       import tasks
+                download_stream = requests.get(self.import_url, stream=True, timeout=10)
+                content_length = download_stream.headers.get('content-length')
+                total_length = int(content_length) if content_length is not None else None
+                downloaded = 0
+                last_update = 0
+
+                with open(zip_path, 'wb') as fd:
+                    for chunk in download_stream.iter_content(4096):
+                        downloaded += len(chunk)
+
+                        if time.time() - last_update >= 2:
+                            # Update progress
+                            if total_length is not None:
+                                Task.objects.filter(pk=self.id).update(running_progress=(float(downloaded) / total_length) * 0.9)
+
+                            self.check_if_canceled()
+                            last_update = time.time()
+
+                        fd.write(chunk)
+
+            except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, ReadTimeoutError) as e:
+                raise ProcessingError(e)
+
+        self.refresh_from_db()
         self.extract_assets_and_complete()
+
+        images_json = self.assets_path("images.json")
+        if os.path.exists(images_json):
+            try:
+                with open(images_json) as f:
+                    images = json.load(f)
+                    self.images_count = len(images)
+            except:
+                logger.warning("Cannot read images count from imported task {}".format(self))
+                pass
+
         self.pending_action = None
         self.processing_time = 0
         self.save()
@@ -441,6 +484,11 @@ class Task(models.Model):
                         except ProcessingException:
                             logger.warning("Could not cancel {} on processing node. We'll proceed anyway...".format(self))
 
+                        self.status = status_codes.CANCELED
+                        self.pending_action = None
+                        self.save()
+                    elif self.import_url:
+                        # Imported tasks need no special action
                         self.status = status_codes.CANCELED
                         self.pending_action = None
                         self.save()
@@ -613,8 +661,11 @@ class Task(models.Model):
         zip_path = self.assets_path("all.zip")
 
         # Extract from zip
-        with zipfile.ZipFile(zip_path, "r") as zip_h:
+        try:
-            zip_h.extractall(assets_dir)
+            with zipfile.ZipFile(zip_path, "r") as zip_h:
+                zip_h.extractall(assets_dir)
+        except zipfile.BadZipFile:
+            raise ProcessingError("Corrupted zip file")
 
         logger.info("Extracted all.zip for {}".format(self))
 

app/static/app/js/components/ImportTaskPanel.jsx

@@ -3,13 +3,15 @@ import React from 'react';
 import PropTypes from 'prop-types';
 import Dropzone from '../vendor/dropzone';
 import csrf from '../django/csrf';
+import ErrorMessage from './ErrorMessage';
+import UploadProgressBar from './UploadProgressBar';
 
 class ImportTaskPanel extends React.Component {
   static defaultProps = {
   };
 
   static propTypes = {
-      // onSave: PropTypes.func.isRequired,
+      onImported: PropTypes.func.isRequired,
       onCancel: PropTypes.func,
       projectId: PropTypes.number.isRequired
   };
@@ -18,9 +20,20 @@ class ImportTaskPanel extends React.Component {
     super(props);
 
     this.state = {
+      error: "",
+      typeUrl: false,
+      uploading: false,
+      importingFromUrl: false,
+      progress: 0,
+      bytesSent: 0,
+      importUrl: ""
     };
   }
 
+  defaultTaskName = () => {
+    return `Task of ${new Date().toISOString()}`;
+  }
+
   componentDidMount(){
     Dropzone.autoDiscover = false;
 
@@ -42,24 +55,90 @@ class ImportTaskPanel extends React.Component {
         }
     });
 
-    this.dz.on("error", function(file){
+    this.dz.on("error", (file) => {
-        // Show 
+        if (this.state.uploading) this.setState({error: "Cannot upload file. Check your internet connection and try again."});
       })
-      .on("uploadprogress", function(file, progress){
+      .on("sending", () => {
-          console.log(progress);
+        this.setState({typeUrl: false, uploading: true, totalCount: 1});
       })
-      .on("complete", function(file){
+      .on("reset", () => {
-          if (file.status === "success"){
+        this.setState({uploading: false, progress: 0, totalBytes: 0, totalBytesSent: 0});
-          }else{
+      })
-            // error
+      .on("uploadprogress", (file, progress, bytesSent) => {
+          this.setState({
+            progress,
+            totalBytes: file.size,
+            totalBytesSent: bytesSent
+          });
+      })
+      .on("sending", (file, xhr, formData) => {
+        // Safari does not have support for has on FormData
+        // as of December 2017
+        if (!formData.has || !formData.has("name")) formData.append("name", this.defaultTaskName());
+      })
+      .on("complete", (file) => {
+        if (file.status === "success"){
+          this.setState({uploading: false});
+          try{
+            let response = JSON.parse(file.xhr.response);
+            if (!response.id) throw new Error(`Expected id field, but none given (${response})`);
+            this.props.onImported();
+          }catch(e){
+            this.setState({error: `Invalid response from server: ${e.message}`});
           }
+        }else if (this.state.uploading){
+          this.setState({uploading: false, error: "An error occured while uploading the file. Please try again."});
+        }
       });
   }
 
   cancel = (e) => {
+    this.cancelUpload();
     this.props.onCancel();
   }
 
+  cancelUpload = (e) => {
+    this.setState({uploading: false});
+    setTimeout(() => {
+      this.dz.removeAllFiles(true);
+    }, 0);
+  }
+
+  handleImportFromUrl = () => {
+    this.setState({typeUrl: !this.state.typeUrl});
+  }
+
+  handleCancelImportFromURL = () => {
+    this.setState({typeUrl: false});
+  }
+
+  handleChangeImportUrl = (e) => {
+    this.setState({importUrl: e.target.value});
+  }
+
+  handleConfirmImportUrl = () => {
+    this.setState({importingFromUrl: true});
+
+    $.post(`/api/projects/${this.props.projectId}/tasks/import`,
+      {
+        url: this.state.importUrl,
+        name: this.defaultTaskName()
+      }
+    ).done(json => {
+      if (json.id){
+        this.props.onImported();
+      }else{
+        this.setState({error: json.error || `Cannot import from URL, server responded: ${JSON.stringify(json)}`});
+      }
+    })
+    .fail(() => {
+        this.setState({error: "Cannot import from URL. Check your internet connection."});
+    })
+    .always(() => {
+      this.setState({importingFromUrl: false});
+    });
+  }
+
   setRef = (prop) => {
     return (domNode) => {
       if (domNode != null) this[prop] = domNode;
@@ -70,23 +149,47 @@ class ImportTaskPanel extends React.Component {
     return (
       <div ref={this.setRef("dropzone")} className="import-task-panel theme-background-highlight">
         <div className="form-horizontal">
+          <ErrorMessage bind={[this, 'error']} />
+
           <button type="button" className="close theme-color-primary" aria-label="Close" onClick={this.cancel}><span aria-hidden="true">&times;</span></button>
           <h4>Import Existing Assets</h4>
           <p>You can import .zip files that have been exported from existing tasks via Download Assets <i className="glyphicon glyphicon-arrow-right"></i> All Assets.</p>
-          <button type="button" 
+          
+          <button disabled={this.state.uploading}
+                  type="button" 
                   className="btn btn-primary"
-                  onClick={this.handleUpload}
                   ref={this.setRef("uploadButton")}>
             <i className="glyphicon glyphicon-upload"></i>
             Upload a File
           </button>
-          <button type="button" 
+          <button disabled={this.state.uploading}
+                  type="button" 
                   className="btn btn-primary"
                   onClick={this.handleImportFromUrl}
                   ref={this.setRef("importFromUrlButton")}>
             <i className="glyphicon glyphicon-cloud-download"></i>
             Import From URL
           </button>
+
+          {this.state.typeUrl ? 
+            <div className="form-inline">
+              <div className="form-group">
+                <input disabled={this.state.importingFromUrl} onChange={this.handleChangeImportUrl} size="45" type="text" className="form-control" placeholder="http://" value={this.state.importUrl} />
+                <button onClick={this.handleConfirmImportUrl}
+                        disabled={this.state.importUrl.length < 4 || this.state.importingFromUrl} 
+                        className="btn-import btn btn-primary"><i className="glyphicon glyphicon-cloud-download"></i> Import</button>
+              </div>
+            </div> : ""}
+
+          {this.state.uploading ? <div>
+            <UploadProgressBar {...this.state}/>
+            <button type="button"
+                    className="btn btn-danger btn-sm" 
+                    onClick={this.cancelUpload}>
+              <i className="glyphicon glyphicon-remove-circle"></i>
+              Cancel Upload
+            </button> 
+          </div> : ""}
         </div>
       </div>
     );

app/static/app/js/components/ProjectListItem.jsx

@@ -204,13 +204,7 @@ class ProjectListItem extends React.Component {
               let response = JSON.parse(files[0].xhr.response);
               if (!response.id) throw new Error(`Expected id field, but none given (${response})`);
               
-              if (this.state.showTaskList){
+              this.newTaskAdded();
-                this.taskList.refresh();
-              }else{
-                this.setState({showTaskList: true});
-              }
-              this.resetUploadState();
-              this.refresh();
             }catch(e){
               this.setUploadState({error: `Invalid response from server: ${e.message}`, uploading: false})
             }
@@ -247,6 +241,18 @@ class ProjectListItem extends React.Component {
     }
   }
 
+  newTaskAdded = () => {
+    this.setState({importing: false});
+    
+    if (this.state.showTaskList){
+      this.taskList.refresh();
+    }else{
+      this.setState({showTaskList: true});
+    }
+    this.resetUploadState();
+    this.refresh();
+  }
+
   setRef(prop){
     return (domNode) => {
       if (domNode != null) this[prop] = domNode;
@@ -448,6 +454,7 @@ class ProjectListItem extends React.Component {
 
           {this.state.importing ? 
             <ImportTaskPanel
+              onImported={this.newTaskAdded}
               onCancel={this.handleCancelImportTask}
               projectId={this.state.data.id}
             />

app/static/app/js/components/TaskListItem.jsx

@@ -406,12 +406,13 @@ class TaskListItem extends React.Component {
       }
 
       if ([statusCodes.QUEUED, statusCodes.RUNNING, null].indexOf(task.status) !== -1 &&
-          task.processing_node){
+          (task.processing_node || imported)){
         addActionButton("Cancel", "btn-primary", "glyphicon glyphicon-remove-circle", this.genActionApiCall("cancel", {defaultError: "Cannot cancel task."}));
       }
 
       if ([statusCodes.FAILED, statusCodes.COMPLETED, statusCodes.CANCELED].indexOf(task.status) !== -1 &&
-            task.processing_node){
+            task.processing_node &&
+            !imported){
           // By default restart reruns every pipeline
           // step from the beginning
           const rerunFrom = task.can_rerun_from.length > 1 ?

app/static/app/js/css/ImportTaskPanel.scss

@@ -17,4 +17,13 @@
     .close:hover, .close:focus{
     	color: inherit;
     }
+
+    .upload-progress-bar{
+    	margin-top: 12px;
+    }
+
+    .btn-import{
+	    margin-top: 8px;
+    	margin-left: 8px;
+    }
 }

app/static/app/js/css/TaskListItem.scss

@@ -15,6 +15,7 @@
 
     .name{
         padding-left: 0;
+        margin-top: 4px;
     }
     
     .details{