Chunked file upload support, progress bar color fix

app/api/tasks.py

@@ -158,28 +158,78 @@ class TaskViewSet(viewsets.ViewSet):
         serializer = TaskSerializer(task)
         return Response(serializer.data)
 
-    def create(self, request, project_pk=None):
+    @detail_route(methods=['post'])
-        project = get_and_check_project(request, project_pk, ('change_project', ))
+    def commit(self, request, pk=None, project_pk=None):
-        
+        """
+        Commit a task after all images have been uploaded
+        """
+        get_and_check_project(request, project_pk, ('change_project', ))
+        try:
+            task = self.queryset.get(pk=pk, project=project_pk)
+        except (ObjectDoesNotExist, ValidationError):
+            raise exceptions.NotFound()
+
+        task.partial = False
+        task.images_count = models.ImageUpload.objects.filter(task=task).count()
+        task.save()
+        worker_tasks.process_task.delay(task.id)
+
+        serializer = TaskSerializer(task)
+        return Response(serializer.data, status=status.HTTP_200_OK)
+
+    @detail_route(methods=['post'])
+    def upload(self, request, pk=None, project_pk=None):
+        """
+        Add images to a task
+        """
+        get_and_check_project(request, project_pk, ('change_project', ))
+        try:
+            task = self.queryset.get(pk=pk, project=project_pk)
+        except (ObjectDoesNotExist, ValidationError):
+            raise exceptions.NotFound()
+
         files = flatten_files(request.FILES)
 
-        if len(files) <= 1:
+        if len(files) == 0:
-            raise exceptions.ValidationError(detail="Cannot create task, you need at least 2 images")
+            raise exceptions.ValidationError(detail="No files uploaded")
 
         with transaction.atomic():
-            task = models.Task.objects.create(project=project,
-                                              pending_action=pending_actions.RESIZE if 'resize_to' in request.data else None)
-
             for image in files:
                 models.ImageUpload.objects.create(task=task, image=image)
-            task.images_count = len(files)
 
-            # Update other parameters such as processing node, task name, etc.
+        return Response({'success': True}, status=status.HTTP_200_OK)
+
+    def create(self, request, project_pk=None):
+        project = get_and_check_project(request, project_pk, ('change_project', ))
+
+        # If this is a partial task, we're going to upload images later
+        # for now we just create a placeholder task.
+        if request.data.get('partial'):
+            task = models.Task.objects.create(project=project,
+                                              pending_action=pending_actions.RESIZE if 'resize_to' in request.data else None)
             serializer = TaskSerializer(task, data=request.data, partial=True)
             serializer.is_valid(raise_exception=True)
             serializer.save()
+        else:
+            files = flatten_files(request.FILES)
 
-            worker_tasks.process_task.delay(task.id)
+            if len(files) <= 1:
+                raise exceptions.ValidationError(detail="Cannot create task, you need at least 2 images")
+
+            with transaction.atomic():
+                task = models.Task.objects.create(project=project,
+                                                  pending_action=pending_actions.RESIZE if 'resize_to' in request.data else None)
+
+                for image in files:
+                    models.ImageUpload.objects.create(task=task, image=image)
+                task.images_count = len(files)
+
+                # Update other parameters such as processing node, task name, etc.
+                serializer = TaskSerializer(task, data=request.data, partial=True)
+                serializer.is_valid(raise_exception=True)
+                serializer.save()
+
+                worker_tasks.process_task.delay(task.id)
 
         return Response(serializer.data, status=status.HTTP_201_CREATED)
 

app/migrations/0028_task_partial.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.1.7 on 2019-06-26 18:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('app', '0027_plugin'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='task',
+            name='partial',
+            field=models.BooleanField(default=False, help_text='A flag indicating whether this task is currently waiting for information or files to be uploaded before being considered for processing.'),
+        ),
+    ]

app/models/task.py

@@ -213,6 +213,7 @@ class Task(models.Model):
                                         blank=True)
     import_url = models.TextField(null=False, default="", blank=True, help_text="URL this task is imported from (only for imported tasks)")
     images_count = models.IntegerField(null=False, blank=True, default=0, help_text="Number of images associated with this task")
+    partial = models.BooleanField(default=False, help_text="A flag indicating whether this task is currently waiting for information or files to be uploaded before being considered for processing.")
 
     def __init__(self, *args, **kwargs):
         super(Task, self).__init__(*args, **kwargs)

app/static/app/css/theme.scss

@@ -72,6 +72,9 @@ body,
 a, a:hover, a:focus{
 	color: theme("tertiary");
 }
+.progress-bar-success{
+    background-color: theme("tertiary");
+}
 
 /* Button primary */
 #navbar-top .navbar-top-links,{

app/static/app/js/components/ProjectListItem.jsx

@@ -81,6 +81,7 @@ class ProjectListItem extends React.Component {
       progress: 0,
       files: [],
       totalCount: 0,
+      uploadedCount: 0,
       totalBytes: 0,
       totalBytesSent: 0,
       lastUpdated: 0
@@ -109,9 +110,9 @@ class ProjectListItem extends React.Component {
     if (this.hasPermission("add")){
       this.dz = new Dropzone(this.dropzone, {
           paramName: "images",
-          url : `/api/projects/${this.state.data.id}/tasks/`,
+          url : 'TO_BE_CHANGED',
-          parallelUploads: 2147483647,
+          parallelUploads: 10,
-          uploadMultiple: true,
+          uploadMultiple: false,
           acceptedFiles: "image/*,text/*",
           autoProcessQueue: false,
           createImageThumbnails: false,
@@ -155,27 +156,39 @@ class ProjectListItem extends React.Component {
           }
       });
 
-      this.dz.on("totaluploadprogress", (progress, totalBytes, totalBytesSent) => {
+      this.dz.on("addedfiles", files => {
-          // Limit updates since this gets called a lot
+          let totalBytes = 0;
-          let now = (new Date()).getTime();
+          for (let i = 0; i < files.length; i++){
-
+              totalBytes += files[i].size;
-          // Progress 100 is sent multiple times at the end
+              files[i].deltaBytesSent = 0;
-          // this makes it so that we update the state only once.
+              files[i].trackedBytesSent = 0;
-          if (progress === 100) now = now + 9999999999;
+              files[i].retries = 0;
-
-          if (this.state.upload.lastUpdated + 500 < now){
-              this.setUploadState({
-                progress, totalBytes, totalBytesSent, lastUpdated: now
-              });
           }
-        })
+
-        .on("addedfiles", files => {
           this.setUploadState({
             editing: true,
             totalCount: this.state.upload.totalCount + files.length,
-            files
+            files,
+            totalBytes: this.state.upload.totalBytes + totalBytes
           });
         })
+        .on("uploadprogress", (file, progress, bytesSent) => {
+            const now = new Date().getTime();
+
+            if (now - this.state.upload.lastUpdated > 500){
+                file.deltaBytesSent = bytesSent - file.deltaBytesSent;
+                file.trackedBytesSent += file.deltaBytesSent;
+
+                const totalBytesSent = this.state.upload.totalBytesSent + file.deltaBytesSent;
+                const progress = totalBytesSent / this.state.upload.totalBytes * 100;
+
+                this.setUploadState({
+                    progress,
+                    totalBytesSent,
+                    lastUpdated: now
+                });
+            }
+        })
         .on("transformcompleted", (file, total) => {
           if (this.dz._resizeMap) this.dz._resizeMap[file.name] = this.dz._taskInfo.resizeSize / Math.max(file.width, file.height);
           if (this.dz.options.resizeWidth) this.setUploadState({resizedImages: total});
@@ -192,29 +205,89 @@ class ProjectListItem extends React.Component {
         .on("transformend", () => {
           this.setUploadState({resizing: false, uploading: true});
         })
-        .on("completemultiple", (files) => {
+        .on("complete", (file) => {
-          // Check
+            // Retry
-          const invalidFilesCount = files.filter(file => file.status !== "success").length;
+            const retry = () => {
-          let success = files.length > 0 && invalidFilesCount === 0;
+                const MAX_RETRIES = 10;
+
+                if (file.retries < MAX_RETRIES){
+                    // Update progress
+                    const totalBytesSent = this.state.upload.totalBytesSent - file.trackedBytesSent;
+                    const progress = totalBytesSent / this.state.upload.totalBytes * 100;
+        
+                    this.setUploadState({
+                        progress,
+                        totalBytesSent,
+                    });
+        
+                    file.status = Dropzone.QUEUED;
+                    file.deltaBytesSent = 0;
+                    file.trackedBytesSent = 0;
+                    file.retries++;
+                    this.dz.processQueue();
+                }else{
+                    throw new Error(`Cannot upload ${file.name}, exceeded max retries (${MAX_RETRIES})`);
+                }
+            };
 
-          // All files have uploaded!
-          if (success){
-            this.setUploadState({uploading: false});
             try{
-              let response = JSON.parse(files[0].xhr.response);
+                if (file.status === "error"){
-              if (!response.id) throw new Error(`Expected id field, but none given (${response})`);
+                    retry();
-              
+                }else{
-              this.newTaskAdded();
+                    // Check response
+                    let response = JSON.parse(file.xhr.response);
+                    if (response.success){
+                        // Update progress by removing the tracked progress and 
+                        // use the file size as the true number of bytes
+                        let totalBytesSent = this.state.upload.totalBytesSent + file.size;
+                        if (file.trackedBytesSent) totalBytesSent -= file.trackedBytesSent;
+        
+                        const progress = totalBytesSent / this.state.upload.totalBytes * 100;
+        
+                        this.setUploadState({
+                            progress,
+                            totalBytesSent,
+                            uploadedCount: this.state.upload.uploadedCount + 1
+                        });
+
+                        this.dz.processQueue();
+                    }else{
+                        retry();
+                    }
+                }
             }catch(e){
-              this.setUploadState({error: `Invalid response from server: ${e.message}`, uploading: false})
+                this.setUploadState({error: `${e.message}`, uploading: false});
+                this.dz.cancelUpload();
+            }
+        })
+        .on("queuecomplete", () => {
+            const remainingFilesCount = this.state.upload.totalCount - this.state.upload.uploadedCount;
+            if (remainingFilesCount === 0){
+                // All files have uploaded!
+                this.setUploadState({uploading: false});
+
+                $.ajax({
+                    url: `/api/projects/${this.state.data.id}/tasks/${this.dz._taskInfo.id}/commit/`,
+                    contentType: 'application/json',
+                    dataType: 'json',
+                    type: 'POST'
+                  }).done((task) => {
+                    if (task && task.id){
+                        this.newTaskAdded();
+                    }else{
+                        this.setUploadState({error: `Cannot create new task. Invalid response from server: ${JSON.stringify(task)}`});
+                    }
+                  }).fail(() => {
+                    this.setUploadState({error: "Cannot create new task. Please try again later."});
+                  });
+            }else if (this.dz.getQueuedFiles() === 0){
+                // Done but didn't upload all?
+                this.setUploadState({
+                    totalCount: this.state.upload.totalCount - remainingFilesCount,
+                    uploading: false,
+                    error: `${remainingFilesCount} files cannot be uploaded. As a reminder, only images (.jpg, .png) and GCP files (.txt) can be uploaded. Try again.`
+                });
             }
-          }else{
-            this.setUploadState({
-              totalCount: this.state.upload.totalCount - invalidFilesCount,
-              uploading: false,
-              error: `${invalidFilesCount} files cannot be uploaded. As a reminder, only images (.jpg, .png) and GCP files (.txt) can be uploaded. Try again.`
-            });
-          }
         })
         .on("reset", () => {
           this.resetUploadState();
@@ -223,20 +296,6 @@ class ProjectListItem extends React.Component {
           if (!this.state.upload.editing){
             this.resetUploadState();
           }
-        })
-        .on("sending", (file, xhr, formData) => {
-          const taskInfo = this.dz._taskInfo;
-
-          // Safari does not have support for has on FormData
-          // as of December 2017
-          if (!formData.has || !formData.has("name")) formData.append("name", taskInfo.name);
-          if (!formData.has || !formData.has("options")) formData.append("options", JSON.stringify(taskInfo.options));
-          if (!formData.has || !formData.has("processing_node")) formData.append("processing_node", taskInfo.selectedNode.id);
-          if (!formData.has || !formData.has("auto_processing_node")) formData.append("auto_processing_node", taskInfo.selectedNode.key == "auto");
-
-          if (taskInfo.resizeMode === ResizeModes.YES){
-            if (!formData.has || !formData.has("resize_to")) formData.append("resize_to", taskInfo.resizeSize);
-          }
         });
     }
   }
@@ -303,9 +362,38 @@ class ProjectListItem extends React.Component {
       this.setUploadState({uploading: true, editing: false});
     }
 
-    setTimeout(() => {
+    // Create task
-      this.dz.processQueue();
+    const formData = {
-    }, 1);
+        name: taskInfo.name,
+        options: taskInfo.options,
+        processing_node:  taskInfo.selectedNode.id,
+        auto_processing_node: taskInfo.selectedNode.key == "auto",
+        partial: true
+    };
+    if (taskInfo.resizeMode === ResizeModes.YES){
+        formData["resize_to"] = taskInfo.resizeSize
+    }
+
+    $.ajax({
+        url: `/api/projects/${this.state.data.id}/tasks/`,
+        contentType: 'application/json',
+        data: JSON.stringify(formData),
+        dataType: 'json',
+        type: 'POST'
+      }).done((task) => {
+        if (task && task.id){
+            console.log(this.dz._taskInfo);
+            this.dz._taskInfo.id = task.id;
+            this.dz.options.url = `/api/projects/${this.state.data.id}/tasks/${task.id}/upload/`;
+            this.dz.processQueue();
+        }else{
+            this.setState({error: `Cannot create new task. Invalid response from server: ${JSON.stringify(task)}`});
+            this.handleTaskCanceled();
+        }
+      }).fail(() => {
+        this.setState({error: "Cannot create new task. Please try again later."});
+        this.handleTaskCanceled();
+      });
   }
 
   handleTaskCanceled = () => {

app/static/app/js/components/TaskListItem.jsx

@@ -570,6 +570,9 @@ class TaskListItem extends React.Component {
       statusLabel = getStatusLabel("Set a processing node");
       statusIcon = "fa fa-hourglass-3";
       showEditLink = true;
+    }else if (task.partial && !task.pending_action){
+      statusIcon = "fa fa-hourglass-3";
+      statusLabel = getStatusLabel("Waiting for image upload...");
     }else{
       let progress = 100;
       let type = 'done';

worker/tasks.py

@@ -124,10 +124,11 @@ def get_pending_tasks():
     # Or that need one assigned (via auto)
     # or tasks that need a status update
     # or tasks that have a pending action
-    return Task.objects.filter(Q(processing_node__isnull=True, auto_processing_node=True) |
+    # no partial tasks allowed
+    return Task.objects.filter(Q(processing_node__isnull=True, auto_processing_node=True, partial=False) |
                                 Q(Q(status=None) | Q(status__in=[status_codes.QUEUED, status_codes.RUNNING]),
-                                  processing_node__isnull=False) |
+                                  processing_node__isnull=False, partial=False) |
-                                Q(pending_action__isnull=False))
+                                Q(pending_action__isnull=False, partial=False))
 
 @app.task
 def process_pending_tasks():