From ac649e7940e970bb3d6fc2cf02a8c0b4efdc845f Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 26 Jun 2019 18:41:09 -0400 Subject: [PATCH] Chunked file upload support, progress bar color fix --- app/api/tasks.py | 72 ++++++- app/migrations/0028_task_partial.py | 18 ++ app/models/task.py | 1 + app/static/app/css/theme.scss | 3 + .../app/js/components/ProjectListItem.jsx | 196 +++++++++++++----- app/static/app/js/components/TaskListItem.jsx | 3 + worker/tasks.py | 7 +- 7 files changed, 232 insertions(+), 68 deletions(-) create mode 100644 app/migrations/0028_task_partial.py diff --git a/app/api/tasks.py b/app/api/tasks.py index ce0cfa92..a900e5c2 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -158,28 +158,78 @@ class TaskViewSet(viewsets.ViewSet): serializer = TaskSerializer(task) return Response(serializer.data) - def create(self, request, project_pk=None): - project = get_and_check_project(request, project_pk, ('change_project', )) - + @detail_route(methods=['post']) + def commit(self, request, pk=None, project_pk=None): + """ + Commit a task after all images have been uploaded + """ + get_and_check_project(request, project_pk, ('change_project', )) + try: + task = self.queryset.get(pk=pk, project=project_pk) + except (ObjectDoesNotExist, ValidationError): + raise exceptions.NotFound() + + task.partial = False + task.images_count = models.ImageUpload.objects.filter(task=task).count() + task.save() + worker_tasks.process_task.delay(task.id) + + serializer = TaskSerializer(task) + return Response(serializer.data, status=status.HTTP_200_OK) + + @detail_route(methods=['post']) + def upload(self, request, pk=None, project_pk=None): + """ + Add images to a task + """ + get_and_check_project(request, project_pk, ('change_project', )) + try: + task = self.queryset.get(pk=pk, project=project_pk) + except (ObjectDoesNotExist, ValidationError): + raise exceptions.NotFound() + files = flatten_files(request.FILES) - if len(files) <= 1: - raise exceptions.ValidationError(detail="Cannot create task, you need at least 2 images") + if len(files) == 0: + raise exceptions.ValidationError(detail="No files uploaded") with transaction.atomic(): - task = models.Task.objects.create(project=project, - pending_action=pending_actions.RESIZE if 'resize_to' in request.data else None) - for image in files: models.ImageUpload.objects.create(task=task, image=image) - task.images_count = len(files) - # Update other parameters such as processing node, task name, etc. + return Response({'success': True}, status=status.HTTP_200_OK) + + def create(self, request, project_pk=None): + project = get_and_check_project(request, project_pk, ('change_project', )) + + # If this is a partial task, we're going to upload images later + # for now we just create a placeholder task. + if request.data.get('partial'): + task = models.Task.objects.create(project=project, + pending_action=pending_actions.RESIZE if 'resize_to' in request.data else None) serializer = TaskSerializer(task, data=request.data, partial=True) serializer.is_valid(raise_exception=True) serializer.save() + else: + files = flatten_files(request.FILES) - worker_tasks.process_task.delay(task.id) + if len(files) <= 1: + raise exceptions.ValidationError(detail="Cannot create task, you need at least 2 images") + + with transaction.atomic(): + task = models.Task.objects.create(project=project, + pending_action=pending_actions.RESIZE if 'resize_to' in request.data else None) + + for image in files: + models.ImageUpload.objects.create(task=task, image=image) + task.images_count = len(files) + + # Update other parameters such as processing node, task name, etc. + serializer = TaskSerializer(task, data=request.data, partial=True) + serializer.is_valid(raise_exception=True) + serializer.save() + + worker_tasks.process_task.delay(task.id) return Response(serializer.data, status=status.HTTP_201_CREATED) diff --git a/app/migrations/0028_task_partial.py b/app/migrations/0028_task_partial.py new file mode 100644 index 00000000..e00d5cb1 --- /dev/null +++ b/app/migrations/0028_task_partial.py @@ -0,0 +1,18 @@ +# Generated by Django 2.1.7 on 2019-06-26 18:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0027_plugin'), + ] + + operations = [ + migrations.AddField( + model_name='task', + name='partial', + field=models.BooleanField(default=False, help_text='A flag indicating whether this task is currently waiting for information or files to be uploaded before being considered for processing.'), + ), + ] diff --git a/app/models/task.py b/app/models/task.py index d068813e..4e6b5533 100644 --- a/app/models/task.py +++ b/app/models/task.py @@ -213,6 +213,7 @@ class Task(models.Model): blank=True) import_url = models.TextField(null=False, default="", blank=True, help_text="URL this task is imported from (only for imported tasks)") images_count = models.IntegerField(null=False, blank=True, default=0, help_text="Number of images associated with this task") + partial = models.BooleanField(default=False, help_text="A flag indicating whether this task is currently waiting for information or files to be uploaded before being considered for processing.") def __init__(self, *args, **kwargs): super(Task, self).__init__(*args, **kwargs) diff --git a/app/static/app/css/theme.scss b/app/static/app/css/theme.scss index 5532a265..cb716c81 100644 --- a/app/static/app/css/theme.scss +++ b/app/static/app/css/theme.scss @@ -72,6 +72,9 @@ body, a, a:hover, a:focus{ color: theme("tertiary"); } +.progress-bar-success{ + background-color: theme("tertiary"); +} /* Button primary */ #navbar-top .navbar-top-links,{ diff --git a/app/static/app/js/components/ProjectListItem.jsx b/app/static/app/js/components/ProjectListItem.jsx index 009b8ec4..098a2bde 100644 --- a/app/static/app/js/components/ProjectListItem.jsx +++ b/app/static/app/js/components/ProjectListItem.jsx @@ -81,6 +81,7 @@ class ProjectListItem extends React.Component { progress: 0, files: [], totalCount: 0, + uploadedCount: 0, totalBytes: 0, totalBytesSent: 0, lastUpdated: 0 @@ -109,9 +110,9 @@ class ProjectListItem extends React.Component { if (this.hasPermission("add")){ this.dz = new Dropzone(this.dropzone, { paramName: "images", - url : `/api/projects/${this.state.data.id}/tasks/`, - parallelUploads: 2147483647, - uploadMultiple: true, + url : 'TO_BE_CHANGED', + parallelUploads: 10, + uploadMultiple: false, acceptedFiles: "image/*,text/*", autoProcessQueue: false, createImageThumbnails: false, @@ -155,27 +156,39 @@ class ProjectListItem extends React.Component { } }); - this.dz.on("totaluploadprogress", (progress, totalBytes, totalBytesSent) => { - // Limit updates since this gets called a lot - let now = (new Date()).getTime(); - - // Progress 100 is sent multiple times at the end - // this makes it so that we update the state only once. - if (progress === 100) now = now + 9999999999; - - if (this.state.upload.lastUpdated + 500 < now){ - this.setUploadState({ - progress, totalBytes, totalBytesSent, lastUpdated: now - }); + this.dz.on("addedfiles", files => { + let totalBytes = 0; + for (let i = 0; i < files.length; i++){ + totalBytes += files[i].size; + files[i].deltaBytesSent = 0; + files[i].trackedBytesSent = 0; + files[i].retries = 0; } - }) - .on("addedfiles", files => { + this.setUploadState({ editing: true, totalCount: this.state.upload.totalCount + files.length, - files + files, + totalBytes: this.state.upload.totalBytes + totalBytes }); }) + .on("uploadprogress", (file, progress, bytesSent) => { + const now = new Date().getTime(); + + if (now - this.state.upload.lastUpdated > 500){ + file.deltaBytesSent = bytesSent - file.deltaBytesSent; + file.trackedBytesSent += file.deltaBytesSent; + + const totalBytesSent = this.state.upload.totalBytesSent + file.deltaBytesSent; + const progress = totalBytesSent / this.state.upload.totalBytes * 100; + + this.setUploadState({ + progress, + totalBytesSent, + lastUpdated: now + }); + } + }) .on("transformcompleted", (file, total) => { if (this.dz._resizeMap) this.dz._resizeMap[file.name] = this.dz._taskInfo.resizeSize / Math.max(file.width, file.height); if (this.dz.options.resizeWidth) this.setUploadState({resizedImages: total}); @@ -192,29 +205,89 @@ class ProjectListItem extends React.Component { .on("transformend", () => { this.setUploadState({resizing: false, uploading: true}); }) - .on("completemultiple", (files) => { - // Check - const invalidFilesCount = files.filter(file => file.status !== "success").length; - let success = files.length > 0 && invalidFilesCount === 0; + .on("complete", (file) => { + // Retry + const retry = () => { + const MAX_RETRIES = 10; + + if (file.retries < MAX_RETRIES){ + // Update progress + const totalBytesSent = this.state.upload.totalBytesSent - file.trackedBytesSent; + const progress = totalBytesSent / this.state.upload.totalBytes * 100; + + this.setUploadState({ + progress, + totalBytesSent, + }); + + file.status = Dropzone.QUEUED; + file.deltaBytesSent = 0; + file.trackedBytesSent = 0; + file.retries++; + this.dz.processQueue(); + }else{ + throw new Error(`Cannot upload ${file.name}, exceeded max retries (${MAX_RETRIES})`); + } + }; - // All files have uploaded! - if (success){ - this.setUploadState({uploading: false}); try{ - let response = JSON.parse(files[0].xhr.response); - if (!response.id) throw new Error(`Expected id field, but none given (${response})`); - - this.newTaskAdded(); + if (file.status === "error"){ + retry(); + }else{ + // Check response + let response = JSON.parse(file.xhr.response); + if (response.success){ + // Update progress by removing the tracked progress and + // use the file size as the true number of bytes + let totalBytesSent = this.state.upload.totalBytesSent + file.size; + if (file.trackedBytesSent) totalBytesSent -= file.trackedBytesSent; + + const progress = totalBytesSent / this.state.upload.totalBytes * 100; + + this.setUploadState({ + progress, + totalBytesSent, + uploadedCount: this.state.upload.uploadedCount + 1 + }); + + this.dz.processQueue(); + }else{ + retry(); + } + } }catch(e){ - this.setUploadState({error: `Invalid response from server: ${e.message}`, uploading: false}) + this.setUploadState({error: `${e.message}`, uploading: false}); + this.dz.cancelUpload(); + } + }) + .on("queuecomplete", () => { + const remainingFilesCount = this.state.upload.totalCount - this.state.upload.uploadedCount; + if (remainingFilesCount === 0){ + // All files have uploaded! + this.setUploadState({uploading: false}); + + $.ajax({ + url: `/api/projects/${this.state.data.id}/tasks/${this.dz._taskInfo.id}/commit/`, + contentType: 'application/json', + dataType: 'json', + type: 'POST' + }).done((task) => { + if (task && task.id){ + this.newTaskAdded(); + }else{ + this.setUploadState({error: `Cannot create new task. Invalid response from server: ${JSON.stringify(task)}`}); + } + }).fail(() => { + this.setUploadState({error: "Cannot create new task. Please try again later."}); + }); + }else if (this.dz.getQueuedFiles() === 0){ + // Done but didn't upload all? + this.setUploadState({ + totalCount: this.state.upload.totalCount - remainingFilesCount, + uploading: false, + error: `${remainingFilesCount} files cannot be uploaded. As a reminder, only images (.jpg, .png) and GCP files (.txt) can be uploaded. Try again.` + }); } - }else{ - this.setUploadState({ - totalCount: this.state.upload.totalCount - invalidFilesCount, - uploading: false, - error: `${invalidFilesCount} files cannot be uploaded. As a reminder, only images (.jpg, .png) and GCP files (.txt) can be uploaded. Try again.` - }); - } }) .on("reset", () => { this.resetUploadState(); @@ -223,20 +296,6 @@ class ProjectListItem extends React.Component { if (!this.state.upload.editing){ this.resetUploadState(); } - }) - .on("sending", (file, xhr, formData) => { - const taskInfo = this.dz._taskInfo; - - // Safari does not have support for has on FormData - // as of December 2017 - if (!formData.has || !formData.has("name")) formData.append("name", taskInfo.name); - if (!formData.has || !formData.has("options")) formData.append("options", JSON.stringify(taskInfo.options)); - if (!formData.has || !formData.has("processing_node")) formData.append("processing_node", taskInfo.selectedNode.id); - if (!formData.has || !formData.has("auto_processing_node")) formData.append("auto_processing_node", taskInfo.selectedNode.key == "auto"); - - if (taskInfo.resizeMode === ResizeModes.YES){ - if (!formData.has || !formData.has("resize_to")) formData.append("resize_to", taskInfo.resizeSize); - } }); } } @@ -303,9 +362,38 @@ class ProjectListItem extends React.Component { this.setUploadState({uploading: true, editing: false}); } - setTimeout(() => { - this.dz.processQueue(); - }, 1); + // Create task + const formData = { + name: taskInfo.name, + options: taskInfo.options, + processing_node: taskInfo.selectedNode.id, + auto_processing_node: taskInfo.selectedNode.key == "auto", + partial: true + }; + if (taskInfo.resizeMode === ResizeModes.YES){ + formData["resize_to"] = taskInfo.resizeSize + } + + $.ajax({ + url: `/api/projects/${this.state.data.id}/tasks/`, + contentType: 'application/json', + data: JSON.stringify(formData), + dataType: 'json', + type: 'POST' + }).done((task) => { + if (task && task.id){ + console.log(this.dz._taskInfo); + this.dz._taskInfo.id = task.id; + this.dz.options.url = `/api/projects/${this.state.data.id}/tasks/${task.id}/upload/`; + this.dz.processQueue(); + }else{ + this.setState({error: `Cannot create new task. Invalid response from server: ${JSON.stringify(task)}`}); + this.handleTaskCanceled(); + } + }).fail(() => { + this.setState({error: "Cannot create new task. Please try again later."}); + this.handleTaskCanceled(); + }); } handleTaskCanceled = () => { diff --git a/app/static/app/js/components/TaskListItem.jsx b/app/static/app/js/components/TaskListItem.jsx index d4242952..f751034c 100644 --- a/app/static/app/js/components/TaskListItem.jsx +++ b/app/static/app/js/components/TaskListItem.jsx @@ -570,6 +570,9 @@ class TaskListItem extends React.Component { statusLabel = getStatusLabel("Set a processing node"); statusIcon = "fa fa-hourglass-3"; showEditLink = true; + }else if (task.partial && !task.pending_action){ + statusIcon = "fa fa-hourglass-3"; + statusLabel = getStatusLabel("Waiting for image upload..."); }else{ let progress = 100; let type = 'done'; diff --git a/worker/tasks.py b/worker/tasks.py index 19e9063f..f16da20c 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -124,10 +124,11 @@ def get_pending_tasks(): # Or that need one assigned (via auto) # or tasks that need a status update # or tasks that have a pending action - return Task.objects.filter(Q(processing_node__isnull=True, auto_processing_node=True) | + # no partial tasks allowed + return Task.objects.filter(Q(processing_node__isnull=True, auto_processing_node=True, partial=False) | Q(Q(status=None) | Q(status__in=[status_codes.QUEUED, status_codes.RUNNING]), - processing_node__isnull=False) | - Q(pending_action__isnull=False)) + processing_node__isnull=False, partial=False) | + Q(pending_action__isnull=False, partial=False)) @app.task def process_pending_tasks():