diff --git a/app/api/tasks.py b/app/api/tasks.py index 32645751..08178018 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -36,7 +36,6 @@ class TaskSerializer(serializers.ModelSerializer): project = serializers.PrimaryKeyRelatedField(queryset=models.Project.objects.all()) processing_node = serializers.PrimaryKeyRelatedField(queryset=ProcessingNode.objects.all()) processing_node_name = serializers.SerializerMethodField() - images_count = serializers.SerializerMethodField() can_rerun_from = serializers.SerializerMethodField() def get_processing_node_name(self, obj): @@ -45,10 +44,6 @@ class TaskSerializer(serializers.ModelSerializer): else: return None - def get_images_count(self, obj): - # TODO: create a field in the model for this - return obj.imageupload_set.count() - def get_can_rerun_from(self, obj): """ When a task has been associated with a processing node @@ -164,6 +159,7 @@ class TaskViewSet(viewsets.ViewSet): for image in files: models.ImageUpload.objects.create(task=task, image=image) + task.images_count = len(files) # Update other parameters such as processing node, task name, etc. serializer = TaskSerializer(task, data=request.data, partial=True) @@ -341,23 +337,29 @@ class TaskAssetsImport(APIView): project = get_and_check_project(request, project_pk, ('change_project',)) files = flatten_files(request.FILES) + import_url = request.data.get('url', None) + task_name = request.data.get('name', 'Imported Task') - if len(files) != 1: + if not import_url and len(files) != 1: raise exceptions.ValidationError(detail="Cannot create task, you need to upload 1 file") + if import_url and len(files) > 0: + raise exceptions.ValidationError(detail="Cannot create task, either specify a URL or upload 1 file.") + with transaction.atomic(): task = models.Task.objects.create(project=project, auto_processing_node=False, - name="Imported Task", - import_url="file://all.zip", + name=task_name, + import_url=import_url if import_url else "file://all.zip", status=status_codes.RUNNING, pending_action=pending_actions.IMPORT) task.create_task_directories() - destination_file = task.assets_path("all.zip") - with open(destination_file, 'wb+') as fd: - for chunk in files[0].chunks(): - fd.write(chunk) + if len(files) > 0: + destination_file = task.assets_path("all.zip") + with open(destination_file, 'wb+') as fd: + for chunk in files[0].chunks(): + fd.write(chunk) worker_tasks.process_task.delay(task.id) diff --git a/app/migrations/0025_auto_20190220_1854.py b/app/migrations/0025_auto_20190220_1854.py index 2bbf86ce..ac4d8619 100644 --- a/app/migrations/0025_auto_20190220_1854.py +++ b/app/migrations/0025_auto_20190220_1854.py @@ -17,36 +17,53 @@ class Migration(migrations.Migration): migrations.AddField( model_name='task', name='import_url', - field=models.TextField(blank=True, default='', help_text='URL this task is imported from (only for imported tasks)'), + field=models.TextField(blank=True, default='', + help_text='URL this task is imported from (only for imported tasks)'), ), migrations.AlterField( model_name='preset', name='options', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=list, help_text="Options that define this preset (same format as in a Task's options).", validators=[app.models.task.validate_task_options]), + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=list, + help_text="Options that define this preset (same format as in a Task's options).", + validators=[app.models.task.validate_task_options]), ), migrations.AlterField( model_name='task', name='available_assets', - field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=80), blank=True, default=list, help_text='List of available assets to download', size=None), + field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=80), blank=True, + default=list, + help_text='List of available assets to download', + size=None), ), migrations.AlterField( model_name='task', name='options', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict, help_text='Options that are being used to process this task', validators=[app.models.task.validate_task_options]), + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict, + help_text='Options that are being used to process this task', + validators=[app.models.task.validate_task_options]), ), migrations.AlterField( model_name='task', name='pending_action', - field=models.IntegerField(blank=True, choices=[(1, 'CANCEL'), (2, 'REMOVE'), (3, 'RESTART'), (4, 'RESIZE'), (5, 'IMPORT')], db_index=True, help_text='A requested action to be performed on the task. The selected action will be performed by the worker at the next iteration.', null=True), + field=models.IntegerField(blank=True, choices=[(1, 'CANCEL'), (2, 'REMOVE'), (3, 'RESTART'), (4, 'RESIZE'), + (5, 'IMPORT')], db_index=True, + help_text='A requested action to be performed on the task. The selected action will be performed by the worker at the next iteration.', + null=True), ), migrations.AlterField( model_name='theme', name='header_background', - field=colorfield.fields.ColorField(default='#3498db', help_text="Background color of the site's header.", max_length=18), + field=colorfield.fields.ColorField(default='#3498db', help_text="Background color of the site's header.", + max_length=18), ), migrations.AlterField( model_name='theme', name='tertiary', field=colorfield.fields.ColorField(default='#3498db', help_text='Navigation links.', max_length=18), ), + migrations.AddField( + model_name='task', + name='images_count', + field=models.IntegerField(blank=True, default=0, help_text='Number of images associated with this task'), + ), ] diff --git a/app/models/task.py b/app/models/task.py index a654d5b0..c2111477 100644 --- a/app/models/task.py +++ b/app/models/task.py @@ -226,7 +226,7 @@ class Task(models.Model): help_text="Value between 0 and 1 indicating the running progress (estimated) of this task", blank=True) import_url = models.TextField(null=False, default="", blank=True, help_text="URL this task is imported from (only for imported tasks)") - + images_count = models.IntegerField(null=False, blank=True, default=0, help_text="Number of images associated with this task") def __init__(self, *args, **kwargs): super(Task, self).__init__(*args, **kwargs) @@ -340,7 +340,50 @@ class Task(models.Model): def handle_import(self): self.console_output += "Importing assets...\n" self.save() + + zip_path = self.assets_path("all.zip") + + if self.import_url and not os.path.exists(zip_path): + try: + # TODO: this is potentially vulnerable to a zip bomb attack + # mitigated by the fact that a valid account is needed to + # import tasks + download_stream = requests.get(self.import_url, stream=True, timeout=10) + content_length = download_stream.headers.get('content-length') + total_length = int(content_length) if content_length is not None else None + downloaded = 0 + last_update = 0 + + with open(zip_path, 'wb') as fd: + for chunk in download_stream.iter_content(4096): + downloaded += len(chunk) + + if time.time() - last_update >= 2: + # Update progress + if total_length is not None: + Task.objects.filter(pk=self.id).update(running_progress=(float(downloaded) / total_length) * 0.9) + + self.check_if_canceled() + last_update = time.time() + + fd.write(chunk) + + except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, ReadTimeoutError) as e: + raise ProcessingError(e) + + self.refresh_from_db() self.extract_assets_and_complete() + + images_json = self.assets_path("images.json") + if os.path.exists(images_json): + try: + with open(images_json) as f: + images = json.load(f) + self.images_count = len(images) + except: + logger.warning("Cannot read images count from imported task {}".format(self)) + pass + self.pending_action = None self.processing_time = 0 self.save() @@ -441,6 +484,11 @@ class Task(models.Model): except ProcessingException: logger.warning("Could not cancel {} on processing node. We'll proceed anyway...".format(self)) + self.status = status_codes.CANCELED + self.pending_action = None + self.save() + elif self.import_url: + # Imported tasks need no special action self.status = status_codes.CANCELED self.pending_action = None self.save() @@ -613,8 +661,11 @@ class Task(models.Model): zip_path = self.assets_path("all.zip") # Extract from zip - with zipfile.ZipFile(zip_path, "r") as zip_h: - zip_h.extractall(assets_dir) + try: + with zipfile.ZipFile(zip_path, "r") as zip_h: + zip_h.extractall(assets_dir) + except zipfile.BadZipFile: + raise ProcessingError("Corrupted zip file") logger.info("Extracted all.zip for {}".format(self)) diff --git a/app/static/app/js/components/ImportTaskPanel.jsx b/app/static/app/js/components/ImportTaskPanel.jsx index d39094e6..b7b953f7 100644 --- a/app/static/app/js/components/ImportTaskPanel.jsx +++ b/app/static/app/js/components/ImportTaskPanel.jsx @@ -3,13 +3,15 @@ import React from 'react'; import PropTypes from 'prop-types'; import Dropzone from '../vendor/dropzone'; import csrf from '../django/csrf'; +import ErrorMessage from './ErrorMessage'; +import UploadProgressBar from './UploadProgressBar'; class ImportTaskPanel extends React.Component { static defaultProps = { }; static propTypes = { - // onSave: PropTypes.func.isRequired, + onImported: PropTypes.func.isRequired, onCancel: PropTypes.func, projectId: PropTypes.number.isRequired }; @@ -18,9 +20,20 @@ class ImportTaskPanel extends React.Component { super(props); this.state = { + error: "", + typeUrl: false, + uploading: false, + importingFromUrl: false, + progress: 0, + bytesSent: 0, + importUrl: "" }; } + defaultTaskName = () => { + return `Task of ${new Date().toISOString()}`; + } + componentDidMount(){ Dropzone.autoDiscover = false; @@ -42,24 +55,90 @@ class ImportTaskPanel extends React.Component { } }); - this.dz.on("error", function(file){ - // Show + this.dz.on("error", (file) => { + if (this.state.uploading) this.setState({error: "Cannot upload file. Check your internet connection and try again."}); }) - .on("uploadprogress", function(file, progress){ - console.log(progress); + .on("sending", () => { + this.setState({typeUrl: false, uploading: true, totalCount: 1}); }) - .on("complete", function(file){ - if (file.status === "success"){ - }else{ - // error + .on("reset", () => { + this.setState({uploading: false, progress: 0, totalBytes: 0, totalBytesSent: 0}); + }) + .on("uploadprogress", (file, progress, bytesSent) => { + this.setState({ + progress, + totalBytes: file.size, + totalBytesSent: bytesSent + }); + }) + .on("sending", (file, xhr, formData) => { + // Safari does not have support for has on FormData + // as of December 2017 + if (!formData.has || !formData.has("name")) formData.append("name", this.defaultTaskName()); + }) + .on("complete", (file) => { + if (file.status === "success"){ + this.setState({uploading: false}); + try{ + let response = JSON.parse(file.xhr.response); + if (!response.id) throw new Error(`Expected id field, but none given (${response})`); + this.props.onImported(); + }catch(e){ + this.setState({error: `Invalid response from server: ${e.message}`}); } + }else if (this.state.uploading){ + this.setState({uploading: false, error: "An error occured while uploading the file. Please try again."}); + } }); } cancel = (e) => { + this.cancelUpload(); this.props.onCancel(); } + cancelUpload = (e) => { + this.setState({uploading: false}); + setTimeout(() => { + this.dz.removeAllFiles(true); + }, 0); + } + + handleImportFromUrl = () => { + this.setState({typeUrl: !this.state.typeUrl}); + } + + handleCancelImportFromURL = () => { + this.setState({typeUrl: false}); + } + + handleChangeImportUrl = (e) => { + this.setState({importUrl: e.target.value}); + } + + handleConfirmImportUrl = () => { + this.setState({importingFromUrl: true}); + + $.post(`/api/projects/${this.props.projectId}/tasks/import`, + { + url: this.state.importUrl, + name: this.defaultTaskName() + } + ).done(json => { + if (json.id){ + this.props.onImported(); + }else{ + this.setState({error: json.error || `Cannot import from URL, server responded: ${JSON.stringify(json)}`}); + } + }) + .fail(() => { + this.setState({error: "Cannot import from URL. Check your internet connection."}); + }) + .always(() => { + this.setState({importingFromUrl: false}); + }); + } + setRef = (prop) => { return (domNode) => { if (domNode != null) this[prop] = domNode; @@ -70,23 +149,47 @@ class ImportTaskPanel extends React.Component { return (
+ +

Import Existing Assets

You can import .zip files that have been exported from existing tasks via Download Assets All Assets.

- - + + {this.state.typeUrl ? +
+
+ + +
+
: ""} + + {this.state.uploading ?
+ + +
: ""}
); diff --git a/app/static/app/js/components/ProjectListItem.jsx b/app/static/app/js/components/ProjectListItem.jsx index 15fece74..cabce614 100644 --- a/app/static/app/js/components/ProjectListItem.jsx +++ b/app/static/app/js/components/ProjectListItem.jsx @@ -204,13 +204,7 @@ class ProjectListItem extends React.Component { let response = JSON.parse(files[0].xhr.response); if (!response.id) throw new Error(`Expected id field, but none given (${response})`); - if (this.state.showTaskList){ - this.taskList.refresh(); - }else{ - this.setState({showTaskList: true}); - } - this.resetUploadState(); - this.refresh(); + this.newTaskAdded(); }catch(e){ this.setUploadState({error: `Invalid response from server: ${e.message}`, uploading: false}) } @@ -247,6 +241,18 @@ class ProjectListItem extends React.Component { } } + newTaskAdded = () => { + this.setState({importing: false}); + + if (this.state.showTaskList){ + this.taskList.refresh(); + }else{ + this.setState({showTaskList: true}); + } + this.resetUploadState(); + this.refresh(); + } + setRef(prop){ return (domNode) => { if (domNode != null) this[prop] = domNode; @@ -448,6 +454,7 @@ class ProjectListItem extends React.Component { {this.state.importing ? diff --git a/app/static/app/js/components/TaskListItem.jsx b/app/static/app/js/components/TaskListItem.jsx index 095c06bb..24f29aa2 100644 --- a/app/static/app/js/components/TaskListItem.jsx +++ b/app/static/app/js/components/TaskListItem.jsx @@ -406,12 +406,13 @@ class TaskListItem extends React.Component { } if ([statusCodes.QUEUED, statusCodes.RUNNING, null].indexOf(task.status) !== -1 && - task.processing_node){ + (task.processing_node || imported)){ addActionButton("Cancel", "btn-primary", "glyphicon glyphicon-remove-circle", this.genActionApiCall("cancel", {defaultError: "Cannot cancel task."})); } if ([statusCodes.FAILED, statusCodes.COMPLETED, statusCodes.CANCELED].indexOf(task.status) !== -1 && - task.processing_node){ + task.processing_node && + !imported){ // By default restart reruns every pipeline // step from the beginning const rerunFrom = task.can_rerun_from.length > 1 ? diff --git a/app/static/app/js/css/ImportTaskPanel.scss b/app/static/app/js/css/ImportTaskPanel.scss index a8047266..bbcc68c5 100644 --- a/app/static/app/js/css/ImportTaskPanel.scss +++ b/app/static/app/js/css/ImportTaskPanel.scss @@ -17,4 +17,13 @@ .close:hover, .close:focus{ color: inherit; } + + .upload-progress-bar{ + margin-top: 12px; + } + + .btn-import{ + margin-top: 8px; + margin-left: 8px; + } } \ No newline at end of file diff --git a/app/static/app/js/css/TaskListItem.scss b/app/static/app/js/css/TaskListItem.scss index d712c97d..a3064759 100644 --- a/app/static/app/js/css/TaskListItem.scss +++ b/app/static/app/js/css/TaskListItem.scss @@ -15,6 +15,7 @@ .name{ padding-left: 0; + margin-top: 4px; } .details{