From 6b4230f23313264a56f2dd0de12ad3ea4ec81cd8 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 6 Nov 2023 10:35:02 -0500 Subject: [PATCH 1/4] Improve cancel task logic --- .../app/js/components/ProjectListItem.jsx | 33 ++++++++++++++++--- app/static/app/js/vendor/dropzone.js | 2 ++ package.json | 2 +- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/app/static/app/js/components/ProjectListItem.jsx b/app/static/app/js/components/ProjectListItem.jsx index 31d6ec42..7c7bdf23 100644 --- a/app/static/app/js/components/ProjectListItem.jsx +++ b/app/static/app/js/components/ProjectListItem.jsx @@ -60,6 +60,7 @@ class ProjectListItem extends React.Component { this.toggleTaskList = this.toggleTaskList.bind(this); this.closeUploadError = this.closeUploadError.bind(this); this.cancelUpload = this.cancelUpload.bind(this); + this.handleCancel = this.handleCancel.bind(this); this.handleTaskSaved = this.handleTaskSaved.bind(this); this.viewMap = this.viewMap.bind(this); this.handleDelete = this.handleDelete.bind(this); @@ -248,13 +249,19 @@ class ProjectListItem extends React.Component { } } }catch(e){ - this.setUploadState({error: `${e.message}`, uploading: false}); - this.dz.cancelUpload(); + if (this.manuallyCanceled){ + // Manually canceled, ignore error + this.setUploadState({uploading: false}); + }else{ + this.setUploadState({error: `${e.message}`, uploading: false}); + } + + if (this.dz.files.length) this.dz.cancelUpload(); } }) .on("queuecomplete", () => { const remainingFilesCount = this.state.upload.totalCount - this.state.upload.uploadedCount; - if (remainingFilesCount === 0){ + if (remainingFilesCount === 0 && this.state.upload.uploadedCount > 0){ // All files have uploaded! this.setUploadState({uploading: false}); @@ -332,10 +339,26 @@ class ProjectListItem extends React.Component { this.setUploadState({error: ""}); } - cancelUpload(e){ + cancelUpload(){ this.dz.removeAllFiles(true); } + handleCancel(){ + this.manuallyCanceled = true; + this.cancelUpload(); + if (this.dz._taskInfo && this.dz._taskInfo.id !== undefined){ + $.ajax({ + url: `/api/projects/${this.state.data.id}/tasks/${this.dz._taskInfo.id}/remove/`, + contentType: 'application/json', + dataType: 'json', + type: 'POST' + }); + } + setTimeout(() => { + this.manuallyCanceled = false; + }, 500); + } + taskDeleted(){ this.refresh(); } @@ -628,7 +651,7 @@ class ProjectListItem extends React.Component { diff --git a/app/static/app/js/vendor/dropzone.js b/app/static/app/js/vendor/dropzone.js index 0beabbb3..e904e1f7 100644 --- a/app/static/app/js/vendor/dropzone.js +++ b/app/static/app/js/vendor/dropzone.js @@ -2199,6 +2199,8 @@ var Dropzone = function (_Emitter) { }, { key: "cancelUpload", value: function cancelUpload(file) { + if (file === undefined) return; + if (file.status === Dropzone.UPLOADING) { var groupedFiles = this._getFilesWithXhr(file.xhr); for (var _iterator19 = groupedFiles, _isArray19 = true, _i20 = 0, _iterator19 = _isArray19 ? _iterator19 : _iterator19[Symbol.iterator]();;) { diff --git a/package.json b/package.json index 1018c1b4..d18eabf8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "WebODM", - "version": "2.2.0", + "version": "2.2.1", "description": "User-friendly, extendable application and API for processing aerial imagery.", "main": "index.js", "scripts": { From 9fb0c6db67bb2f1029497650c71460c5d37e9ea7 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 6 Nov 2023 11:21:10 -0500 Subject: [PATCH 2/4] Cleanup partial tasks --- app/admin.py | 4 ++-- webodm/settings.py | 4 ++++ worker/celery.py | 8 ++++++++ worker/tasks.py | 12 ++++++++++++ 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/app/admin.py b/app/admin.py index 7ad77d95..7bb98a5d 100644 --- a/app/admin.py +++ b/app/admin.py @@ -40,9 +40,9 @@ class TaskAdmin(admin.ModelAdmin): def has_add_permission(self, request): return False - list_display = ('id', 'project', 'processing_node', 'created_at', 'status', 'last_error') + list_display = ('id', 'name', 'project', 'processing_node', 'created_at', 'status', 'last_error') list_filter = ('status', 'project',) - search_fields = ('id', 'project__name') + search_fields = ('id', 'name', 'project__name') admin.site.register(Task, TaskAdmin) diff --git a/webodm/settings.py b/webodm/settings.py index 49806762..5287252f 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -411,6 +411,10 @@ QUOTA_EXCEEDED_GRACE_PERIOD = 8 # Maximum number of processing nodes to show in "Processing Nodes" menus/dropdowns UI_MAX_PROCESSING_NODES = None +# Number of hours before partial tasks +# are removed (or None to disable) +CLEANUP_PARTIAL_TASKS = None + if TESTING or FLUSHING: CELERY_TASK_ALWAYS_EAGER = True EXTERNAL_AUTH_ENDPOINT = 'http://0.0.0.0:5555/auth' diff --git a/worker/celery.py b/worker/celery.py index cb9209e1..7903f2b5 100644 --- a/worker/celery.py +++ b/worker/celery.py @@ -28,6 +28,14 @@ app.conf.beat_schedule = { 'retry': False } }, + 'cleanup-tasks': { + 'task': 'worker.tasks.cleanup_tasks', + 'schedule': 3600, + 'options': { + 'expires': 1799, + 'retry': False + } + }, 'cleanup-tmp-directory': { 'task': 'worker.tasks.cleanup_tmp_directory', 'schedule': 3600, diff --git a/worker/tasks.py b/worker/tasks.py index 5ca8f24c..5ca53841 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -23,6 +23,8 @@ import worker from .celery import app from app.raster_utils import export_raster as export_raster_sync, extension_for_export_format from app.pointcloud_utils import export_pointcloud as export_pointcloud_sync +from django.utils import timezone +from datetime import timedelta import redis logger = get_task_logger("app.logger") @@ -67,6 +69,16 @@ def cleanup_projects(): if total > 0 and 'app.Project' in count_dict: logger.info("Deleted {} projects".format(count_dict['app.Project'])) +@app.task(ignore_result=True) +def cleanup_tasks(): + # Delete tasks that are older than + if settings.CLEANUP_PARTIAL_TASKS is None: + return + + tasks_to_delete = Task.objects.filter(partial=True, created_at__lte=timezone.now() - timedelta(hours=settings.CLEANUP_PARTIAL_TASKS)) + for t in tasks_to_delete: + logger.info("Cleaning up partial task {}".format(t)) + t.delete() @app.task(ignore_result=True) def cleanup_tmp_directory(): From 58dcc46e403900293ff4646a5b4663f257d4d3ff Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 6 Nov 2023 11:22:39 -0500 Subject: [PATCH 3/4] Cleanup partial tasks --- webodm/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webodm/settings.py b/webodm/settings.py index 5287252f..216289cd 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -413,7 +413,7 @@ UI_MAX_PROCESSING_NODES = None # Number of hours before partial tasks # are removed (or None to disable) -CLEANUP_PARTIAL_TASKS = None +CLEANUP_PARTIAL_TASKS = 72 if TESTING or FLUSHING: CELERY_TASK_ALWAYS_EAGER = True From ce9cd0a8b9e9c94bd8a052e83af751a536b53144 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 6 Nov 2023 12:19:41 -0500 Subject: [PATCH 4/4] Add configurable docs/task options links --- app/static/app/js/components/ProcessingNodeOption.jsx | 9 ++++++++- app/static/app/js/components/TaskListItem.jsx | 4 ++-- app/static/app/js/css/ProcessingNodeOption.scss | 4 ++++ app/templates/app/base.html | 3 +++ app/templates/app/dashboard.html | 2 +- app/templatetags/settings.py | 8 ++++++++ app/views/app.py | 2 +- webodm/settings.py | 9 +++++++++ 8 files changed, 36 insertions(+), 5 deletions(-) diff --git a/app/static/app/js/components/ProcessingNodeOption.jsx b/app/static/app/js/components/ProcessingNodeOption.jsx index 84a918c1..d643706e 100644 --- a/app/static/app/js/components/ProcessingNodeOption.jsx +++ b/app/static/app/js/components/ProcessingNodeOption.jsx @@ -106,6 +106,13 @@ class ProcessingNodeOption extends React.Component { } } + handleHelp = e => { + e.preventDefault(); + if (window.__taskOptionsDocsLink){ + window.open(window.__taskOptionsDocsLink + "#" + encodeURIComponent(this.props.name), "task-options") + } + } + render() { let inputControl = ""; let warningMsg = ""; @@ -168,7 +175,7 @@ class ProcessingNodeOption extends React.Component { return (
-
+
{inputControl} {loadFileControl} diff --git a/app/static/app/js/components/TaskListItem.jsx b/app/static/app/js/components/TaskListItem.jsx index a8f58114..54ddf17b 100644 --- a/app/static/app/js/components/TaskListItem.jsx +++ b/app/static/app/js/components/TaskListItem.jsx @@ -605,14 +605,14 @@ class TaskListItem extends React.Component {
{_("An orthophoto could not be generated. To generate one, make sure GPS information is embedded in the EXIF tags of your images, or use a Ground Control Points (GCP) file.")}
: ""} {showMemoryErrorWarning ? -
${_("enough RAM allocated")}`, cloudlink: `${_("cloud processing node")}` }}>{_("It looks like your processing node ran out of memory. If you are using docker, make sure that your docker environment has %(memlink)s. Alternatively, make sure you have enough physical RAM, reduce the number of images, make your images smaller, or reduce the max-concurrency parameter from the task's options. You can also try to use a %(cloudlink)s.")}
: ""} +
${_("enough RAM allocated")}`, cloudlink: `${_("cloud processing node")}` }}>{_("It looks like your processing node ran out of memory. If you are using docker, make sure that your docker environment has %(memlink)s. Alternatively, make sure you have enough physical RAM, reduce the number of images, make your images smaller, or reduce the max-concurrency parameter from the task's options. You can also try to use a %(cloudlink)s.")}
: ""} {showTaskWarning ?
: ""} {showExitedWithCodeOneHints ?
- docs.opendronemap.org` }}>{_("\"Process exited with code 1\" means that part of the processing failed. Sometimes it's a problem with the dataset, sometimes it can be solved by tweaking the Task Options. Check the documentation at %(link)")} + ${window.__taskOptionsDocsLink.replace("https://", "")}` }}>{_("\"Process exited with code 1\" means that part of the processing failed. Sometimes it's a problem with the dataset, sometimes it can be solved by tweaking the Task Options. Check the documentation at %(link)s")}
: ""} diff --git a/app/static/app/js/css/ProcessingNodeOption.scss b/app/static/app/js/css/ProcessingNodeOption.scss index dd3c37d7..a09e2cb5 100644 --- a/app/static/app/js/css/ProcessingNodeOption.scss +++ b/app/static/app/js/css/ProcessingNodeOption.scss @@ -29,4 +29,8 @@ padding: 2px 4px 2px 4px; margin-top: 12px; } + + .help-button:hover{ + cursor: pointer; + } } diff --git a/app/templates/app/base.html b/app/templates/app/base.html index 53b6d013..8ed5296f 100644 --- a/app/templates/app/base.html +++ b/app/templates/app/base.html @@ -115,6 +115,9 @@