Minor tweaks

Clean tasks randomly canceled

Two tasks identical tasks no longer be submitted

Loading button on submit tasks

Rename project

plugins/cesium-ion/api_views.py

@@ -4,12 +4,14 @@ import logging
 import requests
 from os import path
 from enum import Enum
+from itertools import chain as iter_chain
 
 from app.plugins.views import TaskView
 from app.plugins.worker import task
 from app.plugins.data_store import GlobalDataStore
 from app.plugins import logger, signals as plugin_signals
 
+from worker.celery import app
 from django.dispatch import receiver
 from django.utils.translation import ugettext_lazy as _
 from rest_framework.fields import ChoiceField, CharField, JSONField
@@ -67,6 +69,33 @@ def get_asset_info(task_id, asset_type, default=None, ds=None):
     return ds.get_json(get_key_for(task_id, asset_type.value), default)
 
 
+def is_asset_task(asset_meta):
+    is_error = len(asset_meta["error"]) > 0
+    return asset_meta["upload"]["active"] or asset_meta["process"]["active"] or is_error
+
+
+def get_processing_assets(task_id):
+    ispc = app.control.inspect()
+    ion_tasks = set()
+    active = set()
+    from uuid import UUID
+
+    for wtask in iter_chain(*ispc.active().values(), *ispc.reserved().values()):
+        args = eval(wtask["args"])
+        if len(args) < 2:
+            continue
+        ion_tasks.add((str(args[0]), AssetType[args[1]]))
+
+    for asset_type in AssetType:
+        asset_info = get_asset_info(task_id, asset_type)
+        ion_task_id = (task_id, asset_type)
+        if not is_asset_task(asset_info) or ion_task_id in ion_tasks:
+            continue
+        active.add(asset_type)
+
+    return active
+
+
 ###                        ###
 #      MODEL CONFIG          #
 ###                        ###
@@ -182,11 +211,7 @@ class ShareTaskView(TaskView):
             asset_info = get_asset_info(task.id, asset_type)
             ion_id = asset_info["id"]
             is_error = len(asset_info["error"]) > 0
-            is_task = (
-                asset_info["upload"]["active"]
-                or asset_info["process"]["active"]
-                or is_error
-            )
+            is_task = is_asset_task(asset_info)
             is_exported = asset_info["id"] is not None and not is_task
 
             assets.append(
@@ -217,21 +242,26 @@ class ShareTaskView(TaskView):
         )
         asset_path = task.get_asset_download_path(ASSET_TO_FILE[asset_type])
 
-        del_asset_info(task.id, asset_type)
-        asset_info = get_asset_info(task.id, asset_type)
-        asset_info["upload"]["active"] = True
-        set_asset_info(task.id, asset_type, asset_info)
+        # Skip already processing tasks
+        if asset_type not in get_processing_assets(task.id):
+            del_asset_info(task.id, asset_type)
+            asset_info = get_asset_info(task.id, asset_type)
+            asset_info["upload"]["active"] = True
+            set_asset_info(task.id, asset_type, asset_info)
+
+            upload_to_ion.delay(
+                task.id,
+                asset_type,
+                token,
+                asset_path,
+                name,
+                description,
+                attribution,
+                options,
+            )
+        else:
+            print(f"Ignore running ion task {task.id} {str(asset_type)}")
 
-        upload_to_ion.delay(
-            task.id,
-            token,
-            asset_type,
-            asset_path,
-            name,
-            description,
-            attribution,
-            options,
-        )
         return Response(status=status.HTTP_200_OK)
 
 
@@ -245,6 +275,7 @@ class RefreshIonTaskView(TaskView):
         headers = {"Authorization": f"Bearer {token}"}
 
         is_updated = False
+        # ion cleanup check
         for asset_type in AssetType:
             asset_info = get_asset_info(task.id, asset_type)
             ion_id = asset_info["id"]
@@ -255,6 +286,11 @@ class RefreshIonTaskView(TaskView):
                 del_asset_info(task.id, asset_type)
                 is_updated = True
 
+        # dead task cleanup
+        for asset_type in get_processing_assets(task.id):
+            del_asset_info(task.id, asset_type)
+            is_updated = True
+
         return Response({"updated": is_updated}, status=status.HTTP_200_OK)
 
 
@@ -274,7 +310,7 @@ class ClearErrorsTaskView(TaskView):
 #       CELERY TASK(S)       #
 ###                        ###
 class TaskUploadProgress(object):
-    def __init__(self, file_path, task_id, asset_type, logger=None):
+    def __init__(self, file_path, task_id, asset_type, logger=None, log_step_size=0.05):
         self._task_id = task_id
         self._asset_type = asset_type
         self._logger = logger
@@ -283,6 +319,9 @@ class TaskUploadProgress(object):
         self._total_bytes = float(path.getsize(file_path))
         self._asset_info = get_asset_info(task_id, asset_type)
 
+        self._last_log = 0
+        self._log_step_size = log_step_size
+
     @property
     def asset_info(self):
         return self._asset_info
@@ -294,17 +333,20 @@ class TaskUploadProgress(object):
             progress = 1
 
         self._asset_info["upload"]["progress"] = progress
-        if self._logger is not None:
+        if self._logger is not None and progress - self._last_log > self._log_step_size:
             self._logger.info(f"Upload progress: {progress * 100}%")
+            self._last_log = progress
 
         set_asset_info(self._task_id, self._asset_type, self._asset_info)
 
 
+# Arg order is very important for task deconstruction.
+# If order is changed make sure that the refresh API call is updated
 @task
 def upload_to_ion(
     task_id,
-    token,
     asset_type,
+    token,
     asset_path,
     name,
     description="",

plugins/cesium-ion/app_views.py

@@ -51,20 +51,21 @@ def HomeView(plugin):
 
 
 def LoadButtonView(plugin):
-    def load_buttons_callback(request):
-        if request.user.is_authenticated:
-            ds = plugin.get_user_data_store(request.user)
-            token = ds.get_string("token")
-            if len(token) <= 0:
-                return False
+    @login_required
+    def view(request):
+        ds = plugin.get_user_data_store(request.user)
+        token = ds.get_string("token")
 
-            return {
+        return render(
+            request,
+            plugin.template_path("load_buttons.js"),
+            {
                 "token": token,
                 "app_name": plugin.get_name(),
                 "api_url": plugin.public_url("").rstrip("/"),
                 "ion_url": ION_API_URL,
-            }
-        else:
-            return False
+            },
+            content_type="text/javascript",
+        )
 
-    return plugin.get_dynamic_script("templates/load_buttons.js", load_buttons_callback)
+    return view

plugins/cesium-ion/public/TaskView.jsx

@@ -18,7 +18,8 @@ export default class TaskView extends Component {
 	state = {
 		error: "",
 		currentAsset: null,
-		isTasksDialog: false
+		isTasksDialog: false,
+		isUploadDialogLoading: false
 	};
 
 	cancelableFetch = null;
@@ -27,7 +28,8 @@ export default class TaskView extends Component {
 
 	onOpenUploadDialog = asset => this.setState({ currentAsset: asset });
 
-	onHideUploadDialog = () => this.setState({ currentAsset: null });
+	onHideUploadDialog = () =>
+		this.setState({ currentAsset: null, isUploadDialogLoading: false });
 
 	showTaskDialog = () => this.setState({ isTasksDialog: true });
 
@@ -46,6 +48,7 @@ export default class TaskView extends Component {
 		payload.token = token;
 		payload.asset_type = currentAsset;
 
+		this.setState({ isUploadDialogLoading: true });
 		this.cancelableFetch = fetchCancelable(
 			`/api${apiURL}/task/${task.id}/share`,
 			{
@@ -126,7 +129,12 @@ export default class TaskView extends Component {
 
 	render() {
 		const { task, token } = this.props;
-		const { isTasksDialog, isRefreshTask, currentAsset } = this.state;
+		const {
+			isTasksDialog,
+			isUploadDialogLoading,
+			isRefreshTask,
+			currentAsset
+		} = this.state;
 		const isUploadDialog = currentAsset !== null;
 		const assetName = isUploadDialog ? AssetStyles[currentAsset].name : "";
 
@@ -219,7 +227,7 @@ export default class TaskView extends Component {
 					</TaskFetcher>
 				</div>
 
-				<APIFetcher path={"projects"} params={{ id: task.project }}>
+				<APIFetcher path={"projects/"} params={{ id: task.project }}>
 					{({ isLoading, isError, data }) => {
 						const initialValues = {};
 
@@ -234,6 +242,7 @@ export default class TaskView extends Component {
 								title={`Tile in Cesium ion — ${assetName}`}
 								initialValues={initialValues}
 								show={isUploadDialog}
+								loading={isUploadDialogLoading}
 								asset={currentAsset}
 								onHide={this.onHideUploadDialog}
 								onSubmit={this.onUploadAsset}

plugins/cesium-ion/public/components/UploadDialog.jsx

@@ -23,6 +23,7 @@ export default class UploadDialog extends Component {
 	static defaultProps = {
 		show: true,
 		asset: null,
+		loading: false,
 		initialValues: {
 			name: "",
 			description: "",
@@ -142,7 +143,13 @@ export default class UploadDialog extends Component {
 	}
 
 	render() {
-		const { initialValues, onHide, title, ...options } = this.props;
+		const {
+			initialValues,
+			onHide,
+			title,
+			loading: isLoading,
+			...options
+		} = this.props;
 
 		delete options["asset"];
 		delete options["onSubmit"];
@@ -191,13 +198,23 @@ export default class UploadDialog extends Component {
 
 							<Modal.Footer>
 								<Button onClick={onHide}>Close</Button>
-								<Button
-									bsStyle="primary"
-									onClick={handleSubmit}
-								>
-									<i className={"fa fa-upload"} />
-									Submit
-								</Button>
+								{isLoading && (
+									<Button bsStyle="primary" disabled>
+										<i
+											className={"fa fa-refresh fa-spin"}
+										/>
+										Submitting...
+									</Button>
+								)}
+								{!isLoading && (
+									<Button
+										bsStyle="primary"
+										onClick={handleSubmit}
+									>
+										<i className={"fa fa-upload"} />
+										Submit
+									</Button>
+								)}
 							</Modal.Footer>
 						</form>
 					)}

plugins/cesium-ion/templates/load_buttons.js

@@ -1,11 +1,12 @@
 PluginsAPI.Dashboard.addTaskActionButton(
-	["${app_name}/build/TaskView.js"],
+	["{{ app_name }}/build/TaskView.js"],
 	function(args, TaskView) {
+		console.log(args);
 		return React.createElement(TaskView, {
 			task: args.task,
-			token: "${token}",
-			apiURL: "${api_url}",
-			ionURL: "${ion_url}"
+			token: "{{ token }}",
+			apiURL: "{{ api_url }}",
+			ionURL: "{{ ion_url }}"
 		});
 	}
 );