From 23f65bf9cd2260d48e9c96d0f6f9601bf579aa22 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 15 Nov 2016 11:51:19 -0500 Subject: [PATCH] Can delete projects --- app/api/projects.py | 4 +- app/api/tasks.py | 8 +- app/models.py | 73 ++++++++------- app/pending_actions.py | 3 + app/scheduler.py | 15 +++- .../app/js/components/EditProjectDialog.jsx | 35 ++++++-- app/static/app/js/components/ProjectList.jsx | 9 +- .../app/js/components/ProjectListItem.jsx | 89 ++++++++++++------- app/static/app/js/css/ProjectListItem.scss | 7 ++ app/tests/test_api.py | 8 +- 10 files changed, 169 insertions(+), 82 deletions(-) create mode 100644 app/pending_actions.py diff --git a/app/api/projects.py b/app/api/projects.py index a2865a1b..c614f90e 100644 --- a/app/api/projects.py +++ b/app/api/projects.py @@ -14,7 +14,7 @@ class ProjectSerializer(serializers.ModelSerializer): class Meta: model = models.Project - fields = '__all__' + exclude = ('deleting', ) class ProjectViewSet(viewsets.ModelViewSet): @@ -28,5 +28,5 @@ class ProjectViewSet(viewsets.ModelViewSet): """ filter_fields = ('id', 'name', 'description', 'created_at') serializer_class = ProjectSerializer - queryset = models.Project.objects.all() + queryset = models.Project.objects.filter(deleting=False) ordering_fields = '__all__' \ No newline at end of file diff --git a/app/api/tasks.py b/app/api/tasks.py index a9616367..e6606b15 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -9,7 +9,7 @@ from rest_framework.response import Response from rest_framework.decorators import detail_route from rest_framework.views import APIView -from app import models, scheduler +from app import models, scheduler, pending_actions from nodeodm.models import ProcessingNode @@ -76,15 +76,15 @@ class TaskViewSet(viewsets.ViewSet): @detail_route(methods=['post']) def cancel(self, *args, **kwargs): - return self.set_pending_action(models.Task.PendingActions.CANCEL, *args, **kwargs) + return self.set_pending_action(pending_actions.CANCEL, *args, **kwargs) @detail_route(methods=['post']) def restart(self, *args, **kwargs): - return self.set_pending_action(models.Task.PendingActions.RESTART, *args, **kwargs) + return self.set_pending_action(pending_actions.RESTART, *args, **kwargs) @detail_route(methods=['post']) def remove(self, *args, **kwargs): - return self.set_pending_action(models.Task.PendingActions.REMOVE, *args, perms=('delete_project', ), **kwargs) + return self.set_pending_action(pending_actions.REMOVE, *args, perms=('delete_project', ), **kwargs) @detail_route(methods=['get']) def output(self, request, pk=None, project_pk=None): diff --git a/app/models.py b/app/models.py index 9c87a344..e8c31689 100644 --- a/app/models.py +++ b/app/models.py @@ -1,23 +1,27 @@ -import time, os +import logging +import os +import shutil +import zipfile -from django.contrib.gis.gdal import GDALRaster -from django.db import models -from django.db.models import signals -from django.contrib.gis.db import models as gismodels -from django.utils import timezone from django.contrib.auth.models import User +from django.contrib.gis.db import models as gismodels +from django.contrib.gis.gdal import GDALRaster from django.contrib.postgres import fields -from nodeodm.models import ProcessingNode -from guardian.shortcuts import get_perms_for_model, assign_perm -from guardian.models import UserObjectPermissionBase -from guardian.models import GroupObjectPermissionBase from django.core.exceptions import ValidationError -from django.dispatch import receiver -from nodeodm.exceptions import ProcessingException +from django.db import models from django.db import transaction +from django.db.models import signals +from django.dispatch import receiver +from django.utils import timezone +from guardian.models import GroupObjectPermissionBase +from guardian.models import UserObjectPermissionBase +from guardian.shortcuts import get_perms_for_model, assign_perm + +from app import pending_actions from nodeodm import status_codes +from nodeodm.exceptions import ProcessingException +from nodeodm.models import ProcessingNode from webodm import settings -import logging, zipfile, shutil logger = logging.getLogger('app.logger') @@ -36,12 +40,29 @@ class Project(models.Model): name = models.CharField(max_length=255, help_text="A label used to describe the project") description = models.TextField(null=True, blank=True, help_text="More in-depth description of the project") created_at = models.DateTimeField(default=timezone.now, help_text="Creation date") + deleting = models.BooleanField(db_index=True, default=False, + help_text="Whether this project has been marked for deletion. Projects that have running tasks need to wait for tasks to be properly cleaned up before they can be deleted.") + + def delete(self, *args): + # No tasks? + if self.task_set.count() == 0: + # Just delete normally + logger.info("Deleted project {}".format(self.id)) + super().delete(*args) + else: + # Need to remove all tasks before we can remove this project + # which will be deleted on the scheduler after pending actions + # have been completed + self.task_set.update(pending_action=pending_actions.REMOVE) + self.deleting = True + self.save() + logger.info("Tasks pending, set project {} deleting flag".format(self.id)) def __str__(self): return self.name def tasks(self, pk=None): - return Task.objects.filter(project=self).only('id') + return self.task_set.only('id') class Meta: permissions = ( @@ -86,11 +107,6 @@ def validate_task_options(value): class Task(models.Model): - class PendingActions: - CANCEL = 1 - REMOVE = 2 - RESTART = 3 - STATUS_CODES = ( (status_codes.QUEUED, 'QUEUED'), (status_codes.RUNNING, 'RUNNING'), @@ -100,9 +116,9 @@ class Task(models.Model): ) PENDING_ACTIONS = ( - (PendingActions.CANCEL, 'CANCEL'), - (PendingActions.REMOVE, 'REMOVE'), - (PendingActions.RESTART, 'RESTART'), + (pending_actions.CANCEL, 'CANCEL'), + (pending_actions.REMOVE, 'REMOVE'), + (pending_actions.RESTART, 'RESTART'), ) uuid = models.CharField(max_length=255, db_index=True, default='', blank=True, help_text="Identifier of the task (as returned by OpenDroneMap's REST API)") @@ -122,7 +138,7 @@ class Task(models.Model): # textured_model # mission created_at = models.DateTimeField(default=timezone.now, help_text="Creation date") - pending_action = models.IntegerField(choices=PENDING_ACTIONS, db_index=True, null=True, blank=True, help_text="A requested action to be performed on the task. When set to a value other than NONE, the selected action will be performed by the scheduler at the next iteration.") + pending_action = models.IntegerField(choices=PENDING_ACTIONS, db_index=True, null=True, blank=True, help_text="A requested action to be performed on the task. The selected action will be performed by the scheduler at the next iteration.") def __str__(self): return 'Task ID: {}'.format(self.id) @@ -157,8 +173,6 @@ class Task(models.Model): return task - # In case of error - return None def process(self): """ @@ -189,10 +203,9 @@ class Task(models.Model): except ProcessingException as e: self.set_failure(str(e)) - if self.pending_action is not None: try: - if self.pending_action == self.PendingActions.CANCEL: + if self.pending_action == pending_actions.CANCEL: # Do we need to cancel the task on the processing node? logger.info("Canceling task {}".format(self)) if self.processing_node and self.uuid: @@ -202,7 +215,7 @@ class Task(models.Model): else: raise ProcessingException("Cannot cancel a task that has no processing node or UUID") - elif self.pending_action == self.PendingActions.RESTART: + elif self.pending_action == pending_actions.RESTART: logger.info("Restarting task {}".format(self)) if self.processing_node and self.uuid: @@ -235,7 +248,7 @@ class Task(models.Model): else: raise ProcessingException("Cannot restart a task that has no processing node or UUID") - elif self.pending_action == self.PendingActions.REMOVE: + elif self.pending_action == pending_actions.REMOVE: logger.info("Removing task {}".format(self)) if self.processing_node and self.uuid: # Attempt to delete the resources on the processing node @@ -256,7 +269,6 @@ class Task(models.Model): self.last_error = str(e) self.save() - if self.processing_node: # Need to update status (first time, queued or running?) if self.uuid and self.status in [None, status_codes.QUEUED, status_codes.RUNNING]: @@ -333,7 +345,6 @@ class Task(models.Model): self.status = status_codes.FAILED self.save() - class Meta: permissions = ( ('view_task', 'Can view task'), diff --git a/app/pending_actions.py b/app/pending_actions.py new file mode 100644 index 00000000..b5c7bdd8 --- /dev/null +++ b/app/pending_actions.py @@ -0,0 +1,3 @@ +CANCEL = 1 +REMOVE = 2 +RESTART = 3 \ No newline at end of file diff --git a/app/scheduler.py b/app/scheduler.py index 2d610d58..8f599e3a 100644 --- a/app/scheduler.py +++ b/app/scheduler.py @@ -6,8 +6,8 @@ from apscheduler.schedulers import SchedulerAlreadyRunningError, SchedulerNotRun from threading import Thread, Lock from multiprocessing.dummy import Pool as ThreadPool from nodeodm.models import ProcessingNode -from app.models import Task -from django.db.models import Q +from app.models import Task, Project +from django.db.models import Q, Count from django import db from nodeodm import status_codes import random @@ -95,12 +95,23 @@ def process_pending_tasks(): pool.close() pool.join() + +def cleanup_projects(): + # Delete all projects that are marked for deletion + # and that have no tasks left + total, count_dict = Project.objects.filter(deleting=True).annotate( + tasks_count=Count('task') + ).filter(tasks_count=0).delete() + if total > 0 and 'app.Project' in count_dict: + logger.info("Deleted {} projects".format(count_dict['app.Project'])) + def setup(): logger.info("Starting background scheduler...") try: scheduler.start() scheduler.add_job(update_nodes_info, 'interval', seconds=30) scheduler.add_job(process_pending_tasks, 'interval', seconds=5) + scheduler.add_job(cleanup_projects, 'interval', seconds=15) except SchedulerAlreadyRunningError: logger.warn("Scheduler already running (this is OK while testing)") diff --git a/app/static/app/js/components/EditProjectDialog.jsx b/app/static/app/js/components/EditProjectDialog.jsx index a031a8a0..46054f01 100644 --- a/app/static/app/js/components/EditProjectDialog.jsx +++ b/app/static/app/js/components/EditProjectDialog.jsx @@ -33,12 +33,14 @@ class EditProjectDialog extends React.Component { this.state = { showModal: props.show, saving: false, + deleting: false, error: "" }; this.show = this.show.bind(this); this.hide = this.hide.bind(this); this.handleSave = this.handleSave.bind(this); + this.handleDelete = this.handleDelete.bind(this); } componentDidMount(){ @@ -91,12 +93,24 @@ class EditProjectDialog extends React.Component { name: this.nameInput.value, descr: this.descrInput.value }).fail(e => { - this.setState({error: e.message || e.responseText || e}); - }).done(() => { - this.hide(); + this.setState({error: e.message || e.responseText || "Could not apply changes"}); }).always(() => { this.setState({saving: false}); - }) + }).done(() => { + this.hide(); + }); + } + + handleDelete(){ + if (this.props.deleteAction){ + if (window.confirm("All tasks, images and models associated with this project will be permanently deleted. Are you sure you want to continue?")){ + this.setState({deleting: true}); + this.props.deleteAction() + .fail(e => { + this.setState({error: e.message || e.responseText || "Could not delete project", deleting: false}); + }); + } + } } render(){ @@ -143,7 +157,18 @@ class EditProjectDialog extends React.Component { {this.props.deleteAction ?
- +
: ""} diff --git a/app/static/app/js/components/ProjectList.jsx b/app/static/app/js/components/ProjectList.jsx index 842f8497..46d0ec30 100644 --- a/app/static/app/js/components/ProjectList.jsx +++ b/app/static/app/js/components/ProjectList.jsx @@ -12,6 +12,8 @@ class ProjectList extends React.Component { error: "", projects: null } + + this.handleDelete = this.handleDelete.bind(this); } componentDidMount(){ @@ -47,6 +49,11 @@ class ProjectList extends React.Component { this.serverRequest.abort(); } + handleDelete(projectId){ + let projects = this.state.projects.filter(p => p.id !== projectId); + this.setState({projects: projects}); + } + render() { if (this.state.loading){ return (
Loading projects...
); @@ -54,7 +61,7 @@ class ProjectList extends React.Component { else if (this.state.projects){ return (); }else if (this.state.error){ diff --git a/app/static/app/js/components/ProjectListItem.jsx b/app/static/app/js/components/ProjectListItem.jsx index 7c20ba0e..5b191d37 100644 --- a/app/static/app/js/components/ProjectListItem.jsx +++ b/app/static/app/js/components/ProjectListItem.jsx @@ -19,7 +19,8 @@ class ProjectListItem extends React.Component { updatingTask: false, upload: this.getDefaultUploadState(), error: "", - numTasks: props.data.tasks.length + data: props.data, + refreshing: false }; this.toggleTaskList = this.toggleTaskList.bind(this); @@ -34,9 +35,27 @@ class ProjectListItem extends React.Component { this.taskDeleted = this.taskDeleted.bind(this); } + refresh(){ + // Update project information based on server + this.setState({refreshing: true}); + + this.refreshRequest = + $.getJSON(`/api/projects/${this.state.data.id}/`) + .done((json) => { + this.setState({data: json}); + }) + .fail((_, __, e) => { + this.setState({error: e.message}); + }) + .always(() => { + this.setState({refreshing: false}); + }); + } + componentWillUnmount(){ if (this.updateTaskRequest) this.updateTaskRequest.abort(); if (this.deleteProjectRequest) this.deleteProjectRequest.abort(); + if (this.refreshRequest) this.refreshRequest.abort(); } getDefaultUploadState(){ @@ -70,7 +89,7 @@ class ProjectListItem extends React.Component { this.dz = new Dropzone(this.dropzone, { paramName: "images", - url : `/api/projects/${this.props.data.id}/tasks/`, + url : `/api/projects/${this.state.data.id}/tasks/`, parallelUploads: 9999999, uploadMultiple: true, acceptedFiles: "image/*", @@ -148,7 +167,7 @@ class ProjectListItem extends React.Component { this.updateTaskRequest = $.ajax({ - url: `/api/projects/${this.props.data.id}/tasks/${this.state.upload.taskId}/`, + url: `/api/projects/${this.state.data.id}/tasks/${this.state.upload.taskId}/`, contentType: 'application/json', data: JSON.stringify({ name: taskInfo.name, @@ -157,13 +176,13 @@ class ProjectListItem extends React.Component { }), dataType: 'json', type: 'PATCH' - }).done(() => { + }).done((json) => { if (this.state.showTaskList){ this.taskList.refresh(); }else{ this.setState({showTaskList: true}); } - this.setState({numTasks: this.state.numTasks + 1}); + this.refresh(); }).fail(() => { this.setUploadState({error: "Could not update task information. Plese try again."}); }).always(() => { @@ -196,28 +215,16 @@ class ProjectListItem extends React.Component { } taskDeleted(){ - this.setState({numTasks: this.state.numTasks - 1}); - if (this.state.numTasks === 0){ - this.setState({showTaskList: false}); - } + this.refresh(); } handleDelete(){ - this.setState({error: "HI!" + Math.random()}); - // if (window.confirm("All tasks, images and models associated with this project will be permanently deleted. Are you sure you want to continue?")){ - // return; - // this.deleteProjectRequest = - // $.ajax({ - // url: `/api/projects/${this.props.data.id}/`, - // contentType: 'application/json', - // dataType: 'json', - // type: 'DELETE' - // }).done((json) => { - // console.log("REs", json); - // }).fail(() => { - // this.setState({error: "The project could not be deleted"}); - // }); - // } + return $.ajax({ + url: `/api/projects/${this.state.data.id}/`, + type: 'DELETE' + }).done(() => { + if (this.props.onDelete) this.props.onDelete(this.state.data.id); + }); } handleTaskSaved(taskInfo){ @@ -234,16 +241,30 @@ class ProjectListItem extends React.Component { } updateProject(project){ - console.log("OK", project); + return $.ajax({ + url: `/api/projects/${this.state.data.id}/`, + contentType: 'application/json', + data: JSON.stringify({ + name: project.name, + description: project.descr, + }), + dataType: 'json', + type: 'PATCH' + }).done(() => { + this.refresh(); + }); } viewMap(){ - location.href = `/map/?project=${this.props.data.id}`; + location.href = `/map/project/${this.state.data.id}/`; } render() { + const { refreshing, data } = this.state; + const numTasks = data.tasks.length; + return ( -
  • @@ -253,8 +274,8 @@ class ProjectListItem extends React.Component { saveLabel="Save Changes" savingLabel="Saving changes..." saveIcon="fa fa-edit" - projectName={this.props.data.name} - projectDescr={this.props.data.description} + projectName={data.name} + projectDescr={data.description} saveAction={this.updateProject} deleteAction={this.handleDelete} /> @@ -290,17 +311,17 @@ class ProjectListItem extends React.Component { - {this.props.data.name} + {data.name}
    - {this.props.data.description} + {data.description}
    - {this.state.numTasks > 0 ? + {numTasks > 0 ? - {this.state.numTasks} Tasks + {numTasks} Tasks : ""} @@ -336,7 +357,7 @@ class ProjectListItem extends React.Component { {this.state.showTaskList ? : ""} diff --git a/app/static/app/js/css/ProjectListItem.scss b/app/static/app/js/css/ProjectListItem.scss index c8e4045a..6b9aa888 100644 --- a/app/static/app/js/css/ProjectListItem.scss +++ b/app/static/app/js/css/ProjectListItem.scss @@ -31,6 +31,13 @@ 100% {opacity: 0.5;} } + &.refreshing{ + background-color: #eee; + } + + -webkit-transition: background-color 1s ease; + transition: background-color 1s ease; + &.dz-drag-hover{ .drag-drop-icon{ display: block; diff --git a/app/tests/test_api.py b/app/tests/test_api.py index 3089816c..29a0a286 100644 --- a/app/tests/test_api.py +++ b/app/tests/test_api.py @@ -1,3 +1,4 @@ +from app import pending_actions from .classes import BootTestCase from rest_framework.test import APIClient from rest_framework import status @@ -168,13 +169,13 @@ class TestApi(BootTestCase): self.assertTrue(res.data["success"]) task.refresh_from_db() self.assertTrue(task.last_error is None) - self.assertTrue(task.pending_action == task.PendingActions.CANCEL) + self.assertTrue(task.pending_action == pending_actions.CANCEL) res = client.post('/api/projects/{}/tasks/{}/restart/'.format(project.id, task.id)) self.assertTrue(res.data["success"]) task.refresh_from_db() self.assertTrue(task.last_error is None) - self.assertTrue(task.pending_action == task.PendingActions.RESTART) + self.assertTrue(task.pending_action == pending_actions.RESTART) # Cannot cancel, restart or delete a task for which we don't have permission for action in ['cancel', 'remove', 'restart']: @@ -186,7 +187,7 @@ class TestApi(BootTestCase): self.assertTrue(res.data["success"]) task.refresh_from_db() self.assertTrue(task.last_error is None) - self.assertTrue(task.pending_action == task.PendingActions.REMOVE) + self.assertTrue(task.pending_action == pending_actions.REMOVE) # TODO test: @@ -195,6 +196,7 @@ class TestApi(BootTestCase): # - scheduler processing steps # - tiles API urls (permissions, 404s) # - assets download + # - project deletion def test_processingnodes(self): client = APIClient()