Can delete projects

pull/50/head
Piero Toffanin 2016-11-15 11:51:19 -05:00
rodzic 7c0fc4ffd5
commit 23f65bf9cd
10 zmienionych plików z 169 dodań i 82 usunięć

Wyświetl plik

@ -14,7 +14,7 @@ class ProjectSerializer(serializers.ModelSerializer):
class Meta:
model = models.Project
fields = '__all__'
exclude = ('deleting', )
class ProjectViewSet(viewsets.ModelViewSet):
@ -28,5 +28,5 @@ class ProjectViewSet(viewsets.ModelViewSet):
"""
filter_fields = ('id', 'name', 'description', 'created_at')
serializer_class = ProjectSerializer
queryset = models.Project.objects.all()
queryset = models.Project.objects.filter(deleting=False)
ordering_fields = '__all__'

Wyświetl plik

@ -9,7 +9,7 @@ from rest_framework.response import Response
from rest_framework.decorators import detail_route
from rest_framework.views import APIView
from app import models, scheduler
from app import models, scheduler, pending_actions
from nodeodm.models import ProcessingNode
@ -76,15 +76,15 @@ class TaskViewSet(viewsets.ViewSet):
@detail_route(methods=['post'])
def cancel(self, *args, **kwargs):
return self.set_pending_action(models.Task.PendingActions.CANCEL, *args, **kwargs)
return self.set_pending_action(pending_actions.CANCEL, *args, **kwargs)
@detail_route(methods=['post'])
def restart(self, *args, **kwargs):
return self.set_pending_action(models.Task.PendingActions.RESTART, *args, **kwargs)
return self.set_pending_action(pending_actions.RESTART, *args, **kwargs)
@detail_route(methods=['post'])
def remove(self, *args, **kwargs):
return self.set_pending_action(models.Task.PendingActions.REMOVE, *args, perms=('delete_project', ), **kwargs)
return self.set_pending_action(pending_actions.REMOVE, *args, perms=('delete_project', ), **kwargs)
@detail_route(methods=['get'])
def output(self, request, pk=None, project_pk=None):

Wyświetl plik

@ -1,23 +1,27 @@
import time, os
import logging
import os
import shutil
import zipfile
from django.contrib.gis.gdal import GDALRaster
from django.db import models
from django.db.models import signals
from django.contrib.gis.db import models as gismodels
from django.utils import timezone
from django.contrib.auth.models import User
from django.contrib.gis.db import models as gismodels
from django.contrib.gis.gdal import GDALRaster
from django.contrib.postgres import fields
from nodeodm.models import ProcessingNode
from guardian.shortcuts import get_perms_for_model, assign_perm
from guardian.models import UserObjectPermissionBase
from guardian.models import GroupObjectPermissionBase
from django.core.exceptions import ValidationError
from django.dispatch import receiver
from nodeodm.exceptions import ProcessingException
from django.db import models
from django.db import transaction
from django.db.models import signals
from django.dispatch import receiver
from django.utils import timezone
from guardian.models import GroupObjectPermissionBase
from guardian.models import UserObjectPermissionBase
from guardian.shortcuts import get_perms_for_model, assign_perm
from app import pending_actions
from nodeodm import status_codes
from nodeodm.exceptions import ProcessingException
from nodeodm.models import ProcessingNode
from webodm import settings
import logging, zipfile, shutil
logger = logging.getLogger('app.logger')
@ -36,12 +40,29 @@ class Project(models.Model):
name = models.CharField(max_length=255, help_text="A label used to describe the project")
description = models.TextField(null=True, blank=True, help_text="More in-depth description of the project")
created_at = models.DateTimeField(default=timezone.now, help_text="Creation date")
deleting = models.BooleanField(db_index=True, default=False,
help_text="Whether this project has been marked for deletion. Projects that have running tasks need to wait for tasks to be properly cleaned up before they can be deleted.")
def delete(self, *args):
# No tasks?
if self.task_set.count() == 0:
# Just delete normally
logger.info("Deleted project {}".format(self.id))
super().delete(*args)
else:
# Need to remove all tasks before we can remove this project
# which will be deleted on the scheduler after pending actions
# have been completed
self.task_set.update(pending_action=pending_actions.REMOVE)
self.deleting = True
self.save()
logger.info("Tasks pending, set project {} deleting flag".format(self.id))
def __str__(self):
return self.name
def tasks(self, pk=None):
return Task.objects.filter(project=self).only('id')
return self.task_set.only('id')
class Meta:
permissions = (
@ -86,11 +107,6 @@ def validate_task_options(value):
class Task(models.Model):
class PendingActions:
CANCEL = 1
REMOVE = 2
RESTART = 3
STATUS_CODES = (
(status_codes.QUEUED, 'QUEUED'),
(status_codes.RUNNING, 'RUNNING'),
@ -100,9 +116,9 @@ class Task(models.Model):
)
PENDING_ACTIONS = (
(PendingActions.CANCEL, 'CANCEL'),
(PendingActions.REMOVE, 'REMOVE'),
(PendingActions.RESTART, 'RESTART'),
(pending_actions.CANCEL, 'CANCEL'),
(pending_actions.REMOVE, 'REMOVE'),
(pending_actions.RESTART, 'RESTART'),
)
uuid = models.CharField(max_length=255, db_index=True, default='', blank=True, help_text="Identifier of the task (as returned by OpenDroneMap's REST API)")
@ -122,7 +138,7 @@ class Task(models.Model):
# textured_model
# mission
created_at = models.DateTimeField(default=timezone.now, help_text="Creation date")
pending_action = models.IntegerField(choices=PENDING_ACTIONS, db_index=True, null=True, blank=True, help_text="A requested action to be performed on the task. When set to a value other than NONE, the selected action will be performed by the scheduler at the next iteration.")
pending_action = models.IntegerField(choices=PENDING_ACTIONS, db_index=True, null=True, blank=True, help_text="A requested action to be performed on the task. The selected action will be performed by the scheduler at the next iteration.")
def __str__(self):
return 'Task ID: {}'.format(self.id)
@ -157,8 +173,6 @@ class Task(models.Model):
return task
# In case of error
return None
def process(self):
"""
@ -189,10 +203,9 @@ class Task(models.Model):
except ProcessingException as e:
self.set_failure(str(e))
if self.pending_action is not None:
try:
if self.pending_action == self.PendingActions.CANCEL:
if self.pending_action == pending_actions.CANCEL:
# Do we need to cancel the task on the processing node?
logger.info("Canceling task {}".format(self))
if self.processing_node and self.uuid:
@ -202,7 +215,7 @@ class Task(models.Model):
else:
raise ProcessingException("Cannot cancel a task that has no processing node or UUID")
elif self.pending_action == self.PendingActions.RESTART:
elif self.pending_action == pending_actions.RESTART:
logger.info("Restarting task {}".format(self))
if self.processing_node and self.uuid:
@ -235,7 +248,7 @@ class Task(models.Model):
else:
raise ProcessingException("Cannot restart a task that has no processing node or UUID")
elif self.pending_action == self.PendingActions.REMOVE:
elif self.pending_action == pending_actions.REMOVE:
logger.info("Removing task {}".format(self))
if self.processing_node and self.uuid:
# Attempt to delete the resources on the processing node
@ -256,7 +269,6 @@ class Task(models.Model):
self.last_error = str(e)
self.save()
if self.processing_node:
# Need to update status (first time, queued or running?)
if self.uuid and self.status in [None, status_codes.QUEUED, status_codes.RUNNING]:
@ -333,7 +345,6 @@ class Task(models.Model):
self.status = status_codes.FAILED
self.save()
class Meta:
permissions = (
('view_task', 'Can view task'),

Wyświetl plik

@ -0,0 +1,3 @@
CANCEL = 1
REMOVE = 2
RESTART = 3

Wyświetl plik

@ -6,8 +6,8 @@ from apscheduler.schedulers import SchedulerAlreadyRunningError, SchedulerNotRun
from threading import Thread, Lock
from multiprocessing.dummy import Pool as ThreadPool
from nodeodm.models import ProcessingNode
from app.models import Task
from django.db.models import Q
from app.models import Task, Project
from django.db.models import Q, Count
from django import db
from nodeodm import status_codes
import random
@ -95,12 +95,23 @@ def process_pending_tasks():
pool.close()
pool.join()
def cleanup_projects():
# Delete all projects that are marked for deletion
# and that have no tasks left
total, count_dict = Project.objects.filter(deleting=True).annotate(
tasks_count=Count('task')
).filter(tasks_count=0).delete()
if total > 0 and 'app.Project' in count_dict:
logger.info("Deleted {} projects".format(count_dict['app.Project']))
def setup():
logger.info("Starting background scheduler...")
try:
scheduler.start()
scheduler.add_job(update_nodes_info, 'interval', seconds=30)
scheduler.add_job(process_pending_tasks, 'interval', seconds=5)
scheduler.add_job(cleanup_projects, 'interval', seconds=15)
except SchedulerAlreadyRunningError:
logger.warn("Scheduler already running (this is OK while testing)")

Wyświetl plik

@ -33,12 +33,14 @@ class EditProjectDialog extends React.Component {
this.state = {
showModal: props.show,
saving: false,
deleting: false,
error: ""
};
this.show = this.show.bind(this);
this.hide = this.hide.bind(this);
this.handleSave = this.handleSave.bind(this);
this.handleDelete = this.handleDelete.bind(this);
}
componentDidMount(){
@ -91,12 +93,24 @@ class EditProjectDialog extends React.Component {
name: this.nameInput.value,
descr: this.descrInput.value
}).fail(e => {
this.setState({error: e.message || e.responseText || e});
}).done(() => {
this.hide();
this.setState({error: e.message || e.responseText || "Could not apply changes"});
}).always(() => {
this.setState({saving: false});
})
}).done(() => {
this.hide();
});
}
handleDelete(){
if (this.props.deleteAction){
if (window.confirm("All tasks, images and models associated with this project will be permanently deleted. Are you sure you want to continue?")){
this.setState({deleting: true});
this.props.deleteAction()
.fail(e => {
this.setState({error: e.message || e.responseText || "Could not delete project", deleting: false});
});
}
}
}
render(){
@ -143,7 +157,18 @@ class EditProjectDialog extends React.Component {
</div>
{this.props.deleteAction ?
<div className="text-left">
<button className="btn btn-danger" onClick={this.props.deleteAction}><i className="glyphicon glyphicon-trash"></i> Delete</button>
<button
disabled={this.state.deleting}
className="btn btn-danger"
onClick={this.handleDelete}>
{this.state.deleting ?
<span>
<i className="fa fa-circle-o-notch fa-spin"></i> Deleting...
</span>
: <span>
<i className="glyphicon glyphicon-trash"></i> Delete
</span>}
</button>
</div>
: ""}
</div>

Wyświetl plik

@ -12,6 +12,8 @@ class ProjectList extends React.Component {
error: "",
projects: null
}
this.handleDelete = this.handleDelete.bind(this);
}
componentDidMount(){
@ -47,6 +49,11 @@ class ProjectList extends React.Component {
this.serverRequest.abort();
}
handleDelete(projectId){
let projects = this.state.projects.filter(p => p.id !== projectId);
this.setState({projects: projects});
}
render() {
if (this.state.loading){
return (<div>Loading projects... <i className="fa fa-refresh fa-spin fa-fw"></i></div>);
@ -54,7 +61,7 @@ class ProjectList extends React.Component {
else if (this.state.projects){
return (<ul className="list-group">
{this.state.projects.map(p => (
<ProjectListItem key={p.id} data={p} />
<ProjectListItem key={p.id} data={p} onDelete={this.handleDelete} />
))}
</ul>);
}else if (this.state.error){

Wyświetl plik

@ -19,7 +19,8 @@ class ProjectListItem extends React.Component {
updatingTask: false,
upload: this.getDefaultUploadState(),
error: "",
numTasks: props.data.tasks.length
data: props.data,
refreshing: false
};
this.toggleTaskList = this.toggleTaskList.bind(this);
@ -34,9 +35,27 @@ class ProjectListItem extends React.Component {
this.taskDeleted = this.taskDeleted.bind(this);
}
refresh(){
// Update project information based on server
this.setState({refreshing: true});
this.refreshRequest =
$.getJSON(`/api/projects/${this.state.data.id}/`)
.done((json) => {
this.setState({data: json});
})
.fail((_, __, e) => {
this.setState({error: e.message});
})
.always(() => {
this.setState({refreshing: false});
});
}
componentWillUnmount(){
if (this.updateTaskRequest) this.updateTaskRequest.abort();
if (this.deleteProjectRequest) this.deleteProjectRequest.abort();
if (this.refreshRequest) this.refreshRequest.abort();
}
getDefaultUploadState(){
@ -70,7 +89,7 @@ class ProjectListItem extends React.Component {
this.dz = new Dropzone(this.dropzone, {
paramName: "images",
url : `/api/projects/${this.props.data.id}/tasks/`,
url : `/api/projects/${this.state.data.id}/tasks/`,
parallelUploads: 9999999,
uploadMultiple: true,
acceptedFiles: "image/*",
@ -148,7 +167,7 @@ class ProjectListItem extends React.Component {
this.updateTaskRequest =
$.ajax({
url: `/api/projects/${this.props.data.id}/tasks/${this.state.upload.taskId}/`,
url: `/api/projects/${this.state.data.id}/tasks/${this.state.upload.taskId}/`,
contentType: 'application/json',
data: JSON.stringify({
name: taskInfo.name,
@ -157,13 +176,13 @@ class ProjectListItem extends React.Component {
}),
dataType: 'json',
type: 'PATCH'
}).done(() => {
}).done((json) => {
if (this.state.showTaskList){
this.taskList.refresh();
}else{
this.setState({showTaskList: true});
}
this.setState({numTasks: this.state.numTasks + 1});
this.refresh();
}).fail(() => {
this.setUploadState({error: "Could not update task information. Plese try again."});
}).always(() => {
@ -196,28 +215,16 @@ class ProjectListItem extends React.Component {
}
taskDeleted(){
this.setState({numTasks: this.state.numTasks - 1});
if (this.state.numTasks === 0){
this.setState({showTaskList: false});
}
this.refresh();
}
handleDelete(){
this.setState({error: "HI!" + Math.random()});
// if (window.confirm("All tasks, images and models associated with this project will be permanently deleted. Are you sure you want to continue?")){
// return;
// this.deleteProjectRequest =
// $.ajax({
// url: `/api/projects/${this.props.data.id}/`,
// contentType: 'application/json',
// dataType: 'json',
// type: 'DELETE'
// }).done((json) => {
// console.log("REs", json);
// }).fail(() => {
// this.setState({error: "The project could not be deleted"});
// });
// }
return $.ajax({
url: `/api/projects/${this.state.data.id}/`,
type: 'DELETE'
}).done(() => {
if (this.props.onDelete) this.props.onDelete(this.state.data.id);
});
}
handleTaskSaved(taskInfo){
@ -234,16 +241,30 @@ class ProjectListItem extends React.Component {
}
updateProject(project){
console.log("OK", project);
return $.ajax({
url: `/api/projects/${this.state.data.id}/`,
contentType: 'application/json',
data: JSON.stringify({
name: project.name,
description: project.descr,
}),
dataType: 'json',
type: 'PATCH'
}).done(() => {
this.refresh();
});
}
viewMap(){
location.href = `/map/?project=${this.props.data.id}`;
location.href = `/map/project/${this.state.data.id}/`;
}
render() {
const { refreshing, data } = this.state;
const numTasks = data.tasks.length;
return (
<li className="project-list-item list-group-item"
<li className={"project-list-item list-group-item " + (refreshing ? "refreshing" : "")}
href="javascript:void(0);"
ref={this.setRef("dropzone")}>
@ -253,8 +274,8 @@ class ProjectListItem extends React.Component {
saveLabel="Save Changes"
savingLabel="Saving changes..."
saveIcon="fa fa-edit"
projectName={this.props.data.name}
projectDescr={this.props.data.description}
projectName={data.name}
projectDescr={data.description}
saveAction={this.updateProject}
deleteAction={this.handleDelete}
/>
@ -290,17 +311,17 @@ class ProjectListItem extends React.Component {
</div>
<span className="project-name">
{this.props.data.name}
{data.name}
</span>
<div className="project-description">
{this.props.data.description}
{data.description}
</div>
<div className="row project-links">
{this.state.numTasks > 0 ?
{numTasks > 0 ?
<span>
<i className='fa fa-tasks'>
</i> <a href="javascript:void(0);" onClick={this.toggleTaskList}>
{this.state.numTasks} Tasks <i className={'fa fa-caret-' + (this.state.showTaskList ? 'down' : 'right')}></i>
{numTasks} Tasks <i className={'fa fa-caret-' + (this.state.showTaskList ? 'down' : 'right')}></i>
</a>
</span>
: ""}
@ -336,7 +357,7 @@ class ProjectListItem extends React.Component {
{this.state.showTaskList ?
<TaskList
ref={this.setRef("taskList")}
source={`/api/projects/${this.props.data.id}/tasks/?ordering=-created_at`}
source={`/api/projects/${data.id}/tasks/?ordering=-created_at`}
onDelete={this.taskDeleted}
/> : ""}

Wyświetl plik

@ -31,6 +31,13 @@
100% {opacity: 0.5;}
}
&.refreshing{
background-color: #eee;
}
-webkit-transition: background-color 1s ease;
transition: background-color 1s ease;
&.dz-drag-hover{
.drag-drop-icon{
display: block;

Wyświetl plik

@ -1,3 +1,4 @@
from app import pending_actions
from .classes import BootTestCase
from rest_framework.test import APIClient
from rest_framework import status
@ -168,13 +169,13 @@ class TestApi(BootTestCase):
self.assertTrue(res.data["success"])
task.refresh_from_db()
self.assertTrue(task.last_error is None)
self.assertTrue(task.pending_action == task.PendingActions.CANCEL)
self.assertTrue(task.pending_action == pending_actions.CANCEL)
res = client.post('/api/projects/{}/tasks/{}/restart/'.format(project.id, task.id))
self.assertTrue(res.data["success"])
task.refresh_from_db()
self.assertTrue(task.last_error is None)
self.assertTrue(task.pending_action == task.PendingActions.RESTART)
self.assertTrue(task.pending_action == pending_actions.RESTART)
# Cannot cancel, restart or delete a task for which we don't have permission
for action in ['cancel', 'remove', 'restart']:
@ -186,7 +187,7 @@ class TestApi(BootTestCase):
self.assertTrue(res.data["success"])
task.refresh_from_db()
self.assertTrue(task.last_error is None)
self.assertTrue(task.pending_action == task.PendingActions.REMOVE)
self.assertTrue(task.pending_action == pending_actions.REMOVE)
# TODO test:
@ -195,6 +196,7 @@ class TestApi(BootTestCase):
# - scheduler processing steps
# - tiles API urls (permissions, 404s)
# - assets download
# - project deletion
def test_processingnodes(self):
client = APIClient()