Merge pull request #1648 from pierotofy/borgup

Cleanup empty projects when quota = 0

app/management/commands/borg.py

@@ -35,7 +35,7 @@ class Command(BaseCommand):
             print("")
             print("# Skip projects")
             for sp in skip_projects:
-                print("- " + os.path.join("project", str(sp.id)))
+                print("! " + os.path.join("project", str(sp.id)))
 
             if options.get('skip_images'):
                 print("")

app/tests/test_cleanup.py

@@ -0,0 +1,105 @@
+from django.contrib.auth.models import User, Group
+from rest_framework import status
+from rest_framework.test import APIClient
+from app.models import Task, Project
+from django.utils import timezone
+from datetime import timedelta
+from worker.tasks import cleanup_tasks, cleanup_projects
+from .classes import BootTestCase
+from webodm import settings
+
+class TestCleanup(BootTestCase):
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def test_quota(self):
+        c = APIClient()
+        c.login(username="testuser", password="test1234")
+
+        user = User.objects.get(username="testuser")
+        
+        # Create a task with size
+        p = Project.objects.create(owner=user, name='Test')
+        p.save()
+        t = Task.objects.create(project=p, partial=True, name='Now', created_at=timezone.now())
+        t.save()
+        t = Task.objects.create(project=p, partial=True, name='2 hours ago', created_at=timezone.now() - timedelta(hours=2))
+        t.save()
+        t = Task.objects.create(project=p, name='2 hours ago but not partial', created_at=timezone.now() - timedelta(hours=2))
+        t.save()
+
+        # Simulate call to worker cleanup process
+        settings.CLEANUP_PARTIAL_TASKS = None
+        cleanup_tasks()
+        self.assertEqual(Task.objects.filter(project=p).count(), 3)
+
+        settings.CLEANUP_PARTIAL_TASKS = 3
+        cleanup_tasks()
+        self.assertEqual(Task.objects.filter(project=p).count(), 3)
+
+        settings.CLEANUP_PARTIAL_TASKS = 1
+        cleanup_tasks()
+        self.assertEqual(Task.objects.filter(project=p).count(), 2)
+        names = [t.name for t in Task.objects.filter(project=p)]
+        self.assertTrue("Now" in names)
+        self.assertTrue("2 hours ago but not partial" in names)
+    
+    def test_project_cleanup(self):
+        user = User.objects.get(username="testuser")
+        
+        # Create a task
+        p = Project.objects.create(owner=user, name='With tasks 2 hours ago', created_at=timezone.now() - timedelta(hours=2))
+        p.save()
+        t = Task.objects.create(project=p, name='Task')
+        t.save()
+        p2 = Project.objects.create(owner=user, name='Empty 2 hours ago', created_at=timezone.now() - timedelta(hours=2))
+        p2.save()
+        p3 = Project.objects.create(owner=user, name='Empty Now', created_at=timezone.now())
+        p3.save()
+
+        self.assertEqual(p.task_set.count(), 1)
+        self.assertEqual(user.profile.quota, -1)
+        self.assertEqual(user.project_set.count(), 4)
+
+        settings.CLEANUP_EMPTY_PROJECTS = 1
+
+        # Dont cleanup projects if quota is unlimited (-1) 
+        cleanup_projects()
+        self.assertEqual(user.project_set.count(), 4)
+
+        user.profile.quota = 100
+        user.profile.save()
+
+        # Dont cleanup projects if there's a quota
+        cleanup_projects()
+        self.assertEqual(user.project_set.count(), 4)
+        
+        user.profile.quota = 0
+        user.profile.save()
+
+        # If user's quota is zero
+        # Cleanup empty projects that are older than 1 hour
+        for _ in range(2):
+            cleanup_projects()
+            self.assertEqual(user.project_set.count(), 3)
+            
+            self.assertFalse(('Empty 2 hours ago' in [p.name for p in user.project_set.all()]))
+            self.assertTrue(('With tasks 2 hours ago' in [p.name for p in user.project_set.all()]))
+            self.assertTrue(('Empty Now' in [p.name for p in user.project_set.all()]))
+
+        # Recreate..
+        p2 = Project.objects.create(owner=user, name='Empty 2 hours ago', created_at=timezone.now() - timedelta(hours=2))
+        p2.save()
+
+        # Disable
+        settings.CLEANUP_EMPTY_PROJECTS = None
+        self.assertEqual(user.project_set.count(), 4)
+
+        # Dont'cleanup if disabled
+        cleanup_projects()
+        self.assertEqual(user.project_set.count(), 4)
+        
+

webodm/settings.py

@@ -384,6 +384,10 @@ UI_MAX_PROCESSING_NODES = None
 # are removed (or None to disable)
 CLEANUP_PARTIAL_TASKS = 72
 
+# Number of hours before empty projects
+# are removed for users that have zero quotas
+CLEANUP_EMPTY_PROJECTS = None
+
 # Maximum number of threads that a worker should use for processing
 WORKERS_MAX_THREADS = 1
 

worker/tasks.py

@@ -68,6 +68,18 @@ def cleanup_projects():
     if total > 0 and 'app.Project' in count_dict:
         logger.info("Deleted {} projects".format(count_dict['app.Project']))
 
+    # Delete projects that have no tasks and are owned by users with
+    # no disk quota
+    if settings.CLEANUP_EMPTY_PROJECTS is not None:
+        total, count_dict = Project.objects.filter(
+            owner__profile__quota=0,
+            created_at__lte=timezone.now() - timedelta(hours=settings.CLEANUP_EMPTY_PROJECTS)
+        ).annotate(
+            tasks_count=Count('task')
+        ).filter(tasks_count=0).delete()
+        if total > 0 and 'app.Project' in count_dict:
+            logger.info("Deleted {} projects".format(count_dict['app.Project']))
+
 @app.task(ignore_result=True)
 def cleanup_tasks():
     # Delete tasks that are older than