Project folder cleanups

app/models/project.py

@@ -1,5 +1,7 @@
 import logging
 import uuid
+import shutil
+import os
 
 from django.conf import settings
 from django.db import models
@@ -16,6 +18,7 @@ from django.db import transaction
 from app import pending_actions
 
 from nodeodm import status_codes
+from webodm import settings as wo_settings
 
 logger = logging.getLogger('app.logger')
 
@@ -36,7 +39,28 @@ class Project(models.Model):
         # No tasks?
         if self.task_set.count() == 0:
             # Just delete normally
+
+            project_dir = os.path.join(wo_settings.MEDIA_ROOT, "project", str(self.id))
+            if os.path.isdir(project_dir):
+                entries = os.listdir(project_dir)
+                empty_project_folder = False
+
+                if len(entries) == 0:
+                    empty_project_folder = True
+                elif len(entries) == 1 and entries[0] == "task":
+                    empty_project_folder = len(os.listdir(os.path.join(project_dir, "task"))) == 0
+
+                if empty_project_folder:
+                    logger.info(f"Deleting {project_dir}")
+                    try:
+                        shutil.rmtree(project_dir)
+                    except Exception as e:
+                        logger.warning(f"Cannot delete {project_dir}: {str(e)}")
+                else:
+                    logger.warning(f"Project {self.id} is being deleted, but data is stored on disk. We will keep the data at {project_dir}, but will become orphaned")
+
             logger.info("Deleted project {}".format(self.id))
+
             super().delete(*args)
         else:
             # Need to remove all tasks before we can remove this project

app/models/task.py

@@ -980,7 +980,6 @@ class Task(models.Model):
             # Check if the zip file contained a top level directory
             # which shouldn't be there and try to fix the structure
             top_level = [os.path.join(assets_dir, d) for d in os.listdir(assets_dir)]
-            logger.info(top_level)
             if len(top_level) == 1 and os.path.isdir(top_level[0]):
                 second_level = [os.path.join(top_level[0], f) for f in os.listdir(top_level[0])]
                 if len(second_level) > 0:

worker/tasks.py

@@ -62,23 +62,24 @@ def update_nodes_info():
 def cleanup_projects():
     # Delete all projects that are marked for deletion
     # and that have no tasks left
-    total, count_dict = Project.objects.filter(deleting=True).annotate(
+    deletion_projects = Project.objects.filter(deleting=True).annotate(
         tasks_count=Count('task')
-    ).filter(tasks_count=0).delete()
-    if total > 0 and 'app.Project' in count_dict:
-        logger.info("Deleted {} projects".format(count_dict['app.Project']))
+    ).filter(tasks_count=0)
+    for p in deletion_projects:
+        p.delete()     
 
     # Delete projects that have no tasks and are owned by users with
     # no disk quota
     if settings.CLEANUP_EMPTY_PROJECTS is not None:
-        total, count_dict = Project.objects.filter(
+        empty_projects = Project.objects.filter(
             owner__profile__quota=0,
             created_at__lte=timezone.now() - timedelta(hours=settings.CLEANUP_EMPTY_PROJECTS)
         ).annotate(
             tasks_count=Count('task')
-        ).filter(tasks_count=0).delete()
-        if total > 0 and 'app.Project' in count_dict:
-            logger.info("Deleted {} projects".format(count_dict['app.Project']))
+        ).filter(tasks_count=0)
+        for p in empty_projects:
+            p.delete()
+
 
 @app.task(ignore_result=True)
 def cleanup_tasks():