diff --git a/app/models.py b/app/models.py index 95ab5bf9..9908f8b0 100644 --- a/app/models.py +++ b/app/models.py @@ -217,6 +217,7 @@ class Task(models.Model): if self.processing_node and self.uuid: self.processing_node.cancel_task(self.uuid) self.pending_action = None + self.status = None self.save() else: raise ProcessingException("Cannot cancel a task that has no processing node or UUID") @@ -373,7 +374,7 @@ class Task(models.Model): try: shutil.rmtree(directory_to_delete) except FileNotFoundError as e: - logger.warn(e) + logger.warning(e) def set_failure(self, error_message): diff --git a/app/tests/test_api_task.py b/app/tests/test_api_task.py index 355677c3..56440ed7 100644 --- a/app/tests/test_api_task.py +++ b/app/tests/test_api_task.py @@ -2,11 +2,15 @@ import os import subprocess import time +import shutil + +import logging from django.contrib.auth.models import User from rest_framework import status from rest_framework.test import APIClient -from app.models import Project, Task, ImageUpload +from app import scheduler +from app.models import Project, Task, ImageUpload, task_directory_path from app.tests.classes import BootTransactionTestCase from nodeodm import status_codes from nodeodm.models import ProcessingNode @@ -16,7 +20,20 @@ from app.testwatch import testWatch # task processing happens on a separate thread, and normal TestCases # do not commit changes to the DB, so spawning a new thread will show no # data in it. +from webodm import settings +logger = logging.getLogger('app.logger') + class TestApiTask(BootTransactionTestCase): + def setUp(self): + # We need to clear previous media_root content + # This points to the test directory, but just in case + # we double check that the directory is indeed a test directory + if "_test" in settings.MEDIA_ROOT: + logger.info("Cleaning up {}".format(settings.MEDIA_ROOT)) + shutil.rmtree(settings.MEDIA_ROOT) + else: + logger.warning("We did not remove MEDIA_ROOT because we couldn't find a _test suffix in its path.") + def test_task(self): DELAY = 1 # time to sleep for during process launch, background processing, etc. client = APIClient() @@ -64,7 +81,6 @@ class TestApiTask(BootTransactionTestCase): res = client.post("/api/projects/0/tasks/", { 'images': [image1, image2] }, format="multipart") - print(res.status_code) self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) # Cannot create a task for a project for which we have no access to @@ -131,7 +147,6 @@ class TestApiTask(BootTransactionTestCase): # Neither should an individual tile # Z/X/Y coords are choosen based on node-odm test dataset for orthophoto_tiles/ res = client.get("/api/projects/{}/tasks/{}/tiles/16/16020/42443.png".format(project.id, task.id)) - print(res.status_code) self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) # Cannot access a tiles.json we have no access to @@ -161,6 +176,9 @@ class TestApiTask(BootTransactionTestCase): testWatch.clear() + # No UUID at this point + self.assertTrue(len(task.uuid) == 0) + # Assign processing node to task via API res = client.patch("/api/projects/{}/tasks/{}/".format(project.id, task.id), { 'processing_node': pnode.id @@ -170,14 +188,71 @@ class TestApiTask(BootTransactionTestCase): # On update scheduler.processing_pending_tasks should have been called in the background testWatch.wait_until_call("app.scheduler.process_pending_tasks", timeout=5) - # Processing should have completed + # Processing should have started and a UUID is assigned task.refresh_from_db() self.assertTrue(task.status == status_codes.RUNNING) + self.assertTrue(len(task.uuid) > 0) + + # Calling process pending tasks should finish the process + scheduler.process_pending_tasks() + task.refresh_from_db() + self.assertTrue(task.status == status_codes.COMPLETED) + + # Can download assets + for asset in assets: + res = client.get("/api/projects/{}/tasks/{}/download/{}/".format(project.id, task.id, asset)) + self.assertTrue(res.status_code == status.HTTP_200_OK) + + # Can download raw assets + res = client.get("/api/projects/{}/tasks/{}/assets/odm_orthophoto/odm_orthophoto.tif".format(project.id, task.id)) + self.assertTrue(res.status_code == status.HTTP_200_OK) + + # Can access tiles.json and individual tiles + res = client.get("/api/projects/{}/tasks/{}/tiles.json".format(project.id, task.id)) + self.assertTrue(res.status_code == status.HTTP_200_OK) + + res = client.get("/api/projects/{}/tasks/{}/tiles/16/16020/42443.png".format(project.id, task.id)) + self.assertTrue(res.status_code == status.HTTP_200_OK) + + # Restart a task + testWatch.clear() + res = client.post("/api/projects/{}/tasks/{}/restart/".format(project.id, task.id)) + self.assertTrue(res.status_code == status.HTTP_200_OK) + testWatch.wait_until_call("app.scheduler.process_pending_tasks", timeout=5) + task.refresh_from_db() + + self.assertTrue(task.status in [status_codes.RUNNING, status_codes.COMPLETED]) + + # Cancel a task + testWatch.clear() + res = client.post("/api/projects/{}/tasks/{}/cancel/".format(project.id, task.id)) + self.assertTrue(res.status_code == status.HTTP_200_OK) + testWatch.wait_until_call("app.scheduler.process_pending_tasks", timeout=5) + + # Should have been canceled + task.refresh_from_db() + self.assertTrue(task.status == status_codes.CANCELED) + + # Remove a task + res = client.post("/api/projects/{}/tasks/{}/remove/".format(project.id, task.id)) + self.assertTrue(res.status_code == status.HTTP_200_OK) + testWatch.wait_until_call("app.scheduler.process_pending_tasks", 2, timeout=5) + + # Has been removed along with assets + self.assertFalse(Task.objects.filter(pk=task.id).exists()) + self.assertFalse(ImageUpload.objects.filter(task=task).exists()) + + task_assets_path = os.path.join(settings.MEDIA_ROOT, + task_directory_path(task.id, task.project.id)) + self.assertFalse(os.path.exists(task_assets_path)) + + testWatch.clear() + testWatch.intercept("app.scheduler.process_pending_tasks") + + - # TODO: check # TODO: what happens when nodes go offline, or an offline node is assigned to a task - # TODO: check raw/non-raw assets once task is finished processing - # TODO: recheck tiles, tiles.json urls, etc. + # TODO: timeout issues # Teardown processing node node_odm.terminate()