From e510e2fc9b15e4b8e31d778bcd3fc60d19db8b9d Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 11 Sep 2023 16:35:54 -0400 Subject: [PATCH] Move task.console_output --- app/api/tasks.py | 9 ++-- app/classes/console.py | 48 +++++++++++++++++++ .../0038_remove_task_console_output.py | 42 ++++++++++++++++ app/models/task.py | 22 ++++++--- app/tests/test_api.py | 6 +-- coreplugins/cloudimport/api_views.py | 2 +- coreplugins/dronedb/api_views.py | 2 +- coreplugins/tasknotification/signals.py | 6 +-- package.json | 2 +- 9 files changed, 118 insertions(+), 21 deletions(-) create mode 100644 app/classes/console.py create mode 100644 app/migrations/0038_remove_task_console_output.py diff --git a/app/api/tasks.py b/app/api/tasks.py index bb2d4a7c..191178b6 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -74,7 +74,7 @@ class TaskSerializer(serializers.ModelSerializer): class Meta: model = models.Task - exclude = ('console_output', 'orthophoto_extent', 'dsm_extent', 'dtm_extent', ) + exclude = ('orthophoto_extent', 'dsm_extent', 'dtm_extent', ) read_only_fields = ('processing_time', 'status', 'last_error', 'created_at', 'pending_action', 'available_assets', 'size', ) class TaskViewSet(viewsets.ViewSet): @@ -83,7 +83,7 @@ class TaskViewSet(viewsets.ViewSet): A task represents a set of images and other input to be sent to a processing node. Once a processing node completes processing, results are stored in the task. """ - queryset = models.Task.objects.all().defer('orthophoto_extent', 'dsm_extent', 'dtm_extent', 'console_output', ) + queryset = models.Task.objects.all().defer('orthophoto_extent', 'dsm_extent', 'dtm_extent', ) parser_classes = (parsers.MultiPartParser, parsers.JSONParser, parsers.FormParser, ) ordering_fields = '__all__' @@ -145,8 +145,7 @@ class TaskViewSet(viewsets.ViewSet): raise exceptions.NotFound() line_num = max(0, int(request.query_params.get('line', 0))) - output = task.console_output or "" - return Response('\n'.join(output.rstrip().split('\n')[line_num:])) + return Response('\n'.join(task.console.output().rstrip().split('\n')[line_num:])) def list(self, request, project_pk=None): get_and_check_project(request, project_pk) @@ -296,7 +295,7 @@ class TaskViewSet(viewsets.ViewSet): class TaskNestedView(APIView): - queryset = models.Task.objects.all().defer('orthophoto_extent', 'dtm_extent', 'dsm_extent', 'console_output', ) + queryset = models.Task.objects.all().defer('orthophoto_extent', 'dtm_extent', 'dsm_extent', ) permission_classes = (AllowAny, ) def get_and_check_task(self, request, pk, annotate={}): diff --git a/app/classes/console.py b/app/classes/console.py new file mode 100644 index 00000000..d9f0a1e2 --- /dev/null +++ b/app/classes/console.py @@ -0,0 +1,48 @@ +import os +import logging +logger = logging.getLogger('app.logger') + +class Console: + def __init__(self, file): + self.file = file + self.base_dir = os.path.dirname(self.file) + self.parent_dir = os.path.dirname(self.base_dir) + + def __repr__(self): + return "" % self.file + + def __str__(self): + if not os.path.isfile(self.file): + return "" + + try: + with open(self.file, 'r') as f: + return f.read() + except IOError: + logger.warn("Cannot read console file: %s" % self.file) + return "" + + def __add__(self, other): + self.append(other) + return self + + def output(self): + return str(self) + + def append(self, text): + if os.path.isdir(self.parent_dir): + # Write + if not os.path.isdir(self.base_dir): + os.makedirs(self.base_dir, exist_ok=True) + + with open(self.file, "a") as f: + f.write(text) + + def reset(self, text = ""): + if os.path.isdir(self.parent_dir): + if not os.path.isdir(self.base_dir): + os.makedirs(self.base_dir, exist_ok=True) + + with open(self.file, "w") as f: + f.write(text) + diff --git a/app/migrations/0038_remove_task_console_output.py b/app/migrations/0038_remove_task_console_output.py new file mode 100644 index 00000000..88fc024f --- /dev/null +++ b/app/migrations/0038_remove_task_console_output.py @@ -0,0 +1,42 @@ +# Generated by Django 2.2.27 on 2023-09-11 19:11 +import os +from django.db import migrations +from webodm import settings + +def data_path(project_id, task_id, *args): + return os.path.join(settings.MEDIA_ROOT, + "project", + str(project_id), + "task", + str(task_id), + "data", + *args) + +def dump_console_outputs(apps, schema_editor): + Task = apps.get_model('app', 'Task') + + for t in Task.objects.all(): + if t.console_output is not None and len(t.console_output) > 0: + dp = data_path(t.project.id, t.id) + os.makedirs(dp, exist_ok=True) + outfile = os.path.join(dp, "console_output.txt") + + with open(outfile, "w") as f: + f.write(t.console_output) + print("Wrote console output for %s to %s" % (t, outfile)) + else: + print("No task output for %s" % t) + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0037_profile'), + ] + + operations = [ + migrations.RunPython(dump_console_outputs), + migrations.RemoveField( + model_name='task', + name='console_output', + ), + ] diff --git a/app/models/task.py b/app/models/task.py index 4235011c..1f0c4c0c 100644 --- a/app/models/task.py +++ b/app/models/task.py @@ -46,6 +46,7 @@ from django.utils.translation import gettext_lazy as _, gettext from functools import partial import subprocess +from app.classes.console import Console logger = logging.getLogger('app.logger') @@ -247,7 +248,6 @@ class Task(models.Model): last_error = models.TextField(null=True, blank=True, help_text=_("The last processing error received"), verbose_name=_("Last Error")) options = fields.JSONField(default=dict, blank=True, help_text=_("Options that are being used to process this task"), validators=[validate_task_options], verbose_name=_("Options")) available_assets = fields.ArrayField(models.CharField(max_length=80), default=list, blank=True, help_text=_("List of available assets to download"), verbose_name=_("Available Assets")) - console_output = models.TextField(null=False, default="", blank=True, help_text=_("Console output of the processing node"), verbose_name=_("Console Output")) orthophoto_extent = GeometryField(null=True, blank=True, srid=4326, help_text=_("Extent of the orthophoto"), verbose_name=_("Orthophoto Extent")) dsm_extent = GeometryField(null=True, blank=True, srid=4326, help_text="Extent of the DSM", verbose_name=_("DSM Extent")) @@ -290,6 +290,8 @@ class Task(models.Model): # To help keep track of changes to the project id self.__original_project_id = self.project.id + + self.console = Console(self.data_path("console_output.txt")) def __str__(self): name = self.name if self.name is not None else gettext("unnamed") @@ -354,6 +356,12 @@ class Task(models.Model): """ return self.task_path("assets", *args) + def data_path(self, *args): + """ + Path to task data that does not fit in database fields (e.g. console output) + """ + return self.task_path("data", *args) + def task_path(self, *args): """ Get path relative to the root task directory @@ -490,7 +498,7 @@ class Task(models.Model): raise FileNotFoundError("{} is not a valid asset".format(asset)) def handle_import(self): - self.console_output += gettext("Importing assets...") + "\n" + self.console += gettext("Importing assets...") + "\n" self.save() zip_path = self.assets_path("all.zip") @@ -709,7 +717,7 @@ class Task(models.Model): self.options = list(filter(lambda d: d['name'] != 'rerun-from', self.options)) self.upload_progress = 0 - self.console_output = "" + self.console.reset() self.processing_time = -1 self.status = None self.last_error = None @@ -740,10 +748,10 @@ class Task(models.Model): # Need to update status (first time, queued or running?) if self.uuid and self.status in [None, status_codes.QUEUED, status_codes.RUNNING]: # Update task info from processing node - if not self.console_output: + if not self.console.output(): current_lines_count = 0 else: - current_lines_count = len(self.console_output.split("\n")) + current_lines_count = len(self.console.output().split("\n")) info = self.processing_node.get_task_info(self.uuid, current_lines_count) @@ -751,7 +759,7 @@ class Task(models.Model): self.status = info.status.value if len(info.output) > 0: - self.console_output += "\n".join(info.output) + '\n' + self.console += "\n".join(info.output) + '\n' # Update running progress self.running_progress = (info.progress / 100.0) * self.TASK_PROGRESS_LAST_VALUE @@ -891,7 +899,7 @@ class Task(models.Model): self.update_size() self.potree_scene = {} self.running_progress = 1.0 - self.console_output += gettext("Done!") + "\n" + self.console += gettext("Done!") + "\n" self.status = status_codes.COMPLETED self.save() diff --git a/app/tests/test_api.py b/app/tests/test_api.py index fd392a35..c9791683 100644 --- a/app/tests/test_api.py +++ b/app/tests/test_api.py @@ -140,12 +140,12 @@ class TestApi(BootTestCase): self.assertEqual(res.status_code, status.HTTP_200_OK) self.assertTrue(res.data == "") - task.console_output = "line1\nline2\nline3" + task.console.reset("line1\nline2\nline3") task.save() res = client.get('/api/projects/{}/tasks/{}/output/'.format(project.id, task.id)) self.assertEqual(res.status_code, status.HTTP_200_OK) - self.assertTrue(res.data == task.console_output) + self.assertTrue(res.data == task.console.output()) # Console output with line num res = client.get('/api/projects/{}/tasks/{}/output/?line=2'.format(project.id, task.id)) @@ -155,7 +155,7 @@ class TestApi(BootTestCase): res = client.get('/api/projects/{}/tasks/{}/output/?line=3'.format(project.id, task.id)) self.assertTrue(res.data == "") res = client.get('/api/projects/{}/tasks/{}/output/?line=-1'.format(project.id, task.id)) - self.assertTrue(res.data == task.console_output) + self.assertTrue(res.data == task.console.output()) # Cannot list task details for a task belonging to a project we don't have access to res = client.get('/api/projects/{}/tasks/{}/'.format(other_project.id, other_task.id)) diff --git a/coreplugins/cloudimport/api_views.py b/coreplugins/cloudimport/api_views.py index a4100dc0..46969104 100644 --- a/coreplugins/cloudimport/api_views.py +++ b/coreplugins/cloudimport/api_views.py @@ -41,7 +41,7 @@ class ImportFolderTaskView(TaskView): files = platform.import_from_folder(folder_url) # Update the task with the new information - task.console_output += "Importing {} images...\n".format(len(files)) + task.console += "Importing {} images...\n".format(len(files)) task.images_count = len(files) task.pending_action = pending_actions.IMPORT task.save() diff --git a/coreplugins/dronedb/api_views.py b/coreplugins/dronedb/api_views.py index 1b7bec6b..673556ad 100644 --- a/coreplugins/dronedb/api_views.py +++ b/coreplugins/dronedb/api_views.py @@ -181,7 +181,7 @@ class ImportDatasetTaskView(TaskView): return Response({'error': 'Empty dataset or folder.'}, status=status.HTTP_400_BAD_REQUEST) # Update the task with the new information - task.console_output += "Importing {} images...\n".format(len(files)) + task.console += "Importing {} images...\n".format(len(files)) task.images_count = len(files) task.pending_action = pending_actions.IMPORT task.save() diff --git a/coreplugins/tasknotification/signals.py b/coreplugins/tasknotification/signals.py index ad49e4c1..18236775 100644 --- a/coreplugins/tasknotification/signals.py +++ b/coreplugins/tasknotification/signals.py @@ -22,7 +22,7 @@ def handle_task_completed(sender, task_id, **kwargs): setting = Setting.objects.first() notification_app_name = config_data['notification_app_name'] or settings.app_name - console_output = reverse_output(task.console_output) + console_output = reverse_output(task.console.output()) notification.send( f"{notification_app_name} - {task.project.name} Task Completed", f"{task.project.name}\n{task.name} Completed\nProcessing time:{hours_minutes_secs(task.processing_time)}\n\nConsole Output:{console_output}", @@ -41,7 +41,7 @@ def handle_task_removed(sender, task_id, **kwargs): task = Task.objects.get(id=task_id) setting = Setting.objects.first() notification_app_name = config_data['notification_app_name'] or settings.app_name - console_output = reverse_output(task.console_output) + console_output = reverse_output(task.console.output()) notification.send( f"{notification_app_name} - {task.project.name} Task removed", f"{task.project.name}\n{task.name} was removed\nProcessing time:{hours_minutes_secs(task.processing_time)}\n\nConsole Output:{console_output}", @@ -60,7 +60,7 @@ def handle_task_failed(sender, task_id, **kwargs): task = Task.objects.get(id=task_id) setting = Setting.objects.first() notification_app_name = config_data['notification_app_name'] or settings.app_name - console_output = reverse_output(task.console_output) + console_output = reverse_output(task.console.output()) notification.send( f"{notification_app_name} - {task.project.name} Task Failed", f"{task.project.name}\n{task.name} Failed with error: {task.last_error}\nProcessing time:{hours_minutes_secs(task.processing_time)}\n\nConsole Output:{console_output}", diff --git a/package.json b/package.json index 3887114c..7019d92b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "WebODM", - "version": "2.1.0", + "version": "2.1.1", "description": "User-friendly, extendable application and API for processing aerial imagery.", "main": "index.js", "scripts": {