kopia lustrzana https://github.com/OpenDroneMap/WebODM
Move task.console_output
rodzic
53079dbd30
commit
e510e2fc9b
|
@ -74,7 +74,7 @@ class TaskSerializer(serializers.ModelSerializer):
|
|||
|
||||
class Meta:
|
||||
model = models.Task
|
||||
exclude = ('console_output', 'orthophoto_extent', 'dsm_extent', 'dtm_extent', )
|
||||
exclude = ('orthophoto_extent', 'dsm_extent', 'dtm_extent', )
|
||||
read_only_fields = ('processing_time', 'status', 'last_error', 'created_at', 'pending_action', 'available_assets', 'size', )
|
||||
|
||||
class TaskViewSet(viewsets.ViewSet):
|
||||
|
@ -83,7 +83,7 @@ class TaskViewSet(viewsets.ViewSet):
|
|||
A task represents a set of images and other input to be sent to a processing node.
|
||||
Once a processing node completes processing, results are stored in the task.
|
||||
"""
|
||||
queryset = models.Task.objects.all().defer('orthophoto_extent', 'dsm_extent', 'dtm_extent', 'console_output', )
|
||||
queryset = models.Task.objects.all().defer('orthophoto_extent', 'dsm_extent', 'dtm_extent', )
|
||||
|
||||
parser_classes = (parsers.MultiPartParser, parsers.JSONParser, parsers.FormParser, )
|
||||
ordering_fields = '__all__'
|
||||
|
@ -145,8 +145,7 @@ class TaskViewSet(viewsets.ViewSet):
|
|||
raise exceptions.NotFound()
|
||||
|
||||
line_num = max(0, int(request.query_params.get('line', 0)))
|
||||
output = task.console_output or ""
|
||||
return Response('\n'.join(output.rstrip().split('\n')[line_num:]))
|
||||
return Response('\n'.join(task.console.output().rstrip().split('\n')[line_num:]))
|
||||
|
||||
def list(self, request, project_pk=None):
|
||||
get_and_check_project(request, project_pk)
|
||||
|
@ -296,7 +295,7 @@ class TaskViewSet(viewsets.ViewSet):
|
|||
|
||||
|
||||
class TaskNestedView(APIView):
|
||||
queryset = models.Task.objects.all().defer('orthophoto_extent', 'dtm_extent', 'dsm_extent', 'console_output', )
|
||||
queryset = models.Task.objects.all().defer('orthophoto_extent', 'dtm_extent', 'dsm_extent', )
|
||||
permission_classes = (AllowAny, )
|
||||
|
||||
def get_and_check_task(self, request, pk, annotate={}):
|
||||
|
|
|
@ -0,0 +1,48 @@
|
|||
import os
|
||||
import logging
|
||||
logger = logging.getLogger('app.logger')
|
||||
|
||||
class Console:
|
||||
def __init__(self, file):
|
||||
self.file = file
|
||||
self.base_dir = os.path.dirname(self.file)
|
||||
self.parent_dir = os.path.dirname(self.base_dir)
|
||||
|
||||
def __repr__(self):
|
||||
return "<Console output: %s>" % self.file
|
||||
|
||||
def __str__(self):
|
||||
if not os.path.isfile(self.file):
|
||||
return ""
|
||||
|
||||
try:
|
||||
with open(self.file, 'r') as f:
|
||||
return f.read()
|
||||
except IOError:
|
||||
logger.warn("Cannot read console file: %s" % self.file)
|
||||
return ""
|
||||
|
||||
def __add__(self, other):
|
||||
self.append(other)
|
||||
return self
|
||||
|
||||
def output(self):
|
||||
return str(self)
|
||||
|
||||
def append(self, text):
|
||||
if os.path.isdir(self.parent_dir):
|
||||
# Write
|
||||
if not os.path.isdir(self.base_dir):
|
||||
os.makedirs(self.base_dir, exist_ok=True)
|
||||
|
||||
with open(self.file, "a") as f:
|
||||
f.write(text)
|
||||
|
||||
def reset(self, text = ""):
|
||||
if os.path.isdir(self.parent_dir):
|
||||
if not os.path.isdir(self.base_dir):
|
||||
os.makedirs(self.base_dir, exist_ok=True)
|
||||
|
||||
with open(self.file, "w") as f:
|
||||
f.write(text)
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
# Generated by Django 2.2.27 on 2023-09-11 19:11
|
||||
import os
|
||||
from django.db import migrations
|
||||
from webodm import settings
|
||||
|
||||
def data_path(project_id, task_id, *args):
|
||||
return os.path.join(settings.MEDIA_ROOT,
|
||||
"project",
|
||||
str(project_id),
|
||||
"task",
|
||||
str(task_id),
|
||||
"data",
|
||||
*args)
|
||||
|
||||
def dump_console_outputs(apps, schema_editor):
|
||||
Task = apps.get_model('app', 'Task')
|
||||
|
||||
for t in Task.objects.all():
|
||||
if t.console_output is not None and len(t.console_output) > 0:
|
||||
dp = data_path(t.project.id, t.id)
|
||||
os.makedirs(dp, exist_ok=True)
|
||||
outfile = os.path.join(dp, "console_output.txt")
|
||||
|
||||
with open(outfile, "w") as f:
|
||||
f.write(t.console_output)
|
||||
print("Wrote console output for %s to %s" % (t, outfile))
|
||||
else:
|
||||
print("No task output for %s" % t)
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('app', '0037_profile'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RunPython(dump_console_outputs),
|
||||
migrations.RemoveField(
|
||||
model_name='task',
|
||||
name='console_output',
|
||||
),
|
||||
]
|
|
@ -46,6 +46,7 @@ from django.utils.translation import gettext_lazy as _, gettext
|
|||
|
||||
from functools import partial
|
||||
import subprocess
|
||||
from app.classes.console import Console
|
||||
|
||||
logger = logging.getLogger('app.logger')
|
||||
|
||||
|
@ -247,7 +248,6 @@ class Task(models.Model):
|
|||
last_error = models.TextField(null=True, blank=True, help_text=_("The last processing error received"), verbose_name=_("Last Error"))
|
||||
options = fields.JSONField(default=dict, blank=True, help_text=_("Options that are being used to process this task"), validators=[validate_task_options], verbose_name=_("Options"))
|
||||
available_assets = fields.ArrayField(models.CharField(max_length=80), default=list, blank=True, help_text=_("List of available assets to download"), verbose_name=_("Available Assets"))
|
||||
console_output = models.TextField(null=False, default="", blank=True, help_text=_("Console output of the processing node"), verbose_name=_("Console Output"))
|
||||
|
||||
orthophoto_extent = GeometryField(null=True, blank=True, srid=4326, help_text=_("Extent of the orthophoto"), verbose_name=_("Orthophoto Extent"))
|
||||
dsm_extent = GeometryField(null=True, blank=True, srid=4326, help_text="Extent of the DSM", verbose_name=_("DSM Extent"))
|
||||
|
@ -291,6 +291,8 @@ class Task(models.Model):
|
|||
# To help keep track of changes to the project id
|
||||
self.__original_project_id = self.project.id
|
||||
|
||||
self.console = Console(self.data_path("console_output.txt"))
|
||||
|
||||
def __str__(self):
|
||||
name = self.name if self.name is not None else gettext("unnamed")
|
||||
|
||||
|
@ -354,6 +356,12 @@ class Task(models.Model):
|
|||
"""
|
||||
return self.task_path("assets", *args)
|
||||
|
||||
def data_path(self, *args):
|
||||
"""
|
||||
Path to task data that does not fit in database fields (e.g. console output)
|
||||
"""
|
||||
return self.task_path("data", *args)
|
||||
|
||||
def task_path(self, *args):
|
||||
"""
|
||||
Get path relative to the root task directory
|
||||
|
@ -490,7 +498,7 @@ class Task(models.Model):
|
|||
raise FileNotFoundError("{} is not a valid asset".format(asset))
|
||||
|
||||
def handle_import(self):
|
||||
self.console_output += gettext("Importing assets...") + "\n"
|
||||
self.console += gettext("Importing assets...") + "\n"
|
||||
self.save()
|
||||
|
||||
zip_path = self.assets_path("all.zip")
|
||||
|
@ -709,7 +717,7 @@ class Task(models.Model):
|
|||
self.options = list(filter(lambda d: d['name'] != 'rerun-from', self.options))
|
||||
self.upload_progress = 0
|
||||
|
||||
self.console_output = ""
|
||||
self.console.reset()
|
||||
self.processing_time = -1
|
||||
self.status = None
|
||||
self.last_error = None
|
||||
|
@ -740,10 +748,10 @@ class Task(models.Model):
|
|||
# Need to update status (first time, queued or running?)
|
||||
if self.uuid and self.status in [None, status_codes.QUEUED, status_codes.RUNNING]:
|
||||
# Update task info from processing node
|
||||
if not self.console_output:
|
||||
if not self.console.output():
|
||||
current_lines_count = 0
|
||||
else:
|
||||
current_lines_count = len(self.console_output.split("\n"))
|
||||
current_lines_count = len(self.console.output().split("\n"))
|
||||
|
||||
info = self.processing_node.get_task_info(self.uuid, current_lines_count)
|
||||
|
||||
|
@ -751,7 +759,7 @@ class Task(models.Model):
|
|||
self.status = info.status.value
|
||||
|
||||
if len(info.output) > 0:
|
||||
self.console_output += "\n".join(info.output) + '\n'
|
||||
self.console += "\n".join(info.output) + '\n'
|
||||
|
||||
# Update running progress
|
||||
self.running_progress = (info.progress / 100.0) * self.TASK_PROGRESS_LAST_VALUE
|
||||
|
@ -891,7 +899,7 @@ class Task(models.Model):
|
|||
self.update_size()
|
||||
self.potree_scene = {}
|
||||
self.running_progress = 1.0
|
||||
self.console_output += gettext("Done!") + "\n"
|
||||
self.console += gettext("Done!") + "\n"
|
||||
self.status = status_codes.COMPLETED
|
||||
self.save()
|
||||
|
||||
|
|
|
@ -140,12 +140,12 @@ class TestApi(BootTestCase):
|
|||
self.assertEqual(res.status_code, status.HTTP_200_OK)
|
||||
self.assertTrue(res.data == "")
|
||||
|
||||
task.console_output = "line1\nline2\nline3"
|
||||
task.console.reset("line1\nline2\nline3")
|
||||
task.save()
|
||||
|
||||
res = client.get('/api/projects/{}/tasks/{}/output/'.format(project.id, task.id))
|
||||
self.assertEqual(res.status_code, status.HTTP_200_OK)
|
||||
self.assertTrue(res.data == task.console_output)
|
||||
self.assertTrue(res.data == task.console.output())
|
||||
|
||||
# Console output with line num
|
||||
res = client.get('/api/projects/{}/tasks/{}/output/?line=2'.format(project.id, task.id))
|
||||
|
@ -155,7 +155,7 @@ class TestApi(BootTestCase):
|
|||
res = client.get('/api/projects/{}/tasks/{}/output/?line=3'.format(project.id, task.id))
|
||||
self.assertTrue(res.data == "")
|
||||
res = client.get('/api/projects/{}/tasks/{}/output/?line=-1'.format(project.id, task.id))
|
||||
self.assertTrue(res.data == task.console_output)
|
||||
self.assertTrue(res.data == task.console.output())
|
||||
|
||||
# Cannot list task details for a task belonging to a project we don't have access to
|
||||
res = client.get('/api/projects/{}/tasks/{}/'.format(other_project.id, other_task.id))
|
||||
|
|
|
@ -41,7 +41,7 @@ class ImportFolderTaskView(TaskView):
|
|||
files = platform.import_from_folder(folder_url)
|
||||
|
||||
# Update the task with the new information
|
||||
task.console_output += "Importing {} images...\n".format(len(files))
|
||||
task.console += "Importing {} images...\n".format(len(files))
|
||||
task.images_count = len(files)
|
||||
task.pending_action = pending_actions.IMPORT
|
||||
task.save()
|
||||
|
|
|
@ -181,7 +181,7 @@ class ImportDatasetTaskView(TaskView):
|
|||
return Response({'error': 'Empty dataset or folder.'}, status=status.HTTP_400_BAD_REQUEST)
|
||||
|
||||
# Update the task with the new information
|
||||
task.console_output += "Importing {} images...\n".format(len(files))
|
||||
task.console += "Importing {} images...\n".format(len(files))
|
||||
task.images_count = len(files)
|
||||
task.pending_action = pending_actions.IMPORT
|
||||
task.save()
|
||||
|
|
|
@ -22,7 +22,7 @@ def handle_task_completed(sender, task_id, **kwargs):
|
|||
setting = Setting.objects.first()
|
||||
notification_app_name = config_data['notification_app_name'] or settings.app_name
|
||||
|
||||
console_output = reverse_output(task.console_output)
|
||||
console_output = reverse_output(task.console.output())
|
||||
notification.send(
|
||||
f"{notification_app_name} - {task.project.name} Task Completed",
|
||||
f"{task.project.name}\n{task.name} Completed\nProcessing time:{hours_minutes_secs(task.processing_time)}\n\nConsole Output:{console_output}",
|
||||
|
@ -41,7 +41,7 @@ def handle_task_removed(sender, task_id, **kwargs):
|
|||
task = Task.objects.get(id=task_id)
|
||||
setting = Setting.objects.first()
|
||||
notification_app_name = config_data['notification_app_name'] or settings.app_name
|
||||
console_output = reverse_output(task.console_output)
|
||||
console_output = reverse_output(task.console.output())
|
||||
notification.send(
|
||||
f"{notification_app_name} - {task.project.name} Task removed",
|
||||
f"{task.project.name}\n{task.name} was removed\nProcessing time:{hours_minutes_secs(task.processing_time)}\n\nConsole Output:{console_output}",
|
||||
|
@ -60,7 +60,7 @@ def handle_task_failed(sender, task_id, **kwargs):
|
|||
task = Task.objects.get(id=task_id)
|
||||
setting = Setting.objects.first()
|
||||
notification_app_name = config_data['notification_app_name'] or settings.app_name
|
||||
console_output = reverse_output(task.console_output)
|
||||
console_output = reverse_output(task.console.output())
|
||||
notification.send(
|
||||
f"{notification_app_name} - {task.project.name} Task Failed",
|
||||
f"{task.project.name}\n{task.name} Failed with error: {task.last_error}\nProcessing time:{hours_minutes_secs(task.processing_time)}\n\nConsole Output:{console_output}",
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "WebODM",
|
||||
"version": "2.1.0",
|
||||
"version": "2.1.1",
|
||||
"description": "User-friendly, extendable application and API for processing aerial imagery.",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
|
|
Ładowanie…
Reference in New Issue