kopia lustrzana https://github.com/OpenDroneMap/WebODM
Move task.console_output
rodzic
53079dbd30
commit
e510e2fc9b
|
@ -74,7 +74,7 @@ class TaskSerializer(serializers.ModelSerializer):
|
||||||
|
|
||||||
class Meta:
|
class Meta:
|
||||||
model = models.Task
|
model = models.Task
|
||||||
exclude = ('console_output', 'orthophoto_extent', 'dsm_extent', 'dtm_extent', )
|
exclude = ('orthophoto_extent', 'dsm_extent', 'dtm_extent', )
|
||||||
read_only_fields = ('processing_time', 'status', 'last_error', 'created_at', 'pending_action', 'available_assets', 'size', )
|
read_only_fields = ('processing_time', 'status', 'last_error', 'created_at', 'pending_action', 'available_assets', 'size', )
|
||||||
|
|
||||||
class TaskViewSet(viewsets.ViewSet):
|
class TaskViewSet(viewsets.ViewSet):
|
||||||
|
@ -83,7 +83,7 @@ class TaskViewSet(viewsets.ViewSet):
|
||||||
A task represents a set of images and other input to be sent to a processing node.
|
A task represents a set of images and other input to be sent to a processing node.
|
||||||
Once a processing node completes processing, results are stored in the task.
|
Once a processing node completes processing, results are stored in the task.
|
||||||
"""
|
"""
|
||||||
queryset = models.Task.objects.all().defer('orthophoto_extent', 'dsm_extent', 'dtm_extent', 'console_output', )
|
queryset = models.Task.objects.all().defer('orthophoto_extent', 'dsm_extent', 'dtm_extent', )
|
||||||
|
|
||||||
parser_classes = (parsers.MultiPartParser, parsers.JSONParser, parsers.FormParser, )
|
parser_classes = (parsers.MultiPartParser, parsers.JSONParser, parsers.FormParser, )
|
||||||
ordering_fields = '__all__'
|
ordering_fields = '__all__'
|
||||||
|
@ -145,8 +145,7 @@ class TaskViewSet(viewsets.ViewSet):
|
||||||
raise exceptions.NotFound()
|
raise exceptions.NotFound()
|
||||||
|
|
||||||
line_num = max(0, int(request.query_params.get('line', 0)))
|
line_num = max(0, int(request.query_params.get('line', 0)))
|
||||||
output = task.console_output or ""
|
return Response('\n'.join(task.console.output().rstrip().split('\n')[line_num:]))
|
||||||
return Response('\n'.join(output.rstrip().split('\n')[line_num:]))
|
|
||||||
|
|
||||||
def list(self, request, project_pk=None):
|
def list(self, request, project_pk=None):
|
||||||
get_and_check_project(request, project_pk)
|
get_and_check_project(request, project_pk)
|
||||||
|
@ -296,7 +295,7 @@ class TaskViewSet(viewsets.ViewSet):
|
||||||
|
|
||||||
|
|
||||||
class TaskNestedView(APIView):
|
class TaskNestedView(APIView):
|
||||||
queryset = models.Task.objects.all().defer('orthophoto_extent', 'dtm_extent', 'dsm_extent', 'console_output', )
|
queryset = models.Task.objects.all().defer('orthophoto_extent', 'dtm_extent', 'dsm_extent', )
|
||||||
permission_classes = (AllowAny, )
|
permission_classes = (AllowAny, )
|
||||||
|
|
||||||
def get_and_check_task(self, request, pk, annotate={}):
|
def get_and_check_task(self, request, pk, annotate={}):
|
||||||
|
|
|
@ -0,0 +1,48 @@
|
||||||
|
import os
|
||||||
|
import logging
|
||||||
|
logger = logging.getLogger('app.logger')
|
||||||
|
|
||||||
|
class Console:
|
||||||
|
def __init__(self, file):
|
||||||
|
self.file = file
|
||||||
|
self.base_dir = os.path.dirname(self.file)
|
||||||
|
self.parent_dir = os.path.dirname(self.base_dir)
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
return "<Console output: %s>" % self.file
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
if not os.path.isfile(self.file):
|
||||||
|
return ""
|
||||||
|
|
||||||
|
try:
|
||||||
|
with open(self.file, 'r') as f:
|
||||||
|
return f.read()
|
||||||
|
except IOError:
|
||||||
|
logger.warn("Cannot read console file: %s" % self.file)
|
||||||
|
return ""
|
||||||
|
|
||||||
|
def __add__(self, other):
|
||||||
|
self.append(other)
|
||||||
|
return self
|
||||||
|
|
||||||
|
def output(self):
|
||||||
|
return str(self)
|
||||||
|
|
||||||
|
def append(self, text):
|
||||||
|
if os.path.isdir(self.parent_dir):
|
||||||
|
# Write
|
||||||
|
if not os.path.isdir(self.base_dir):
|
||||||
|
os.makedirs(self.base_dir, exist_ok=True)
|
||||||
|
|
||||||
|
with open(self.file, "a") as f:
|
||||||
|
f.write(text)
|
||||||
|
|
||||||
|
def reset(self, text = ""):
|
||||||
|
if os.path.isdir(self.parent_dir):
|
||||||
|
if not os.path.isdir(self.base_dir):
|
||||||
|
os.makedirs(self.base_dir, exist_ok=True)
|
||||||
|
|
||||||
|
with open(self.file, "w") as f:
|
||||||
|
f.write(text)
|
||||||
|
|
|
@ -0,0 +1,42 @@
|
||||||
|
# Generated by Django 2.2.27 on 2023-09-11 19:11
|
||||||
|
import os
|
||||||
|
from django.db import migrations
|
||||||
|
from webodm import settings
|
||||||
|
|
||||||
|
def data_path(project_id, task_id, *args):
|
||||||
|
return os.path.join(settings.MEDIA_ROOT,
|
||||||
|
"project",
|
||||||
|
str(project_id),
|
||||||
|
"task",
|
||||||
|
str(task_id),
|
||||||
|
"data",
|
||||||
|
*args)
|
||||||
|
|
||||||
|
def dump_console_outputs(apps, schema_editor):
|
||||||
|
Task = apps.get_model('app', 'Task')
|
||||||
|
|
||||||
|
for t in Task.objects.all():
|
||||||
|
if t.console_output is not None and len(t.console_output) > 0:
|
||||||
|
dp = data_path(t.project.id, t.id)
|
||||||
|
os.makedirs(dp, exist_ok=True)
|
||||||
|
outfile = os.path.join(dp, "console_output.txt")
|
||||||
|
|
||||||
|
with open(outfile, "w") as f:
|
||||||
|
f.write(t.console_output)
|
||||||
|
print("Wrote console output for %s to %s" % (t, outfile))
|
||||||
|
else:
|
||||||
|
print("No task output for %s" % t)
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('app', '0037_profile'),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.RunPython(dump_console_outputs),
|
||||||
|
migrations.RemoveField(
|
||||||
|
model_name='task',
|
||||||
|
name='console_output',
|
||||||
|
),
|
||||||
|
]
|
|
@ -46,6 +46,7 @@ from django.utils.translation import gettext_lazy as _, gettext
|
||||||
|
|
||||||
from functools import partial
|
from functools import partial
|
||||||
import subprocess
|
import subprocess
|
||||||
|
from app.classes.console import Console
|
||||||
|
|
||||||
logger = logging.getLogger('app.logger')
|
logger = logging.getLogger('app.logger')
|
||||||
|
|
||||||
|
@ -247,7 +248,6 @@ class Task(models.Model):
|
||||||
last_error = models.TextField(null=True, blank=True, help_text=_("The last processing error received"), verbose_name=_("Last Error"))
|
last_error = models.TextField(null=True, blank=True, help_text=_("The last processing error received"), verbose_name=_("Last Error"))
|
||||||
options = fields.JSONField(default=dict, blank=True, help_text=_("Options that are being used to process this task"), validators=[validate_task_options], verbose_name=_("Options"))
|
options = fields.JSONField(default=dict, blank=True, help_text=_("Options that are being used to process this task"), validators=[validate_task_options], verbose_name=_("Options"))
|
||||||
available_assets = fields.ArrayField(models.CharField(max_length=80), default=list, blank=True, help_text=_("List of available assets to download"), verbose_name=_("Available Assets"))
|
available_assets = fields.ArrayField(models.CharField(max_length=80), default=list, blank=True, help_text=_("List of available assets to download"), verbose_name=_("Available Assets"))
|
||||||
console_output = models.TextField(null=False, default="", blank=True, help_text=_("Console output of the processing node"), verbose_name=_("Console Output"))
|
|
||||||
|
|
||||||
orthophoto_extent = GeometryField(null=True, blank=True, srid=4326, help_text=_("Extent of the orthophoto"), verbose_name=_("Orthophoto Extent"))
|
orthophoto_extent = GeometryField(null=True, blank=True, srid=4326, help_text=_("Extent of the orthophoto"), verbose_name=_("Orthophoto Extent"))
|
||||||
dsm_extent = GeometryField(null=True, blank=True, srid=4326, help_text="Extent of the DSM", verbose_name=_("DSM Extent"))
|
dsm_extent = GeometryField(null=True, blank=True, srid=4326, help_text="Extent of the DSM", verbose_name=_("DSM Extent"))
|
||||||
|
@ -290,6 +290,8 @@ class Task(models.Model):
|
||||||
|
|
||||||
# To help keep track of changes to the project id
|
# To help keep track of changes to the project id
|
||||||
self.__original_project_id = self.project.id
|
self.__original_project_id = self.project.id
|
||||||
|
|
||||||
|
self.console = Console(self.data_path("console_output.txt"))
|
||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
name = self.name if self.name is not None else gettext("unnamed")
|
name = self.name if self.name is not None else gettext("unnamed")
|
||||||
|
@ -354,6 +356,12 @@ class Task(models.Model):
|
||||||
"""
|
"""
|
||||||
return self.task_path("assets", *args)
|
return self.task_path("assets", *args)
|
||||||
|
|
||||||
|
def data_path(self, *args):
|
||||||
|
"""
|
||||||
|
Path to task data that does not fit in database fields (e.g. console output)
|
||||||
|
"""
|
||||||
|
return self.task_path("data", *args)
|
||||||
|
|
||||||
def task_path(self, *args):
|
def task_path(self, *args):
|
||||||
"""
|
"""
|
||||||
Get path relative to the root task directory
|
Get path relative to the root task directory
|
||||||
|
@ -490,7 +498,7 @@ class Task(models.Model):
|
||||||
raise FileNotFoundError("{} is not a valid asset".format(asset))
|
raise FileNotFoundError("{} is not a valid asset".format(asset))
|
||||||
|
|
||||||
def handle_import(self):
|
def handle_import(self):
|
||||||
self.console_output += gettext("Importing assets...") + "\n"
|
self.console += gettext("Importing assets...") + "\n"
|
||||||
self.save()
|
self.save()
|
||||||
|
|
||||||
zip_path = self.assets_path("all.zip")
|
zip_path = self.assets_path("all.zip")
|
||||||
|
@ -709,7 +717,7 @@ class Task(models.Model):
|
||||||
self.options = list(filter(lambda d: d['name'] != 'rerun-from', self.options))
|
self.options = list(filter(lambda d: d['name'] != 'rerun-from', self.options))
|
||||||
self.upload_progress = 0
|
self.upload_progress = 0
|
||||||
|
|
||||||
self.console_output = ""
|
self.console.reset()
|
||||||
self.processing_time = -1
|
self.processing_time = -1
|
||||||
self.status = None
|
self.status = None
|
||||||
self.last_error = None
|
self.last_error = None
|
||||||
|
@ -740,10 +748,10 @@ class Task(models.Model):
|
||||||
# Need to update status (first time, queued or running?)
|
# Need to update status (first time, queued or running?)
|
||||||
if self.uuid and self.status in [None, status_codes.QUEUED, status_codes.RUNNING]:
|
if self.uuid and self.status in [None, status_codes.QUEUED, status_codes.RUNNING]:
|
||||||
# Update task info from processing node
|
# Update task info from processing node
|
||||||
if not self.console_output:
|
if not self.console.output():
|
||||||
current_lines_count = 0
|
current_lines_count = 0
|
||||||
else:
|
else:
|
||||||
current_lines_count = len(self.console_output.split("\n"))
|
current_lines_count = len(self.console.output().split("\n"))
|
||||||
|
|
||||||
info = self.processing_node.get_task_info(self.uuid, current_lines_count)
|
info = self.processing_node.get_task_info(self.uuid, current_lines_count)
|
||||||
|
|
||||||
|
@ -751,7 +759,7 @@ class Task(models.Model):
|
||||||
self.status = info.status.value
|
self.status = info.status.value
|
||||||
|
|
||||||
if len(info.output) > 0:
|
if len(info.output) > 0:
|
||||||
self.console_output += "\n".join(info.output) + '\n'
|
self.console += "\n".join(info.output) + '\n'
|
||||||
|
|
||||||
# Update running progress
|
# Update running progress
|
||||||
self.running_progress = (info.progress / 100.0) * self.TASK_PROGRESS_LAST_VALUE
|
self.running_progress = (info.progress / 100.0) * self.TASK_PROGRESS_LAST_VALUE
|
||||||
|
@ -891,7 +899,7 @@ class Task(models.Model):
|
||||||
self.update_size()
|
self.update_size()
|
||||||
self.potree_scene = {}
|
self.potree_scene = {}
|
||||||
self.running_progress = 1.0
|
self.running_progress = 1.0
|
||||||
self.console_output += gettext("Done!") + "\n"
|
self.console += gettext("Done!") + "\n"
|
||||||
self.status = status_codes.COMPLETED
|
self.status = status_codes.COMPLETED
|
||||||
self.save()
|
self.save()
|
||||||
|
|
||||||
|
|
|
@ -140,12 +140,12 @@ class TestApi(BootTestCase):
|
||||||
self.assertEqual(res.status_code, status.HTTP_200_OK)
|
self.assertEqual(res.status_code, status.HTTP_200_OK)
|
||||||
self.assertTrue(res.data == "")
|
self.assertTrue(res.data == "")
|
||||||
|
|
||||||
task.console_output = "line1\nline2\nline3"
|
task.console.reset("line1\nline2\nline3")
|
||||||
task.save()
|
task.save()
|
||||||
|
|
||||||
res = client.get('/api/projects/{}/tasks/{}/output/'.format(project.id, task.id))
|
res = client.get('/api/projects/{}/tasks/{}/output/'.format(project.id, task.id))
|
||||||
self.assertEqual(res.status_code, status.HTTP_200_OK)
|
self.assertEqual(res.status_code, status.HTTP_200_OK)
|
||||||
self.assertTrue(res.data == task.console_output)
|
self.assertTrue(res.data == task.console.output())
|
||||||
|
|
||||||
# Console output with line num
|
# Console output with line num
|
||||||
res = client.get('/api/projects/{}/tasks/{}/output/?line=2'.format(project.id, task.id))
|
res = client.get('/api/projects/{}/tasks/{}/output/?line=2'.format(project.id, task.id))
|
||||||
|
@ -155,7 +155,7 @@ class TestApi(BootTestCase):
|
||||||
res = client.get('/api/projects/{}/tasks/{}/output/?line=3'.format(project.id, task.id))
|
res = client.get('/api/projects/{}/tasks/{}/output/?line=3'.format(project.id, task.id))
|
||||||
self.assertTrue(res.data == "")
|
self.assertTrue(res.data == "")
|
||||||
res = client.get('/api/projects/{}/tasks/{}/output/?line=-1'.format(project.id, task.id))
|
res = client.get('/api/projects/{}/tasks/{}/output/?line=-1'.format(project.id, task.id))
|
||||||
self.assertTrue(res.data == task.console_output)
|
self.assertTrue(res.data == task.console.output())
|
||||||
|
|
||||||
# Cannot list task details for a task belonging to a project we don't have access to
|
# Cannot list task details for a task belonging to a project we don't have access to
|
||||||
res = client.get('/api/projects/{}/tasks/{}/'.format(other_project.id, other_task.id))
|
res = client.get('/api/projects/{}/tasks/{}/'.format(other_project.id, other_task.id))
|
||||||
|
|
|
@ -41,7 +41,7 @@ class ImportFolderTaskView(TaskView):
|
||||||
files = platform.import_from_folder(folder_url)
|
files = platform.import_from_folder(folder_url)
|
||||||
|
|
||||||
# Update the task with the new information
|
# Update the task with the new information
|
||||||
task.console_output += "Importing {} images...\n".format(len(files))
|
task.console += "Importing {} images...\n".format(len(files))
|
||||||
task.images_count = len(files)
|
task.images_count = len(files)
|
||||||
task.pending_action = pending_actions.IMPORT
|
task.pending_action = pending_actions.IMPORT
|
||||||
task.save()
|
task.save()
|
||||||
|
|
|
@ -181,7 +181,7 @@ class ImportDatasetTaskView(TaskView):
|
||||||
return Response({'error': 'Empty dataset or folder.'}, status=status.HTTP_400_BAD_REQUEST)
|
return Response({'error': 'Empty dataset or folder.'}, status=status.HTTP_400_BAD_REQUEST)
|
||||||
|
|
||||||
# Update the task with the new information
|
# Update the task with the new information
|
||||||
task.console_output += "Importing {} images...\n".format(len(files))
|
task.console += "Importing {} images...\n".format(len(files))
|
||||||
task.images_count = len(files)
|
task.images_count = len(files)
|
||||||
task.pending_action = pending_actions.IMPORT
|
task.pending_action = pending_actions.IMPORT
|
||||||
task.save()
|
task.save()
|
||||||
|
|
|
@ -22,7 +22,7 @@ def handle_task_completed(sender, task_id, **kwargs):
|
||||||
setting = Setting.objects.first()
|
setting = Setting.objects.first()
|
||||||
notification_app_name = config_data['notification_app_name'] or settings.app_name
|
notification_app_name = config_data['notification_app_name'] or settings.app_name
|
||||||
|
|
||||||
console_output = reverse_output(task.console_output)
|
console_output = reverse_output(task.console.output())
|
||||||
notification.send(
|
notification.send(
|
||||||
f"{notification_app_name} - {task.project.name} Task Completed",
|
f"{notification_app_name} - {task.project.name} Task Completed",
|
||||||
f"{task.project.name}\n{task.name} Completed\nProcessing time:{hours_minutes_secs(task.processing_time)}\n\nConsole Output:{console_output}",
|
f"{task.project.name}\n{task.name} Completed\nProcessing time:{hours_minutes_secs(task.processing_time)}\n\nConsole Output:{console_output}",
|
||||||
|
@ -41,7 +41,7 @@ def handle_task_removed(sender, task_id, **kwargs):
|
||||||
task = Task.objects.get(id=task_id)
|
task = Task.objects.get(id=task_id)
|
||||||
setting = Setting.objects.first()
|
setting = Setting.objects.first()
|
||||||
notification_app_name = config_data['notification_app_name'] or settings.app_name
|
notification_app_name = config_data['notification_app_name'] or settings.app_name
|
||||||
console_output = reverse_output(task.console_output)
|
console_output = reverse_output(task.console.output())
|
||||||
notification.send(
|
notification.send(
|
||||||
f"{notification_app_name} - {task.project.name} Task removed",
|
f"{notification_app_name} - {task.project.name} Task removed",
|
||||||
f"{task.project.name}\n{task.name} was removed\nProcessing time:{hours_minutes_secs(task.processing_time)}\n\nConsole Output:{console_output}",
|
f"{task.project.name}\n{task.name} was removed\nProcessing time:{hours_minutes_secs(task.processing_time)}\n\nConsole Output:{console_output}",
|
||||||
|
@ -60,7 +60,7 @@ def handle_task_failed(sender, task_id, **kwargs):
|
||||||
task = Task.objects.get(id=task_id)
|
task = Task.objects.get(id=task_id)
|
||||||
setting = Setting.objects.first()
|
setting = Setting.objects.first()
|
||||||
notification_app_name = config_data['notification_app_name'] or settings.app_name
|
notification_app_name = config_data['notification_app_name'] or settings.app_name
|
||||||
console_output = reverse_output(task.console_output)
|
console_output = reverse_output(task.console.output())
|
||||||
notification.send(
|
notification.send(
|
||||||
f"{notification_app_name} - {task.project.name} Task Failed",
|
f"{notification_app_name} - {task.project.name} Task Failed",
|
||||||
f"{task.project.name}\n{task.name} Failed with error: {task.last_error}\nProcessing time:{hours_minutes_secs(task.processing_time)}\n\nConsole Output:{console_output}",
|
f"{task.project.name}\n{task.name} Failed with error: {task.last_error}\nProcessing time:{hours_minutes_secs(task.processing_time)}\n\nConsole Output:{console_output}",
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "WebODM",
|
"name": "WebODM",
|
||||||
"version": "2.1.0",
|
"version": "2.1.1",
|
||||||
"description": "User-friendly, extendable application and API for processing aerial imagery.",
|
"description": "User-friendly, extendable application and API for processing aerial imagery.",
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
|
Ładowanie…
Reference in New Issue