From 8263d060c0480ee5ba157ffd25a9c1fe1a4e1727 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 20 May 2019 10:05:56 -0400 Subject: [PATCH] NodeODM 1.5.1 support, simplified task view, progress reporting via API --- app/api/processingnodes.py | 2 +- app/boot.py | 9 +- app/models/task.py | 35 +-- .../app/js/components/BasicTaskView.jsx | 235 ------------------ app/static/app/js/components/TaskListItem.jsx | 61 ++--- .../components/tests/BasicTaskView.test.jsx | 10 - app/static/app/js/css/BasicTaskView.scss | 38 --- app/static/app/js/css/TaskListItem.scss | 25 +- app/templates/app/processing_node.html | 8 +- app/tests/test_api.py | 7 +- nodeodm/migrations/0007_auto_20190520_1258.py | 27 ++ nodeodm/models.py | 16 +- requirements.txt | 2 +- .../includes/reference/_processingnode.md | 7 +- 14 files changed, 108 insertions(+), 374 deletions(-) delete mode 100644 app/static/app/js/components/BasicTaskView.jsx delete mode 100644 app/static/app/js/components/tests/BasicTaskView.test.jsx delete mode 100644 app/static/app/js/css/BasicTaskView.scss create mode 100644 nodeodm/migrations/0007_auto_20190520_1258.py diff --git a/app/api/processingnodes.py b/app/api/processingnodes.py index 0b1ea188..ed37c384 100644 --- a/app/api/processingnodes.py +++ b/app/api/processingnodes.py @@ -34,7 +34,7 @@ class ProcessingNodeFilter(FilterSet): class Meta: model = ProcessingNode - fields = ['has_available_options', 'id', 'hostname', 'port', 'api_version', 'queue_count', 'max_images', 'label', ] + fields = ['has_available_options', 'id', 'hostname', 'port', 'api_version', 'queue_count', 'max_images', 'label', 'engine', 'engine_version', ] class ProcessingNodeViewSet(viewsets.ModelViewSet): """ diff --git a/app/boot.py b/app/boot.py index e2d80352..e8a5cb6f 100644 --- a/app/boot.py +++ b/app/boot.py @@ -97,11 +97,9 @@ def boot(): def add_default_presets(): try: Preset.objects.update_or_create(name='Volume Analysis', system=True, - defaults={'options': [{'name': 'use-opensfm-dense', 'value': True}, - {'name': 'dsm', 'value': True}, + defaults={'options': [{'name': 'dsm', 'value': True}, {'name': 'dem-resolution', 'value': '2'}, - {'name': 'depthmap-resolution', 'value': '1000'}, - {'name': 'opensfm-depthmap-min-patch-sd', 'value': '0'}]}) + {'name': 'depthmap-resolution', 'value': '1000'}]}) Preset.objects.update_or_create(name='3D Model', system=True, defaults={'options': [{'name': 'mesh-octree-depth', 'value': "11"}, {'name': 'use-3dmesh', 'value': True}, @@ -126,7 +124,8 @@ def add_default_presets(): Preset.objects.update_or_create(name='Fast Orthophoto', system=True, defaults={'options': [{'name': 'fast-orthophoto', 'value': True}]}) Preset.objects.update_or_create(name='High Resolution', system=True, - defaults={'options': [{'name': 'dsm', 'value': True}, + defaults={'options': [{'name': 'ignore-gsd', 'value': True}, + {'name': 'dsm', 'value': True}, {'name': 'depthmap-resolution', 'value': '1000'}, {'name': 'dem-resolution', 'value': "2.0"}, {'name': 'orthophoto-resolution', 'value': "2.0"}, diff --git a/app/models/task.py b/app/models/task.py index 8103fc67..877c6576 100644 --- a/app/models/task.py +++ b/app/models/task.py @@ -175,22 +175,7 @@ class Task(models.Model): (pending_actions.IMPORT, 'IMPORT'), ) - # Not an exact science - TASK_OUTPUT_MILESTONES_LAST_VALUE = 0.85 - TASK_OUTPUT_MILESTONES = { - 'Running ODM Load Dataset Cell': 0.01, - 'Running ODM Load Dataset Cell - Finished': 0.05, - 'opensfm/bin/opensfm match_features': 0.10, - 'opensfm/bin/opensfm reconstruct': 0.20, - 'opensfm/bin/opensfm export_visualsfm': 0.30, - 'Running ODM Meshing Cell': 0.50, - 'Running MVS Texturing Cell': 0.55, - 'Running ODM Georeferencing Cell': 0.60, - 'Running ODM DEM Cell': 0.70, - 'Running ODM Orthophoto Cell': 0.75, - 'Running ODM OrthoPhoto Cell - Finished': 0.80, - 'Compressing all.zip:': TASK_OUTPUT_MILESTONES_LAST_VALUE - } + TASK_PROGRESS_LAST_VALUE = 0.85 id = models.UUIDField(primary_key=True, default=uuid_module.uuid4, unique=True, serialize=False, editable=False) @@ -578,22 +563,18 @@ class Task(models.Model): # Need to update status (first time, queued or running?) if self.uuid and self.status in [None, status_codes.QUEUED, status_codes.RUNNING]: # Update task info from processing node - info = self.processing_node.get_task_info(self.uuid) + current_lines_count = len(self.console_output.split("\n")) + + info = self.processing_node.get_task_info(self.uuid, current_lines_count) self.processing_time = info.processing_time self.status = info.status.value - current_lines_count = len(self.console_output.split("\n")) - console_output = self.processing_node.get_task_console_output(self.uuid, current_lines_count) - if len(console_output) > 0: - self.console_output += "\n".join(console_output) + '\n' + if len(info.output) > 0: + self.console_output += "\n".join(info.output) + '\n' # Update running progress - for line in console_output: - for line_match, value in self.TASK_OUTPUT_MILESTONES.items(): - if line_match in line: - self.running_progress = value - break + self.running_progress = (info.progress / 100.0) * self.TASK_PROGRESS_LAST_VALUE if info.last_error != "": self.last_error = info.last_error @@ -625,7 +606,7 @@ class Task(models.Model): if time_has_elapsed or int(progress) == 100: Task.objects.filter(pk=self.id).update(running_progress=( - self.TASK_OUTPUT_MILESTONES_LAST_VALUE + (float(progress) / 100.0) * 0.1)) + self.TASK_PROGRESS_LAST_VALUE + (float(progress) / 100.0) * 0.1)) last_update = time.time() while not extracted: diff --git a/app/static/app/js/components/BasicTaskView.jsx b/app/static/app/js/components/BasicTaskView.jsx deleted file mode 100644 index a5ce0d4f..00000000 --- a/app/static/app/js/components/BasicTaskView.jsx +++ /dev/null @@ -1,235 +0,0 @@ -import React from 'react'; -import PropTypes from 'prop-types'; -import '../css/BasicTaskView.scss'; -import update from 'immutability-helper'; -import PipelineSteps from '../classes/PipelineSteps'; -import StatusCodes from '../classes/StatusCodes'; -import $ from 'jquery'; - -class BasicTaskView extends React.Component { - static defaultProps = {}; - - static propTypes = { - source: PropTypes.oneOfType([ - PropTypes.string.isRequired, - PropTypes.func.isRequired - ]), - taskStatus: PropTypes.number, - onAddLines: PropTypes.func - }; - - constructor(props){ - super(); - - this.state = { - lines: [], - currentRf: 0, - rf: PipelineSteps.get(), - loaded: false - }; - - this.imageUpload = { - action: "imageupload", - label: "Image Resize / Upload", - icon: "fa fa-image" - }; - - // Add a post processing step - this.state.rf.push({ - action: "postprocessing", - label: "Post Processing", - icon: "fa fa-file-archive-o", - beginsWith: "Running ODM OrthoPhoto Cell - Finished", - endsWith: null - }); - - this.addLines = this.addLines.bind(this); - this.setupDynamicSource = this.setupDynamicSource.bind(this); - this.reset = this.reset.bind(this); - this.tearDownDynamicSource = this.tearDownDynamicSource.bind(this); - this.getRfEndStatus = this.getRfEndStatus.bind(this); - this.getRfRunningStatus = this.getRfRunningStatus.bind(this); - this.suffixFor = this.suffixFor.bind(this); - this.updateRfState = this.updateRfState.bind(this); - this.getInitialStatus = this.getInitialStatus.bind(this); - } - - componentDidMount(){ - this.reset(); - } - - setupDynamicSource(){ - const updateFromSource = () => { - let sourceUrl = typeof this.props.source === 'function' ? - this.props.source(this.state.lines.length) : - this.props.source; - - // Fetch - this.sourceRequest = $.get(sourceUrl, text => { - if (text !== ""){ - let lines = text.split("\n"); - this.addLines(lines); - } - }) - .always((_, textStatus) => { - if (textStatus !== "abort" && this.props.refreshInterval !== undefined){ - this.sourceTimeout = setTimeout(updateFromSource, this.props.refreshInterval); - } - if (!this.state.loaded) this.setState({loaded: true}); - }); - }; - - updateFromSource(); - } - - getRfEndStatus(){ - return this.props.taskStatus === StatusCodes.COMPLETED ? - 'completed' : - 'errored'; - } - - getRfRunningStatus(){ - return [StatusCodes.RUNNING, StatusCodes.QUEUED].indexOf(this.props.taskStatus) !== -1 ? - 'running' : - 'errored'; - } - - getInitialStatus(){ - if ([null, StatusCodes.QUEUED, StatusCodes.RUNNING].indexOf(this.props.taskStatus) !== -1){ - return 'queued'; - }else{ - return this.getRfEndStatus(); - } - } - - reset(){ - this.state.rf.forEach(p => { - p.state = this.getInitialStatus(); - }); - - if ([StatusCodes.RUNNING].indexOf(this.props.taskStatus) !== -1){ - this.state.rf[0].state = 'running'; - } - - this.tearDownDynamicSource(); - this.setState({lines: [], currentRf: 0, loaded: false, rf: this.state.rf}); - this.setupDynamicSource(); - } - - tearDownDynamicSource(){ - if (this.sourceTimeout) clearTimeout(this.sourceTimeout); - if (this.sourceRequest) this.sourceRequest.abort(); - } - - componentDidUpdate(prevProps){ - - let taskFailed; - let taskCompleted; - let taskRestarted; - - taskFailed = [StatusCodes.FAILED, StatusCodes.CANCELED].indexOf(this.props.taskStatus) !== -1; - taskCompleted = this.props.taskStatus === StatusCodes.COMPLETED; - - if (prevProps.taskStatus !== this.props.taskStatus){ - taskRestarted = this.props.taskStatus === null; - } - - this.updateRfState(taskFailed, taskCompleted, taskRestarted); - } - - componentWillUnmount(){ - this.tearDownDynamicSource(); - } - - addLines(lines){ - if (!Array.isArray(lines)) lines = [lines]; - - let currentRf = this.state.currentRf; - - const updateRf = (rfIndex, line) => { - const current = this.state.rf[rfIndex]; - if (!current) return; - - if (current.beginsWith && line.endsWith(current.beginsWith)){ - current.state = this.getRfRunningStatus(); - - // Set previous as done - if (this.state.rf[rfIndex - 1]){ - this.state.rf[rfIndex - 1].state = 'completed'; - } - }else if (current.endsWith && line.endsWith(current.endsWith)){ - current.state = this.getRfEndStatus(); - - // Check next - updateRf(rfIndex + 1, line); - - currentRf = rfIndex + 1; - } - }; - - lines.forEach(line => { - updateRf(currentRf, line); - }); - - this.setState(update(this.state, { - lines: {$push: lines} - })); - this.setState({ - rf: this.state.rf, - currentRf - }); - this.updateRfState(); - - if (this.props.onAddLines) this.props.onAddLines(lines); - } - - updateRfState(taskFailed, taskCompleted, taskRestarted){ - // If the task has just failed, update all items that were either running or in queued state - if (taskFailed){ - this.state.rf.forEach(p => { - if (p.state === 'queued' || p.state === 'running') p.state = this.getInitialStatus(); - }); - } - - // If completed, all steps must have completed - if (taskCompleted){ - this.state.rf.forEach(p => p.state = 'completed'); - } - - if (taskRestarted){ - this.state.rf.forEach(p => p.state = 'queued'); - } - } - - suffixFor(state){ - if (state === 'running'){ - return (...); - }else if (state === 'completed'){ - return (); - }else if (state === 'errored'){ - return (); - } - } - - render() { - const { rf, loaded } = this.state; - const imageUploadState = this.props.taskStatus === null - ? 'running' - : 'completed'; - - return (
-
- {this.imageUpload.label} {this.suffixFor(imageUploadState)} -
- {rf.map(p => { - const state = loaded ? p.state : 'queued'; - - return (
- {p.label} {this.suffixFor(state)} -
); - })} -
); - } -} - -export default BasicTaskView; diff --git a/app/static/app/js/components/TaskListItem.jsx b/app/static/app/js/components/TaskListItem.jsx index 24f29aa2..b8955393 100644 --- a/app/static/app/js/components/TaskListItem.jsx +++ b/app/static/app/js/components/TaskListItem.jsx @@ -10,7 +10,6 @@ import HistoryNav from '../classes/HistoryNav'; import PropTypes from 'prop-types'; import TaskPluginActionButtons from './TaskPluginActionButtons'; import PipelineSteps from '../classes/PipelineSteps'; -import BasicTaskView from './BasicTaskView'; import Css from '../classes/Css'; class TaskListItem extends React.Component { @@ -468,33 +467,33 @@ class TaskListItem extends React.Component { expanded = (
-
-
- Created on: {(new Date(task.created_at)).toLocaleString()}
+
+
+
Task Output:
    +
  • +
    + + +
    +
    • +
-
- Processing Node: {task.processing_node_name || "-"} ({task.auto_processing_node ? "auto" : "manual"})
-
- {Array.isArray(task.options) ? -
- Options: {this.optionsToList(task.options)}
+ +
+
+ Created on: {(new Date(task.created_at)).toLocaleString()}
- : ""} - {/* TODO: List of images? */} - - {showOrthophotoMissingWarning ? -
An orthophoto could not be generated. To generate one, make sure GPS information is embedded in the EXIF tags of your images, or use a Ground Control Points (GCP) file.
: ""} - -
-
-
- Simple - | - Console -
- +
+ Processing Node: {task.processing_node_name || "-"} ({task.auto_processing_node ? "auto" : "manual"})
+
+ {Array.isArray(task.options) ? +
+ Options: {this.optionsToList(task.options)}
+
+ : ""} + {/* TODO: List of images? */} +
+ {this.state.view === 'console' ? : ""} - {this.state.view === 'basic' ? - this.basicView = domNode} - refreshInterval={this.shouldRefresh() ? 3000 : undefined} - onAddLines={this.checkForCommonErrors} - taskStatus={task.status} - /> : ""} + {showOrthophotoMissingWarning ? +
An orthophoto could not be generated. To generate one, make sure GPS information is embedded in the EXIF tags of your images, or use a Ground Control Points (GCP) file.
: ""} {showMemoryErrorWarning ?
It looks like your processing node ran out of memory. If you are using docker, make sure that your docker environment has enough RAM allocated. Alternatively, make sure you have enough physical RAM, reduce the number of images, make your images smaller, or reduce the max-concurrency parameter from the task's options. You can also try to use a cloud processing node.
: ""} diff --git a/app/static/app/js/components/tests/BasicTaskView.test.jsx b/app/static/app/js/components/tests/BasicTaskView.test.jsx deleted file mode 100644 index ad459c28..00000000 --- a/app/static/app/js/components/tests/BasicTaskView.test.jsx +++ /dev/null @@ -1,10 +0,0 @@ -import React from 'react'; -import { mount } from 'enzyme'; -import BasicTaskView from '../BasicTaskView'; - -describe('', () => { - it('renders without exploding', () => { - const wrapper = mount(); - expect(wrapper.exists()).toBe(true); - }) -}); \ No newline at end of file diff --git a/app/static/app/js/css/BasicTaskView.scss b/app/static/app/js/css/BasicTaskView.scss deleted file mode 100644 index 8b1eb9a5..00000000 --- a/app/static/app/js/css/BasicTaskView.scss +++ /dev/null @@ -1,38 +0,0 @@ -.basic-task-view{ - margin-bottom: 8px; - opacity: 0.7; - &.loaded{ - opacity: 1; - } - - .processing-step{ - opacity: 0.7; - - &.completed{ - opacity: 1; - /* font-weight: bold; */ - } - &.running{ - opacity: 1; - animation: pulse ease-in-out 1.4833s infinite; - } - &.queued{ - - } - &.errored{ - opacity: 1; - } - } - - @keyframes pulse { - 0% { - opacity: 0.7; - } - 50% { - opacity: 1; - } - 100% { - opacity: 0.7; - } - } -} \ No newline at end of file diff --git a/app/static/app/js/css/TaskListItem.scss b/app/static/app/js/css/TaskListItem.scss index a3064759..e9a85d68 100644 --- a/app/static/app/js/css/TaskListItem.scss +++ b/app/static/app/js/css/TaskListItem.scss @@ -57,6 +57,8 @@ .task-warning{ margin-top: 16px; + margin-bottom: 16px; + i.fa.fa-warning{ color: #ff8000; } @@ -100,25 +102,20 @@ display: inline; } - .switch-view{ + .console-switch{ + margin-right: 10px; margin-bottom: 8px; - font-size: 90%; position: relative; z-index: 99; - - a{ + .console-output-label{ margin-right: 8px; - &.selected{ - font-weight: bold; - } - &:last-child{ - margin-right: 0; - } } - - i{ - margin-left: 8px; + .console-output-label, ul{ + display: inline-block; } } - + + .mb{ + margin-bottom: 12px; + } } \ No newline at end of file diff --git a/app/templates/app/processing_node.html b/app/templates/app/processing_node.html index e8328c76..c74012d8 100644 --- a/app/templates/app/processing_node.html +++ b/app/templates/app/processing_node.html @@ -20,8 +20,12 @@ {{ processing_node.api_version }} - {% trans "ODM Version" %} - {{ processing_node.odm_version }} + {% trans "Engine" %} + {{ processing_node.engine }} + + + {% trans "Engine Version" %} + {{ processing_node.engine_version }} {% trans "Queue Count" %} diff --git a/app/tests/test_api.py b/app/tests/test_api.py index af5cfd4d..fe98c60f 100644 --- a/app/tests/test_api.py +++ b/app/tests/test_api.py @@ -355,8 +355,11 @@ class TestApi(BootTestCase): # Verify max images field self.assertTrue("max_images" in res.data) - # Verify odm version - self.assertTrue("odm_version" in res.data) + # Verify engine version + self.assertTrue("engine_version" in res.data) + + # Verify engine + self.assertTrue("engine" in res.data) # label should be hostname:port (since no label is set) self.assertEqual(res.data['label'], pnode.hostname + ":" + str(pnode.port)) diff --git a/nodeodm/migrations/0007_auto_20190520_1258.py b/nodeodm/migrations/0007_auto_20190520_1258.py new file mode 100644 index 00000000..1b36b748 --- /dev/null +++ b/nodeodm/migrations/0007_auto_20190520_1258.py @@ -0,0 +1,27 @@ +# Generated by Django 2.1.7 on 2019-05-20 12:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('nodeodm', '0006_auto_20190220_1842'), + ] + + operations = [ + migrations.RemoveField( + model_name='processingnode', + name='odm_version', + ), + migrations.AddField( + model_name='processingnode', + name='engine', + field=models.CharField(help_text='Engine used by the node.', max_length=255, null=True), + ), + migrations.AddField( + model_name='processingnode', + name='engine_version', + field=models.CharField(help_text='Engine version used by the node.', max_length=32, null=True), + ), + ] diff --git a/nodeodm/models.py b/nodeodm/models.py index 3d200027..82c65367 100644 --- a/nodeodm/models.py +++ b/nodeodm/models.py @@ -25,8 +25,9 @@ class ProcessingNode(models.Model): available_options = fields.JSONField(default=dict, help_text="Description of the options that can be used for processing") token = models.CharField(max_length=1024, blank=True, default="", help_text="Token to use for authentication. If the node doesn't have authentication, you can leave this field blank.") max_images = models.PositiveIntegerField(help_text="Maximum number of images accepted by this node.", blank=True, null=True) - odm_version = models.CharField(max_length=32, null=True, help_text="ODM version used by the node.") + engine_version = models.CharField(max_length=32, null=True, help_text="Engine version used by the node.") label = models.CharField(max_length=255, default="", blank=True, help_text="Optional label for this node. When set, this label will be shown instead of the hostname:port name.") + engine = models.CharField(max_length=255, null=True, help_text="Engine used by the node.") def __str__(self): if self.label != "": @@ -61,7 +62,8 @@ class ProcessingNode(models.Model): self.api_version = info.version self.queue_count = info.task_queue_count self.max_images = info.max_images - self.odm_version = info.odm_version + self.engine_version = info.engine_version + self.engine = info.engine options = list(map(lambda o: o.__dict__, api_client.options())) self.available_options = options @@ -116,7 +118,7 @@ class ProcessingNode(models.Model): task = api_client.create_task(images, opts, name, progress_callback) return task.uuid - def get_task_info(self, uuid): + def get_task_info(self, uuid, with_output=None): """ Gets information about this task, such as name, creation date, processing time, status, command line options and number of @@ -124,7 +126,13 @@ class ProcessingNode(models.Model): """ api_client = self.api_client() task = api_client.get_task(uuid) - return task.info() + task_info = task.info(with_output) + + # Output support for older clients + if not api_client.version_greater_or_equal_than("1.5.1") and with_output: + task_info.output = self.get_task_console_output(uuid, with_output) + + return task_info def get_task_console_output(self, uuid, line): """ diff --git a/requirements.txt b/requirements.txt index 67c9369e..97bae866 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,7 +37,7 @@ pip-autoremove==0.9.0 psycopg2==2.7.4 psycopg2-binary==2.7.4 PyJWT==1.5.3 -pyodm==1.4.0 +pyodm==1.5.1 pyparsing==2.1.10 pytz==2018.3 rcssmin==1.0.6 diff --git a/slate/source/includes/reference/_processingnode.md b/slate/source/includes/reference/_processingnode.md index 1a0d53a6..94512b14 100644 --- a/slate/source/includes/reference/_processingnode.md +++ b/slate/source/includes/reference/_processingnode.md @@ -9,6 +9,8 @@ "hostname": "nodeodm.masseranolabs.com", "port": 80, "api_version": "1.0.1", + "engine_version": "0.6.0", + "engine": "odm", "last_refreshed": "2017-03-01T21:14:49.918276Z", "queue_count": 0, "max_images": null, @@ -34,7 +36,8 @@ online | bool | Whether the processing node could be reached in the last 5 minut hostname | string | Hostname/IP address port | int | Port api_version | string | Version of NodeODM currently running -odm_version | string | Version of ODM currently being used +engine_version | string | Version of processing engine currently being used +engine | string | Lowercase identifier of processing engine last_refreshed | string | Date and time this node was last seen online. This value is typically refreshed every 15-30 seconds and is used to decide whether a node is offline or not queue_count | int | Number of [Task](#task) items currently being processed/queued on this node. max_images | int | Optional maximum number of images this processing node can accept. null indicates no limit. @@ -89,6 +92,8 @@ port | | "" | Filter by port api_version | | "" | Filter by API version queue_count | | "" | Filter by queue count max_images | | "" | Filter by max images +engine_version | | "" | Filter by engine version +engine | | "" | Filter by engine identifier ordering | | "" | Ordering field to sort results by has_available_options | | "" | Return only processing nodes that have a valid set of processing options (check that the `available_options` field is populated). Either `true` or `false`.