mirror
/
OpenDroneMap-WebODM
kopia lustrzana https://github.com/OpenDroneMap/WebODM
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

NodeODM 1.5.1 support, simplified task view, progress reporting via API

pull/663/head
Piero Toffanin 2019-05-20 10:05:56 -04:00
rodzic 9440fe2d44
commit 8263d060c0
14 zmienionych plików z 108 dodań i 374 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

2
app/api/processingnodes.py
Wyświetl plik

@ -34,7 +34,7 @@ class ProcessingNodeFilter(FilterSet):
class Meta:
model = ProcessingNode
fields = ['has_available_options', 'id', 'hostname', 'port', 'api_version', 'queue_count', 'max_images', 'label', ]
fields = ['has_available_options', 'id', 'hostname', 'port', 'api_version', 'queue_count', 'max_images', 'label', 'engine', 'engine_version', ]
class ProcessingNodeViewSet(viewsets.ModelViewSet):
"""

9
app/boot.py
Wyświetl plik

@ -97,11 +97,9 @@ def boot():
def add_default_presets():
try:
Preset.objects.update_or_create(name='Volume Analysis', system=True,
defaults={'options': [{'name': 'use-opensfm-dense', 'value': True},
{'name': 'dsm', 'value': True},
defaults={'options': [{'name': 'dsm', 'value': True},
{'name': 'dem-resolution', 'value': '2'},
{'name': 'depthmap-resolution', 'value': '1000'},
{'name': 'opensfm-depthmap-min-patch-sd', 'value': '0'}]})
{'name': 'depthmap-resolution', 'value': '1000'}]})
Preset.objects.update_or_create(name='3D Model', system=True,
defaults={'options': [{'name': 'mesh-octree-depth', 'value': "11"},
{'name': 'use-3dmesh', 'value': True},
@ -126,7 +124,8 @@ def add_default_presets():
Preset.objects.update_or_create(name='Fast Orthophoto', system=True,
defaults={'options': [{'name': 'fast-orthophoto', 'value': True}]})
Preset.objects.update_or_create(name='High Resolution', system=True,
defaults={'options': [{'name': 'dsm', 'value': True},
defaults={'options': [{'name': 'ignore-gsd', 'value': True},
{'name': 'dsm', 'value': True},
{'name': 'depthmap-resolution', 'value': '1000'},
{'name': 'dem-resolution', 'value': "2.0"},
{'name': 'orthophoto-resolution', 'value': "2.0"},

35
app/models/task.py
Wyświetl plik

@ -175,22 +175,7 @@ class Task(models.Model):
(pending_actions.IMPORT, 'IMPORT'),
)
# Not an exact science
TASK_OUTPUT_MILESTONES_LAST_VALUE = 0.85
TASK_OUTPUT_MILESTONES = {
'Running ODM Load Dataset Cell': 0.01,
'Running ODM Load Dataset Cell - Finished': 0.05,
'opensfm/bin/opensfm match_features': 0.10,
'opensfm/bin/opensfm reconstruct': 0.20,
'opensfm/bin/opensfm export_visualsfm': 0.30,
'Running ODM Meshing Cell': 0.50,
'Running MVS Texturing Cell': 0.55,
'Running ODM Georeferencing Cell': 0.60,
'Running ODM DEM Cell': 0.70,
'Running ODM Orthophoto Cell': 0.75,
'Running ODM OrthoPhoto Cell - Finished': 0.80,
'Compressing all.zip:': TASK_OUTPUT_MILESTONES_LAST_VALUE
}
TASK_PROGRESS_LAST_VALUE = 0.85
id = models.UUIDField(primary_key=True, default=uuid_module.uuid4, unique=True, serialize=False, editable=False)
@ -578,22 +563,18 @@ class Task(models.Model):
# Need to update status (first time, queued or running?)
if self.uuid and self.status in [None, status_codes.QUEUED, status_codes.RUNNING]:
# Update task info from processing node
info = self.processing_node.get_task_info(self.uuid)
current_lines_count = len(self.console_output.split("\n"))
info = self.processing_node.get_task_info(self.uuid, current_lines_count)
self.processing_time = info.processing_time
self.status = info.status.value
current_lines_count = len(self.console_output.split("\n"))
console_output = self.processing_node.get_task_console_output(self.uuid, current_lines_count)
if len(console_output) > 0:
self.console_output += "\n".join(console_output) + '\n'
if len(info.output) > 0:
self.console_output += "\n".join(info.output) + '\n'
# Update running progress
for line in console_output:
for line_match, value in self.TASK_OUTPUT_MILESTONES.items():
if line_match in line:
self.running_progress = value
break
self.running_progress = (info.progress / 100.0) * self.TASK_PROGRESS_LAST_VALUE
if info.last_error != "":
self.last_error = info.last_error
@ -625,7 +606,7 @@ class Task(models.Model):
if time_has_elapsed or int(progress) == 100:
Task.objects.filter(pk=self.id).update(running_progress=(
self.TASK_OUTPUT_MILESTONES_LAST_VALUE + (float(progress) / 100.0) * 0.1))
self.TASK_PROGRESS_LAST_VALUE + (float(progress) / 100.0) * 0.1))
last_update = time.time()
while not extracted:

235
app/static/app/js/components/BasicTaskView.jsx
Wyświetl plik

@ -1,235 +0,0 @@
import React from 'react';
import PropTypes from 'prop-types';
import '../css/BasicTaskView.scss';
import update from 'immutability-helper';
import PipelineSteps from '../classes/PipelineSteps';
import StatusCodes from '../classes/StatusCodes';
import $ from 'jquery';
class BasicTaskView extends React.Component {
static defaultProps = {};
static propTypes = {
source: PropTypes.oneOfType([
PropTypes.string.isRequired,
PropTypes.func.isRequired
]),
taskStatus: PropTypes.number,
onAddLines: PropTypes.func
};
constructor(props){
super();
this.state = {
lines: [],
currentRf: 0,
rf: PipelineSteps.get(),
loaded: false
};
this.imageUpload = {
action: "imageupload",
label: "Image Resize / Upload",
icon: "fa fa-image"
};
// Add a post processing step
this.state.rf.push({
action: "postprocessing",
label: "Post Processing",
icon: "fa fa-file-archive-o",
beginsWith: "Running ODM OrthoPhoto Cell - Finished",
endsWith: null
});
this.addLines = this.addLines.bind(this);
this.setupDynamicSource = this.setupDynamicSource.bind(this);
this.reset = this.reset.bind(this);
this.tearDownDynamicSource = this.tearDownDynamicSource.bind(this);
this.getRfEndStatus = this.getRfEndStatus.bind(this);
this.getRfRunningStatus = this.getRfRunningStatus.bind(this);
this.suffixFor = this.suffixFor.bind(this);
this.updateRfState = this.updateRfState.bind(this);
this.getInitialStatus = this.getInitialStatus.bind(this);
}
componentDidMount(){
this.reset();
}
setupDynamicSource(){
const updateFromSource = () => {
let sourceUrl = typeof this.props.source === 'function' ?
this.props.source(this.state.lines.length) :
this.props.source;
// Fetch
this.sourceRequest = $.get(sourceUrl, text => {
if (text !== ""){
let lines = text.split("\n");
this.addLines(lines);
}
})
.always((_, textStatus) => {
if (textStatus !== "abort" && this.props.refreshInterval !== undefined){
this.sourceTimeout = setTimeout(updateFromSource, this.props.refreshInterval);
}
if (!this.state.loaded) this.setState({loaded: true});
});
};
updateFromSource();
}
getRfEndStatus(){
return this.props.taskStatus === StatusCodes.COMPLETED ?
'completed' :
'errored';
}
getRfRunningStatus(){
return [StatusCodes.RUNNING, StatusCodes.QUEUED].indexOf(this.props.taskStatus) !== -1 ?
'running' :
'errored';
}
getInitialStatus(){
if ([null, StatusCodes.QUEUED, StatusCodes.RUNNING].indexOf(this.props.taskStatus) !== -1){
return 'queued';
}else{
return this.getRfEndStatus();
}
}
reset(){
this.state.rf.forEach(p => {
p.state = this.getInitialStatus();
});
if ([StatusCodes.RUNNING].indexOf(this.props.taskStatus) !== -1){
this.state.rf[0].state = 'running';
}
this.tearDownDynamicSource();
this.setState({lines: [], currentRf: 0, loaded: false, rf: this.state.rf});
this.setupDynamicSource();
}
tearDownDynamicSource(){
if (this.sourceTimeout) clearTimeout(this.sourceTimeout);
if (this.sourceRequest) this.sourceRequest.abort();
}
componentDidUpdate(prevProps){
let taskFailed;
let taskCompleted;
let taskRestarted;
taskFailed = [StatusCodes.FAILED, StatusCodes.CANCELED].indexOf(this.props.taskStatus) !== -1;
taskCompleted = this.props.taskStatus === StatusCodes.COMPLETED;
if (prevProps.taskStatus !== this.props.taskStatus){
taskRestarted = this.props.taskStatus === null;
}
this.updateRfState(taskFailed, taskCompleted, taskRestarted);
}
componentWillUnmount(){
this.tearDownDynamicSource();
}
addLines(lines){
if (!Array.isArray(lines)) lines = [lines];
let currentRf = this.state.currentRf;
const updateRf = (rfIndex, line) => {
const current = this.state.rf[rfIndex];
if (!current) return;
if (current.beginsWith && line.endsWith(current.beginsWith)){
current.state = this.getRfRunningStatus();
// Set previous as done
if (this.state.rf[rfIndex - 1]){
this.state.rf[rfIndex - 1].state = 'completed';
}
}else if (current.endsWith && line.endsWith(current.endsWith)){
current.state = this.getRfEndStatus();
// Check next
updateRf(rfIndex + 1, line);
currentRf = rfIndex + 1;
}
};
lines.forEach(line => {
updateRf(currentRf, line);
});
this.setState(update(this.state, {
lines: {$push: lines}
}));
this.setState({
rf: this.state.rf,
currentRf
});
this.updateRfState();
if (this.props.onAddLines) this.props.onAddLines(lines);
}
updateRfState(taskFailed, taskCompleted, taskRestarted){
// If the task has just failed, update all items that were either running or in queued state
if (taskFailed){
this.state.rf.forEach(p => {
if (p.state === 'queued' || p.state === 'running') p.state = this.getInitialStatus();
});
}
// If completed, all steps must have completed
if (taskCompleted){
this.state.rf.forEach(p => p.state = 'completed');
}
if (taskRestarted){
this.state.rf.forEach(p => p.state = 'queued');
}
}
suffixFor(state){
if (state === 'running'){
return (<span>...</span>);
}else if (state === 'completed'){
return (<i className="fa fa-check"></i>);
}else if (state === 'errored'){
return (<i className="fa fa-times"></i>);
}
}
render() {
const { rf, loaded } = this.state;
const imageUploadState = this.props.taskStatus === null
? 'running'
: 'completed';
return (<div className={"basic-task-view " + (loaded ? 'loaded' : '')}>
<div className={imageUploadState + " processing-step"}>
<i className={this.imageUpload.icon + " fa-fw"}></i> {this.imageUpload.label} {this.suffixFor(imageUploadState)}
</div>
{rf.map(p => {
const state = loaded ? p.state : 'queued';
return (<div key={p.action} className={state + " processing-step"}>
<i className={p.icon + " fa-fw"}></i> {p.label} {this.suffixFor(state)}
</div>);
})}
</div>);
}
}
export default BasicTaskView;

61
app/static/app/js/components/TaskListItem.jsx
Wyświetl plik

@ -10,7 +10,6 @@ import HistoryNav from '../classes/HistoryNav';
import PropTypes from 'prop-types';
import TaskPluginActionButtons from './TaskPluginActionButtons';
import PipelineSteps from '../classes/PipelineSteps';
import BasicTaskView from './BasicTaskView';
import Css from '../classes/Css';
class TaskListItem extends React.Component {
@ -468,33 +467,33 @@ class TaskListItem extends React.Component {
expanded = (
<div className="expanded-panel">
<div className="row">
<div className="col-md-3 no-padding">
<div className="labels">
<strong>Created on: </strong> {(new Date(task.created_at)).toLocaleString()}<br/>
<div className="col-md-12 no-padding">
<div className="console-switch text-right pull-right">
<div className="console-output-label">Task Output: </div><ul className="list-inline">
<li>
<div className="btn-group btn-toggle">
<button onClick={this.setView("console")} className={"btn btn-xs " + (this.state.view === "basic" ? "btn-default" : "btn-primary")}>On</button>
<button onClick={this.setView("basic")} className={"btn btn-xs " + (this.state.view === "console" ? "btn-default" : "btn-primary")}>Off</button>
</div>
</li>
</ul>
</div>
<div className="labels">
<strong>Processing Node: </strong> {task.processing_node_name || "-"} ({task.auto_processing_node ? "auto" : "manual"})<br/>
</div>
{Array.isArray(task.options) ?
<div className="labels">
<strong>Options: </strong> {this.optionsToList(task.options)}<br/>
<div className="mb">
<div className="labels">
<strong>Created on: </strong> {(new Date(task.created_at)).toLocaleString()}<br/>
</div>
: ""}
{/* TODO: List of images? */}
{showOrthophotoMissingWarning ?
<div className="task-warning"><i className="fa fa-warning"></i> <span>An orthophoto could not be generated. To generate one, make sure GPS information is embedded in the EXIF tags of your images, or use a Ground Control Points (GCP) file.</span></div> : ""}
</div>
<div className="col-md-9">
<div className="switch-view text-right pull-right">
<i className="fa fa-list-ul"></i> <a href="javascript:void(0);" onClick={this.setView("basic")}
className={this.state.view === 'basic' ? "selected" : ""}>Simple</a>
|
<i className="fa fa-desktop"></i> <a href="javascript:void(0);" onClick={this.setView("console")}
className={this.state.view === 'console' ? "selected" : ""}>Console</a>
</div>
<div className="labels">
<strong>Processing Node: </strong> {task.processing_node_name || "-"} ({task.auto_processing_node ? "auto" : "manual"})<br/>
</div>
{Array.isArray(task.options) ?
<div className="labels">
<strong>Options: </strong> {this.optionsToList(task.options)}<br/>
</div>
: ""}
{/* TODO: List of images? */}
</div>
{this.state.view === 'console' ?
<Console
className="floatfix"
@ -508,14 +507,8 @@ class TaskListItem extends React.Component {
maximumLines={500}
/> : ""}
{this.state.view === 'basic' ?
<BasicTaskView
source={this.consoleOutputUrl}
ref={domNode => this.basicView = domNode}
refreshInterval={this.shouldRefresh() ? 3000 : undefined}
onAddLines={this.checkForCommonErrors}
taskStatus={task.status}
/> : ""}
{showOrthophotoMissingWarning ?
<div className="task-warning"><i className="fa fa-warning"></i> <span>An orthophoto could not be generated. To generate one, make sure GPS information is embedded in the EXIF tags of your images, or use a Ground Control Points (GCP) file.</span></div> : ""}
{showMemoryErrorWarning ?
<div className="task-warning"><i className="fa fa-support"></i> <span>It looks like your processing node ran out of memory. If you are using docker, make sure that your docker environment has <a href={memoryErrorLink} target="_blank">enough RAM allocated</a>. Alternatively, make sure you have enough physical RAM, reduce the number of images, make your images smaller, or reduce the max-concurrency parameter from the task's <a href="javascript:void(0);" onClick={this.startEditing}>options</a>. You can also try to use a <a href="https://www.opendronemap.org/webodm/lightning/" target="_blank">cloud processing node</a>.</span></div> : ""}

10
app/static/app/js/components/tests/BasicTaskView.test.jsx
Wyświetl plik

@ -1,10 +0,0 @@
import React from 'react';
import { mount } from 'enzyme';
import BasicTaskView from '../BasicTaskView';
describe('<BasicTaskView />', () => {
it('renders without exploding', () => {
const wrapper = mount(<BasicTaskView source="http://localhost/output" taskStatus={40} />);
expect(wrapper.exists()).toBe(true);
})
});

38
app/static/app/js/css/BasicTaskView.scss
Wyświetl plik

@ -1,38 +0,0 @@
.basic-task-view{
margin-bottom: 8px;
opacity: 0.7;
&.loaded{
opacity: 1;
}
.processing-step{
opacity: 0.7;
&.completed{
opacity: 1;
/* font-weight: bold; */
}
&.running{
opacity: 1;
animation: pulse ease-in-out 1.4833s infinite;
}
&.queued{
}
&.errored{
opacity: 1;
}
}
@keyframes pulse {
0% {
opacity: 0.7;
}
50% {
opacity: 1;
}
100% {
opacity: 0.7;
}
}
}

25
app/static/app/js/css/TaskListItem.scss
Wyświetl plik

@ -57,6 +57,8 @@
.task-warning{
margin-top: 16px;
margin-bottom: 16px;
i.fa.fa-warning{
color: #ff8000;
}
@ -100,25 +102,20 @@
display: inline;
}
.switch-view{
.console-switch{
margin-right: 10px;
margin-bottom: 8px;
font-size: 90%;
position: relative;
z-index: 99;
a{
.console-output-label{
margin-right: 8px;
&.selected{
font-weight: bold;
}
&:last-child{
margin-right: 0;
}
}
i{
margin-left: 8px;
.console-output-label, ul{
display: inline-block;
}
}
.mb{
margin-bottom: 12px;
}
}

8
app/templates/app/processing_node.html
Wyświetl plik

@ -20,8 +20,12 @@
<td>{{ processing_node.api_version }}</td>
</tr>
<tr>
<td>{% trans "ODM Version" %}</td>
<td>{{ processing_node.odm_version }}</td>
<td>{% trans "Engine" %}</td>
<td>{{ processing_node.engine }}</td>
</tr>
<tr>
<td>{% trans "Engine Version" %}</td>
<td>{{ processing_node.engine_version }}</td>
</tr>
<tr>
<td>{% trans "Queue Count" %}</td>

7
app/tests/test_api.py
Wyświetl plik

@ -355,8 +355,11 @@ class TestApi(BootTestCase):
# Verify max images field
self.assertTrue("max_images" in res.data)
# Verify odm version
self.assertTrue("odm_version" in res.data)
# Verify engine version
self.assertTrue("engine_version" in res.data)
# Verify engine
self.assertTrue("engine" in res.data)
# label should be hostname:port (since no label is set)
self.assertEqual(res.data['label'], pnode.hostname + ":" + str(pnode.port))

27
nodeodm/migrations/0007_auto_20190520_1258.py 100644
Wyświetl plik

@ -0,0 +1,27 @@
# Generated by Django 2.1.7 on 2019-05-20 12:58
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('nodeodm', '0006_auto_20190220_1842'),
]
operations = [
migrations.RemoveField(
model_name='processingnode',
name='odm_version',
),
migrations.AddField(
model_name='processingnode',
name='engine',
field=models.CharField(help_text='Engine used by the node.', max_length=255, null=True),
),
migrations.AddField(
model_name='processingnode',
name='engine_version',
field=models.CharField(help_text='Engine version used by the node.', max_length=32, null=True),
),
]

16
nodeodm/models.py
Wyświetl plik

@ -25,8 +25,9 @@ class ProcessingNode(models.Model):
available_options = fields.JSONField(default=dict, help_text="Description of the options that can be used for processing")
token = models.CharField(max_length=1024, blank=True, default="", help_text="Token to use for authentication. If the node doesn't have authentication, you can leave this field blank.")
max_images = models.PositiveIntegerField(help_text="Maximum number of images accepted by this node.", blank=True, null=True)
odm_version = models.CharField(max_length=32, null=True, help_text="ODM version used by the node.")
engine_version = models.CharField(max_length=32, null=True, help_text="Engine version used by the node.")
label = models.CharField(max_length=255, default="", blank=True, help_text="Optional label for this node. When set, this label will be shown instead of the hostname:port name.")
engine = models.CharField(max_length=255, null=True, help_text="Engine used by the node.")
def __str__(self):
if self.label != "":
@ -61,7 +62,8 @@ class ProcessingNode(models.Model):
self.api_version = info.version
self.queue_count = info.task_queue_count
self.max_images = info.max_images
self.odm_version = info.odm_version
self.engine_version = info.engine_version
self.engine = info.engine
options = list(map(lambda o: o.__dict__, api_client.options()))
self.available_options = options
@ -116,7 +118,7 @@ class ProcessingNode(models.Model):
task = api_client.create_task(images, opts, name, progress_callback)
return task.uuid
def get_task_info(self, uuid):
def get_task_info(self, uuid, with_output=None):
"""
Gets information about this task, such as name, creation date,
processing time, status, command line options and number of
@ -124,7 +126,13 @@ class ProcessingNode(models.Model):
"""
api_client = self.api_client()
task = api_client.get_task(uuid)
return task.info()
task_info = task.info(with_output)
# Output support for older clients
if not api_client.version_greater_or_equal_than("1.5.1") and with_output:
task_info.output = self.get_task_console_output(uuid, with_output)
return task_info
def get_task_console_output(self, uuid, line):
"""

2
requirements.txt
Wyświetl plik

@ -37,7 +37,7 @@ pip-autoremove==0.9.0
psycopg2==2.7.4
psycopg2-binary==2.7.4
PyJWT==1.5.3
pyodm==1.4.0
pyodm==1.5.1
pyparsing==2.1.10
pytz==2018.3
rcssmin==1.0.6

7
slate/source/includes/reference/_processingnode.md
Wyświetl plik

@ -9,6 +9,8 @@
"hostname": "nodeodm.masseranolabs.com",
"port": 80,
"api_version": "1.0.1",
"engine_version": "0.6.0",
"engine": "odm",
"last_refreshed": "2017-03-01T21:14:49.918276Z",
"queue_count": 0,
"max_images": null,
@ -34,7 +36,8 @@ online | bool | Whether the processing node could be reached in the last 5 minut
hostname | string | Hostname/IP address
port | int | Port
api_version | string | Version of NodeODM currently running
odm_version | string | Version of ODM currently being used
engine_version | string | Version of processing engine currently being used
engine | string | Lowercase identifier of processing engine
last_refreshed | string | Date and time this node was last seen online. This value is typically refreshed every 15-30 seconds and is used to decide whether a node is offline or not
queue_count | int | Number of [Task](#task) items currently being processed/queued on this node.
max_images | int | Optional maximum number of images this processing node can accept. null indicates no limit.
@ -89,6 +92,8 @@ port | | "" | Filter by port
api_version | | "" | Filter by API version
queue_count | | "" | Filter by queue count
max_images | | "" | Filter by max images
engine_version | | "" | Filter by engine version
engine | | "" | Filter by engine identifier
ordering | | "" | Ordering field to sort results by
has_available_options | | "" | Return only processing nodes that have a valid set of processing options (check that the `available_options` field is populated). Either `true` or `false`.