Running progress bars, tweaks

app/migrations/0023_task_running_progress.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.0.3 on 2018-12-07 18:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('app', '0022_auto_20181205_1644'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='task',
+            name='running_progress',
+            field=models.FloatField(blank=True, default=0.0, help_text="Value between 0 and 1 indicating the running progress (estimated) of this task's."),
+        ),
+    ]

app/models/task.py

@@ -156,6 +156,22 @@ class Task(models.Model):
         (pending_actions.RESIZE, 'RESIZE'),
     )
 
+    # Not an exact science
+    TASK_OUTPUT_MILESTONES = {
+        'Running ODM Load Dataset Cell': 0.01,
+        'Running ODM Load Dataset Cell - Finished': 0.05,
+        'opensfm/bin/opensfm match_features': 0.10,
+        'opensfm/bin/opensfm reconstruct': 0.20,
+        'opensfm/bin/opensfm export_visualsfm': 0.30,
+        'Running ODM Meshing Cell': 0.60,
+        'Running MVS Texturing Cell': 0.65,
+        'Running ODM Georeferencing Cell': 0.70,
+        'Running ODM DEM Cell': 0.80,
+        'Running ODM Orthophoto Cell': 0.85,
+        'Running ODM OrthoPhoto Cell - Finished': 0.90,
+        'Compressing all.zip:': 0.95
+    }
+
     id = models.UUIDField(primary_key=True, default=uuid_module.uuid4, unique=True, serialize=False, editable=False)
 
     uuid = models.CharField(max_length=255, db_index=True, default='', blank=True, help_text="Identifier of the task (as returned by OpenDroneMap's REST API)")
@@ -187,6 +203,9 @@ class Task(models.Model):
     resize_progress = models.FloatField(default=0.0,
                                         help_text="Value between 0 and 1 indicating the resize progress of this task's images.",
                                         blank=True)
+    running_progress = models.FloatField(default=0.0,
+                                        help_text="Value between 0 and 1 indicating the running progress (estimated) of this task's.",
+                                        blank=True)
 
     def __init__(self, *args, **kwargs):
         super(Task, self).__init__(*args, **kwargs)
@@ -429,12 +448,14 @@ class Task(models.Model):
 
                             # We also remove the "rerun-from" parameter if it's set
                             self.options = list(filter(lambda d: d['name'] != 'rerun-from', self.options))
+                            self.upload_progress = 0
 
                         self.console_output = ""
                         self.processing_time = -1
                         self.status = None
                         self.last_error = None
                         self.pending_action = None
+                        self.running_progress = 0
                         self.save()
                     else:
                         raise ProcessingError("Cannot restart a task that has no processing node")
@@ -468,7 +489,14 @@ class Task(models.Model):
                     current_lines_count = len(self.console_output.split("\n"))
                     console_output = self.processing_node.get_task_console_output(self.uuid, current_lines_count)
                     if len(console_output) > 0:
-                        self.console_output += console_output + '\n'
+                        self.console_output += "\n".join(console_output) + '\n'
+
+                        # Update running progress
+                        for line in console_output:
+                            for line_match, value in self.TASK_OUTPUT_MILESTONES.items():
+                                if line_match in line:
+                                    self.running_progress = value
+                                    break
 
                     if "errorMessage" in info["status"]:
                         self.last_error = info["status"]["errorMessage"]
@@ -527,6 +555,7 @@ class Task(models.Model):
                                     logger.info("Populated extent field with {} for {}".format(raster_path, self))
 
                             self.update_available_assets_field()
+                            self.running_progress = 1.0
                             self.save()
 
                             from app.plugins import signals as plugin_signals

app/static/app/js/classes/Css.js

@@ -0,0 +1,16 @@
+const values = {};
+export default {
+    getValue: function(className, property, element = 'div'){
+        const k = className + '|' + property;
+        if (values[k]) return values[k];
+        else{
+            let d = document.createElement(element);
+            d.style.display = "none";
+            d.className = className;
+            document.body.appendChild(d);
+            values[k] = getComputedStyle(d)[property];
+            document.body.removeChild(d);
+            return values[k];
+        }
+    }
+}

app/static/app/js/components/BasicTaskView.jsx

@@ -112,7 +112,7 @@ class BasicTaskView extends React.Component {
         }
 
         this.tearDownDynamicSource();
-        this.setState({lines: [], currentRf: 0, loaded: false});
+        this.setState({lines: [], currentRf: 0, loaded: false, rf: this.state.rf});
         this.setupDynamicSource();
     }
 
@@ -122,9 +122,18 @@ class BasicTaskView extends React.Component {
     }
 
     componentDidUpdate(prevProps){
-        let taskFailed = [StatusCodes.RUNNING, StatusCodes.QUEUED].indexOf(prevProps.taskStatus) !== -1 && 
+        
-                         [StatusCodes.FAILED, StatusCodes.CANCELED].indexOf(this.props.taskStatus) !== -1;
+        let taskFailed;
-        this.updateRfState(taskFailed);
+        let taskCompleted;
+        let taskRestarted;
+
+        if (prevProps.taskStatus !== this.props.taskStatus){
+            taskFailed = [StatusCodes.FAILED, StatusCodes.CANCELED].indexOf(this.props.taskStatus) !== -1;
+            taskCompleted = this.props.taskStatus === StatusCodes.COMPLETED;
+            taskRestarted = this.props.taskStatus === null;
+        }
+        
+        this.updateRfState(taskFailed, taskCompleted, taskRestarted);
     }
 
     componentWillUnmount(){
@@ -173,7 +182,7 @@ class BasicTaskView extends React.Component {
         if (this.props.onAddLines) this.props.onAddLines(lines);
     }
 
-    updateRfState(taskFailed){
+    updateRfState(taskFailed, taskCompleted, taskRestarted){
         // If the task has just failed, update all items that were either running or in queued state
         if (taskFailed){
             this.state.rf.forEach(p => {
@@ -181,9 +190,14 @@ class BasicTaskView extends React.Component {
             });
         }
 
-        // The last is always dependent on the task status
+        // If completed, all steps must have completed
-        this.state.rf[this.state.rf.length - 1].state = this.getInitialStatus();
+        if (taskCompleted){
+            this.state.rf.forEach(p => p.state = 'completed');
+        }
 
+        if (taskRestarted){
+            this.state.rf.forEach(p => p.state = 'queued');
+        }
     }
 
     suffixFor(state){

app/static/app/js/components/TaskListItem.jsx

@@ -11,6 +11,7 @@ import PropTypes from 'prop-types';
 import TaskPluginActionButtons from './TaskPluginActionButtons';
 import PipelineSteps from '../classes/PipelineSteps';
 import BasicTaskView from './BasicTaskView';
+import Css from '../classes/Css';
 
 class TaskListItem extends React.Component {
   static propTypes = {
@@ -49,6 +50,10 @@ class TaskListItem extends React.Component {
     this.checkForCommonErrors = this.checkForCommonErrors.bind(this);
     this.handleEditTaskSave = this.handleEditTaskSave.bind(this);
     this.setView = this.setView.bind(this);
+
+    // Retrieve CSS values for status bar colors
+    this.backgroundSuccessColor = Css.getValue('theme-background-success', 'backgroundColor');
+    this.backgroundFailedColor = Css.getValue('theme-background-failed', 'backgroundColor');
   }
 
   shouldRefresh(){
@@ -351,7 +356,7 @@ class TaskListItem extends React.Component {
     const name = task.name !== null ? task.name : `Task #${task.id}`;
 
     let status = statusCodes.description(task.status);
-    if (status === "") status = "Uploading images";
+    if (status === "") status = "Uploading images to processing node";
 
     if (!task.processing_node) status = "Waiting for a node...";
     if (task.pending_action !== null) status = pendingActions.description(task.pending_action);
@@ -464,10 +469,6 @@ class TaskListItem extends React.Component {
               <div className="labels">
                   <strong>Processing Node: </strong> {task.processing_node_name || "-"} ({task.auto_processing_node ? "auto" : "manual"})<br/>
               </div>
-              {status ? <div className="labels">
-                  <strong>Status: </strong> {status}<br/>
-                </div>
-              : ""}
               {Array.isArray(task.options) ?
                  <div className="labels">
                   <strong>Options: </strong> {this.optionsToList(task.options)}<br/>
@@ -482,7 +483,7 @@ class TaskListItem extends React.Component {
             <div className="col-md-9">
               <div className="switch-view text-right pull-right">
                     <i className="fa fa-list-ul"></i> <a href="javascript:void(0);" onClick={this.setView("basic")}
-                            className={this.state.view === 'basic' ? "selected" : ""}>Basic</a>
+                            className={this.state.view === 'basic' ? "selected" : ""}>Simple</a>
                     | 
                     <i className="fa fa-desktop"></i> <a href="javascript:void(0);" onClick={this.setView("console")}
                             className={this.state.view === 'console' ? "selected" : ""}>Console</a>
@@ -551,8 +552,15 @@ class TaskListItem extends React.Component {
       }
     }
 
-    const getStatusLabel = (text, classes = "") => {
+    // @param type {String} one of: ['neutral', 'done', 'error']
-      return (<div className={"status-label " + classes} title={text}>{text}</div>);
+    const getStatusLabel = (text, type = 'neutral', progress = 100) => {
+      let color = 'rgba(255, 255, 255, 0.0)';
+      if (type === 'done') color = this.backgroundSuccessColor;
+      else if (type === 'error') color = this.backgroundFailedColor;
+      return (<div 
+            className={"status-label theme-border-primary " + type} 
+            style={{background: `linear-gradient(90deg, ${color} ${progress}%, rgba(255, 255, 255, 0) ${progress}%)`}}
+            title={text}>{text}</div>);
     }
 
     let statusLabel = "";
@@ -560,13 +568,28 @@ class TaskListItem extends React.Component {
     let showEditLink = false;
 
     if (task.last_error){
-      statusLabel = getStatusLabel(task.last_error, "error");
+      statusLabel = getStatusLabel(task.last_error, 'error');
     }else if (!task.processing_node){
       statusLabel = getStatusLabel("Set a processing node");
       statusIcon = "fa fa-hourglass-3";
       showEditLink = true;
     }else{
-      statusLabel = getStatusLabel(status, task.status == 40 ? "done" : "");
+      let progress = 100;
+      let type = 'done';
+
+      if (task.pending_action === pendingActions.RESIZE){
+          progress = task.resize_progress * 100;
+      }else if (task.status === null){
+          progress = task.upload_progress * 100;
+      }else if (task.status === statusCodes.RUNNING){
+          progress = task.running_progress * 100;
+      }else if (task.status === statusCodes.FAILED){
+          type = 'error';
+      }else if (task.status !== statusCodes.COMPLETED){
+          type = 'neutral';
+      }
+
+      statusLabel = getStatusLabel(status, type, progress);
     }
 
     return (

app/static/app/js/css/TaskListItem.scss

@@ -41,6 +41,8 @@
         padding: 4px;
         width: 100%;
         font-size: 90%;
+        border-style: solid;
+        border-width: 1px;
     }
 
     .clickable:hover{

nodeodm/models.py

@@ -153,7 +153,7 @@ class ProcessingNode(models.Model):
         if isinstance(result, dict) and 'error' in result:
             raise ProcessingError(result['error'])
         elif isinstance(result, list):
-            return "\n".join(result)
+            return result
         else:
             raise ProcessingError("Unknown response for console output: {}".format(result))
 

nodeodm/tests.py

@@ -121,7 +121,7 @@ class TestClientApi(TestCase):
 
         # task_output
         self.assertTrue(isinstance(api.task_output(uuid, 0), list))
-        self.assertTrue(isinstance(online_node.get_task_console_output(uuid, 0), str))
+        self.assertTrue(isinstance(online_node.get_task_console_output(uuid, 0), list))
 
         self.assertRaises(ProcessingError, online_node.get_task_console_output, "wrong-uuid", 0)
 

start.sh

@@ -47,6 +47,9 @@ if [ "$1" = "--setup-devenv" ] || [ "$2" = "--setup-devenv" ]; then
     npm install
     cd /webodm
 
+    echo Setup pip requirements...
+    pip install -r requirements.txt
+
     echo Setup webpack watch...
     webpack --watch &
 fi