Merge pull request #82 from OpenDroneMap/sprint

Progress receiver class, stages information

docs/index.adoc

@@ -8,7 +8,7 @@ REST API to access ODM
 
 === Version information
 [%hardbreaks]
-_Version_ : 1.5.0
+_Version_ : 1.5.1
 
 
 === Contact information
@@ -180,12 +180,14 @@ _optional_|Token required for authentication (when authentication is required).|
 _optional_|Amount of RAM available in bytes|integer
 |*cpuCores* +
 _optional_|Number of CPU cores (virtual)|integer
+|*engine* +
+_required_|Lowercase identifier of processing engine|string
+|*engineVersion* +
+_required_|Current version of processing engine|string
 |*maxImages* +
-_optional_|Maximum number of images allowed for new tasks or null if there's no limit.|integer
+_required_|Maximum number of images allowed for new tasks or null if there's no limit.|integer
 |*maxParallelTasks* +
 _optional_|Maximum number of tasks that can be processed simultaneously|integer
-|*odmVersion* +
-_optional_|Current version of ODM|string
 |*taskQueueCount* +
 _required_|Number of tasks currently being processed or waiting to be processed|integer
 |*totalMemory* +
@@ -594,6 +596,8 @@ Gets information about this task, such as name, creation date, processing time,
 _required_|UUID of the task|string|
 |*Query*|*token* +
 _optional_|Token required for authentication (when authentication is required).|string|
+|*Query*|*with_output* +
+_optional_|Optionally retrieve the console output for this task. The parameter specifies the line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. By default no console output is added to the response.|integer|`"0"`
 |*FormData*|*options* +
 _optional_|Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, …]. For example, [{"name":"cmvs-maxImages","value":"500"},{"name":"time","value":true}]. For a list of all options, call /options|string|
 |===
@@ -622,10 +626,12 @@ _required_|Number of images|integer
 _required_|Name|string
 |*options* +
 _required_|List of options used to process this task|< <<_task_uuid_info_get_options,options>> > array
+|*output* +
+_optional_|Console output for the task (only if requested via ?output=<linenum>)|< string > array
 |*processingTime* +
 _required_|Milliseconds that have elapsed since the task started being processed.|integer
 |*status* +
-_required_|Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)|integer
+_required_||<<_task_uuid_info_get_status,status>>
 |*uuid* +
 _required_|UUID|string
 |===
@@ -642,6 +648,16 @@ _required_|Option name (example: "odm_meshing-octreeDepth")|string
 _required_|Value (example: 9)|string
 |===
 
+[[_task_uuid_info_get_status]]
+*status*
+
+[options="header", cols=".^3,.^11,.^4"]
+|===
+|Name|Description|Schema
+|*code* +
+_required_|Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)|integer
+|===
+
 
 ==== Tags
 

libs/ProgressReceiver.js

@@ -0,0 +1,59 @@
+/*
+Node-OpenDroneMap Node.js App and REST API to access OpenDroneMap.
+Copyright (C) 2016 Node-OpenDroneMap Contributors
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+"use strict";
+const logger = require('./logger');
+const dgram = require('dgram');
+
+module.exports = class ProgressReceiver{
+    constructor(){
+        const server = dgram.createSocket('udp4');
+        this.callbacks = [];
+
+        server.on('error', (err) => {
+            logger.warn(`Progress listener server error: ${err.stack}`);
+            server.close();
+        });
+
+        server.on('message', (msg) => {
+            const parts = String(msg).split("/");
+            if (parts.length === 4){
+                const cmd = parts[0];
+                if (cmd === 'PGUP'){
+                    let [_, pid, uuid, globalProgress] = parts;
+                    globalProgress = parseFloat(globalProgress);
+
+                    if (!isNaN(globalProgress)){
+                        this.callbacks.forEach(callback => callback(uuid, globalProgress));
+                    }
+                }
+            }
+        });
+
+        server.on('listening', () => {
+            const address = server.address();
+            logger.info(`Listening on ${address.address}:${address.port} UDP for progress updates`);
+        });
+
+        server.bind(6367);
+    }
+
+    addListener(callback){
+        this.callbacks.push(callback);
+    }
+};
+

libs/Task.js

@@ -26,6 +26,7 @@ const glob = require("glob");
 const path = require('path');
 const rmdir = require('rimraf');
 const odmRunner = require('./odmRunner');
+const odmInfo = require('./odmInfo');
 const processRunner = require('./processRunner');
 const archiver = require('archiver');
 const Directories = require('./Directories');
@@ -53,7 +54,8 @@ module.exports = class Task{
         this.webhook = webhook;
         this.skipPostProcessing = skipPostProcessing;
         this.outputs = utils.parseUnsafePathsList(outputs);
-
+        this.progress = 0;
+        
         async.series([
             // Read images info
             cb => {
@@ -163,6 +165,17 @@ module.exports = class Task{
         }
     }
 
+    updateProgress(globalProgress){
+        globalProgress = Math.min(100, Math.max(0, globalProgress));
+        
+        // Progress updates are asynchronous (via UDP)
+        // so things could be out of order. We ignore all progress
+        // updates that are lower than what we might have previously received.
+        if (globalProgress >= this.progress){
+            this.progress = globalProgress;
+        }
+    }
+
     updateProcessingTime(resetTime){
         this.processingTime = resetTime ?
                                 -1		:
@@ -224,6 +237,7 @@ module.exports = class Task{
     // This will spawn a new process.
     start(done){
         const finished = err => {
+            this.updateProgress(100);
             this.stopTrackingProcessingTime();
             done(err);
         };
@@ -239,6 +253,7 @@ module.exports = class Task{
                         });
 
                     archive.on('finish', () => {
+                        this.updateProgress(97);
                         // TODO: is this being fired twice?
                         done();
                     });
@@ -310,8 +325,10 @@ module.exports = class Task{
                         }, (err, code, signal) => {
                             if (err) done(err);
                             else{
-                                if (code === 0) done();
-                                else done(new Error(`Process exited with code ${code}`));
+                                if (code === 0){
+                                    this.updateProgress(93);
+                                    done();
+                                }else done(new Error(`Process exited with code ${code}`));
                             }
                         }, output => {
                             this.output.push(output);
@@ -463,7 +480,8 @@ module.exports = class Task{
             processingTime: this.processingTime,
             status: this.status,
             options: this.options,
-            imagesCount: this.images.length
+            imagesCount: this.images.length,
+            progress: this.progress
         };
     }
 

libs/TaskManager.js

@@ -27,6 +27,7 @@ const statusCodes = require('./statusCodes');
 const async = require('async');
 const schedule = require('node-schedule');
 const Directories = require('./Directories');
+const ProgressReceiver = require('./ProgressReceiver');
 
 const TASKS_DUMP_FILE = path.join(Directories.data, "tasks.json");
 const CLEANUP_TASKS_IF_OLDER_THAN = 1000 * 60 * config.cleanupTasksAfter; // minutes
@@ -38,6 +39,9 @@ class TaskManager{
     constructor(done){
         this.tasks = {};
         this.runningQueue = [];
+        
+        const progressReceiver = new ProgressReceiver();
+        progressReceiver.addListener(this.onProgressUpdate.bind(this));
 
         async.series([
             cb => this.restoreTaskListFromDump(cb),
@@ -61,6 +65,13 @@ class TaskManager{
         ], done);
     }
 
+    onProgressUpdate(uuid, globalProgress){
+        const task = this.tasks[uuid];
+
+        // Keep 10% for special postprocessing step
+        if (task) task.updateProgress(globalProgress * 0.9);
+    }
+
     // Removes old tasks that have either failed, are completed, or
     // have been canceled.
     removeOldTasks(done){

libs/odmInfo.js

@@ -58,7 +58,7 @@ module.exports = {
                     if (["-h", "--project-path", "--cmvs-maxImages", "--time",
                         "--zip-results", "--pmvs-num-cores",
                         "--start-with", "--gcp", "--images",
-                        "--rerun-all", "--rerun", "--end-with",
+                        "--rerun-all", "--rerun",
                         "--slam-config", "--video", "--version", "name"].indexOf(option) !== -1) continue;
 
                     let values = json[option];

package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-opendronemap",
-  "version": "1.5.0",
+  "version": "1.5.1",
   "description": "REST API to access ODM",
   "main": "index.js",
   "scripts": {

public/index.html

@@ -157,6 +157,10 @@
 
                             <span data-bind="css: 'statusIcon glyphicon ' + icon()"></span>
 
+                            <div data-bind="visible: info().status && info().status.code === 20" class="progress" style="margin-top: 12px;">
+                                <div class="progress-bar progress-bar-info" role="progressbar" data-bind="text: parseInt(info().progress) + '%', style: {width: info().progress + '%'}"></div>
+                            </div>
+
                             <div class="actionButtons">
                                 <button data-bind="click: cancel, visible: showCancel()" type="button" class="btn btn-primary btn-sm">
                                     <span class="glyphicon glyphicon-remove-circle"></span> Cancel