Merge pull request #82 from OpenDroneMap/sprint

Progress receiver class, stages information

docs/index.adoc

@@ -8,7 +8,7 @@ REST API to access ODM
 
 === Version information
 [%hardbreaks]
-_Version_ : 1.5.0
+_Version_ : 1.5.1
 
 
 === Contact information
@@ -180,12 +180,14 @@ _optional_|Token required for authentication (when authentication is required).|
 _optional_|Amount of RAM available in bytes|integer
 |*cpuCores* +
 _optional_|Number of CPU cores (virtual)|integer
+|*engine* +
+_required_|Lowercase identifier of processing engine|string
+|*engineVersion* +
+_required_|Current version of processing engine|string
 |*maxImages* +
-_optional_|Maximum number of images allowed for new tasks or null if there's no limit.|integer
+_required_|Maximum number of images allowed for new tasks or null if there's no limit.|integer
 |*maxParallelTasks* +
 _optional_|Maximum number of tasks that can be processed simultaneously|integer
-|*odmVersion* +
-_optional_|Current version of ODM|string
 |*taskQueueCount* +
 _required_|Number of tasks currently being processed or waiting to be processed|integer
 |*totalMemory* +
@@ -594,6 +596,8 @@ Gets information about this task, such as name, creation date, processing time,
 _required_|UUID of the task|string|
 |*Query*|*token* +
 _optional_|Token required for authentication (when authentication is required).|string|
+|*Query*|*with_output* +
+_optional_|Optionally retrieve the console output for this task. The parameter specifies the line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. By default no console output is added to the response.|integer|`"0"`
 |*FormData*|*options* +
 _optional_|Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, …]. For example, [{"name":"cmvs-maxImages","value":"500"},{"name":"time","value":true}]. For a list of all options, call /options|string|
 |===
@@ -622,10 +626,12 @@ _required_|Number of images|integer
 _required_|Name|string
 |*options* +
 _required_|List of options used to process this task|< <<_task_uuid_info_get_options,options>> > array
+|*output* +
+_optional_|Console output for the task (only if requested via ?output=<linenum>)|< string > array
 |*processingTime* +
 _required_|Milliseconds that have elapsed since the task started being processed.|integer
 |*status* +
-_required_|Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)|integer
+_required_||<<_task_uuid_info_get_status,status>>
 |*uuid* +
 _required_|UUID|string
 |===
@@ -642,6 +648,16 @@ _required_|Option name (example: "odm_meshing-octreeDepth")|string
 _required_|Value (example: 9)|string
 |===
 
+[[_task_uuid_info_get_status]]
+*status*
+
+[options="header", cols=".^3,.^11,.^4"]
+|===
+|Name|Description|Schema
+|*code* +
+_required_|Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)|integer
+|===
+
 
 ==== Tags
 

index.js

@@ -303,13 +303,20 @@ let getTaskFromUuid = (req, res, next) => {
  *          description: 'Token required for authentication (when authentication is required).'
  *          required: false
  *          type: string
+ *        -
+ *          name: with_output
+ *          in: query
+ *          description: Optionally retrieve the console output for this task. The parameter specifies the line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. By default no console output is added to the response.
+ *          default: 0
+ *          required: false
+ *          type: integer
  *       responses:
  *        200:
  *         description: Task Information
  *         schema:
  *           title: TaskInfo
  *           type: object
- *           required: [uuid, name, dateCreated, processingTime, status, options, imagesCount]
+ *           required: [uuid, name, dateCreated, processingTime, status, options, imagesCount, progress]
  *           properties:
  *            uuid:
  *              type: string
@@ -324,9 +331,13 @@ let getTaskFromUuid = (req, res, next) => {
  *              type: integer
  *              description: Milliseconds that have elapsed since the task started being processed.
  *            status:
- *              type: integer
- *              description: Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)
- *              enum: [10, 20, 30, 40, 50]
+ *              type: object
+ *              required: [code]
+ *              properties:
+ *                code:
+ *                  type: integer
+ *                  description: Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)
+ *                  enum: [10, 20, 30, 40, 50]
  *            options:
  *              type: array
  *              description: List of options used to process this task
@@ -343,13 +354,23 @@ let getTaskFromUuid = (req, res, next) => {
  *            imagesCount:
  *              type: integer
  *              description: Number of images
+ *            progress:
+ *              type: float
+ *              description: Percentage progress (estimated) of the task
+ *            output:
+ *              type: array
+ *              description: Console output for the task (only if requested via ?output=<linenum>)
+ *              items:
+ *                type: string
  *        default:
  *          description: Error
  *          schema:
  *            $ref: '#/definitions/Error'
  */
 app.get('/task/:uuid/info', authCheck, getTaskFromUuid, (req, res) => {
-    res.json(req.task.getInfo());
+    const info = req.task.getInfo();
+    if (req.query.with_output !== undefined) info.output = req.task.getOutput(req.query.with_output);
+    res.json(info);
 });
 
 /** @swagger
@@ -648,7 +669,7 @@ app.get('/options', authCheck, (req, res) => {
  *         description: Info
  *         schema:
  *           type: object
- *           required: [version, taskQueueCount]
+ *           required: [version, taskQueueCount, maxImages, engineVersion, engine]
  *           properties:
  *             version:
  *               type: string
@@ -671,17 +692,20 @@ app.get('/options', authCheck, (req, res) => {
  *             maxParallelTasks:
  *               type: integer
  *               description: Maximum number of tasks that can be processed simultaneously
- *             odmVersion:
+ *             engineVersion:
  *               type: string
- *               description: Current version of ODM
+ *               description: Current version of processing engine
+ *             engine:
+ *               type: string
+ *               description: Lowercase identifier of processing engine
  */
 app.get('/info', authCheck, (req, res) => {
     async.parallel({
         cpu: cb => si.cpu(data => cb(null, data)),
         mem: cb => si.mem(data => cb(null, data)),
-        odmVersion: odmInfo.getVersion
+        engineVersion: odmInfo.getVersion
     }, (_, data) => {
-        const { cpu, mem, odmVersion } = data;
+        const { cpu, mem, engineVersion } = data;
 
         // For testing
         if (req.query._debugUnauthorized){
@@ -698,7 +722,8 @@ app.get('/info', authCheck, (req, res) => {
             cpuCores: cpu.cores,
             maxImages: config.maxImages,
             maxParallelTasks: config.parallelQueueProcessing,
-            odmVersion: odmVersion
+            engineVersion: engineVersion,
+            engine: 'odm'
         });
     });
 });

libs/ProgressReceiver.js

@@ -0,0 +1,59 @@
+/*
+Node-OpenDroneMap Node.js App and REST API to access OpenDroneMap.
+Copyright (C) 2016 Node-OpenDroneMap Contributors
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+"use strict";
+const logger = require('./logger');
+const dgram = require('dgram');
+
+module.exports = class ProgressReceiver{
+    constructor(){
+        const server = dgram.createSocket('udp4');
+        this.callbacks = [];
+
+        server.on('error', (err) => {
+            logger.warn(`Progress listener server error: ${err.stack}`);
+            server.close();
+        });
+
+        server.on('message', (msg) => {
+            const parts = String(msg).split("/");
+            if (parts.length === 4){
+                const cmd = parts[0];
+                if (cmd === 'PGUP'){
+                    let [_, pid, uuid, globalProgress] = parts;
+                    globalProgress = parseFloat(globalProgress);
+
+                    if (!isNaN(globalProgress)){
+                        this.callbacks.forEach(callback => callback(uuid, globalProgress));
+                    }
+                }
+            }
+        });
+
+        server.on('listening', () => {
+            const address = server.address();
+            logger.info(`Listening on ${address.address}:${address.port} UDP for progress updates`);
+        });
+
+        server.bind(6367);
+    }
+
+    addListener(callback){
+        this.callbacks.push(callback);
+    }
+};
+

libs/Task.js

@@ -26,6 +26,7 @@ const glob = require("glob");
 const path = require('path');
 const rmdir = require('rimraf');
 const odmRunner = require('./odmRunner');
+const odmInfo = require('./odmInfo');
 const processRunner = require('./processRunner');
 const archiver = require('archiver');
 const Directories = require('./Directories');
@@ -53,6 +54,7 @@ module.exports = class Task{
         this.webhook = webhook;
         this.skipPostProcessing = skipPostProcessing;
         this.outputs = utils.parseUnsafePathsList(outputs);
+        this.progress = 0;
         
         async.series([
             // Read images info
@@ -163,6 +165,17 @@ module.exports = class Task{
         }
     }
 
+    updateProgress(globalProgress){
+        globalProgress = Math.min(100, Math.max(0, globalProgress));
+        
+        // Progress updates are asynchronous (via UDP)
+        // so things could be out of order. We ignore all progress
+        // updates that are lower than what we might have previously received.
+        if (globalProgress >= this.progress){
+            this.progress = globalProgress;
+        }
+    }
+
     updateProcessingTime(resetTime){
         this.processingTime = resetTime ?
                                 -1		:
@@ -224,6 +237,7 @@ module.exports = class Task{
     // This will spawn a new process.
     start(done){
         const finished = err => {
+            this.updateProgress(100);
             this.stopTrackingProcessingTime();
             done(err);
         };
@@ -239,6 +253,7 @@ module.exports = class Task{
                         });
 
                     archive.on('finish', () => {
+                        this.updateProgress(97);
                         // TODO: is this being fired twice?
                         done();
                     });
@@ -310,8 +325,10 @@ module.exports = class Task{
                         }, (err, code, signal) => {
                             if (err) done(err);
                             else{
-                                if (code === 0) done();
-                                else done(new Error(`Process exited with code ${code}`));
+                                if (code === 0){
+                                    this.updateProgress(93);
+                                    done();
+                                }else done(new Error(`Process exited with code ${code}`));
                             }
                         }, output => {
                             this.output.push(output);
@@ -463,7 +480,8 @@ module.exports = class Task{
             processingTime: this.processingTime,
             status: this.status,
             options: this.options,
-            imagesCount: this.images.length
+            imagesCount: this.images.length,
+            progress: this.progress
         };
     }
 

libs/TaskManager.js

@@ -27,6 +27,7 @@ const statusCodes = require('./statusCodes');
 const async = require('async');
 const schedule = require('node-schedule');
 const Directories = require('./Directories');
+const ProgressReceiver = require('./ProgressReceiver');
 
 const TASKS_DUMP_FILE = path.join(Directories.data, "tasks.json");
 const CLEANUP_TASKS_IF_OLDER_THAN = 1000 * 60 * config.cleanupTasksAfter; // minutes
@@ -39,6 +40,9 @@ class TaskManager{
         this.tasks = {};
         this.runningQueue = [];
         
+        const progressReceiver = new ProgressReceiver();
+        progressReceiver.addListener(this.onProgressUpdate.bind(this));
+
         async.series([
             cb => this.restoreTaskListFromDump(cb),
             cb => this.removeOldTasks(cb),
@@ -61,6 +65,13 @@ class TaskManager{
         ], done);
     }
 
+    onProgressUpdate(uuid, globalProgress){
+        const task = this.tasks[uuid];
+
+        // Keep 10% for special postprocessing step
+        if (task) task.updateProgress(globalProgress * 0.9);
+    }
+
     // Removes old tasks that have either failed, are completed, or
     // have been canceled.
     removeOldTasks(done){

libs/odmInfo.js

@@ -58,7 +58,7 @@ module.exports = {
                     if (["-h", "--project-path", "--cmvs-maxImages", "--time",
                         "--zip-results", "--pmvs-num-cores",
                         "--start-with", "--gcp", "--images",
-                        "--rerun-all", "--rerun", "--end-with",
+                        "--rerun-all", "--rerun",
                         "--slam-config", "--video", "--version", "name"].indexOf(option) !== -1) continue;
 
                     let values = json[option];

package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-opendronemap",
-  "version": "1.5.0",
+  "version": "1.5.1",
   "description": "REST API to access ODM",
   "main": "index.js",
   "scripts": {

public/index.html

@@ -157,6 +157,10 @@
 
                             <span data-bind="css: 'statusIcon glyphicon ' + icon()"></span>
 
+                            <div data-bind="visible: info().status && info().status.code === 20" class="progress" style="margin-top: 12px;">
+                                <div class="progress-bar progress-bar-info" role="progressbar" data-bind="text: parseInt(info().progress) + '%', style: {width: info().progress + '%'}"></div>
+                            </div>
+
                             <div class="actionButtons">
                                 <button data-bind="click: cancel, visible: showCancel()" type="button" class="btn btn-primary btn-sm">
                                     <span class="glyphicon glyphicon-remove-circle"></span> Cancel