Stage update code, API docs update

docs/index.adoc

@@ -8,7 +8,7 @@ REST API to access ODM
 
 === Version information
 [%hardbreaks]
-_Version_ : 1.5.0
+_Version_ : 1.5.1
 
 
 === Contact information
@@ -592,6 +592,8 @@ Gets information about this task, such as name, creation date, processing time,
 |Type|Name|Description|Schema|Default
 |*Path*|*uuid* +
 _required_|UUID of the task|string|
+|*Query*|*output* +
+_optional_|Optionally retrieve the console output for this task. The parameter specifies the line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. By default no console output is added to the response.|integer|`"0"`
 |*Query*|*token* +
 _optional_|Token required for authentication (when authentication is required).|string|
 |*FormData*|*options* +
@@ -622,10 +624,14 @@ _required_|Number of images|integer
 _required_|Name|string
 |*options* +
 _required_|List of options used to process this task|< <<_task_uuid_info_get_options,options>> > array
+|*output* +
+_optional_|Console output for the task (only if requested via ?output=<linenum>)|< string > array
 |*processingTime* +
 _required_|Milliseconds that have elapsed since the task started being processed.|integer
+|*stages* +
+_required_|Progress information about each stage of the task|< <<_task_uuid_info_get_stages,stages>> > array
 |*status* +
-_required_|Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)|integer
+_required_||<<_task_uuid_info_get_status,status>>
 |*uuid* +
 _required_|UUID|string
 |===
@@ -642,6 +648,28 @@ _required_|Option name (example: "odm_meshing-octreeDepth")|string
 _required_|Value (example: 9)|string
 |===
 
+[[_task_uuid_info_get_stages]]
+*stages*
+
+[options="header", cols=".^3,.^11,.^4"]
+|===
+|Name|Description|Schema
+|*id* +
+_required_|The stage key (same as the value used in the –rerun-from parameter)|string
+|*status_code* +
+_required_|Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)|integer
+|===
+
+[[_task_uuid_info_get_status]]
+*status*
+
+[options="header", cols=".^3,.^11,.^4"]
+|===
+|Name|Description|Schema
+|*code* +
+_required_|Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)|integer
+|===
+
 
 ==== Tags
 

index.js

@@ -303,13 +303,20 @@ let getTaskFromUuid = (req, res, next) => {
  *          description: 'Token required for authentication (when authentication is required).'
  *          required: false
  *          type: string
+ *        -
+ *          name: with_output
+ *          in: query
+ *          description: Optionally retrieve the console output for this task. The parameter specifies the line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. By default no console output is added to the response.
+ *          default: 0
+ *          required: false
+ *          type: integer
  *       responses:
  *        200:
  *         description: Task Information
  *         schema:
  *           title: TaskInfo
  *           type: object
- *           required: [uuid, name, dateCreated, processingTime, status, options, imagesCount]
+ *           required: [uuid, name, dateCreated, processingTime, status, options, imagesCount, progress, stages]
  *           properties:
  *            uuid:
  *              type: string
@@ -324,9 +331,13 @@ let getTaskFromUuid = (req, res, next) => {
  *              type: integer
  *              description: Milliseconds that have elapsed since the task started being processed.
  *            status:
- *              type: integer
- *              description: Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)
- *              enum: [10, 20, 30, 40, 50]
+ *              type: object
+ *              required: [code]
+ *              properties:
+ *                code:
+ *                  type: integer
+ *                  description: Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)
+ *                  enum: [10, 20, 30, 40, 50]
  *            options:
  *              type: array
  *              description: List of options used to process this task
@@ -343,13 +354,37 @@ let getTaskFromUuid = (req, res, next) => {
  *            imagesCount:
  *              type: integer
  *              description: Number of images
+ *            progress:
+ *              type: float
+ *              description: Percentage progress (estimated) of the task
+ *            stages:
+ *              type: array
+ *              description: Progress information about each stage of the task
+ *              items:
+ *                type: object
+ *                required: [id, status, progress]
+ *                properties:
+ *                  id:
+ *                    type: string
+ *                    description: The stage key (same as the value used in the --rerun-from parameter)
+ *                  status:
+ *                    type: integer
+ *                    description: Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)
+ *                    enum: [10, 20, 30, 40, 50]
+ *            output:
+ *              type: array
+ *              description: Console output for the task (only if requested via ?output=<linenum>)
+ *              items:
+ *                type: string
  *        default:
  *          description: Error
  *          schema:
  *            $ref: '#/definitions/Error'
  */
 app.get('/task/:uuid/info', authCheck, getTaskFromUuid, (req, res) => {
-    res.json(req.task.getInfo());
+    const info = req.task.getInfo();
+    if (req.query.with_output !== undefined) info.output = req.task.getOutput(req.query.with_output);
+    res.json(info);
 });
 
 /** @swagger

libs/Task.js

@@ -26,6 +26,7 @@ const glob = require("glob");
 const path = require('path');
 const rmdir = require('rimraf');
 const odmRunner = require('./odmRunner');
+const odmInfo = require('./odmInfo');
 const processRunner = require('./processRunner');
 const archiver = require('archiver');
 const Directories = require('./Directories');
@@ -53,7 +54,7 @@ module.exports = class Task{
         this.webhook = webhook;
         this.skipPostProcessing = skipPostProcessing;
         this.outputs = utils.parseUnsafePathsList(outputs);
-        // TODO: add stages getPipelineStages
+        this.progress = 0;
         
         async.series([
             // Read images info
@@ -82,6 +83,19 @@ module.exports = class Task{
                         cb(null);
                     }
                 });
+            },
+
+            // Populate stage progress
+            cb => {
+                odmInfo.getPipelineStages((err, pstages) => {
+                    if (!err) this.stages = pstages.map(ps => { return {
+                                                            id: ps,
+                                                            status_code: statusCodes.QUEUED,
+                                                            progress: 0
+                                                        }});
+                    else this.stages = [];
+                    cb();
+                }); 
             }
         ], err => {
             done(err, this);
@@ -165,7 +179,48 @@ module.exports = class Task{
     }
 
     updateProgress(globalProgress, stageProgress, stage){
-        // TODO
+        return;
+        globalProgress = min(100, max(0, globalProgress));
+        stageProgress = min(100, max(0, stageProgress));
+        
+        // Progress updates are asynchronous (via UDP)
+        // so things could be out of order. We ignore all progress
+        // updates that are lower than what we might have previously received.
+        if (globalProgress >= this.progress){
+            this.progress = globalProgress;
+            
+            // Process only if we don't know what the stages are
+            if (this.stages.length){
+                let i = 0;
+                for (i = 0; i < this.stages.length; i++){
+                    let s = this.stages[i];
+                    if (s.id === stage){
+                        // Update progress
+                        s.progress = stageProgress;
+                        
+                        // If this one completed, make sure previous stages are also completed
+                        // and that the next stage (if any) is running
+                        if (stageProgress === 100){
+                            s.status = statusCodes.COMPLETED;
+                            for (let j = i; j >= 0; j--){
+                                this.stages[j].status = s.status;
+                                this.stages[j].progress = 100
+                            }
+                            if (i < this.stages.length - 1){
+                                this.stages[i + 1].status = statusCodes.RUNNING;
+                                this.stages[i + 1].progress = 0
+                            }
+                        }else{
+                            s.status = statusCodes.RUNNING;
+                        }
+                        return;
+                    }
+                }
+
+                // This should never happen
+                logger.warn(`Invalid progress update for stage: ${stage}|${globalProgress}|${stageProgress}`);
+            }
+        }
     }
 
     updateProcessingTime(resetTime){
@@ -468,7 +523,9 @@ module.exports = class Task{
             processingTime: this.processingTime,
             status: this.status,
             options: this.options,
-            imagesCount: this.images.length
+            imagesCount: this.images.length,
+            progress: this.progress,
+            stages: this.stages
         };
     }
 

libs/TaskManager.js

@@ -67,9 +67,9 @@ class TaskManager{
 
     onProgressUpdate(uuid, globalProgress, stageProgress, stage){
         const task = this.tasks[uuid];
-        if (task){
 
-        }
+        // Keep 10% for special postprocessing step
+        if (task) task.updateProgress(globalProgress * 0.9, stageProgress, stage);
     }
 
     // Removes old tasks that have either failed, are completed, or

libs/odmInfo.js

@@ -136,14 +136,18 @@ module.exports = {
     },
 
     // Returns a list of stages that tasks go through
-    // In OpenDroneMap this is the same as the --rerun-from domain
+    // In OpenDroneMap this is the same as the --rerun-from domain,
+    // plus a special "postprocess" step
     getPipelineStages: function(done){
         this.getOptions((err, odmOptions) => {
             if (err) done(err);
             else{
                 const rerunFrom = odmOptions.find(opt => opt.name === 'rerun-from' && opt.type === 'enum');
-                if (rerunFrom) done(null, rerunFrom.domain);
-                else done(null, []);
+                if (rerunFrom){
+                    let stages = rerunFrom.domain.filter(d => d !== "");
+                    stages.push("postprocess");
+                    done(null, stages);
+                }else done(null, []);
             }
         });
     },

package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-opendronemap",
-  "version": "1.5.0",
+  "version": "1.5.1",
   "description": "REST API to access ODM",
   "main": "index.js",
   "scripts": {