Stage update code, API docs update

pull/82/head
Piero Toffanin 2019-05-17 19:24:47 -04:00
rodzic c574e2b83d
commit 1a4048402f
7 zmienionych plików z 143 dodań i 19 usunięć

Wyświetl plik

@ -8,7 +8,7 @@ REST API to access ODM
=== Version information
[%hardbreaks]
_Version_ : 1.5.0
_Version_ : 1.5.1
=== Contact information
@ -592,6 +592,8 @@ Gets information about this task, such as name, creation date, processing time,
|Type|Name|Description|Schema|Default
|*Path*|*uuid* +
_required_|UUID of the task|string|
|*Query*|*output* +
_optional_|Optionally retrieve the console output for this task. The parameter specifies the line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. By default no console output is added to the response.|integer|`"0"`
|*Query*|*token* +
_optional_|Token required for authentication (when authentication is required).|string|
|*FormData*|*options* +
@ -622,10 +624,14 @@ _required_|Number of images|integer
_required_|Name|string
|*options* +
_required_|List of options used to process this task|< <<_task_uuid_info_get_options,options>> > array
|*output* +
_optional_|Console output for the task (only if requested via ?output=<linenum>)|< string > array
|*processingTime* +
_required_|Milliseconds that have elapsed since the task started being processed.|integer
|*stages* +
_required_|Progress information about each stage of the task|< <<_task_uuid_info_get_stages,stages>> > array
|*status* +
_required_|Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)|integer
_required_||<<_task_uuid_info_get_status,status>>
|*uuid* +
_required_|UUID|string
|===
@ -642,6 +648,28 @@ _required_|Option name (example: "odm_meshing-octreeDepth")|string
_required_|Value (example: 9)|string
|===
[[_task_uuid_info_get_stages]]
*stages*
[options="header", cols=".^3,.^11,.^4"]
|===
|Name|Description|Schema
|*id* +
_required_|The stage key (same as the value used in the –rerun-from parameter)|string
|*status_code* +
_required_|Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)|integer
|===
[[_task_uuid_info_get_status]]
*status*
[options="header", cols=".^3,.^11,.^4"]
|===
|Name|Description|Schema
|*code* +
_required_|Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)|integer
|===
==== Tags

File diff suppressed because one or more lines are too long

Wyświetl plik

@ -303,13 +303,20 @@ let getTaskFromUuid = (req, res, next) => {
* description: 'Token required for authentication (when authentication is required).'
* required: false
* type: string
* -
* name: with_output
* in: query
* description: Optionally retrieve the console output for this task. The parameter specifies the line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. By default no console output is added to the response.
* default: 0
* required: false
* type: integer
* responses:
* 200:
* description: Task Information
* schema:
* title: TaskInfo
* type: object
* required: [uuid, name, dateCreated, processingTime, status, options, imagesCount]
* required: [uuid, name, dateCreated, processingTime, status, options, imagesCount, progress, stages]
* properties:
* uuid:
* type: string
@ -324,9 +331,13 @@ let getTaskFromUuid = (req, res, next) => {
* type: integer
* description: Milliseconds that have elapsed since the task started being processed.
* status:
* type: integer
* description: Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)
* enum: [10, 20, 30, 40, 50]
* type: object
* required: [code]
* properties:
* code:
* type: integer
* description: Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)
* enum: [10, 20, 30, 40, 50]
* options:
* type: array
* description: List of options used to process this task
@ -343,13 +354,37 @@ let getTaskFromUuid = (req, res, next) => {
* imagesCount:
* type: integer
* description: Number of images
* progress:
* type: float
* description: Percentage progress (estimated) of the task
* stages:
* type: array
* description: Progress information about each stage of the task
* items:
* type: object
* required: [id, status, progress]
* properties:
* id:
* type: string
* description: The stage key (same as the value used in the --rerun-from parameter)
* status:
* type: integer
* description: Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)
* enum: [10, 20, 30, 40, 50]
* output:
* type: array
* description: Console output for the task (only if requested via ?output=<linenum>)
* items:
* type: string
* default:
* description: Error
* schema:
* $ref: '#/definitions/Error'
*/
app.get('/task/:uuid/info', authCheck, getTaskFromUuid, (req, res) => {
res.json(req.task.getInfo());
const info = req.task.getInfo();
if (req.query.with_output !== undefined) info.output = req.task.getOutput(req.query.with_output);
res.json(info);
});
/** @swagger

Wyświetl plik

@ -26,6 +26,7 @@ const glob = require("glob");
const path = require('path');
const rmdir = require('rimraf');
const odmRunner = require('./odmRunner');
const odmInfo = require('./odmInfo');
const processRunner = require('./processRunner');
const archiver = require('archiver');
const Directories = require('./Directories');
@ -53,7 +54,7 @@ module.exports = class Task{
this.webhook = webhook;
this.skipPostProcessing = skipPostProcessing;
this.outputs = utils.parseUnsafePathsList(outputs);
// TODO: add stages getPipelineStages
this.progress = 0;
async.series([
// Read images info
@ -82,6 +83,19 @@ module.exports = class Task{
cb(null);
}
});
},
// Populate stage progress
cb => {
odmInfo.getPipelineStages((err, pstages) => {
if (!err) this.stages = pstages.map(ps => { return {
id: ps,
status_code: statusCodes.QUEUED,
progress: 0
}});
else this.stages = [];
cb();
});
}
], err => {
done(err, this);
@ -165,7 +179,48 @@ module.exports = class Task{
}
updateProgress(globalProgress, stageProgress, stage){
// TODO
return;
globalProgress = min(100, max(0, globalProgress));
stageProgress = min(100, max(0, stageProgress));
// Progress updates are asynchronous (via UDP)
// so things could be out of order. We ignore all progress
// updates that are lower than what we might have previously received.
if (globalProgress >= this.progress){
this.progress = globalProgress;
// Process only if we don't know what the stages are
if (this.stages.length){
let i = 0;
for (i = 0; i < this.stages.length; i++){
let s = this.stages[i];
if (s.id === stage){
// Update progress
s.progress = stageProgress;
// If this one completed, make sure previous stages are also completed
// and that the next stage (if any) is running
if (stageProgress === 100){
s.status = statusCodes.COMPLETED;
for (let j = i; j >= 0; j--){
this.stages[j].status = s.status;
this.stages[j].progress = 100
}
if (i < this.stages.length - 1){
this.stages[i + 1].status = statusCodes.RUNNING;
this.stages[i + 1].progress = 0
}
}else{
s.status = statusCodes.RUNNING;
}
return;
}
}
// This should never happen
logger.warn(`Invalid progress update for stage: ${stage}|${globalProgress}|${stageProgress}`);
}
}
}
updateProcessingTime(resetTime){
@ -468,7 +523,9 @@ module.exports = class Task{
processingTime: this.processingTime,
status: this.status,
options: this.options,
imagesCount: this.images.length
imagesCount: this.images.length,
progress: this.progress,
stages: this.stages
};
}

Wyświetl plik

@ -67,9 +67,9 @@ class TaskManager{
onProgressUpdate(uuid, globalProgress, stageProgress, stage){
const task = this.tasks[uuid];
if (task){
}
// Keep 10% for special postprocessing step
if (task) task.updateProgress(globalProgress * 0.9, stageProgress, stage);
}
// Removes old tasks that have either failed, are completed, or

Wyświetl plik

@ -136,14 +136,18 @@ module.exports = {
},
// Returns a list of stages that tasks go through
// In OpenDroneMap this is the same as the --rerun-from domain
// In OpenDroneMap this is the same as the --rerun-from domain,
// plus a special "postprocess" step
getPipelineStages: function(done){
this.getOptions((err, odmOptions) => {
if (err) done(err);
else{
const rerunFrom = odmOptions.find(opt => opt.name === 'rerun-from' && opt.type === 'enum');
if (rerunFrom) done(null, rerunFrom.domain);
else done(null, []);
if (rerunFrom){
let stages = rerunFrom.domain.filter(d => d !== "");
stages.push("postprocess");
done(null, stages);
}else done(null, []);
}
});
},

Wyświetl plik

@ -1,6 +1,6 @@
{
"name": "node-opendronemap",
"version": "1.5.0",
"version": "1.5.1",
"description": "REST API to access ODM",
"main": "index.js",
"scripts": {