From e87eabe9e3aa5af80af46668fd17d955d89cab06 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 10 Jan 2019 10:07:32 -0500 Subject: [PATCH 1/7] Better help text parsing --- libs/odmInfo.js | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/odmInfo.js b/libs/odmInfo.js index ff76b76..7b339e7 100644 --- a/libs/odmInfo.js +++ b/libs/odmInfo.js @@ -123,6 +123,7 @@ module.exports = { if (domain.indexOf(value) === -1) domain.unshift(value); } + help = help.replace(/^One of: \%\(choices\)s. /, ""); help = help.replace(/\%\(default\)s/g, value); odmOptions.push({ From c7a742307978773f66ebbbc755d9658867ee861f Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 30 Jan 2019 15:56:09 -0500 Subject: [PATCH 2/7] Started refactoring to accomodate chunked upload API --- index.js | 108 +++++++++++++++--------- libs/TaskManager.js | 14 ++- libs/taskNew.js | 201 ++++++++++++++++++++++++++++++++++++++++++++ package.json | 2 +- 4 files changed, 281 insertions(+), 44 deletions(-) create mode 100644 libs/taskNew.js diff --git a/index.js b/index.js index 5813443..f9aedcf 100644 --- a/index.js +++ b/index.js @@ -30,7 +30,6 @@ const rmdir = require('rimraf'); const express = require('express'); const app = express(); -const multer = require('multer'); const bodyParser = require('body-parser'); const TaskManager = require('./libs/TaskManager'); @@ -44,7 +43,7 @@ const S3 = require('./libs/S3'); const auth = require('./libs/auth/factory').fromConfig(config); const authCheck = auth.getMiddleware(); -const uuidv4 = require('uuid/v4'); +const taskNew = require('./libs/taskNew'); // zip files let request = require('request'); @@ -61,31 +60,72 @@ let download = function(uri, filename, callback) { app.use(express.static('public')); app.use('/swagger.json', express.static('docs/swagger.json')); -const upload = multer({ - storage: multer.diskStorage({ - destination: (req, file, cb) => { - let dstPath = path.join("tmp", req.id); - fs.exists(dstPath, exists => { - if (!exists) { - fs.mkdir(dstPath, undefined, () => { - cb(null, dstPath); - }); - } else { - cb(null, dstPath); - } - }); - }, - filename: (req, file, cb) => { - cb(null, file.originalname); - } - }) -}); - const urlEncodedBodyParser = bodyParser.urlencoded({extended: false}); let taskManager; let server; +/** @swagger + * /task/new/init: + * post: + * description: Initialize the upload of a new task. If successful, a user can start uploading files via /task/new/upload. The task will not start until /task/new/commit is called. + * tags: [task] + * parameters: + * name: name + * in: formData + * description: An optional name to be associated with the task + * required: false + * type: string + * - + * name: options + * in: formData + * description: 'Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{"name":"cmvs-maxImages","value":"500"},{"name":"time","value":true}]. For a list of all options, call /options' + * required: false + * type: string + * - + * name: skipPostProcessing + * in: formData + * description: 'When set, skips generation of map tiles, derivate assets, point cloud tiles.' + * required: false + * type: boolean + * - + * name: token + * in: query + * description: 'Token required for authentication (when authentication is required).' + * required: false + * type: string + * - + * name: set-uuid + * in: header + * description: 'An optional UUID string that will be used as UUID for this task instead of generating a random one.' + * required: false + * type: string + * responses: + * 200: + * description: Success + * schema: + * type: object + * required: [uuid] + * properties: + * uuid: + * type: string + * description: UUID of the newly created task + * default: + * description: Error + * schema: + * $ref: '#/definitions/Error' + */ +app.post('/task/new/init', authCheck, taskNew.assignUUID, (req, res) => { + +}); + +app.post('/task/new/upload/:uuid', authCheck, (req, res) => { +}); + +app.post('/task/new/commit/:uuid', authCheck, (req, res) => { + +}); + /** @swagger * /task/new: * post: @@ -151,24 +191,7 @@ let server; * schema: * $ref: '#/definitions/Error' */ -app.post('/task/new', authCheck, (req, res, next) => { - // A user can optionally suggest a UUID instead of letting - // nodeODM pick one. - if (req.get('set-uuid')){ - const userUuid = req.get('set-uuid'); - - // Valid UUID and no other task with same UUID? - if (/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(userUuid) && !taskManager.find(userUuid)){ - req.id = userUuid; - next(); - }else{ - res.json({error: `Invalid set-uuid: ${userUuid}`}) - } - }else{ - req.id = uuidv4(); - next(); - } -}, upload.array('images'), (req, res) => { +app.post('/task/new', authCheck, taskNew.assignUUID, taskNew.uploadImages, (req, res) => { // TODO: consider doing the file moving in the background // and return a response more quickly instead of a long timeout. req.setTimeout(1000 * 60 * 20); @@ -863,7 +886,10 @@ let commands = [ cb => odmInfo.initialize(cb), cb => auth.initialize(cb), cb => S3.initialize(cb), - cb => { taskManager = new TaskManager(cb); }, + cb => { + TaskManager.initialize(cb); + taskManager = TaskManager.singleton(); + }, cb => { server = app.listen(config.port, err => { if (!err) logger.info('Server has started on port ' + String(config.port)); diff --git a/libs/TaskManager.js b/libs/TaskManager.js index 9b1ced5..db9622c 100644 --- a/libs/TaskManager.js +++ b/libs/TaskManager.js @@ -31,7 +31,9 @@ const Directories = require('./Directories'); const TASKS_DUMP_FILE = path.join(Directories.data, "tasks.json"); const CLEANUP_TASKS_IF_OLDER_THAN = 1000 * 60 * config.cleanupTasksAfter; // minutes -module.exports = class TaskManager{ +let taskManager; + +class TaskManager{ constructor(done){ this.tasks = {}; this.runningQueue = []; @@ -82,6 +84,7 @@ module.exports = class TaskManager{ // Removes directories that don't have a corresponding // task associated with it (maybe as a cause of an abrupt exit) + // TODO: do not delete /task/new/init directories!!! removeOrphanedDirectories(done){ logger.info("Checking for orphaned directories to be removed..."); @@ -264,4 +267,11 @@ module.exports = class TaskManager{ } return count; } -}; +} + +module.exports = { + singleton: function(){ return taskManager; }, + initialize: function(cb){ + taskManager = new TaskManager(cb); + } +}; \ No newline at end of file diff --git a/libs/taskNew.js b/libs/taskNew.js new file mode 100644 index 0000000..ff8ad33 --- /dev/null +++ b/libs/taskNew.js @@ -0,0 +1,201 @@ +/* +Node-OpenDroneMap Node.js App and REST API to access OpenDroneMap. +Copyright (C) 2016 Node-OpenDroneMap Contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +const multer = require('multer'); +const fs = require('fs'); +const path = require('path'); +const TaskManager = require('./TaskManager'); +const uuidv4 = require('uuid/v4'); + +const upload = multer({ + storage: multer.diskStorage({ + destination: (req, file, cb) => { + let dstPath = path.join("tmp", req.id); + fs.exists(dstPath, exists => { + if (!exists) { + fs.mkdir(dstPath, undefined, () => { + cb(null, dstPath); + }); + } else { + cb(null, dstPath); + } + }); + }, + filename: (req, file, cb) => { + cb(null, file.originalname); + } + }) +}); + +module.exports = { + assignUUID: (req, res, next) => { + // A user can optionally suggest a UUID instead of letting + // nodeODM pick one. + if (req.get('set-uuid')){ + const userUuid = req.get('set-uuid'); + + // Valid UUID and no other task with same UUID? + if (/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(userUuid) && !TaskManager.singleton().find(userUuid)){ + req.id = userUuid; + next(); + }else{ + res.json({error: `Invalid set-uuid: ${userUuid}`}) + } + }else{ + req.id = uuidv4(); + next(); + } + }, + + uploadImages: upload.array("images"), + + handleTaskNew: (res, res) => { + // TODO: consider doing the file moving in the background + // and return a response more quickly instead of a long timeout. + req.setTimeout(1000 * 60 * 20); + + let srcPath = path.join("tmp", req.id); + + // Print error message and cleanup + const die = (error) => { + res.json({error}); + + // Check if tmp/ directory needs to be cleaned + if (fs.stat(srcPath, (err, stats) => { + if (!err && stats.isDirectory()) rmdir(srcPath, () => {}); // ignore errors, don't wait + })); + }; + + if ((!req.files || req.files.length === 0) && !req.body.zipurl) die("Need at least 1 file or a zip file url."); + else if (config.maxImages && req.files && req.files.length > config.maxImages) die(`${req.files.length} images uploaded, but this node can only process up to ${config.maxImages}.`); + + else { + let destPath = path.join(Directories.data, req.id); + let destImagesPath = path.join(destPath, "images"); + let destGpcPath = path.join(destPath, "gpc"); + + async.series([ + cb => { + odmInfo.filterOptions(req.body.options, (err, options) => { + if (err) cb(err); + else { + req.body.options = options; + cb(null); + } + }); + }, + + // Move all uploads to data//images dir (if any) + cb => { + if (req.files && req.files.length > 0) { + fs.stat(destPath, (err, stat) => { + if (err && err.code === 'ENOENT') cb(); + else cb(new Error(`Directory exists (should not have happened: ${err.code})`)); + }); + } else { + cb(); + } + }, + + // Unzips zip URL to tmp// (if any) + cb => { + if (req.body.zipurl) { + let archive = "zipurl.zip"; + + upload.storage.getDestination(req, archive, (err, dstPath) => { + if (err) cb(err); + else{ + let archiveDestPath = path.join(dstPath, archive); + + download(req.body.zipurl, archiveDestPath, cb); + } + }); + } else { + cb(); + } + }, + + cb => fs.mkdir(destPath, undefined, cb), + cb => fs.mkdir(destGpcPath, undefined, cb), + cb => mv(srcPath, destImagesPath, cb), + + cb => { + // Find any *.zip file and extract + fs.readdir(destImagesPath, (err, entries) => { + if (err) cb(err); + else { + async.eachSeries(entries, (entry, cb) => { + if (/\.zip$/gi.test(entry)) { + let filesCount = 0; + fs.createReadStream(path.join(destImagesPath, entry)).pipe(unzip.Parse()) + .on('entry', function(entry) { + if (entry.type === 'File') { + filesCount++; + entry.pipe(fs.createWriteStream(path.join(destImagesPath, path.basename(entry.path)))); + } else { + entry.autodrain(); + } + }) + .on('close', () => { + // Verify max images limit + if (config.maxImages && filesCount > config.maxImages) cb(`${filesCount} images uploaded, but this node can only process up to ${config.maxImages}.`); + else cb(); + }) + .on('error', cb); + } else cb(); + }, cb); + } + }); + }, + + cb => { + // Find any *.txt (GPC) file and move it to the data//gpc directory + // also remove any lingering zipurl.zip + fs.readdir(destImagesPath, (err, entries) => { + if (err) cb(err); + else { + async.eachSeries(entries, (entry, cb) => { + if (/\.txt$/gi.test(entry)) { + mv(path.join(destImagesPath, entry), path.join(destGpcPath, entry), cb); + }else if (/\.zip$/gi.test(entry)){ + fs.unlink(path.join(destImagesPath, entry), cb); + } else cb(); + }, cb); + } + }); + }, + + // Create task + cb => { + new Task(req.id, req.body.name, (err, task) => { + if (err) cb(err); + else { + taskManager.addNew(task); + res.json({ uuid: req.id }); + cb(); + } + }, req.body.options, + req.body.webhook, + req.body.skipPostProcessing === 'true'); + } + ], err => { + if (err) die(err.message); + }); + } + } +} diff --git a/package.json b/package.json index d2595ff..d057b68 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-opendronemap", - "version": "1.3.1", + "version": "1.4.0", "description": "REST API to access ODM", "main": "index.js", "scripts": { From 9fb1097e87fd736d4a2a4d5966678a1cf464bda0 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 31 Jan 2019 09:45:29 -0500 Subject: [PATCH 3/7] more refactoring, changed gpc --> gcp --- README.md | 7 +-- index.js | 158 ++++++------------------------------------------ libs/Task.js | 18 +++--- libs/taskNew.js | 35 +++++++---- 4 files changed, 51 insertions(+), 167 deletions(-) diff --git a/README.md b/README.md index 0a6eb67..9e2272a 100644 --- a/README.md +++ b/README.md @@ -156,9 +156,4 @@ Make a pull request for small contributions. For big contributions, please open ## Roadmap -- [X] Command line options for OpenDroneMap -- [X] GPC List support -- [ ] Video support when the [SLAM module](https://github.com/OpenDroneMap/OpenDroneMap/pull/317) becomes available -- [ ] Continuous Integration Setup -- [X] Documentation -- [ ] Unit Testing +See the [list of wanted features](https://github.com/OpenDroneMap/NodeODM/issues?q=is%3Aopen+is%3Aissue+label%3A%22new+feature%22). diff --git a/index.js b/index.js index f9aedcf..df6fdea 100644 --- a/index.js +++ b/index.js @@ -22,10 +22,8 @@ const config = require('./config.js'); const packageJson = JSON.parse(fs.readFileSync('./package.json')); const logger = require('./libs/logger'); -const path = require('path'); const async = require('async'); const mime = require('mime'); -const rmdir = require('rimraf'); const express = require('express'); const app = express(); @@ -33,12 +31,8 @@ const app = express(); const bodyParser = require('body-parser'); const TaskManager = require('./libs/TaskManager'); -const Task = require('./libs/Task'); const odmInfo = require('./libs/odmInfo'); -const Directories = require('./libs/Directories'); -const unzip = require('node-unzip-2'); const si = require('systeminformation'); -const mv = require('mv'); const S3 = require('./libs/S3'); const auth = require('./libs/auth/factory').fromConfig(config); @@ -89,6 +83,12 @@ let server; * required: false * type: boolean * - + * name: webhook + * in: formData + * description: Optional URL to call when processing has ended (either successfully or unsuccessfully). + * required: false + * type: string + * - * name: token * in: query * description: 'Token required for authentication (when authentication is required).' @@ -165,6 +165,12 @@ app.post('/task/new/commit/:uuid', authCheck, (req, res) => { * required: false * type: boolean * - + * name: webhook + * in: formData + * description: Optional URL to call when processing has ended (either successfully or unsuccessfully). + * required: false + * type: string + * - * name: token * in: query * description: 'Token required for authentication (when authentication is required).' @@ -191,141 +197,11 @@ app.post('/task/new/commit/:uuid', authCheck, (req, res) => { * schema: * $ref: '#/definitions/Error' */ -app.post('/task/new', authCheck, taskNew.assignUUID, taskNew.uploadImages, (req, res) => { - // TODO: consider doing the file moving in the background - // and return a response more quickly instead of a long timeout. - req.setTimeout(1000 * 60 * 20); - - let srcPath = path.join("tmp", req.id); - - // Print error message and cleanup - const die = (error) => { - res.json({error}); - - // Check if tmp/ directory needs to be cleaned - if (fs.stat(srcPath, (err, stats) => { - if (!err && stats.isDirectory()) rmdir(srcPath, () => {}); // ignore errors, don't wait - })); - }; - - if ((!req.files || req.files.length === 0) && !req.body.zipurl) die("Need at least 1 file or a zip file url."); - else if (config.maxImages && req.files && req.files.length > config.maxImages) die(`${req.files.length} images uploaded, but this node can only process up to ${config.maxImages}.`); - - else { - let destPath = path.join(Directories.data, req.id); - let destImagesPath = path.join(destPath, "images"); - let destGpcPath = path.join(destPath, "gpc"); - - async.series([ - cb => { - odmInfo.filterOptions(req.body.options, (err, options) => { - if (err) cb(err); - else { - req.body.options = options; - cb(null); - } - }); - }, - - // Move all uploads to data//images dir (if any) - cb => { - if (req.files && req.files.length > 0) { - fs.stat(destPath, (err, stat) => { - if (err && err.code === 'ENOENT') cb(); - else cb(new Error(`Directory exists (should not have happened: ${err.code})`)); - }); - } else { - cb(); - } - }, - - // Unzips zip URL to tmp// (if any) - cb => { - if (req.body.zipurl) { - let archive = "zipurl.zip"; - - upload.storage.getDestination(req, archive, (err, dstPath) => { - if (err) cb(err); - else{ - let archiveDestPath = path.join(dstPath, archive); - - download(req.body.zipurl, archiveDestPath, cb); - } - }); - } else { - cb(); - } - }, - - cb => fs.mkdir(destPath, undefined, cb), - cb => fs.mkdir(destGpcPath, undefined, cb), - cb => mv(srcPath, destImagesPath, cb), - - cb => { - // Find any *.zip file and extract - fs.readdir(destImagesPath, (err, entries) => { - if (err) cb(err); - else { - async.eachSeries(entries, (entry, cb) => { - if (/\.zip$/gi.test(entry)) { - let filesCount = 0; - fs.createReadStream(path.join(destImagesPath, entry)).pipe(unzip.Parse()) - .on('entry', function(entry) { - if (entry.type === 'File') { - filesCount++; - entry.pipe(fs.createWriteStream(path.join(destImagesPath, path.basename(entry.path)))); - } else { - entry.autodrain(); - } - }) - .on('close', () => { - // Verify max images limit - if (config.maxImages && filesCount > config.maxImages) cb(`${filesCount} images uploaded, but this node can only process up to ${config.maxImages}.`); - else cb(); - }) - .on('error', cb); - } else cb(); - }, cb); - } - }); - }, - - cb => { - // Find any *.txt (GPC) file and move it to the data//gpc directory - // also remove any lingering zipurl.zip - fs.readdir(destImagesPath, (err, entries) => { - if (err) cb(err); - else { - async.eachSeries(entries, (entry, cb) => { - if (/\.txt$/gi.test(entry)) { - mv(path.join(destImagesPath, entry), path.join(destGpcPath, entry), cb); - }else if (/\.zip$/gi.test(entry)){ - fs.unlink(path.join(destImagesPath, entry), cb); - } else cb(); - }, cb); - } - }); - }, - - // Create task - cb => { - new Task(req.id, req.body.name, (err, task) => { - if (err) cb(err); - else { - taskManager.addNew(task); - res.json({ uuid: req.id }); - cb(); - } - }, req.body.options, - req.body.webhook, - req.body.skipPostProcessing === 'true'); - } - ], err => { - if (err) die(err.message); - }); - } - -}); +app.post('/task/new', authCheck, taskNew.assignUUID, taskNew.uploadImages, (req, res, next) => { + if ((!req.files || req.files.length === 0) && !req.body.zipurl) req.error = "Need at least 1 file or a zip file url."; + else if (config.maxImages && req.files && req.files.length > config.maxImages) req.error = `${req.files.length} images uploaded, but this node can only process up to ${config.maxImages}.`; + next(); +}, taskNew.handleTaskNew); let getTaskFromUuid = (req, res, next) => { let task = taskManager.find(req.params.uuid); diff --git a/libs/Task.js b/libs/Task.js index d4c1266..ad5a6df 100644 --- a/libs/Task.js +++ b/libs/Task.js @@ -46,7 +46,7 @@ module.exports = class Task{ this.processingTime = -1; this.setStatus(statusCodes.QUEUED); this.options = options; - this.gpcFiles = []; + this.gcpFiles = []; this.output = []; this.runningProcesses = []; this.webhook = webhook; @@ -67,15 +67,15 @@ module.exports = class Task{ // Find GCP (if any) cb => { - fs.readdir(this.getGpcFolderPath(), (err, files) => { + fs.readdir(this.getGcpFolderPath(), (err, files) => { if (err) cb(err); else{ files.forEach(file => { if (/\.txt$/gi.test(file)){ - this.gpcFiles.push(file); + this.gcpFiles.push(file); } }); - logger.debug(`Found ${this.gpcFiles.length} GPC files (${this.gpcFiles.join(" ")}) for ${this.uuid}`); + logger.debug(`Found ${this.gcpFiles.length} GCP files (${this.gcpFiles.join(" ")}) for ${this.uuid}`); cb(null); } }); @@ -110,10 +110,10 @@ module.exports = class Task{ return path.join(this.getProjectFolderPath(), "images"); } - // Get path where GPC file(s) are stored + // Get path where GCP file(s) are stored // (relative to nodejs process CWD) - getGpcFolderPath(){ - return path.join(this.getProjectFolderPath(), "gpc"); + getGcpFolderPath(){ + return path.join(this.getProjectFolderPath(), "gcp"); } // Get path of project (where all images and assets folder are contained) @@ -385,8 +385,8 @@ module.exports = class Task{ runnerOptions["project-path"] = fs.realpathSync(Directories.data); - if (this.gpcFiles.length > 0){ - runnerOptions.gcp = fs.realpathSync(path.join(this.getGpcFolderPath(), this.gpcFiles[0])); + if (this.gcpFiles.length > 0){ + runnerOptions.gcp = fs.realpathSync(path.join(this.getGcpFolderPath(), this.gcpFiles[0])); } this.runningProcesses.push(odmRunner.run(runnerOptions, this.uuid, (err, code, signal) => { diff --git a/libs/taskNew.js b/libs/taskNew.js index ff8ad33..0d84735 100644 --- a/libs/taskNew.js +++ b/libs/taskNew.js @@ -21,6 +21,12 @@ const fs = require('fs'); const path = require('path'); const TaskManager = require('./TaskManager'); const uuidv4 = require('uuid/v4'); +const config = require('../config.js'); +const rmdir = require('rimraf'); +const Directories = require('./Directories'); +const unzip = require('node-unzip-2'); +const mv = require('mv'); +const Task = require('./Task'); const upload = multer({ storage: multer.diskStorage({ @@ -64,7 +70,14 @@ module.exports = { uploadImages: upload.array("images"), - handleTaskNew: (res, res) => { + setupFiles: (req, res, next) => { + // populate req.id (here or somehwere else) + // populate req.files from directory + // populate req.body from metadata file + + }, + + handleTaskNew: (req, res) => { // TODO: consider doing the file moving in the background // and return a response more quickly instead of a long timeout. req.setTimeout(1000 * 60 * 20); @@ -81,13 +94,12 @@ module.exports = { })); }; - if ((!req.files || req.files.length === 0) && !req.body.zipurl) die("Need at least 1 file or a zip file url."); - else if (config.maxImages && req.files && req.files.length > config.maxImages) die(`${req.files.length} images uploaded, but this node can only process up to ${config.maxImages}.`); - - else { + if (req.error !== undefined){ + die(req.error); + }else{ let destPath = path.join(Directories.data, req.id); let destImagesPath = path.join(destPath, "images"); - let destGpcPath = path.join(destPath, "gpc"); + let destGcpPath = path.join(destPath, "gcp"); async.series([ cb => { @@ -100,7 +112,7 @@ module.exports = { }); }, - // Move all uploads to data//images dir (if any) + // Check if dest directory already exists cb => { if (req.files && req.files.length > 0) { fs.stat(destPath, (err, stat) => { @@ -130,8 +142,9 @@ module.exports = { } }, + // Move all uploads to data//images dir (if any) cb => fs.mkdir(destPath, undefined, cb), - cb => fs.mkdir(destGpcPath, undefined, cb), + cb => fs.mkdir(destGcpPath, undefined, cb), cb => mv(srcPath, destImagesPath, cb), cb => { @@ -164,14 +177,14 @@ module.exports = { }, cb => { - // Find any *.txt (GPC) file and move it to the data//gpc directory + // Find any *.txt (GCP) file and move it to the data//gcp directory // also remove any lingering zipurl.zip fs.readdir(destImagesPath, (err, entries) => { if (err) cb(err); else { async.eachSeries(entries, (entry, cb) => { if (/\.txt$/gi.test(entry)) { - mv(path.join(destImagesPath, entry), path.join(destGpcPath, entry), cb); + mv(path.join(destImagesPath, entry), path.join(destGcpPath, entry), cb); }else if (/\.zip$/gi.test(entry)){ fs.unlink(path.join(destImagesPath, entry), cb); } else cb(); @@ -185,7 +198,7 @@ module.exports = { new Task(req.id, req.body.name, (err, task) => { if (err) cb(err); else { - taskManager.addNew(task); + TaskManager.singleton().addNew(task); res.json({ uuid: req.id }); cb(); } From 5c78f8c5066e17aa30dada63008842427476d857 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 31 Jan 2019 13:33:50 -0500 Subject: [PATCH 4/7] Chunked API draft working --- config.js | 2 + docs/index.adoc | 143 +++++++++++++++++++++++++++++++++++++++++++- docs/swagger.json | 2 +- index.js | 101 ++++++++++++++++++++++++------- libs/TaskManager.js | 28 ++++++++- libs/taskNew.js | 138 +++++++++++++++++++++++++++++++++++++----- 6 files changed, 372 insertions(+), 42 deletions(-) diff --git a/config.js b/config.js index f86b957..4170e81 100644 --- a/config.js +++ b/config.js @@ -33,6 +33,7 @@ Options: -d, --deamonize Set process to run as a deamon --parallel_queue_processing Number of simultaneous processing tasks (default: 2) --cleanup_tasks_after Number of minutes that elapse before deleting finished and canceled tasks (default: 2880) + --cleanup_uploads_after Number of minutes that elapse before deleting unfinished uploads. Set this value to the maximum time you expect a dataset to be uploaded. (default: 2880) --test Enable test mode. In test mode, no commands are sent to OpenDroneMap. This can be useful during development or testing (default: false) --test_skip_orthophotos If test mode is enabled, skip orthophoto results when generating assets. (default: false) --test_skip_dems If test mode is enabled, skip dems results when generating assets. (default: false) @@ -90,6 +91,7 @@ config.port = parseInt(argv.port || argv.p || fromConfigFile("port", process.env config.deamon = argv.deamonize || argv.d || fromConfigFile("daemon", false); config.parallelQueueProcessing = argv.parallel_queue_processing || fromConfigFile("parallelQueueProcessing", 2); config.cleanupTasksAfter = parseInt(argv.cleanup_tasks_after || fromConfigFile("cleanupTasksAfter", 2880)); +config.cleanupUploadsAfter = parseInt(argv.cleanup_uploads_after || fromConfigFile("cleanupUploadsAfter", 2880)); config.test = argv.test || fromConfigFile("test", false); config.testSkipOrthophotos = argv.test_skip_orthophotos || fromConfigFile("testSkipOrthophotos", false); config.testSkipDems = argv.test_skip_dems || fromConfigFile("testSkipDems", false); diff --git a/docs/index.adoc b/docs/index.adoc index 1060c84..6b0b507 100644 --- a/docs/index.adoc +++ b/docs/index.adoc @@ -8,7 +8,7 @@ REST API to access ODM === Version information [%hardbreaks] -_Version_ : 1.3.1 +_Version_ : 1.4.0 === Contact information @@ -281,7 +281,7 @@ _required_|UUID of the task|string| === POST /task/new ==== Description -Creates a new task and places it at the end of the processing queue +Creates a new task and places it at the end of the processing queue. For uploading really large tasks, see /task/new/init instead. ==== Parameters @@ -301,6 +301,8 @@ _optional_|An optional name to be associated with the task|string| _optional_|Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, …]. For example, [{"name":"cmvs-maxImages","value":"500"},{"name":"time","value":true}]. For a list of all options, call /options|string| |*FormData*|*skipPostProcessing* + _optional_|When set, skips generation of map tiles, derivate assets, point cloud tiles.|boolean| +|*FormData*|*webhook* + +_optional_|Optional URL to call when processing has ended (either successfully or unsuccessfully).|string| |*FormData*|*zipurl* + _optional_|URL of the zip file containing the images to process, plus an optional GCP file. If included, the GCP file should have .txt extension|string| |=== @@ -336,6 +338,143 @@ _required_|UUID of the newly created task|string * task +[[_task_new_commit_uuid_post]] +=== POST /task/new/commit/{uuid} + +==== Description +Creates a new task for which images have been uploaded via /task/new/upload. + + +==== Parameters + +[options="header", cols=".^2,.^3,.^9,.^4,.^2"] +|=== +|Type|Name|Description|Schema|Default +|*Path*|*uuid* + +_required_|UUID of the task|string| +|*Query*|*token* + +_optional_|Token required for authentication (when authentication is required).|string| +|=== + + +==== Responses + +[options="header", cols=".^2,.^14,.^4"] +|=== +|HTTP Code|Description|Schema +|*200*|Success|<<_task_new_commit_uuid_post_response_200,Response 200>> +|*default*|Error|<<_error,Error>> +|=== + +[[_task_new_commit_uuid_post_response_200]] +*Response 200* + +[options="header", cols=".^3,.^11,.^4"] +|=== +|Name|Description|Schema +|*uuid* + +_required_|UUID of the newly created task|string +|=== + + +==== Tags + +* task + + +[[_task_new_init_post]] +=== POST /task/new/init + +==== Description +Initialize the upload of a new task. If successful, a user can start uploading files via /task/new/upload. The task will not start until /task/new/commit is called. + + +==== Parameters + +[options="header", cols=".^2,.^3,.^9,.^4,.^2"] +|=== +|Type|Name|Description|Schema|Default +|*Header*|*set-uuid* + +_optional_|An optional UUID string that will be used as UUID for this task instead of generating a random one.|string| +|*Query*|*token* + +_optional_|Token required for authentication (when authentication is required).|string| +|*FormData*|*name* + +_optional_|An optional name to be associated with the task|string| +|*FormData*|*options* + +_optional_|Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, …]. For example, [{"name":"cmvs-maxImages","value":"500"},{"name":"time","value":true}]. For a list of all options, call /options|string| +|*FormData*|*skipPostProcessing* + +_optional_|When set, skips generation of map tiles, derivate assets, point cloud tiles.|boolean| +|*FormData*|*webhook* + +_optional_|Optional URL to call when processing has ended (either successfully or unsuccessfully).|string| +|=== + + +==== Responses + +[options="header", cols=".^2,.^14,.^4"] +|=== +|HTTP Code|Description|Schema +|*200*|Success|<<_task_new_init_post_response_200,Response 200>> +|*default*|Error|<<_error,Error>> +|=== + +[[_task_new_init_post_response_200]] +*Response 200* + +[options="header", cols=".^3,.^11,.^4"] +|=== +|Name|Description|Schema +|*uuid* + +_required_|UUID of the newly created task|string +|=== + + +==== Tags + +* task + + +[[_task_new_upload_uuid_post]] +=== POST /task/new/upload/{uuid} + +==== Description +Adds one or more files to the task created via /task/new/init. It does not start the task. To start the task, call /task/new/commit. + + +==== Parameters + +[options="header", cols=".^2,.^3,.^9,.^4,.^2"] +|=== +|Type|Name|Description|Schema|Default +|*Path*|*uuid* + +_required_|UUID of the task|string| +|*Query*|*token* + +_optional_|Token required for authentication (when authentication is required).|string| +|*FormData*|*images* + +_required_|Images to process, plus an optional GCP file. If included, the GCP file should have .txt extension|file| +|=== + + +==== Responses + +[options="header", cols=".^2,.^14,.^4"] +|=== +|HTTP Code|Description|Schema +|*200*|File Received|<<_response,Response>> +|*default*|Error|<<_error,Error>> +|=== + + +==== Consumes + +* `multipart/form-data` + + +==== Tags + +* task + + [[_task_remove_post]] === POST /task/remove diff --git a/docs/swagger.json b/docs/swagger.json index 0563f64..b819577 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -1 +1 @@ -{"info":{"title":"node-opendronemap","version":"1.3.1","description":"REST API to access ODM","license":{"name":"GPL-3.0"},"contact":{"name":"Piero Toffanin"}},"consumes":["application/json"],"produces":["application/json","application/zip"],"basePath":"/","schemes":["http"],"swagger":"2.0","paths":{"/task/new":{"post":{"description":"Creates a new task and places it at the end of the processing queue","tags":["task"],"consumes":["multipart/form-data"],"parameters":[{"name":"images","in":"formData","description":"Images to process, plus an optional GCP file. If included, the GCP file should have .txt extension","required":false,"type":"file"},{"name":"zipurl","in":"formData","description":"URL of the zip file containing the images to process, plus an optional GCP file. If included, the GCP file should have .txt extension","required":false,"type":"string"},{"name":"name","in":"formData","description":"An optional name to be associated with the task","required":false,"type":"string"},{"name":"options","in":"formData","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options","required":false,"type":"string"},{"name":"skipPostProcessing","in":"formData","description":"When set, skips generation of map tiles, derivate assets, point cloud tiles.","required":false,"type":"boolean"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"},{"name":"set-uuid","in":"header","description":"An optional UUID string that will be used as UUID for this task instead of generating a random one.","required":false,"type":"string"}],"responses":{"200":{"description":"Success","schema":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID of the newly created task"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/info":{"get":{"description":"Gets information about this task, such as name, creation date, processing time, status, command line options and number of images being processed. See schema definition for a full list.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"options","in":"formData","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options","required":false,"type":"string"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Task Information","schema":{"title":"TaskInfo","type":"object","required":["uuid","name","dateCreated","processingTime","status","options","imagesCount"],"properties":{"uuid":{"type":"string","description":"UUID"},"name":{"type":"string","description":"Name"},"dateCreated":{"type":"integer","description":"Timestamp"},"processingTime":{"type":"integer","description":"Milliseconds that have elapsed since the task started being processed."},"status":{"type":"integer","description":"Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)","enum":[10,20,30,40,50]},"options":{"type":"array","description":"List of options used to process this task","items":{"type":"object","required":["name","value"],"properties":{"name":{"type":"string","description":"Option name (example: \"odm_meshing-octreeDepth\")"},"value":{"type":"string","description":"Value (example: 9)"}}}},"imagesCount":{"type":"integer","description":"Number of images"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/output":{"get":{"description":"Retrieves the console output of the OpenDroneMap's process. Useful for monitoring execution and to provide updates to the user.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"line","in":"query","description":"Optional line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. Defaults to 0 (retrieve all console output).","default":0,"required":false,"type":"integer"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Console Output","schema":{"type":"string"}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/download/{asset}":{"get":{"description":"Retrieves an asset (the output of OpenDroneMap's processing) associated with a task","tags":["task"],"produces":["application/zip"],"parameters":[{"name":"uuid","in":"path","type":"string","description":"UUID of the task","required":true},{"name":"asset","in":"path","type":"string","description":"Type of asset to download. Use \"all.zip\" for zip file containing all assets.","required":true,"enum":["all.zip","orthophoto.tif"]},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Asset File","schema":{"type":"file"}},"default":{"description":"Error message","schema":{"$ref":"#/definitions/Error"}}}}},"/task/cancel":{"post":{"description":"Cancels a task (stops its execution, or prevents it from being executed)","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/task/remove":{"post":{"description":"Removes a task and deletes all of its assets","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/task/restart":{"post":{"description":"Restarts a task that was previously canceled, that had failed to process or that successfully completed","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"options","in":"body","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options. Overrides the previous options set for this task.","required":false,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/options":{"get":{"description":"Retrieves the command line options that can be passed to process a task","parameters":[{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"tags":["server"],"responses":{"200":{"description":"Options","schema":{"type":"array","items":{"title":"Option","type":"object","required":["name","type","value","domain","help"],"properties":{"name":{"type":"string","description":"Command line option (exactly as it is passed to the OpenDroneMap process, minus the leading '--')"},"type":{"type":"string","description":"Datatype of the value of this option","enum":["int","float","string","bool"]},"value":{"type":"string","description":"Default value of this option"},"domain":{"type":"string","description":"Valid range of values (for example, \"positive integer\" or \"float > 0.0\")"},"help":{"type":"string","description":"Description of what this option does"}}}}}}}},"/info":{"get":{"description":"Retrieves information about this node","parameters":[{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"tags":["server"],"responses":{"200":{"description":"Info","schema":{"type":"object","required":["version","taskQueueCount"],"properties":{"version":{"type":"string","description":"Current API version"},"taskQueueCount":{"type":"integer","description":"Number of tasks currently being processed or waiting to be processed"},"availableMemory":{"type":"integer","description":"Amount of RAM available in bytes"},"totalMemory":{"type":"integer","description":"Amount of total RAM in the system in bytes"},"cpuCores":{"type":"integer","description":"Number of CPU cores (virtual)"},"maxImages":{"type":"integer","description":"Maximum number of images allowed for new tasks or null if there's no limit."},"maxParallelTasks":{"type":"integer","description":"Maximum number of tasks that can be processed simultaneously"},"odmVersion":{"type":"string","description":"Current version of ODM"}}}}}}},"/auth/info":{"get":{"description":"Retrieves login information for this node.","tags":["auth"],"responses":{"200":{"description":"LoginInformation","schema":{"type":"object","required":["message","loginUrl","registerUrl"],"properties":{"message":{"type":"string","description":"Message to be displayed to the user prior to login/registration. This might include instructions on how to register or login, or to communicate that authentication is not available."},"loginUrl":{"type":"string","description":"URL (absolute or relative) where to make a POST request to obtain a token, or null if login is disabled."},"registerUrl":{"type":"string","description":"URL (absolute or relative) where to make a POST request to register a user, or null if registration is disabled."}}}}}}},"/auth/login":{"post":{"description":"Retrieve a token from a username/password pair.","parameters":[{"name":"username","in":"body","description":"Username","required":true,"schema":{"type":"string"}},{"name":"password","in":"body","description":"Password","required":true,"type":"string"}],"responses":{"200":{"description":"Login Succeeded","schema":{"type":"object","required":["token"],"properties":{"token":{"type":"string","description":"Token to be passed as a query parameter to other API calls."}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/auth/register":{"post":{"description":"Register a new username/password.","parameters":[{"name":"username","in":"body","description":"Username","required":true,"schema":{"type":"string"}},{"name":"password","in":"body","description":"Password","required":true,"type":"string"}],"responses":{"200":{"description":"Response","schema":{"$ref":"#/definitions/Response"}}}}}},"definitions":{"Error":{"type":"object","required":["error"],"properties":{"error":{"type":"string","description":"Description of the error"}}},"Response":{"type":"object","required":["success"],"properties":{"success":{"type":"boolean","description":"true if the command succeeded, false otherwise"},"error":{"type":"string","description":"Error message if an error occured"}}}},"responses":{},"parameters":{},"securityDefinitions":{},"tags":[]} \ No newline at end of file +{"info":{"title":"node-opendronemap","version":"1.4.0","description":"REST API to access ODM","license":{"name":"GPL-3.0"},"contact":{"name":"Piero Toffanin"}},"consumes":["application/json"],"produces":["application/json","application/zip"],"basePath":"/","schemes":["http"],"swagger":"2.0","paths":{"/task/new/init":{"post":{"description":"Initialize the upload of a new task. If successful, a user can start uploading files via /task/new/upload. The task will not start until /task/new/commit is called.","tags":["task"],"parameters":[{"name":"name","in":"formData","description":"An optional name to be associated with the task","required":false,"type":"string"},{"name":"options","in":"formData","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options","required":false,"type":"string"},{"name":"skipPostProcessing","in":"formData","description":"When set, skips generation of map tiles, derivate assets, point cloud tiles.","required":false,"type":"boolean"},{"name":"webhook","in":"formData","description":"Optional URL to call when processing has ended (either successfully or unsuccessfully).","required":false,"type":"string"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"},{"name":"set-uuid","in":"header","description":"An optional UUID string that will be used as UUID for this task instead of generating a random one.","required":false,"type":"string"}],"responses":{"200":{"description":"Success","schema":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID of the newly created task"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/new/upload/{uuid}":{"post":{"description":"Adds one or more files to the task created via /task/new/init. It does not start the task. To start the task, call /task/new/commit.","tags":["task"],"consumes":["multipart/form-data"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"images","in":"formData","description":"Images to process, plus an optional GCP file. If included, the GCP file should have .txt extension","required":true,"type":"file"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"File Received","schema":{"$ref":"#/definitions/Response"}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/new/commit/{uuid}":{"post":{"description":"Creates a new task for which images have been uploaded via /task/new/upload.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Success","schema":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID of the newly created task"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/new":{"post":{"description":"Creates a new task and places it at the end of the processing queue. For uploading really large tasks, see /task/new/init instead.","tags":["task"],"consumes":["multipart/form-data"],"parameters":[{"name":"images","in":"formData","description":"Images to process, plus an optional GCP file. If included, the GCP file should have .txt extension","required":false,"type":"file"},{"name":"zipurl","in":"formData","description":"URL of the zip file containing the images to process, plus an optional GCP file. If included, the GCP file should have .txt extension","required":false,"type":"string"},{"name":"name","in":"formData","description":"An optional name to be associated with the task","required":false,"type":"string"},{"name":"options","in":"formData","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options","required":false,"type":"string"},{"name":"skipPostProcessing","in":"formData","description":"When set, skips generation of map tiles, derivate assets, point cloud tiles.","required":false,"type":"boolean"},{"name":"webhook","in":"formData","description":"Optional URL to call when processing has ended (either successfully or unsuccessfully).","required":false,"type":"string"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"},{"name":"set-uuid","in":"header","description":"An optional UUID string that will be used as UUID for this task instead of generating a random one.","required":false,"type":"string"}],"responses":{"200":{"description":"Success","schema":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID of the newly created task"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/info":{"get":{"description":"Gets information about this task, such as name, creation date, processing time, status, command line options and number of images being processed. See schema definition for a full list.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"options","in":"formData","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options","required":false,"type":"string"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Task Information","schema":{"title":"TaskInfo","type":"object","required":["uuid","name","dateCreated","processingTime","status","options","imagesCount"],"properties":{"uuid":{"type":"string","description":"UUID"},"name":{"type":"string","description":"Name"},"dateCreated":{"type":"integer","description":"Timestamp"},"processingTime":{"type":"integer","description":"Milliseconds that have elapsed since the task started being processed."},"status":{"type":"integer","description":"Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)","enum":[10,20,30,40,50]},"options":{"type":"array","description":"List of options used to process this task","items":{"type":"object","required":["name","value"],"properties":{"name":{"type":"string","description":"Option name (example: \"odm_meshing-octreeDepth\")"},"value":{"type":"string","description":"Value (example: 9)"}}}},"imagesCount":{"type":"integer","description":"Number of images"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/output":{"get":{"description":"Retrieves the console output of the OpenDroneMap's process. Useful for monitoring execution and to provide updates to the user.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"line","in":"query","description":"Optional line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. Defaults to 0 (retrieve all console output).","default":0,"required":false,"type":"integer"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Console Output","schema":{"type":"string"}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/download/{asset}":{"get":{"description":"Retrieves an asset (the output of OpenDroneMap's processing) associated with a task","tags":["task"],"produces":["application/zip"],"parameters":[{"name":"uuid","in":"path","type":"string","description":"UUID of the task","required":true},{"name":"asset","in":"path","type":"string","description":"Type of asset to download. Use \"all.zip\" for zip file containing all assets.","required":true,"enum":["all.zip","orthophoto.tif"]},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Asset File","schema":{"type":"file"}},"default":{"description":"Error message","schema":{"$ref":"#/definitions/Error"}}}}},"/task/cancel":{"post":{"description":"Cancels a task (stops its execution, or prevents it from being executed)","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/task/remove":{"post":{"description":"Removes a task and deletes all of its assets","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/task/restart":{"post":{"description":"Restarts a task that was previously canceled, that had failed to process or that successfully completed","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"options","in":"body","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options. Overrides the previous options set for this task.","required":false,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/options":{"get":{"description":"Retrieves the command line options that can be passed to process a task","parameters":[{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"tags":["server"],"responses":{"200":{"description":"Options","schema":{"type":"array","items":{"title":"Option","type":"object","required":["name","type","value","domain","help"],"properties":{"name":{"type":"string","description":"Command line option (exactly as it is passed to the OpenDroneMap process, minus the leading '--')"},"type":{"type":"string","description":"Datatype of the value of this option","enum":["int","float","string","bool"]},"value":{"type":"string","description":"Default value of this option"},"domain":{"type":"string","description":"Valid range of values (for example, \"positive integer\" or \"float > 0.0\")"},"help":{"type":"string","description":"Description of what this option does"}}}}}}}},"/info":{"get":{"description":"Retrieves information about this node","parameters":[{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"tags":["server"],"responses":{"200":{"description":"Info","schema":{"type":"object","required":["version","taskQueueCount"],"properties":{"version":{"type":"string","description":"Current API version"},"taskQueueCount":{"type":"integer","description":"Number of tasks currently being processed or waiting to be processed"},"availableMemory":{"type":"integer","description":"Amount of RAM available in bytes"},"totalMemory":{"type":"integer","description":"Amount of total RAM in the system in bytes"},"cpuCores":{"type":"integer","description":"Number of CPU cores (virtual)"},"maxImages":{"type":"integer","description":"Maximum number of images allowed for new tasks or null if there's no limit."},"maxParallelTasks":{"type":"integer","description":"Maximum number of tasks that can be processed simultaneously"},"odmVersion":{"type":"string","description":"Current version of ODM"}}}}}}},"/auth/info":{"get":{"description":"Retrieves login information for this node.","tags":["auth"],"responses":{"200":{"description":"LoginInformation","schema":{"type":"object","required":["message","loginUrl","registerUrl"],"properties":{"message":{"type":"string","description":"Message to be displayed to the user prior to login/registration. This might include instructions on how to register or login, or to communicate that authentication is not available."},"loginUrl":{"type":"string","description":"URL (absolute or relative) where to make a POST request to obtain a token, or null if login is disabled."},"registerUrl":{"type":"string","description":"URL (absolute or relative) where to make a POST request to register a user, or null if registration is disabled."}}}}}}},"/auth/login":{"post":{"description":"Retrieve a token from a username/password pair.","parameters":[{"name":"username","in":"body","description":"Username","required":true,"schema":{"type":"string"}},{"name":"password","in":"body","description":"Password","required":true,"type":"string"}],"responses":{"200":{"description":"Login Succeeded","schema":{"type":"object","required":["token"],"properties":{"token":{"type":"string","description":"Token to be passed as a query parameter to other API calls."}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/auth/register":{"post":{"description":"Register a new username/password.","parameters":[{"name":"username","in":"body","description":"Username","required":true,"schema":{"type":"string"}},{"name":"password","in":"body","description":"Password","required":true,"type":"string"}],"responses":{"200":{"description":"Response","schema":{"$ref":"#/definitions/Response"}}}}}},"definitions":{"Error":{"type":"object","required":["error"],"properties":{"error":{"type":"string","description":"Description of the error"}}},"Response":{"type":"object","required":["success"],"properties":{"success":{"type":"boolean","description":"true if the command succeeded, false otherwise"},"error":{"type":"string","description":"Error message if an error occured"}}}},"responses":{},"parameters":{},"securityDefinitions":{},"tags":[]} \ No newline at end of file diff --git a/index.js b/index.js index df6fdea..3dc957c 100644 --- a/index.js +++ b/index.js @@ -29,6 +29,7 @@ const express = require('express'); const app = express(); const bodyParser = require('body-parser'); +const multer = require('multer'); const TaskManager = require('./libs/TaskManager'); const odmInfo = require('./libs/odmInfo'); @@ -39,21 +40,10 @@ const auth = require('./libs/auth/factory').fromConfig(config); const authCheck = auth.getMiddleware(); const taskNew = require('./libs/taskNew'); -// zip files -let request = require('request'); - -let download = function(uri, filename, callback) { - request.head(uri, function(err, res, body) { - if (err) callback(err); - else{ - request(uri).pipe(fs.createWriteStream(filename)).on('close', callback); - } - }); -}; - app.use(express.static('public')); app.use('/swagger.json', express.static('docs/swagger.json')); +const formDataParser = multer().none(); const urlEncodedBodyParser = bodyParser.urlencoded({extended: false}); let taskManager; @@ -65,6 +55,7 @@ let server; * description: Initialize the upload of a new task. If successful, a user can start uploading files via /task/new/upload. The task will not start until /task/new/commit is called. * tags: [task] * parameters: + * - * name: name * in: formData * description: An optional name to be associated with the task @@ -115,21 +106,85 @@ let server; * schema: * $ref: '#/definitions/Error' */ -app.post('/task/new/init', authCheck, taskNew.assignUUID, (req, res) => { - -}); +app.post('/task/new/init', authCheck, taskNew.assignUUID, formDataParser, taskNew.handleInit); -app.post('/task/new/upload/:uuid', authCheck, (req, res) => { -}); +/** @swagger + * /task/new/upload/{uuid}: + * post: + * description: Adds one or more files to the task created via /task/new/init. It does not start the task. To start the task, call /task/new/commit. + * tags: [task] + * consumes: + * - multipart/form-data + * parameters: + * - + * name: uuid + * in: path + * description: UUID of the task + * required: true + * type: string + * - + * name: images + * in: formData + * description: Images to process, plus an optional GCP file. If included, the GCP file should have .txt extension + * required: true + * type: file + * - + * name: token + * in: query + * description: 'Token required for authentication (when authentication is required).' + * required: false + * type: string + * responses: + * 200: + * description: File Received + * schema: + * $ref: "#/definitions/Response" + * default: + * description: Error + * schema: + * $ref: '#/definitions/Error' + */ +app.post('/task/new/upload/:uuid', authCheck, taskNew.getUUID, taskNew.uploadImages, taskNew.handleUpload); -app.post('/task/new/commit/:uuid', authCheck, (req, res) => { - -}); +/** @swagger + * /task/new/commit/{uuid}: + * post: + * description: Creates a new task for which images have been uploaded via /task/new/upload. + * tags: [task] + * parameters: + * - + * name: uuid + * in: path + * description: UUID of the task + * required: true + * type: string + * - + * name: token + * in: query + * description: 'Token required for authentication (when authentication is required).' + * required: false + * type: string + * responses: + * 200: + * description: Success + * schema: + * type: object + * required: [uuid] + * properties: + * uuid: + * type: string + * description: UUID of the newly created task + * default: + * description: Error + * schema: + * $ref: '#/definitions/Error' + */ +app.post('/task/new/commit/:uuid', authCheck, taskNew.getUUID, taskNew.handleCommit, taskNew.createTask); /** @swagger * /task/new: * post: - * description: Creates a new task and places it at the end of the processing queue + * description: Creates a new task and places it at the end of the processing queue. For uploading really large tasks, see /task/new/init instead. * tags: [task] * consumes: * - multipart/form-data @@ -198,10 +253,12 @@ app.post('/task/new/commit/:uuid', authCheck, (req, res) => { * $ref: '#/definitions/Error' */ app.post('/task/new', authCheck, taskNew.assignUUID, taskNew.uploadImages, (req, res, next) => { + console.log(req.body); + req.body = req.body || {}; if ((!req.files || req.files.length === 0) && !req.body.zipurl) req.error = "Need at least 1 file or a zip file url."; else if (config.maxImages && req.files && req.files.length > config.maxImages) req.error = `${req.files.length} images uploaded, but this node can only process up to ${config.maxImages}.`; next(); -}, taskNew.handleTaskNew); +}, taskNew.createTask); let getTaskFromUuid = (req, res, next) => { let task = taskManager.find(req.params.uuid); diff --git a/libs/TaskManager.js b/libs/TaskManager.js index db9622c..e01ec48 100644 --- a/libs/TaskManager.js +++ b/libs/TaskManager.js @@ -30,6 +30,7 @@ const Directories = require('./Directories'); const TASKS_DUMP_FILE = path.join(Directories.data, "tasks.json"); const CLEANUP_TASKS_IF_OLDER_THAN = 1000 * 60 * config.cleanupTasksAfter; // minutes +const CLEANUP_STALE_UPLOADS_AFTER = 1000 * 60 * config.cleanupUploadsAfter; // minutes let taskManager; @@ -42,6 +43,7 @@ class TaskManager{ cb => this.restoreTaskListFromDump(cb), cb => this.removeOldTasks(cb), cb => this.removeOrphanedDirectories(cb), + cb => this.removeStaleUploads(cb), cb => { this.processNextTask(); cb(); @@ -51,6 +53,7 @@ class TaskManager{ schedule.scheduleJob('0 * * * *', () => { this.removeOldTasks(); this.dumpTaskList(); + this.removeStaleUploads(); }); cb(); @@ -84,7 +87,6 @@ class TaskManager{ // Removes directories that don't have a corresponding // task associated with it (maybe as a cause of an abrupt exit) - // TODO: do not delete /task/new/init directories!!! removeOrphanedDirectories(done){ logger.info("Checking for orphaned directories to be removed..."); @@ -104,6 +106,30 @@ class TaskManager{ }); } + removeStaleUploads(done){ + logger.info("Checking for stale uploads..."); + fs.readdir("tmp", (err, entries) => { + if (err) done(err); + else{ + const now = new Date(); + async.eachSeries(entries, (entry, cb) => { + let dirPath = path.join("tmp", entry); + if (entry.match(/^[\w\d]+\-[\w\d]+\-[\w\d]+\-[\w\d]+\-[\w\d]+$/)){ + fs.stat(dirPath, (err, stats) => { + if (err) cb(err); + else{ + if (stats.isDirectory() && stats.ctime.getTime() + CLEANUP_STALE_UPLOADS_AFTER < now.getTime()){ + logger.info(`Found stale upload directory: ${entry}, removing...`); + rmdir(dirPath, cb); + }else cb(); + } + }); + }else cb(); + }, done); + } + }); + } + // Load tasks that already exists (if any) restoreTaskListFromDump(done){ fs.readFile(TASKS_DUMP_FILE, (err, data) => { diff --git a/libs/taskNew.js b/libs/taskNew.js index 0d84735..9620ba7 100644 --- a/libs/taskNew.js +++ b/libs/taskNew.js @@ -27,6 +27,25 @@ const Directories = require('./Directories'); const unzip = require('node-unzip-2'); const mv = require('mv'); const Task = require('./Task'); +const async = require('async'); +const odmInfo = require('./odmInfo'); +const request = require('request'); + +const download = function(uri, filename, callback) { + request.head(uri, function(err, res, body) { + if (err) callback(err); + else{ + request(uri).pipe(fs.createWriteStream(filename)).on('close', callback); + } + }); +}; + +const removeDirectory = function(dir, cb = () => {}){ + fs.stat(dir, (err, stats) => { + if (!err && stats.isDirectory()) rmdir(dir, cb); // ignore errors, don't wait + else cb(err); + }); +}; const upload = multer({ storage: multer.diskStorage({ @@ -43,7 +62,9 @@ const upload = multer({ }); }, filename: (req, file, cb) => { - cb(null, file.originalname); + let filename = file.originalname; + if (filename === "body.json") filename = "_body.json"; + cb(null, filename); } }) }); @@ -68,30 +89,115 @@ module.exports = { } }, - uploadImages: upload.array("images"), + getUUID: (req, res, next) => { + req.id = req.params.uuid; + if (!req.id) res.json({error: `Invalid uuid (not set)`}); - setupFiles: (req, res, next) => { - // populate req.id (here or somehwere else) - // populate req.files from directory - // populate req.body from metadata file + const srcPath = path.join("tmp", req.id); + const bodyFile = path.join(srcPath, "body.json"); + fs.access(bodyFile, fs.F_OK, err => { + if (err) res.json({error: `Invalid uuid (not found)`}); + else next(); + }); }, - handleTaskNew: (req, res) => { - // TODO: consider doing the file moving in the background - // and return a response more quickly instead of a long timeout. - req.setTimeout(1000 * 60 * 20); + uploadImages: upload.array("images"), - let srcPath = path.join("tmp", req.id); + handleUpload: (req, res) => { + // IMPROVEMENT: check files count limits ahead of handleTaskNew + if (req.files && req.files.length > 0){ + res.json({success: true}); + }else{ + res.json({error: "Need at least 1 file."}); + } + }, + + handleCommit: (req, res, next) => { + const srcPath = path.join("tmp", req.id); + const bodyFile = path.join(srcPath, "body.json"); + + async.series([ + cb => { + fs.readFile(bodyFile, 'utf8', (err, data) => { + if (err) cb(err); + else{ + try{ + const body = JSON.parse(data); + fs.unlink(bodyFile, err => { + if (err) cb(err); + else cb(null, body); + }); + }catch(e){ + cb("Malformed body.json"); + } + } + }); + }, + cb => fs.readdir(srcPath, cb), + ], (err, [ body, files ]) => { + if (err) res.json({error: err.message}); + else{ + req.body = body; + req.files = files; + next(); + } + }); + }, + + handleInit: (req, res) => { + console.log(req.body); + req.body = req.body || {}; + + const srcPath = path.join("tmp", req.id); + const bodyFile = path.join(srcPath, "body.json"); // Print error message and cleanup const die = (error) => { res.json({error}); + removeDirectory(srcPath); + }; - // Check if tmp/ directory needs to be cleaned - if (fs.stat(srcPath, (err, stats) => { - if (!err && stats.isDirectory()) rmdir(srcPath, () => {}); // ignore errors, don't wait - })); + async.series([ + cb => { + // Check for problems before file uploads + if (req.body && req.body.options){ + odmInfo.filterOptions(req.body.options, err => { + if (err) cb(err); + else cb(); + }); + }else cb(); + }, + cb => { + fs.stat(srcPath, (err, stat) => { + if (err && err.code === 'ENOENT') cb(); + else cb(new Error(`Directory exists (should not have happened: ${err.code})`)); + }); + }, + cb => fs.mkdir(srcPath, undefined, cb), + cb => { + fs.writeFile(bodyFile, JSON.stringify(req.body), {encoding: 'utf8'}, cb); + }, + cb => { + res.json({uuid: req.id}); + cb(); + } + ], err => { + if (err) die(err.message); + }); + }, + + createTask: (req, res) => { + // IMPROVEMENT: consider doing the file moving in the background + // and return a response more quickly instead of a long timeout. + req.setTimeout(1000 * 60 * 20); + + const srcPath = path.join("tmp", req.id); + + // Print error message and cleanup + const die = (error) => { + res.json({error}); + removeDirectory(srcPath); }; if (req.error !== undefined){ @@ -202,7 +308,7 @@ module.exports = { res.json({ uuid: req.id }); cb(); } - }, req.body.options, + }, req.body.options, req.body.webhook, req.body.skipPostProcessing === 'true'); } From 5be741bef237766993d73b1a1fea8f7162b67a59 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 31 Jan 2019 17:16:37 -0500 Subject: [PATCH 5/7] Started rewriting UI, added dropzone, tweaks, drop uploads test mode --- config.js | 2 + index.js | 10 +- libs/taskNew.js | 16 +- public/index.html | 58 +- public/js/dropzone.js | 3530 ++++++++++++++++++++++++++++++++++++ public/js/fileinput.js | 3263 --------------------------------- public/js/fileinput.min.js | 12 - public/js/main.js | 244 ++- 8 files changed, 3756 insertions(+), 3379 deletions(-) create mode 100644 public/js/dropzone.js delete mode 100644 public/js/fileinput.js delete mode 100644 public/js/fileinput.min.js diff --git a/config.js b/config.js index 4170e81..770327c 100644 --- a/config.js +++ b/config.js @@ -37,6 +37,7 @@ Options: --test Enable test mode. In test mode, no commands are sent to OpenDroneMap. This can be useful during development or testing (default: false) --test_skip_orthophotos If test mode is enabled, skip orthophoto results when generating assets. (default: false) --test_skip_dems If test mode is enabled, skip dems results when generating assets. (default: false) + --test_drop_uploads If test mode is enabled, drop /task/new/upload requests with 50% probability. (default: false) --powercycle When set, the application exits immediately after powering up. Useful for testing launch and compilation issues. --token Sets a token that needs to be passed for every request. This can be used to limit access to the node only to token holders. (default: none) --max_images Specify the maximum number of images that this processing node supports. (default: unlimited) @@ -95,6 +96,7 @@ config.cleanupUploadsAfter = parseInt(argv.cleanup_uploads_after || fromConfigFi config.test = argv.test || fromConfigFile("test", false); config.testSkipOrthophotos = argv.test_skip_orthophotos || fromConfigFile("testSkipOrthophotos", false); config.testSkipDems = argv.test_skip_dems || fromConfigFile("testSkipDems", false); +config.testDropUploads = argv.test_drop_uploads || fromConfigFile("testDropUploads", false); config.powercycle = argv.powercycle || fromConfigFile("powercycle", false); config.token = argv.token || fromConfigFile("token", ""); config.maxImages = parseInt(argv.max_images || fromConfigFile("maxImages", "")) || null; diff --git a/index.js b/index.js index 3dc957c..0539ec1 100644 --- a/index.js +++ b/index.js @@ -144,7 +144,7 @@ app.post('/task/new/init', authCheck, taskNew.assignUUID, formDataParser, taskNe * schema: * $ref: '#/definitions/Error' */ -app.post('/task/new/upload/:uuid', authCheck, taskNew.getUUID, taskNew.uploadImages, taskNew.handleUpload); +app.post('/task/new/upload/:uuid', authCheck, taskNew.getUUID, taskNew.preUpload, taskNew.uploadImages, taskNew.handleUpload); /** @swagger * /task/new/commit/{uuid}: @@ -253,7 +253,6 @@ app.post('/task/new/commit/:uuid', authCheck, taskNew.getUUID, taskNew.handleCom * $ref: '#/definitions/Error' */ app.post('/task/new', authCheck, taskNew.assignUUID, taskNew.uploadImages, (req, res, next) => { - console.log(req.body); req.body = req.body || {}; if ((!req.files || req.files.length === 0) && !req.body.zipurl) req.error = "Need at least 1 file or a zip file url."; else if (config.maxImages && req.files && req.files.length > config.maxImages) req.error = `${req.files.length} images uploaded, but this node can only process up to ${config.maxImages}.`; @@ -813,7 +812,12 @@ process.on('SIGTERM', gracefulShutdown); process.on('SIGINT', gracefulShutdown); // Startup -if (config.test) logger.info("Running in test mode"); +if (config.test) { + logger.info("Running in test mode"); + if (config.testSkipOrthophotos) logger.info("Orthophotos will be skipped"); + if (config.testSkipDems) logger.info("DEMs will be skipped"); + if (config.testDropUploads) logger.info("Uploads will drop at random"); +} let commands = [ cb => odmInfo.initialize(cb), diff --git a/libs/taskNew.js b/libs/taskNew.js index 9620ba7..83ac25d 100644 --- a/libs/taskNew.js +++ b/libs/taskNew.js @@ -102,6 +102,17 @@ module.exports = { }); }, + preUpload: (req, res, next) => { + // Testing stuff + if (!config.test) next(); + else{ + if (config.testDropUploads){ + if (Math.random() < 0.5) res.sendStatus(500); + else next(); + } + } + }, + uploadImages: upload.array("images"), handleUpload: (req, res) => { @@ -140,13 +151,16 @@ module.exports = { else{ req.body = body; req.files = files; + + if (req.files.length === 0){ + req.error = "Need at least 1 file."; + } next(); } }); }, handleInit: (req, res) => { - console.log(req.body); req.body = req.body || {}; const srcPath = path.join("tmp", req.id); diff --git a/public/index.html b/public/index.html index cb1839f..2a212a9 100644 --- a/public/index.html +++ b/public/index.html @@ -14,8 +14,19 @@ padding-top: 50px; padding-bottom: 20px; } + .navbar{ + background-color: #3498db; + } + a:hover, a:focus, a:active, a{ + color: #3498db; + } + #images{ + font-weight: bold; + } + #btnSelectFiles, #images{ + display: inline-block; + } - @@ -29,34 +40,34 @@
NodeODM -

Open Source Drone Aerial Imagery Processing -

-
-

New Task

-
-
-