From c7a742307978773f66ebbbc755d9658867ee861f Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 30 Jan 2019 15:56:09 -0500 Subject: [PATCH] Started refactoring to accomodate chunked upload API --- index.js | 108 +++++++++++++++--------- libs/TaskManager.js | 14 ++- libs/taskNew.js | 201 ++++++++++++++++++++++++++++++++++++++++++++ package.json | 2 +- 4 files changed, 281 insertions(+), 44 deletions(-) create mode 100644 libs/taskNew.js diff --git a/index.js b/index.js index 5813443..f9aedcf 100644 --- a/index.js +++ b/index.js @@ -30,7 +30,6 @@ const rmdir = require('rimraf'); const express = require('express'); const app = express(); -const multer = require('multer'); const bodyParser = require('body-parser'); const TaskManager = require('./libs/TaskManager'); @@ -44,7 +43,7 @@ const S3 = require('./libs/S3'); const auth = require('./libs/auth/factory').fromConfig(config); const authCheck = auth.getMiddleware(); -const uuidv4 = require('uuid/v4'); +const taskNew = require('./libs/taskNew'); // zip files let request = require('request'); @@ -61,31 +60,72 @@ let download = function(uri, filename, callback) { app.use(express.static('public')); app.use('/swagger.json', express.static('docs/swagger.json')); -const upload = multer({ - storage: multer.diskStorage({ - destination: (req, file, cb) => { - let dstPath = path.join("tmp", req.id); - fs.exists(dstPath, exists => { - if (!exists) { - fs.mkdir(dstPath, undefined, () => { - cb(null, dstPath); - }); - } else { - cb(null, dstPath); - } - }); - }, - filename: (req, file, cb) => { - cb(null, file.originalname); - } - }) -}); - const urlEncodedBodyParser = bodyParser.urlencoded({extended: false}); let taskManager; let server; +/** @swagger + * /task/new/init: + * post: + * description: Initialize the upload of a new task. If successful, a user can start uploading files via /task/new/upload. The task will not start until /task/new/commit is called. + * tags: [task] + * parameters: + * name: name + * in: formData + * description: An optional name to be associated with the task + * required: false + * type: string + * - + * name: options + * in: formData + * description: 'Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{"name":"cmvs-maxImages","value":"500"},{"name":"time","value":true}]. For a list of all options, call /options' + * required: false + * type: string + * - + * name: skipPostProcessing + * in: formData + * description: 'When set, skips generation of map tiles, derivate assets, point cloud tiles.' + * required: false + * type: boolean + * - + * name: token + * in: query + * description: 'Token required for authentication (when authentication is required).' + * required: false + * type: string + * - + * name: set-uuid + * in: header + * description: 'An optional UUID string that will be used as UUID for this task instead of generating a random one.' + * required: false + * type: string + * responses: + * 200: + * description: Success + * schema: + * type: object + * required: [uuid] + * properties: + * uuid: + * type: string + * description: UUID of the newly created task + * default: + * description: Error + * schema: + * $ref: '#/definitions/Error' + */ +app.post('/task/new/init', authCheck, taskNew.assignUUID, (req, res) => { + +}); + +app.post('/task/new/upload/:uuid', authCheck, (req, res) => { +}); + +app.post('/task/new/commit/:uuid', authCheck, (req, res) => { + +}); + /** @swagger * /task/new: * post: @@ -151,24 +191,7 @@ let server; * schema: * $ref: '#/definitions/Error' */ -app.post('/task/new', authCheck, (req, res, next) => { - // A user can optionally suggest a UUID instead of letting - // nodeODM pick one. - if (req.get('set-uuid')){ - const userUuid = req.get('set-uuid'); - - // Valid UUID and no other task with same UUID? - if (/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(userUuid) && !taskManager.find(userUuid)){ - req.id = userUuid; - next(); - }else{ - res.json({error: `Invalid set-uuid: ${userUuid}`}) - } - }else{ - req.id = uuidv4(); - next(); - } -}, upload.array('images'), (req, res) => { +app.post('/task/new', authCheck, taskNew.assignUUID, taskNew.uploadImages, (req, res) => { // TODO: consider doing the file moving in the background // and return a response more quickly instead of a long timeout. req.setTimeout(1000 * 60 * 20); @@ -863,7 +886,10 @@ let commands = [ cb => odmInfo.initialize(cb), cb => auth.initialize(cb), cb => S3.initialize(cb), - cb => { taskManager = new TaskManager(cb); }, + cb => { + TaskManager.initialize(cb); + taskManager = TaskManager.singleton(); + }, cb => { server = app.listen(config.port, err => { if (!err) logger.info('Server has started on port ' + String(config.port)); diff --git a/libs/TaskManager.js b/libs/TaskManager.js index 9b1ced5..db9622c 100644 --- a/libs/TaskManager.js +++ b/libs/TaskManager.js @@ -31,7 +31,9 @@ const Directories = require('./Directories'); const TASKS_DUMP_FILE = path.join(Directories.data, "tasks.json"); const CLEANUP_TASKS_IF_OLDER_THAN = 1000 * 60 * config.cleanupTasksAfter; // minutes -module.exports = class TaskManager{ +let taskManager; + +class TaskManager{ constructor(done){ this.tasks = {}; this.runningQueue = []; @@ -82,6 +84,7 @@ module.exports = class TaskManager{ // Removes directories that don't have a corresponding // task associated with it (maybe as a cause of an abrupt exit) + // TODO: do not delete /task/new/init directories!!! removeOrphanedDirectories(done){ logger.info("Checking for orphaned directories to be removed..."); @@ -264,4 +267,11 @@ module.exports = class TaskManager{ } return count; } -}; +} + +module.exports = { + singleton: function(){ return taskManager; }, + initialize: function(cb){ + taskManager = new TaskManager(cb); + } +}; \ No newline at end of file diff --git a/libs/taskNew.js b/libs/taskNew.js new file mode 100644 index 0000000..ff8ad33 --- /dev/null +++ b/libs/taskNew.js @@ -0,0 +1,201 @@ +/* +Node-OpenDroneMap Node.js App and REST API to access OpenDroneMap. +Copyright (C) 2016 Node-OpenDroneMap Contributors + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +const multer = require('multer'); +const fs = require('fs'); +const path = require('path'); +const TaskManager = require('./TaskManager'); +const uuidv4 = require('uuid/v4'); + +const upload = multer({ + storage: multer.diskStorage({ + destination: (req, file, cb) => { + let dstPath = path.join("tmp", req.id); + fs.exists(dstPath, exists => { + if (!exists) { + fs.mkdir(dstPath, undefined, () => { + cb(null, dstPath); + }); + } else { + cb(null, dstPath); + } + }); + }, + filename: (req, file, cb) => { + cb(null, file.originalname); + } + }) +}); + +module.exports = { + assignUUID: (req, res, next) => { + // A user can optionally suggest a UUID instead of letting + // nodeODM pick one. + if (req.get('set-uuid')){ + const userUuid = req.get('set-uuid'); + + // Valid UUID and no other task with same UUID? + if (/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(userUuid) && !TaskManager.singleton().find(userUuid)){ + req.id = userUuid; + next(); + }else{ + res.json({error: `Invalid set-uuid: ${userUuid}`}) + } + }else{ + req.id = uuidv4(); + next(); + } + }, + + uploadImages: upload.array("images"), + + handleTaskNew: (res, res) => { + // TODO: consider doing the file moving in the background + // and return a response more quickly instead of a long timeout. + req.setTimeout(1000 * 60 * 20); + + let srcPath = path.join("tmp", req.id); + + // Print error message and cleanup + const die = (error) => { + res.json({error}); + + // Check if tmp/ directory needs to be cleaned + if (fs.stat(srcPath, (err, stats) => { + if (!err && stats.isDirectory()) rmdir(srcPath, () => {}); // ignore errors, don't wait + })); + }; + + if ((!req.files || req.files.length === 0) && !req.body.zipurl) die("Need at least 1 file or a zip file url."); + else if (config.maxImages && req.files && req.files.length > config.maxImages) die(`${req.files.length} images uploaded, but this node can only process up to ${config.maxImages}.`); + + else { + let destPath = path.join(Directories.data, req.id); + let destImagesPath = path.join(destPath, "images"); + let destGpcPath = path.join(destPath, "gpc"); + + async.series([ + cb => { + odmInfo.filterOptions(req.body.options, (err, options) => { + if (err) cb(err); + else { + req.body.options = options; + cb(null); + } + }); + }, + + // Move all uploads to data//images dir (if any) + cb => { + if (req.files && req.files.length > 0) { + fs.stat(destPath, (err, stat) => { + if (err && err.code === 'ENOENT') cb(); + else cb(new Error(`Directory exists (should not have happened: ${err.code})`)); + }); + } else { + cb(); + } + }, + + // Unzips zip URL to tmp// (if any) + cb => { + if (req.body.zipurl) { + let archive = "zipurl.zip"; + + upload.storage.getDestination(req, archive, (err, dstPath) => { + if (err) cb(err); + else{ + let archiveDestPath = path.join(dstPath, archive); + + download(req.body.zipurl, archiveDestPath, cb); + } + }); + } else { + cb(); + } + }, + + cb => fs.mkdir(destPath, undefined, cb), + cb => fs.mkdir(destGpcPath, undefined, cb), + cb => mv(srcPath, destImagesPath, cb), + + cb => { + // Find any *.zip file and extract + fs.readdir(destImagesPath, (err, entries) => { + if (err) cb(err); + else { + async.eachSeries(entries, (entry, cb) => { + if (/\.zip$/gi.test(entry)) { + let filesCount = 0; + fs.createReadStream(path.join(destImagesPath, entry)).pipe(unzip.Parse()) + .on('entry', function(entry) { + if (entry.type === 'File') { + filesCount++; + entry.pipe(fs.createWriteStream(path.join(destImagesPath, path.basename(entry.path)))); + } else { + entry.autodrain(); + } + }) + .on('close', () => { + // Verify max images limit + if (config.maxImages && filesCount > config.maxImages) cb(`${filesCount} images uploaded, but this node can only process up to ${config.maxImages}.`); + else cb(); + }) + .on('error', cb); + } else cb(); + }, cb); + } + }); + }, + + cb => { + // Find any *.txt (GPC) file and move it to the data//gpc directory + // also remove any lingering zipurl.zip + fs.readdir(destImagesPath, (err, entries) => { + if (err) cb(err); + else { + async.eachSeries(entries, (entry, cb) => { + if (/\.txt$/gi.test(entry)) { + mv(path.join(destImagesPath, entry), path.join(destGpcPath, entry), cb); + }else if (/\.zip$/gi.test(entry)){ + fs.unlink(path.join(destImagesPath, entry), cb); + } else cb(); + }, cb); + } + }); + }, + + // Create task + cb => { + new Task(req.id, req.body.name, (err, task) => { + if (err) cb(err); + else { + taskManager.addNew(task); + res.json({ uuid: req.id }); + cb(); + } + }, req.body.options, + req.body.webhook, + req.body.skipPostProcessing === 'true'); + } + ], err => { + if (err) die(err.message); + }); + } + } +} diff --git a/package.json b/package.json index d2595ff..d057b68 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-opendronemap", - "version": "1.3.1", + "version": "1.4.0", "description": "REST API to access ODM", "main": "index.js", "scripts": {