kopia lustrzana https://github.com/OpenDroneMap/NodeODM
Started refactoring to accomodate chunked upload API
Piero Toffanin
rodzic
e87eabe9e3
commit
c7a7423079
108
index.js
108
index.js
|
|
@ -30,7 +30,6 @@ const rmdir = require('rimraf');
|
|||
const express = require('express');
|
||||
const app = express();
|
||||
|
||||
const multer = require('multer');
|
||||
const bodyParser = require('body-parser');
|
||||
|
||||
const TaskManager = require('./libs/TaskManager');
|
||||
|
|
@ -44,7 +43,7 @@ const S3 = require('./libs/S3');
|
|||
|
||||
const auth = require('./libs/auth/factory').fromConfig(config);
|
||||
const authCheck = auth.getMiddleware();
|
||||
const uuidv4 = require('uuid/v4');
|
||||
const taskNew = require('./libs/taskNew');
|
||||
|
||||
// zip files
|
||||
let request = require('request');
|
||||
|
|
@ -61,31 +60,72 @@ let download = function(uri, filename, callback) {
|
|||
app.use(express.static('public'));
|
||||
app.use('/swagger.json', express.static('docs/swagger.json'));
|
||||
|
||||
const upload = multer({
|
||||
storage: multer.diskStorage({
|
||||
destination: (req, file, cb) => {
|
||||
let dstPath = path.join("tmp", req.id);
|
||||
fs.exists(dstPath, exists => {
|
||||
if (!exists) {
|
||||
fs.mkdir(dstPath, undefined, () => {
|
||||
cb(null, dstPath);
|
||||
});
|
||||
} else {
|
||||
cb(null, dstPath);
|
||||
}
|
||||
});
|
||||
},
|
||||
filename: (req, file, cb) => {
|
||||
cb(null, file.originalname);
|
||||
}
|
||||
})
|
||||
});
|
||||
|
||||
const urlEncodedBodyParser = bodyParser.urlencoded({extended: false});
|
||||
|
||||
let taskManager;
|
||||
let server;
|
||||
|
||||
/** @swagger
|
||||
* /task/new/init:
|
||||
* post:
|
||||
* description: Initialize the upload of a new task. If successful, a user can start uploading files via /task/new/upload. The task will not start until /task/new/commit is called.
|
||||
* tags: [task]
|
||||
* parameters:
|
||||
* name: name
|
||||
* in: formData
|
||||
* description: An optional name to be associated with the task
|
||||
* required: false
|
||||
* type: string
|
||||
* -
|
||||
* name: options
|
||||
* in: formData
|
||||
* description: 'Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{"name":"cmvs-maxImages","value":"500"},{"name":"time","value":true}]. For a list of all options, call /options'
|
||||
* required: false
|
||||
* type: string
|
||||
* -
|
||||
* name: skipPostProcessing
|
||||
* in: formData
|
||||
* description: 'When set, skips generation of map tiles, derivate assets, point cloud tiles.'
|
||||
* required: false
|
||||
* type: boolean
|
||||
* -
|
||||
* name: token
|
||||
* in: query
|
||||
* description: 'Token required for authentication (when authentication is required).'
|
||||
* required: false
|
||||
* type: string
|
||||
* -
|
||||
* name: set-uuid
|
||||
* in: header
|
||||
* description: 'An optional UUID string that will be used as UUID for this task instead of generating a random one.'
|
||||
* required: false
|
||||
* type: string
|
||||
* responses:
|
||||
* 200:
|
||||
* description: Success
|
||||
* schema:
|
||||
* type: object
|
||||
* required: [uuid]
|
||||
* properties:
|
||||
* uuid:
|
||||
* type: string
|
||||
* description: UUID of the newly created task
|
||||
* default:
|
||||
* description: Error
|
||||
* schema:
|
||||
* $ref: '#/definitions/Error'
|
||||
*/
|
||||
app.post('/task/new/init', authCheck, taskNew.assignUUID, (req, res) => {
|
||||
|
||||
});
|
||||
|
||||
app.post('/task/new/upload/:uuid', authCheck, (req, res) => {
|
||||
});
|
||||
|
||||
app.post('/task/new/commit/:uuid', authCheck, (req, res) => {
|
||||
|
||||
});
|
||||
|
||||
/** @swagger
|
||||
* /task/new:
|
||||
* post:
|
||||
|
|
@ -151,24 +191,7 @@ let server;
|
|||
* schema:
|
||||
* $ref: '#/definitions/Error'
|
||||
*/
|
||||
app.post('/task/new', authCheck, (req, res, next) => {
|
||||
// A user can optionally suggest a UUID instead of letting
|
||||
// nodeODM pick one.
|
||||
if (req.get('set-uuid')){
|
||||
const userUuid = req.get('set-uuid');
|
||||
|
||||
// Valid UUID and no other task with same UUID?
|
||||
if (/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(userUuid) && !taskManager.find(userUuid)){
|
||||
req.id = userUuid;
|
||||
next();
|
||||
}else{
|
||||
res.json({error: `Invalid set-uuid: ${userUuid}`})
|
||||
}
|
||||
}else{
|
||||
req.id = uuidv4();
|
||||
next();
|
||||
}
|
||||
}, upload.array('images'), (req, res) => {
|
||||
app.post('/task/new', authCheck, taskNew.assignUUID, taskNew.uploadImages, (req, res) => {
|
||||
// TODO: consider doing the file moving in the background
|
||||
// and return a response more quickly instead of a long timeout.
|
||||
req.setTimeout(1000 * 60 * 20);
|
||||
|
|
@ -863,7 +886,10 @@ let commands = [
|
|||
cb => odmInfo.initialize(cb),
|
||||
cb => auth.initialize(cb),
|
||||
cb => S3.initialize(cb),
|
||||
cb => { taskManager = new TaskManager(cb); },
|
||||
cb => {
|
||||
TaskManager.initialize(cb);
|
||||
taskManager = TaskManager.singleton();
|
||||
},
|
||||
cb => {
|
||||
server = app.listen(config.port, err => {
|
||||
if (!err) logger.info('Server has started on port ' + String(config.port));
|
||||
|
|
|
|||
|
|
@ -31,7 +31,9 @@ const Directories = require('./Directories');
|
|||
const TASKS_DUMP_FILE = path.join(Directories.data, "tasks.json");
|
||||
const CLEANUP_TASKS_IF_OLDER_THAN = 1000 * 60 * config.cleanupTasksAfter; // minutes
|
||||
|
||||
module.exports = class TaskManager{
|
||||
let taskManager;
|
||||
|
||||
class TaskManager{
|
||||
constructor(done){
|
||||
this.tasks = {};
|
||||
this.runningQueue = [];
|
||||
|
|
@ -82,6 +84,7 @@ module.exports = class TaskManager{
|
|||
|
||||
// Removes directories that don't have a corresponding
|
||||
// task associated with it (maybe as a cause of an abrupt exit)
|
||||
// TODO: do not delete /task/new/init directories!!!
|
||||
removeOrphanedDirectories(done){
|
||||
logger.info("Checking for orphaned directories to be removed...");
|
||||
|
||||
|
|
@ -264,4 +267,11 @@ module.exports = class TaskManager{
|
|||
}
|
||||
return count;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
singleton: function(){ return taskManager; },
|
||||
initialize: function(cb){
|
||||
taskManager = new TaskManager(cb);
|
||||
}
|
||||
};
|
||||
|
|
@ -0,0 +1,201 @@
|
|||
/*
|
||||
Node-OpenDroneMap Node.js App and REST API to access OpenDroneMap.
|
||||
Copyright (C) 2016 Node-OpenDroneMap Contributors
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
const multer = require('multer');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const TaskManager = require('./TaskManager');
|
||||
const uuidv4 = require('uuid/v4');
|
||||
|
||||
const upload = multer({
|
||||
storage: multer.diskStorage({
|
||||
destination: (req, file, cb) => {
|
||||
let dstPath = path.join("tmp", req.id);
|
||||
fs.exists(dstPath, exists => {
|
||||
if (!exists) {
|
||||
fs.mkdir(dstPath, undefined, () => {
|
||||
cb(null, dstPath);
|
||||
});
|
||||
} else {
|
||||
cb(null, dstPath);
|
||||
}
|
||||
});
|
||||
},
|
||||
filename: (req, file, cb) => {
|
||||
cb(null, file.originalname);
|
||||
}
|
||||
})
|
||||
});
|
||||
|
||||
module.exports = {
|
||||
assignUUID: (req, res, next) => {
|
||||
// A user can optionally suggest a UUID instead of letting
|
||||
// nodeODM pick one.
|
||||
if (req.get('set-uuid')){
|
||||
const userUuid = req.get('set-uuid');
|
||||
|
||||
// Valid UUID and no other task with same UUID?
|
||||
if (/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(userUuid) && !TaskManager.singleton().find(userUuid)){
|
||||
req.id = userUuid;
|
||||
next();
|
||||
}else{
|
||||
res.json({error: `Invalid set-uuid: ${userUuid}`})
|
||||
}
|
||||
}else{
|
||||
req.id = uuidv4();
|
||||
next();
|
||||
}
|
||||
},
|
||||
|
||||
uploadImages: upload.array("images"),
|
||||
|
||||
handleTaskNew: (res, res) => {
|
||||
// TODO: consider doing the file moving in the background
|
||||
// and return a response more quickly instead of a long timeout.
|
||||
req.setTimeout(1000 * 60 * 20);
|
||||
|
||||
let srcPath = path.join("tmp", req.id);
|
||||
|
||||
// Print error message and cleanup
|
||||
const die = (error) => {
|
||||
res.json({error});
|
||||
|
||||
// Check if tmp/ directory needs to be cleaned
|
||||
if (fs.stat(srcPath, (err, stats) => {
|
||||
if (!err && stats.isDirectory()) rmdir(srcPath, () => {}); // ignore errors, don't wait
|
||||
}));
|
||||
};
|
||||
|
||||
if ((!req.files || req.files.length === 0) && !req.body.zipurl) die("Need at least 1 file or a zip file url.");
|
||||
else if (config.maxImages && req.files && req.files.length > config.maxImages) die(`${req.files.length} images uploaded, but this node can only process up to ${config.maxImages}.`);
|
||||
|
||||
else {
|
||||
let destPath = path.join(Directories.data, req.id);
|
||||
let destImagesPath = path.join(destPath, "images");
|
||||
let destGpcPath = path.join(destPath, "gpc");
|
||||
|
||||
async.series([
|
||||
cb => {
|
||||
odmInfo.filterOptions(req.body.options, (err, options) => {
|
||||
if (err) cb(err);
|
||||
else {
|
||||
req.body.options = options;
|
||||
cb(null);
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
// Move all uploads to data/<uuid>/images dir (if any)
|
||||
cb => {
|
||||
if (req.files && req.files.length > 0) {
|
||||
fs.stat(destPath, (err, stat) => {
|
||||
if (err && err.code === 'ENOENT') cb();
|
||||
else cb(new Error(`Directory exists (should not have happened: ${err.code})`));
|
||||
});
|
||||
} else {
|
||||
cb();
|
||||
}
|
||||
},
|
||||
|
||||
// Unzips zip URL to tmp/<uuid>/ (if any)
|
||||
cb => {
|
||||
if (req.body.zipurl) {
|
||||
let archive = "zipurl.zip";
|
||||
|
||||
upload.storage.getDestination(req, archive, (err, dstPath) => {
|
||||
if (err) cb(err);
|
||||
else{
|
||||
let archiveDestPath = path.join(dstPath, archive);
|
||||
|
||||
download(req.body.zipurl, archiveDestPath, cb);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
cb();
|
||||
}
|
||||
},
|
||||
|
||||
cb => fs.mkdir(destPath, undefined, cb),
|
||||
cb => fs.mkdir(destGpcPath, undefined, cb),
|
||||
cb => mv(srcPath, destImagesPath, cb),
|
||||
|
||||
cb => {
|
||||
// Find any *.zip file and extract
|
||||
fs.readdir(destImagesPath, (err, entries) => {
|
||||
if (err) cb(err);
|
||||
else {
|
||||
async.eachSeries(entries, (entry, cb) => {
|
||||
if (/\.zip$/gi.test(entry)) {
|
||||
let filesCount = 0;
|
||||
fs.createReadStream(path.join(destImagesPath, entry)).pipe(unzip.Parse())
|
||||
.on('entry', function(entry) {
|
||||
if (entry.type === 'File') {
|
||||
filesCount++;
|
||||
entry.pipe(fs.createWriteStream(path.join(destImagesPath, path.basename(entry.path))));
|
||||
} else {
|
||||
entry.autodrain();
|
||||
}
|
||||
})
|
||||
.on('close', () => {
|
||||
// Verify max images limit
|
||||
if (config.maxImages && filesCount > config.maxImages) cb(`${filesCount} images uploaded, but this node can only process up to ${config.maxImages}.`);
|
||||
else cb();
|
||||
})
|
||||
.on('error', cb);
|
||||
} else cb();
|
||||
}, cb);
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
cb => {
|
||||
// Find any *.txt (GPC) file and move it to the data/<uuid>/gpc directory
|
||||
// also remove any lingering zipurl.zip
|
||||
fs.readdir(destImagesPath, (err, entries) => {
|
||||
if (err) cb(err);
|
||||
else {
|
||||
async.eachSeries(entries, (entry, cb) => {
|
||||
if (/\.txt$/gi.test(entry)) {
|
||||
mv(path.join(destImagesPath, entry), path.join(destGpcPath, entry), cb);
|
||||
}else if (/\.zip$/gi.test(entry)){
|
||||
fs.unlink(path.join(destImagesPath, entry), cb);
|
||||
} else cb();
|
||||
}, cb);
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
// Create task
|
||||
cb => {
|
||||
new Task(req.id, req.body.name, (err, task) => {
|
||||
if (err) cb(err);
|
||||
else {
|
||||
taskManager.addNew(task);
|
||||
res.json({ uuid: req.id });
|
||||
cb();
|
||||
}
|
||||
}, req.body.options,
|
||||
req.body.webhook,
|
||||
req.body.skipPostProcessing === 'true');
|
||||
}
|
||||
], err => {
|
||||
if (err) die(err.message);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "node-opendronemap",
|
||||
"version": "1.3.1",
|
||||
"version": "1.4.0",
|
||||
"description": "REST API to access ODM",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue