mirror
/
OpenDroneMap-NodeODM
kopia lustrzana https://github.com/OpenDroneMap/NodeODM
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Modify: Dockerfile for local build 

Add: Webhook callback option
Modify: .gitignore for vscode
pull/6/head^2
Lee Pepper 2017-04-07 10:55:24 -06:00
rodzic 9725cba1a2
commit 93d59292dd
8 zmienionych plików z 1320 dodań i 1206 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

3
.dockerignore 100644
Wyświetl plik

@ -0,0 +1,3 @@
node_modules
tests
tmp

2
.gitignore vendored
Wyświetl plik

@ -39,3 +39,5 @@ jspm_packages
# Elastic Beanstalk # Elastic Beanstalk
.elasticbeanstalk .elasticbeanstalk
.vscode

7
Dockerfile
Wyświetl plik

@ -30,8 +30,13 @@ RUN cd /staging/PotreeConverter && \
RUN mkdir /var/www RUN mkdir /var/www
WORKDIR "/var/www" WORKDIR "/var/www"
RUN git clone https://github.com/OpenDroneMap/node-OpenDroneMap . # RUN git clone https://github.com/OpenDroneMap/node-OpenDroneMap .
COPY . /var/www
RUN npm install RUN npm install
RUN mkdir tmp
# Fix old version of gdal2tiles.py # Fix old version of gdal2tiles.py
# RUN (cd / && patch -p0) <patches/gdal2tiles.patch # RUN (cd / && patch -p0) <patches/gdal2tiles.patch

833
index.js
Wyświetl plik

@ -40,453 +40,455 @@ let odmOptions = require('./libs/odmOptions');
let Directories = require('./libs/Directories'); let Directories = require('./libs/Directories');
let winstonStream = { let winstonStream = {
write: function(message, encoding){ write: function(message, encoding) {
logger.debug(message.slice(0, -1)); logger.debug(message.slice(0, -1));
} }
}; };
app.use(morgan('combined', { stream : winstonStream })); app.use(morgan('combined', { stream: winstonStream }));
app.use(bodyParser.urlencoded({extended: true})); app.use(bodyParser.urlencoded({ extended: true }));
app.use(bodyParser.json()); app.use(bodyParser.json());
app.use(express.static('public')); app.use(express.static('public'));
app.use('/swagger.json', express.static('docs/swagger.json')); app.use('/swagger.json', express.static('docs/swagger.json'));
let upload = multer({ let upload = multer({
storage: multer.diskStorage({ storage: multer.diskStorage({
destination: (req, file, cb) => { destination: (req, file, cb) => {
let dstPath = path.join("tmp", req.id); let dstPath = path.join("tmp", req.id);
fs.exists(dstPath, exists => { fs.exists(dstPath, exists => {
if (!exists){ if (!exists) {
fs.mkdir(dstPath, undefined, () => { fs.mkdir(dstPath, undefined, () => {
cb(null, dstPath); cb(null, dstPath);
}); });
}else{ } else {
cb(null, dstPath); cb(null, dstPath);
} }
}); });
}, },
filename: (req, file, cb) => { filename: (req, file, cb) => {
cb(null, file.originalname); cb(null, file.originalname);
} }
}) })
}); });
let taskManager; let taskManager;
let server; let server;
/** @swagger /** @swagger
* /task/new: * /task/new:
* post: * post:
* description: Creates a new task and places it at the end of the processing queue * description: Creates a new task and places it at the end of the processing queue
* tags: [task] * tags: [task]
* consumes: * consumes:
* - multipart/form-data * - multipart/form-data
* parameters: * parameters:
* - * -
* name: images * name: images
* in: formData * in: formData
* description: Images to process, plus an optional GPC file. If included, the GPC file should have .txt extension * description: Images to process, plus an optional GPC file. If included, the GPC file should have .txt extension
* required: true * required: true
* type: file * type: file
* - * -
* name: name * name: name
* in: formData * in: formData
* description: An optional name to be associated with the task * description: An optional name to be associated with the task
* required: false * required: false
* type: string * type: string
* - * -
* name: options * name: options
* in: formData * in: formData
* description: 'Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{"name":"cmvs-maxImages","value":"500"},{"name":"time","value":true}]. For a list of all options, call /options' * description: 'Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{"name":"cmvs-maxImages","value":"500"},{"name":"time","value":true}]. For a list of all options, call /options'
* required: false * required: false
* type: string * type: string
* responses: * responses:
* 200: * 200:
* description: Success * description: Success
* schema: * schema:
* type: object * type: object
* required: [uuid] * required: [uuid]
* properties: * properties:
* uuid: * uuid:
* type: string * type: string
* description: UUID of the newly created task * description: UUID of the newly created task
* default: * default:
* description: Error * description: Error
* schema: * schema:
* $ref: '#/definitions/Error' * $ref: '#/definitions/Error'
*/ */
app.post('/task/new', addRequestId, upload.array('images'), (req, res) => { app.post('/task/new', addRequestId, upload.array('images'), (req, res) => {
if (!req.files || req.files.length === 0) res.json({error: "Need at least 1 file."}); if (!req.files || req.files.length === 0) res.json({ error: "Need at least 1 file." });
else{ else {
let srcPath = path.join("tmp", req.id); let srcPath = path.join("tmp", req.id);
let destPath = path.join(Directories.data, req.id); let destPath = path.join(Directories.data, req.id);
let destImagesPath = path.join(destPath, "images"); let destImagesPath = path.join(destPath, "images");
let destGpcPath = path.join(destPath, "gpc"); let destGpcPath = path.join(destPath, "gpc");
async.series([ async.series([
cb => { cb => {
odmOptions.filterOptions(req.body.options, (err, options) => { odmOptions.filterOptions(req.body.options, (err, options) => {
if (err) cb(err); if (err) cb(err);
else{ else {
req.body.options = options; req.body.options = options;
cb(null); cb(null);
} }
}); });
}, },
// Move all uploads to data/<uuid>/images dir // Move all uploads to data/<uuid>/images dir
cb => { cb => {
fs.stat(destPath, (err, stat) => { setTimeout(function() {
if (err && err.code === 'ENOENT') cb(); fs.stat(destPath, (err, stat) => {
else cb(new Error(`Directory exists (should not have happened: ${err.code})`)); if (err && err.code === 'ENOENT') cb();
}); else cb(new Error(`Directory exists (should not have happened: ${err.code})`));
}, });
cb => fs.mkdir(destPath, undefined, cb), }, 300);
cb => fs.mkdir(destGpcPath, undefined, cb), },
cb => fs.rename(srcPath, destImagesPath, cb), cb => fs.mkdir(destPath, undefined, cb),
cb => { cb => fs.mkdir(destGpcPath, undefined, cb),
// Find any *.txt (GPC) file and move it to the data/<uuid>/gpc directory cb => fs.rename(srcPath, destImagesPath, cb),
fs.readdir(destImagesPath, (err, entries) => { cb => {
if (err) cb(err); // Find any *.txt (GPC) file and move it to the data/<uuid>/gpc directory
else{ fs.readdir(destImagesPath, (err, entries) => {
async.eachSeries(entries, (entry, cb) => { if (err) cb(err);
if (/\.txt$/gi.test(entry)){ else {
fs.rename(path.join(destImagesPath, entry), path.join(destGpcPath, entry), cb); async.eachSeries(entries, (entry, cb) => {
}else cb(); if (/\.txt$/gi.test(entry)) {
}, cb); fs.rename(path.join(destImagesPath, entry), path.join(destGpcPath, entry), cb);
} } else cb();
}); }, cb);
}, }
});
},
// Create task // Create task
cb => { cb => {
new Task(req.id, req.body.name, (err, task) => { new Task(req.id, req.body.name, (err, task) => {
if (err) cb(err); if (err) cb(err);
else{ else {
taskManager.addNew(task); taskManager.addNew(task);
res.json({uuid: req.id}); res.json({ uuid: req.id });
cb(); cb();
} }
}, req.body.options); }, req.body.options);
} }
], err => { ], err => {
if (err) res.json({error: err.message}); if (err) res.json({ error: err.message });
}); });
} }
}); });
let getTaskFromUuid = (req, res, next) => { let getTaskFromUuid = (req, res, next) => {
let task = taskManager.find(req.params.uuid); let task = taskManager.find(req.params.uuid);
if (task){ if (task) {
req.task = task; req.task = task;
next(); next();
}else res.json({error: `${req.params.uuid} not found`}); } else res.json({ error: `${req.params.uuid} not found` });
}; };
/** @swagger /** @swagger
* /task/{uuid}/info: * /task/{uuid}/info:
* get: * get:
* description: Gets information about this task, such as name, creation date, processing time, status, command line options and number of images being processed. See schema definition for a full list. * description: Gets information about this task, such as name, creation date, processing time, status, command line options and number of images being processed. See schema definition for a full list.
* tags: [task] * tags: [task]
* parameters: * parameters:
* - * -
* name: uuid * name: uuid
* in: path * in: path
* description: UUID of the task * description: UUID of the task
* required: true * required: true
* type: string * type: string
* responses: * responses:
* 200: * 200:
* description: Task Information * description: Task Information
* schema: * schema:
* title: TaskInfo * title: TaskInfo
* type: object * type: object
* required: [uuid, name, dateCreated, processingTime, status, options, imagesCount] * required: [uuid, name, dateCreated, processingTime, status, options, imagesCount]
* properties: * properties:
* uuid: * uuid:
* type: string * type: string
* description: UUID * description: UUID
* name: * name:
* type: string * type: string
* description: Name * description: Name
* dateCreated: * dateCreated:
* type: integer * type: integer
* description: Timestamp * description: Timestamp
* processingTime: * processingTime:
* type: integer * type: integer
* description: Milliseconds that have elapsed since the task started being processed. * description: Milliseconds that have elapsed since the task started being processed.
* status: * status:
* type: integer * type: integer
* description: Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED) * description: Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)
* enum: [10, 20, 30, 40, 50] * enum: [10, 20, 30, 40, 50]
* options: * options:
* type: array * type: array
* description: List of options used to process this task * description: List of options used to process this task
* items: * items:
* type: object * type: object
* required: [name, value] * required: [name, value]
* properties: * properties:
* name: * name:
* type: string * type: string
* description: 'Option name (example: "odm_meshing-octreeDepth")' * description: 'Option name (example: "odm_meshing-octreeDepth")'
* value: * value:
* type: string * type: string
* description: 'Value (example: 9)' * description: 'Value (example: 9)'
* imagesCount: * imagesCount:
* type: integer * type: integer
* description: Number of images * description: Number of images
* default: * default:
* description: Error * description: Error
* schema: * schema:
* $ref: '#/definitions/Error' * $ref: '#/definitions/Error'
*/ */
app.get('/task/:uuid/info', getTaskFromUuid, (req, res) => { app.get('/task/:uuid/info', getTaskFromUuid, (req, res) => {
res.json(req.task.getInfo()); res.json(req.task.getInfo());
}); });
/** @swagger /** @swagger
* /task/{uuid}/output: * /task/{uuid}/output:
* get: * get:
* description: Retrieves the console output of the OpenDroneMap's process. Useful for monitoring execution and to provide updates to the user. * description: Retrieves the console output of the OpenDroneMap's process. Useful for monitoring execution and to provide updates to the user.
* tags: [task] * tags: [task]
* parameters: * parameters:
* - * -
* name: uuid * name: uuid
* in: path * in: path
* description: UUID of the task * description: UUID of the task
* required: true * required: true
* type: string * type: string
* - * -
* name: line * name: line
* in: query * in: query
* description: Optional line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. Defaults to 0 (retrieve all console output). * description: Optional line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. Defaults to 0 (retrieve all console output).
* default: 0 * default: 0
* required: false * required: false
* type: integer * type: integer
* responses: * responses:
* 200: * 200:
* description: Console Output * description: Console Output
* schema: * schema:
* type: string * type: string
* default: * default:
* description: Error * description: Error
* schema: * schema:
* $ref: '#/definitions/Error' * $ref: '#/definitions/Error'
*/ */
app.get('/task/:uuid/output', getTaskFromUuid, (req, res) => { app.get('/task/:uuid/output', getTaskFromUuid, (req, res) => {
res.json(req.task.getOutput(req.query.line)); res.json(req.task.getOutput(req.query.line));
}); });
/** @swagger /** @swagger
* /task/{uuid}/download/{asset}: * /task/{uuid}/download/{asset}:
* get: * get:
* description: Retrieves an asset (the output of OpenDroneMap's processing) associated with a task * description: Retrieves an asset (the output of OpenDroneMap's processing) associated with a task
* tags: [task] * tags: [task]
* produces: [application/zip] * produces: [application/zip]
* parameters: * parameters:
* - name: uuid * - name: uuid
* in: path * in: path
* type: string * type: string
* description: UUID of the task * description: UUID of the task
* required: true * required: true
* - name: asset * - name: asset
* in: path * in: path
* type: string * type: string
* description: Type of asset to download. Use "all.zip" for zip file containing all assets. * description: Type of asset to download. Use "all.zip" for zip file containing all assets.
* required: true * required: true
* enum: * enum:
* - all.zip * - all.zip
* - orthophoto.tif * - orthophoto.tif
* responses: * responses:
* 200: * 200:
* description: Asset File * description: Asset File
* schema: * schema:
* type: file * type: file
* default: * default:
* description: Error message * description: Error message
* schema: * schema:
* $ref: '#/definitions/Error' * $ref: '#/definitions/Error'
*/ */
app.get('/task/:uuid/download/:asset', getTaskFromUuid, (req, res) => { app.get('/task/:uuid/download/:asset', getTaskFromUuid, (req, res) => {
let asset = req.params.asset !== undefined ? req.params.asset : "all.zip"; let asset = req.params.asset !== undefined ? req.params.asset : "all.zip";
let filePath = req.task.getAssetsArchivePath(asset); let filePath = req.task.getAssetsArchivePath(asset);
if (filePath){ if (filePath) {
if (fs.existsSync(filePath)){ if (fs.existsSync(filePath)) {
res.setHeader('Content-Disposition', `attachment; filename=${asset}`); res.setHeader('Content-Disposition', `attachment; filename=${asset}`);
res.setHeader('Content-Type', mime.lookup(asset)); res.setHeader('Content-Type', mime.lookup(asset));
res.setHeader('Content-Length', fs.statSync(filePath)["size"]); res.setHeader('Content-Length', fs.statSync(filePath)["size"]);
const filestream = fs.createReadStream(filePath); const filestream = fs.createReadStream(filePath);
filestream.pipe(res); filestream.pipe(res);
}else{ } else {
res.json({error: "Asset not ready"}); res.json({ error: "Asset not ready" });
} }
}else{ } else {
res.json({error: "Invalid asset"}); res.json({ error: "Invalid asset" });
} }
}); });
/** @swagger /** @swagger
* definition: * definition:
* Error: * Error:
* type: object * type: object
* required: * required:
* - error * - error
* properties: * properties:
* error: * error:
* type: string * type: string
* description: Description of the error * description: Description of the error
* Response: * Response:
* type: object * type: object
* required: * required:
* - success * - success
* properties: * properties:
* success: * success:
* type: boolean * type: boolean
* description: true if the command succeeded, false otherwise * description: true if the command succeeded, false otherwise
* error: * error:
* type: string * type: string
* description: Error message if an error occured * description: Error message if an error occured
*/ */
let uuidCheck = (req, res, next) => { let uuidCheck = (req, res, next) => {
if (!req.body.uuid) res.json({error: "uuid param missing."}); if (!req.body.uuid) res.json({ error: "uuid param missing." });
else next(); else next();
}; };
let successHandler = res => { let successHandler = res => {
return err => { return err => {
if (!err) res.json({success: true}); if (!err) res.json({ success: true });
else res.json({success: false, error: err.message}); else res.json({ success: false, error: err.message });
}; };
}; };
/** @swagger /** @swagger
* /task/cancel: * /task/cancel:
* post: * post:
* description: Cancels a task (stops its execution, or prevents it from being executed) * description: Cancels a task (stops its execution, or prevents it from being executed)
* parameters: * parameters:
* - * -
* name: uuid * name: uuid
* in: body * in: body
* description: UUID of the task * description: UUID of the task
* required: true * required: true
* schema: * schema:
* type: string * type: string
* responses: * responses:
* 200: * 200:
* description: Command Received * description: Command Received
* schema: * schema:
* $ref: "#/definitions/Response" * $ref: "#/definitions/Response"
*/ */
app.post('/task/cancel', uuidCheck, (req, res) => { app.post('/task/cancel', uuidCheck, (req, res) => {
taskManager.cancel(req.body.uuid, successHandler(res)); taskManager.cancel(req.body.uuid, successHandler(res));
}); });
/** @swagger /** @swagger
* /task/remove: * /task/remove:
* post: * post:
* description: Removes a task and deletes all of its assets * description: Removes a task and deletes all of its assets
* parameters: * parameters:
* - * -
* name: uuid * name: uuid
* in: body * in: body
* description: UUID of the task * description: UUID of the task
* required: true * required: true
* schema: * schema:
* type: string * type: string
* responses: * responses:
* 200: * 200:
* description: Command Received * description: Command Received
* schema: * schema:
* $ref: "#/definitions/Response" * $ref: "#/definitions/Response"
*/ */
app.post('/task/remove', uuidCheck, (req, res) => { app.post('/task/remove', uuidCheck, (req, res) => {
taskManager.remove(req.body.uuid, successHandler(res)); taskManager.remove(req.body.uuid, successHandler(res));
}); });
/** @swagger /** @swagger
* /task/restart: * /task/restart:
* post: * post:
* description: Restarts a task that was previously canceled or that had failed to process * description: Restarts a task that was previously canceled or that had failed to process
* parameters: * parameters:
* - * -
* name: uuid * name: uuid
* in: body * in: body
* description: UUID of the task * description: UUID of the task
* required: true * required: true
* schema: * schema:
* type: string * type: string
* responses: * responses:
* 200: * 200:
* description: Command Received * description: Command Received
* schema: * schema:
* $ref: "#/definitions/Response" * $ref: "#/definitions/Response"
*/ */
app.post('/task/restart', uuidCheck, (req, res) => { app.post('/task/restart', uuidCheck, (req, res) => {
taskManager.restart(req.body.uuid, successHandler(res)); taskManager.restart(req.body.uuid, successHandler(res));
}); });
/** @swagger /** @swagger
* /options: * /options:
* get: * get:
* description: Retrieves the command line options that can be passed to process a task * description: Retrieves the command line options that can be passed to process a task
* tags: [server] * tags: [server]
* responses: * responses:
* 200: * 200:
* description: Options * description: Options
* schema: * schema:
* type: array * type: array
* items: * items:
* title: Option * title: Option
* type: object * type: object
* required: [name, type, value, domain, help] * required: [name, type, value, domain, help]
* properties: * properties:
* name: * name:
* type: string * type: string
* description: Command line option (exactly as it is passed to the OpenDroneMap process, minus the leading '--') * description: Command line option (exactly as it is passed to the OpenDroneMap process, minus the leading '--')
* type: * type:
* type: string * type: string
* description: Datatype of the value of this option * description: Datatype of the value of this option
* enum: * enum:
* - int * - int
* - float * - float
* - string * - string
* - bool * - bool
* value: * value:
* type: string * type: string
* description: Default value of this option * description: Default value of this option
* domain: * domain:
* type: string * type: string
* description: Valid range of values (for example, "positive integer" or "float > 0.0") * description: Valid range of values (for example, "positive integer" or "float > 0.0")
* help: * help:
* type: string * type: string
* description: Description of what this option does * description: Description of what this option does
*/ */
app.get('/options', (req, res) => { app.get('/options', (req, res) => {
odmOptions.getOptions((err, options) => { odmOptions.getOptions((err, options) => {
if (err) res.json({error: err.message}); if (err) res.json({ error: err.message });
else res.json(options); else res.json(options);
}); });
}); });
/** @swagger /** @swagger
* /info: * /info:
* get: * get:
* description: Retrieves information about this node * description: Retrieves information about this node
* tags: [server] * tags: [server]
* responses: * responses:
* 200: * 200:
* description: Info * description: Info
* schema: * schema:
* type: object * type: object
* required: [version, taskQueueCount] * required: [version, taskQueueCount]
* properties: * properties:
* version: * version:
* type: string * type: string
* description: Current version * description: Current version
* taskQueueCount: * taskQueueCount:
* type: integer * type: integer
* description: Number of tasks currently being processed or waiting to be processed * description: Number of tasks currently being processed or waiting to be processed
*/ */
app.get('/info', (req, res) => { app.get('/info', (req, res) => {
res.json({ res.json({
version: packageJson.version, version: packageJson.version,
@ -495,46 +497,47 @@ app.get('/info', (req, res) => {
}); });
let gracefulShutdown = done => { let gracefulShutdown = done => {
async.series([ async.series([
cb => taskManager.dumpTaskList(cb), cb => taskManager.dumpTaskList(cb),
cb => { cb => {
logger.info("Closing server"); logger.info("Closing server");
server.close(); server.close();
logger.info("Exiting..."); logger.info("Exiting...");
process.exit(0); process.exit(0);
} }
], done); ], done);
}; };
// listen for TERM signal .e.g. kill // listen for TERM signal .e.g. kill
process.on ('SIGTERM', gracefulShutdown); process.on('SIGTERM', gracefulShutdown);
// listen for INT signal e.g. Ctrl-C // listen for INT signal e.g. Ctrl-C
process.on ('SIGINT', gracefulShutdown); process.on('SIGINT', gracefulShutdown);
// Startup // Startup
if (config.test) logger.info("Running in test mode"); if (config.test) logger.info("Running in test mode");
let commands = [ let commands = [
cb => odmOptions.initialize(cb), cb => odmOptions.initialize(cb),
cb => { taskManager = new TaskManager(cb); }, cb => { taskManager = new TaskManager(cb); },
cb => { server = app.listen(config.port, err => { cb => {
if (!err) logger.info('Server has started on port ' + String(config.port)); server = app.listen(config.port, err => {
cb(err); if (!err) logger.info('Server has started on port ' + String(config.port));
}); cb(err);
} });
}
]; ];
if (config.powercycle){ if (config.powercycle) {
commands.push(cb => { commands.push(cb => {
logger.info("Power cycling is set, application will shut down..."); logger.info("Power cycling is set, application will shut down...");
process.exit(0); process.exit(0);
}); });
} }
async.series(commands, err => { async.series(commands, err => {
if (err){ if (err) {
logger.error("Error during startup: " + err.message); logger.error("Error during startup: " + err.message);
process.exit(1); process.exit(1);
} }
}); });

788
libs/Task.js
Wyświetl plik

@ -33,449 +33,453 @@ let Directories = require('./Directories');
let statusCodes = require('./statusCodes'); let statusCodes = require('./statusCodes');
module.exports = class Task{ module.exports = class Task {
constructor(uuid, name, done, options = []){ constructor(uuid, name, done, options = []) {
assert(uuid !== undefined, "uuid must be set"); assert(uuid !== undefined, "uuid must be set");
assert(done !== undefined, "ready must be set"); assert(done !== undefined, "ready must be set");
this.uuid = uuid; this.uuid = uuid;
this.name = name !== "" ? name : "Task of " + (new Date()).toISOString(); this.name = name !== "" ? name : "Task of " + (new Date()).toISOString();
this.dateCreated = new Date().getTime(); this.dateCreated = new Date().getTime();
this.processingTime = -1; this.processingTime = -1;
this.setStatus(statusCodes.QUEUED); this.setStatus(statusCodes.QUEUED);
this.options = options; this.options = options;
this.gpcFiles = []; this.gpcFiles = [];
this.output = []; this.output = [];
this.runningProcesses = []; this.runningProcesses = [];
async.series([ async.series([
// Read images info // Read images info
cb => { cb => {
fs.readdir(this.getImagesFolderPath(), (err, files) => { fs.readdir(this.getImagesFolderPath(), (err, files) => {
if (err) cb(err); if (err) cb(err);
else{ else {
this.images = files; this.images = files;
logger.debug(`Found ${this.images.length} images for ${this.uuid}`); logger.debug(`Found ${this.images.length} images for ${this.uuid}`);
cb(null); cb(null);
} }
}); });
}, },
// Find GCP (if any) // Find GCP (if any)
cb => { cb => {
fs.readdir(this.getGpcFolderPath(), (err, files) => { fs.readdir(this.getGpcFolderPath(), (err, files) => {
if (err) cb(err); if (err) cb(err);
else{ else {
files.forEach(file => { files.forEach(file => {
if (/\.txt$/gi.test(file)){ if (/\.txt$/gi.test(file)) {
this.gpcFiles.push(file); this.gpcFiles.push(file);
} }
}); });
logger.debug(`Found ${this.gpcFiles.length} GPC files (${this.gpcFiles.join(" ")}) for ${this.uuid}`); logger.debug(`Found ${this.gpcFiles.length} GPC files (${this.gpcFiles.join(" ")}) for ${this.uuid}`);
cb(null); cb(null);
} }
}); });
} }
], err => { ], err => {
done(err, this); done(err, this);
}); });
} }
static CreateFromSerialized(taskJson, done){ static CreateFromSerialized(taskJson, done) {
new Task(taskJson.uuid, taskJson.name, (err, task) => { new Task(taskJson.uuid, taskJson.name, (err, task) => {
if (err) done(err); if (err) done(err);
else{ else {
// Override default values with those // Override default values with those
// provided in the taskJson // provided in the taskJson
for (let k in taskJson){ for (let k in taskJson) {
task[k] = taskJson[k]; task[k] = taskJson[k];
} }
// Tasks that were running should be put back to QUEUED state // Tasks that were running should be put back to QUEUED state
if (task.status.code === statusCodes.RUNNING){ if (task.status.code === statusCodes.RUNNING) {
task.status.code = statusCodes.QUEUED; task.status.code = statusCodes.QUEUED;
} }
done(null, task); done(null, task);
} }
}, taskJson.options); }, taskJson.options);
} }
// Get path where images are stored for this task // Get path where images are stored for this task
// (relative to nodejs process CWD) // (relative to nodejs process CWD)
getImagesFolderPath(){ getImagesFolderPath() {
return path.join(this.getProjectFolderPath(), "images"); return path.join(this.getProjectFolderPath(), "images");
} }
// Get path where GPC file(s) are stored // Get path where GPC file(s) are stored
// (relative to nodejs process CWD) // (relative to nodejs process CWD)
getGpcFolderPath(){ getGpcFolderPath() {
return path.join(this.getProjectFolderPath(), "gpc"); return path.join(this.getProjectFolderPath(), "gpc");
} }
// Get path of project (where all images and assets folder are contained) // Get path of project (where all images and assets folder are contained)
// (relative to nodejs process CWD) // (relative to nodejs process CWD)
getProjectFolderPath(){ getProjectFolderPath() {
return path.join(Directories.data, this.uuid); return path.join(Directories.data, this.uuid);
} }
// Get the path of the archive where all assets // Get the path of the archive where all assets
// outputted by this task are stored. // outputted by this task are stored.
getAssetsArchivePath(filename){ getAssetsArchivePath(filename) {
if (filename == 'all.zip'){ if (filename == 'all.zip') {
// OK, do nothing // OK, do nothing
}else if (filename == 'orthophoto.tif'){ } else if (filename == 'orthophoto.tif') {
if (config.test){ if (config.test) {
if (config.testSkipOrthophotos) return false; if (config.testSkipOrthophotos) return false;
else filename = path.join('..', '..', 'processing_results', 'odm_orthophoto', `odm_${filename}`); else filename = path.join('..', '..', 'processing_results', 'odm_orthophoto', `odm_${filename}`);
}else{ } else {
filename = path.join('odm_orthophoto', `odm_${filename}`); filename = path.join('odm_orthophoto', `odm_${filename}`);
} }
}else{ } else {
return false; // Invalid return false; // Invalid
} }
return path.join(this.getProjectFolderPath(), filename);
}
// Deletes files and folders related to this task return path.join(this.getProjectFolderPath(), filename);
cleanup(cb){ }
rmdir(this.getProjectFolderPath(), cb);
}
setStatus(code, extra){ // Deletes files and folders related to this task
this.status = { cleanup(cb) {
code: code rmdir(this.getProjectFolderPath(), cb);
}; }
for (let k in extra){
this.status[k] = extra[k];
}
}
updateProcessingTime(resetTime){ setStatus(code, extra) {
this.processingTime = resetTime ? this.status = {
-1 : code: code
new Date().getTime() - this.dateCreated; };
} for (let k in extra) {
this.status[k] = extra[k];
}
}
startTrackingProcessingTime(){ updateProcessingTime(resetTime) {
this.updateProcessingTime(); this.processingTime = resetTime ?
if (!this._updateProcessingTimeInterval){ -1 :
this._updateProcessingTimeInterval = setInterval(() => { new Date().getTime() - this.dateCreated;
this.updateProcessingTime(); }
}, 1000);
}
}
stopTrackingProcessingTime(resetTime){ startTrackingProcessingTime() {
this.updateProcessingTime(resetTime); this.updateProcessingTime();
if (this._updateProcessingTimeInterval){ if (!this._updateProcessingTimeInterval) {
clearInterval(this._updateProcessingTimeInterval); this._updateProcessingTimeInterval = setInterval(() => {
this._updateProcessingTimeInterval = null; this.updateProcessingTime();
} }, 1000);
} }
}
getStatus(){ stopTrackingProcessingTime(resetTime) {
return this.status.code; this.updateProcessingTime(resetTime);
} if (this._updateProcessingTimeInterval) {
clearInterval(this._updateProcessingTimeInterval);
this._updateProcessingTimeInterval = null;
}
}
isCanceled(){ getStatus() {
return this.status.code === statusCodes.CANCELED; return this.status.code;
} }
// Cancels the current task (unless it's already canceled) isCanceled() {
cancel(cb){ return this.status.code === statusCodes.CANCELED;
if (this.status.code !== statusCodes.CANCELED){ }
let wasRunning = this.status.code === statusCodes.RUNNING;
this.setStatus(statusCodes.CANCELED);
if (wasRunning){ // Cancels the current task (unless it's already canceled)
this.runningProcesses.forEach(proc => { cancel(cb) {
// TODO: this does NOT guarantee that if (this.status.code !== statusCodes.CANCELED) {
// the process will immediately terminate. let wasRunning = this.status.code === statusCodes.RUNNING;
// For eaxmple in the case of the ODM process, the process will continue running for a while this.setStatus(statusCodes.CANCELED);
// This might need to be fixed on ODM's end.
proc.kill('SIGINT');
});
this.runningProcesses = [];
}
this.stopTrackingProcessingTime(true); if (wasRunning) {
cb(null); this.runningProcesses.forEach(proc => {
}else{ // TODO: this does NOT guarantee that
cb(new Error("Task already cancelled")); // the process will immediately terminate.
} // For eaxmple in the case of the ODM process, the process will continue running for a while
} // This might need to be fixed on ODM's end.
proc.kill('SIGINT');
});
this.runningProcesses = [];
}
// Starts processing the task with OpenDroneMap this.stopTrackingProcessingTime(true);
// This will spawn a new process. cb(null);
start(done){ } else {
const finished = err => { cb(new Error("Task already cancelled"));
this.stopTrackingProcessingTime(); }
done(err); }
};
const postProcess = () => {
const createZipArchive = (outputFilename, files) => {
return (done) => {
this.output.push(`Compressing ${outputFilename}\n`);
let output = fs.createWriteStream(this.getAssetsArchivePath(outputFilename)); // Starts processing the task with OpenDroneMap
let archive = archiver.create('zip', {}); // This will spawn a new process.
start(done) {
const finished = err => {
this.stopTrackingProcessingTime();
done(err);
};
archive.on('finish', () => { const postProcess = () => {
// TODO: is this being fired twice? const createZipArchive = (outputFilename, files) => {
done(); return (done) => {
}); this.output.push(`Compressing ${outputFilename}\n`);
archive.on('error', err => { let output = fs.createWriteStream(this.getAssetsArchivePath(outputFilename));
logger.error(`Could not archive .zip file: ${err.message}`); let archive = archiver.create('zip', {});
done(err);
});
archive.pipe(output); archive.on('finish', () => {
let globs = []; // TODO: is this being fired twice?
done();
});
// Process files and directories first archive.on('error', err => {
files.forEach(file => { logger.error(`Could not archive .zip file: ${err.message}`);
let sourcePath = !config.test ? done(err);
this.getProjectFolderPath() : });
path.join("tests", "processing_results");
let filePath = path.join(sourcePath, file);
// Skip non-existing items
if (!fs.existsSync(filePath)) return;
let isGlob = /\*/.test(file), archive.pipe(output);
isDirectory = !isGlob && fs.lstatSync(filePath).isDirectory(); let globs = [];
if (isDirectory){ // Process files and directories first
archive.directory(filePath, file); files.forEach(file => {
}else if (isGlob){ let sourcePath = !config.test ?
globs.push(filePath); this.getProjectFolderPath() :
}else{ path.join("tests", "processing_results");
archive.file(filePath, {name: path.basename(file)}); let filePath = path.join(sourcePath, file);
}
});
// Check for globs // Skip non-existing items
if (globs.length !== 0){ if (!fs.existsSync(filePath)) return;
let pending = globs.length;
globs.forEach(pattern => { let isGlob = /\*/.test(file),
glob(pattern, (err, files) => { isDirectory = !isGlob && fs.lstatSync(filePath).isDirectory();
if (err) done(err);
else{
files.forEach(file => {
if (fs.lstatSync(file).isFile()){
archive.file(file, {name: path.basename(file)});
}else{
logger.debug(`Could not add ${file} from glob`);
}
});
if (--pending === 0){ if (isDirectory) {
archive.finalize(); archive.directory(filePath, file);
} } else if (isGlob) {
} globs.push(filePath);
}); } else {
}); archive.file(filePath, { name: path.basename(file) });
}else{ }
archive.finalize(); });
}
};
};
const handleProcessExit = (done) => { // Check for globs
return (err, code, signal) => { if (globs.length !== 0) {
if (err) done(err); let pending = globs.length;
else{
// Don't evaluate if we caused the process to exit via SIGINT?
if (code === 0) done();
else done(new Error(`Process exited with code ${code}`));
}
};
};
const handleOutput = output => { globs.forEach(pattern => {
this.output.push(output); glob(pattern, (err, files) => {
}; if (err) done(err);
else {
files.forEach(file => {
if (fs.lstatSync(file).isFile()) {
archive.file(file, { name: path.basename(file) });
} else {
logger.debug(`Could not add ${file} from glob`);
}
});
const generateTiles = (inputFile, outputDir) => { if (--pending === 0) {
return (done) => { archive.finalize();
const inputFilePath = path.join(this.getProjectFolderPath(), inputFile); }
}
// Not all datasets generate an orthophoto, so we skip });
// tiling if the orthophoto is missing });
if (fs.existsSync(inputFilePath)){ } else {
this.runningProcesses.push(processRunner.runTiler({ archive.finalize();
zoomLevels: "12-21", }
inputFile: inputFilePath, };
outputDir: path.join(this.getProjectFolderPath(), outputDir) };
}, handleProcessExit(done), handleOutput));
}else{
handleOutput(`${inputFilePath} file not found, skipping tiles generation\n`);
done();
}
};
};
const generatePotreeCloud = (inputFile, outputDir) => { const handleProcessExit = (done) => {
return (done) => { return (err, code, signal) => {
this.runningProcesses.push(processRunner.runPotreeConverter({ if (err) done(err);
inputFile: path.join(this.getProjectFolderPath(), inputFile), else {
outputDir: path.join(this.getProjectFolderPath(), outputDir) // Don't evaluate if we caused the process to exit via SIGINT?
}, handleProcessExit(done), handleOutput)); if (code === 0) done();
}; else done(new Error(`Process exited with code ${code}`));
}; }
};
};
const pdalTranslate = (inputPath, outputPath, filters) => { const handleOutput = output => {
return (done) => { this.output.push(output);
this.runningProcesses.push(processRunner.runPdalTranslate({ };
inputFile: inputPath,
outputFile: outputPath,
filters: filters
}, handleProcessExit(done), handleOutput));
};
};
// All paths are relative to the project directory (./data/<uuid>/) const generateTiles = (inputFile, outputDir) => {
let allFolders = ['odm_orthophoto', 'odm_georeferencing', 'odm_texturing', 'odm_meshing', 'orthophoto_tiles', 'potree_pointcloud']; return (done) => {
const inputFilePath = path.join(this.getProjectFolderPath(), inputFile);
if (config.test && config.testSkipOrthophotos){
logger.info("Test mode will skip orthophoto generation");
// Exclude these folders from the all.zip archive // Not all datasets generate an orthophoto, so we skip
['odm_orthophoto', 'orthophoto_tiles'].forEach(dir => { // tiling if the orthophoto is missing
allFolders.splice(allFolders.indexOf(dir), 1); if (fs.existsSync(inputFilePath)) {
}); this.runningProcesses.push(processRunner.runTiler({
} zoomLevels: "12-21",
inputFile: inputFilePath,
outputDir: path.join(this.getProjectFolderPath(), outputDir)
}, handleProcessExit(done), handleOutput));
} else {
handleOutput(`${inputFilePath} file not found, skipping tiles generation\n`);
done();
}
};
};
let orthophotoPath = path.join('odm_orthophoto', 'odm_orthophoto.tif'), const generatePotreeCloud = (inputFile, outputDir) => {
lasPointCloudPath = path.join('odm_georeferencing', 'odm_georeferenced_model.ply.las'), return (done) => {
projectFolderPath = this.getProjectFolderPath(); this.runningProcesses.push(processRunner.runPotreeConverter({
inputFile: path.join(this.getProjectFolderPath(), inputFile),
outputDir: path.join(this.getProjectFolderPath(), outputDir)
}, handleProcessExit(done), handleOutput));
};
};
let commands = [ const pdalTranslate = (inputPath, outputPath, filters) => {
return (done) => {
this.runningProcesses.push(processRunner.runPdalTranslate({
inputFile: inputPath,
outputFile: outputPath,
filters: filters
}, handleProcessExit(done), handleOutput));
};
};
// All paths are relative to the project directory (./data/<uuid>/)
let allFolders = ['odm_orthophoto', 'odm_georeferencing', 'odm_texturing', 'odm_meshing', 'orthophoto_tiles', 'potree_pointcloud'];
if (config.test && config.testSkipOrthophotos) {
logger.info("Test mode will skip orthophoto generation");
// Exclude these folders from the all.zip archive
['odm_orthophoto', 'orthophoto_tiles'].forEach(dir => {
allFolders.splice(allFolders.indexOf(dir), 1);
});
}
let orthophotoPath = path.join('odm_orthophoto', 'odm_orthophoto.tif'),
lasPointCloudPath = path.join('odm_georeferencing', 'odm_georeferenced_model.ply.las'),
projectFolderPath = this.getProjectFolderPath();
let commands = [
generateTiles(orthophotoPath, 'orthophoto_tiles'), generateTiles(orthophotoPath, 'orthophoto_tiles'),
generatePotreeCloud(lasPointCloudPath, 'potree_pointcloud'), generatePotreeCloud(lasPointCloudPath, 'potree_pointcloud'),
createZipArchive('all.zip', allFolders) createZipArchive('all.zip', allFolders)
]; ];
// If point cloud file does not exist, it's likely because location (GPS/GPC) information // If point cloud file does not exist, it's likely because location (GPS/GPC) information
// was missing and the file was not generated. // was missing and the file was not generated.
let fullLasPointCloudPath = path.join(projectFolderPath, lasPointCloudPath); let fullLasPointCloudPath = path.join(projectFolderPath, lasPointCloudPath);
if (!fs.existsSync(fullLasPointCloudPath)){ if (!fs.existsSync(fullLasPointCloudPath)) {
let unreferencedPointCloudPath = path.join(projectFolderPath, "opensfm", "depthmaps", "merged.ply"); let unreferencedPointCloudPath = path.join(projectFolderPath, "opensfm", "depthmaps", "merged.ply");
if (fs.existsSync(unreferencedPointCloudPath)){ if (fs.existsSync(unreferencedPointCloudPath)) {
logger.info(`${lasPointCloudPath} is missing, will attempt to generate it from ${unreferencedPointCloudPath}`); logger.info(`${lasPointCloudPath} is missing, will attempt to generate it from ${unreferencedPointCloudPath}`);
commands.unshift(pdalTranslate(unreferencedPointCloudPath, fullLasPointCloudPath, [ commands.unshift(pdalTranslate(unreferencedPointCloudPath, fullLasPointCloudPath, [{
{ // opensfm's ply files map colors with the diffuse_ prefix
// opensfm's ply files map colors with the diffuse_ prefix dimensions: "diffuse_red = red, diffuse_green = green, diffuse_blue = blue",
dimensions: "diffuse_red = red, diffuse_green = green, diffuse_blue = blue", type: "filters.ferry"
type: "filters.ferry" }]));
} }
])); }
}
}
async.series(commands, (err) => { async.series(commands, (err) => {
if (!err){ if (!err) {
this.setStatus(statusCodes.COMPLETED); this.setStatus(statusCodes.COMPLETED);
finished(); finished();
}else{ } else {
this.setStatus(statusCodes.FAILED); this.setStatus(statusCodes.FAILED);
finished(err); finished(err);
} }
}); });
}; };
if (this.status.code === statusCodes.QUEUED){ if (this.status.code === statusCodes.QUEUED) {
this.startTrackingProcessingTime(); this.startTrackingProcessingTime();
this.setStatus(statusCodes.RUNNING); this.setStatus(statusCodes.RUNNING);
// rmeove webhoook
var optionsToRun = this.options.filter(function(opt) {
if (opt.name !== 'webhook') {
return opt;
}
let runnerOptions = this.options.reduce((result, opt) => { });
result[opt.name] = opt.value;
return result;
}, {});
runnerOptions["project-path"] = fs.realpathSync(Directories.data); let runnerOptions = optionsToRun.reduce((result, opt) => {
runnerOptions["pmvs-num-cores"] = os.cpus().length; result[opt.name] = opt.value;
return result;
}, {});
if (this.gpcFiles.length > 0){ runnerOptions["project-path"] = fs.realpathSync(Directories.data);
runnerOptions.gcp = fs.realpathSync(path.join(this.getGpcFolderPath(), this.gpcFiles[0])); runnerOptions["pmvs-num-cores"] = os.cpus().length;
}
this.runningProcesses.push(odmRunner.run(runnerOptions, this.uuid, (err, code, signal) => { if (this.gpcFiles.length > 0) {
if (err){ runnerOptions.gcp = fs.realpathSync(path.join(this.getGpcFolderPath(), this.gpcFiles[0]));
this.setStatus(statusCodes.FAILED, {errorMessage: `Could not start process (${err.message})`}); }
finished(err);
}else{
// Don't evaluate if we caused the process to exit via SIGINT?
if (this.status.code !== statusCodes.CANCELED){
if (code === 0){
postProcess();
}else{
this.setStatus(statusCodes.FAILED, {errorMessage: `Process exited with code ${code}`});
finished();
}
}else{
finished();
}
}
}, output => {
// Replace console colors
output = output.replace(/\x1b\[[0-9;]*m/g, "");
this.output.push(output);
})
);
return true; this.runningProcesses.push(odmRunner.run(runnerOptions, this.uuid, (err, code, signal) => {
}else{ if (err) {
return false; this.setStatus(statusCodes.FAILED, { errorMessage: `Could not start process (${err.message})` });
} finished(err);
} } else {
// Don't evaluate if we caused the process to exit via SIGINT?
if (this.status.code !== statusCodes.CANCELED) {
if (code === 0) {
postProcess();
} else {
this.setStatus(statusCodes.FAILED, { errorMessage: `Process exited with code ${code}` });
finished();
}
} else {
finished();
}
}
}, output => {
// Replace console colors
output = output.replace(/\x1b\[[0-9;]*m/g, "");
this.output.push(output);
}));
// Re-executes the task (by setting it's state back to QUEUED) return true;
// Only tasks that have been canceled, completed or have failed can be restarted. } else {
restart(cb){ return false;
if ([statusCodes.CANCELED, statusCodes.FAILED, statusCodes.COMPLETED].indexOf(this.status.code) !== -1){ }
this.setStatus(statusCodes.QUEUED); }
this.dateCreated = new Date().getTime();
this.output = [];
this.stopTrackingProcessingTime(true);
cb(null);
}else{
cb(new Error("Task cannot be restarted"));
}
}
// Returns the description of the task. // Re-executes the task (by setting it's state back to QUEUED)
getInfo(){ // Only tasks that have been canceled, completed or have failed can be restarted.
return { restart(cb) {
uuid: this.uuid, if ([statusCodes.CANCELED, statusCodes.FAILED, statusCodes.COMPLETED].indexOf(this.status.code) !== -1) {
name: this.name, this.setStatus(statusCodes.QUEUED);
dateCreated: this.dateCreated, this.dateCreated = new Date().getTime();
processingTime: this.processingTime, this.output = [];
status: this.status, this.stopTrackingProcessingTime(true);
options: this.options, cb(null);
imagesCount: this.images.length } else {
}; cb(new Error("Task cannot be restarted"));
} }
}
// Returns the output of the OpenDroneMap process // Returns the description of the task.
// Optionally starting from a certain line number getInfo() {
getOutput(startFromLine = 0){ return {
return this.output.slice(startFromLine, this.output.length); uuid: this.uuid,
} name: this.name,
dateCreated: this.dateCreated,
processingTime: this.processingTime,
status: this.status,
options: this.options,
imagesCount: this.images.length
};
}
// Returns the data necessary to serialize this // Returns the output of the OpenDroneMap process
// task to restore it later. // Optionally starting from a certain line number
serialize(){ getOutput(startFromLine = 0) {
return { return this.output.slice(startFromLine, this.output.length);
uuid: this.uuid, }
name: this.name,
dateCreated: this.dateCreated, // Returns the data necessary to serialize this
status: this.status, // task to restore it later.
options: this.options serialize() {
}; return {
} uuid: this.uuid,
}; name: this.name,
dateCreated: this.dateCreated,
status: this.status,
options: this.options
};
}
};

471
libs/TaskManager.js
Wyświetl plik

@ -27,237 +27,316 @@ let statusCodes = require('./statusCodes');
let async = require('async'); let async = require('async');
let schedule = require('node-schedule'); let schedule = require('node-schedule');
let Directories = require('./Directories'); let Directories = require('./Directories');
// webhook reqs
let request = require('request');
const TASKS_DUMP_FILE = path.join(Directories.data, "tasks.json"); const TASKS_DUMP_FILE = path.join(Directories.data, "tasks.json");
const CLEANUP_TASKS_IF_OLDER_THAN = 1000 * 60 * 60 * 24 * config.cleanupTasksAfter; // days const CLEANUP_TASKS_IF_OLDER_THAN = 1000 * 60 * 60 * 24 * config.cleanupTasksAfter; // days
module.exports = class TaskManager{ module.exports = class TaskManager {
constructor(done){ constructor(done) {
this.tasks = {}; this.tasks = {};
this.runningQueue = []; this.runningQueue = [];
async.series([ async.series([
cb => this.restoreTaskListFromDump(cb), cb => this.restoreTaskListFromDump(cb),
cb => this.removeOldTasks(cb), cb => this.removeOldTasks(cb),
cb => this.removeOrphanedDirectories(cb), cb => this.removeOrphanedDirectories(cb),
cb => { cb => {
this.processNextTask(); this.processNextTask();
cb(); cb();
}, },
cb => { cb => {
// Every hour // Every hour
schedule.scheduleJob('0 * * * *', () => { schedule.scheduleJob('0 * * * *', () => {
this.removeOldTasks(); this.removeOldTasks();
this.dumpTaskList(); this.dumpTaskList();
}); });
cb(); cb();
} }
], done); ], done);
} }
// Removes old tasks that have either failed, are completed, or // Removes old tasks that have either failed, are completed, or
// have been canceled. // have been canceled.
removeOldTasks(done){ removeOldTasks(done) {
let list = []; let list = [];
let now = new Date().getTime(); let now = new Date().getTime();
logger.debug("Checking for old tasks to be removed..."); logger.debug("Checking for old tasks to be removed...");
for (let uuid in this.tasks){ for (let uuid in this.tasks) {
let task = this.tasks[uuid]; let task = this.tasks[uuid];
if ([statusCodes.FAILED, if ([statusCodes.FAILED,
statusCodes.COMPLETED, statusCodes.COMPLETED,
statusCodes.CANCELED].indexOf(task.status.code) !== -1 && statusCodes.CANCELED
now - task.dateCreated > CLEANUP_TASKS_IF_OLDER_THAN){ ].indexOf(task.status.code) !== -1 &&
list.push(task.uuid); now - task.dateCreated > CLEANUP_TASKS_IF_OLDER_THAN) {
} list.push(task.uuid);
} }
}
async.eachSeries(list, (uuid, cb) => { async.eachSeries(list, (uuid, cb) => {
logger.info(`Cleaning up old task ${uuid}`); logger.info(`Cleaning up old task ${uuid}`);
this.remove(uuid, cb); this.remove(uuid, cb);
}, done); }, done);
} }
// Removes directories that don't have a corresponding // Removes directories that don't have a corresponding
// task associated with it (maybe as a cause of an abrupt exit) // task associated with it (maybe as a cause of an abrupt exit)
removeOrphanedDirectories(done){ removeOrphanedDirectories(done) {
logger.info("Checking for orphaned directories to be removed..."); logger.info("Checking for orphaned directories to be removed...");
fs.readdir(Directories.data, (err, entries) => { fs.readdir(Directories.data, (err, entries) => {
if (err) done(err); if (err) done(err);
else{ else {
async.eachSeries(entries, (entry, cb) => { async.eachSeries(entries, (entry, cb) => {
let dirPath = path.join(Directories.data, entry); let dirPath = path.join(Directories.data, entry);
if (fs.statSync(dirPath).isDirectory() && if (fs.statSync(dirPath).isDirectory() &&
entry.match(/^[\w\d]+\-[\w\d]+\-[\w\d]+\-[\w\d]+\-[\w\d]+$/) && entry.match(/^[\w\d]+\-[\w\d]+\-[\w\d]+\-[\w\d]+\-[\w\d]+$/) &&
!this.tasks[entry]){ !this.tasks[entry]) {
logger.info(`Found orphaned directory: ${entry}, removing...`); logger.info(`Found orphaned directory: ${entry}, removing...`);
rmdir(dirPath, cb); rmdir(dirPath, cb);
}else cb(); } else cb();
}, done); }, done);
} }
}); });
} }
// Load tasks that already exists (if any) // Load tasks that already exists (if any)
restoreTaskListFromDump(done){ restoreTaskListFromDump(done) {
fs.readFile(TASKS_DUMP_FILE, (err, data) => { fs.readFile(TASKS_DUMP_FILE, (err, data) => {
if (!err){ if (!err) {
let tasks; let tasks;
try{ try {
tasks = JSON.parse(data.toString()); tasks = JSON.parse(data.toString());
}catch(e){ } catch (e) {
done(new Error(`Could not load task list. It looks like the ${TASKS_DUMP_FILE} is corrupted (${e.message}). Please manually delete the file and try again.`)); done(new Error(`Could not load task list. It looks like the ${TASKS_DUMP_FILE} is corrupted (${e.message}). Please manually delete the file and try again.`));
return; return;
} }
async.each(tasks, (taskJson, done) => { async.each(tasks, (taskJson, done) => {
Task.CreateFromSerialized(taskJson, (err, task) => { Task.CreateFromSerialized(taskJson, (err, task) => {
if (err) done(err); if (err) done(err);
else{ else {
this.tasks[task.uuid] = task; this.tasks[task.uuid] = task;
done(); done();
} }
}); });
}, err => { }, err => {
logger.info(`Initialized ${tasks.length} tasks`); logger.info(`Initialized ${tasks.length} tasks`);
if (done !== undefined) done(); if (done !== undefined) done();
}); });
}else{ } else {
logger.info("No tasks dump found"); logger.info("No tasks dump found");
if (done !== undefined) done(); if (done !== undefined) done();
} }
}); });
} }
// Finds the first QUEUED task. // Finds the first QUEUED task.
findNextTaskToProcess(){ findNextTaskToProcess() {
for (let uuid in this.tasks){ for (let uuid in this.tasks) {
if (this.tasks[uuid].getStatus() === statusCodes.QUEUED){ if (this.tasks[uuid].getStatus() === statusCodes.QUEUED) {
return this.tasks[uuid]; return this.tasks[uuid];
} }
} }
} }
// Finds the next tasks, adds them to the running queue, // Finds the next tasks, adds them to the running queue,
// and starts the tasks (up to the limit). // and starts the tasks (up to the limit).
processNextTask(){ processNextTask() {
if (this.runningQueue.length < config.parallelQueueProcessing){ if (this.runningQueue.length < config.parallelQueueProcessing) {
let task = this.findNextTaskToProcess(); let task = this.findNextTaskToProcess();
if (task){ if (task) {
this.addToRunningQueue(task); this.addToRunningQueue(task);
task.start(() => { for (var i = 0; i < task.options.length; i++) {
this.removeFromRunningQueue(task); if (task.options[i].name === "webhook") {
this.processNextTask(); // call back webhook
}); request({
url: task.options[i].value,
method: 'POST',
headers: {
'User-Agent': 'node-OpenDroneMap',
'Content-Type': 'application/json'
},
json: task.getInfo(),
}, function(error, response, body) {
if (this.runningQueue.length < config.parallelQueueProcessing) this.processNextTask(); // ignore error handling for now
}
}else{
// Do nothing
}
}
addToRunningQueue(task){ // if (!error && response.statusCode == 200) {
assert(task.constructor.name === "Task", "Must be a Task object");
this.runningQueue.push(task);
}
removeFromRunningQueue(task){
assert(task.constructor.name === "Task", "Must be a Task object");
this.runningQueue = this.runningQueue.filter(t => t !== task);
}
addNew(task){ // }
assert(task.constructor.name === "Task", "Must be a Task object"); })
this.tasks[task.uuid] = task;
this.processNextTask(); }
} }
// Stops the execution of a task task.start(() => {
// (without removing it from the system). for (var i = 0; i < task.options.length; i++) {
cancel(uuid, cb){ if (task.options[i].name === "webhook") {
let task = this.find(uuid, cb); // call back webhook
if (task){ request({
if (!task.isCanceled()){ url: task.options[i].value,
task.cancel(err => { method: 'POST',
this.removeFromRunningQueue(task); headers: {
this.processNextTask(); 'User-Agent': 'node-OpenDroneMap',
cb(err); 'Content-Type': 'application/json'
}); },
}else{ json: task.getInfo(),
cb(null); // Nothing to be done }, function(error, response, body) {
}
}
}
// Removes a task from the system. // ignore error handling for now
// Before being removed, the task is canceled.
remove(uuid, cb){
this.cancel(uuid, err => {
if (!err){
let task = this.find(uuid, cb);
if (task){
task.cleanup(err => {
if (!err){
delete(this.tasks[uuid]);
this.processNextTask();
cb(null);
}else cb(err);
});
}else; // cb is called by find on error
}else cb(err);
});
}
// Restarts (puts back into QUEUED state) // if (!error && response.statusCode == 200) {
// a task that is either in CANCELED or FAILED state.
restart(uuid, cb){
let task = this.find(uuid, cb);
if (task){
task.restart(err => {
if (!err) this.processNextTask();
cb(err);
});
}
}
// Finds a task by its UUID string.
find(uuid, cb){
let task = this.tasks[uuid];
if (!task && cb) cb(new Error(`${uuid} not found`));
return task;
}
// Serializes the list of tasks and saves it // }
// to disk })
dumpTaskList(done){
let output = [];
for (let uuid in this.tasks){ }
output.push(this.tasks[uuid].serialize()); }
}
fs.writeFile(TASKS_DUMP_FILE, JSON.stringify(output), err => {
if (err) logger.error(`Could not dump tasks: ${err.message}`);
else logger.debug("Dumped tasks list.");
if (done !== undefined) done();
});
}
getQueueCount(){ this.removeFromRunningQueue(task);
this.processNextTask();
});
if (this.runningQueue.length < config.parallelQueueProcessing) this.processNextTask();
}
} else {
// Do nothing
}
}
addToRunningQueue(task) {
assert(task.constructor.name === "Task", "Must be a Task object");
this.runningQueue.push(task);
}
removeFromRunningQueue(task) {
assert(task.constructor.name === "Task", "Must be a Task object");
this.runningQueue = this.runningQueue.filter(t => t !== task);
}
addNew(task) {
assert(task.constructor.name === "Task", "Must be a Task object");
this.tasks[task.uuid] = task;
this.processNextTask();
}
// Stops the execution of a task
// (without removing it from the system).
cancel(uuid, cb) {
let task = this.find(uuid, cb);
if (task) {
if (!task.isCanceled()) {
task.cancel(err => {
this.removeFromRunningQueue(task);
this.processNextTask();
for (var i = 0; i < task.options.length; i++) {
if (task.options[i].name === "webhook") {
// call back webhook
request({
url: task.options[i].value,
method: 'POST',
headers: {
'User-Agent': 'node-OpenDroneMap',
'Content-Type': 'application/json'
},
json: task.getInfo(),
}, function(error, response, body) {
// ignore error handling for now
// if (!error && response.statusCode == 200) {
// }
})
}
}
cb(err);
});
} else {
cb(null); // Nothing to be done
}
}
}
// Removes a task from the system.
// Before being removed, the task is canceled.
remove(uuid, cb) {
this.cancel(uuid, err => {
if (!err) {
let task = this.find(uuid, cb);
if (task) {
task.cleanup(err => {
if (!err) {
delete(this.tasks[uuid]);
this.processNextTask();
cb(null);
} else cb(err);
});
} else; // cb is called by find on error
} else cb(err);
});
}
// Restarts (puts back into QUEUED state)
// a task that is either in CANCELED or FAILED state.
restart(uuid, cb) {
let task = this.find(uuid, cb);
if (task) {
task.restart(err => {
if (!err) this.processNextTask();
cb(err);
});
}
}
// Finds a task by its UUID string.
find(uuid, cb) {
let task = this.tasks[uuid];
if (!task && cb) cb(new Error(`${uuid} not found`));
return task;
}
// Serializes the list of tasks and saves it
// to disk
dumpTaskList(done) {
let output = [];
for (let uuid in this.tasks) {
output.push(this.tasks[uuid].serialize());
}
fs.writeFile(TASKS_DUMP_FILE, JSON.stringify(output), err => {
if (err) logger.error(`Could not dump tasks: ${err.message}`);
else logger.debug("Dumped tasks list.");
if (done !== undefined) done();
});
}
getQueueCount() {
let count = 0; let count = 0;
for (let uuid in this.tasks){ for (let uuid in this.tasks) {
let task = this.tasks[uuid]; let task = this.tasks[uuid];
if ([statusCodes.QUEUED, if ([statusCodes.QUEUED,
statusCodes.RUNNING].indexOf(task.status.code) !== -1){ statusCodes.RUNNING
].indexOf(task.status.code) !== -1) {
count++; count++;
} }
} }
return count; return count;
} }
}; };

421
libs/odmOptions.js
Wyświetl plik

@ -23,229 +23,246 @@ let logger = require('./logger');
let odmOptions = null; let odmOptions = null;
module.exports = { module.exports = {
initialize: function(done){ initialize: function(done) {
this.getOptions(done); this.getOptions(done);
}, },
getOptions: function(done){ getOptions: function(done) {
if (odmOptions){ if (odmOptions) {
done(null, odmOptions); done(null, odmOptions);
return; return;
} }
odmRunner.getJsonOptions((err, json) => { odmRunner.getJsonOptions((err, json) => {
if (err) done(err); if (err) done(err);
else{ else {
odmOptions = []; odmOptions = [];
for (let option in json){ for (let option in json) {
// Not all options are useful to the end user // Not all options are useful to the end user
// (num cores can be set programmatically, so can gcpFile, etc.) // (num cores can be set programmatically, so can gcpFile, etc.)
if (["-h", "--project-path", "--cmvs-maxImages", "--time", if (["-h", "--project-path", "--cmvs-maxImages", "--time",
"--zip-results", "--pmvs-num-cores", "--zip-results", "--pmvs-num-cores",
"--start-with", "--gcp", "--end-with", "--images", "--start-with", "--gcp", "--end-with", "--images",
"--slam-config", "--video"].indexOf(option) !== -1) continue; "--slam-config", "--video"
].indexOf(option) !== -1) continue;
let values = json[option]; let values = json[option];
let name = option.replace(/^--/, ""); let name = option.replace(/^--/, "");
let type = ""; let type = "";
let value = ""; let value = "";
let help = values.help || ""; let help = values.help || "";
let domain = values.metavar !== undefined ? let domain = values.metavar !== undefined ?
values.metavar.replace(/^[<>]/g, "") values.metavar.replace(/^[<>]/g, "")
.replace(/[<>]$/g, "") .replace(/[<>]$/g, "")
.trim() : .trim() :
""; "";
switch((values.type || "").trim()){ switch ((values.type || "").trim()) {
case "<type 'int'>": case "<type 'int'>":
type = "int"; type = "int";
value = values['default'] !== undefined ? value = values['default'] !== undefined ?
parseInt(values['default']) : parseInt(values['default']) :
0; 0;
break; break;
case "<type 'float'>": case "<type 'float'>":
type = "float"; type = "float";
value = values['default'] !== undefined ? value = values['default'] !== undefined ?
parseFloat(values['default']) : parseFloat(values['default']) :
0.0; 0.0;
break; break;
default: default:
type = "string"; type = "string";
value = values['default'] !== undefined ? value = values['default'] !== undefined ?
values['default'].trim() : values['default'].trim() :
""; "";
} }
if (values['default'] === "True"){ if (values['default'] === "True") {
type = "bool"; type = "bool";
value = true; value = true;
}else if (values['default'] === "False"){ } else if (values['default'] === "False") {
type = "bool"; type = "bool";
value = false; value = false;
} }
// If 'choices' is specified, try to convert it to array // If 'choices' is specified, try to convert it to array
if (values.choices){ if (values.choices) {
try{ try {
values.choices = JSON.parse(values.choices.replace(/'/g, '"')); // Convert ' to " values.choices = JSON.parse(values.choices.replace(/'/g, '"')); // Convert ' to "
}catch(e){ } catch (e) {
logger.warn(`Cannot parse choices: ${values.choices}`); logger.warn(`Cannot parse choices: ${values.choices}`);
} }
} }
if (Array.isArray(values.choices)){ if (Array.isArray(values.choices)) {
type = "string"; // TODO: change to enum type = "string"; // TODO: change to enum
domain = values.choices; domain = values.choices;
} }
help = help.replace(/\%\(default\)s/g, value); help = help.replace(/\%\(default\)s/g, value);
// In the end, all values must be converted back // In the end, all values must be converted back
// to strings (per OpenAPI spec which doesn't allow mixed types) // to strings (per OpenAPI spec which doesn't allow mixed types)
value = String(value); value = String(value);
odmOptions.push({ odmOptions.push({
name, type, value, domain, help name,
}); type,
} value,
done(null, odmOptions); domain,
} help
}); });
},
// Checks that the options (as received from the rest endpoint) }
// Are valid and within proper ranges.
// The result of filtering is passed back via callback
// @param options[]
filterOptions: function(options, done){
assert(odmOptions !== null, "odmOptions is not set. Have you initialized odmOptions properly?");
try{
if (typeof options === "string") options = JSON.parse(options);
if (!Array.isArray(options)) options = [];
let result = [];
let errors = [];
let addError = function(opt, descr){
errors.push({
name: opt.name,
error: descr
});
};
let typeConversion = {
'float': Number.parseFloat,
'int': Number.parseInt,
'bool': function(value){
if (value === 'true') return true;
else if (value === 'false') return false;
else if (typeof value === 'boolean') return value;
else throw new Error(`Cannot convert ${value} to boolean`);
},
'string': function(value){
return value; // No conversion needed
},
'path': function(value){
return value; // No conversion needed
}
};
let domainChecks = [
{
regex: /^(positive |negative )?(integer|float)$/,
validate: function(matches, value){
if (matches[1] === 'positive ') return value >= 0;
else if (matches[1] === 'negative ') return value <= 0;
else if (matches[2] === 'integer') return Number.isInteger(value);
else if (matches[2] === 'float') return Number.isFinite(value);
}
},
{
regex: /^percent$/,
validate: function(matches, value){
return value >= 0 && value <= 100;
}
},
{
regex: /^(float): ([\-\+\.\d]+) <= x <= ([\-\+\.\d]+)$/,
validate: function(matches, value){
let [str, type, lower, upper] = matches;
lower = parseFloat(lower);
upper = parseFloat(upper);
return value >= lower && value <= upper;
}
},
{
regex: /^(float) (>=|>|<|<=) ([\-\+\.\d]+)$/,
validate: function(matches, value){
let [str, type, oper, bound] = matches;
bound = parseFloat(bound);
switch(oper){
case '>=':
return value >= bound;
case '>':
return value > bound;
case '<=':
return value <= bound;
case '<':
return value < bound;
default:
return false;
}
}
},
{
regex: /^(string|path)$/,
validate: function(){
return true; // All strings/paths are fine
}
}
// TODO: handle enum odmOptions.push({
]; name: 'webhook',
type: 'string',
value: '',
domain: '',
help: 'On task complete, fail, etc. Call the above url with a post of the task serialized'
});
let checkDomain = function(domain, value){ // local (non odm) options
let matches,
dc = domainChecks.find(dc => matches = domain.match(dc.regex));
if (dc){ done(null, odmOptions);
if (!dc.validate(matches, value)) throw new Error(`Invalid value ${value} (out of range)`); }
}else{ });
throw new Error(`Domain value cannot be handled: '${domain}' : '${value}'`); },
}
};
// Scan through all possible options // Checks that the options (as received from the rest endpoint)
for (let odmOption of odmOptions){ // Are valid and within proper ranges.
// Was this option selected by the user? // The result of filtering is passed back via callback
/*jshint loopfunc: true */ // @param options[]
let opt = options.find(o => o.name === odmOption.name); filterOptions: function(options, done) {
if (opt){ assert(odmOptions !== null, "odmOptions is not set. Have you initialized odmOptions properly?");
try{
// Convert to proper data type
let value = typeConversion[odmOption.type](opt.value);
// Domain check try {
if (odmOption.domain){ if (typeof options === "string") options = JSON.parse(options);
checkDomain(odmOption.domain, value); if (!Array.isArray(options)) options = [];
}
result.push({ let result = [];
name: odmOption.name, let errors = [];
value: value let addError = function(opt, descr) {
}); errors.push({
}catch(e){ name: opt.name,
addError(opt, e.message); error: descr
} });
} };
}
if (errors.length > 0) done(new Error(JSON.stringify(errors))); let typeConversion = {
else done(null, result); 'float': Number.parseFloat,
}catch(e){ 'int': Number.parseInt,
done(e); 'bool': function(value) {
} if (value === 'true') return true;
} else if (value === 'false') return false;
else if (typeof value === 'boolean') return value;
else throw new Error(`Cannot convert ${value} to boolean`);
},
'string': function(value) {
return value; // No conversion needed
},
'path': function(value) {
return value; // No conversion needed
}
};
let domainChecks = [{
regex: /^(positive |negative )?(integer|float)$/,
validate: function(matches, value) {
if (matches[1] === 'positive ') return value >= 0;
else if (matches[1] === 'negative ') return value <= 0;
else if (matches[2] === 'integer') return Number.isInteger(value);
else if (matches[2] === 'float') return Number.isFinite(value);
}
},
{
regex: /^percent$/,
validate: function(matches, value) {
return value >= 0 && value <= 100;
}
},
{
regex: /^(float): ([\-\+\.\d]+) <= x <= ([\-\+\.\d]+)$/,
validate: function(matches, value) {
let [str, type, lower, upper] = matches;
lower = parseFloat(lower);
upper = parseFloat(upper);
return value >= lower && value <= upper;
}
},
{
regex: /^(float) (>=|>|<|<=) ([\-\+\.\d]+)$/,
validate: function(matches, value) {
let [str, type, oper, bound] = matches;
bound = parseFloat(bound);
switch (oper) {
case '>=':
return value >= bound;
case '>':
return value > bound;
case '<=':
return value <= bound;
case '<':
return value < bound;
default:
return false;
}
}
},
{
regex: /^(string|path)$/,
validate: function() {
return true; // All strings/paths are fine
}
}
// TODO: handle enum
];
let checkDomain = function(domain, value) {
let matches,
dc = domainChecks.find(dc => matches = domain.match(dc.regex));
if (dc) {
if (!dc.validate(matches, value)) throw new Error(`Invalid value ${value} (out of range)`);
} else {
throw new Error(`Domain value cannot be handled: '${domain}' : '${value}'`);
}
};
// Scan through all possible options
for (let odmOption of odmOptions) {
// Was this option selected by the user?
/*jshint loopfunc: true */
let opt = options.find(o => o.name === odmOption.name);
if (opt) {
try {
// Convert to proper data type
let value = typeConversion[odmOption.type](opt.value);
// Domain check
if (odmOption.domain) {
checkDomain(odmOption.domain, value);
}
result.push({
name: odmOption.name,
value: value
});
} catch (e) {
addError(opt, e.message);
}
}
}
if (errors.length > 0) done(new Error(JSON.stringify(errors)));
else done(null, result);
} catch (e) {
done(e);
}
}
}; };

1
package.json
Wyświetl plik

@ -31,6 +31,7 @@
"multer": "^1.1.0", "multer": "^1.1.0",
"node-schedule": "^1.1.1", "node-schedule": "^1.1.1",
"node-uuid": "^1.4.7", "node-uuid": "^1.4.7",
"request": "^2.81.0",
"rimraf": "^2.5.3", "rimraf": "^2.5.3",
"swagger-jsdoc": "^1.3.1", "swagger-jsdoc": "^1.3.1",
"winston": "^2.2.0" "winston": "^2.2.0"