OpenDroneMap-NodeODM/index.js

/*
Node-OpenDroneMap Node.js App and REST API to access OpenDroneMap.
Copyright (C) 2016 Node-OpenDroneMap Contributors

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/
"use strict";

let fs = require('fs');
let config = require('./config.js');
let packageJson = JSON.parse(fs.readFileSync('./package.json'));

let logger = require('./libs/logger');
let path = require('path');
let async = require('async');

let express = require('express');
let app = express();

let addRequestId = require('./libs/expressRequestId')();
let multer = require('multer');
let bodyParser = require('body-parser');
let morgan = require('morgan');

let TaskManager = require('./libs/TaskManager');
let Task = require('./libs/Task');
let odmOptions = require('./libs/odmOptions');
let Directories = require('./libs/Directories');

let winstonStream = {
    write: function(message, encoding){
    	logger.debug(message.slice(0, -1));
    }
};
app.use(morgan('combined', { stream : winstonStream }));
app.use(bodyParser.urlencoded({extended: true}));
app.use(bodyParser.json());
app.use(express.static('public'));
app.use('/swagger.json', express.static('docs/swagger.json'));

let upload = multer({
	storage: multer.diskStorage({
	  destination: (req, file, cb) => {
	  	let dstPath = path.join("tmp", req.id);
	  	fs.exists(dstPath, exists => {
	  		if (!exists){
	  			fs.mkdir(dstPath, undefined, () => {
	  				cb(null, dstPath);
	  			});
	  		}else{
	    		cb(null, dstPath);
	  		}
	  	});
	  },
	  filename: (req, file, cb) => {
	    cb(null, file.originalname);
	  }
	})
});

let taskManager;
let server;

/** @swagger
*  /task/new:
*    post:
*      description: Creates a new task and places it at the end of the processing queue
*      tags: [task]
*      consumes:
*        - multipart/form-data
*      parameters:
*        -
*          name: images
*          in: formData
*          description: Images to process, plus an optional GPC file. If included, the GPC file should have .txt extension
*          required: true
*          type: file
*        - 
*          name: name
*          in: formData
*          description: An optional name to be associated with the task
*          required: false
*          type: string
*        - 
*          name: options
*          in: formData
*          description: 'Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{"name":"cmvs-maxImages","value":"500"},{"name":"time","value":true}]. For a list of all options, call /options'
*          required: false
*          type: string
*      responses:
*        200:
*          description: Success
*          schema:
*            type: object
*            required: [uuid]
*            properties:
*              uuid:
*                type: string
*                description: UUID of the newly created task
*        default:
*          description: Error
*          schema:
*            $ref: '#/definitions/Error'
*/
app.post('/task/new', addRequestId, upload.array('images'), (req, res) => {
	if (!req.files || req.files.length === 0) res.json({error: "Need at least 1 file."});
	else{
		let srcPath = path.join("tmp", req.id);
		let destPath = path.join(Directories.data, req.id);
		let destImagesPath = path.join(destPath, "images");
		let destGpcPath = path.join(destPath, "gpc");

		async.series([
			cb => {
				odmOptions.filterOptions(req.body.options, (err, options) => {
					if (err) cb(err);
					else{
						req.body.options = options;
						cb(null);
					}
				});
			},

			// Move all uploads to data/<uuid>/images dir
			cb => {
				fs.stat(destPath, (err, stat) => {
					if (err && err.code === 'ENOENT') cb();
					else cb(new Error(`Directory exists (should not have happened: ${err.code})`));
				});
			},
			cb => fs.mkdir(destPath, undefined, cb),
			cb => fs.mkdir(destGpcPath, undefined, cb),
			cb => fs.rename(srcPath, destImagesPath, cb),
			cb => {
				// Find any *.txt (GPC) file and move it to the data/<uuid>/gpc directory
				fs.readdir(destImagesPath, (err, entries) => {
					if (err) cb(err);
					else{
						async.eachSeries(entries, (entry, cb) => {
							if (/\.txt$/gi.test(entry)){
								fs.rename(path.join(destImagesPath, entry), path.join(destGpcPath, entry), cb);		
							}else cb();
						}, cb);
					}
				});
			},

			// Create task
			cb => {
				new Task(req.id, req.body.name, (err, task) => {
					if (err) cb(err);
					else{
						taskManager.addNew(task);
						res.json({uuid: req.id});
						cb();
					}
				}, req.body.options);
			}
		], err => {
			if (err) res.json({error: err.message});
		});
	}
});

let getTaskFromUuid = (req, res, next) => {
	let task = taskManager.find(req.params.uuid);
	if (task){
		req.task = task;
		next();
	}else res.json({error: `${req.params.uuid} not found`});
};

/** @swagger
*  /task/{uuid}/info:
*     get:
*       description: Gets information about this task, such as name, creation date, processing time, status, command line options and number of images being processed. See schema definition for a full list.
*       tags: [task]
*       parameters:
*        -
*           name: uuid
*           in: path
*           description: UUID of the task
*           required: true
*           type: string
*       responses:
*        200:
*         description: Task Information
*         schema:
*           title: TaskInfo
*           type: object
*           required: [uuid, name, dateCreated, processingTime, status, options, imagesCount]
*           properties:
*            uuid:
*              type: string
*              description: UUID
*            name:
*              type: string
*              description: Name
*            dateCreated:
*              type: integer
*              description: Timestamp
*            processingTime:
*              type: integer
*              description: Milliseconds that have elapsed since the task started being processed.
*            status:
*              type: integer
*              description: Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)
*              enum: [10, 20, 30, 40, 50]
*            options:
*              type: array
*              description: List of options used to process this task
*              items:
*                type: object
*                required: [name, value]
*                properties:
*                  name:
*                    type: string
*                    description: 'Option name (example: "odm_meshing-octreeDepth")'
*                  value:
*                    type: string
*                    description: 'Value (example: 9)'
*            imagesCount:
*              type: integer
*              description: Number of images
*        default:
*          description: Error
*          schema:
*            $ref: '#/definitions/Error'
*/
app.get('/task/:uuid/info', getTaskFromUuid, (req, res) => {
	res.json(req.task.getInfo());
});

/** @swagger
*  /task/{uuid}/output:
*     get:
*       description: Retrieves the console output of the OpenDroneMap's process. Useful for monitoring execution and to provide updates to the user.
*       tags: [task]
*       parameters:
*        -
*           name: uuid
*           in: path
*           description: UUID of the task
*           required: true
*           type: string
*        -
*         name: line
*         in: query
*         description: Optional line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. Defaults to 0 (retrieve all console output).
*         default: 0
*         required: false
*         type: integer
*       responses:
*        200:
*         description: Console Output
*         schema:
*           type: string
*        default:
*          description: Error
*          schema:
*            $ref: '#/definitions/Error'
*/
app.get('/task/:uuid/output', getTaskFromUuid, (req, res) => {
	res.json(req.task.getOutput(req.query.line));
});

/** @swagger
*  /task/{uuid}/download/{asset}:
*    get:
*      description: Retrieves an asset (the output of OpenDroneMap's processing) associated with a task
*      tags: [task]
*      produces: [application/zip]
*      parameters:
*        - name: uuid
*          in: path
*          type: string
*          description: UUID of the task
*          required: true
*        - name: asset
*          in: path
*          type: string
*          description: Type of asset to download. Use "all.zip" for zip file containing all assets.
*          required: true
*          enum:
*            - all.zip
*            - georeferenced_model.ply.zip
*            - georeferenced_model.las.zip
*            - georeferenced_model.csv.zip
*            - orthophoto.png
*            - orthophoto.tif
*            - textured_model.zip
*            - orthophoto_tiles.zip
*      responses:
*        200:
*          description: Asset File
*          schema:
*            type: file
*        default:
*          description: Error message
*          schema:
*            $ref: '#/definitions/Error'
*/
app.get('/task/:uuid/download/:asset', getTaskFromUuid, (req, res) => {
	let asset = req.params.asset !== undefined ? req.params.asset : "all.zip";
	let filePath = req.task.getAssetsArchivePath(asset);
	if (filePath){
		res.download(filePath, filePath, err => {
			if (err) res.json({error: "Asset not ready"});
		});
	}else{
		res.json({error: "Invalid asset"});
	}
});

/** @swagger
* definition:
*   Error:
*     type: object
*     required:
*       - error
*     properties:
*       error:
*         type: string
*         description: Description of the error
*   Response:
*     type: object
*     required:
*       - success
*     properties:
*       success:
*         type: boolean
*         description: true if the command succeeded, false otherwise
*       error:
*         type: string
*         description: Error message if an error occured
*/
let uuidCheck = (req, res, next) => {
    if (!req.body.uuid) res.json({error: "uuid param missing."});
    else next();
};

let successHandler = res => {
	return err => {
		if (!err) res.json({success: true});
		else res.json({success: false, error: err.message});
	};
};

/** @swagger
* /task/cancel:
*    post:
*      description: Cancels a task (stops its execution, or prevents it from being executed)
*      parameters:
*        -
*          name: uuid
*          in: body
*          description: UUID of the task
*          required: true
*          schema:
*            type: string
*      responses:
*        200:
*          description: Command Received
*          schema:
*            $ref: "#/definitions/Response"
*/
app.post('/task/cancel', uuidCheck, (req, res) => {
	taskManager.cancel(req.body.uuid, successHandler(res));
});

/** @swagger
* /task/remove:
*    post:
*      description: Removes a task and deletes all of its assets
*      parameters:
*        -
*          name: uuid
*          in: body
*          description: UUID of the task
*          required: true
*          schema:
*            type: string
*      responses:
*        200:
*          description: Command Received
*          schema:
*            $ref: "#/definitions/Response"
*/
app.post('/task/remove', uuidCheck, (req, res) => {
	taskManager.remove(req.body.uuid, successHandler(res));
});

/** @swagger
* /task/restart:
*    post:
*      description: Restarts a task that was previously canceled or that had failed to process
*      parameters:
*        -
*          name: uuid
*          in: body
*          description: UUID of the task
*          required: true
*          schema:
*            type: string
*      responses:
*        200:
*          description: Command Received
*          schema:
*            $ref: "#/definitions/Response"
*/
app.post('/task/restart', uuidCheck, (req, res) => {
	taskManager.restart(req.body.uuid, successHandler(res));
});

/** @swagger
* /options:
*   get:
*     description: Retrieves the command line options that can be passed to process a task
*     tags: [server]
*     responses:
*       200:
*         description: Options
*         schema:
*           type: array
*           items:
*             title: Option
*             type: object
*             required: [name, type, value, domain, help]
*             properties:
*               name:
*                 type: string
*                 description: Command line option (exactly as it is passed to the OpenDroneMap process, minus the leading '--')
*               type:
*                 type: string
*                 description: Datatype of the value of this option
*                 enum:
*                   - int
*                   - float
*                   - string
*                   - bool
*               value:
*                 type: string
*                 description: Default value of this option
*               domain:
*                 type: string
*                 description: Valid range of values (for example, "positive integer" or "float > 0.0")
*               help:
*                 type: string
*                 description: Description of what this option does
*/
app.get('/options', (req, res) => {
	odmOptions.getOptions((err, options) => {
		if (err) res.json({error: err.message});
		else res.json(options);
	});
});

/** @swagger
* /info:
*   get:
*     description: Retrieves information about this node
*     tags: [server]
*     responses:
*       200:
*         description: Info
*         schema:
*           type: object
*           required: [version, taskQueueCount]
*           properties:
*             version:
*               type: string
*               description: Current version
*             taskQueueCount:
*               type: integer
*               description: Number of tasks currently being processed or waiting to be processed
*/
app.get('/info', (req, res) => {
    res.json({
        version: packageJson.version,
        taskQueueCount: taskManager.getQueueCount()
    });
});

let gracefulShutdown = done => {
	async.series([
		cb => taskManager.dumpTaskList(cb),
		cb => {
			logger.info("Closing server");
			server.close();
			logger.info("Exiting...");
			process.exit(0);
		}
	], done);
};

// listen for TERM signal .e.g. kill
process.on ('SIGTERM', gracefulShutdown);

// listen for INT signal e.g. Ctrl-C
process.on ('SIGINT', gracefulShutdown);

// Startup
if (config.test) logger.info("Running in test mode");

async.series([
	cb => odmOptions.initialize(cb),
	cb => { taskManager = new TaskManager(cb); },
	cb => { server = app.listen(config.port, err => {
			if (!err) logger.info('Server has started on port ' + String(config.port));
			cb(err);
		});
	}
], err => {
	if (err) logger.error("Error during startup: " + err.message);
});