diff --git a/.jshintrc b/.jshintrc new file mode 100644 index 0000000..439082e --- /dev/null +++ b/.jshintrc @@ -0,0 +1,4 @@ +{ + "esnext": true, + "node": true +} \ No newline at end of file diff --git a/README.md b/README.md index 8fc067a..97240c5 100644 --- a/README.md +++ b/README.md @@ -63,11 +63,39 @@ For other command line options you can run: node index.js --help ``` -These and other options (including logging options) can also be set in [config.js](config.js). +You can also specify configuration values via a JSON file: + +``` +node index.js --config config.default.json +``` + +Command line arguments always take precedence over the configuration file. + +### Run it using PM2 + +The app can also be run as a background process using the [pm2 process manager](https://github.com/Unitech/pm2), which can also assist you with system startup scripts and process monitoring. + +To install pm2, run (using `sudo` if required): +```shell +npm install pm2 -g +``` +The app can then be started using +```shell +pm2 start processes.json +``` +To have pm2 started on OS startup run +```shell +pm2 save +pm2 startup +``` +and then run the command as per the instructions that prints out. If that command errors then you may have to specify the system (note that systemd should be used on CentOS 7). Note that if the process is not running as root (recommended) you will need to change `/etc/init.d/pm2-init.sh` to set `export PM2_HOME="/path/to/user/home/.pm2"`, as per [these instructions]( +http://www.buildsucceeded.com/2015/solved-pm2-startup-at-boot-time-centos-7-red-hat-linux/) + +You can monitor the process using `pm2 status`. ### Test Images -You can find some test drone images from [OpenDroneMap's Test Data Folder](https://github.com/OpenDroneMap/OpenDroneMap/tree/master/tests/test_data/images). +You can find some test drone images [here](https://github.com/dakotabenjamin/odm_data). ## Contributing diff --git a/config-default.json b/config-default.json new file mode 100644 index 0000000..7f71512 --- /dev/null +++ b/config-default.json @@ -0,0 +1,16 @@ +{ + "instance": "node-OpenDroneMap", + "odm_path": "/code", + + "logger": { + "level": "info", + "maxFileSize": 104857600, + "maxFiles": 10, + "logDirectory": "" + }, + + "port": 3000, + "deamon": false, + "parallelQueueProcessing": 2, + "cleanupTasksAfter": 3 +} \ No newline at end of file diff --git a/config.js b/config.js index 40de1f2..2422751 100644 --- a/config.js +++ b/config.js @@ -17,19 +17,22 @@ along with this program. If not, see . */ 'use strict'; +let fs = require('fs'); let argv = require('minimist')(process.argv.slice(2)); +let utils = require('./libs/utils'); + if (argv.help){ console.log(` Usage: node index.js [options] Options: + --config Path to the configuration file (default: config-default.json) -p, --port Port to bind the server to (default: 3000) --odm_path Path to OpenDroneMap's code (default: /code) --log_level Set log level verbosity (default: info) -d, --deamonize Set process to run as a deamon --parallel_queue_processing Number of simultaneous processing tasks (default: 2) --cleanup_tasks_after Number of days that elapse before deleting finished and canceled tasks (default: 3) - Log Levels: error | debug | info | verbose | debug | silly `); @@ -38,20 +41,40 @@ error | debug | info | verbose | debug | silly let config = {}; +// Read configuration from file +let configFilePath = argv.config || "config-default.json"; +let configFile = {}; + +if (/\.json$/i.test(configFilePath)){ + try{ + let data = fs.readFileSync(configFilePath); + configFile = JSON.parse(data.toString()); + }catch(e){ + console.log(`Invalid configuration file ${configFilePath}`); + process.exit(1); + } +} + +// Gets a property that might not exist from configuration file +// example: fromConfigFile("logger.maxFileSize", 1000); +function fromConfigFile(prop, defaultValue){ + return utils.get(configFile, prop, defaultValue); +} + // Instance name - default name for this configuration -config.instance = 'node-OpenDroneMap'; -config.odm_path = argv.odm_path || '/code'; +config.instance = fromConfigFile("instance", 'node-OpenDroneMap'); +config.odm_path = argv.odm_path || fromConfigFile("odm_path", '/code'); // Logging configuration config.logger = {}; -config.logger.level = argv.log_level || 'info'; // What level to log at; info, verbose or debug are most useful. Levels are (npm defaults): silly, debug, verbose, info, warn, error. -config.logger.maxFileSize = 1024 * 1024 * 100; // Max file size in bytes of each log file; default 100MB -config.logger.maxFiles = 10; // Max number of log files kept -config.logger.logDirectory = ''; // Set this to a full path to a directory - if not set logs will be written to the application directory. +config.logger.level = argv.log_level || fromConfigFile("logger.level", 'info'); // What level to log at; info, verbose or debug are most useful. Levels are (npm defaults): silly, debug, verbose, info, warn, error. +config.logger.maxFileSize = fromConfigFile("logger.maxFileSize", 1024 * 1024 * 100); // Max file size in bytes of each log file; default 100MB +config.logger.maxFiles = fromConfigFile("logger.maxFiles", 10); // Max number of log files kept +config.logger.logDirectory = fromConfigFile("logger.logDirectory", ''); // Set this to a full path to a directory - if not set logs will be written to the application directory. -config.port = parseInt(argv.port || argv.p || process.env.PORT || 3000); -config.deamon = argv.deamonize || argv.d; -config.parallelQueueProcessing = argv.parallel_queue_processing || 2; -config.cleanupTasksAfter = argv.cleanup_tasks_after || 3; +config.port = parseInt(argv.port || argv.p || fromConfigFile("port", process.env.PORT || 3000)); +config.deamon = argv.deamonize || argv.d || fromConfigFile("daemon", false); +config.parallelQueueProcessing = argv.parallel_queue_processing || fromConfigFile("parallelQueueProcessing", 2); +config.cleanupTasksAfter = argv.cleanup_tasks_after || fromConfigFile("cleanupTasksAfter", 3); module.exports = config; diff --git a/index.js b/index.js index 5078b3a..0a21e84 100644 --- a/index.js +++ b/index.js @@ -61,11 +61,14 @@ let upload = multer({ }); }, filename: (req, file, cb) => { - cb(null, file.originalname) + cb(null, file.originalname); } }) }); +let taskManager; +let server; + app.post('/task/new', addRequestId, upload.array('images'), (req, res) => { if (req.files.length === 0) res.json({error: "Need at least 1 file."}); else{ @@ -121,7 +124,7 @@ app.post('/task/new', addRequestId, upload.array('images'), (req, res) => { }, req.body.options); } ], err => { - if (err) res.json({error: err.message}) + if (err) res.json({error: err.message}); }); } }); @@ -132,7 +135,7 @@ let getTaskFromUuid = (req, res, next) => { req.task = task; next(); }else res.json({error: `${req.params.uuid} not found`}); -} +}; app.get('/task/:uuid/info', getTaskFromUuid, (req, res) => { res.json(req.task.getInfo()); @@ -200,9 +203,6 @@ process.on ('SIGTERM', gracefulShutdown); process.on ('SIGINT', gracefulShutdown); // Startup -let taskManager; -let server; - async.series([ cb => odmOptions.initialize(cb), cb => { taskManager = new TaskManager(cb); }, diff --git a/libs/Task.js b/libs/Task.js index 7f1fda3..27d08e7 100644 --- a/libs/Task.js +++ b/libs/Task.js @@ -35,7 +35,7 @@ module.exports = class Task{ assert(done !== undefined, "ready must be set"); this.uuid = uuid; - this.name = name != "" ? name : "Task of " + (new Date()).toISOString(); + this.name = name !== "" ? name : "Task of " + (new Date()).toISOString(); this.dateCreated = new Date().getTime(); this.processingTime = -1; this.setStatus(statusCodes.QUEUED); @@ -51,7 +51,7 @@ module.exports = class Task{ if (err) cb(err); else{ this.images = files; - logger.debug(`Found ${this.images.length} images for ${this.uuid}`) + logger.debug(`Found ${this.images.length} images for ${this.uuid}`); cb(null); } }); @@ -190,6 +190,11 @@ module.exports = class Task{ // Starts processing the task with OpenDroneMap // This will spawn a new process. start(done){ + const finished = err => { + this.stopTrackingProcessingTime(); + done(err); + }; + const postProcess = () => { let output = fs.createWriteStream(this.getAssetsArchivePath()); let archive = archiver.create('zip', {}); @@ -214,11 +219,6 @@ module.exports = class Task{ .finalize(); }; - const finished = err => { - this.stopTrackingProcessingTime(); - done(err); - }; - if (this.status.code === statusCodes.QUEUED){ this.startTrackingProcessingTime(); this.setStatus(statusCodes.RUNNING); @@ -289,7 +289,7 @@ module.exports = class Task{ status: this.status, options: this.options, imagesCount: this.images.length - } + }; } // Returns the output of the OpenDroneMap process @@ -307,6 +307,6 @@ module.exports = class Task{ dateCreated: this.dateCreated, status: this.status, options: this.options - } + }; } }; diff --git a/libs/TaskManager.js b/libs/TaskManager.js index dfa9449..eb98afd 100644 --- a/libs/TaskManager.js +++ b/libs/TaskManager.js @@ -61,7 +61,7 @@ module.exports = class TaskManager{ removeOldTasks(done){ let list = []; let now = new Date().getTime(); - logger.info("Checking for old tasks to be removed..."); + logger.debug("Checking for old tasks to be removed..."); for (let uuid in this.tasks){ let task = this.tasks[uuid]; @@ -75,7 +75,7 @@ module.exports = class TaskManager{ } async.eachSeries(list, (uuid, cb) => { - logger.info(`Cleaning up old task ${uuid}`) + logger.info(`Cleaning up old task ${uuid}`); this.remove(uuid, cb); }, done); } @@ -174,8 +174,8 @@ module.exports = class TaskManager{ // Stops the execution of a task // (without removing it from the system). cancel(uuid, cb){ - let task; - if (task = this.find(uuid, cb)){ + let task = this.find(uuid, cb); + if (task){ if (!task.isCanceled()){ task.cancel(err => { this.removeFromRunningQueue(task); @@ -193,8 +193,8 @@ module.exports = class TaskManager{ remove(uuid, cb){ this.cancel(uuid, err => { if (!err){ - let task; - if (task = this.find(uuid, cb)){ + let task = this.find(uuid, cb); + if (task){ task.cleanup(err => { if (!err){ delete(this.tasks[uuid]); @@ -210,8 +210,8 @@ module.exports = class TaskManager{ // Restarts (puts back into QUEUED state) // a task that is either in CANCELED or FAILED state. restart(uuid, cb){ - let task; - if (task = this.find(uuid, cb)){ + let task = this.find(uuid, cb); + if (task){ task.restart(err => { if (!err) this.processNextTask(); cb(err); @@ -237,8 +237,8 @@ module.exports = class TaskManager{ fs.writeFile(TASKS_DUMP_FILE, JSON.stringify(output), err => { if (err) logger.error(`Could not dump tasks: ${err.message}`); - else logger.info("Dumped tasks list."); + else logger.debug("Dumped tasks list."); if (done !== undefined) done(); - }) + }); } }; diff --git a/libs/logger.js b/libs/logger.js index 9eb26ba..5aba9bf 100644 --- a/libs/logger.js +++ b/libs/logger.js @@ -49,7 +49,7 @@ logger.add(winston.transports.File, { maxsize: config.logger.maxFileSize, // Max size of each file maxFiles: config.logger.maxFiles, // Max number of files level: config.logger.level // Level of log messages - }) + }); if (config.deamon){ // Console transport is no use to us when running as a daemon diff --git a/libs/odmOptions.js b/libs/odmOptions.js index 74ce542..532352c 100644 --- a/libs/odmOptions.js +++ b/libs/odmOptions.js @@ -106,12 +106,12 @@ module.exports = { let result = []; let errors = []; - function addError(opt, descr){ + let addError = function(opt, descr){ errors.push({ name: opt.name, error: descr }); - } + }; let typeConversion = { 'float': Number.parseFloat, @@ -171,21 +171,22 @@ module.exports = { } ]; - function checkDomain(domain, value){ - let dc, matches; + let checkDomain = function(domain, value){ + let matches, + dc = domainChecks.find(dc => matches = domain.match(dc.regex)); - if (dc = domainChecks.find(dc => matches = domain.match(dc.regex))){ + if (dc){ if (!dc.validate(matches, value)) throw new Error(`Invalid value ${value} (out of range)`); }else{ throw new Error(`Domain value cannot be handled: '${domain}' : '${value}'`); } - } + }; // Scan through all possible options for (let odmOption of odmOptions){ // Was this option selected by the user? - let opt; - if (opt = options.find(o => o.name === odmOption.name)){ + let opt = options.find(o => o.name === odmOption.name); + if (opt){ try{ // Convert to proper data type let value = typeConversion[odmOption.type](opt.value); diff --git a/libs/odmRunner.js b/libs/odmRunner.js index db2c48a..4a59094 100644 --- a/libs/odmRunner.js +++ b/libs/odmRunner.js @@ -16,14 +16,14 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ "use strict"; +let assert = require('assert'); let spawn = require('child_process').spawn; let config = require('../config.js'); let logger = require('./logger'); module.exports = { - run: function(options = { - "project-path": "/images" - }, done, outputReceived){ + run: function(options, done, outputReceived){ + assert(options["project-path"] !== undefined, "project-path must be defined"); let command = [`${config.odm_path}/run.py`]; for (var name in options){ diff --git a/libs/utils.js b/libs/utils.js new file mode 100644 index 0000000..466550b --- /dev/null +++ b/libs/utils.js @@ -0,0 +1,16 @@ +module.exports = { + get: function(scope, prop, defaultValue){ + let parts = prop.split("."); + let current = scope; + for (let i = 0; i < parts.length; i++){ + if (current[parts[i]] !== undefined && i < parts.length - 1){ + current = current[parts[i]]; + }else if (current[parts[i]] !== undefined && i < parts.length){ + return current[parts[i]]; + }else{ + return defaultValue; + } + } + return defaultValue; + } +}; \ No newline at end of file diff --git a/processes.json b/processes.json new file mode 100644 index 0000000..127d447 --- /dev/null +++ b/processes.json @@ -0,0 +1,6 @@ +[{ + "script" : "index.js", + "name" : "node-OpenDroneMap", + "watch" : false, + "ignore_watch" : ["[\\/\\\\]\\./", "node_modules", "git-hooks", "test", "tmp", "data", "^.*\.log$"] +}]