From f22e572a91d1324e0cc28c5411ce9bd5dc602bf2 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 29 Jul 2016 17:16:22 -0500 Subject: [PATCH] Logger cleanup, more command line arguments, options passing to ODM working --- config.js | 9 +++++++-- index.js | 5 +++-- libs/Task.js | 16 +++++++++++----- libs/TaskManager.js | 38 ++++++++++++++++++++++++++++++++------ libs/logger.js | 15 +++++++++++---- libs/odmOptions.js | 4 ++++ libs/odmRunner.js | 21 +++++++++++++++++---- 7 files changed, 85 insertions(+), 23 deletions(-) diff --git a/config.js b/config.js index 30ba23e..40de1f2 100644 --- a/config.js +++ b/config.js @@ -25,8 +25,11 @@ Usage: node index.js [options] Options: -p, --port Port to bind the server to (default: 3000) --odm_path Path to OpenDroneMap's code (default: /code) - --log_level Set log level verbosity (default: debug) + --log_level Set log level verbosity (default: info) -d, --deamonize Set process to run as a deamon + --parallel_queue_processing Number of simultaneous processing tasks (default: 2) + --cleanup_tasks_after Number of days that elapse before deleting finished and canceled tasks (default: 3) + Log Levels: error | debug | info | verbose | debug | silly `); @@ -41,12 +44,14 @@ config.odm_path = argv.odm_path || '/code'; // Logging configuration config.logger = {}; -config.logger.level = argv.log_level || 'debug'; // What level to log at; info, verbose or debug are most useful. Levels are (npm defaults): silly, debug, verbose, info, warn, error. +config.logger.level = argv.log_level || 'info'; // What level to log at; info, verbose or debug are most useful. Levels are (npm defaults): silly, debug, verbose, info, warn, error. config.logger.maxFileSize = 1024 * 1024 * 100; // Max file size in bytes of each log file; default 100MB config.logger.maxFiles = 10; // Max number of log files kept config.logger.logDirectory = ''; // Set this to a full path to a directory - if not set logs will be written to the application directory. config.port = parseInt(argv.port || argv.p || process.env.PORT || 3000); config.deamon = argv.deamonize || argv.d; +config.parallelQueueProcessing = argv.parallel_queue_processing || 2; +config.cleanupTasksAfter = argv.cleanup_tasks_after || 3; module.exports = config; diff --git a/index.js b/index.js index 58f5a41..77a6e15 100644 --- a/index.js +++ b/index.js @@ -17,7 +17,7 @@ along with this program. If not, see . */ "use strict"; -let config = require('./config.js') +let config = require('./config.js'); let logger = require('./libs/logger'); let fs = require('fs'); @@ -37,7 +37,7 @@ let odmOptions = require('./libs/odmOptions'); let winstonStream = { write: function(message, encoding){ - logger.info(message.slice(0, -1)); + logger.debug(message.slice(0, -1)); } }; app.use(morgan('combined', { stream : winstonStream })); @@ -189,6 +189,7 @@ let taskManager; let server; async.series([ + cb => odmOptions.initialize(cb), cb => { taskManager = new TaskManager(cb); }, cb => { server = app.listen(config.port, err => { if (!err) logger.info('Server has started on port ' + String(config.port)); diff --git a/libs/Task.js b/libs/Task.js index 8d9fd58..2eda63b 100644 --- a/libs/Task.js +++ b/libs/Task.js @@ -21,6 +21,7 @@ let fs = require('fs'); let rmdir = require('rimraf'); let odmRunner = require('./odmRunner'); let archiver = require('archiver'); +let os = require('os'); let statusCodes = require('./statusCodes'); @@ -38,8 +39,6 @@ module.exports = class Task{ this.output = []; this.runnerProcess = null; - this.options.forEach(option => { console.log(option); }); - // Read images info fs.readdir(this.getImagesFolderPath(), (err, files) => { if (err) done(err); @@ -189,9 +188,16 @@ module.exports = class Task{ if (this.status.code === statusCodes.QUEUED){ this.startTrackingProcessingTime(); this.setStatus(statusCodes.RUNNING); - this.runnerProcess = odmRunner.run({ - projectPath: `${__dirname}/../${this.getProjectFolderPath()}` - }, (err, code, signal) => { + + let runnerOptions = this.options.reduce((result, opt) => { + result[opt.name] = opt.value; + return result; + }, {}); + + runnerOptions["project-path"] = `${__dirname}/../${this.getProjectFolderPath()}`; + runnerOptions["pmvs-num-cores"] = os.cpus().length; + + this.runnerProcess = odmRunner.run(runnerOptions, (err, code, signal) => { if (err){ this.setStatus(statusCodes.FAILED, {errorMessage: `Could not start process (${err.message})`}); finished(); diff --git a/libs/TaskManager.js b/libs/TaskManager.js index 7549a89..8475746 100644 --- a/libs/TaskManager.js +++ b/libs/TaskManager.js @@ -17,16 +17,19 @@ along with this program. If not, see . */ "use strict"; let assert = require('assert'); +let config = require('../config'); +let rmdir = require('rimraf'); let fs = require('fs'); +let path = require('path'); let logger = require('./logger'); let Task = require('./Task'); let statusCodes = require('./statusCodes'); let async = require('async'); let schedule = require('node-schedule'); -const PARALLEL_QUEUE_PROCESS_LIMIT = 2; -const TASKS_DUMP_FILE = "data/tasks.json"; -const CLEANUP_TASKS_IF_OLDER_THAN = 1000 * 60 * 60 * 24 * 3; // 3 days +const DATA_DIR = "data"; +const TASKS_DUMP_FILE = `${DATA_DIR}/tasks.json`; +const CLEANUP_TASKS_IF_OLDER_THAN = 1000 * 60 * 60 * 24 * config.cleanupTasksAfter; // days module.exports = class TaskManager{ constructor(done){ @@ -36,13 +39,14 @@ module.exports = class TaskManager{ async.series([ cb => this.restoreTaskListFromDump(cb), cb => this.removeOldTasks(cb), + cb => this.removeOrphanedDirectories(cb), cb => { this.processNextTask(); cb(); }, cb => { // Every hour - schedule.scheduleJob('* 0 * * * *', () => { + schedule.scheduleJob('0 * * * *', () => { this.removeOldTasks(); this.dumpTaskList(); }); @@ -76,6 +80,28 @@ module.exports = class TaskManager{ }, done); } + // Removes directories that don't have a corresponding + // task associated with it (maybe as a cause of an abrupt exit) + removeOrphanedDirectories(done){ + logger.info("Checking for orphaned directories to be removed..."); + + fs.readdir(DATA_DIR, (err, entries) => { + if (err) done(err); + else{ + async.eachSeries(entries, (entry, cb) => { + let dirPath = path.join(DATA_DIR, entry); + if (fs.statSync(dirPath).isDirectory() && + entry.match(/[\w\d]+\-[\w\d]+\-[\w\d]+\-[\w\d]+\-[\w\d]+/) && + !this.tasks[entry]){ + logger.info(`Found orphaned directory: ${entry}, removing...`); + rmdir(dirPath, cb); + }else cb(); + }, done); + } + }); + } + + // Load tasks that already exists (if any) restoreTaskListFromDump(done){ fs.readFile(TASKS_DUMP_FILE, (err, data) => { @@ -113,7 +139,7 @@ module.exports = class TaskManager{ // Finds the next tasks, adds them to the running queue, // and starts the tasks (up to the limit). processNextTask(){ - if (this.runningQueue.length < PARALLEL_QUEUE_PROCESS_LIMIT){ + if (this.runningQueue.length < config.parallelQueueProcessing){ let task = this.findNextTaskToProcess(); if (task){ this.addToRunningQueue(task); @@ -122,7 +148,7 @@ module.exports = class TaskManager{ this.processNextTask(); }); - if (this.runningQueue.length < PARALLEL_QUEUE_PROCESS_LIMIT) this.processNextTask(); + if (this.runningQueue.length < config.parallelQueueProcessing) this.processNextTask(); } }else{ // Do nothing diff --git a/libs/logger.js b/libs/logger.js index 7067291..9eb26ba 100644 --- a/libs/logger.js +++ b/libs/logger.js @@ -24,7 +24,9 @@ let path = require('path'); // Set up logging // Configure custom File transport to write plain text messages -let logPath = ( config.logger.logDirectory ? config.logger.logDirectory : __dirname ); +let logPath = ( config.logger.logDirectory ? + config.logger.logDirectory : + `${__dirname}/../` ); // Check that log file directory can be written to try { @@ -36,7 +38,12 @@ try { logPath += path.sep; logPath += config.instance + ".log"; -winston.add(winston.transports.File, { +let logger = new (winston.Logger)({ + transports: [ + new (winston.transports.Console)({ level: config.logger.level }), + ] +}); +logger.add(winston.transports.File, { filename: logPath, // Write to projectname.log json: false, // Write in plain text, not JSON maxsize: config.logger.maxFileSize, // Max size of each file @@ -46,7 +53,7 @@ winston.add(winston.transports.File, { if (config.deamon){ // Console transport is no use to us when running as a daemon - winston.remove(winston.transports.Console); + logger.remove(winston.transports.Console); } -module.exports = winston; \ No newline at end of file +module.exports = logger; \ No newline at end of file diff --git a/libs/odmOptions.js b/libs/odmOptions.js index b8d346b..74ce542 100644 --- a/libs/odmOptions.js +++ b/libs/odmOptions.js @@ -22,6 +22,10 @@ let assert = require('assert'); let odmOptions = null; module.exports = { + initialize: function(done){ + this.getOptions(done); + }, + getOptions: function(done){ if (odmOptions){ done(null, odmOptions); diff --git a/libs/odmRunner.js b/libs/odmRunner.js index b428efa..69af062 100644 --- a/libs/odmRunner.js +++ b/libs/odmRunner.js @@ -21,13 +21,26 @@ let config = require('../config.js'); module.exports = { run: function(options = { - projectPath: "/images" + "project-path": "/images" }, done, outputReceived){ + let command = [`${config.odm_path}/run.py`]; + for (var name in options){ + let value = options[name]; + + // Skip false booleans + if (value === false) continue; + + command.push("--" + name); + + // We don't specify "--time true" (just "--time") + if (typeof value !== 'boolean'){ + command.push(value); + } + } + // Launch - let childProcess = spawn("python", [`${config.odm_path}/run.py`, - "--project-path", options.projectPath - ], {cwd: config.odm_path}); + let childProcess = spawn("python", command, {cwd: config.odm_path}); childProcess .on('exit', (code, signal) => done(null, code, signal))