Logger cleanup, more command line arguments, options passing to ODM working

config.js

@@ -25,8 +25,11 @@ Usage: node index.js [options]
 Options:
 	-p, --port <number> 	Port to bind the server to (default: 3000)
 	--odm_path <path>	Path to OpenDroneMap's code	(default: /code)
-	--log_level <logLevel>	Set log level verbosity (default: debug)
+	--log_level <logLevel>	Set log level verbosity (default: info)
 	-d, --deamonize 	Set process to run as a deamon
+	--parallel_queue_processing <number> Number of simultaneous processing tasks (default: 2)
+	--cleanup_tasks_after <number> Number of days that elapse before deleting finished and canceled tasks (default: 3) 
+
 Log Levels: 
 error | debug | info | verbose | debug | silly 
 `);
@@ -41,12 +44,14 @@ config.odm_path = argv.odm_path || '/code';
 
 // Logging configuration
 config.logger = {};
-config.logger.level = argv.log_level || 'debug'; // What level to log at; info, verbose or debug are most useful. Levels are (npm defaults): silly, debug, verbose, info, warn, error.
+config.logger.level = argv.log_level || 'info'; // What level to log at; info, verbose or debug are most useful. Levels are (npm defaults): silly, debug, verbose, info, warn, error.
 config.logger.maxFileSize = 1024 * 1024 * 100; // Max file size in bytes of each log file; default 100MB
 config.logger.maxFiles = 10; // Max number of log files kept
 config.logger.logDirectory = ''; // Set this to a full path to a directory - if not set logs will be written to the application directory.
 
 config.port = parseInt(argv.port || argv.p || process.env.PORT || 3000);
 config.deamon = argv.deamonize || argv.d;
+config.parallelQueueProcessing = argv.parallel_queue_processing || 2;
+config.cleanupTasksAfter = argv.cleanup_tasks_after || 3;
 
 module.exports = config;

index.js

@@ -17,7 +17,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 "use strict";
 
-let config = require('./config.js')
+let config = require('./config.js');
 
 let logger = require('./libs/logger');
 let fs = require('fs');
@@ -37,7 +37,7 @@ let odmOptions = require('./libs/odmOptions');
 
 let winstonStream = {
     write: function(message, encoding){
-    	logger.info(message.slice(0, -1));
+    	logger.debug(message.slice(0, -1));
     }
 };
 app.use(morgan('combined', { stream : winstonStream }));
@@ -189,6 +189,7 @@ let taskManager;
 let server;
 
 async.series([
+	cb => odmOptions.initialize(cb),
 	cb => { taskManager = new TaskManager(cb); },
 	cb => { server = app.listen(config.port, err => {
 			if (!err) logger.info('Server has started on port ' + String(config.port));

libs/Task.js

@@ -21,6 +21,7 @@ let fs = require('fs');
 let rmdir = require('rimraf');
 let odmRunner = require('./odmRunner');
 let archiver = require('archiver');
+let os = require('os');
 
 let statusCodes = require('./statusCodes');
 
@@ -38,8 +39,6 @@ module.exports = class Task{
 		this.output = [];
 		this.runnerProcess = null;
 
-		this.options.forEach(option => { console.log(option); });
-
 		// Read images info
 		fs.readdir(this.getImagesFolderPath(), (err, files) => {
 			if (err) done(err);
@@ -189,9 +188,16 @@ module.exports = class Task{
 		if (this.status.code === statusCodes.QUEUED){
 			this.startTrackingProcessingTime();
 			this.setStatus(statusCodes.RUNNING);
-			this.runnerProcess = odmRunner.run({
-					projectPath: `${__dirname}/../${this.getProjectFolderPath()}`
-				}, (err, code, signal) => {
+
+			let runnerOptions = this.options.reduce((result, opt) => {
+				result[opt.name] = opt.value;
+				return result;
+			}, {});
+
+			runnerOptions["project-path"] = `${__dirname}/../${this.getProjectFolderPath()}`;
+			runnerOptions["pmvs-num-cores"] = os.cpus().length;
+
+			this.runnerProcess = odmRunner.run(runnerOptions, (err, code, signal) => {
 					if (err){
 						this.setStatus(statusCodes.FAILED, {errorMessage: `Could not start process (${err.message})`});
 						finished();

libs/TaskManager.js

@@ -17,16 +17,19 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 "use strict";
 let assert = require('assert');
+let config = require('../config');
+let rmdir = require('rimraf');
 let fs = require('fs');
+let path = require('path');
 let logger = require('./logger');
 let Task = require('./Task');
 let statusCodes = require('./statusCodes');
 let async = require('async');
 let schedule = require('node-schedule');
 
-const PARALLEL_QUEUE_PROCESS_LIMIT = 2;
-const TASKS_DUMP_FILE = "data/tasks.json";
-const CLEANUP_TASKS_IF_OLDER_THAN = 1000 * 60 * 60 * 24 * 3; // 3 days
+const DATA_DIR = "data";
+const TASKS_DUMP_FILE = `${DATA_DIR}/tasks.json`;
+const CLEANUP_TASKS_IF_OLDER_THAN = 1000 * 60 * 60 * 24 * config.cleanupTasksAfter; // days
 
 module.exports = class TaskManager{
 	constructor(done){
@@ -36,13 +39,14 @@ module.exports = class TaskManager{
 		async.series([
 			cb => this.restoreTaskListFromDump(cb),
 			cb => this.removeOldTasks(cb),
+			cb => this.removeOrphanedDirectories(cb),
 			cb => {
 				this.processNextTask();
 				cb();
 			},
 			cb => {
 				// Every hour
-				schedule.scheduleJob('* 0 * * * *', () => {
+				schedule.scheduleJob('0 * * * *', () => {
 					this.removeOldTasks();
 					this.dumpTaskList();
 				});
@@ -76,6 +80,28 @@ module.exports = class TaskManager{
 		}, done);
 	}
 
+	// Removes directories that don't have a corresponding
+	// task associated with it (maybe as a cause of an abrupt exit)
+	removeOrphanedDirectories(done){
+		logger.info("Checking for orphaned directories to be removed...");
+
+		fs.readdir(DATA_DIR, (err, entries) => {
+			if (err) done(err);
+			else{
+				async.eachSeries(entries, (entry, cb) => {
+					let dirPath = path.join(DATA_DIR, entry);
+					if (fs.statSync(dirPath).isDirectory() &&
+						entry.match(/[\w\d]+\-[\w\d]+\-[\w\d]+\-[\w\d]+\-[\w\d]+/) &&
+						!this.tasks[entry]){
+						logger.info(`Found orphaned directory: ${entry}, removing...`);
+						rmdir(dirPath, cb);
+					}else cb();
+				}, done);
+			}
+		});
+	}
+
+
 	// Load tasks that already exists (if any)
 	restoreTaskListFromDump(done){
 		fs.readFile(TASKS_DUMP_FILE, (err, data) => {
@@ -113,7 +139,7 @@ module.exports = class TaskManager{
 	// Finds the next tasks, adds them to the running queue,
 	// and starts the tasks (up to the limit).
 	processNextTask(){
-		if (this.runningQueue.length < PARALLEL_QUEUE_PROCESS_LIMIT){
+		if (this.runningQueue.length < config.parallelQueueProcessing){
 			let task = this.findNextTaskToProcess();
 			if (task){
 				this.addToRunningQueue(task);
@@ -122,7 +148,7 @@ module.exports = class TaskManager{
 					this.processNextTask();
 				});
 
-				if (this.runningQueue.length < PARALLEL_QUEUE_PROCESS_LIMIT) this.processNextTask();
+				if (this.runningQueue.length < config.parallelQueueProcessing) this.processNextTask();
 			}
 		}else{
 			// Do nothing

libs/logger.js

@@ -24,7 +24,9 @@ let path = require('path');
 
 // Set up logging
 // Configure custom File transport to write plain text messages
-let logPath = ( config.logger.logDirectory ? config.logger.logDirectory : __dirname );
+let logPath = ( config.logger.logDirectory ? 
+				config.logger.logDirectory : 
+				`${__dirname}/../` );
 
 // Check that log file directory can be written to
 try {
@@ -36,7 +38,12 @@ try {
 logPath += path.sep;
 logPath += config.instance + ".log";
 
-winston.add(winston.transports.File, {
+let logger = new (winston.Logger)({
+  transports: [
+    new (winston.transports.Console)({ level: config.logger.level }),
+  ]
+});
+logger.add(winston.transports.File, {
 		filename: logPath, // Write to projectname.log
 		json: false, // Write in plain text, not JSON
 		maxsize: config.logger.maxFileSize, // Max size of each file
@@ -46,7 +53,7 @@ winston.add(winston.transports.File, {
 
 if (config.deamon){
 	// Console transport is no use to us when running as a daemon
-	winston.remove(winston.transports.Console);
+	logger.remove(winston.transports.Console);
 }
 
-module.exports = winston;
+module.exports = logger;

libs/odmOptions.js

@@ -22,6 +22,10 @@ let assert = require('assert');
 let odmOptions = null;
 
 module.exports = {
+	initialize: function(done){
+		this.getOptions(done);
+	},
+
 	getOptions: function(done){
 		if (odmOptions){
 			done(null, odmOptions);

libs/odmRunner.js

@@ -21,13 +21,26 @@ let config = require('../config.js');
 
 module.exports = {
 	run: function(options = {
-			projectPath: "/images"
+			"project-path": "/images"
 		}, done, outputReceived){
 
+		let command = [`${config.odm_path}/run.py`];
+		for (var name in options){
+			let value = options[name];
+
+			// Skip false booleans
+			if (value === false) continue;
+
+			command.push("--" + name);
+
+			// We don't specify "--time true" (just "--time")
+			if (typeof value !== 'boolean'){
+				command.push(value);
+			}
+		}
+		
 		// Launch
-		let childProcess = spawn("python", [`${config.odm_path}/run.py`,
-				"--project-path", options.projectPath
-			], {cwd: config.odm_path});
+		let childProcess = spawn("python", command, {cwd: config.odm_path});
 
 		childProcess
 			.on('exit', (code, signal) => done(null, code, signal))