Merged dev into master

2016-08-02 11:28:17 -05:00 · 2016-08-02 11:28:17 -05:00 · 5286132e09
commit 5286132e09
--- a/.jshintrc
+++ b/.jshintrc
@ -0,0 +1,4 @@
+{
+	"esnext": true,
+	"node": true
+}
--- a/README.md
+++ b/README.md
@ -1,11 +1,15 @@
 # Open Source Drone Aerial Imagery Processing
 node-OpenDroneMap is a Node.js App and REST API to access [OpenDroneMap](https://github.com/OpenDroneMap/OpenDroneMap)

+[http://nodeodm.masseranolabs.com](http://nodeodm.masseranolabs.com)
+
 ![Alt text](/screenshots/main.png?raw=true "Node-OpenDroneMap")

 ## Getting Started

-The quickest way is to use [Docker](https://www.docker.com/).
+For a quick taste of the application, we have setup a test environment at [http://nodeodm.masseranolabs.com](http://nodeodm.masseranolabs.com). Please note that **this is not a production environment**, and that processing on this server will be slow (you are sharing the server's resources with everyone else in the world).
+
+If you want to do your own imagery processing, we recommend that you setup your own instance via [Docker](https://www.docker.com/).

 * From the Docker Quickstart Terminal (Windows / OSX) or from the command line (Linux) type:
 ```
@ -41,6 +45,54 @@ npm install
 node index.js
 ```

+You may need to specify your ODM project path to start the server:
+
+```
+node index.js --odm_path /home/username/OpenDroneMap
+```
+
+If you want to start node ODM on a different port you can do the following:
+
+```
+node index.js --port 8000 --odm_path /home/username/OpenDroneMap
+```
+
+For other command line options you can run:
+
+```
+node index.js --help
+```
+
+You can also specify configuration values via a JSON file:
+
+```
+node index.js --config config.default.json
+```
+
+Command line arguments always take precedence over the configuration file.
+
+### Run it using PM2
+
+The app can also be run as a background process using the [pm2 process manager](https://github.com/Unitech/pm2), which can also assist you with system startup scripts and process monitoring.
+
+To install pm2, run (using `sudo` if required):
+```shell
+npm install pm2 -g
+```
+The app can then be started using
+```shell
+pm2 start processes.json
+```
+To have pm2 started on OS startup run
+```shell
+pm2 save
+pm2 startup
+```
+and then run the command as per the instructions that prints out. If that command errors then you may have to specify the system (note that systemd should be used on CentOS 7). Note that if the process is not running as root (recommended) you will need to change `/etc/init.d/pm2-init.sh` to set `export PM2_HOME="/path/to/user/home/.pm2"`, as per [these instructions](
+http://www.buildsucceeded.com/2015/solved-pm2-startup-at-boot-time-centos-7-red-hat-linux/)
+
+You can monitor the process using `pm2 status`.
+
 ### Test Images

 You can find some test drone images from [OpenDroneMap's Test Data Folder](https://github.com/OpenDroneMap/OpenDroneMap/tree/master/tests/test_data/images).
@ -51,8 +103,10 @@ Make a pull request to the dev branch for small contributions. For big contribut

 ## Roadmap

- [ ] Command line options for OpenDroneMap (in progress)
- [ ] Cluster tasks distribution to multiple servers
+- [X] Command line options for OpenDroneMap
+- [X] GPC List support
+- [ ] Autoremove Abandoned Tasks
+- [ ] Continuous Integration Setup
 - [ ] Documentation
 - [ ] Unit Testing

--- a/config-default.json
+++ b/config-default.json
@ -0,0 +1,16 @@
+{
+	"instance": "node-OpenDroneMap",
+	"odm_path": "/code",
+
+	"logger": {
+		"level": "info",
+		"maxFileSize": 104857600,
+		"maxFiles": 10,
+		"logDirectory": ""
+	},
+
+	"port": 3000,
+	"deamon": false,
+	"parallelQueueProcessing": 2,
+	"cleanupTasksAfter": 3
+}
--- a/config.js
+++ b/config.js
@ -17,19 +17,22 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 'use strict';

+let fs = require('fs');
 let argv = require('minimist')(process.argv.slice(2));
+let utils = require('./libs/utils');
+
 if (argv.help){
 	console.log(`
 Usage: node index.js [options]

 Options:
+	--config	<path>	Path to the configuration file (default: config-default.json)	
 	-p, --port <number> 	Port to bind the server to (default: 3000)
 	--odm_path <path>	Path to OpenDroneMap's code	(default: /code)
 	--log_level <logLevel>	Set log level verbosity (default: info)
 	-d, --deamonize 	Set process to run as a deamon
 	--parallel_queue_processing <number> Number of simultaneous processing tasks (default: 2)
 	--cleanup_tasks_after <number> Number of days that elapse before deleting finished and canceled tasks (default: 3) 
-
 Log Levels: 
 error | debug | info | verbose | debug | silly 
 `);
@ -38,20 +41,40 @@ error | debug | info | verbose | debug | silly

 let config = {};

+// Read configuration from file
+let configFilePath = argv.config || "config-default.json";
+let configFile = {};
+
+if (/\.json$/i.test(configFilePath)){
+	try{
+		let data = fs.readFileSync(configFilePath);
+		configFile = JSON.parse(data.toString());
+	}catch(e){
+		console.log(`Invalid configuration file ${configFilePath}`);
+		process.exit(1);
+	}
+}
+
+// Gets a property that might not exist from configuration file
+// example: fromConfigFile("logger.maxFileSize", 1000);
+function fromConfigFile(prop, defaultValue){
+	return utils.get(configFile, prop, defaultValue);
+}
+
 // Instance name - default name for this configuration
-config.instance = 'node-OpenDroneMap';
-config.odm_path = argv.odm_path || '/code';
+config.instance = fromConfigFile("instance", 'node-OpenDroneMap');
+config.odm_path = argv.odm_path || fromConfigFile("odm_path", '/code');

 // Logging configuration
 config.logger = {};
-config.logger.level = argv.log_level || 'info'; // What level to log at; info, verbose or debug are most useful. Levels are (npm defaults): silly, debug, verbose, info, warn, error.
-config.logger.maxFileSize = 1024 * 1024 * 100; // Max file size in bytes of each log file; default 100MB
-config.logger.maxFiles = 10; // Max number of log files kept
-config.logger.logDirectory = ''; // Set this to a full path to a directory - if not set logs will be written to the application directory.
+config.logger.level = argv.log_level || fromConfigFile("logger.level", 'info'); // What level to log at; info, verbose or debug are most useful. Levels are (npm defaults): silly, debug, verbose, info, warn, error.
+config.logger.maxFileSize = fromConfigFile("logger.maxFileSize", 1024 * 1024 * 100); // Max file size in bytes of each log file; default 100MB
+config.logger.maxFiles = fromConfigFile("logger.maxFiles", 10); // Max number of log files kept
+config.logger.logDirectory = fromConfigFile("logger.logDirectory", ''); // Set this to a full path to a directory - if not set logs will be written to the application directory.

-config.port = parseInt(argv.port || argv.p || process.env.PORT || 3000);
-config.deamon = argv.deamonize || argv.d;
-config.parallelQueueProcessing = argv.parallel_queue_processing || 2;
-config.cleanupTasksAfter = argv.cleanup_tasks_after || 3;
+config.port = parseInt(argv.port || argv.p || fromConfigFile("port", process.env.PORT || 3000));
+config.deamon = argv.deamonize || argv.d || fromConfigFile("daemon", false);
+config.parallelQueueProcessing = argv.parallel_queue_processing || fromConfigFile("parallelQueueProcessing", 2);
+config.cleanupTasksAfter = argv.cleanup_tasks_after || fromConfigFile("cleanupTasksAfter", 3);

 module.exports = config;
--- a/index.js
+++ b/index.js
@ -61,11 +61,14 @@ let upload = multer({
 	  	});
 	  },
 	  filename: (req, file, cb) => {
-	    cb(null, file.originalname)
+	    cb(null, file.originalname);
 	  }
 	})
 });

+let taskManager;
+let server;
+
 app.post('/task/new', addRequestId, upload.array('images'), (req, res) => {
 	if (req.files.length === 0) res.json({error: "Need at least 1 file."});
 	else{
@ -121,7 +124,7 @@ app.post('/task/new', addRequestId, upload.array('images'), (req, res) => {
 				}, req.body.options);
 			}
 		], err => {
-			if (err) res.json({error: err.message})
+			if (err) res.json({error: err.message});
 		});
 	}
 });
@ -132,7 +135,7 @@ let getTaskFromUuid = (req, res, next) => {
 		req.task = task;
 		next();
 	}else res.json({error: `${req.params.uuid} not found`});
-}
+};

 app.get('/task/:uuid/info', getTaskFromUuid, (req, res) => {
 	res.json(req.task.getInfo());
@ -200,9 +203,6 @@ process.on ('SIGTERM', gracefulShutdown);
 process.on ('SIGINT', gracefulShutdown);

 // Startup
-let taskManager;
-let server;
-
 async.series([
 	cb => odmOptions.initialize(cb),
 	cb => { taskManager = new TaskManager(cb); },
--- a/libs/Task.js
+++ b/libs/Task.js
@ -35,7 +35,7 @@ module.exports = class Task{
 		assert(done !== undefined, "ready must be set");

 		this.uuid = uuid;
-		this.name = name != "" ? name : "Task of " + (new Date()).toISOString();
+		this.name = name !== "" ? name : "Task of " + (new Date()).toISOString();
 		this.dateCreated = new Date().getTime();
 		this.processingTime = -1;
 		this.setStatus(statusCodes.QUEUED);
@ -51,7 +51,7 @@ module.exports = class Task{
 					if (err) cb(err);
 					else{
 						this.images = files;
-						logger.debug(`Found ${this.images.length} images for ${this.uuid}`)
+						logger.debug(`Found ${this.images.length} images for ${this.uuid}`);
 						cb(null);
 					}
 				});
@ -190,6 +190,11 @@ module.exports = class Task{
 	// Starts processing the task with OpenDroneMap
 	// This will spawn a new process.
 	start(done){
+		const finished = err => {
+			this.stopTrackingProcessingTime();
+			done(err);
+		};
+		
 		const postProcess = () => {
 			let output = fs.createWriteStream(this.getAssetsArchivePath());
 			let archive = archiver.create('zip', {});
@ -214,11 +219,6 @@ module.exports = class Task{
 			  .finalize();
 		};

-		const finished = err => {
-			this.stopTrackingProcessingTime();
-			done(err);
-		};
-
 		if (this.status.code === statusCodes.QUEUED){
 			this.startTrackingProcessingTime();
 			this.setStatus(statusCodes.RUNNING);
@ -289,7 +289,7 @@ module.exports = class Task{
 			status: this.status,
 			options: this.options,
 			imagesCount: this.images.length
-		}
+		};
 	}

 	// Returns the output of the OpenDroneMap process
@ -307,6 +307,6 @@ module.exports = class Task{
 			dateCreated: this.dateCreated,
 			status: this.status,
 			options: this.options
-		}
+		};
 	}
 };
--- a/libs/TaskManager.js
+++ b/libs/TaskManager.js
@ -61,7 +61,7 @@ module.exports = class TaskManager{
 	removeOldTasks(done){
 		let list = [];
 		let now = new Date().getTime();
-		logger.info("Checking for old tasks to be removed...");
+		logger.debug("Checking for old tasks to be removed...");

 		for (let uuid in this.tasks){
 			let task = this.tasks[uuid];
@ -75,7 +75,7 @@ module.exports = class TaskManager{
 		}

 		async.eachSeries(list, (uuid, cb) => {
-			logger.info(`Cleaning up old task ${uuid}`)
+			logger.info(`Cleaning up old task ${uuid}`);
 			this.remove(uuid, cb);
 		}, done);
 	}
@ -174,8 +174,8 @@ module.exports = class TaskManager{
 	// Stops the execution of a task
 	// (without removing it from the system).
 	cancel(uuid, cb){
-		let task;
-		if (task = this.find(uuid, cb)){
+		let task = this.find(uuid, cb);
+		if (task){
 			if (!task.isCanceled()){
 				task.cancel(err => {
 					this.removeFromRunningQueue(task);
@ -193,8 +193,8 @@ module.exports = class TaskManager{
 	remove(uuid, cb){
 		this.cancel(uuid, err => {
 			if (!err){
-				let task;
-				if (task = this.find(uuid, cb)){
+				let task = this.find(uuid, cb);
+				if (task){
 					task.cleanup(err => {
 						if (!err){
 							delete(this.tasks[uuid]);
@ -210,8 +210,8 @@ module.exports = class TaskManager{
 	// Restarts (puts back into QUEUED state)
 	// a task that is either in CANCELED or FAILED state.
 	restart(uuid, cb){
-		let task;
-		if (task = this.find(uuid, cb)){
+		let task = this.find(uuid, cb);
+		if (task){
 			task.restart(err => {
 				if (!err) this.processNextTask();
 				cb(err);
@ -237,8 +237,8 @@ module.exports = class TaskManager{

 		fs.writeFile(TASKS_DUMP_FILE, JSON.stringify(output), err => {
 			if (err) logger.error(`Could not dump tasks: ${err.message}`);
-			else logger.info("Dumped tasks list.");
+			else logger.debug("Dumped tasks list.");
 			if (done !== undefined) done();
-		})
+		});
 	}
 };
--- a/libs/logger.js
+++ b/libs/logger.js
@ -49,7 +49,7 @@ logger.add(winston.transports.File, {
 		maxsize: config.logger.maxFileSize, // Max size of each file
 		maxFiles: config.logger.maxFiles, // Max number of files
 		level: config.logger.level // Level of log messages
-	})
+	});

 if (config.deamon){
 	// Console transport is no use to us when running as a daemon
--- a/libs/odmOptions.js
+++ b/libs/odmOptions.js
@ -106,12 +106,12 @@ module.exports = {
 			
 			let result = [];
 			let errors = [];
-			function addError(opt, descr){
+			let addError = function(opt, descr){
 				errors.push({
 					name: opt.name,
 					error: descr
 				});
-			}
+			};

 			let typeConversion = {
 				'float': Number.parseFloat,
@ -171,21 +171,22 @@ module.exports = {
 				}			
 			];

-			function checkDomain(domain, value){
-				let dc, matches;
+			let checkDomain = function(domain, value){
+				let matches,
+					dc = domainChecks.find(dc => matches = domain.match(dc.regex));

-				if (dc = domainChecks.find(dc => matches = domain.match(dc.regex))){
+				if (dc){
 					if (!dc.validate(matches, value)) throw new Error(`Invalid value ${value} (out of range)`);
 				}else{
 					throw new Error(`Domain value cannot be handled: '${domain}' : '${value}'`);
 				}
-			}
+			};

 			// Scan through all possible options
 			for (let odmOption of odmOptions){
 				// Was this option selected by the user?
-				let opt;
-				if (opt = options.find(o => o.name === odmOption.name)){
+				let opt = options.find(o => o.name === odmOption.name);
+				if (opt){
 					try{
 						// Convert to proper data type
 						let value = typeConversion[odmOption.type](opt.value);
--- a/libs/odmRunner.js
+++ b/libs/odmRunner.js
@ -16,14 +16,14 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 "use strict";
+let assert = require('assert');
 let spawn = require('child_process').spawn;
 let config = require('../config.js');
 let logger = require('./logger');

 module.exports = {
-	run: function(options = {
-			"project-path": "/images"
-		}, done, outputReceived){
+	run: function(options, done, outputReceived){
+		assert(options["project-path"] !== undefined, "project-path must be defined");

 		let command = [`${config.odm_path}/run.py`];
 		for (var name in options){
--- a/libs/utils.js
+++ b/libs/utils.js
@ -0,0 +1,16 @@
+module.exports = {
+	get: function(scope, prop, defaultValue){
+		let parts = prop.split(".");
+		let current = scope;
+		for (let i = 0; i < parts.length; i++){
+			if (current[parts[i]] !== undefined && i < parts.length - 1){
+				current = current[parts[i]];
+			}else if (current[parts[i]] !== undefined && i < parts.length){
+				return current[parts[i]];
+			}else{
+				return defaultValue;
+			}
+		}	
+		return defaultValue;
+	}
+};
--- a/processes.json
+++ b/processes.json
@ -0,0 +1,6 @@
+[{
+  "script"  : "index.js",
+  "name"    : "node-OpenDroneMap",
+  "watch"   : false,
+  "ignore_watch" : ["[\\/\\\\]\\./", "node_modules", "git-hooks", "test", "tmp", "data", "^.*\.log$"]
+}]
--- a/public/index.html
+++ b/public/index.html
@ -43,7 +43,7 @@
 					<label for="taskName">Project Name:</lable> <input type="text" class="form-control" value="" id="taskName" />
 				</div>
 				<div class="form-group">
-            <label for="images">Aerial Imageries and GCP List (optional):</label> <input id="images" name="images" multiple type="file">
+            <label for="images">Aerial Images and GCP List (optional):</label> <input id="images" name="images" multiple type="file">
            <div id="errorBlock" class="help-block"></div>
        </div>
        <div class="text-right"><input type="submit" class="btn btn-success" value="Start Task" id="btnUpload" /></div>