Merge remote-tracking branch 'pierotofy/dev' into dev

helpers/odmOptionsToJson.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+'''
+Node-OpenDroneMap Node.js App and REST API to access OpenDroneMap. 
+Copyright (C) 2016 Node-OpenDroneMap Contributors
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+'''
+
+import sys
+import imp
+import argparse
+import json
+
+imp.load_source('context', sys.argv[2] + '/opendm/context.py')
+odm = imp.load_source('config', sys.argv[2] + '/opendm/config.py')
+
+options = {}
+class ArgumentParserStub(argparse.ArgumentParser):
+	def add_argument(self, *args, **kwargs):
+		argparse.ArgumentParser.add_argument(self, *args, **kwargs)
+		options[args[0]] = {}
+		for name, value in kwargs.items():
+			options[args[0]][str(name)] = str(value)
+			
+odm.parser = ArgumentParserStub()
+odm.config()
+print json.dumps(options)

index.js

@@ -64,6 +64,7 @@ let winstonStream = {
 
 let TaskManager = require('./libs/taskManager');
 let Task = require('./libs/Task');
+let odmOptions = require('./libs/odmOptions');
 
 app.use(morgan('combined', { stream : winstonStream }));
 app.use(bodyParser.urlencoded({extended: true}));
@@ -93,8 +94,18 @@ let upload = multer({
 app.post('/task/new', addRequestId, upload.array('images'), (req, res) => {
 	if (req.files.length === 0) res.json({error: "Need at least 1 file."});
 	else{
-		// Move to data
 		async.series([
+			cb => {
+				odmOptions.filterOptions(req.body.options, (err, options) => {
+					if (err) cb(err);
+					else{
+						req.body.options = options;
+						cb(null);
+					}
+				});
+			},
+
+			// Move uploads to data dir
 			cb => {
 				fs.stat(`data/${req.id}`, (err, stat) => {
 					if (err && err.code === 'ENOENT') cb();
@@ -104,7 +115,13 @@ app.post('/task/new', addRequestId, upload.array('images'), (req, res) => {
 			cb => { fs.mkdir(`data/${req.id}`, undefined, cb); },
 			cb => {
 				fs.rename(`tmp/${req.id}`, `data/${req.id}/images`, err => {
-					if (!err){
+					if (!err) cb();
+					else cb(new Error("Could not move images folder."))
+				});
+			},
+
+			// Create task
+			cb => {
 				new Task(req.id, req.body.name, (err, task) => {
 					if (err) cb(err);
 					else{
@@ -112,11 +129,7 @@ app.post('/task/new', addRequestId, upload.array('images'), (req, res) => {
 						res.json({uuid: req.id, success: true});
 						cb();
 					}
-						});
+				}, req.body.options);
-					}else{
-						cb(new Error("Could not move images folder."))
-					}
-				});
 			}
 		], err => {
 			if (err) res.json({error: err.message})
@@ -172,9 +185,16 @@ app.post('/task/restart', uuidCheck, (req, res) => {
 	taskManager.restart(req.body.uuid, successHandler(res));
 });
 
+app.get('/getOptions', (req, res) => {
+	odmOptions.getOptions((err, options) => {
+		if (err) res.json({error: err.message});
+		else res.json(options);
+	});
+});
+
 let gracefulShutdown = done => {
 	async.series([
-		cb => { taskManager.dumpTaskList(cb) },
+		cb => taskManager.dumpTaskList(cb),
 		cb => {
 			logger.info("Closing server");
 			server.close();

libs/Task.js

@@ -25,7 +25,7 @@ let archiver = require('archiver');
 let statusCodes = require('./statusCodes');
 
 module.exports = class Task{
-	constructor(uuid, name, done){
+	constructor(uuid, name, done, options = []){
 		assert(uuid !== undefined, "uuid must be set");
 		assert(done !== undefined, "ready must be set");
 
@@ -34,10 +34,12 @@ module.exports = class Task{
 		this.dateCreated = new Date().getTime();
 		this.processingTime = -1;
 		this.setStatus(statusCodes.QUEUED);
-		this.options = {};
+		this.options = options;
 		this.output = [];
 		this.runnerProcess = null;
 
+		this.options.forEach(option => { console.log(option); });
+
 		// Read images info
 		fs.readdir(this.getImagesFolderPath(), (err, files) => {
 			if (err) done(err);
@@ -64,7 +66,7 @@ module.exports = class Task{
 				}
 				done(null, task);
 			}
-		})
+		}, taskJson.options);
 	}
 
 	// Get path where images are stored for this task
@@ -137,7 +139,7 @@ module.exports = class Task{
 			this.setStatus(statusCodes.CANCELED);
 
 			if (wasRunning && this.runnerProcess){
-				// TODO: this does guarantee that
+				// TODO: this does NOT guarantee that
 				// the process will immediately terminate.
 				// In fact, often times ODM will continue running for a while
 				// This might need to be fixed on ODM's end.

libs/TaskManager.js

@@ -34,8 +34,8 @@ module.exports = class TaskManager{
 		this.runningQueue = [];
 
 		async.series([
-			cb => { this.restoreTaskListFromDump(cb); },
+			cb => this.restoreTaskListFromDump(cb),
-			cb => { this.removeOldTasks(cb); },
+			cb => this.removeOldTasks(cb),
 			cb => {
 				this.processNextTask();
 				cb();
@@ -136,10 +136,7 @@ module.exports = class TaskManager{
 
 	removeFromRunningQueue(task){
 		assert(task.constructor.name === "Task", "Must be a Task object");
-
+		this.runningQueue = this.runningQueue.filter(t => t !== task);
-		this.runningQueue = this.runningQueue.filter(t => {
-			return t !== task;
-		});
 	}
 
 	addNew(task){

libs/odmOptions.js

@@ -0,0 +1,210 @@
+/* 
+Node-OpenDroneMap Node.js App and REST API to access OpenDroneMap. 
+Copyright (C) 2016 Node-OpenDroneMap Contributors
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+"use strict";
+let odmRunner = require('./odmRunner');
+let assert = require('assert');
+
+let odmOptions = null;
+
+module.exports = {
+	getOptions: function(done){
+		if (odmOptions){
+			done(null, odmOptions);
+			return;
+		}
+
+		odmRunner.getJsonOptions((err, json) => {
+			if (err) done(err);
+			else{
+				odmOptions = [];
+				for (let option in json){
+					// Not all options are useful to the end user
+					// (num cores can be set programmatically, so can gcpFile, etc.)
+					if (["-h", "--project-path", 
+						"--zip-results", "--pmvs-num-cores", "--odm_georeferencing-useGcp",
+						"--start-with", "--odm_georeferencing-gcpFile", "--end-with"].indexOf(option) !== -1) continue;
+
+					let values = json[option];
+
+					let name = option.replace(/^--/, "");
+					let type = "";
+					let value = "";
+					let help = values.help || "";
+					let domain = values.metavar !== undefined ? 
+								 values.metavar.replace(/^[<>]/g, "")
+								 				.replace(/[<>]$/g, "")
+							 					.trim() : 
+								 "";
+
+					switch((values.type || "").trim()){
+						case "<type 'int'>":
+							type = "int";
+							value = values['default'] !== undefined ? 
+											parseInt(values['default']) :
+											0;
+							break;
+						case "<type 'float'>":
+							type = "float";
+							value = values['default'] !== undefined ? 
+											parseFloat(values['default']) :
+											0.0;
+							break;
+						default:
+							type = "string";
+							value = values['default'] !== undefined ? 
+								    values['default'].trim() :
+								    "";
+					}
+
+					if (values['default'] === "True"){
+						type = "bool";
+						value = true;
+					}else if (values['default'] === "False"){
+						type = "bool";
+						value = false;
+					}
+
+					help = help.replace(/\%\(default\)s/g, value);
+
+					odmOptions.push({
+						name, type, value, domain, help
+					});
+				}
+				done(null, odmOptions);
+			}
+		});
+	},
+
+	// Checks that the options (as received from the rest endpoint)
+	// Are valid and within proper ranges.
+	// The result of filtering is passed back via callback
+	// @param options[]
+	filterOptions: function(options, done){
+		assert(odmOptions !== null, "odmOptions is not set. Have you initialized odmOptions properly?");
+		
+		try{
+			if (typeof options === "string") options = JSON.parse(options);
+			
+			let result = [];
+			let errors = [];
+			function addError(opt, descr){
+				errors.push({
+					name: opt.name,
+					error: descr
+				});
+			}
+
+			let typeConversion = {
+				'float': Number.parseFloat,
+				'int': Number.parseInt,
+				'bool': function(value){
+					if (value === 'true') return true;
+					else if (value === 'false') return false;
+					else if (typeof value === 'boolean') return value;
+					else throw new Error(`Cannot convert ${value} to boolean`);
+				}
+			};
+			
+			let domainChecks = [
+				{
+					regex: /^(positive |negative )?(integer|float)$/, 
+					validate: function(matches, value){
+						if (matches[1] === 'positive ') return value >= 0;
+						else if (matches[1] === 'negative ') return value <= 0;
+						
+						else if (matches[2] === 'integer') return Number.isInteger(value);
+						else if (matches[2] === 'float') return Number.isFinite(value);
+					}
+				},
+				{
+					regex: /^percent$/,
+					validate: function(matches, value){
+						return value >= 0 && value <= 100;
+					}
+				},
+				{
+					regex: /^(float): ([\-\+\.\d]+) <= x <= ([\-\+\.\d]+)$/,
+					validate: function(matches, value){
+						let [str, type, lower, upper] = matches;
+						lower = parseFloat(lower);
+						upper = parseFloat(upper);
+						return value >= lower && value <= upper;						
+					}
+				},
+				{
+					regex: /^(float) (>=|>|<|<=) ([\-\+\.\d]+)$/,
+					validate: function(matches, value){
+						let [str, type, oper, bound] = matches;
+						bound = parseFloat(bound);
+						switch(oper){
+							case '>=':
+								return value >= bound;
+							case '>':
+								return value > bound;
+							case '<=':
+								return value <= bound;
+							case '<':
+								return value < bound;
+							default:
+								return false;
+						}
+					}
+				}			
+			];
+
+			function checkDomain(domain, value){
+				let dc, matches;
+
+				if (dc = domainChecks.find(dc => matches = domain.match(dc.regex))){
+					if (!dc.validate(matches, value)) throw new Error(`Invalid value ${value} (out of range)`);
+				}else{
+					throw new Error(`Domain value cannot be handled: '${domain}' : '${value}'`);
+				}
+			}
+
+			// Scan through all possible options
+			for (let odmOption of odmOptions){
+				// Was this option selected by the user?
+				let opt;
+				if (opt = options.find(o => o.name === odmOption.name)){
+					try{
+						// Convert to proper data type
+						let value = typeConversion[odmOption.type](opt.value);
+
+						// Domain check
+						if (odmOption.domain){
+							checkDomain(odmOption.domain, value);
+						}
+
+						result.push({
+							name: odmOption.name,
+							value: value
+						});
+					}catch(e){
+						addError(opt, e.message);						
+					}
+				}
+			}
+
+			if (errors.length > 0) done(new Error(JSON.stringify(errors)));
+			else done(null, result);
+		}catch(e){
+			done(e);
+		}
+	}
+};

libs/odmRunner.js

@@ -30,19 +30,36 @@ module.exports = {
 				"--project-path", options.projectPath
 			], {cwd: ODM_PATH});
 
+		childProcess
+			.on('exit', (code, signal) => done(null, code, signal))
+			.on('error', done);
+
+		childProcess.stdout.on('data', chunk => outputReceived(chunk.toString()));
+		childProcess.stderr.on('data', chunk => outputReceived(chunk.toString()));
+
+		return childProcess;
+	},
+
+	getJsonOptions: function(done){
+		// Launch
+		let childProcess = spawn("python", [`${__dirname}/../helpers/odmOptionsToJson.py`, 
+				"--project-path", ODM_PATH]);
+		let output = [];
+
 		childProcess
 			.on('exit', (code, signal) => {
-				done(null, code, signal);
+				try{
+					let json = JSON.parse(output.join(""));
+					done(null, json);
+				}catch(err){
+					done(err);	
+				}
 			})
 			.on('error', done);
 
-		childProcess.stdout.on('data', chunk => {
+		let processOutput = chunk => output.push(chunk.toString());
-		  outputReceived(chunk.toString());
-		});
-		childProcess.stderr.on('data', chunk => {
-		  outputReceived(chunk.toString());
-		});
 
-		return childProcess;
+		childProcess.stdout.on('data', processOutput);
+		childProcess.stderr.on('data', processOutput);
 	}
 };

public/css/main.css

@@ -56,3 +56,11 @@
     font-family: monospace;
     font-size: 90%;
 }
+
+.selectric-items li{
+    background: #fff;
+}
+
+#options .checkbox{
+    margin-right: 143px;
+}

public/index.html

@@ -15,7 +15,6 @@
             }
         </style>
         <link href="css/fileinput.css" media="all" rel="stylesheet" type="text/css" />
-
         <link rel="stylesheet" href="css/main.css">
 
         <script src="js/vendor/modernizr-2.8.3.min.js"></script>
@@ -47,14 +46,36 @@
             <label for="images">Aerial Imageries:</label> <input id="images" name="images" multiple type="file">
             <div id="errorBlock" class="help-block"></div>
         </div>
-                <div class="form-group">
-					<!-- <label>Options:</label> -->
-                </div>
-
         <div class="text-right"><input type="submit" class="btn btn-success" value="Start Task" id="btnUpload" /></div>
+        <div id="options">
+          <div class="form-inline form-group form-horizontal">
+             <div data-bind="visible: error(), text: error()" class="alert alert-warning" role="alert"></div>
+             <button style="position: relative; top: -45px;" type="submit" class="btn btn-default" data-bind="visible: !error(), click: function(){ showOptions(!showOptions()); }, text: (showOptions() ? 'Hide' : 'Show') + ' Options'"></button>
+            
+             <div data-bind="visible: showOptions()">
+                <div data-bind="foreach: options">
+                 <label data-bind="text: properties.name + (properties.domain ? ' (' + properties.domain + ')' : '')"></label><br/>
+                 <!-- ko if: properties.type !== 'bool' -->
+                 <input type="text" class="form-control" data-bind="attr: {placeholder: properties.value}, value: value">
+                 <!-- /ko -->
+                 <!-- ko if: properties.type === 'bool' -->
+                 <div class="checkbox">
+                  <label>
+                    <input type="checkbox" data-bind="checked: value"> Enable
+                  </label>
+                </div>
+                 <!-- /ko -->
+                 <button type="submit" class="btn glyphicon glyphicon-info-sign btn-info" data-toggle="tooltip" data-placement="top" data-bind="attr: {title: properties.help}"></button>
+                 <button type="submit" class="btn glyphicon glyphicon glyphicon-repeat btn-default" data-toggle="tooltip" data-placement="top" title="Reset to default" data-bind="click: resetToDefault"></button>
+
+                 <br/><br/>
+              </div>
+            </div>
+          </div>
+        </div>
       </form>
 		</div>
-        <div class="col-md-7">
+        <div class="col-md-7" id="taskList">
           <h2>Current Tasks (<span data-bind="text: tasks().length"></span>)</h2>
           <p data-bind="visible: tasks().length === 0">No running tasks.</p>
           <div data-bind="foreach: tasks">
@@ -113,6 +134,7 @@
         <script src="js/vendor/bootstrap.min.js"></script>
         <script src="js/vendor/knockout-3.4.0.js"></script>
         <script src="js/fileinput.js" type="text/javascript"></script>
+
         <script src="js/main.js"></script>
     </body>
 </html>

public/js/main.js

@@ -149,8 +149,8 @@ $(function(){
          })
          .always(function(){ self.loading(false); });
     };
-    Task.prototype.consoleMouseOver = function(){ this.autoScrollOutput = false; }
+    Task.prototype.consoleMouseOver = function(){ this.autoScrollOutput = false; };
-    Task.prototype.consoleMouseOut = function(){ this.autoScrollOutput = true; } 
+    Task.prototype.consoleMouseOut = function(){ this.autoScrollOutput = true; };
     Task.prototype.resetOutput = function(){
         this.viewOutputLine = 0;
         this.autoScrollOutput = true;
@@ -170,7 +170,7 @@ $(function(){
                     self.viewOutputLine += output.length;
                     if (self.autoScrollOutput){
                         var $console = $("#console_" + self.uuid);
-                        $console.scrollTop($console[0].scrollHeight - $console.height())
+                        $console.scrollTop($console[0].scrollHeight - $console.height());
                     }
                 }
              })
@@ -243,8 +243,8 @@ $(function(){
                 self.info({error: url + " is unreachable."});
                 self.stopRefreshingInfo();
             });
-        }
         };
+    }
     Task.prototype.cancel = genApiCall("/task/cancel");
     Task.prototype.restart = genApiCall("/task/restart", function(task){
         task.resetOutput();
@@ -254,7 +254,7 @@ $(function(){
     };
 
     var taskList = new TaskList();
-    ko.applyBindings(taskList);
+    ko.applyBindings(taskList, document.getElementById('taskList'));
 
     // Handle uploads
     $("#images").fileinput({
@@ -266,7 +266,8 @@ $(function(){
         uploadAsync: false,
         uploadExtraData: function(){
             return {
-                name: $("#taskName").val()
+                name: $("#taskName").val(),
+                options: JSON.stringify(optionsModel.getUserOptions())
             };
         }
     });
@@ -294,4 +295,55 @@ $(function(){
         })
         .on('filebatchuploaderror', function(e, data, msg){
         });
+
+    // Load options
+    function Option(properties){
+        this.properties = properties;
+        this.value = ko.observable();
+    }
+    Option.prototype.resetToDefault = function(){
+        this.value(undefined);
+    };
+
+    function OptionsModel(){
+        var self = this;
+
+        this.options = ko.observableArray();
+        this.options.subscribe(function(){
+            setTimeout(function(){
+                $('#options [data-toggle="tooltip"]').tooltip();
+            }, 100);
+        });
+        this.showOptions = ko.observable(false);
+        this.error = ko.observable();
+
+        $.get("/getOptions")
+         .done(function(json){
+            if (json.error) self.error(json.error);
+            else{
+                for (var i in json){
+                    self.options.push(new Option(json[i]));
+                }
+            }
+         })
+         .fail(function(){
+            self.error("options are not available.");
+         });
+    }
+    OptionsModel.prototype.getUserOptions = function(){
+        var result = [];
+        for (var i = 0; i < this.options().length; i++){
+            var opt = this.options()[i];
+            if (opt.value() !== undefined){
+                result.push({
+                    name: opt.properties.name,
+                    value: opt.value()
+                });
+            }
+        }
+        return result;
+    };
+
+    var optionsModel = new OptionsModel();
+    ko.applyBindings(optionsModel, document.getElementById("options"));
 });