kopia lustrzana https://github.com/OpenDroneMap/NodeODM
Support to load configuration from .json file
rodzic
3a45bac30a
commit
0d6ae6362d
|
@ -0,0 +1,4 @@
|
||||||
|
{
|
||||||
|
"esnext": true,
|
||||||
|
"node": true
|
||||||
|
}
|
|
@ -0,0 +1,16 @@
|
||||||
|
{
|
||||||
|
"instance": "node-OpenDroneMap",
|
||||||
|
"odm_path": "/code",
|
||||||
|
|
||||||
|
"logger": {
|
||||||
|
"level": "info",
|
||||||
|
"maxFileSize": 104857600,
|
||||||
|
"maxFiles": 10,
|
||||||
|
"logDirectory": ""
|
||||||
|
},
|
||||||
|
|
||||||
|
"port": 3000,
|
||||||
|
"deamon": false,
|
||||||
|
"parallelQueueProcessing": 2,
|
||||||
|
"cleanupTasksAfter": 3
|
||||||
|
}
|
45
config.js
45
config.js
|
@ -17,19 +17,22 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
*/
|
*/
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
|
let fs = require('fs');
|
||||||
let argv = require('minimist')(process.argv.slice(2));
|
let argv = require('minimist')(process.argv.slice(2));
|
||||||
|
let utils = require('./libs/utils');
|
||||||
|
|
||||||
if (argv.help){
|
if (argv.help){
|
||||||
console.log(`
|
console.log(`
|
||||||
Usage: node index.js [options]
|
Usage: node index.js [options]
|
||||||
|
|
||||||
Options:
|
Options:
|
||||||
|
--config <path> Path to the configuration file (default: config-default.json)
|
||||||
-p, --port <number> Port to bind the server to (default: 3000)
|
-p, --port <number> Port to bind the server to (default: 3000)
|
||||||
--odm_path <path> Path to OpenDroneMap's code (default: /code)
|
--odm_path <path> Path to OpenDroneMap's code (default: /code)
|
||||||
--log_level <logLevel> Set log level verbosity (default: info)
|
--log_level <logLevel> Set log level verbosity (default: info)
|
||||||
-d, --deamonize Set process to run as a deamon
|
-d, --deamonize Set process to run as a deamon
|
||||||
--parallel_queue_processing <number> Number of simultaneous processing tasks (default: 2)
|
--parallel_queue_processing <number> Number of simultaneous processing tasks (default: 2)
|
||||||
--cleanup_tasks_after <number> Number of days that elapse before deleting finished and canceled tasks (default: 3)
|
--cleanup_tasks_after <number> Number of days that elapse before deleting finished and canceled tasks (default: 3)
|
||||||
|
|
||||||
Log Levels:
|
Log Levels:
|
||||||
error | debug | info | verbose | debug | silly
|
error | debug | info | verbose | debug | silly
|
||||||
`);
|
`);
|
||||||
|
@ -38,20 +41,40 @@ error | debug | info | verbose | debug | silly
|
||||||
|
|
||||||
let config = {};
|
let config = {};
|
||||||
|
|
||||||
|
// Read configuration from file
|
||||||
|
let configFilePath = argv.config || "config-default.json";
|
||||||
|
let configFile = {};
|
||||||
|
|
||||||
|
if (/\.json$/i.test(configFilePath)){
|
||||||
|
try{
|
||||||
|
let data = fs.readFileSync(configFilePath);
|
||||||
|
configFile = JSON.parse(data.toString());
|
||||||
|
}catch(e){
|
||||||
|
console.log(`Invalid configuration file ${configFilePath}`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Gets a property that might not exist from configuration file
|
||||||
|
// example: fromConfigFile("logger.maxFileSize", 1000);
|
||||||
|
function fromConfigFile(prop, defaultValue){
|
||||||
|
return utils.get(configFile, prop, defaultValue);
|
||||||
|
}
|
||||||
|
|
||||||
// Instance name - default name for this configuration
|
// Instance name - default name for this configuration
|
||||||
config.instance = 'node-OpenDroneMap';
|
config.instance = fromConfigFile("instance", 'node-OpenDroneMap');
|
||||||
config.odm_path = argv.odm_path || '/code';
|
config.odm_path = argv.odm_path || fromConfigFile("odm_path", '/code');
|
||||||
|
|
||||||
// Logging configuration
|
// Logging configuration
|
||||||
config.logger = {};
|
config.logger = {};
|
||||||
config.logger.level = argv.log_level || 'info'; // What level to log at; info, verbose or debug are most useful. Levels are (npm defaults): silly, debug, verbose, info, warn, error.
|
config.logger.level = argv.log_level || fromConfigFile("logger.level", 'info'); // What level to log at; info, verbose or debug are most useful. Levels are (npm defaults): silly, debug, verbose, info, warn, error.
|
||||||
config.logger.maxFileSize = 1024 * 1024 * 100; // Max file size in bytes of each log file; default 100MB
|
config.logger.maxFileSize = fromConfigFile("logger.maxFileSize", 1024 * 1024 * 100); // Max file size in bytes of each log file; default 100MB
|
||||||
config.logger.maxFiles = 10; // Max number of log files kept
|
config.logger.maxFiles = fromConfigFile("logger.maxFiles", 10); // Max number of log files kept
|
||||||
config.logger.logDirectory = ''; // Set this to a full path to a directory - if not set logs will be written to the application directory.
|
config.logger.logDirectory = fromConfigFile("logger.logDirectory", ''); // Set this to a full path to a directory - if not set logs will be written to the application directory.
|
||||||
|
|
||||||
config.port = parseInt(argv.port || argv.p || process.env.PORT || 3000);
|
config.port = parseInt(argv.port || argv.p || fromConfigFile("port", process.env.PORT || 3000));
|
||||||
config.deamon = argv.deamonize || argv.d;
|
config.deamon = argv.deamonize || argv.d || fromConfigFile("daemon", false);
|
||||||
config.parallelQueueProcessing = argv.parallel_queue_processing || 2;
|
config.parallelQueueProcessing = argv.parallel_queue_processing || fromConfigFile("parallelQueueProcessing", 2);
|
||||||
config.cleanupTasksAfter = argv.cleanup_tasks_after || 3;
|
config.cleanupTasksAfter = argv.cleanup_tasks_after || fromConfigFile("cleanupTasksAfter", 3);
|
||||||
|
|
||||||
module.exports = config;
|
module.exports = config;
|
||||||
|
|
12
index.js
12
index.js
|
@ -61,11 +61,14 @@ let upload = multer({
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
filename: (req, file, cb) => {
|
filename: (req, file, cb) => {
|
||||||
cb(null, file.originalname)
|
cb(null, file.originalname);
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
});
|
});
|
||||||
|
|
||||||
|
let taskManager;
|
||||||
|
let server;
|
||||||
|
|
||||||
app.post('/task/new', addRequestId, upload.array('images'), (req, res) => {
|
app.post('/task/new', addRequestId, upload.array('images'), (req, res) => {
|
||||||
if (req.files.length === 0) res.json({error: "Need at least 1 file."});
|
if (req.files.length === 0) res.json({error: "Need at least 1 file."});
|
||||||
else{
|
else{
|
||||||
|
@ -121,7 +124,7 @@ app.post('/task/new', addRequestId, upload.array('images'), (req, res) => {
|
||||||
}, req.body.options);
|
}, req.body.options);
|
||||||
}
|
}
|
||||||
], err => {
|
], err => {
|
||||||
if (err) res.json({error: err.message})
|
if (err) res.json({error: err.message});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
@ -132,7 +135,7 @@ let getTaskFromUuid = (req, res, next) => {
|
||||||
req.task = task;
|
req.task = task;
|
||||||
next();
|
next();
|
||||||
}else res.json({error: `${req.params.uuid} not found`});
|
}else res.json({error: `${req.params.uuid} not found`});
|
||||||
}
|
};
|
||||||
|
|
||||||
app.get('/task/:uuid/info', getTaskFromUuid, (req, res) => {
|
app.get('/task/:uuid/info', getTaskFromUuid, (req, res) => {
|
||||||
res.json(req.task.getInfo());
|
res.json(req.task.getInfo());
|
||||||
|
@ -200,9 +203,6 @@ process.on ('SIGTERM', gracefulShutdown);
|
||||||
process.on ('SIGINT', gracefulShutdown);
|
process.on ('SIGINT', gracefulShutdown);
|
||||||
|
|
||||||
// Startup
|
// Startup
|
||||||
let taskManager;
|
|
||||||
let server;
|
|
||||||
|
|
||||||
async.series([
|
async.series([
|
||||||
cb => odmOptions.initialize(cb),
|
cb => odmOptions.initialize(cb),
|
||||||
cb => { taskManager = new TaskManager(cb); },
|
cb => { taskManager = new TaskManager(cb); },
|
||||||
|
|
18
libs/Task.js
18
libs/Task.js
|
@ -35,7 +35,7 @@ module.exports = class Task{
|
||||||
assert(done !== undefined, "ready must be set");
|
assert(done !== undefined, "ready must be set");
|
||||||
|
|
||||||
this.uuid = uuid;
|
this.uuid = uuid;
|
||||||
this.name = name != "" ? name : "Task of " + (new Date()).toISOString();
|
this.name = name !== "" ? name : "Task of " + (new Date()).toISOString();
|
||||||
this.dateCreated = new Date().getTime();
|
this.dateCreated = new Date().getTime();
|
||||||
this.processingTime = -1;
|
this.processingTime = -1;
|
||||||
this.setStatus(statusCodes.QUEUED);
|
this.setStatus(statusCodes.QUEUED);
|
||||||
|
@ -51,7 +51,7 @@ module.exports = class Task{
|
||||||
if (err) cb(err);
|
if (err) cb(err);
|
||||||
else{
|
else{
|
||||||
this.images = files;
|
this.images = files;
|
||||||
logger.debug(`Found ${this.images.length} images for ${this.uuid}`)
|
logger.debug(`Found ${this.images.length} images for ${this.uuid}`);
|
||||||
cb(null);
|
cb(null);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
@ -190,6 +190,11 @@ module.exports = class Task{
|
||||||
// Starts processing the task with OpenDroneMap
|
// Starts processing the task with OpenDroneMap
|
||||||
// This will spawn a new process.
|
// This will spawn a new process.
|
||||||
start(done){
|
start(done){
|
||||||
|
const finished = err => {
|
||||||
|
this.stopTrackingProcessingTime();
|
||||||
|
done(err);
|
||||||
|
};
|
||||||
|
|
||||||
const postProcess = () => {
|
const postProcess = () => {
|
||||||
let output = fs.createWriteStream(this.getAssetsArchivePath());
|
let output = fs.createWriteStream(this.getAssetsArchivePath());
|
||||||
let archive = archiver.create('zip', {});
|
let archive = archiver.create('zip', {});
|
||||||
|
@ -214,11 +219,6 @@ module.exports = class Task{
|
||||||
.finalize();
|
.finalize();
|
||||||
};
|
};
|
||||||
|
|
||||||
const finished = err => {
|
|
||||||
this.stopTrackingProcessingTime();
|
|
||||||
done(err);
|
|
||||||
};
|
|
||||||
|
|
||||||
if (this.status.code === statusCodes.QUEUED){
|
if (this.status.code === statusCodes.QUEUED){
|
||||||
this.startTrackingProcessingTime();
|
this.startTrackingProcessingTime();
|
||||||
this.setStatus(statusCodes.RUNNING);
|
this.setStatus(statusCodes.RUNNING);
|
||||||
|
@ -289,7 +289,7 @@ module.exports = class Task{
|
||||||
status: this.status,
|
status: this.status,
|
||||||
options: this.options,
|
options: this.options,
|
||||||
imagesCount: this.images.length
|
imagesCount: this.images.length
|
||||||
}
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// Returns the output of the OpenDroneMap process
|
// Returns the output of the OpenDroneMap process
|
||||||
|
@ -307,6 +307,6 @@ module.exports = class Task{
|
||||||
dateCreated: this.dateCreated,
|
dateCreated: this.dateCreated,
|
||||||
status: this.status,
|
status: this.status,
|
||||||
options: this.options
|
options: this.options
|
||||||
}
|
};
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
|
@ -75,7 +75,7 @@ module.exports = class TaskManager{
|
||||||
}
|
}
|
||||||
|
|
||||||
async.eachSeries(list, (uuid, cb) => {
|
async.eachSeries(list, (uuid, cb) => {
|
||||||
logger.info(`Cleaning up old task ${uuid}`)
|
logger.info(`Cleaning up old task ${uuid}`);
|
||||||
this.remove(uuid, cb);
|
this.remove(uuid, cb);
|
||||||
}, done);
|
}, done);
|
||||||
}
|
}
|
||||||
|
@ -174,8 +174,8 @@ module.exports = class TaskManager{
|
||||||
// Stops the execution of a task
|
// Stops the execution of a task
|
||||||
// (without removing it from the system).
|
// (without removing it from the system).
|
||||||
cancel(uuid, cb){
|
cancel(uuid, cb){
|
||||||
let task;
|
let task = this.find(uuid, cb);
|
||||||
if (task = this.find(uuid, cb)){
|
if (task){
|
||||||
if (!task.isCanceled()){
|
if (!task.isCanceled()){
|
||||||
task.cancel(err => {
|
task.cancel(err => {
|
||||||
this.removeFromRunningQueue(task);
|
this.removeFromRunningQueue(task);
|
||||||
|
@ -193,8 +193,8 @@ module.exports = class TaskManager{
|
||||||
remove(uuid, cb){
|
remove(uuid, cb){
|
||||||
this.cancel(uuid, err => {
|
this.cancel(uuid, err => {
|
||||||
if (!err){
|
if (!err){
|
||||||
let task;
|
let task = this.find(uuid, cb);
|
||||||
if (task = this.find(uuid, cb)){
|
if (task){
|
||||||
task.cleanup(err => {
|
task.cleanup(err => {
|
||||||
if (!err){
|
if (!err){
|
||||||
delete(this.tasks[uuid]);
|
delete(this.tasks[uuid]);
|
||||||
|
@ -210,8 +210,8 @@ module.exports = class TaskManager{
|
||||||
// Restarts (puts back into QUEUED state)
|
// Restarts (puts back into QUEUED state)
|
||||||
// a task that is either in CANCELED or FAILED state.
|
// a task that is either in CANCELED or FAILED state.
|
||||||
restart(uuid, cb){
|
restart(uuid, cb){
|
||||||
let task;
|
let task = this.find(uuid, cb);
|
||||||
if (task = this.find(uuid, cb)){
|
if (task){
|
||||||
task.restart(err => {
|
task.restart(err => {
|
||||||
if (!err) this.processNextTask();
|
if (!err) this.processNextTask();
|
||||||
cb(err);
|
cb(err);
|
||||||
|
@ -239,6 +239,6 @@ module.exports = class TaskManager{
|
||||||
if (err) logger.error(`Could not dump tasks: ${err.message}`);
|
if (err) logger.error(`Could not dump tasks: ${err.message}`);
|
||||||
else logger.debug("Dumped tasks list.");
|
else logger.debug("Dumped tasks list.");
|
||||||
if (done !== undefined) done();
|
if (done !== undefined) done();
|
||||||
})
|
});
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
|
@ -49,7 +49,7 @@ logger.add(winston.transports.File, {
|
||||||
maxsize: config.logger.maxFileSize, // Max size of each file
|
maxsize: config.logger.maxFileSize, // Max size of each file
|
||||||
maxFiles: config.logger.maxFiles, // Max number of files
|
maxFiles: config.logger.maxFiles, // Max number of files
|
||||||
level: config.logger.level // Level of log messages
|
level: config.logger.level // Level of log messages
|
||||||
})
|
});
|
||||||
|
|
||||||
if (config.deamon){
|
if (config.deamon){
|
||||||
// Console transport is no use to us when running as a daemon
|
// Console transport is no use to us when running as a daemon
|
||||||
|
|
|
@ -106,12 +106,12 @@ module.exports = {
|
||||||
|
|
||||||
let result = [];
|
let result = [];
|
||||||
let errors = [];
|
let errors = [];
|
||||||
function addError(opt, descr){
|
let addError = function(opt, descr){
|
||||||
errors.push({
|
errors.push({
|
||||||
name: opt.name,
|
name: opt.name,
|
||||||
error: descr
|
error: descr
|
||||||
});
|
});
|
||||||
}
|
};
|
||||||
|
|
||||||
let typeConversion = {
|
let typeConversion = {
|
||||||
'float': Number.parseFloat,
|
'float': Number.parseFloat,
|
||||||
|
@ -171,21 +171,22 @@ module.exports = {
|
||||||
}
|
}
|
||||||
];
|
];
|
||||||
|
|
||||||
function checkDomain(domain, value){
|
let checkDomain = function(domain, value){
|
||||||
let dc, matches;
|
let matches,
|
||||||
|
dc = domainChecks.find(dc => matches = domain.match(dc.regex));
|
||||||
|
|
||||||
if (dc = domainChecks.find(dc => matches = domain.match(dc.regex))){
|
if (dc){
|
||||||
if (!dc.validate(matches, value)) throw new Error(`Invalid value ${value} (out of range)`);
|
if (!dc.validate(matches, value)) throw new Error(`Invalid value ${value} (out of range)`);
|
||||||
}else{
|
}else{
|
||||||
throw new Error(`Domain value cannot be handled: '${domain}' : '${value}'`);
|
throw new Error(`Domain value cannot be handled: '${domain}' : '${value}'`);
|
||||||
}
|
}
|
||||||
}
|
};
|
||||||
|
|
||||||
// Scan through all possible options
|
// Scan through all possible options
|
||||||
for (let odmOption of odmOptions){
|
for (let odmOption of odmOptions){
|
||||||
// Was this option selected by the user?
|
// Was this option selected by the user?
|
||||||
let opt;
|
let opt = options.find(o => o.name === odmOption.name);
|
||||||
if (opt = options.find(o => o.name === odmOption.name)){
|
if (opt){
|
||||||
try{
|
try{
|
||||||
// Convert to proper data type
|
// Convert to proper data type
|
||||||
let value = typeConversion[odmOption.type](opt.value);
|
let value = typeConversion[odmOption.type](opt.value);
|
||||||
|
|
|
@ -16,14 +16,14 @@ You should have received a copy of the GNU General Public License
|
||||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
*/
|
*/
|
||||||
"use strict";
|
"use strict";
|
||||||
|
let assert = require('assert');
|
||||||
let spawn = require('child_process').spawn;
|
let spawn = require('child_process').spawn;
|
||||||
let config = require('../config.js');
|
let config = require('../config.js');
|
||||||
let logger = require('./logger');
|
let logger = require('./logger');
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
run: function(options = {
|
run: function(options, done, outputReceived){
|
||||||
"project-path": "/images"
|
assert(options["project-path"] !== undefined, "project-path must be defined");
|
||||||
}, done, outputReceived){
|
|
||||||
|
|
||||||
let command = [`${config.odm_path}/run.py`];
|
let command = [`${config.odm_path}/run.py`];
|
||||||
for (var name in options){
|
for (var name in options){
|
||||||
|
|
|
@ -0,0 +1,16 @@
|
||||||
|
module.exports = {
|
||||||
|
get: function(scope, prop, defaultValue){
|
||||||
|
let parts = prop.split(".");
|
||||||
|
let current = scope;
|
||||||
|
for (let i = 0; i < parts.length; i++){
|
||||||
|
if (current[parts[i]] !== undefined && i < parts.length - 1){
|
||||||
|
current = current[parts[i]];
|
||||||
|
}else if (current[parts[i]] !== undefined && i < parts.length){
|
||||||
|
return current[parts[i]];
|
||||||
|
}else{
|
||||||
|
return defaultValue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return defaultValue;
|
||||||
|
}
|
||||||
|
};
|
|
@ -2,5 +2,5 @@
|
||||||
"script" : "index.js",
|
"script" : "index.js",
|
||||||
"name" : "node-OpenDroneMap",
|
"name" : "node-OpenDroneMap",
|
||||||
"watch" : false,
|
"watch" : false,
|
||||||
"ignore_watch" : ["[\\/\\\\]\\./", "node_modules", "git-hooks", "test", "tmp", "data", "twitter-reply-test", "twitter-send-test", "^.*\.log$"]
|
"ignore_watch" : ["[\\/\\\\]\\./", "node_modules", "git-hooks", "test", "tmp", "data", "^.*\.log$"]
|
||||||
}]
|
}]
|
||||||
|
|
Ładowanie…
Reference in New Issue