mirror
/
OpenDroneMap-NodeODM
kopia lustrzana https://github.com/OpenDroneMap/NodeODM
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Added skippostprocess parameter, converted tabs to spaces in files

pull/67/head
Piero Toffanin 2019-01-04 11:36:14 -05:00
rodzic 43558492f1
commit cdf175f16c
15 zmienionych plików z 1095 dodań i 1067 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

4
docs/index.adoc
Wyświetl plik

@ -8,7 +8,7 @@ REST API to access ODM
=== Version information
[%hardbreaks]
_Version_ : 1.3.0
_Version_ : 1.3.1
=== Contact information
@ -299,6 +299,8 @@ _optional_|Images to process, plus an optional GCP file. If included, the GCP fi
_optional_|An optional name to be associated with the task|string|
|*FormData*|*options* +
_optional_|Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, …]. For example, [{"name":"cmvs-maxImages","value":"500"},{"name":"time","value":true}]. For a list of all options, call /options|string|
|*FormData*|*skipPostProcessing* +
_optional_|When set, skips generation of map tiles, derivate assets, point cloud tiles.|boolean|
|*FormData*|*zipurl* +
_optional_|URL of the zip file containing the images to process, plus an optional GCP file. If included, the GCP file should have .txt extension|string|
|===

2
docs/swagger.json
Wyświetl plik

File diff suppressed because one or more lines are too long

10
index.js
Wyświetl plik

@ -119,6 +119,12 @@ let server;
* required: false
* type: string
* -
* name: skipPostProcessing
* in: formData
* description: 'When set, skips generation of map tiles, derivate assets, point cloud tiles.'
* required: false
* type: boolean
* -
* name: token
* in: query
* description: 'Token required for authentication (when authentication is required).'
@ -287,7 +293,9 @@ app.post('/task/new', authCheck, (req, res, next) => {
res.json({ uuid: req.id });
cb();
}
}, req.body.options, req.body.webhook);
}, req.body.options,
req.body.webhook,
req.body.skipPostProcessing === 'true');
}
], err => {
if (err) die(err.message);

6
libs/Directories.js
Wyświetl plik

@ -20,9 +20,9 @@ let config = require('../config');
let path = require('path');
class Directories{
static get data(){
return !config.test ? "data" : path.join("tests", "data");
}
static get data(){
return !config.test ? "data" : path.join("tests", "data");
}
}
module.exports = Directories;

878
libs/Task.js
Wyświetl plik

@ -36,492 +36,494 @@ const request = require('request');
const statusCodes = require('./statusCodes');
module.exports = class Task{
constructor(uuid, name, done, options = [], webhook = null){
assert(uuid !== undefined, "uuid must be set");
assert(done !== undefined, "ready must be set");
constructor(uuid, name, done, options = [], webhook = null, skipPostProcessing = false){
assert(uuid !== undefined, "uuid must be set");
assert(done !== undefined, "ready must be set");
this.uuid = uuid;
this.name = name !== "" ? name : "Task of " + (new Date()).toISOString();
this.dateCreated = new Date().getTime();
this.processingTime = -1;
this.setStatus(statusCodes.QUEUED);
this.options = options;
this.gpcFiles = [];
this.output = [];
this.runningProcesses = [];
this.webhook = webhook;
this.uuid = uuid;
this.name = name !== "" ? name : "Task of " + (new Date()).toISOString();
this.dateCreated = new Date().getTime();
this.processingTime = -1;
this.setStatus(statusCodes.QUEUED);
this.options = options;
this.gpcFiles = [];
this.output = [];
this.runningProcesses = [];
this.webhook = webhook;
this.skipPostProcessing = skipPostProcessing;
async.series([
// Read images info
cb => {
fs.readdir(this.getImagesFolderPath(), (err, files) => {
if (err) cb(err);
else{
this.images = files;
logger.debug(`Found ${this.images.length} images for ${this.uuid}`);
cb(null);
}
});
},
async.series([
// Read images info
cb => {
fs.readdir(this.getImagesFolderPath(), (err, files) => {
if (err) cb(err);
else{
this.images = files;
logger.debug(`Found ${this.images.length} images for ${this.uuid}`);
cb(null);
}
});
},
// Find GCP (if any)
cb => {
fs.readdir(this.getGpcFolderPath(), (err, files) => {
if (err) cb(err);
else{
files.forEach(file => {
if (/\.txt$/gi.test(file)){
this.gpcFiles.push(file);
}
});
logger.debug(`Found ${this.gpcFiles.length} GPC files (${this.gpcFiles.join(" ")}) for ${this.uuid}`);
cb(null);
}
});
}
], err => {
done(err, this);
});
}
// Find GCP (if any)
cb => {
fs.readdir(this.getGpcFolderPath(), (err, files) => {
if (err) cb(err);
else{
files.forEach(file => {
if (/\.txt$/gi.test(file)){
this.gpcFiles.push(file);
}
});
logger.debug(`Found ${this.gpcFiles.length} GPC files (${this.gpcFiles.join(" ")}) for ${this.uuid}`);
cb(null);
}
});
}
], err => {
done(err, this);
});
}
static CreateFromSerialized(taskJson, done){
new Task(taskJson.uuid, taskJson.name, (err, task) => {
if (err) done(err);
else{
// Override default values with those
// provided in the taskJson
for (let k in taskJson){
task[k] = taskJson[k];
}
static CreateFromSerialized(taskJson, done){
new Task(taskJson.uuid, taskJson.name, (err, task) => {
if (err) done(err);
else{
// Override default values with those
// provided in the taskJson
for (let k in taskJson){
task[k] = taskJson[k];
}
// Tasks that were running should be put back to QUEUED state
if (task.status.code === statusCodes.RUNNING){
task.status.code = statusCodes.QUEUED;
}
done(null, task);
}
}, taskJson.options, taskJson.webhook);
}
// Tasks that were running should be put back to QUEUED state
if (task.status.code === statusCodes.RUNNING){
task.status.code = statusCodes.QUEUED;
}
done(null, task);
}
}, taskJson.options, taskJson.webhook, taskJson.skipPostProcessing);
}
// Get path where images are stored for this task
// (relative to nodejs process CWD)
getImagesFolderPath(){
return path.join(this.getProjectFolderPath(), "images");
}
// Get path where images are stored for this task
// (relative to nodejs process CWD)
getImagesFolderPath(){
return path.join(this.getProjectFolderPath(), "images");
}
// Get path where GPC file(s) are stored
// (relative to nodejs process CWD)
getGpcFolderPath(){
return path.join(this.getProjectFolderPath(), "gpc");
}
// Get path where GPC file(s) are stored
// (relative to nodejs process CWD)
getGpcFolderPath(){
return path.join(this.getProjectFolderPath(), "gpc");
}
// Get path of project (where all images and assets folder are contained)
// (relative to nodejs process CWD)
getProjectFolderPath(){
return path.join(Directories.data, this.uuid);
}
// Get path of project (where all images and assets folder are contained)
// (relative to nodejs process CWD)
getProjectFolderPath(){
return path.join(Directories.data, this.uuid);
}
// Get the path of the archive where all assets
// outputted by this task are stored.
getAssetsArchivePath(filename){
if (filename == 'all.zip'){
// OK, do nothing
}else if (filename == 'orthophoto.tif'){
if (config.test){
if (config.testSkipOrthophotos) return false;
else filename = path.join('..', '..', 'processing_results', 'odm_orthophoto', `odm_${filename}`);
}else{
filename = path.join('odm_orthophoto', `odm_${filename}`);
}
}else{
return false; // Invalid
}
return path.join(this.getProjectFolderPath(), filename);
}
// Get the path of the archive where all assets
// outputted by this task are stored.
getAssetsArchivePath(filename){
if (filename == 'all.zip'){
// OK, do nothing
}else if (filename == 'orthophoto.tif'){
if (config.test){
if (config.testSkipOrthophotos) return false;
else filename = path.join('..', '..', 'processing_results', 'odm_orthophoto', `odm_${filename}`);
}else{
filename = path.join('odm_orthophoto', `odm_${filename}`);
}
}else{
return false; // Invalid
}
return path.join(this.getProjectFolderPath(), filename);
}
// Deletes files and folders related to this task
cleanup(cb){
rmdir(this.getProjectFolderPath(), cb);
}
// Deletes files and folders related to this task
cleanup(cb){
rmdir(this.getProjectFolderPath(), cb);
}
setStatus(code, extra){
this.status = {
code: code
};
for (let k in extra){
this.status[k] = extra[k];
}
}
setStatus(code, extra){
this.status = {
code: code
};
for (let k in extra){
this.status[k] = extra[k];
}
}
updateProcessingTime(resetTime){
this.processingTime = resetTime ?
-1 :
new Date().getTime() - this.dateCreated;
}
updateProcessingTime(resetTime){
this.processingTime = resetTime ?
-1 :
new Date().getTime() - this.dateCreated;
}
startTrackingProcessingTime(){
this.updateProcessingTime();
if (!this._updateProcessingTimeInterval){
this._updateProcessingTimeInterval = setInterval(() => {
this.updateProcessingTime();
}, 1000);
}
}
startTrackingProcessingTime(){
this.updateProcessingTime();
if (!this._updateProcessingTimeInterval){
this._updateProcessingTimeInterval = setInterval(() => {
this.updateProcessingTime();
}, 1000);
}
}
stopTrackingProcessingTime(resetTime){
this.updateProcessingTime(resetTime);
if (this._updateProcessingTimeInterval){
clearInterval(this._updateProcessingTimeInterval);
this._updateProcessingTimeInterval = null;
}
}
stopTrackingProcessingTime(resetTime){
this.updateProcessingTime(resetTime);
if (this._updateProcessingTimeInterval){
clearInterval(this._updateProcessingTimeInterval);
this._updateProcessingTimeInterval = null;
}
}
getStatus(){
return this.status.code;
}
getStatus(){
return this.status.code;
}
isCanceled(){
return this.status.code === statusCodes.CANCELED;
}
isCanceled(){
return this.status.code === statusCodes.CANCELED;
}
// Cancels the current task (unless it's already canceled)
cancel(cb){
if (this.status.code !== statusCodes.CANCELED){
let wasRunning = this.status.code === statusCodes.RUNNING;
this.setStatus(statusCodes.CANCELED);
// Cancels the current task (unless it's already canceled)
cancel(cb){
if (this.status.code !== statusCodes.CANCELED){
let wasRunning = this.status.code === statusCodes.RUNNING;
this.setStatus(statusCodes.CANCELED);
if (wasRunning){
this.runningProcesses.forEach(proc => {
// TODO: this does NOT guarantee that
// the process will immediately terminate.
// For eaxmple in the case of the ODM process, the process will continue running for a while
// This might need to be fixed on ODM's end.
// During testing, proc is undefined
if (proc) kill(proc.pid);
});
this.runningProcesses = [];
}
if (wasRunning){
this.runningProcesses.forEach(proc => {
// TODO: this does NOT guarantee that
// the process will immediately terminate.
// For eaxmple in the case of the ODM process, the process will continue running for a while
// This might need to be fixed on ODM's end.
// During testing, proc is undefined
if (proc) kill(proc.pid);
});
this.runningProcesses = [];
}
this.stopTrackingProcessingTime(true);
cb(null);
}else{
cb(new Error("Task already cancelled"));
}
}
this.stopTrackingProcessingTime(true);
cb(null);
}else{
cb(new Error("Task already cancelled"));
}
}
// Starts processing the task with OpenDroneMap
// This will spawn a new process.
start(done){
const finished = err => {
this.stopTrackingProcessingTime();
done(err);
};
const postProcess = () => {
const createZipArchive = (outputFilename, files) => {
return (done) => {
this.output.push(`Compressing ${outputFilename}\n`);
// Starts processing the task with OpenDroneMap
// This will spawn a new process.
start(done){
const finished = err => {
this.stopTrackingProcessingTime();
done(err);
};
const postProcess = () => {
const createZipArchive = (outputFilename, files) => {
return (done) => {
this.output.push(`Compressing ${outputFilename}\n`);
let output = fs.createWriteStream(this.getAssetsArchivePath(outputFilename));
let archive = archiver.create('zip', {
zlib: { level: 1 } // Sets the compression level (1 = best speed since most assets are already compressed)
});
let output = fs.createWriteStream(this.getAssetsArchivePath(outputFilename));
let archive = archiver.create('zip', {
zlib: { level: 1 } // Sets the compression level (1 = best speed since most assets are already compressed)
});
archive.on('finish', () => {
// TODO: is this being fired twice?
done();
});
archive.on('finish', () => {
// TODO: is this being fired twice?
done();
});
archive.on('error', err => {
logger.error(`Could not archive .zip file: ${err.message}`);
done(err);
});
archive.on('error', err => {
logger.error(`Could not archive .zip file: ${err.message}`);
done(err);
});
archive.pipe(output);
let globs = [];
const sourcePath = !config.test ?
this.getProjectFolderPath() :
path.join("tests", "processing_results");
archive.pipe(output);
let globs = [];
const sourcePath = !config.test ?
this.getProjectFolderPath() :
path.join("tests", "processing_results");
// Process files and directories first
files.forEach(file => {
let filePath = path.join(sourcePath, file);
// Skip non-existing items
if (!fs.existsSync(filePath)) return;
// Process files and directories first
files.forEach(file => {
let filePath = path.join(sourcePath, file);
// Skip non-existing items
if (!fs.existsSync(filePath)) return;
let isGlob = /\*/.test(file),
isDirectory = !isGlob && fs.lstatSync(filePath).isDirectory();
let isGlob = /\*/.test(file),
isDirectory = !isGlob && fs.lstatSync(filePath).isDirectory();
if (isDirectory){
archive.directory(filePath, file);
}else if (isGlob){
globs.push(filePath);
}else{
archive.file(filePath, {name: file});
}
});
if (isDirectory){
archive.directory(filePath, file);
}else if (isGlob){
globs.push(filePath);
}else{
archive.file(filePath, {name: file});
}
});
// Check for globs
if (globs.length !== 0){
let pending = globs.length;
// Check for globs
if (globs.length !== 0){
let pending = globs.length;
globs.forEach(pattern => {
glob(pattern, (err, files) => {
if (err) done(err);
else{
files.forEach(file => {
if (fs.lstatSync(file).isFile()){
archive.file(file, {name: path.basename(file)});
}else{
logger.debug(`Could not add ${file} from glob`);
}
});
globs.forEach(pattern => {
glob(pattern, (err, files) => {
if (err) done(err);
else{
files.forEach(file => {
if (fs.lstatSync(file).isFile()){
archive.file(file, {name: path.basename(file)});
}else{
logger.debug(`Could not add ${file} from glob`);
}
});
if (--pending === 0){
archive.finalize();
}
}
});
});
}else{
archive.finalize();
}
};
};
if (--pending === 0){
archive.finalize();
}
}
});
});
}else{
archive.finalize();
}
};
};
const runPostProcessingScript = () => {
return (done) => {
this.runningProcesses.push(
processRunner.runPostProcessingScript({
projectFolderPath: this.getProjectFolderPath()
}, (err, code, signal) => {
if (err) done(err);
else{
if (code === 0) done();
else done(new Error(`Process exited with code ${code}`));
}
}, output => {
this.output.push(output);
})
);
};
};
const runPostProcessingScript = () => {
return (done) => {
this.runningProcesses.push(
processRunner.runPostProcessingScript({
projectFolderPath: this.getProjectFolderPath()
}, (err, code, signal) => {
if (err) done(err);
else{
if (code === 0) done();
else done(new Error(`Process exited with code ${code}`));
}
}, output => {
this.output.push(output);
})
);
};
};
// All paths are relative to the project directory (./data/<uuid>/)
let allPaths = ['odm_orthophoto', 'odm_georeferencing', 'odm_texturing',
'odm_dem/dsm.tif', 'odm_dem/dtm.tif', 'dsm_tiles', 'dtm_tiles',
'odm_meshing', 'orthophoto_tiles', 'potree_pointcloud', 'images.json'];
if (config.test){
if (config.testSkipOrthophotos){
logger.info("Test mode will skip orthophoto generation");
// All paths are relative to the project directory (./data/<uuid>/)
let allPaths = ['odm_orthophoto', 'odm_georeferencing', 'odm_texturing',
'odm_dem/dsm.tif', 'odm_dem/dtm.tif', 'dsm_tiles', 'dtm_tiles',
'odm_meshing', 'orthophoto_tiles', 'potree_pointcloud', 'images.json'];
if (config.test){
if (config.testSkipOrthophotos){
logger.info("Test mode will skip orthophoto generation");
// Exclude these folders from the all.zip archive
['odm_orthophoto', 'orthophoto_tiles'].forEach(dir => {
allPaths.splice(allPaths.indexOf(dir), 1);
});
}
if (config.testSkipDems){
logger.info("Test mode will skip DEMs generation");
// Exclude these folders from the all.zip archive
['odm_orthophoto', 'orthophoto_tiles'].forEach(dir => {
allPaths.splice(allPaths.indexOf(dir), 1);
});
}
if (config.testSkipDems){
logger.info("Test mode will skip DEMs generation");
// Exclude these folders from the all.zip archive
['odm_dem/dsm.tif', 'odm_dem/dtm.tif', 'dsm_tiles', 'dtm_tiles'].forEach(p => {
allPaths.splice(allPaths.indexOf(p), 1);
});
}
}
let tasks = [
runPostProcessingScript(),
createZipArchive('all.zip', allPaths)
];
// Upload to S3 all paths + all.zip file (if config says so)
if (S3.enabled()){
tasks.push((done) => {
let s3Paths;
if (config.test){
s3Paths = ['all.zip']; // During testing only upload all.zip
}else if (config.s3UploadEverything){
s3Paths = ['all.zip'].concat(allPaths);
}else{
s3Paths = ['all.zip', 'odm_orthophoto/odm_orthophoto.tif'];
}
S3.uploadPaths(this.getProjectFolderPath(), config.s3Bucket, this.uuid, s3Paths,
err => {
if (!err) this.output.push("Done uploading to S3!");
done(err);
}, output => this.output.push(output));
});
}
// Exclude these folders from the all.zip archive
['odm_dem/dsm.tif', 'odm_dem/dtm.tif', 'dsm_tiles', 'dtm_tiles'].forEach(p => {
allPaths.splice(allPaths.indexOf(p), 1);
});
}
}
let tasks = [];
if (!this.skipPostProcessing) tasks.push(runPostProcessingScript());
tasks.push(createZipArchive('all.zip', allPaths));
// Upload to S3 all paths + all.zip file (if config says so)
if (S3.enabled()){
tasks.push((done) => {
let s3Paths;
if (config.test){
s3Paths = ['all.zip']; // During testing only upload all.zip
}else if (config.s3UploadEverything){
s3Paths = ['all.zip'].concat(allPaths);
}else{
s3Paths = ['all.zip', 'odm_orthophoto/odm_orthophoto.tif'];
}
S3.uploadPaths(this.getProjectFolderPath(), config.s3Bucket, this.uuid, s3Paths,
err => {
if (!err) this.output.push("Done uploading to S3!");
done(err);
}, output => this.output.push(output));
});
}
async.series(tasks, (err) => {
if (!err){
this.setStatus(statusCodes.COMPLETED);
finished();
}else{
this.setStatus(statusCodes.FAILED);
finished(err);
}
});
};
async.series(tasks, (err) => {
if (!err){
this.setStatus(statusCodes.COMPLETED);
finished();
}else{
this.setStatus(statusCodes.FAILED);
finished(err);
}
});
};
if (this.status.code === statusCodes.QUEUED){
this.startTrackingProcessingTime();
this.setStatus(statusCodes.RUNNING);
if (this.status.code === statusCodes.QUEUED){
this.startTrackingProcessingTime();
this.setStatus(statusCodes.RUNNING);
let runnerOptions = this.options.reduce((result, opt) => {
result[opt.name] = opt.value;
return result;
}, {});
let runnerOptions = this.options.reduce((result, opt) => {
result[opt.name] = opt.value;
return result;
}, {});
runnerOptions["project-path"] = fs.realpathSync(Directories.data);
runnerOptions["project-path"] = fs.realpathSync(Directories.data);
if (this.gpcFiles.length > 0){
runnerOptions.gcp = fs.realpathSync(path.join(this.getGpcFolderPath(), this.gpcFiles[0]));
}
if (this.gpcFiles.length > 0){
runnerOptions.gcp = fs.realpathSync(path.join(this.getGpcFolderPath(), this.gpcFiles[0]));
}
this.runningProcesses.push(odmRunner.run(runnerOptions, this.uuid, (err, code, signal) => {
if (err){
this.setStatus(statusCodes.FAILED, {errorMessage: `Could not start process (${err.message})`});
finished(err);
}else{
// Don't evaluate if we caused the process to exit via SIGINT?
if (this.status.code !== statusCodes.CANCELED){
if (code === 0){
postProcess();
}else{
this.setStatus(statusCodes.FAILED, {errorMessage: `Process exited with code ${code}`});
finished();
}
}else{
finished();
}
}
}, output => {
// Replace console colors
output = output.replace(/\x1b\[[0-9;]*m/g, "");
this.runningProcesses.push(odmRunner.run(runnerOptions, this.uuid, (err, code, signal) => {
if (err){
this.setStatus(statusCodes.FAILED, {errorMessage: `Could not start process (${err.message})`});
finished(err);
}else{
// Don't evaluate if we caused the process to exit via SIGINT?
if (this.status.code !== statusCodes.CANCELED){
if (code === 0){
postProcess();
}else{
this.setStatus(statusCodes.FAILED, {errorMessage: `Process exited with code ${code}`});
finished();
}
}else{
finished();
}
}
}, output => {
// Replace console colors
output = output.replace(/\x1b\[[0-9;]*m/g, "");
// Split lines and trim
output.trim().split('\n').forEach(line => {
this.output.push(line.trim());
});
})
);
// Split lines and trim
output.trim().split('\n').forEach(line => {
this.output.push(line.trim());
});
})
);
return true;
}else{
return false;
}
}
return true;
}else{
return false;
}
}
// Re-executes the task (by setting it's state back to QUEUED)
// Only tasks that have been canceled, completed or have failed can be restarted.
restart(options, cb){
if ([statusCodes.CANCELED, statusCodes.FAILED, statusCodes.COMPLETED].indexOf(this.status.code) !== -1){
this.setStatus(statusCodes.QUEUED);
this.dateCreated = new Date().getTime();
this.output = [];
this.stopTrackingProcessingTime(true);
if (options !== undefined) this.options = options;
cb(null);
}else{
cb(new Error("Task cannot be restarted"));
}
}
// Re-executes the task (by setting it's state back to QUEUED)
// Only tasks that have been canceled, completed or have failed can be restarted.
restart(options, cb){
if ([statusCodes.CANCELED, statusCodes.FAILED, statusCodes.COMPLETED].indexOf(this.status.code) !== -1){
this.setStatus(statusCodes.QUEUED);
this.dateCreated = new Date().getTime();
this.output = [];
this.stopTrackingProcessingTime(true);
if (options !== undefined) this.options = options;
cb(null);
}else{
cb(new Error("Task cannot be restarted"));
}
}
// Returns the description of the task.
getInfo(){
return {
uuid: this.uuid,
name: this.name,
dateCreated: this.dateCreated,
processingTime: this.processingTime,
status: this.status,
options: this.options,
imagesCount: this.images.length
};
}
// Returns the description of the task.
getInfo(){
return {
uuid: this.uuid,
name: this.name,
dateCreated: this.dateCreated,
processingTime: this.processingTime,
status: this.status,
options: this.options,
imagesCount: this.images.length
};
}
// Returns the output of the OpenDroneMap process
// Optionally starting from a certain line number
getOutput(startFromLine = 0){
return this.output.slice(startFromLine, this.output.length);
}
// Reads the contents of the tasks's
// images.json and returns its JSON representation
readImagesDatabase(callback){
const imagesDbPath = !config.test ?
path.join(this.getProjectFolderPath(), 'images.json') :
path.join('tests', 'processing_results', 'images.json');
fs.readFile(imagesDbPath, 'utf8', (err, data) => {
if (err) callback(err);
else{
try{
const json = JSON.parse(data);
callback(null, json);
}catch(e){
callback(e);
}
}
});
}
// Returns the output of the OpenDroneMap process
// Optionally starting from a certain line number
getOutput(startFromLine = 0){
return this.output.slice(startFromLine, this.output.length);
}
// Reads the contents of the tasks's
// images.json and returns its JSON representation
readImagesDatabase(callback){
const imagesDbPath = !config.test ?
path.join(this.getProjectFolderPath(), 'images.json') :
path.join('tests', 'processing_results', 'images.json');
fs.readFile(imagesDbPath, 'utf8', (err, data) => {
if (err) callback(err);
else{
try{
const json = JSON.parse(data);
callback(null, json);
}catch(e){
callback(e);
}
}
});
}
callWebhooks(){
// Hooks can be passed via command line
// or for each individual task
const hooks = [this.webhook, config.webhook];
callWebhooks(){
// Hooks can be passed via command line
// or for each individual task
const hooks = [this.webhook, config.webhook];
this.readImagesDatabase((err, images) => {
if (err) logger.warn(err); // Continue with callback
if (!images) images = [];
this.readImagesDatabase((err, images) => {
if (err) logger.warn(err); // Continue with callback
if (!images) images = [];
let json = this.getInfo();
json.images = images;
let json = this.getInfo();
json.images = images;
hooks.forEach(hook => {
if (hook && hook.length > 3){
const notifyCallback = (attempt) => {
if (attempt > 5){
logger.warn(`Webhook invokation failed, will not retry: ${hook}`);
return;
}
request.post(hook, { json },
(error, response) => {
if (error || response.statusCode != 200){
logger.warn(`Webhook invokation failed, will retry in a bit: ${hook}`);
setTimeout(() => {
notifyCallback(attempt + 1);
}, attempt * 5000);
}else{
logger.debug(`Webhook invoked: ${hook}`);
}
});
};
notifyCallback(0);
}
});
});
}
hooks.forEach(hook => {
if (hook && hook.length > 3){
const notifyCallback = (attempt) => {
if (attempt > 5){
logger.warn(`Webhook invokation failed, will not retry: ${hook}`);
return;
}
request.post(hook, { json },
(error, response) => {
if (error || response.statusCode != 200){
logger.warn(`Webhook invokation failed, will retry in a bit: ${hook}`);
setTimeout(() => {
notifyCallback(attempt + 1);
}, attempt * 5000);
}else{
logger.debug(`Webhook invoked: ${hook}`);
}
});
};
notifyCallback(0);
}
});
});
}
// Returns the data necessary to serialize this
// task to restore it later.
serialize(){
return {
uuid: this.uuid,
name: this.name,
dateCreated: this.dateCreated,
status: this.status,
options: this.options,
webhook: this.webhook
};
}
// Returns the data necessary to serialize this
// task to restore it later.
serialize(){
return {
uuid: this.uuid,
name: this.name,
dateCreated: this.dateCreated,
status: this.status,
options: this.options,
webhook: this.webhook,
skipPostProcessing: !!this.skipPostProcessing
};
}
};

408
libs/TaskManager.js
Wyświetl plik

@ -32,236 +32,236 @@ const TASKS_DUMP_FILE = path.join(Directories.data, "tasks.json");
const CLEANUP_TASKS_IF_OLDER_THAN = 1000 * 60 * config.cleanupTasksAfter; // minutes
module.exports = class TaskManager{
constructor(done){
this.tasks = {};
this.runningQueue = [];
constructor(done){
this.tasks = {};
this.runningQueue = [];
async.series([
cb => this.restoreTaskListFromDump(cb),
cb => this.removeOldTasks(cb),
cb => this.removeOrphanedDirectories(cb),
cb => {
this.processNextTask();
cb();
},
cb => {
// Every hour
schedule.scheduleJob('0 * * * *', () => {
this.removeOldTasks();
this.dumpTaskList();
});
async.series([
cb => this.restoreTaskListFromDump(cb),
cb => this.removeOldTasks(cb),
cb => this.removeOrphanedDirectories(cb),
cb => {
this.processNextTask();
cb();
},
cb => {
// Every hour
schedule.scheduleJob('0 * * * *', () => {
this.removeOldTasks();
this.dumpTaskList();
});
cb();
}
], done);
}
cb();
}
], done);
}
// Removes old tasks that have either failed, are completed, or
// have been canceled.
removeOldTasks(done){
let list = [];
let now = new Date().getTime();
logger.debug("Checking for old tasks to be removed...");
// Removes old tasks that have either failed, are completed, or
// have been canceled.
removeOldTasks(done){
let list = [];
let now = new Date().getTime();
logger.debug("Checking for old tasks to be removed...");
for (let uuid in this.tasks){
let task = this.tasks[uuid];
for (let uuid in this.tasks){
let task = this.tasks[uuid];
if ([statusCodes.FAILED,
statusCodes.COMPLETED,
statusCodes.CANCELED].indexOf(task.status.code) !== -1 &&
now - task.dateCreated > CLEANUP_TASKS_IF_OLDER_THAN){
list.push(task.uuid);
}
}
if ([statusCodes.FAILED,
statusCodes.COMPLETED,
statusCodes.CANCELED].indexOf(task.status.code) !== -1 &&
now - task.dateCreated > CLEANUP_TASKS_IF_OLDER_THAN){
list.push(task.uuid);
}
}
async.eachSeries(list, (uuid, cb) => {
logger.info(`Cleaning up old task ${uuid}`);
this.remove(uuid, cb);
}, done);
}
async.eachSeries(list, (uuid, cb) => {
logger.info(`Cleaning up old task ${uuid}`);
this.remove(uuid, cb);
}, done);
}
// Removes directories that don't have a corresponding
// task associated with it (maybe as a cause of an abrupt exit)
removeOrphanedDirectories(done){
logger.info("Checking for orphaned directories to be removed...");
// Removes directories that don't have a corresponding
// task associated with it (maybe as a cause of an abrupt exit)
removeOrphanedDirectories(done){
logger.info("Checking for orphaned directories to be removed...");
fs.readdir(Directories.data, (err, entries) => {
if (err) done(err);
else{
async.eachSeries(entries, (entry, cb) => {
let dirPath = path.join(Directories.data, entry);
if (fs.statSync(dirPath).isDirectory() &&
entry.match(/^[\w\d]+\-[\w\d]+\-[\w\d]+\-[\w\d]+\-[\w\d]+$/) &&
!this.tasks[entry]){
logger.info(`Found orphaned directory: ${entry}, removing...`);
rmdir(dirPath, cb);
}else cb();
}, done);
}
});
}
fs.readdir(Directories.data, (err, entries) => {
if (err) done(err);
else{
async.eachSeries(entries, (entry, cb) => {
let dirPath = path.join(Directories.data, entry);
if (fs.statSync(dirPath).isDirectory() &&
entry.match(/^[\w\d]+\-[\w\d]+\-[\w\d]+\-[\w\d]+\-[\w\d]+$/) &&
!this.tasks[entry]){
logger.info(`Found orphaned directory: ${entry}, removing...`);
rmdir(dirPath, cb);
}else cb();
}, done);
}
});
}
// Load tasks that already exists (if any)
restoreTaskListFromDump(done){
fs.readFile(TASKS_DUMP_FILE, (err, data) => {
if (!err){
let tasks;
try{
tasks = JSON.parse(data.toString());
}catch(e){
done(new Error(`Could not load task list. It looks like the ${TASKS_DUMP_FILE} is corrupted (${e.message}). Please manually delete the file and try again.`));
return;
}
// Load tasks that already exists (if any)
restoreTaskListFromDump(done){
fs.readFile(TASKS_DUMP_FILE, (err, data) => {
if (!err){
let tasks;
try{
tasks = JSON.parse(data.toString());
}catch(e){
done(new Error(`Could not load task list. It looks like the ${TASKS_DUMP_FILE} is corrupted (${e.message}). Please manually delete the file and try again.`));
return;
}
async.each(tasks, (taskJson, done) => {
Task.CreateFromSerialized(taskJson, (err, task) => {
if (err) done(err);
else{
this.tasks[task.uuid] = task;
done();
}
});
}, err => {
logger.info(`Initialized ${tasks.length} tasks`);
if (done !== undefined) done();
});
}else{
logger.info("No tasks dump found");
if (done !== undefined) done();
}
});
}
async.each(tasks, (taskJson, done) => {
Task.CreateFromSerialized(taskJson, (err, task) => {
if (err) done(err);
else{
this.tasks[task.uuid] = task;
done();
}
});
}, err => {
logger.info(`Initialized ${tasks.length} tasks`);
if (done !== undefined) done();
});
}else{
logger.info("No tasks dump found");
if (done !== undefined) done();
}
});
}
// Finds the first QUEUED task.
findNextTaskToProcess(){
for (let uuid in this.tasks){
if (this.tasks[uuid].getStatus() === statusCodes.QUEUED){
return this.tasks[uuid];
}
}
}
// Finds the first QUEUED task.
findNextTaskToProcess(){
for (let uuid in this.tasks){
if (this.tasks[uuid].getStatus() === statusCodes.QUEUED){
return this.tasks[uuid];
}
}
}
// Finds the next tasks, adds them to the running queue,
// and starts the tasks (up to the limit).
processNextTask(){
if (this.runningQueue.length < config.parallelQueueProcessing){
let task = this.findNextTaskToProcess();
if (task){
this.addToRunningQueue(task);
task.start(() => {
// Finds the next tasks, adds them to the running queue,
// and starts the tasks (up to the limit).
processNextTask(){
if (this.runningQueue.length < config.parallelQueueProcessing){
let task = this.findNextTaskToProcess();
if (task){
this.addToRunningQueue(task);
task.start(() => {
task.callWebhooks();
task.callWebhooks();
this.removeFromRunningQueue(task);
this.processNextTask();
});
this.removeFromRunningQueue(task);
this.processNextTask();
});
if (this.runningQueue.length < config.parallelQueueProcessing) this.processNextTask();
}
}else{
// Do nothing
}
}
if (this.runningQueue.length < config.parallelQueueProcessing) this.processNextTask();
}
}else{
// Do nothing
}
}
addToRunningQueue(task){
assert(task.constructor.name === "Task", "Must be a Task object");
this.runningQueue.push(task);
}
addToRunningQueue(task){
assert(task.constructor.name === "Task", "Must be a Task object");
this.runningQueue.push(task);
}
removeFromRunningQueue(task){
assert(task.constructor.name === "Task", "Must be a Task object");
this.runningQueue = this.runningQueue.filter(t => t !== task);
}
removeFromRunningQueue(task){
assert(task.constructor.name === "Task", "Must be a Task object");
this.runningQueue = this.runningQueue.filter(t => t !== task);
}
addNew(task){
assert(task.constructor.name === "Task", "Must be a Task object");
this.tasks[task.uuid] = task;
addNew(task){
assert(task.constructor.name === "Task", "Must be a Task object");
this.tasks[task.uuid] = task;
this.processNextTask();
}
this.processNextTask();
}
// Stops the execution of a task
// (without removing it from the system).
cancel(uuid, cb){
let task = this.find(uuid, cb);
if (task){
if (!task.isCanceled()){
task.cancel(err => {
this.removeFromRunningQueue(task);
this.processNextTask();
cb(err);
});
}else{
cb(null); // Nothing to be done
}
}
}
// Stops the execution of a task
// (without removing it from the system).
cancel(uuid, cb){
let task = this.find(uuid, cb);
if (task){
if (!task.isCanceled()){
task.cancel(err => {
this.removeFromRunningQueue(task);
this.processNextTask();
cb(err);
});
}else{
cb(null); // Nothing to be done
}
}
}
// Removes a task from the system.
// Before being removed, the task is canceled.
remove(uuid, cb){
this.cancel(uuid, err => {
if (!err){
let task = this.find(uuid, cb);
if (task){
task.cleanup(err => {
if (!err){
delete(this.tasks[uuid]);
this.processNextTask();
cb(null);
}else cb(err);
});
}else; // cb is called by find on error
}else cb(err);
});
}
// Removes a task from the system.
// Before being removed, the task is canceled.
remove(uuid, cb){
this.cancel(uuid, err => {
if (!err){
let task = this.find(uuid, cb);
if (task){
task.cleanup(err => {
if (!err){
delete(this.tasks[uuid]);
this.processNextTask();
cb(null);
}else cb(err);
});
}else; // cb is called by find on error
}else cb(err);
});
}
// Restarts (puts back into QUEUED state)
// a task that is either in CANCELED or FAILED state.
// When options is set, the task's options are overriden
restart(uuid, options, cb){
let task = this.find(uuid, cb);
if (task){
task.restart(options, err => {
if (!err) this.processNextTask();
cb(err);
});
}
}
// Restarts (puts back into QUEUED state)
// a task that is either in CANCELED or FAILED state.
// When options is set, the task's options are overriden
restart(uuid, options, cb){
let task = this.find(uuid, cb);
if (task){
task.restart(options, err => {
if (!err) this.processNextTask();
cb(err);
});
}
}
// Finds a task by its UUID string.
find(uuid, cb){
let task = this.tasks[uuid];
if (!task && cb) cb(new Error(`${uuid} not found`));
return task;
}
// Finds a task by its UUID string.
find(uuid, cb){
let task = this.tasks[uuid];
if (!task && cb) cb(new Error(`${uuid} not found`));
return task;
}
// Serializes the list of tasks and saves it
// to disk
dumpTaskList(done){
let output = [];
// Serializes the list of tasks and saves it
// to disk
dumpTaskList(done){
let output = [];
for (let uuid in this.tasks){
output.push(this.tasks[uuid].serialize());
}
for (let uuid in this.tasks){
output.push(this.tasks[uuid].serialize());
}
fs.writeFile(TASKS_DUMP_FILE, JSON.stringify(output), err => {
if (err) logger.error(`Could not dump tasks: ${err.message}`);
else logger.debug("Dumped tasks list.");
if (done !== undefined) done();
});
}
fs.writeFile(TASKS_DUMP_FILE, JSON.stringify(output), err => {
if (err) logger.error(`Could not dump tasks: ${err.message}`);
else logger.debug("Dumped tasks list.");
if (done !== undefined) done();
});
}
getQueueCount(){
let count = 0;
for (let uuid in this.tasks){
let task = this.tasks[uuid];
getQueueCount(){
let count = 0;
for (let uuid in this.tasks){
let task = this.tasks[uuid];
if ([statusCodes.QUEUED,
statusCodes.RUNNING].indexOf(task.status.code) !== -1){
count++;
}
}
return count;
}
if ([statusCodes.QUEUED,
statusCodes.RUNNING].indexOf(task.status.code) !== -1){
count++;
}
}
return count;
}
};

26
libs/logger.js
Wyświetl plik

@ -25,15 +25,15 @@ let path = require('path');
// Set up logging
// Configure custom File transport to write plain text messages
let logPath = ( config.logger.logDirectory ?
config.logger.logDirectory :
path.join(__dirname, "..") );
config.logger.logDirectory :
path.join(__dirname, "..") );
// Check that log file directory can be written to
try {
fs.accessSync(logPath, fs.W_OK);
fs.accessSync(logPath, fs.W_OK);
} catch (e) {
console.log( "Log directory '" + logPath + "' cannot be written to" );
throw e;
console.log( "Log directory '" + logPath + "' cannot be written to" );
throw e;
}
logPath += path.sep;
logPath += config.instance + ".log";
@ -44,16 +44,16 @@ let logger = new (winston.Logger)({
]
});
logger.add(winston.transports.File, {
filename: logPath, // Write to projectname.log
json: false, // Write in plain text, not JSON
maxsize: config.logger.maxFileSize, // Max size of each file
maxFiles: config.logger.maxFiles, // Max number of files
level: config.logger.level // Level of log messages
});
filename: logPath, // Write to projectname.log
json: false, // Write in plain text, not JSON
maxsize: config.logger.maxFileSize, // Max size of each file
maxFiles: config.logger.maxFiles, // Max number of files
level: config.logger.level // Level of log messages
});
if (config.deamon){
// Console transport is no use to us when running as a daemon
logger.remove(winston.transports.Console);
// Console transport is no use to us when running as a daemon
logger.remove(winston.transports.Console);
}
module.exports = logger;

498
libs/odmInfo.js
Wyświetl plik

@ -26,279 +26,279 @@ let odmOptions = null;
let odmVersion = null;
module.exports = {
initialize: function(done){
async.parallel([
this.getOptions,
this.getVersion
], done);
},
getVersion: function(done){
if (odmVersion){
done(null, odmVersion);
return;
}
initialize: function(done){
async.parallel([
this.getOptions,
this.getVersion
], done);
},
getVersion: function(done){
if (odmVersion){
done(null, odmVersion);
return;
}
odmRunner.getVersion(done);
},
odmRunner.getVersion(done);
},
getOptions: function(done){
if (odmOptions){
done(null, odmOptions);
return;
}
getOptions: function(done){
if (odmOptions){
done(null, odmOptions);
return;
}
odmRunner.getJsonOptions((err, json) => {
if (err) done(err);
else{
odmOptions = [];
for (let option in json){
// Not all options are useful to the end user
// (num cores can be set programmatically, so can gcpFile, etc.)
if (["-h", "--project-path", "--cmvs-maxImages", "--time",
"--zip-results", "--pmvs-num-cores",
"--start-with", "--gcp", "--end-with", "--images",
"--rerun-all", "--rerun",
"--slam-config", "--video", "--version", "--name"].indexOf(option) !== -1) continue;
odmRunner.getJsonOptions((err, json) => {
if (err) done(err);
else{
odmOptions = [];
for (let option in json){
// Not all options are useful to the end user
// (num cores can be set programmatically, so can gcpFile, etc.)
if (["-h", "--project-path", "--cmvs-maxImages", "--time",
"--zip-results", "--pmvs-num-cores",
"--start-with", "--gcp", "--end-with", "--images",
"--rerun-all", "--rerun",
"--slam-config", "--video", "--version", "--name"].indexOf(option) !== -1) continue;
let values = json[option];
let values = json[option];
let name = option.replace(/^--/, "");
let type = "";
let value = "";
let help = values.help || "";
let domain = values.metavar !== undefined ?
values.metavar.replace(/^[<>]/g, "")
.replace(/[<>]$/g, "")
.trim() :
"";
let name = option.replace(/^--/, "");
let type = "";
let value = "";
let help = values.help || "";
let domain = values.metavar !== undefined ?
values.metavar.replace(/^[<>]/g, "")
.replace(/[<>]$/g, "")
.trim() :
"";
switch((values.type || "").trim()){
case "<type 'int'>":
type = "int";
value = values['default'] !== undefined ?
parseInt(values['default']) :
0;
break;
case "<type 'float'>":
type = "float";
value = values['default'] !== undefined ?
parseFloat(values['default']) :
0.0;
break;
default:
type = "string";
value = values['default'] !== undefined ?
values['default'].trim() :
"";
}
switch((values.type || "").trim()){
case "<type 'int'>":
type = "int";
value = values['default'] !== undefined ?
parseInt(values['default']) :
0;
break;
case "<type 'float'>":
type = "float";
value = values['default'] !== undefined ?
parseFloat(values['default']) :
0.0;
break;
default:
type = "string";
value = values['default'] !== undefined ?
values['default'].trim() :
"";
}
if (values['default'] === "True"){
type = "bool";
value = true;
}else if (values['default'] === "False"){
type = "bool";
value = false;
}
if (values['default'] === "True"){
type = "bool";
value = true;
}else if (values['default'] === "False"){
type = "bool";
value = false;
}
// If 'choices' is specified, try to convert it to array
if (values.choices){
try{
values.choices = JSON.parse(values.choices.replace(/'/g, '"')); // Convert ' to "
}catch(e){
logger.warn(`Cannot parse choices: ${values.choices}`);
}
}
// If 'choices' is specified, try to convert it to array
if (values.choices){
try{
values.choices = JSON.parse(values.choices.replace(/'/g, '"')); // Convert ' to "
}catch(e){
logger.warn(`Cannot parse choices: ${values.choices}`);
}
}
// In the end, all values must be converted back
// to strings (per OpenAPI spec which doesn't allow mixed types)
value = String(value);
// In the end, all values must be converted back
// to strings (per OpenAPI spec which doesn't allow mixed types)
value = String(value);
if (Array.isArray(values.choices)){
type = "enum";
domain = values.choices;
if (Array.isArray(values.choices)){
type = "enum";
domain = values.choices;
// Make sure that the default value
// is in the list of choices
if (domain.indexOf(value) === -1) domain.unshift(value);
}
// Make sure that the default value
// is in the list of choices
if (domain.indexOf(value) === -1) domain.unshift(value);
}
help = help.replace(/\%\(default\)s/g, value);
help = help.replace(/\%\(default\)s/g, value);
odmOptions.push({
name, type, value, domain, help
});
}
done(null, odmOptions);
}
});
},
odmOptions.push({
name, type, value, domain, help
});
}
done(null, odmOptions);
}
});
},
// Checks that the options (as received from the rest endpoint)
// Are valid and within proper ranges.
// The result of filtering is passed back via callback
// @param options[]
filterOptions: function(options, done){
assert(odmOptions !== null, "odmOptions is not set. Have you initialized odmOptions properly?");
// Checks that the options (as received from the rest endpoint)
// Are valid and within proper ranges.
// The result of filtering is passed back via callback
// @param options[]
filterOptions: function(options, done){
assert(odmOptions !== null, "odmOptions is not set. Have you initialized odmOptions properly?");
try{
if (typeof options === "string") options = JSON.parse(options);
if (!Array.isArray(options)) options = [];
let result = [];
let errors = [];
let addError = function(opt, descr){
errors.push({
name: opt.name,
error: descr
});
};
try{
if (typeof options === "string") options = JSON.parse(options);
if (!Array.isArray(options)) options = [];
let result = [];
let errors = [];
let addError = function(opt, descr){
errors.push({
name: opt.name,
error: descr
});
};
let typeConversion = {
'float': Number.parseFloat,
'int': Number.parseInt,
'bool': function(value){
if (value === 'true' || value === '1') return true;
else if (value === 'false' || value === '0') return false;
else if (typeof value === 'boolean') return value;
else throw new Error(`Cannot convert ${value} to boolean`);
},
'string': function(value){
return value; // No conversion needed
},
'path': function(value){
return value; // No conversion needed
},
'enum': function(value){
return value; // No conversion needed
}
};
let domainChecks = [
{
regex: /^(positive |negative )?(integer|float)$/,
validate: function(matches, value){
if (matches[1] === 'positive ') return value >= 0;
else if (matches[1] === 'negative ') return value <= 0;
else if (matches[2] === 'integer') return Number.isInteger(value);
else if (matches[2] === 'float') return Number.isFinite(value);
}
},
{
regex: /^percent$/,
validate: function(matches, value){
return value >= 0 && value <= 100;
}
},
{
regex: /^(float|integer): ([\-\+\.\d]+) <= x <= ([\-\+\.\d]+)$/,
validate: function(matches, value){
let [str, type, lower, upper] = matches;
let parseFunc = type === 'float' ? parseFloat : parseInt;
lower = parseFunc(lower);
upper = parseFunc(upper);
return value >= lower && value <= upper;
}
},
{
regex: /^(float|integer) (>=|>|<|<=) ([\-\+\.\d]+)$/,
validate: function(matches, value){
let [str, type, oper, bound] = matches;
let parseFunc = type === 'float' ? parseFloat : parseInt;
bound = parseFunc(bound);
switch(oper){
case '>=':
return value >= bound;
case '>':
return value > bound;
case '<=':
return value <= bound;
case '<':
return value < bound;
default:
return false;
}
}
},
{
regex: /^(string|path)$/,
validate: function(){
return true; // All strings/paths are fine
}
}
];
let typeConversion = {
'float': Number.parseFloat,
'int': Number.parseInt,
'bool': function(value){
if (value === 'true' || value === '1') return true;
else if (value === 'false' || value === '0') return false;
else if (typeof value === 'boolean') return value;
else throw new Error(`Cannot convert ${value} to boolean`);
},
'string': function(value){
return value; // No conversion needed
},
'path': function(value){
return value; // No conversion needed
},
'enum': function(value){
return value; // No conversion needed
}
};
let domainChecks = [
{
regex: /^(positive |negative )?(integer|float)$/,
validate: function(matches, value){
if (matches[1] === 'positive ') return value >= 0;
else if (matches[1] === 'negative ') return value <= 0;
else if (matches[2] === 'integer') return Number.isInteger(value);
else if (matches[2] === 'float') return Number.isFinite(value);
}
},
{
regex: /^percent$/,
validate: function(matches, value){
return value >= 0 && value <= 100;
}
},
{
regex: /^(float|integer): ([\-\+\.\d]+) <= x <= ([\-\+\.\d]+)$/,
validate: function(matches, value){
let [str, type, lower, upper] = matches;
let parseFunc = type === 'float' ? parseFloat : parseInt;
lower = parseFunc(lower);
upper = parseFunc(upper);
return value >= lower && value <= upper;
}
},
{
regex: /^(float|integer) (>=|>|<|<=) ([\-\+\.\d]+)$/,
validate: function(matches, value){
let [str, type, oper, bound] = matches;
let parseFunc = type === 'float' ? parseFloat : parseInt;
bound = parseFunc(bound);
switch(oper){
case '>=':
return value >= bound;
case '>':
return value > bound;
case '<=':
return value <= bound;
case '<':
return value < bound;
default:
return false;
}
}
},
{
regex: /^(string|path)$/,
validate: function(){
return true; // All strings/paths are fine
}
}
];
let checkDomain = function(domain, value){
if (Array.isArray(domain)){
// Special case for enum checks
if (domain.indexOf(value) === -1) throw new Error(`Invalid value ${value} (not in enum)`);
}else{
let matches,
dc = domainChecks.find(dc => matches = domain.match(dc.regex));
let checkDomain = function(domain, value){
if (Array.isArray(domain)){
// Special case for enum checks
if (domain.indexOf(value) === -1) throw new Error(`Invalid value ${value} (not in enum)`);
}else{
let matches,
dc = domainChecks.find(dc => matches = domain.match(dc.regex));
if (dc){
if (!dc.validate(matches, value)) throw new Error(`Invalid value ${value} (out of range)`);
}else{
throw new Error(`Domain value cannot be handled: '${domain}' : '${value}'`);
}
}
};
if (dc){
if (!dc.validate(matches, value)) throw new Error(`Invalid value ${value} (out of range)`);
}else{
throw new Error(`Domain value cannot be handled: '${domain}' : '${value}'`);
}
}
};
// Scan through all possible options
let maxConcurrencyFound = false;
let maxConcurrencyIsAnOption = false;
// Scan through all possible options
let maxConcurrencyFound = false;
let maxConcurrencyIsAnOption = false;
for (let odmOption of odmOptions){
if (odmOption.name === 'max-concurrency') maxConcurrencyIsAnOption = true;
// Was this option selected by the user?
/*jshint loopfunc: true */
let opt = options.find(o => o.name === odmOption.name);
if (opt){
try{
// Convert to proper data type
for (let odmOption of odmOptions){
if (odmOption.name === 'max-concurrency') maxConcurrencyIsAnOption = true;
// Was this option selected by the user?
/*jshint loopfunc: true */
let opt = options.find(o => o.name === odmOption.name);
if (opt){
try{
// Convert to proper data type
let value = typeConversion[odmOption.type](opt.value);
let value = typeConversion[odmOption.type](opt.value);
// Domain check
if (odmOption.domain){
checkDomain(odmOption.domain, value);
}
// Max concurrency check
if (opt.name === 'max-concurrency'){
maxConcurrencyFound = true;
// Domain check
if (odmOption.domain){
checkDomain(odmOption.domain, value);
}
// Max concurrency check
if (opt.name === 'max-concurrency'){
maxConcurrencyFound = true;
// Cap
if (config.maxConcurrency){
value = Math.min(value, config.maxConcurrency);
}
}
// Cap
if (config.maxConcurrency){
value = Math.min(value, config.maxConcurrency);
}
}
result.push({
name: odmOption.name,
value: value
});
}catch(e){
addError(opt, e.message);
}
}
}
result.push({
name: odmOption.name,
value: value
});
}catch(e){
addError(opt, e.message);
}
}
}
// If no max concurrency was passed by the user
// but our configuration sets a limit, pass it.
if (!maxConcurrencyFound && maxConcurrencyIsAnOption && config.maxConcurrency){
result.push({
name: "max-concurrency",
value: config.maxConcurrency
});
}
// If no max concurrency was passed by the user
// but our configuration sets a limit, pass it.
if (!maxConcurrencyFound && maxConcurrencyIsAnOption && config.maxConcurrency){
result.push({
name: "max-concurrency",
value: config.maxConcurrency
});
}
if (errors.length > 0) done(new Error(JSON.stringify(errors)));
else done(null, result);
}catch(e){
done(e);
}
}
if (errors.length > 0) done(new Error(JSON.stringify(errors)));
else done(null, result);
}catch(e){
done(e);
}
}
};

176
libs/odmRunner.js
Wyświetl plik

@ -25,112 +25,112 @@ let logger = require('./logger');
module.exports = {
run: function(options, projectName, done, outputReceived){
assert(projectName !== undefined, "projectName must be specified");
assert(options["project-path"] !== undefined, "project-path must be defined");
run: function(options, projectName, done, outputReceived){
assert(projectName !== undefined, "projectName must be specified");
assert(options["project-path"] !== undefined, "project-path must be defined");
const command = path.join(config.odm_path, "run.sh"),
params = [];
const command = path.join(config.odm_path, "run.sh"),
params = [];
for (var name in options){
let value = options[name];
for (var name in options){
let value = options[name];
// Skip false booleans
if (value === false) continue;
// Skip false booleans
if (value === false) continue;
params.push("--" + name);
params.push("--" + name);
// We don't specify "--time true" (just "--time")
if (typeof value !== 'boolean'){
params.push(value);
}
}
// We don't specify "--time true" (just "--time")
if (typeof value !== 'boolean'){
params.push(value);
}
}
params.push(projectName);
params.push(projectName);
logger.info(`About to run: ${command} ${params.join(" ")}`);
logger.info(`About to run: ${command} ${params.join(" ")}`);
if (config.test){
logger.info("Test mode is on, command will not execute");
if (config.test){
logger.info("Test mode is on, command will not execute");
let outputTestFile = path.join("..", "tests", "odm_output.txt");
fs.readFile(path.resolve(__dirname, outputTestFile), 'utf8', (err, text) => {
if (!err){
let lines = text.split("\n");
lines.forEach(line => outputReceived(line));
done(null, 0, null);
}else{
logger.warn(`Error: ${err.message}`);
done(err);
}
});
let outputTestFile = path.join("..", "tests", "odm_output.txt");
fs.readFile(path.resolve(__dirname, outputTestFile), 'utf8', (err, text) => {
if (!err){
let lines = text.split("\n");
lines.forEach(line => outputReceived(line));
done(null, 0, null);
}else{
logger.warn(`Error: ${err.message}`);
done(err);
}
});
return; // Skip rest
}
return; // Skip rest
}
// Launch
let childProcess = spawn(command, params, {cwd: config.odm_path});
// Launch
let childProcess = spawn(command, params, {cwd: config.odm_path});
childProcess
.on('exit', (code, signal) => done(null, code, signal))
.on('error', done);
childProcess
.on('exit', (code, signal) => done(null, code, signal))
.on('error', done);
childProcess.stdout.on('data', chunk => outputReceived(chunk.toString()));
childProcess.stderr.on('data', chunk => outputReceived(chunk.toString()));
childProcess.stdout.on('data', chunk => outputReceived(chunk.toString()));
childProcess.stderr.on('data', chunk => outputReceived(chunk.toString()));
return childProcess;
},
getVersion: function(done){
fs.readFile(path.join(config.odm_path, 'VERSION'), {encoding: 'utf8'}, (err, content) => {
if (err) done(null, "?");
else done(null, content.split("\n").map(l => l.trim())[0]);
});
},
return childProcess;
},
getVersion: function(done){
fs.readFile(path.join(config.odm_path, 'VERSION'), {encoding: 'utf8'}, (err, content) => {
if (err) done(null, "?");
else done(null, content.split("\n").map(l => l.trim())[0]);
});
},
getJsonOptions: function(done){
// In test mode, we don't call ODM,
// instead we return a mock
if (config.test){
let optionsTestFile = path.join("..", "tests", "odm_options.json");
fs.readFile(path.resolve(__dirname, optionsTestFile), 'utf8', (err, json) => {
if (!err){
try{
let options = JSON.parse(json);
done(null, options);
}catch(e){
logger.warn(`Invalid test options ${optionsTestFile}: ${err.message}`);
done(e);
}
}else{
logger.warn(`Error: ${err.message}`);
done(err);
}
});
getJsonOptions: function(done){
// In test mode, we don't call ODM,
// instead we return a mock
if (config.test){
let optionsTestFile = path.join("..", "tests", "odm_options.json");
fs.readFile(path.resolve(__dirname, optionsTestFile), 'utf8', (err, json) => {
if (!err){
try{
let options = JSON.parse(json);
done(null, options);
}catch(e){
logger.warn(`Invalid test options ${optionsTestFile}: ${err.message}`);
done(e);
}
}else{
logger.warn(`Error: ${err.message}`);
done(err);
}
});
return; // Skip rest
}
return; // Skip rest
}
// Launch
let childProcess = spawn("python", [path.join(__dirname, "..", "helpers", "odmOptionsToJson.py"),
"--project-path", config.odm_path, "bogusname"]);
let output = [];
// Launch
let childProcess = spawn("python", [path.join(__dirname, "..", "helpers", "odmOptionsToJson.py"),
"--project-path", config.odm_path, "bogusname"]);
let output = [];
childProcess
.on('exit', (code, signal) => {
try{
let json = JSON.parse(output.join(""));
done(null, json);
}catch(err){
done(new Error(`Could not load list of options from OpenDroneMap. Is OpenDroneMap installed in ${config.odm_path}? Make sure that OpenDroneMap is installed and that --odm_path is set properly: ${err.message}`));
}
})
.on('error', done);
childProcess
.on('exit', (code, signal) => {
try{
let json = JSON.parse(output.join(""));
done(null, json);
}catch(err){
done(new Error(`Could not load list of options from OpenDroneMap. Is OpenDroneMap installed in ${config.odm_path}? Make sure that OpenDroneMap is installed and that --odm_path is set properly: ${err.message}`));
}
})
.on('error', done);
let processOutput = chunk => output.push(chunk.toString());
let processOutput = chunk => output.push(chunk.toString());
childProcess.stdout.on('data', processOutput);
childProcess.stderr.on('data', processOutput);
}
childProcess.stdout.on('data', processOutput);
childProcess.stderr.on('data', processOutput);
}
};

80
libs/processRunner.js
Wyświetl plik

@ -25,56 +25,56 @@ let logger = require('./logger');
function makeRunner(command, args, requiredOptions = [], outputTestFile = null){
return function(options, done, outputReceived){
for (let requiredOption of requiredOptions){
assert(options[requiredOption] !== undefined, `${requiredOption} must be defined`);
}
return function(options, done, outputReceived){
for (let requiredOption of requiredOptions){
assert(options[requiredOption] !== undefined, `${requiredOption} must be defined`);
}
let commandArgs = args;
if (typeof commandArgs === 'function') commandArgs = commandArgs(options);
let commandArgs = args;
if (typeof commandArgs === 'function') commandArgs = commandArgs(options);
logger.info(`About to run: ${command} ${commandArgs.join(" ")}`);
logger.info(`About to run: ${command} ${commandArgs.join(" ")}`);
if (config.test){
logger.info("Test mode is on, command will not execute");
if (config.test){
logger.info("Test mode is on, command will not execute");
if (outputTestFile){
fs.readFile(path.resolve(__dirname, outputTestFile), 'utf8', (err, text) => {
if (!err){
let lines = text.split("\n");
lines.forEach(line => outputReceived(line));
done(null, 0, null);
}else{
logger.warn(`Error: ${err.message}`);
done(err);
}
});
}else{
done(null, 0, null);
}
if (outputTestFile){
fs.readFile(path.resolve(__dirname, outputTestFile), 'utf8', (err, text) => {
if (!err){
let lines = text.split("\n");
lines.forEach(line => outputReceived(line));
done(null, 0, null);
}else{
logger.warn(`Error: ${err.message}`);
done(err);
}
});
}else{
done(null, 0, null);
}
return;// Skip rest
}
return;// Skip rest
}
// Launch
let childProcess = spawn(command, commandArgs);
// Launch
let childProcess = spawn(command, commandArgs);
childProcess
.on('exit', (code, signal) => done(null, code, signal))
.on('error', done);
childProcess
.on('exit', (code, signal) => done(null, code, signal))
.on('error', done);
childProcess.stdout.on('data', chunk => outputReceived(chunk.toString()));
childProcess.stderr.on('data', chunk => outputReceived(chunk.toString()));
childProcess.stdout.on('data', chunk => outputReceived(chunk.toString()));
childProcess.stderr.on('data', chunk => outputReceived(chunk.toString()));
return childProcess;
};
return childProcess;
};
}
module.exports = {
runPostProcessingScript: makeRunner(path.join(__dirname, "..", "scripts", "postprocess.sh"),
function(options){
return [options.projectFolderPath];
},
["projectFolderPath"])
runPostProcessingScript: makeRunner(path.join(__dirname, "..", "scripts", "postprocess.sh"),
function(options){
return [options.projectFolderPath];
},
["projectFolderPath"])
};

10
libs/statusCodes.js
Wyświetl plik

@ -17,9 +17,9 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
"use strict";
module.exports = {
QUEUED: 10,
RUNNING: 20,
FAILED: 30,
COMPLETED: 40,
CANCELED: 50
QUEUED: 10,
RUNNING: 20,
FAILED: 30,
COMPLETED: 40,
CANCELED: 50
};

28
libs/utils.js
Wyświetl plik

@ -1,17 +1,17 @@
"use strict";
module.exports = {
get: function(scope, prop, defaultValue){
let parts = prop.split(".");
let current = scope;
for (let i = 0; i < parts.length; i++){
if (current[parts[i]] !== undefined && i < parts.length - 1){
current = current[parts[i]];
}else if (current[parts[i]] !== undefined && i < parts.length){
return current[parts[i]];
}else{
return defaultValue;
}
}
return defaultValue;
}
get: function(scope, prop, defaultValue){
let parts = prop.split(".");
let current = scope;
for (let i = 0; i < parts.length; i++){
if (current[parts[i]] !== undefined && i < parts.length - 1){
current = current[parts[i]];
}else if (current[parts[i]] !== undefined && i < parts.length){
return current[parts[i]];
}else{
return defaultValue;
}
}
return defaultValue;
}
};

2
package.json
Wyświetl plik

@ -1,6 +1,6 @@
{
"name": "node-opendronemap",
"version": "1.3.0",
"version": "1.3.1",
"description": "REST API to access ODM",
"main": "index.js",
"scripts": {

29
public/index.html
Wyświetl plik

@ -44,16 +44,14 @@
<form enctype="multipart/form-data" onsubmit="return false;">
<div class="form-group form-inline">
<label for="taskName">Project Name:</lable> <input type="text" class="form-control" value="" id="taskName" />
</div>
<div id="imagesInput" class="form-group">
<label for="images">Aerial Images and GCP List (optional):</label> <input id="images" name="images" multiple accept="image/*|.txt|.zip" type="file">
</div>
<div id="imagesInput" class="form-group">
<label for="images">Aerial Images and GCP List (optional):</label> <input id="images" name="images" multiple accept="image/*|.txt|.zip" type="file">
</div>
<div id="zipFileInput" class="form-group hidden">
<label for="zipurl">URL to zip file with Aerial Images and GCP List (optional):</label> <input id="zipurl" name="zipurl" class="form-control" type="text">
</div>
<div id="errorBlock" class="help-block"></div>
<div id="errorBlock" class="help-block"></div>
<div class="text-right"><input type="button" class="btn btn-info" value="Switch to URL" id="btnShowImport" />
<input type="button" class="btn btn-info hidden" value="Switch to File Upload" id="btnShowUpload" />
@ -65,16 +63,29 @@
<button style="position: relative; top: -45px;" type="submit" class="btn btn-default" data-bind="visible: !error(), click: function(){ showOptions(!showOptions()); }, text: (showOptions() ? 'Hide' : 'Show') + ' Options'"></button>
<div data-bind="visible: showOptions()">
<div>
<label for="doPostProcessing">generate 2D and potree point cloud tiles:</label>
<br/>
<div class="checkbox">
<label>
<input type="checkbox" id="doPostProcessing"> Enable
</label>
</div>
<button type="submit" class="btn glyphicon glyphicon-info-sign btn-info" data-toggle="tooltip" data-placement="top" title="Generate 2D and Potree Point Cloud Tiles" ></button>
<button id="resetDoPostProcessing" type="submit" class="btn glyphicon glyphicon glyphicon-repeat btn-default" data-toggle="tooltip" data-placement="top" title="Reset to default" ></button>
<br/><br/>
</div>
<div>
<label for="webhook">webhook callback url (optional):</label>
<br/>
<input id="webhook" name="webhook" class="form-control" type="text">
<button type="submit" class="btn glyphicon glyphicon-info-sign btn-info" data-toggle="tooltip" data-placement="top" title="Optional webhook" ></button>
<button type="submit" class="btn glyphicon glyphicon-info-sign btn-info" data-toggle="tooltip" data-placement="top" title="Optional webhook" ></button>
<button id="resetWebhook" type="submit" class="btn glyphicon glyphicon glyphicon-repeat btn-default" data-toggle="tooltip" data-placement="top" title="Reset to default" ></button>
<br/><br/>
</div>
<div data-bind="foreach: options">
<label data-bind="text: properties.name + (properties.domain ? ' (' + properties.domain + ')' : '')"></label><br/>

5
public/js/main.js
Wyświetl plik

@ -299,6 +299,7 @@ $(function() {
name: $("#taskName").val(),
zipurl: $("#zipurl").val(),
webhook: $("#webhook").val(),
skipPostProcessing: !$("#doPostProcessing").prop('checked'),
options: JSON.stringify(optionsModel.getUserOptions())
};
}
@ -326,6 +327,10 @@ $(function() {
$("#webhook").val('');
});
$('#resetDoPostProcessing').on('click', function(){
$("#doPostProcessing").prop('checked', false);
});
// zip file control
$('#btnShowImport').on('click', function(e){
e.preventDefault();