kopia lustrzana https://github.com/OpenDroneMap/NodeODM
Spaces --> tabs
Piero Toffanin
rodzic
133994bb91
commit
bb66d2852f
186
libs/Task.js
186
libs/Task.js
|
|
@ -132,8 +132,8 @@ module.exports = class Task{
|
||||||
else filename = path.join('..', '..', 'processing_results', 'odm_orthophoto', `odm_${filename}`);
|
else filename = path.join('..', '..', 'processing_results', 'odm_orthophoto', `odm_${filename}`);
|
||||||
}else{
|
}else{
|
||||||
filename = path.join('odm_orthophoto', `odm_${filename}`);
|
filename = path.join('odm_orthophoto', `odm_${filename}`);
|
||||||
}
|
}
|
||||||
}else{
|
}else{
|
||||||
return false; // Invalid
|
return false; // Invalid
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -196,9 +196,9 @@ module.exports = class Task{
|
||||||
// TODO: this does NOT guarantee that
|
// TODO: this does NOT guarantee that
|
||||||
// the process will immediately terminate.
|
// the process will immediately terminate.
|
||||||
// For eaxmple in the case of the ODM process, the process will continue running for a while
|
// For eaxmple in the case of the ODM process, the process will continue running for a while
|
||||||
// This might need to be fixed on ODM's end.
|
// This might need to be fixed on ODM's end.
|
||||||
|
|
||||||
// During testing, proc is undefined
|
// During testing, proc is undefined
|
||||||
if (proc) kill(proc.pid);
|
if (proc) kill(proc.pid);
|
||||||
});
|
});
|
||||||
this.runningProcesses = [];
|
this.runningProcesses = [];
|
||||||
|
|
@ -217,7 +217,7 @@ module.exports = class Task{
|
||||||
const finished = err => {
|
const finished = err => {
|
||||||
this.stopTrackingProcessingTime();
|
this.stopTrackingProcessingTime();
|
||||||
done(err);
|
done(err);
|
||||||
};
|
};
|
||||||
|
|
||||||
const postProcess = () => {
|
const postProcess = () => {
|
||||||
const createZipArchive = (outputFilename, files) => {
|
const createZipArchive = (outputFilename, files) => {
|
||||||
|
|
@ -226,8 +226,8 @@ module.exports = class Task{
|
||||||
|
|
||||||
let output = fs.createWriteStream(this.getAssetsArchivePath(outputFilename));
|
let output = fs.createWriteStream(this.getAssetsArchivePath(outputFilename));
|
||||||
let archive = archiver.create('zip', {
|
let archive = archiver.create('zip', {
|
||||||
zlib: { level: 1 } // Sets the compression level (1 = best speed since most assets are already compressed)
|
zlib: { level: 1 } // Sets the compression level (1 = best speed since most assets are already compressed)
|
||||||
});
|
});
|
||||||
|
|
||||||
archive.on('finish', () => {
|
archive.on('finish', () => {
|
||||||
// TODO: is this being fired twice?
|
// TODO: is this being fired twice?
|
||||||
|
|
@ -240,11 +240,11 @@ module.exports = class Task{
|
||||||
});
|
});
|
||||||
|
|
||||||
archive.pipe(output);
|
archive.pipe(output);
|
||||||
let globs = [];
|
let globs = [];
|
||||||
|
|
||||||
const sourcePath = !config.test ?
|
const sourcePath = !config.test ?
|
||||||
this.getProjectFolderPath() :
|
this.getProjectFolderPath() :
|
||||||
path.join("tests", "processing_results");
|
path.join("tests", "processing_results");
|
||||||
|
|
||||||
// Process files and directories first
|
// Process files and directories first
|
||||||
files.forEach(file => {
|
files.forEach(file => {
|
||||||
|
|
@ -334,32 +334,32 @@ module.exports = class Task{
|
||||||
allPaths.splice(allPaths.indexOf(p), 1);
|
allPaths.splice(allPaths.indexOf(p), 1);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let tasks = [
|
let tasks = [
|
||||||
runPostProcessingScript(),
|
runPostProcessingScript(),
|
||||||
createZipArchive('all.zip', allPaths)
|
createZipArchive('all.zip', allPaths)
|
||||||
];
|
];
|
||||||
|
|
||||||
// Upload to S3 all paths + all.zip file (if config says so)
|
// Upload to S3 all paths + all.zip file (if config says so)
|
||||||
if (S3.enabled()){
|
if (S3.enabled()){
|
||||||
tasks.push((done) => {
|
tasks.push((done) => {
|
||||||
let s3Paths;
|
let s3Paths;
|
||||||
if (config.test){
|
if (config.test){
|
||||||
s3Paths = ['all.zip']; // During testing only upload all.zip
|
s3Paths = ['all.zip']; // During testing only upload all.zip
|
||||||
}else if (config.s3UploadEverything){
|
}else if (config.s3UploadEverything){
|
||||||
s3Paths = ['all.zip'].concat(allPaths);
|
s3Paths = ['all.zip'].concat(allPaths);
|
||||||
}else{
|
}else{
|
||||||
s3Paths = ['all.zip', 'odm_orthophoto/odm_orthophoto.tif'];
|
s3Paths = ['all.zip', 'odm_orthophoto/odm_orthophoto.tif'];
|
||||||
}
|
}
|
||||||
|
|
||||||
S3.uploadPaths(this.getProjectFolderPath(), config.s3Bucket, this.uuid, s3Paths,
|
S3.uploadPaths(this.getProjectFolderPath(), config.s3Bucket, this.uuid, s3Paths,
|
||||||
err => {
|
err => {
|
||||||
if (!err) this.output.push("Done uploading to S3!");
|
if (!err) this.output.push("Done uploading to S3!");
|
||||||
done(err);
|
done(err);
|
||||||
}, output => this.output.push(output));
|
}, output => this.output.push(output));
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async.series(tasks, (err) => {
|
async.series(tasks, (err) => {
|
||||||
if (!err){
|
if (!err){
|
||||||
|
|
@ -453,64 +453,64 @@ module.exports = class Task{
|
||||||
// Optionally starting from a certain line number
|
// Optionally starting from a certain line number
|
||||||
getOutput(startFromLine = 0){
|
getOutput(startFromLine = 0){
|
||||||
return this.output.slice(startFromLine, this.output.length);
|
return this.output.slice(startFromLine, this.output.length);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reads the contents of the tasks's
|
// Reads the contents of the tasks's
|
||||||
// images.json and returns its JSON representation
|
// images.json and returns its JSON representation
|
||||||
readImagesDatabase(callback){
|
readImagesDatabase(callback){
|
||||||
const imagesDbPath = !config.test ?
|
const imagesDbPath = !config.test ?
|
||||||
path.join(this.getProjectFolderPath(), 'images.json') :
|
path.join(this.getProjectFolderPath(), 'images.json') :
|
||||||
path.join('tests', 'processing_results', 'images.json');
|
path.join('tests', 'processing_results', 'images.json');
|
||||||
|
|
||||||
fs.readFile(imagesDbPath, 'utf8', (err, data) => {
|
fs.readFile(imagesDbPath, 'utf8', (err, data) => {
|
||||||
if (err) callback(err);
|
if (err) callback(err);
|
||||||
else{
|
else{
|
||||||
try{
|
try{
|
||||||
const json = JSON.parse(data);
|
const json = JSON.parse(data);
|
||||||
callback(null, json);
|
callback(null, json);
|
||||||
}catch(e){
|
}catch(e){
|
||||||
callback(e);
|
callback(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
callWebhooks(){
|
callWebhooks(){
|
||||||
// Hooks can be passed via command line
|
// Hooks can be passed via command line
|
||||||
// or for each individual task
|
// or for each individual task
|
||||||
const hooks = [this.webhook, config.webhook];
|
const hooks = [this.webhook, config.webhook];
|
||||||
|
|
||||||
this.readImagesDatabase((err, images) => {
|
this.readImagesDatabase((err, images) => {
|
||||||
if (err) logger.warn(err); // Continue with callback
|
if (err) logger.warn(err); // Continue with callback
|
||||||
if (!images) images = [];
|
if (!images) images = [];
|
||||||
|
|
||||||
let json = this.getInfo();
|
let json = this.getInfo();
|
||||||
json.images = images;
|
json.images = images;
|
||||||
|
|
||||||
hooks.forEach(hook => {
|
hooks.forEach(hook => {
|
||||||
if (hook && hook.length > 3){
|
if (hook && hook.length > 3){
|
||||||
const notifyCallback = (attempt) => {
|
const notifyCallback = (attempt) => {
|
||||||
if (attempt > 5){
|
if (attempt > 5){
|
||||||
logger.warn(`Webhook invokation failed, will not retry: ${hook}`);
|
logger.warn(`Webhook invokation failed, will not retry: ${hook}`);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
request.post(hook, { json },
|
request.post(hook, { json },
|
||||||
(error, response) => {
|
(error, response) => {
|
||||||
if (error || response.statusCode != 200){
|
if (error || response.statusCode != 200){
|
||||||
logger.warn(`Webhook invokation failed, will retry in a bit: ${hook}`);
|
logger.warn(`Webhook invokation failed, will retry in a bit: ${hook}`);
|
||||||
setTimeout(() => {
|
setTimeout(() => {
|
||||||
notifyCallback(attempt + 1);
|
notifyCallback(attempt + 1);
|
||||||
}, attempt * 5000);
|
}, attempt * 5000);
|
||||||
}else{
|
}else{
|
||||||
logger.debug(`Webhook invoked: ${hook}`);
|
logger.debug(`Webhook invoked: ${hook}`);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
notifyCallback(0);
|
notifyCallback(0);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// Returns the data necessary to serialize this
|
// Returns the data necessary to serialize this
|
||||||
// task to restore it later.
|
// task to restore it later.
|
||||||
|
|
|
||||||
Ładowanie…
Reference in New Issue