diff --git a/config.js b/config.js index 95b6beb..20e013c 100644 --- a/config.js +++ b/config.js @@ -51,6 +51,7 @@ Options: --s3_signature_version S3 signature version. (default: 4) --s3_upload_everything Upload all task results to S3. (default: upload only .zip archive and orthophoto) --max_concurrency Place a cap on the max-concurrency option to use for each task. (default: no limit) + --max_runtime Number of minutes that a task is allowed to run before being forcibly canceled (timeout). (default: no limit) Log Levels: error | debug | info | verbose | debug | silly `); @@ -112,5 +113,6 @@ config.s3SecretKey = argv.s3_secret_key || fromConfigFile("s3SecretKey", process config.s3SignatureVersion = argv.s3_signature_version || fromConfigFile("s3SignatureVersion", "4") config.s3UploadEverything = argv.s3_upload_everything || fromConfigFile("s3UploadEverything", false); config.maxConcurrency = parseInt(argv.max_concurrency || fromConfigFile("maxConcurrency", 0)); +config.maxRuntime = parseInt(argv.max_runtime || fromConfigFile("maxRuntime", -1)); module.exports = config; diff --git a/libs/TaskManager.js b/libs/TaskManager.js index 540681e..98cf127 100644 --- a/libs/TaskManager.js +++ b/libs/TaskManager.js @@ -59,6 +59,13 @@ class TaskManager{ this.dumpTaskList(); this.removeStaleUploads(); }); + + if (config.maxRuntime > 0){ + // Every minute + schedule.scheduleJob('* * * * *', () => { + this.checkTimeouts(); + }); + } cb(); } @@ -306,6 +313,21 @@ class TaskManager{ } return count; } + + checkTimeouts(){ + if (config.maxRuntime > 0){ + for (let uuid in this.tasks){ + let task = this.tasks[uuid]; + + if (task.processingTime > config.maxRuntime * 60 * 1000){ + task.output.push(`Task timed out after ${task.processingTime / 60 / 1000} seconds.\n`); + this.cancel(uuid, () => { + logger.warn(`Task ${uuid} timed out`); + }); + } + } + } + } } module.exports = {