From bcbec55c039b02c068da18da466f268d1baa1fce Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 3 Dec 2018 21:46:49 -0500 Subject: [PATCH] Added s3_upload_everything flag --- config.js | 2 ++ libs/S3.js | 2 +- libs/Task.js | 11 ++++++++--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/config.js b/config.js index 01249fe..487ea6c 100644 --- a/config.js +++ b/config.js @@ -45,6 +45,7 @@ Options: --s3_access_key S3 access key, required if --s3_endpoint is set. (default: none) --s3_secret_key S3 secret key, required if --s3_endpoint is set. (default: none) --s3_signature_version S3 signature version. (default: 4) + --s3_upload_everything Upload all task results to S3. (default: upload only .zip archive and orthophoto) --max_concurrency Place a cap on the max-concurrency option to use for each task. (default: no limit) Log Levels: error | debug | info | verbose | debug | silly @@ -101,6 +102,7 @@ config.s3Bucket = argv.s3_bucket || fromConfigFile("s3Bucket", ""); config.s3AccessKey = argv.s3_access_key || fromConfigFile("s3AccessKey", process.env.AWS_ACCESS_KEY_ID || "") config.s3SecretKey = argv.s3_secret_key || fromConfigFile("s3SecretKey", process.env.AWS_SECRET_ACCESS_KEY || "") config.s3SignatureVersion = argv.s3_signature_version || fromConfigFile("s3SignatureVersion", "4") +config.s3UploadEverything = argv.s3_upload_everything || fromConfigFile("s3UploadEverything", false); config.maxConcurrency = parseInt(argv.max_concurrency || fromConfigFile("maxConcurrency", 0)); module.exports = config; diff --git a/libs/S3.js b/libs/S3.js index 24193b6..792792d 100644 --- a/libs/S3.js +++ b/libs/S3.js @@ -66,7 +66,7 @@ module.exports = { uploadPaths: function(srcFolder, bucket, dstFolder, paths, cb, onOutput){ if (!s3) throw new Error("S3 is not initialized"); - const PARALLEL_UPLOADS = 10; + const PARALLEL_UPLOADS = 5; const q = async.queue((file, done) => { logger.debug(`Uploading ${file.src} --> ${file.dest}`); diff --git a/libs/Task.js b/libs/Task.js index 157c6e6..751303c 100644 --- a/libs/Task.js +++ b/libs/Task.js @@ -342,9 +342,14 @@ module.exports = class Task{ // Upload to S3 all paths + all.zip file (if config says so) if (S3.enabled()){ tasks.push((done) => { - const s3Paths = !config.test ? - ['all.zip'].concat(allPaths) : - ['all.zip']; // During testing only upload all.zip + let s3Paths; + if (config.test){ + s3Paths = ['all.zip']; // During testing only upload all.zip + }else if (config.s3UploadEverything){ + s3Paths = ['all.zip'].concat(allPaths) + }else{ + s3Paths = ['all.zip', 'odm_orthophoto/odm_orthophoto.tif']; + } S3.uploadPaths(this.getProjectFolderPath(), config.s3Bucket, this.uuid, s3Paths, err => {