From 98343ddea1ec086cc03a3ff4549f4e9b6af18c1e Mon Sep 17 00:00:00 2001 From: Bibek Joshi <42735993+bibekjoshisrijan@users.noreply.github.com> Date: Thu, 17 Sep 2020 00:05:23 +0530 Subject: [PATCH 1/5] Added the s3 acl option to read from the command line or config file --- config.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config.js b/config.js index b93798f..8d18b73 100644 --- a/config.js +++ b/config.js @@ -51,6 +51,7 @@ Options: --s3_force_path_style Whether to force path style URLs for S3 objects. (default: false) --s3_secret_key S3 secret key, required if --s3_endpoint is set. (default: none) --s3_signature_version S3 signature version. (default: 4) + --s3_acl S3 object acl. (deafult: public-read) --s3_upload_everything Upload all task results to S3. (default: upload only .zip archive and orthophoto) --max_concurrency Place a cap on the max-concurrency option to use for each task. (default: no limit) --max_runtime Number of minutes (approximate) that a task is allowed to run before being forcibly canceled (timeout). (default: no limit) @@ -114,6 +115,7 @@ config.s3ForcePathStyle = argv.s3_force_path_style || fromConfigFile("s3ForcePat config.s3AccessKey = argv.s3_access_key || fromConfigFile("s3AccessKey", process.env.AWS_ACCESS_KEY_ID || "") config.s3SecretKey = argv.s3_secret_key || fromConfigFile("s3SecretKey", process.env.AWS_SECRET_ACCESS_KEY || "") config.s3SignatureVersion = argv.s3_signature_version || fromConfigFile("s3SignatureVersion", "4") +config.s3ACL = argv.s3_acl || fromConfigFile("s3_acl", "public-read") config.s3UploadEverything = argv.s3_upload_everything || fromConfigFile("s3UploadEverything", false); config.maxConcurrency = parseInt(argv.max_concurrency || fromConfigFile("maxConcurrency", 0)); config.maxRuntime = parseInt(argv.max_runtime || fromConfigFile("maxRuntime", -1)); From f7abea2f9b6faf48d42a5d338f4129ec9c27a31a Mon Sep 17 00:00:00 2001 From: Bibek Joshi <42735993+bibekjoshisrijan@users.noreply.github.com> Date: Thu, 17 Sep 2020 00:07:14 +0530 Subject: [PATCH 2/5] Added the support to read the s3 acl value from the config file --- libs/S3.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/S3.js b/libs/S3.js index c2c694c..7e00b18 100644 --- a/libs/S3.js +++ b/libs/S3.js @@ -76,7 +76,7 @@ module.exports = { Bucket: bucket, Key: file.dest, Body: fs.createReadStream(file.src), - ACL: 'public-read' + ACL: config.s3_acl }, {partSize: 5 * 1024 * 1024, queueSize: 1}, err => { if (err){ logger.debug(err); From aa2347ef3b1c5eed72013feeb5d93cc233aa821f Mon Sep 17 00:00:00 2001 From: Bibek Joshi <42735993+bibekjoshisrijan@users.noreply.github.com> Date: Thu, 17 Sep 2020 00:09:01 +0530 Subject: [PATCH 3/5] Added the correct value from config file --- libs/S3.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/S3.js b/libs/S3.js index 7e00b18..a317d2b 100644 --- a/libs/S3.js +++ b/libs/S3.js @@ -76,7 +76,7 @@ module.exports = { Bucket: bucket, Key: file.dest, Body: fs.createReadStream(file.src), - ACL: config.s3_acl + ACL: config.s3ACL }, {partSize: 5 * 1024 * 1024, queueSize: 1}, err => { if (err){ logger.debug(err); From d25627b36d1488bd2eb572bf96350fbfd47e5f30 Mon Sep 17 00:00:00 2001 From: Stephen Mather <1174901+smathermather@users.noreply.github.com> Date: Wed, 16 Sep 2020 15:37:40 -0400 Subject: [PATCH 4/5] typo fix --- config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.js b/config.js index 8d18b73..3a9c5c4 100644 --- a/config.js +++ b/config.js @@ -51,7 +51,7 @@ Options: --s3_force_path_style Whether to force path style URLs for S3 objects. (default: false) --s3_secret_key S3 secret key, required if --s3_endpoint is set. (default: none) --s3_signature_version S3 signature version. (default: 4) - --s3_acl S3 object acl. (deafult: public-read) + --s3_acl S3 object acl. (default: public-read) --s3_upload_everything Upload all task results to S3. (default: upload only .zip archive and orthophoto) --max_concurrency Place a cap on the max-concurrency option to use for each task. (default: no limit) --max_runtime Number of minutes (approximate) that a task is allowed to run before being forcibly canceled (timeout). (default: no limit) From 2ee189612d06df9198ae215105654441a7428349 Mon Sep 17 00:00:00 2001 From: Bibek Joshi <42735993+bibekjoshisrijan@users.noreply.github.com> Date: Thu, 17 Sep 2020 16:36:00 +0530 Subject: [PATCH 5/5] Added the IAM role authentication for the s3 access --- libs/S3.js | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/libs/S3.js b/libs/S3.js index a317d2b..b1e4ffe 100644 --- a/libs/S3.js +++ b/libs/S3.js @@ -32,15 +32,24 @@ module.exports = { }, initialize: function(cb){ - if (config.s3Endpoint && config.s3Bucket && config.s3AccessKey && config.s3SecretKey){ + if (config.s3Endpoint && config.s3Bucket){ const spacesEndpoint = new AWS.Endpoint(config.s3Endpoint); - s3 = new AWS.S3({ + + const s3Config = { endpoint: spacesEndpoint, signatureVersion: ('v' + config.s3SignatureVersion) || 'v4', - accessKeyId: config.s3AccessKey, - secretAccessKey: config.s3SecretKey, s3ForcePathStyle: config.s3ForcePathStyle, - }); + }; + + // If we are not using IAM roles then we need to pass access key and secret key in our config + if (config.s3AccessKey && config.s3SecretKey) { + s3Config['accessKeyId'] = config.s3AccessKey; + s3Config['secretAccessKey'] = config.s3SecretKey; + }else{ + logger.info("Secret Key and Access ID not passed. Using the IAM role"); + }; + + s3 = new AWS.S3(s3Config); // Test connection s3.putObject({