From b2f7a19e1fc991f93551ae0ac27e87de66e00d0d Mon Sep 17 00:00:00 2001
From: Piero Toffanin <pt@masseranolabs.com>
Date: Mon, 2 Nov 2020 12:13:30 -0500
Subject: [PATCH] Faster S3 uploads, progress reporting

---
 libs/S3.js | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/libs/S3.js b/libs/S3.js
index b1e4ffe..c200d58 100644
--- a/libs/S3.js
+++ b/libs/S3.js
@@ -76,9 +76,12 @@ module.exports = {
     uploadPaths: function(srcFolder, bucket, dstFolder, paths, cb, onOutput){
         if (!s3) throw new Error("S3 is not initialized");
 
-        const PARALLEL_UPLOADS = 5;
+        const PARALLEL_UPLOADS = 5; // Upload these many files at the same time
         const MAX_RETRIES = 6;
 
+        let concurrency = 10; // Upload these many parts per file at the same time
+        let progress = 0;
+
         const q = async.queue((file, done) => {
             logger.debug(`Uploading ${file.src} --> ${file.dest}`);
             s3.upload({
@@ -86,13 +89,16 @@ module.exports = {
                 Key: file.dest,
                 Body: fs.createReadStream(file.src),
                 ACL: config.s3ACL
-            }, {partSize: 5 * 1024 * 1024, queueSize: 1}, err => {
+            }, {partSize: 100 * 1024 * 1024, queueSize: concurrency}, err => {
                 if (err){
                     logger.debug(err);
                     const msg = `Cannot upload file to S3: ${err.code}, retrying... ${file.retries}`;
                     if (onOutput) onOutput(msg);
                     if (file.retries < MAX_RETRIES){
                         file.retries++;
+                        concurrency = Math.max(1, Math.floor(concurrency * 0.66));
+                        progress = 0;
+
                         setTimeout(() => {
                             q.push(file, errHandler);
                             done();
@@ -101,6 +107,12 @@ module.exports = {
                         done(new Error(msg));
                     }
                 }else done();
+            }).on('httpUploadProgress', p => {
+                const perc = Math.round((p.loaded / p.total) * 100)
+                if (perc % 5 == 0 && progress < perc){
+                    progress = perc;
+                    if (onOutput) onOutput(`Uploading ${path.basename(file.dest)}... ${progress}%`);
+                }
             });
         }, PARALLEL_UPLOADS);