Merge pull request #59 from pierotofy/s3resiliance

Added retries to S3 uploads, UI improvement

libs/S3.js

@@ -66,7 +66,8 @@ module.exports = {
     uploadPaths: function(srcFolder, bucket, dstFolder, paths, cb, onOutput){
         if (!s3) throw new Error("S3 is not initialized");
 
-        const PARALLEL_UPLOADS = 1;
+        const PARALLEL_UPLOADS = 5;
+        const MAX_RETRIES = 6;
 
         const q = async.queue((file, done) => {
             logger.debug(`Uploading ${file.src} --> ${file.dest}`);
@@ -75,16 +76,34 @@ module.exports = {
                 Key: file.dest,
                 Body: fs.createReadStream(file.src),
                 ACL: 'public-read'
-            }, err => {
+            }, {partSize: 10 * 1024 * 1024, queueSize: 1}, err => {
                 if (err){
                     logger.debug(err);
-                    const msg = "Cannot upload file to S3: " + err.code;
+                    const msg = `Cannot upload file to S3: ${err.code}, retrying... ${file.retries}`;
-                    if (onOutput) onOutput(msg)
+                    if (onOutput) onOutput(msg);
-                    done(new Error(msg));
+                    if (file.retries < MAX_RETRIES){
+                        file.retries++;
+                        setTimeout(() => {
+                            q.push(file, errHandler);
+                            done();
+                        }, (2 ** file.retries) * 1000);
+                    }else{
+                        done(new Error(msg));
+                    }
                 }else done();
             });
         }, PARALLEL_UPLOADS);
 
+        const errHandler = err => {
+            if (err){
+                q.kill();
+                if (!cbCalled){
+                    cbCalled = true;
+                    cb(err);
+                }
+            }
+        };
+
         let uploadList = [];
 
         paths.forEach(p => {
@@ -99,13 +118,15 @@ module.exports = {
                 globPaths.forEach(gp => {
                     uploadList.push({
                         src: path.join(srcFolder, gp),
-                        dest: path.join(dstFolder, gp)
+                        dest: path.join(dstFolder, gp),
+                        retries: 0
                     });
                 });
             }else{
                 uploadList.push({
                     src: fullPath,
-                    dest: path.join(dstFolder, p)
+                    dest: path.join(dstFolder, p),
+                    retries: 0
                 });
             }
         });
@@ -119,14 +140,6 @@ module.exports = {
         };
 
         if (onOutput) onOutput(`Uploading ${uploadList.length} files to S3...`);
-        q.push(uploadList, err => {
+        q.push(uploadList, errHandler);
-            if (err){
-                q.kill();
-                if (!cbCalled){
-                    cbCalled = true;
-                    cb(err);
-                }
-            }
-        });
     }
 };

libs/Task.js

@@ -346,11 +346,11 @@ module.exports = class Task{
                     if (config.test){
                         s3Paths = ['all.zip']; // During testing only upload all.zip
                     }else if (config.s3UploadEverything){
-                        s3Paths = ['all.zip'].concat(allPaths)
+                        s3Paths = ['all.zip'].concat(allPaths);
                     }else{
                         s3Paths = ['all.zip', 'odm_orthophoto/odm_orthophoto.tif'];
                     }
-
+                    
                     S3.uploadPaths(this.getProjectFolderPath(), config.s3Bucket, this.uuid, s3Paths, 
                         err => {
                             if (!err) this.output.push("Done uploading to S3!");

public/js/main.js

@@ -150,6 +150,10 @@ $(function() {
                 if (json.processingTime && json.processingTime !== -1) {
                     self.timeElapsed(hoursMinutesSecs(json.processingTime));
                 }
+                if (json.status && json.status.code && [codes.COMPLETED, codes.FAILED, codes.CANCELED].indexOf(json.status.code) !== -1){
+                    self.stopRefreshingInfo();
+                }
+
                 self.info(json);
             })
             .fail(function() {