diff --git a/libs/S3.js b/libs/S3.js index 720b4b9..56548a1 100644 --- a/libs/S3.js +++ b/libs/S3.js @@ -66,7 +66,8 @@ module.exports = { uploadPaths: function(srcFolder, bucket, dstFolder, paths, cb, onOutput){ if (!s3) throw new Error("S3 is not initialized"); - const PARALLEL_UPLOADS = 1; + const PARALLEL_UPLOADS = 5; + const MAX_RETRIES = 6; const q = async.queue((file, done) => { logger.debug(`Uploading ${file.src} --> ${file.dest}`); @@ -75,16 +76,34 @@ module.exports = { Key: file.dest, Body: fs.createReadStream(file.src), ACL: 'public-read' - }, err => { + }, {partSize: 10 * 1024 * 1024, queueSize: 1}, err => { if (err){ logger.debug(err); - const msg = "Cannot upload file to S3: " + err.code; - if (onOutput) onOutput(msg) - done(new Error(msg)); + const msg = `Cannot upload file to S3: ${err.code}, retrying... ${file.retries}`; + if (onOutput) onOutput(msg); + if (file.retries < MAX_RETRIES){ + file.retries++; + setTimeout(() => { + q.push(file, errHandler); + done(); + }, (2 ** file.retries) * 1000); + }else{ + done(new Error(msg)); + } }else done(); }); }, PARALLEL_UPLOADS); + const errHandler = err => { + if (err){ + q.kill(); + if (!cbCalled){ + cbCalled = true; + cb(err); + } + } + }; + let uploadList = []; paths.forEach(p => { @@ -99,13 +118,15 @@ module.exports = { globPaths.forEach(gp => { uploadList.push({ src: path.join(srcFolder, gp), - dest: path.join(dstFolder, gp) + dest: path.join(dstFolder, gp), + retries: 0 }); }); }else{ uploadList.push({ src: fullPath, - dest: path.join(dstFolder, p) + dest: path.join(dstFolder, p), + retries: 0 }); } }); @@ -119,14 +140,6 @@ module.exports = { }; if (onOutput) onOutput(`Uploading ${uploadList.length} files to S3...`); - q.push(uploadList, err => { - if (err){ - q.kill(); - if (!cbCalled){ - cbCalled = true; - cb(err); - } - } - }); + q.push(uploadList, errHandler); } }; diff --git a/libs/Task.js b/libs/Task.js index 751303c..66d76ef 100644 --- a/libs/Task.js +++ b/libs/Task.js @@ -346,11 +346,11 @@ module.exports = class Task{ if (config.test){ s3Paths = ['all.zip']; // During testing only upload all.zip }else if (config.s3UploadEverything){ - s3Paths = ['all.zip'].concat(allPaths) + s3Paths = ['all.zip'].concat(allPaths); }else{ s3Paths = ['all.zip', 'odm_orthophoto/odm_orthophoto.tif']; } - + S3.uploadPaths(this.getProjectFolderPath(), config.s3Bucket, this.uuid, s3Paths, err => { if (!err) this.output.push("Done uploading to S3!"); diff --git a/public/js/main.js b/public/js/main.js index 7de8a96..3b2adb2 100644 --- a/public/js/main.js +++ b/public/js/main.js @@ -150,6 +150,10 @@ $(function() { if (json.processingTime && json.processingTime !== -1) { self.timeElapsed(hoursMinutesSecs(json.processingTime)); } + if (json.status && json.status.code && [codes.COMPLETED, codes.FAILED, codes.CANCELED].indexOf(json.status.code) !== -1){ + self.stopRefreshingInfo(); + } + self.info(json); }) .fail(function() {