From 802d72a48b1e69b8269e6b689dc1a5ff987b8e08 Mon Sep 17 00:00:00 2001
From: Piero Toffanin <pt@masseranolabs.com>
Date: Thu, 10 Sep 2020 12:24:08 -0400
Subject: [PATCH] Faster unzipping with 7z

---
 libs/processRunner.js | 12 ++++++++++-
 libs/taskNew.js       | 50 +++++++++++++++++++++----------------------
 libs/ziputils.js      | 26 ++++++++++++++++++++++
 3 files changed, 62 insertions(+), 26 deletions(-)
 create mode 100644 libs/ziputils.js

diff --git a/libs/processRunner.js b/libs/processRunner.js
index f5e000c..2669316 100644
--- a/libs/processRunner.js
+++ b/libs/processRunner.js
@@ -93,9 +93,19 @@ module.exports = {
                      ["projectFolderPath"]),
 
     sevenZip: makeRunner("7z", function(options){
-            return ["a", "-mx=0", "-r", "-bd", options.destination].concat(options.pathsToArchive);
+            return ["a", "-mx=0", "-y", "-r", "-bd", options.destination].concat(options.pathsToArchive);
         },
         ["destination", "pathsToArchive", "cwd"],
         null,
+        false),
+
+    sevenUnzip: makeRunner("7z", function(options){
+            let cmd = "x"; // eXtract files with full paths
+            if (options.noDirectories) cmd = "e"; //Extract files from archive (without using directory names)
+
+            return [cmd, "-aoa", "-bd", "-y", `-o${options.destination}`, options.file];
+        },
+        ["destination", "file"],
+        null,
         false)
 };
diff --git a/libs/taskNew.js b/libs/taskNew.js
index 7535e3b..3714e54 100644
--- a/libs/taskNew.js
+++ b/libs/taskNew.js
@@ -24,13 +24,13 @@ const uuidv4 = require('uuid/v4');
 const config = require('../config.js');
 const rmdir = require('rimraf');
 const Directories = require('./Directories');
-const unzip = require('node-unzip-2');
 const mv = require('mv');
 const Task = require('./Task');
 const async = require('async');
 const odmInfo = require('./odmInfo');
 const request = require('request');
-const utils = require('./utils');
+const ziputils = require('./ziputils');
+const { cancelJob } = require('node-schedule');
 
 const download = function(uri, filename, callback) {
     request.head(uri, function(err, res, body) {
@@ -243,6 +243,13 @@ module.exports = {
             let destImagesPath = path.join(destPath, "images");
             let destGcpPath = path.join(destPath, "gcp");
 
+            const checkMaxImageLimits = (cb) => {
+                fs.readdir(destImagesPath, (err, files) => {
+                    if (config.maxImages && files.length > config.maxImages) cb(`${files.length} images uploaded, but this node can only process up to ${config.maxImages}.`);
+                    else cb(err);
+                });
+            };
+
             async.series([
                 cb => {
                     odmInfo.filterOptions(req.body.options, (err, options) => {
@@ -309,17 +316,12 @@ module.exports = {
                             
                             // Extract
                             cb => {
-                                fs.createReadStream(seedFileDst).pipe(unzip.Extract({ path: destPath }))
-                                    .on('close', cb)
-                                    .on('error', cb);
+                                ziputils.unzip(seedFileDst, destPath, cb);
                             },
 
                             // Verify max images limit
                             cb => {
-                                fs.readdir(destImagesPath, (err, files) => {
-                                    if (config.maxImages && files.length > config.maxImages) cb(`${files.length} images uploaded, but this node can only process up to ${config.maxImages}.`);
-                                    else cb(err);
-                                });
+                                checkMaxImageLimits(cb);
                             },
 
                             // Remove
@@ -333,22 +335,20 @@ module.exports = {
                     }
 
                     const handleZipUrl = (cb) => {
-                        let filesCount = 0;
-                        fs.createReadStream(path.join(destImagesPath, "zipurl.zip")).pipe(unzip.Parse())
-                            .on('entry', function(entry) {
-                                if (entry.type === 'File') {
-                                    filesCount++;
-                                    entry.pipe(fs.createWriteStream(path.join(destImagesPath, path.basename(entry.path))));
-                                } else {
-                                    entry.autodrain();
-                                }
-                            })
-                            .on('close', () => {
-                                // Verify max images limit
-                                if (config.maxImages && filesCount > config.maxImages) cb(`${filesCount} images uploaded, but this node can only process up to ${config.maxImages}.`);
-                                else cb();
-                            })
-                            .on('error', cb);
+                        async.series([
+                            // Extract images
+                            cb => {
+                                ziputils.unzip(path.join(destImagesPath, "zipurl.zip"), 
+                                                destImagesPath, 
+                                                cb,
+                                                true);
+                            },
+
+                            // Count files
+                            cb => {
+                                checkMaxImageLimits(cb);
+                            }
+                        ], cb);
                     }
 
                     // Find and handle zip files and extract
diff --git a/libs/ziputils.js b/libs/ziputils.js
new file mode 100644
index 0000000..f7cb620
--- /dev/null
+++ b/libs/ziputils.js
@@ -0,0 +1,26 @@
+const processRunner = require('./processRunner');
+const nodeUnzip = require('node-unzip-2');
+const config = require('../config');
+const fs = require('fs');
+
+module.exports = {
+    unzip: function(file, outputDir, cb, noDirectories = false){
+        if (config.has7z){
+            processRunner.sevenUnzip({
+                file: file,
+                destination: outputDir,
+                noDirectories
+            }, (err, code, _) => {
+                if (err) cb(err);
+                else{
+                    if (code === 0) cb();
+                    else cb(new Error(`Could not extract .zip file, 7z exited with code ${code}`));
+                }
+            });
+        }else{
+            fs.createReadStream(file).pipe(nodeUnzip.Extract({ path: outputDir }))
+                .on('close', cb)
+                .on('error', cb);
+        }
+    }
+}
\ No newline at end of file