Faster unzipping with 7z

libs/processRunner.js

@@ -93,9 +93,19 @@ module.exports = {
                      ["projectFolderPath"]),
 
     sevenZip: makeRunner("7z", function(options){
-            return ["a", "-mx=0", "-r", "-bd", options.destination].concat(options.pathsToArchive);
+            return ["a", "-mx=0", "-y", "-r", "-bd", options.destination].concat(options.pathsToArchive);
         },
         ["destination", "pathsToArchive", "cwd"],
         null,
+        false),
+
+    sevenUnzip: makeRunner("7z", function(options){
+            let cmd = "x"; // eXtract files with full paths
+            if (options.noDirectories) cmd = "e"; //Extract files from archive (without using directory names)
+
+            return [cmd, "-aoa", "-bd", "-y", `-o${options.destination}`, options.file];
+        },
+        ["destination", "file"],
+        null,
         false)
 };

libs/taskNew.js

@@ -24,13 +24,13 @@ const uuidv4 = require('uuid/v4');
 const config = require('../config.js');
 const rmdir = require('rimraf');
 const Directories = require('./Directories');
-const unzip = require('node-unzip-2');
 const mv = require('mv');
 const Task = require('./Task');
 const async = require('async');
 const odmInfo = require('./odmInfo');
 const request = require('request');
-const utils = require('./utils');
+const ziputils = require('./ziputils');
+const { cancelJob } = require('node-schedule');
 
 const download = function(uri, filename, callback) {
     request.head(uri, function(err, res, body) {
@@ -243,6 +243,13 @@ module.exports = {
             let destImagesPath = path.join(destPath, "images");
             let destGcpPath = path.join(destPath, "gcp");
 
+            const checkMaxImageLimits = (cb) => {
+                fs.readdir(destImagesPath, (err, files) => {
+                    if (config.maxImages && files.length > config.maxImages) cb(`${files.length} images uploaded, but this node can only process up to ${config.maxImages}.`);
+                    else cb(err);
+                });
+            };
+
             async.series([
                 cb => {
                     odmInfo.filterOptions(req.body.options, (err, options) => {
@@ -309,17 +316,12 @@ module.exports = {
                             
                             // Extract
                             cb => {
-                                fs.createReadStream(seedFileDst).pipe(unzip.Extract({ path: destPath }))
+                                ziputils.unzip(seedFileDst, destPath, cb);
-                                    .on('close', cb)
-                                    .on('error', cb);
                             },
 
                             // Verify max images limit
                             cb => {
-                                fs.readdir(destImagesPath, (err, files) => {
+                                checkMaxImageLimits(cb);
-                                    if (config.maxImages && files.length > config.maxImages) cb(`${files.length} images uploaded, but this node can only process up to ${config.maxImages}.`);
-                                    else cb(err);
-                                });
                             },
 
                             // Remove
@@ -333,22 +335,20 @@ module.exports = {
                     }
 
                     const handleZipUrl = (cb) => {
-                        let filesCount = 0;
+                        async.series([
-                        fs.createReadStream(path.join(destImagesPath, "zipurl.zip")).pipe(unzip.Parse())
+                            // Extract images
-                            .on('entry', function(entry) {
+                            cb => {
-                                if (entry.type === 'File') {
+                                ziputils.unzip(path.join(destImagesPath, "zipurl.zip"), 
-                                    filesCount++;
+                                                destImagesPath, 
-                                    entry.pipe(fs.createWriteStream(path.join(destImagesPath, path.basename(entry.path))));
+                                                cb,
-                                } else {
+                                                true);
-                                    entry.autodrain();
+                            },
-                                }
+
-                            })
+                            // Count files
-                            .on('close', () => {
+                            cb => {
-                                // Verify max images limit
+                                checkMaxImageLimits(cb);
-                                if (config.maxImages && filesCount > config.maxImages) cb(`${filesCount} images uploaded, but this node can only process up to ${config.maxImages}.`);
+                            }
-                                else cb();
+                        ], cb);
-                            })
-                            .on('error', cb);
                     }
 
                     // Find and handle zip files and extract

libs/ziputils.js

@@ -0,0 +1,26 @@
+const processRunner = require('./processRunner');
+const nodeUnzip = require('node-unzip-2');
+const config = require('../config');
+const fs = require('fs');
+
+module.exports = {
+    unzip: function(file, outputDir, cb, noDirectories = false){
+        if (config.has7z){
+            processRunner.sevenUnzip({
+                file: file,
+                destination: outputDir,
+                noDirectories
+            }, (err, code, _) => {
+                if (err) cb(err);
+                else{
+                    if (code === 0) cb();
+                    else cb(new Error(`Could not extract .zip file, 7z exited with code ${code}`));
+                }
+            });
+        }else{
+            fs.createReadStream(file).pipe(nodeUnzip.Extract({ path: outputDir }))
+                .on('close', cb)
+                .on('error', cb);
+        }
+    }
+}