Merged 7unzip branch

Dockerfile

@@ -5,7 +5,7 @@ EXPOSE 3000
 
 USER root
 RUN curl --silent --location https://deb.nodesource.com/setup_10.x | bash -
-RUN apt-get install -y nodejs p7zip-full && npm install -g nodemon && \
+RUN apt-get install -y nodejs unzip p7zip-full && npm install -g nodemon && \
     ln -s /code/SuperBuild/install/bin/entwine /usr/bin/entwine && \
     ln -s /code/SuperBuild/install/bin/pdal /usr/bin/pdal
 

README.md

@@ -56,11 +56,11 @@ If you are already running [ODM](https://github.com/OpenDroneMap/ODM) on Ubuntu
 
 1) Install Entwine: https://entwine.io/quickstart.html#installation
  
-2) Install node.js, npm dependencies and 7zip:
+2) Install node.js, npm dependencies, 7zip and unzip:
 
 ```bash
 sudo curl --silent --location https://deb.nodesource.com/setup_6.x | sudo bash -
-sudo apt-get install -y nodejs python-gdal p7zip-full
+sudo apt-get install -y nodejs python-gdal p7zip-full unzip
 git clone https://github.com/OpenDroneMap/NodeODM
 cd NodeODM
 npm install

config.js

@@ -119,7 +119,8 @@ config.maxConcurrency = parseInt(argv.max_concurrency || fromConfigFile("maxConc
 config.maxRuntime = parseInt(argv.max_runtime || fromConfigFile("maxRuntime", -1));
 
 // Detect 7z availability
-const childProcess = spawnSync("7z", ['--help']);
+config.has7z = spawnSync("7z", ['--help']).status === 0;
-config.has7z = childProcess.status === 0;
+config.hasUnzip = spawnSync("unzip", ['--help']).status === 0;
+
 
 module.exports = config;

index.js

@@ -900,9 +900,9 @@ if (config.test) {
     if (config.testDropUploads) logger.info("Uploads will drop at random");
 }
 
-if (!config.has7z){
+if (!config.hasUnzip) logger.warn("The unzip program is not installed, (certain unzip operations might be slower)");
-    logger.warn("The 7z program is not installed, falling back to legacy (zipping will be slower)");
+if (!config.has7z) logger.warn("The 7z program is not installed, falling back to legacy (zipping will be slower)");
-}
+
 
 let commands = [
     cb => odmInfo.initialize(cb),

libs/processRunner.js

@@ -93,9 +93,27 @@ module.exports = {
                      ["projectFolderPath"]),
 
     sevenZip: makeRunner("7z", function(options){
-            return ["a", "-mx=0", "-r", "-bd", options.destination].concat(options.pathsToArchive);
+            return ["a", "-mx=0", "-y", "-r", "-bd", options.destination].concat(options.pathsToArchive);
         },
         ["destination", "pathsToArchive", "cwd"],
         null,
+        false),
+
+    sevenUnzip: makeRunner("7z", function(options){
+            let cmd = "x"; // eXtract files with full paths
+            if (options.noDirectories) cmd = "e"; //Extract files from archive (without using directory names)
+
+            return [cmd, "-aoa", "-bd", "-y", `-o${options.destination}`, options.file];
+        },
+        ["destination", "file"],
+        null,
+        false),
+
+    unzip: makeRunner("unzip", function(options){
+            const opts = options.noDirectories ? ["-j"] : [];
+            return opts.concat(["-qq", "-o", options.file, "-d", options.destination]);
+        },
+        ["destination", "file"],
+        null,
         false)
 };

libs/taskNew.js

@@ -24,13 +24,13 @@ const uuidv4 = require('uuid/v4');
 const config = require('../config.js');
 const rmdir = require('rimraf');
 const Directories = require('./Directories');
-const unzip = require('node-unzip-2');
 const mv = require('mv');
 const Task = require('./Task');
 const async = require('async');
 const odmInfo = require('./odmInfo');
 const request = require('request');
-const utils = require('./utils');
+const ziputils = require('./ziputils');
+const { cancelJob } = require('node-schedule');
 
 const download = function(uri, filename, callback) {
     request.head(uri, function(err, res, body) {
@@ -163,7 +163,7 @@ module.exports = {
                                 else cb(null, body);
                             });
                         }catch(e){
-                            cb("Malformed body.json");
+                            cb(new Error("Malformed body.json"));
                         }
                     }
                 });
@@ -243,6 +243,17 @@ module.exports = {
             let destImagesPath = path.join(destPath, "images");
             let destGcpPath = path.join(destPath, "gcp");
 
+            const checkMaxImageLimits = (cb) => {
+                if (!config.maxImages) cb();
+                else{
+                    fs.readdir(destImagesPath, (err, files) => {
+                        if (err) cb(err);
+                        else if (files.length > config.maxImages) cb(new Error(`${files.length} images uploaded, but this node can only process up to ${config.maxImages}.`));
+                        else cb();
+                    });
+                }
+            };
+
             async.series([
                 cb => {
                     odmInfo.filterOptions(req.body.options, (err, options) => {
@@ -309,17 +320,7 @@ module.exports = {
                             
                             // Extract
                             cb => {
-                                fs.createReadStream(seedFileDst).pipe(unzip.Extract({ path: destPath }))
+                                ziputils.unzip(seedFileDst, destPath, cb);
-                                    .on('close', cb)
-                                    .on('error', cb);
-                            },
-
-                            // Verify max images limit
-                            cb => {
-                                fs.readdir(destImagesPath, (err, files) => {
-                                    if (config.maxImages && files.length > config.maxImages) cb(`${files.length} images uploaded, but this node can only process up to ${config.maxImages}.`);
-                                    else cb(err);
-                                });
                             },
 
                             // Remove
@@ -333,22 +334,10 @@ module.exports = {
                     }
 
                     const handleZipUrl = (cb) => {
-                        let filesCount = 0;
+                        // Extract images
-                        fs.createReadStream(path.join(destImagesPath, "zipurl.zip")).pipe(unzip.Parse())
+                        ziputils.unzip(path.join(destImagesPath, "zipurl.zip"), 
-                            .on('entry', function(entry) {
+                                        destImagesPath, 
-                                if (entry.type === 'File') {
+                                        cb, true);
-                                    filesCount++;
-                                    entry.pipe(fs.createWriteStream(path.join(destImagesPath, path.basename(entry.path))));
-                                } else {
-                                    entry.autodrain();
-                                }
-                            })
-                            .on('close', () => {
-                                // Verify max images limit
-                                if (config.maxImages && filesCount > config.maxImages) cb(`${filesCount} images uploaded, but this node can only process up to ${config.maxImages}.`);
-                                else cb();
-                            })
-                            .on('error', cb);
                     }
 
                     // Find and handle zip files and extract
@@ -366,6 +355,11 @@ module.exports = {
                     });
                 },
 
+                // Verify max images limit
+                cb => {
+                    checkMaxImageLimits(cb);
+                },
+
                 cb => {
                     // Find any *.txt (GCP) file and move it to the data/<uuid>/gcp directory
                     // also remove any lingering zipurl.zip

libs/ziputils.js

@@ -0,0 +1,38 @@
+const processRunner = require('./processRunner');
+const nodeUnzip = require('node-unzip-2');
+const config = require('../config');
+const fs = require('fs');
+
+module.exports = {
+    unzip: function(file, outputDir, cb, noDirectories = false){
+        if (config.hasUnzip){
+            processRunner.unzip({
+                file: file,
+                destination: outputDir,
+                noDirectories
+            }, (err, code, _) => {
+                if (err) cb(err);
+                else{
+                    if (code === 0) cb();
+                    else cb(new Error(`Could not extract .zip file, unzip exited with code ${code}`));
+                }
+            });
+        }else if (config.has7z){
+            processRunner.sevenUnzip({
+                file: file,
+                destination: outputDir,
+                noDirectories
+            }, (err, code, _) => {
+                if (err) cb(err);
+                else{
+                    if (code === 0) cb();
+                    else cb(new Error(`Could not extract .zip file, 7z exited with code ${code}`));
+                }
+            });
+        }else{
+            fs.createReadStream(file).pipe(nodeUnzip.Extract({ path: outputDir }))
+                .on('close', cb)
+                .on('error', cb);
+        }
+    }
+}