Use 7zip for compression

Dockerfile

@@ -5,7 +5,7 @@ EXPOSE 3000
 
 USER root
 RUN curl --silent --location https://deb.nodesource.com/setup_10.x | bash -
-RUN apt-get install -y nodejs python-gdal && npm install -g nodemon && \
+RUN apt-get install -y nodejs python-gdal p7zip-full && npm install -g nodemon && \
     ln -s /code/SuperBuild/install/bin/entwine /usr/bin/entwine && \
     ln -s /code/SuperBuild/install/bin/pdal /usr/bin/pdal
 

README.md

@@ -54,11 +54,11 @@ If you are already running [ODM](https://github.com/OpenDroneMap/ODM) on Ubuntu
 
 1) Install Entwine: https://entwine.io/quickstart.html#installation
  
-2) Install node.js and npm dependencies:
+2) Install node.js, npm dependencies and 7zip:
 
 ```bash
 sudo curl --silent --location https://deb.nodesource.com/setup_6.x | sudo bash -
-sudo apt-get install -y nodejs python-gdal
+sudo apt-get install -y nodejs python-gdal p7zip-full
 git clone https://github.com/OpenDroneMap/NodeODM
 cd NodeODM
 npm install

config.js

@@ -20,6 +20,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 let fs = require('fs');
 let argv = require('minimist')(process.argv.slice(2));
 let utils = require('./libs/utils');
+const spawnSync = require('child_process').spawnSync;
 
 if (argv.help){
 	console.log(`
@@ -115,4 +116,8 @@ config.s3UploadEverything = argv.s3_upload_everything || fromConfigFile("s3Uploa
 config.maxConcurrency = parseInt(argv.max_concurrency || fromConfigFile("maxConcurrency", 0));
 config.maxRuntime = parseInt(argv.max_runtime || fromConfigFile("maxRuntime", -1));
 
+// Detect 7z availability
+const childProcess = spawnSync("7z", ['--help']);
+config.has7z = childProcess.status === 0;
+
 module.exports = config;

index.js

@@ -863,6 +863,10 @@ if (config.test) {
     if (config.testDropUploads) logger.info("Uploads will drop at random");
 }
 
+if (!config.has7z){
+    logger.warn("The 7z program is not installed, falling back to legacy (zipping will be slower)");
+}
+
 let commands = [
     cb => odmInfo.initialize(cb),
     cb => auth.initialize(cb),

libs/Task.js

@@ -22,12 +22,11 @@ const async = require('async');
 const assert = require('assert');
 const logger = require('./logger');
 const fs = require('fs');
-const glob = require("glob");
+const si = require("systeminformation");
 const path = require('path');
 const rmdir = require('rimraf');
 const odmRunner = require('./odmRunner');
 const processRunner = require('./processRunner');
-const archiver = require('archiver');
 const Directories = require('./Directories');
 const kill = require('tree-kill');
 const S3 = require('./S3');
@@ -249,6 +248,42 @@ module.exports = class Task{
         
         const postProcess = () => {
             const createZipArchive = (outputFilename, files) => {
+                return (done) => {
+                    this.output.push(`Compressing ${outputFilename}\n`);
+
+                    const zipFile = path.resolve(this.getAssetsArchivePath(outputFilename));
+                    const sourcePath = !config.test ? 
+                                        this.getProjectFolderPath() : 
+                                        path.join("tests", "processing_results");
+
+                    const pathsToArchive = [];
+                    files.forEach(f => {
+                        if (fs.existsSync(path.join(sourcePath, f))){
+                            pathsToArchive.push(f);
+                        }
+                    });
+
+                    processRunner.sevenZip({
+                        destination: zipFile,
+                        pathsToArchive,
+                        cwd: sourcePath
+                    }, (err, code, _) => {
+                        if (err){
+                            logger.error(`Could not archive .zip file: ${err.message}`);
+                            done(err);
+                        }else{
+                            if (code === 0){
+                                this.updateProgress(97);
+                                done();
+                            }else done(new Error(`Could not archive .zip file, 7z exited with code ${code}`));
+                        }
+                    }, output => {
+                        this.output.push(output);
+                    });
+                };
+            };
+
+            const createZipArchiveLegacy = (outputFilename, files) => {
                 return (done) => {
                     this.output.push(`Compressing ${outputFilename}\n`);
 
@@ -327,7 +362,7 @@ module.exports = class Task{
                     this.runningProcesses.push(
                         processRunner.runPostProcessingScript({
                             projectFolderPath: this.getProjectFolderPath() 
-                        }, (err, code, signal) => {
+                        }, (err, code, _) => {
                             if (err) done(err);
                             else{
                                 if (code === 0){
@@ -388,7 +423,9 @@ module.exports = class Task{
             }
             
             if (!this.skipPostProcessing) tasks.push(runPostProcessingScript());
-            tasks.push(createZipArchive('all.zip', allPaths));
+
+            const archiveFunc = config.has7z ? createZipArchive : createZipArchiveLegacy;
+            tasks.push(archiveFunc('all.zip', allPaths));
             
             // Upload to S3 all paths + all.zip file (if config says so)
             if (S3.enabled()){

libs/processRunner.js

@@ -25,7 +25,7 @@ let logger = require('./logger');
 let utils = require('./utils');
 
 
-function makeRunner(command, args, requiredOptions = [], outputTestFile = null){
+function makeRunner(command, args, requiredOptions = [], outputTestFile = null, skipOnTest = true){
     return function(options, done, outputReceived){
         for (let requiredOption of requiredOptions){
             assert(options[requiredOption] !== undefined, `${requiredOption} must be defined`);
@@ -36,7 +36,7 @@ function makeRunner(command, args, requiredOptions = [], outputTestFile = null){
 
         logger.info(`About to run: ${command} ${commandArgs.join(" ")}`);
 
-        if (config.test){
+        if (config.test && skipOnTest){
             logger.info("Test mode is on, command will not execute");
 
             if (outputTestFile){
@@ -61,7 +61,11 @@ function makeRunner(command, args, requiredOptions = [], outputTestFile = null){
         // Launch
         const env = utils.clone(process.env);
         env.LD_LIBRARY_PATH = path.join(config.odm_path, "SuperBuild", "install", "lib");
-        let childProcess = spawn(command, commandArgs, { env });
+        
+        let cwd = undefined;
+        if (options.cwd) cwd = options.cwd;
+
+        let childProcess = spawn(command, commandArgs, { env, cwd });
 
         childProcess
             .on('exit', (code, signal) => done(null, code, signal))
@@ -79,5 +83,12 @@ module.exports = {
                      function(options){
                          return [options.projectFolderPath];
                      },
-                     ["projectFolderPath"])
+                     ["projectFolderPath"]),
+
+    sevenZip: makeRunner("7z", function(options){
+            return ["a", "-r", "-bd", options.destination].concat(options.pathsToArchive);
+        },
+        ["destination", "pathsToArchive", "cwd"],
+        null,
+        false)
 };