From f695c7b44e11268b35838dd260318edf036fec49 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sun, 19 Jan 2020 15:21:54 -0500 Subject: [PATCH] Use 7zip for compression --- Dockerfile | 2 +- README.md | 4 ++-- config.js | 5 +++++ index.js | 4 ++++ libs/Task.js | 45 +++++++++++++++++++++++++++++++++++++++---- libs/processRunner.js | 19 ++++++++++++++---- 6 files changed, 68 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 433504f..e976ebd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ EXPOSE 3000 USER root RUN curl --silent --location https://deb.nodesource.com/setup_10.x | bash - -RUN apt-get install -y nodejs python-gdal && npm install -g nodemon && \ +RUN apt-get install -y nodejs python-gdal p7zip-full && npm install -g nodemon && \ ln -s /code/SuperBuild/install/bin/entwine /usr/bin/entwine && \ ln -s /code/SuperBuild/install/bin/pdal /usr/bin/pdal diff --git a/README.md b/README.md index a38507d..8702d64 100644 --- a/README.md +++ b/README.md @@ -54,11 +54,11 @@ If you are already running [ODM](https://github.com/OpenDroneMap/ODM) on Ubuntu 1) Install Entwine: https://entwine.io/quickstart.html#installation -2) Install node.js and npm dependencies: +2) Install node.js, npm dependencies and 7zip: ```bash sudo curl --silent --location https://deb.nodesource.com/setup_6.x | sudo bash - -sudo apt-get install -y nodejs python-gdal +sudo apt-get install -y nodejs python-gdal p7zip-full git clone https://github.com/OpenDroneMap/NodeODM cd NodeODM npm install diff --git a/config.js b/config.js index bd88ced..cb38e43 100644 --- a/config.js +++ b/config.js @@ -20,6 +20,7 @@ along with this program. If not, see . let fs = require('fs'); let argv = require('minimist')(process.argv.slice(2)); let utils = require('./libs/utils'); +const spawnSync = require('child_process').spawnSync; if (argv.help){ console.log(` @@ -115,4 +116,8 @@ config.s3UploadEverything = argv.s3_upload_everything || fromConfigFile("s3Uploa config.maxConcurrency = parseInt(argv.max_concurrency || fromConfigFile("maxConcurrency", 0)); config.maxRuntime = parseInt(argv.max_runtime || fromConfigFile("maxRuntime", -1)); +// Detect 7z availability +const childProcess = spawnSync("7z", ['--help']); +config.has7z = childProcess.status === 0; + module.exports = config; diff --git a/index.js b/index.js index 0094c06..6da022d 100644 --- a/index.js +++ b/index.js @@ -863,6 +863,10 @@ if (config.test) { if (config.testDropUploads) logger.info("Uploads will drop at random"); } +if (!config.has7z){ + logger.warn("The 7z program is not installed, falling back to legacy (zipping will be slower)"); +} + let commands = [ cb => odmInfo.initialize(cb), cb => auth.initialize(cb), diff --git a/libs/Task.js b/libs/Task.js index eb78fd7..a4c17ed 100644 --- a/libs/Task.js +++ b/libs/Task.js @@ -22,12 +22,11 @@ const async = require('async'); const assert = require('assert'); const logger = require('./logger'); const fs = require('fs'); -const glob = require("glob"); +const si = require("systeminformation"); const path = require('path'); const rmdir = require('rimraf'); const odmRunner = require('./odmRunner'); const processRunner = require('./processRunner'); -const archiver = require('archiver'); const Directories = require('./Directories'); const kill = require('tree-kill'); const S3 = require('./S3'); @@ -249,6 +248,42 @@ module.exports = class Task{ const postProcess = () => { const createZipArchive = (outputFilename, files) => { + return (done) => { + this.output.push(`Compressing ${outputFilename}\n`); + + const zipFile = path.resolve(this.getAssetsArchivePath(outputFilename)); + const sourcePath = !config.test ? + this.getProjectFolderPath() : + path.join("tests", "processing_results"); + + const pathsToArchive = []; + files.forEach(f => { + if (fs.existsSync(path.join(sourcePath, f))){ + pathsToArchive.push(f); + } + }); + + processRunner.sevenZip({ + destination: zipFile, + pathsToArchive, + cwd: sourcePath + }, (err, code, _) => { + if (err){ + logger.error(`Could not archive .zip file: ${err.message}`); + done(err); + }else{ + if (code === 0){ + this.updateProgress(97); + done(); + }else done(new Error(`Could not archive .zip file, 7z exited with code ${code}`)); + } + }, output => { + this.output.push(output); + }); + }; + }; + + const createZipArchiveLegacy = (outputFilename, files) => { return (done) => { this.output.push(`Compressing ${outputFilename}\n`); @@ -327,7 +362,7 @@ module.exports = class Task{ this.runningProcesses.push( processRunner.runPostProcessingScript({ projectFolderPath: this.getProjectFolderPath() - }, (err, code, signal) => { + }, (err, code, _) => { if (err) done(err); else{ if (code === 0){ @@ -388,7 +423,9 @@ module.exports = class Task{ } if (!this.skipPostProcessing) tasks.push(runPostProcessingScript()); - tasks.push(createZipArchive('all.zip', allPaths)); + + const archiveFunc = config.has7z ? createZipArchive : createZipArchiveLegacy; + tasks.push(archiveFunc('all.zip', allPaths)); // Upload to S3 all paths + all.zip file (if config says so) if (S3.enabled()){ diff --git a/libs/processRunner.js b/libs/processRunner.js index 8e50c1a..5739722 100644 --- a/libs/processRunner.js +++ b/libs/processRunner.js @@ -25,7 +25,7 @@ let logger = require('./logger'); let utils = require('./utils'); -function makeRunner(command, args, requiredOptions = [], outputTestFile = null){ +function makeRunner(command, args, requiredOptions = [], outputTestFile = null, skipOnTest = true){ return function(options, done, outputReceived){ for (let requiredOption of requiredOptions){ assert(options[requiredOption] !== undefined, `${requiredOption} must be defined`); @@ -36,7 +36,7 @@ function makeRunner(command, args, requiredOptions = [], outputTestFile = null){ logger.info(`About to run: ${command} ${commandArgs.join(" ")}`); - if (config.test){ + if (config.test && skipOnTest){ logger.info("Test mode is on, command will not execute"); if (outputTestFile){ @@ -61,7 +61,11 @@ function makeRunner(command, args, requiredOptions = [], outputTestFile = null){ // Launch const env = utils.clone(process.env); env.LD_LIBRARY_PATH = path.join(config.odm_path, "SuperBuild", "install", "lib"); - let childProcess = spawn(command, commandArgs, { env }); + + let cwd = undefined; + if (options.cwd) cwd = options.cwd; + + let childProcess = spawn(command, commandArgs, { env, cwd }); childProcess .on('exit', (code, signal) => done(null, code, signal)) @@ -79,5 +83,12 @@ module.exports = { function(options){ return [options.projectFolderPath]; }, - ["projectFolderPath"]) + ["projectFolderPath"]), + + sevenZip: makeRunner("7z", function(options){ + return ["a", "-r", "-bd", options.destination].concat(options.pathsToArchive); + }, + ["destination", "pathsToArchive", "cwd"], + null, + false) };