kopia lustrzana https://github.com/OpenDroneMap/NodeODM
Use 7zip for compression
rodzic
33fc1cf8ec
commit
f695c7b44e
|
@ -5,7 +5,7 @@ EXPOSE 3000
|
||||||
|
|
||||||
USER root
|
USER root
|
||||||
RUN curl --silent --location https://deb.nodesource.com/setup_10.x | bash -
|
RUN curl --silent --location https://deb.nodesource.com/setup_10.x | bash -
|
||||||
RUN apt-get install -y nodejs python-gdal && npm install -g nodemon && \
|
RUN apt-get install -y nodejs python-gdal p7zip-full && npm install -g nodemon && \
|
||||||
ln -s /code/SuperBuild/install/bin/entwine /usr/bin/entwine && \
|
ln -s /code/SuperBuild/install/bin/entwine /usr/bin/entwine && \
|
||||||
ln -s /code/SuperBuild/install/bin/pdal /usr/bin/pdal
|
ln -s /code/SuperBuild/install/bin/pdal /usr/bin/pdal
|
||||||
|
|
||||||
|
|
|
@ -54,11 +54,11 @@ If you are already running [ODM](https://github.com/OpenDroneMap/ODM) on Ubuntu
|
||||||
|
|
||||||
1) Install Entwine: https://entwine.io/quickstart.html#installation
|
1) Install Entwine: https://entwine.io/quickstart.html#installation
|
||||||
|
|
||||||
2) Install node.js and npm dependencies:
|
2) Install node.js, npm dependencies and 7zip:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
sudo curl --silent --location https://deb.nodesource.com/setup_6.x | sudo bash -
|
sudo curl --silent --location https://deb.nodesource.com/setup_6.x | sudo bash -
|
||||||
sudo apt-get install -y nodejs python-gdal
|
sudo apt-get install -y nodejs python-gdal p7zip-full
|
||||||
git clone https://github.com/OpenDroneMap/NodeODM
|
git clone https://github.com/OpenDroneMap/NodeODM
|
||||||
cd NodeODM
|
cd NodeODM
|
||||||
npm install
|
npm install
|
||||||
|
|
|
@ -20,6 +20,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
let fs = require('fs');
|
let fs = require('fs');
|
||||||
let argv = require('minimist')(process.argv.slice(2));
|
let argv = require('minimist')(process.argv.slice(2));
|
||||||
let utils = require('./libs/utils');
|
let utils = require('./libs/utils');
|
||||||
|
const spawnSync = require('child_process').spawnSync;
|
||||||
|
|
||||||
if (argv.help){
|
if (argv.help){
|
||||||
console.log(`
|
console.log(`
|
||||||
|
@ -115,4 +116,8 @@ config.s3UploadEverything = argv.s3_upload_everything || fromConfigFile("s3Uploa
|
||||||
config.maxConcurrency = parseInt(argv.max_concurrency || fromConfigFile("maxConcurrency", 0));
|
config.maxConcurrency = parseInt(argv.max_concurrency || fromConfigFile("maxConcurrency", 0));
|
||||||
config.maxRuntime = parseInt(argv.max_runtime || fromConfigFile("maxRuntime", -1));
|
config.maxRuntime = parseInt(argv.max_runtime || fromConfigFile("maxRuntime", -1));
|
||||||
|
|
||||||
|
// Detect 7z availability
|
||||||
|
const childProcess = spawnSync("7z", ['--help']);
|
||||||
|
config.has7z = childProcess.status === 0;
|
||||||
|
|
||||||
module.exports = config;
|
module.exports = config;
|
||||||
|
|
4
index.js
4
index.js
|
@ -863,6 +863,10 @@ if (config.test) {
|
||||||
if (config.testDropUploads) logger.info("Uploads will drop at random");
|
if (config.testDropUploads) logger.info("Uploads will drop at random");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!config.has7z){
|
||||||
|
logger.warn("The 7z program is not installed, falling back to legacy (zipping will be slower)");
|
||||||
|
}
|
||||||
|
|
||||||
let commands = [
|
let commands = [
|
||||||
cb => odmInfo.initialize(cb),
|
cb => odmInfo.initialize(cb),
|
||||||
cb => auth.initialize(cb),
|
cb => auth.initialize(cb),
|
||||||
|
|
45
libs/Task.js
45
libs/Task.js
|
@ -22,12 +22,11 @@ const async = require('async');
|
||||||
const assert = require('assert');
|
const assert = require('assert');
|
||||||
const logger = require('./logger');
|
const logger = require('./logger');
|
||||||
const fs = require('fs');
|
const fs = require('fs');
|
||||||
const glob = require("glob");
|
const si = require("systeminformation");
|
||||||
const path = require('path');
|
const path = require('path');
|
||||||
const rmdir = require('rimraf');
|
const rmdir = require('rimraf');
|
||||||
const odmRunner = require('./odmRunner');
|
const odmRunner = require('./odmRunner');
|
||||||
const processRunner = require('./processRunner');
|
const processRunner = require('./processRunner');
|
||||||
const archiver = require('archiver');
|
|
||||||
const Directories = require('./Directories');
|
const Directories = require('./Directories');
|
||||||
const kill = require('tree-kill');
|
const kill = require('tree-kill');
|
||||||
const S3 = require('./S3');
|
const S3 = require('./S3');
|
||||||
|
@ -249,6 +248,42 @@ module.exports = class Task{
|
||||||
|
|
||||||
const postProcess = () => {
|
const postProcess = () => {
|
||||||
const createZipArchive = (outputFilename, files) => {
|
const createZipArchive = (outputFilename, files) => {
|
||||||
|
return (done) => {
|
||||||
|
this.output.push(`Compressing ${outputFilename}\n`);
|
||||||
|
|
||||||
|
const zipFile = path.resolve(this.getAssetsArchivePath(outputFilename));
|
||||||
|
const sourcePath = !config.test ?
|
||||||
|
this.getProjectFolderPath() :
|
||||||
|
path.join("tests", "processing_results");
|
||||||
|
|
||||||
|
const pathsToArchive = [];
|
||||||
|
files.forEach(f => {
|
||||||
|
if (fs.existsSync(path.join(sourcePath, f))){
|
||||||
|
pathsToArchive.push(f);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
processRunner.sevenZip({
|
||||||
|
destination: zipFile,
|
||||||
|
pathsToArchive,
|
||||||
|
cwd: sourcePath
|
||||||
|
}, (err, code, _) => {
|
||||||
|
if (err){
|
||||||
|
logger.error(`Could not archive .zip file: ${err.message}`);
|
||||||
|
done(err);
|
||||||
|
}else{
|
||||||
|
if (code === 0){
|
||||||
|
this.updateProgress(97);
|
||||||
|
done();
|
||||||
|
}else done(new Error(`Could not archive .zip file, 7z exited with code ${code}`));
|
||||||
|
}
|
||||||
|
}, output => {
|
||||||
|
this.output.push(output);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const createZipArchiveLegacy = (outputFilename, files) => {
|
||||||
return (done) => {
|
return (done) => {
|
||||||
this.output.push(`Compressing ${outputFilename}\n`);
|
this.output.push(`Compressing ${outputFilename}\n`);
|
||||||
|
|
||||||
|
@ -327,7 +362,7 @@ module.exports = class Task{
|
||||||
this.runningProcesses.push(
|
this.runningProcesses.push(
|
||||||
processRunner.runPostProcessingScript({
|
processRunner.runPostProcessingScript({
|
||||||
projectFolderPath: this.getProjectFolderPath()
|
projectFolderPath: this.getProjectFolderPath()
|
||||||
}, (err, code, signal) => {
|
}, (err, code, _) => {
|
||||||
if (err) done(err);
|
if (err) done(err);
|
||||||
else{
|
else{
|
||||||
if (code === 0){
|
if (code === 0){
|
||||||
|
@ -388,7 +423,9 @@ module.exports = class Task{
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!this.skipPostProcessing) tasks.push(runPostProcessingScript());
|
if (!this.skipPostProcessing) tasks.push(runPostProcessingScript());
|
||||||
tasks.push(createZipArchive('all.zip', allPaths));
|
|
||||||
|
const archiveFunc = config.has7z ? createZipArchive : createZipArchiveLegacy;
|
||||||
|
tasks.push(archiveFunc('all.zip', allPaths));
|
||||||
|
|
||||||
// Upload to S3 all paths + all.zip file (if config says so)
|
// Upload to S3 all paths + all.zip file (if config says so)
|
||||||
if (S3.enabled()){
|
if (S3.enabled()){
|
||||||
|
|
|
@ -25,7 +25,7 @@ let logger = require('./logger');
|
||||||
let utils = require('./utils');
|
let utils = require('./utils');
|
||||||
|
|
||||||
|
|
||||||
function makeRunner(command, args, requiredOptions = [], outputTestFile = null){
|
function makeRunner(command, args, requiredOptions = [], outputTestFile = null, skipOnTest = true){
|
||||||
return function(options, done, outputReceived){
|
return function(options, done, outputReceived){
|
||||||
for (let requiredOption of requiredOptions){
|
for (let requiredOption of requiredOptions){
|
||||||
assert(options[requiredOption] !== undefined, `${requiredOption} must be defined`);
|
assert(options[requiredOption] !== undefined, `${requiredOption} must be defined`);
|
||||||
|
@ -36,7 +36,7 @@ function makeRunner(command, args, requiredOptions = [], outputTestFile = null){
|
||||||
|
|
||||||
logger.info(`About to run: ${command} ${commandArgs.join(" ")}`);
|
logger.info(`About to run: ${command} ${commandArgs.join(" ")}`);
|
||||||
|
|
||||||
if (config.test){
|
if (config.test && skipOnTest){
|
||||||
logger.info("Test mode is on, command will not execute");
|
logger.info("Test mode is on, command will not execute");
|
||||||
|
|
||||||
if (outputTestFile){
|
if (outputTestFile){
|
||||||
|
@ -61,7 +61,11 @@ function makeRunner(command, args, requiredOptions = [], outputTestFile = null){
|
||||||
// Launch
|
// Launch
|
||||||
const env = utils.clone(process.env);
|
const env = utils.clone(process.env);
|
||||||
env.LD_LIBRARY_PATH = path.join(config.odm_path, "SuperBuild", "install", "lib");
|
env.LD_LIBRARY_PATH = path.join(config.odm_path, "SuperBuild", "install", "lib");
|
||||||
let childProcess = spawn(command, commandArgs, { env });
|
|
||||||
|
let cwd = undefined;
|
||||||
|
if (options.cwd) cwd = options.cwd;
|
||||||
|
|
||||||
|
let childProcess = spawn(command, commandArgs, { env, cwd });
|
||||||
|
|
||||||
childProcess
|
childProcess
|
||||||
.on('exit', (code, signal) => done(null, code, signal))
|
.on('exit', (code, signal) => done(null, code, signal))
|
||||||
|
@ -79,5 +83,12 @@ module.exports = {
|
||||||
function(options){
|
function(options){
|
||||||
return [options.projectFolderPath];
|
return [options.projectFolderPath];
|
||||||
},
|
},
|
||||||
["projectFolderPath"])
|
["projectFolderPath"]),
|
||||||
|
|
||||||
|
sevenZip: makeRunner("7z", function(options){
|
||||||
|
return ["a", "-r", "-bd", options.destination].concat(options.pathsToArchive);
|
||||||
|
},
|
||||||
|
["destination", "pathsToArchive", "cwd"],
|
||||||
|
null,
|
||||||
|
false)
|
||||||
};
|
};
|
||||||
|
|
Ładowanie…
Reference in New Issue