kopia lustrzana https://github.com/OpenDroneMap/NodeODM
Merged 7unzip branch
Piero Toffanin
commit
be98fe4d92
|
|
@ -5,7 +5,7 @@ EXPOSE 3000
|
|||
|
||||
USER root
|
||||
RUN curl --silent --location https://deb.nodesource.com/setup_10.x | bash -
|
||||
RUN apt-get install -y nodejs p7zip-full && npm install -g nodemon && \
|
||||
RUN apt-get install -y nodejs unzip p7zip-full && npm install -g nodemon && \
|
||||
ln -s /code/SuperBuild/install/bin/entwine /usr/bin/entwine && \
|
||||
ln -s /code/SuperBuild/install/bin/pdal /usr/bin/pdal
|
||||
|
||||
|
|
|
|||
|
|
@ -56,11 +56,11 @@ If you are already running [ODM](https://github.com/OpenDroneMap/ODM) on Ubuntu
|
|||
|
||||
1) Install Entwine: https://entwine.io/quickstart.html#installation
|
||||
|
||||
2) Install node.js, npm dependencies and 7zip:
|
||||
2) Install node.js, npm dependencies, 7zip and unzip:
|
||||
|
||||
```bash
|
||||
sudo curl --silent --location https://deb.nodesource.com/setup_6.x | sudo bash -
|
||||
sudo apt-get install -y nodejs python-gdal p7zip-full
|
||||
sudo apt-get install -y nodejs python-gdal p7zip-full unzip
|
||||
git clone https://github.com/OpenDroneMap/NodeODM
|
||||
cd NodeODM
|
||||
npm install
|
||||
|
|
|
|||
|
|
@ -119,7 +119,8 @@ config.maxConcurrency = parseInt(argv.max_concurrency || fromConfigFile("maxConc
|
|||
config.maxRuntime = parseInt(argv.max_runtime || fromConfigFile("maxRuntime", -1));
|
||||
|
||||
// Detect 7z availability
|
||||
const childProcess = spawnSync("7z", ['--help']);
|
||||
config.has7z = childProcess.status === 0;
|
||||
config.has7z = spawnSync("7z", ['--help']).status === 0;
|
||||
config.hasUnzip = spawnSync("unzip", ['--help']).status === 0;
|
||||
|
||||
|
||||
module.exports = config;
|
||||
|
|
|
|||
6
index.js
6
index.js
|
|
@ -900,9 +900,9 @@ if (config.test) {
|
|||
if (config.testDropUploads) logger.info("Uploads will drop at random");
|
||||
}
|
||||
|
||||
if (!config.has7z){
|
||||
logger.warn("The 7z program is not installed, falling back to legacy (zipping will be slower)");
|
||||
}
|
||||
if (!config.hasUnzip) logger.warn("The unzip program is not installed, (certain unzip operations might be slower)");
|
||||
if (!config.has7z) logger.warn("The 7z program is not installed, falling back to legacy (zipping will be slower)");
|
||||
|
||||
|
||||
let commands = [
|
||||
cb => odmInfo.initialize(cb),
|
||||
|
|
|
|||
|
|
@ -93,9 +93,27 @@ module.exports = {
|
|||
["projectFolderPath"]),
|
||||
|
||||
sevenZip: makeRunner("7z", function(options){
|
||||
return ["a", "-mx=0", "-r", "-bd", options.destination].concat(options.pathsToArchive);
|
||||
return ["a", "-mx=0", "-y", "-r", "-bd", options.destination].concat(options.pathsToArchive);
|
||||
},
|
||||
["destination", "pathsToArchive", "cwd"],
|
||||
null,
|
||||
false),
|
||||
|
||||
sevenUnzip: makeRunner("7z", function(options){
|
||||
let cmd = "x"; // eXtract files with full paths
|
||||
if (options.noDirectories) cmd = "e"; //Extract files from archive (without using directory names)
|
||||
|
||||
return [cmd, "-aoa", "-bd", "-y", `-o${options.destination}`, options.file];
|
||||
},
|
||||
["destination", "file"],
|
||||
null,
|
||||
false),
|
||||
|
||||
unzip: makeRunner("unzip", function(options){
|
||||
const opts = options.noDirectories ? ["-j"] : [];
|
||||
return opts.concat(["-qq", "-o", options.file, "-d", options.destination]);
|
||||
},
|
||||
["destination", "file"],
|
||||
null,
|
||||
false)
|
||||
};
|
||||
|
|
|
|||
|
|
@ -24,13 +24,13 @@ const uuidv4 = require('uuid/v4');
|
|||
const config = require('../config.js');
|
||||
const rmdir = require('rimraf');
|
||||
const Directories = require('./Directories');
|
||||
const unzip = require('node-unzip-2');
|
||||
const mv = require('mv');
|
||||
const Task = require('./Task');
|
||||
const async = require('async');
|
||||
const odmInfo = require('./odmInfo');
|
||||
const request = require('request');
|
||||
const utils = require('./utils');
|
||||
const ziputils = require('./ziputils');
|
||||
const { cancelJob } = require('node-schedule');
|
||||
|
||||
const download = function(uri, filename, callback) {
|
||||
request.head(uri, function(err, res, body) {
|
||||
|
|
@ -163,7 +163,7 @@ module.exports = {
|
|||
else cb(null, body);
|
||||
});
|
||||
}catch(e){
|
||||
cb("Malformed body.json");
|
||||
cb(new Error("Malformed body.json"));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
|
@ -243,6 +243,17 @@ module.exports = {
|
|||
let destImagesPath = path.join(destPath, "images");
|
||||
let destGcpPath = path.join(destPath, "gcp");
|
||||
|
||||
const checkMaxImageLimits = (cb) => {
|
||||
if (!config.maxImages) cb();
|
||||
else{
|
||||
fs.readdir(destImagesPath, (err, files) => {
|
||||
if (err) cb(err);
|
||||
else if (files.length > config.maxImages) cb(new Error(`${files.length} images uploaded, but this node can only process up to ${config.maxImages}.`));
|
||||
else cb();
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
async.series([
|
||||
cb => {
|
||||
odmInfo.filterOptions(req.body.options, (err, options) => {
|
||||
|
|
@ -309,17 +320,7 @@ module.exports = {
|
|||
|
||||
// Extract
|
||||
cb => {
|
||||
fs.createReadStream(seedFileDst).pipe(unzip.Extract({ path: destPath }))
|
||||
.on('close', cb)
|
||||
.on('error', cb);
|
||||
},
|
||||
|
||||
// Verify max images limit
|
||||
cb => {
|
||||
fs.readdir(destImagesPath, (err, files) => {
|
||||
if (config.maxImages && files.length > config.maxImages) cb(`${files.length} images uploaded, but this node can only process up to ${config.maxImages}.`);
|
||||
else cb(err);
|
||||
});
|
||||
ziputils.unzip(seedFileDst, destPath, cb);
|
||||
},
|
||||
|
||||
// Remove
|
||||
|
|
@ -333,22 +334,10 @@ module.exports = {
|
|||
}
|
||||
|
||||
const handleZipUrl = (cb) => {
|
||||
let filesCount = 0;
|
||||
fs.createReadStream(path.join(destImagesPath, "zipurl.zip")).pipe(unzip.Parse())
|
||||
.on('entry', function(entry) {
|
||||
if (entry.type === 'File') {
|
||||
filesCount++;
|
||||
entry.pipe(fs.createWriteStream(path.join(destImagesPath, path.basename(entry.path))));
|
||||
} else {
|
||||
entry.autodrain();
|
||||
}
|
||||
})
|
||||
.on('close', () => {
|
||||
// Verify max images limit
|
||||
if (config.maxImages && filesCount > config.maxImages) cb(`${filesCount} images uploaded, but this node can only process up to ${config.maxImages}.`);
|
||||
else cb();
|
||||
})
|
||||
.on('error', cb);
|
||||
// Extract images
|
||||
ziputils.unzip(path.join(destImagesPath, "zipurl.zip"),
|
||||
destImagesPath,
|
||||
cb, true);
|
||||
}
|
||||
|
||||
// Find and handle zip files and extract
|
||||
|
|
@ -366,6 +355,11 @@ module.exports = {
|
|||
});
|
||||
},
|
||||
|
||||
// Verify max images limit
|
||||
cb => {
|
||||
checkMaxImageLimits(cb);
|
||||
},
|
||||
|
||||
cb => {
|
||||
// Find any *.txt (GCP) file and move it to the data/<uuid>/gcp directory
|
||||
// also remove any lingering zipurl.zip
|
||||
|
|
|
|||
|
|
@ -0,0 +1,38 @@
|
|||
const processRunner = require('./processRunner');
|
||||
const nodeUnzip = require('node-unzip-2');
|
||||
const config = require('../config');
|
||||
const fs = require('fs');
|
||||
|
||||
module.exports = {
|
||||
unzip: function(file, outputDir, cb, noDirectories = false){
|
||||
if (config.hasUnzip){
|
||||
processRunner.unzip({
|
||||
file: file,
|
||||
destination: outputDir,
|
||||
noDirectories
|
||||
}, (err, code, _) => {
|
||||
if (err) cb(err);
|
||||
else{
|
||||
if (code === 0) cb();
|
||||
else cb(new Error(`Could not extract .zip file, unzip exited with code ${code}`));
|
||||
}
|
||||
});
|
||||
}else if (config.has7z){
|
||||
processRunner.sevenUnzip({
|
||||
file: file,
|
||||
destination: outputDir,
|
||||
noDirectories
|
||||
}, (err, code, _) => {
|
||||
if (err) cb(err);
|
||||
else{
|
||||
if (code === 0) cb();
|
||||
else cb(new Error(`Could not extract .zip file, 7z exited with code ${code}`));
|
||||
}
|
||||
});
|
||||
}else{
|
||||
fs.createReadStream(file).pipe(nodeUnzip.Extract({ path: outputDir }))
|
||||
.on('close', cb)
|
||||
.on('error', cb);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ładowanie…
Reference in New Issue