kopia lustrzana https://github.com/OpenDroneMap/NodeODM
Faster unzipping with 7z
rodzic
ff21268744
commit
802d72a48b
|
@ -93,9 +93,19 @@ module.exports = {
|
|||
["projectFolderPath"]),
|
||||
|
||||
sevenZip: makeRunner("7z", function(options){
|
||||
return ["a", "-mx=0", "-r", "-bd", options.destination].concat(options.pathsToArchive);
|
||||
return ["a", "-mx=0", "-y", "-r", "-bd", options.destination].concat(options.pathsToArchive);
|
||||
},
|
||||
["destination", "pathsToArchive", "cwd"],
|
||||
null,
|
||||
false),
|
||||
|
||||
sevenUnzip: makeRunner("7z", function(options){
|
||||
let cmd = "x"; // eXtract files with full paths
|
||||
if (options.noDirectories) cmd = "e"; //Extract files from archive (without using directory names)
|
||||
|
||||
return [cmd, "-aoa", "-bd", "-y", `-o${options.destination}`, options.file];
|
||||
},
|
||||
["destination", "file"],
|
||||
null,
|
||||
false)
|
||||
};
|
||||
|
|
|
@ -24,13 +24,13 @@ const uuidv4 = require('uuid/v4');
|
|||
const config = require('../config.js');
|
||||
const rmdir = require('rimraf');
|
||||
const Directories = require('./Directories');
|
||||
const unzip = require('node-unzip-2');
|
||||
const mv = require('mv');
|
||||
const Task = require('./Task');
|
||||
const async = require('async');
|
||||
const odmInfo = require('./odmInfo');
|
||||
const request = require('request');
|
||||
const utils = require('./utils');
|
||||
const ziputils = require('./ziputils');
|
||||
const { cancelJob } = require('node-schedule');
|
||||
|
||||
const download = function(uri, filename, callback) {
|
||||
request.head(uri, function(err, res, body) {
|
||||
|
@ -243,6 +243,13 @@ module.exports = {
|
|||
let destImagesPath = path.join(destPath, "images");
|
||||
let destGcpPath = path.join(destPath, "gcp");
|
||||
|
||||
const checkMaxImageLimits = (cb) => {
|
||||
fs.readdir(destImagesPath, (err, files) => {
|
||||
if (config.maxImages && files.length > config.maxImages) cb(`${files.length} images uploaded, but this node can only process up to ${config.maxImages}.`);
|
||||
else cb(err);
|
||||
});
|
||||
};
|
||||
|
||||
async.series([
|
||||
cb => {
|
||||
odmInfo.filterOptions(req.body.options, (err, options) => {
|
||||
|
@ -309,17 +316,12 @@ module.exports = {
|
|||
|
||||
// Extract
|
||||
cb => {
|
||||
fs.createReadStream(seedFileDst).pipe(unzip.Extract({ path: destPath }))
|
||||
.on('close', cb)
|
||||
.on('error', cb);
|
||||
ziputils.unzip(seedFileDst, destPath, cb);
|
||||
},
|
||||
|
||||
// Verify max images limit
|
||||
cb => {
|
||||
fs.readdir(destImagesPath, (err, files) => {
|
||||
if (config.maxImages && files.length > config.maxImages) cb(`${files.length} images uploaded, but this node can only process up to ${config.maxImages}.`);
|
||||
else cb(err);
|
||||
});
|
||||
checkMaxImageLimits(cb);
|
||||
},
|
||||
|
||||
// Remove
|
||||
|
@ -333,22 +335,20 @@ module.exports = {
|
|||
}
|
||||
|
||||
const handleZipUrl = (cb) => {
|
||||
let filesCount = 0;
|
||||
fs.createReadStream(path.join(destImagesPath, "zipurl.zip")).pipe(unzip.Parse())
|
||||
.on('entry', function(entry) {
|
||||
if (entry.type === 'File') {
|
||||
filesCount++;
|
||||
entry.pipe(fs.createWriteStream(path.join(destImagesPath, path.basename(entry.path))));
|
||||
} else {
|
||||
entry.autodrain();
|
||||
}
|
||||
})
|
||||
.on('close', () => {
|
||||
// Verify max images limit
|
||||
if (config.maxImages && filesCount > config.maxImages) cb(`${filesCount} images uploaded, but this node can only process up to ${config.maxImages}.`);
|
||||
else cb();
|
||||
})
|
||||
.on('error', cb);
|
||||
async.series([
|
||||
// Extract images
|
||||
cb => {
|
||||
ziputils.unzip(path.join(destImagesPath, "zipurl.zip"),
|
||||
destImagesPath,
|
||||
cb,
|
||||
true);
|
||||
},
|
||||
|
||||
// Count files
|
||||
cb => {
|
||||
checkMaxImageLimits(cb);
|
||||
}
|
||||
], cb);
|
||||
}
|
||||
|
||||
// Find and handle zip files and extract
|
||||
|
|
|
@ -0,0 +1,26 @@
|
|||
const processRunner = require('./processRunner');
|
||||
const nodeUnzip = require('node-unzip-2');
|
||||
const config = require('../config');
|
||||
const fs = require('fs');
|
||||
|
||||
module.exports = {
|
||||
unzip: function(file, outputDir, cb, noDirectories = false){
|
||||
if (config.has7z){
|
||||
processRunner.sevenUnzip({
|
||||
file: file,
|
||||
destination: outputDir,
|
||||
noDirectories
|
||||
}, (err, code, _) => {
|
||||
if (err) cb(err);
|
||||
else{
|
||||
if (code === 0) cb();
|
||||
else cb(new Error(`Could not extract .zip file, 7z exited with code ${code}`));
|
||||
}
|
||||
});
|
||||
}else{
|
||||
fs.createReadStream(file).pipe(nodeUnzip.Extract({ path: outputDir }))
|
||||
.on('close', cb)
|
||||
.on('error', cb);
|
||||
}
|
||||
}
|
||||
}
|
Ładowanie…
Reference in New Issue