mirror
/
OpenDroneMap-NodeODM
kopia lustrzana https://github.com/OpenDroneMap/NodeODM
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge pull request #79 from pierotofy/splitmerge 

Split merge
pull/82/head
Piero Toffanin 2019-05-09 11:58:26 -07:00 zatwierdzone przez GitHub
rodzic a6eb15c197 cbd629dfcb
commit 72b4434095
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
10 zmienionych plików z 139 dodań i 50 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

10
docs/index.adoc
Wyświetl plik

@ -8,7 +8,7 @@ REST API to access ODM
=== Version information
[%hardbreaks]
_Version_ : 1.4.0
_Version_ : 1.5.0
=== Contact information
@ -294,11 +294,13 @@ _optional_|An optional UUID string that will be used as UUID for this task inste
|*Query*|*token* +
_optional_|Token required for authentication (when authentication is required).|string|
|*FormData*|*images* +
_optional_|Images to process, plus an optional GCP file. If included, the GCP file should have .txt extension|file|
_optional_|Images to process, plus an optional GCP file (*.txt) and/or an optional seed file (seed.zip). If included, the GCP file should have .txt extension. If included, the seed archive pre-polulates the task directory with its contents.|file|
|*FormData*|*name* +
_optional_|An optional name to be associated with the task|string|
|*FormData*|*options* +
_optional_|Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, …]. For example, [{"name":"cmvs-maxImages","value":"500"},{"name":"time","value":true}]. For a list of all options, call /options|string|
|*FormData*|*outputs* +
_optional_|An optional serialized JSON string of paths relative to the project directory that should be included in the all.zip result file, overriding the default behavior.|string|
|*FormData*|*skipPostProcessing* +
_optional_|When set, skips generation of map tiles, derivate assets, point cloud tiles.|boolean|
|*FormData*|*webhook* +
@ -402,6 +404,8 @@ _optional_|Token required for authentication (when authentication is required).|
_optional_|An optional name to be associated with the task|string|
|*FormData*|*options* +
_optional_|Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, …]. For example, [{"name":"cmvs-maxImages","value":"500"},{"name":"time","value":true}]. For a list of all options, call /options|string|
|*FormData*|*outputs* +
_optional_|An optional serialized JSON string of paths relative to the project directory that should be included in the all.zip result file, overriding the default behavior.|string|
|*FormData*|*skipPostProcessing* +
_optional_|When set, skips generation of map tiles, derivate assets, point cloud tiles.|boolean|
|*FormData*|*webhook* +
@ -451,7 +455,7 @@ _required_|UUID of the task|string|
|*Query*|*token* +
_optional_|Token required for authentication (when authentication is required).|string|
|*FormData*|*images* +
_required_|Images to process, plus an optional GCP file. If included, the GCP file should have .txt extension|file|
_required_|Images to process, plus an optional GCP file (*.txt) and/or an optional seed file (seed.zip). If included, the GCP file should have .txt extension. If included, the seed archive pre-polulates the task directory with its contents.|file|
|===

2
docs/swagger.json
Wyświetl plik

File diff suppressed because one or more lines are too long

16
index.js
Wyświetl plik

@ -80,6 +80,12 @@ let server;
* required: false
* type: string
* -
* name: outputs
* in: formData
* description: 'An optional serialized JSON string of paths relative to the project directory that should be included in the all.zip result file, overriding the default behavior.'
* required: false
* type: string
* -
* name: token
* in: query
* description: 'Token required for authentication (when authentication is required).'
@ -125,7 +131,7 @@ app.post('/task/new/init', authCheck, taskNew.assignUUID, formDataParser, taskNe
* -
* name: images
* in: formData
* description: Images to process, plus an optional GCP file. If included, the GCP file should have .txt extension
* description: Images to process, plus an optional GCP file (*.txt) and/or an optional seed file (seed.zip). If included, the GCP file should have .txt extension. If included, the seed archive pre-polulates the task directory with its contents.
* required: true
* type: file
* -
@ -192,7 +198,7 @@ app.post('/task/new/commit/:uuid', authCheck, taskNew.getUUID, taskNew.handleCom
* -
* name: images
* in: formData
* description: Images to process, plus an optional GCP file. If included, the GCP file should have .txt extension
* description: Images to process, plus an optional GCP file (*.txt) and/or an optional seed file (seed.zip). If included, the GCP file should have .txt extension. If included, the seed archive pre-polulates the task directory with its contents.
* required: false
* type: file
* -
@ -226,6 +232,12 @@ app.post('/task/new/commit/:uuid', authCheck, taskNew.getUUID, taskNew.handleCom
* required: false
* type: string
* -
* name: outputs
* in: formData
* description: 'An optional serialized JSON string of paths relative to the project directory that should be included in the all.zip result file, overriding the default behavior.'
* required: false
* type: string
* -
* name: token
* in: query
* description: 'Token required for authentication (when authentication is required).'

47
libs/Task.js
Wyświetl plik

@ -32,11 +32,12 @@ const Directories = require('./Directories');
const kill = require('tree-kill');
const S3 = require('./S3');
const request = require('request');
const utils = require('./utils');
const statusCodes = require('./statusCodes');
module.exports = class Task{
constructor(uuid, name, done, options = [], webhook = null, skipPostProcessing = false){
constructor(uuid, name, options = [], webhook = null, skipPostProcessing = false, outputs = [], done = () => {}){
assert(uuid !== undefined, "uuid must be set");
assert(done !== undefined, "ready must be set");
@ -51,6 +52,7 @@ module.exports = class Task{
this.runningProcesses = [];
this.webhook = webhook;
this.skipPostProcessing = skipPostProcessing;
this.outputs = utils.parseUnsafePathsList(outputs);
async.series([
// Read images info
@ -86,22 +88,28 @@ module.exports = class Task{
}
static CreateFromSerialized(taskJson, done){
new Task(taskJson.uuid, taskJson.name, (err, task) => {
if (err) done(err);
else{
// Override default values with those
// provided in the taskJson
for (let k in taskJson){
task[k] = taskJson[k];
new Task(taskJson.uuid,
taskJson.name,
taskJson.options,
taskJson.webhook,
taskJson.skipPostProcessing,
taskJson.outputs,
(err, task) => {
if (err) done(err);
else{
// Override default values with those
// provided in the taskJson
for (let k in taskJson){
task[k] = taskJson[k];
}
// Tasks that were running should be put back to QUEUED state
if (task.status.code === statusCodes.RUNNING){
task.status.code = statusCodes.QUEUED;
}
done(null, task);
}
// Tasks that were running should be put back to QUEUED state
if (task.status.code === statusCodes.RUNNING){
task.status.code = statusCodes.QUEUED;
}
done(null, task);
}
}, taskJson.options, taskJson.webhook, taskJson.skipPostProcessing);
});
}
// Get path where images are stored for this task
@ -317,6 +325,10 @@ module.exports = class Task{
'odm_georeferencing', 'odm_texturing',
'odm_dem/dsm.tif', 'odm_dem/dtm.tif', 'dsm_tiles', 'dtm_tiles',
'orthophoto_tiles', 'potree_pointcloud', 'images.json'];
// Did the user request different outputs than the default?
if (this.outputs.length > 0) allPaths = this.outputs;
let tasks = [];
if (config.test){
@ -528,7 +540,8 @@ module.exports = class Task{
status: this.status,
options: this.options,
webhook: this.webhook,
skipPostProcessing: !!this.skipPostProcessing
skipPostProcessing: !!this.skipPostProcessing,
outputs: this.outputs || []
};
}
};

4
libs/odmInfo.js
Wyświetl plik

@ -57,8 +57,8 @@ module.exports = {
// (num cores can be set programmatically, so can gcpFile, etc.)
if (["-h", "--project-path", "--cmvs-maxImages", "--time",
"--zip-results", "--pmvs-num-cores",
"--start-with", "--gcp", "--end-with", "--images",
"--rerun-all", "--rerun",
"--start-with", "--gcp", "--images",
"--rerun-all", "--rerun", "--end-with",
"--slam-config", "--video", "--version", "name"].indexOf(option) !== -1) continue;
let values = json[option];

79
libs/taskNew.js
Wyświetl plik

@ -269,30 +269,61 @@ module.exports = {
cb => fs.mkdir(destPath, undefined, cb),
cb => fs.mkdir(destGcpPath, undefined, cb),
cb => mv(srcPath, destImagesPath, cb),
// Zip files handling
cb => {
// Find any *.zip file and extract
const handleSeed = (cb) => {
const seedFileDst = path.join(destPath, "seed.zip");
async.series([
// Move to project root
cb => mv(path.join(destImagesPath, "seed.zip"), seedFileDst, cb),
// Extract
cb => {
fs.createReadStream(seedFileDst).pipe(unzip.Extract({ path: destPath }))
.on('close', cb)
.on('error', cb);
},
// Verify max images limit
cb => {
fs.readdir(destImagesPath, (err, files) => {
if (config.maxImages && files.length > config.maxImages) cb(`${files.length} images uploaded, but this node can only process up to ${config.maxImages}.`);
else cb(err);
});
}
], cb);
}
const handleZipUrl = (cb) => {
let filesCount = 0;
fs.createReadStream(path.join(destImagesPath, "zipurl.zip")).pipe(unzip.Parse())
.on('entry', function(entry) {
if (entry.type === 'File') {
filesCount++;
entry.pipe(fs.createWriteStream(path.join(destImagesPath, path.basename(entry.path))));
} else {
entry.autodrain();
}
})
.on('close', () => {
// Verify max images limit
if (config.maxImages && filesCount > config.maxImages) cb(`${filesCount} images uploaded, but this node can only process up to ${config.maxImages}.`);
else cb();
})
.on('error', cb);
}
// Find and handle zip files and extract
fs.readdir(destImagesPath, (err, entries) => {
if (err) cb(err);
else {
async.eachSeries(entries, (entry, cb) => {
if (/\.zip$/gi.test(entry)) {
let filesCount = 0;
fs.createReadStream(path.join(destImagesPath, entry)).pipe(unzip.Parse())
.on('entry', function(entry) {
if (entry.type === 'File') {
filesCount++;
entry.pipe(fs.createWriteStream(path.join(destImagesPath, path.basename(entry.path))));
} else {
entry.autodrain();
}
})
.on('close', () => {
// Verify max images limit
if (config.maxImages && filesCount > config.maxImages) cb(`${filesCount} images uploaded, but this node can only process up to ${config.maxImages}.`);
else cb();
})
.on('error', cb);
if (entry === "seed.zip"){
handleSeed(cb);
}else if (entry === "zipurl.zip") {
handleZipUrl(cb);
} else cb();
}, cb);
}
@ -318,16 +349,18 @@ module.exports = {
// Create task
cb => {
new Task(req.id, req.body.name, (err, task) => {
new Task(req.id, req.body.name, req.body.options,
req.body.webhook,
req.body.skipPostProcessing === 'true',
req.body.outputs,
(err, task) => {
if (err) cb(err);
else {
TaskManager.singleton().addNew(task);
res.json({ uuid: req.id });
cb();
}
}, req.body.options,
req.body.webhook,
req.body.skipPostProcessing === 'true');
});
}
], err => {
if (err) die(err.message);

26
libs/utils.js
Wyświetl plik

@ -1,4 +1,7 @@
"use strict";
const path = require('path');
module.exports = {
get: function(scope, prop, defaultValue){
let parts = prop.split(".");
@ -18,5 +21,28 @@ module.exports = {
sanitize: function(filePath){
filePath = filePath.replace(/[^\w.-]/g, "_");
return filePath;
},
parseUnsafePathsList: function(paths){
// Parse a list (or a JSON encoded string representing a list)
// of paths and remove all traversals (., ..) and guarantee
// that the paths are relative
if (typeof paths === "string"){
try{
paths = JSON.parse(paths);
}catch(e){
return [];
}
}
if (!Array.isArray(paths)){
return [];
}
return paths.map(p => {
const safeSuffix = path.normalize(p).replace(/^(\.\.(\/|\\|$))+/, '');
return path.join('./', safeSuffix);
});
}
};

2
package.json
Wyświetl plik

@ -1,6 +1,6 @@
{
"name": "node-opendronemap",
"version": "1.4.0",
"version": "1.5.0",
"description": "REST API to access ODM",
"main": "index.js",
"scripts": {

3
public/js/main.js
Wyświetl plik

@ -69,6 +69,7 @@ $(function() {
formData.append("webhook", $("#webhook").val());
formData.append("skipPostProcessing", !$("#doPostProcessing").prop('checked'));
formData.append("options", JSON.stringify(optionsModel.getUserOptions()));
// formData.append("outputs", JSON.stringify(['odm_orthophoto/odm_orthophoto.tif']));
if (this.mode() === 'file'){
if (this.filesCount() > 0){
@ -119,7 +120,7 @@ $(function() {
url : "/task/new/upload/",
parallelUploads: 8, // http://blog.olamisan.com/max-parallel-http-connections-in-a-browser max parallel connections
uploadMultiple: false,
acceptedFiles: "image/*,text/*",
acceptedFiles: "image/*,text/*,application/*",
autoProcessQueue: false,
createImageThumbnails: false,
previewTemplate: '<div style="display:none"></div>',

0
tests/processing_results/opensfm/features/test.txt 100644
Wyświetl plik