Merge pull request #79 from pierotofy/splitmerge

Split merge

docs/index.adoc

@@ -8,7 +8,7 @@ REST API to access ODM
 
 === Version information
 [%hardbreaks]
-_Version_ : 1.4.0
+_Version_ : 1.5.0
 
 
 === Contact information
@@ -294,11 +294,13 @@ _optional_|An optional UUID string that will be used as UUID for this task inste
 |*Query*|*token* +
 _optional_|Token required for authentication (when authentication is required).|string|
 |*FormData*|*images* +
-_optional_|Images to process, plus an optional GCP file. If included, the GCP file should have .txt extension|file|
+_optional_|Images to process, plus an optional GCP file (*.txt) and/or an optional seed file (seed.zip). If included, the GCP file should have .txt extension. If included, the seed archive pre-polulates the task directory with its contents.|file|
 |*FormData*|*name* +
 _optional_|An optional name to be associated with the task|string|
 |*FormData*|*options* +
 _optional_|Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, …]. For example, [{"name":"cmvs-maxImages","value":"500"},{"name":"time","value":true}]. For a list of all options, call /options|string|
+|*FormData*|*outputs* +
+_optional_|An optional serialized JSON string of paths relative to the project directory that should be included in the all.zip result file, overriding the default behavior.|string|
 |*FormData*|*skipPostProcessing* +
 _optional_|When set, skips generation of map tiles, derivate assets, point cloud tiles.|boolean|
 |*FormData*|*webhook* +
@@ -402,6 +404,8 @@ _optional_|Token required for authentication (when authentication is required).|
 _optional_|An optional name to be associated with the task|string|
 |*FormData*|*options* +
 _optional_|Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, …]. For example, [{"name":"cmvs-maxImages","value":"500"},{"name":"time","value":true}]. For a list of all options, call /options|string|
+|*FormData*|*outputs* +
+_optional_|An optional serialized JSON string of paths relative to the project directory that should be included in the all.zip result file, overriding the default behavior.|string|
 |*FormData*|*skipPostProcessing* +
 _optional_|When set, skips generation of map tiles, derivate assets, point cloud tiles.|boolean|
 |*FormData*|*webhook* +
@@ -451,7 +455,7 @@ _required_|UUID of the task|string|
 |*Query*|*token* +
 _optional_|Token required for authentication (when authentication is required).|string|
 |*FormData*|*images* +
-_required_|Images to process, plus an optional GCP file. If included, the GCP file should have .txt extension|file|
+_required_|Images to process, plus an optional GCP file (*.txt) and/or an optional seed file (seed.zip). If included, the GCP file should have .txt extension. If included, the seed archive pre-polulates the task directory with its contents.|file|
 |===
 
 

index.js

@@ -80,6 +80,12 @@ let server;
  *          required: false
  *          type: string
  *        -
+ *          name: outputs
+ *          in: formData
+ *          description: 'An optional serialized JSON string of paths relative to the project directory that should be included in the all.zip result file, overriding the default behavior.'
+ *          required: false
+ *          type: string
+ *        -
  *          name: token
  *          in: query
  *          description: 'Token required for authentication (when authentication is required).'
@@ -125,7 +131,7 @@ app.post('/task/new/init', authCheck, taskNew.assignUUID, formDataParser, taskNe
  *        -
  *          name: images
  *          in: formData
- *          description: Images to process, plus an optional GCP file. If included, the GCP file should have .txt extension
+ *          description: Images to process, plus an optional GCP file (*.txt) and/or an optional seed file (seed.zip). If included, the GCP file should have .txt extension. If included, the seed archive pre-polulates the task directory with its contents.
  *          required: true
  *          type: file
  *        -
@@ -192,7 +198,7 @@ app.post('/task/new/commit/:uuid', authCheck, taskNew.getUUID, taskNew.handleCom
  *        -
  *          name: images
  *          in: formData
- *          description: Images to process, plus an optional GCP file. If included, the GCP file should have .txt extension
+ *          description: Images to process, plus an optional GCP file (*.txt) and/or an optional seed file (seed.zip). If included, the GCP file should have .txt extension. If included, the seed archive pre-polulates the task directory with its contents.
  *          required: false
  *          type: file
  *        -
@@ -226,6 +232,12 @@ app.post('/task/new/commit/:uuid', authCheck, taskNew.getUUID, taskNew.handleCom
  *          required: false
  *          type: string
  *        -
+ *          name: outputs
+ *          in: formData
+ *          description: 'An optional serialized JSON string of paths relative to the project directory that should be included in the all.zip result file, overriding the default behavior.'
+ *          required: false
+ *          type: string
+ *        -
  *          name: token
  *          in: query
  *          description: 'Token required for authentication (when authentication is required).'

libs/Task.js

@@ -32,11 +32,12 @@ const Directories = require('./Directories');
 const kill = require('tree-kill');
 const S3 = require('./S3');
 const request = require('request');
+const utils = require('./utils');
 
 const statusCodes = require('./statusCodes');
 
 module.exports = class Task{
-    constructor(uuid, name, done, options = [], webhook = null, skipPostProcessing = false){
+    constructor(uuid, name, options = [], webhook = null, skipPostProcessing = false, outputs = [], done = () => {}){
         assert(uuid !== undefined, "uuid must be set");
         assert(done !== undefined, "ready must be set");
 
@@ -51,6 +52,7 @@ module.exports = class Task{
         this.runningProcesses = [];
         this.webhook = webhook;
         this.skipPostProcessing = skipPostProcessing;
+        this.outputs = utils.parseUnsafePathsList(outputs);
 
         async.series([
             // Read images info
@@ -86,7 +88,13 @@ module.exports = class Task{
     }
 
     static CreateFromSerialized(taskJson, done){
-        new Task(taskJson.uuid, taskJson.name, (err, task) => {
+        new Task(taskJson.uuid, 
+            taskJson.name, 
+            taskJson.options, 
+            taskJson.webhook, 
+            taskJson.skipPostProcessing,
+            taskJson.outputs,
+            (err, task) => {
                 if (err) done(err);
                 else{
                     // Override default values with those
@@ -101,7 +109,7 @@ module.exports = class Task{
                     }
                     done(null, task);
                 }
-        }, taskJson.options, taskJson.webhook, taskJson.skipPostProcessing);
+            });
     }
 
     // Get path where images are stored for this task
@@ -317,6 +325,10 @@ module.exports = class Task{
                               'odm_georeferencing', 'odm_texturing',
                               'odm_dem/dsm.tif', 'odm_dem/dtm.tif', 'dsm_tiles', 'dtm_tiles',
                               'orthophoto_tiles', 'potree_pointcloud', 'images.json'];
+            
+            // Did the user request different outputs than the default?
+            if (this.outputs.length > 0) allPaths = this.outputs;
+
             let tasks = [];
             
             if (config.test){
@@ -528,7 +540,8 @@ module.exports = class Task{
             status: this.status,
             options: this.options,
             webhook: this.webhook,
-            skipPostProcessing: !!this.skipPostProcessing
+            skipPostProcessing: !!this.skipPostProcessing,
+            outputs: this.outputs || []
         };
     }
 };

libs/odmInfo.js

@@ -57,8 +57,8 @@ module.exports = {
                     // (num cores can be set programmatically, so can gcpFile, etc.)
                     if (["-h", "--project-path", "--cmvs-maxImages", "--time",
                         "--zip-results", "--pmvs-num-cores",
-                        "--start-with", "--gcp", "--end-with", "--images",
+                        "--start-with", "--gcp", "--images",
-                        "--rerun-all", "--rerun",
+                        "--rerun-all", "--rerun", "--end-with",
                         "--slam-config", "--video", "--version", "name"].indexOf(option) !== -1) continue;
 
                     let values = json[option];

libs/taskNew.js

@@ -270,15 +270,35 @@ module.exports = {
                 cb => fs.mkdir(destGcpPath, undefined, cb),
                 cb => mv(srcPath, destImagesPath, cb),
                 
+                // Zip files handling
                 cb => {
-                    // Find any *.zip file and extract
+                    const handleSeed = (cb) => {
-                    fs.readdir(destImagesPath, (err, entries) => {
+                        const seedFileDst = path.join(destPath, "seed.zip");
-                        if (err) cb(err);
+
-                        else {
+                        async.series([
-                            async.eachSeries(entries, (entry, cb) => {
+                            // Move to project root
-                                if (/\.zip$/gi.test(entry)) {
+                            cb => mv(path.join(destImagesPath, "seed.zip"), seedFileDst, cb),
+                            
+                            // Extract
+                            cb => {
+                                fs.createReadStream(seedFileDst).pipe(unzip.Extract({ path: destPath }))
+                                    .on('close', cb)
+                                    .on('error', cb);
+                            },
+
+                            // Verify max images limit
+                            cb => {
+                                fs.readdir(destImagesPath, (err, files) => {
+                                    if (config.maxImages && files.length > config.maxImages) cb(`${files.length} images uploaded, but this node can only process up to ${config.maxImages}.`);
+                                    else cb(err);
+                                });
+                            }
+                        ], cb);
+                    }
+
+                    const handleZipUrl = (cb) => {
                         let filesCount = 0;
-                                    fs.createReadStream(path.join(destImagesPath, entry)).pipe(unzip.Parse())
+                        fs.createReadStream(path.join(destImagesPath, "zipurl.zip")).pipe(unzip.Parse())
                             .on('entry', function(entry) {
                                 if (entry.type === 'File') {
                                     filesCount++;
@@ -293,6 +313,17 @@ module.exports = {
                                 else cb();
                             })
                             .on('error', cb);
+                    }
+
+                    // Find and handle zip files and extract
+                    fs.readdir(destImagesPath, (err, entries) => {
+                        if (err) cb(err);
+                        else {
+                            async.eachSeries(entries, (entry, cb) => {
+                                if (entry === "seed.zip"){
+                                    handleSeed(cb);
+                                }else if (entry === "zipurl.zip") {
+                                    handleZipUrl(cb);
                                 } else cb();
                             }, cb);
                         }
@@ -318,16 +349,18 @@ module.exports = {
 
                 // Create task
                 cb => {
-                    new Task(req.id, req.body.name, (err, task) => {
+                    new Task(req.id, req.body.name, req.body.options,
+                    req.body.webhook,
+                    req.body.skipPostProcessing === 'true',
+                    req.body.outputs,
+                    (err, task) => {
                         if (err) cb(err);
                         else {
                             TaskManager.singleton().addNew(task);
                             res.json({ uuid: req.id });
                             cb();
                         }
-                    }, req.body.options,
+                    });
-                    req.body.webhook,
-                    req.body.skipPostProcessing === 'true');
                 }
             ], err => {
                 if (err) die(err.message);

libs/utils.js

@@ -1,4 +1,7 @@
 "use strict";
+
+const path = require('path');
+
 module.exports = {
     get: function(scope, prop, defaultValue){
         let parts = prop.split(".");
@@ -18,5 +21,28 @@ module.exports = {
     sanitize: function(filePath){
         filePath = filePath.replace(/[^\w.-]/g, "_");
         return filePath;
+    },
+
+    parseUnsafePathsList: function(paths){
+        // Parse a list (or a JSON encoded string representing a list)
+        // of paths and remove all traversals (., ..) and guarantee
+        // that the paths are relative
+
+        if (typeof paths === "string"){
+            try{
+                paths = JSON.parse(paths);
+            }catch(e){
+                return [];
+            }
+        }
+        
+        if (!Array.isArray(paths)){
+            return [];
+        }
+        
+        return paths.map(p => {
+            const safeSuffix = path.normalize(p).replace(/^(\.\.(\/|\\|$))+/, '');
+            return path.join('./', safeSuffix);
+        });
     }
 };

package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-opendronemap",
-  "version": "1.4.0",
+  "version": "1.5.0",
   "description": "REST API to access ODM",
   "main": "index.js",
   "scripts": {

public/js/main.js

@@ -69,6 +69,7 @@ $(function() {
         formData.append("webhook", $("#webhook").val());
         formData.append("skipPostProcessing", !$("#doPostProcessing").prop('checked'));
         formData.append("options", JSON.stringify(optionsModel.getUserOptions()));
+        // formData.append("outputs", JSON.stringify(['odm_orthophoto/odm_orthophoto.tif']));
 
         if (this.mode() === 'file'){
             if (this.filesCount() > 0){
@@ -119,7 +120,7 @@ $(function() {
         url : "/task/new/upload/",
         parallelUploads: 8, // http://blog.olamisan.com/max-parallel-http-connections-in-a-browser max parallel connections
         uploadMultiple: false,
-        acceptedFiles: "image/*,text/*",
+        acceptedFiles: "image/*,text/*,application/*",
         autoProcessQueue: false,
         createImageThumbnails: false,
         previewTemplate: '<div style="display:none"></div>',