diff --git a/config-default.json b/config-default.json index 5a46cbe..e7fad31 100644 --- a/config-default.json +++ b/config-default.json @@ -13,5 +13,6 @@ "deamon": false, "parallelQueueProcessing": 2, "cleanupTasksAfter": 3, - "test": false + "test": false, + "testSkipOrthophotos": false } \ No newline at end of file diff --git a/config.js b/config.js index eed2c45..c475836 100644 --- a/config.js +++ b/config.js @@ -34,6 +34,7 @@ Options: --parallel_queue_processing Number of simultaneous processing tasks (default: 2) --cleanup_tasks_after Number of days that elapse before deleting finished and canceled tasks (default: 3) --test Enable test mode. In test mode, no commands are sent to OpenDroneMap. This can be useful during development or testing (default: false) + --test_skip_orthophotos If test mode is enabled, skip orthophoto results when generating assets. (default: false) Log Levels: error | debug | info | verbose | debug | silly `); @@ -78,5 +79,6 @@ config.deamon = argv.deamonize || argv.d || fromConfigFile("daemon", false); config.parallelQueueProcessing = argv.parallel_queue_processing || fromConfigFile("parallelQueueProcessing", 2); config.cleanupTasksAfter = argv.cleanup_tasks_after || fromConfigFile("cleanupTasksAfter", 3); config.test = argv.test || fromConfigFile("test", false); +config.testSkipOrthophotos = argv.test_skip_orthophotos || fromConfigFile("testSkipOrthophotos", false); module.exports = config; diff --git a/docs/generateDocs.sh b/docs/generateDocs.sh new file mode 100755 index 0000000..32d0784 --- /dev/null +++ b/docs/generateDocs.sh @@ -0,0 +1,2 @@ +node generateSwaggerAPI.js +java -jar swagger2markup-cli-1.0.1.jar convert -i swagger.json -f index diff --git a/docs/index.adoc b/docs/index.adoc index 119ed4e..5f08e82 100644 --- a/docs/index.adoc +++ b/docs/index.adoc @@ -8,7 +8,7 @@ REST API to access OpenDroneMap === Version information [%hardbreaks] -_Version_ : 1.0.1 +_Version_ : 1.0.2 === Contact information @@ -257,7 +257,7 @@ Retrieves an asset (the output of OpenDroneMap's processing) associated with a t |=== |Type|Name|Description|Schema|Default |*Path*|*asset* + -_required_|Type of asset to download. Use "all.zip" for zip file containing all assets.|enum (all.zip, georeferenced_model.ply.zip, georeferenced_model.las.zip, georeferenced_model.csv.zip, orthophoto.png, orthophoto.tif, textured_model.zip, orthophoto_tiles.zip)| +_required_|Type of asset to download. Use "all.zip" for zip file containing all assets.|enum (all.zip, orthophoto.tif)| |*Path*|*uuid* + _required_|UUID of the task|string| |=== diff --git a/docs/swagger.json b/docs/swagger.json index ca0a621..7b53d3d 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -1 +1 @@ -{"info":{"title":"node-opendronemap","version":"1.0.1","description":"REST API to access OpenDroneMap","license":{"name":"GPL-3.0"},"contact":{"name":"Piero Toffanin"}},"consumes":["application/json"],"produces":["application/json","application/zip"],"basePath":"/","schemes":["http"],"swagger":"2.0","paths":{"/task/new":{"post":{"description":"Creates a new task and places it at the end of the processing queue","tags":["task"],"consumes":["multipart/form-data"],"parameters":[{"name":"images","in":"formData","description":"Images to process, plus an optional GPC file. If included, the GPC file should have .txt extension","required":true,"type":"file"},{"name":"name","in":"formData","description":"An optional name to be associated with the task","required":false,"type":"string"},{"name":"options","in":"formData","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options","required":false,"type":"string"}],"responses":{"200":{"description":"Success","schema":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID of the newly created task"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/info":{"get":{"description":"Gets information about this task, such as name, creation date, processing time, status, command line options and number of images being processed. See schema definition for a full list.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"}],"responses":{"200":{"description":"Task Information","schema":{"title":"TaskInfo","type":"object","required":["uuid","name","dateCreated","processingTime","status","options","imagesCount"],"properties":{"uuid":{"type":"string","description":"UUID"},"name":{"type":"string","description":"Name"},"dateCreated":{"type":"integer","description":"Timestamp"},"processingTime":{"type":"integer","description":"Milliseconds that have elapsed since the task started being processed."},"status":{"type":"integer","description":"Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)","enum":[10,20,30,40,50]},"options":{"type":"array","description":"List of options used to process this task","items":{"type":"object","required":["name","value"],"properties":{"name":{"type":"string","description":"Option name (example: \"odm_meshing-octreeDepth\")"},"value":{"type":"string","description":"Value (example: 9)"}}}},"imagesCount":{"type":"integer","description":"Number of images"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/output":{"get":{"description":"Retrieves the console output of the OpenDroneMap's process. Useful for monitoring execution and to provide updates to the user.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"line","in":"query","description":"Optional line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. Defaults to 0 (retrieve all console output).","default":0,"required":false,"type":"integer"}],"responses":{"200":{"description":"Console Output","schema":{"type":"string"}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/download/{asset}":{"get":{"description":"Retrieves an asset (the output of OpenDroneMap's processing) associated with a task","tags":["task"],"produces":["application/zip"],"parameters":[{"name":"uuid","in":"path","type":"string","description":"UUID of the task","required":true},{"name":"asset","in":"path","type":"string","description":"Type of asset to download. Use \"all.zip\" for zip file containing all assets.","required":true,"enum":["all.zip","georeferenced_model.ply.zip","georeferenced_model.las.zip","georeferenced_model.csv.zip","orthophoto.png","orthophoto.tif","textured_model.zip","orthophoto_tiles.zip"]}],"responses":{"200":{"description":"Asset File","schema":{"type":"file"}},"default":{"description":"Error message","schema":{"$ref":"#/definitions/Error"}}}}},"/task/cancel":{"post":{"description":"Cancels a task (stops its execution, or prevents it from being executed)","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/task/remove":{"post":{"description":"Removes a task and deletes all of its assets","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/task/restart":{"post":{"description":"Restarts a task that was previously canceled or that had failed to process","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/options":{"get":{"description":"Retrieves the command line options that can be passed to process a task","tags":["server"],"responses":{"200":{"description":"Options","schema":{"type":"array","items":{"title":"Option","type":"object","required":["name","type","value","domain","help"],"properties":{"name":{"type":"string","description":"Command line option (exactly as it is passed to the OpenDroneMap process, minus the leading '--')"},"type":{"type":"string","description":"Datatype of the value of this option","enum":["int","float","string","bool"]},"value":{"type":"string","description":"Default value of this option"},"domain":{"type":"string","description":"Valid range of values (for example, \"positive integer\" or \"float > 0.0\")"},"help":{"type":"string","description":"Description of what this option does"}}}}}}}},"/info":{"get":{"description":"Retrieves information about this node","tags":["server"],"responses":{"200":{"description":"Info","schema":{"type":"object","required":["version","taskQueueCount"],"properties":{"version":{"type":"string","description":"Current version"},"taskQueueCount":{"type":"integer","description":"Number of tasks currently being processed or waiting to be processed"}}}}}}}},"definitions":{"Error":{"type":"object","required":["error"],"properties":{"error":{"type":"string","description":"Description of the error"}}},"Response":{"type":"object","required":["success"],"properties":{"success":{"type":"boolean","description":"true if the command succeeded, false otherwise"},"error":{"type":"string","description":"Error message if an error occured"}}}},"responses":{},"parameters":{},"securityDefinitions":{},"tags":[]} \ No newline at end of file +{"info":{"title":"node-opendronemap","version":"1.0.2","description":"REST API to access OpenDroneMap","license":{"name":"GPL-3.0"},"contact":{"name":"Piero Toffanin"}},"consumes":["application/json"],"produces":["application/json","application/zip"],"basePath":"/","schemes":["http"],"swagger":"2.0","paths":{"/task/new":{"post":{"description":"Creates a new task and places it at the end of the processing queue","tags":["task"],"consumes":["multipart/form-data"],"parameters":[{"name":"images","in":"formData","description":"Images to process, plus an optional GPC file. If included, the GPC file should have .txt extension","required":true,"type":"file"},{"name":"name","in":"formData","description":"An optional name to be associated with the task","required":false,"type":"string"},{"name":"options","in":"formData","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options","required":false,"type":"string"}],"responses":{"200":{"description":"Success","schema":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID of the newly created task"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/info":{"get":{"description":"Gets information about this task, such as name, creation date, processing time, status, command line options and number of images being processed. See schema definition for a full list.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"}],"responses":{"200":{"description":"Task Information","schema":{"title":"TaskInfo","type":"object","required":["uuid","name","dateCreated","processingTime","status","options","imagesCount"],"properties":{"uuid":{"type":"string","description":"UUID"},"name":{"type":"string","description":"Name"},"dateCreated":{"type":"integer","description":"Timestamp"},"processingTime":{"type":"integer","description":"Milliseconds that have elapsed since the task started being processed."},"status":{"type":"integer","description":"Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)","enum":[10,20,30,40,50]},"options":{"type":"array","description":"List of options used to process this task","items":{"type":"object","required":["name","value"],"properties":{"name":{"type":"string","description":"Option name (example: \"odm_meshing-octreeDepth\")"},"value":{"type":"string","description":"Value (example: 9)"}}}},"imagesCount":{"type":"integer","description":"Number of images"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/output":{"get":{"description":"Retrieves the console output of the OpenDroneMap's process. Useful for monitoring execution and to provide updates to the user.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"line","in":"query","description":"Optional line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. Defaults to 0 (retrieve all console output).","default":0,"required":false,"type":"integer"}],"responses":{"200":{"description":"Console Output","schema":{"type":"string"}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/download/{asset}":{"get":{"description":"Retrieves an asset (the output of OpenDroneMap's processing) associated with a task","tags":["task"],"produces":["application/zip"],"parameters":[{"name":"uuid","in":"path","type":"string","description":"UUID of the task","required":true},{"name":"asset","in":"path","type":"string","description":"Type of asset to download. Use \"all.zip\" for zip file containing all assets.","required":true,"enum":["all.zip","orthophoto.tif"]}],"responses":{"200":{"description":"Asset File","schema":{"type":"file"}},"default":{"description":"Error message","schema":{"$ref":"#/definitions/Error"}}}}},"/task/cancel":{"post":{"description":"Cancels a task (stops its execution, or prevents it from being executed)","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/task/remove":{"post":{"description":"Removes a task and deletes all of its assets","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/task/restart":{"post":{"description":"Restarts a task that was previously canceled or that had failed to process","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/options":{"get":{"description":"Retrieves the command line options that can be passed to process a task","tags":["server"],"responses":{"200":{"description":"Options","schema":{"type":"array","items":{"title":"Option","type":"object","required":["name","type","value","domain","help"],"properties":{"name":{"type":"string","description":"Command line option (exactly as it is passed to the OpenDroneMap process, minus the leading '--')"},"type":{"type":"string","description":"Datatype of the value of this option","enum":["int","float","string","bool"]},"value":{"type":"string","description":"Default value of this option"},"domain":{"type":"string","description":"Valid range of values (for example, \"positive integer\" or \"float > 0.0\")"},"help":{"type":"string","description":"Description of what this option does"}}}}}}}},"/info":{"get":{"description":"Retrieves information about this node","tags":["server"],"responses":{"200":{"description":"Info","schema":{"type":"object","required":["version","taskQueueCount"],"properties":{"version":{"type":"string","description":"Current version"},"taskQueueCount":{"type":"integer","description":"Number of tasks currently being processed or waiting to be processed"}}}}}}}},"definitions":{"Error":{"type":"object","required":["error"],"properties":{"error":{"type":"string","description":"Description of the error"}}},"Response":{"type":"object","required":["success"],"properties":{"success":{"type":"boolean","description":"true if the command succeeded, false otherwise"},"error":{"type":"string","description":"Error message if an error occured"}}}},"responses":{},"parameters":{},"securityDefinitions":{},"tags":[]} \ No newline at end of file diff --git a/index.js b/index.js index c71ef10..1a28c24 100644 --- a/index.js +++ b/index.js @@ -295,13 +295,7 @@ app.get('/task/:uuid/output', getTaskFromUuid, (req, res) => { * required: true * enum: * - all.zip -* - georeferenced_model.ply.zip -* - georeferenced_model.las.zip -* - georeferenced_model.csv.zip -* - orthophoto.png * - orthophoto.tif -* - textured_model.zip -* - orthophoto_tiles.zip * responses: * 200: * description: Asset File diff --git a/libs/Task.js b/libs/Task.js index dd455a9..2fc9e6b 100644 --- a/libs/Task.js +++ b/libs/Task.js @@ -121,27 +121,19 @@ module.exports = class Task{ // Get the path of the archive where all assets // outputted by this task are stored. getAssetsArchivePath(filename){ - switch(filename){ - case "all.zip": - case "georeferenced_model.ply.zip": - case "georeferenced_model.las.zip": - case "georeferenced_model.csv.zip": - case "textured_model.zip": - case "orthophoto_tiles.zip": - // OK - break; - case "orthophoto.png": - case "orthophoto.tif": - // Append missing pieces to the path - filename = !config.test ? - path.join('odm_orthophoto', `odm_${filename}`) : - path.join('..', '..', 'processing_results', 'odm_orthophoto', `odm_${filename}`); - break; - default: - // Invalid - return false; + if (filename == 'all.zip'){ + // OK, do nothing + }else if (filename == 'orthophoto.tif'){ + if (config.test){ + if (config.testSkipOrthophotos) return false; + else filename = path.join('..', '..', 'processing_results', 'odm_orthophoto', `odm_${filename}`); + }else{ + filename = path.join('odm_orthophoto', `odm_${filename}`); + } + }else{ + return false; // Invalid } - + return path.join(this.getProjectFolderPath(), filename); } @@ -306,11 +298,20 @@ module.exports = class Task{ const generateTiles = (inputFile, outputDir) => { return (done) => { - this.runningProcesses.push(processRunner.runTiler({ - zoomLevels: "12-21", - inputFile: path.join(this.getProjectFolderPath(), inputFile), - outputDir: path.join(this.getProjectFolderPath(), outputDir) - }, handleProcessExit(done), handleOutput)); + const inputFilePath = path.join(this.getProjectFolderPath(), inputFile); + + // Not all datasets generate an orthophoto, so we skip + // tiling if the orthophoto is missing + if (fs.existsSync(inputFilePath)){ + this.runningProcesses.push(processRunner.runTiler({ + zoomLevels: "12-21", + inputFile: inputFilePath, + outputDir: path.join(this.getProjectFolderPath(), outputDir) + }, handleProcessExit(done), handleOutput)); + }else{ + handleOutput(`${inputFilePath} file not found, skipping tiles generation\n`); + done(); + } }; }; @@ -324,19 +325,21 @@ module.exports = class Task{ }; // All paths are relative to the project directory (./data//) + let allFolders = ['odm_orthophoto', 'odm_georeferencing', 'odm_texturing', 'odm_meshing', 'orthophoto_tiles', 'potree_pointcloud']; + + if (config.test && config.testSkipOrthophotos){ + logger.info("Test mode will skip orthophoto generation"); + + // Exclude these folders from the all.zip archive + ['odm_orthophoto', 'orthophoto_tiles'].forEach(dir => { + allFolders.splice(allFolders.indexOf(dir), 1); + }); + } + async.series([ generateTiles(path.join('odm_orthophoto', 'odm_orthophoto.tif'), 'orthophoto_tiles'), generatePotreeCloud(path.join('odm_georeferencing', 'odm_georeferenced_model.ply.las'), 'potree_pointcloud'), - createZipArchive('all.zip', ['odm_orthophoto', 'odm_georeferencing', 'odm_texturing', 'odm_meshing', 'orthophoto_tiles', 'potree_pointcloud']), - createZipArchive('georeferenced_model.ply.zip', [path.join('odm_georeferencing', 'odm_georeferenced_model.ply')]), - createZipArchive('georeferenced_model.las.zip', [path.join('odm_georeferencing', 'odm_georeferenced_model.ply.las')]), - createZipArchive('georeferenced_model.csv.zip', [path.join('odm_georeferencing', 'odm_georeferenced_model.csv')]), - createZipArchive('textured_model.zip', [ - path.join('odm_texturing', '*.jpg'), - path.join('odm_texturing', 'odm_textured_model_geo.obj'), - path.join('odm_texturing', 'odm_textured_model_geo.mtl') - ]), - createZipArchive('orthophoto_tiles.zip', ['orthophoto_tiles']) + createZipArchive('all.zip', allFolders) ], (err) => { if (!err){ this.setStatus(statusCodes.COMPLETED); @@ -361,7 +364,7 @@ module.exports = class Task{ runnerOptions["pmvs-num-cores"] = os.cpus().length; if (this.gpcFiles.length > 0){ - runnerOptions["gcp"] = fs.realpathSync(path.join(this.getGpcFolderPath(), this.gpcFiles[0])); + runnerOptions.gcp = fs.realpathSync(path.join(this.getGpcFolderPath(), this.gpcFiles[0])); } this.runningProcesses.push(odmRunner.run(runnerOptions, (err, code, signal) => { diff --git a/libs/odmOptions.js b/libs/odmOptions.js index b2dd37c..7e933e5 100644 --- a/libs/odmOptions.js +++ b/libs/odmOptions.js @@ -41,7 +41,8 @@ module.exports = { // (num cores can be set programmatically, so can gcpFile, etc.) if (["-h", "--project-path", "--cmvs-maxImages", "--time", "--zip-results", "--pmvs-num-cores", - "--start-with", "--gcp", "--end-with"].indexOf(option) !== -1) continue; + "--start-with", "--gcp", "--end-with", "--images", + "--slam-config", "--video"].indexOf(option) !== -1) continue; let values = json[option]; @@ -129,6 +130,9 @@ module.exports = { }, 'string': function(value){ return value; // No conversion needed + }, + 'path': function(value){ + return value; // No conversion needed } }; @@ -178,9 +182,9 @@ module.exports = { } }, { - regex: /^string$/, + regex: /^(string|path)$/, validate: function(){ - return true; // All strings are fine + return true; // All strings/paths are fine } } ]; diff --git a/package.json b/package.json index b72c56f..1b245db 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-opendronemap", - "version": "1.0.1", + "version": "1.0.2", "description": "REST API to access OpenDroneMap", "main": "index.js", "scripts": {