diff --git a/Dockerfile b/Dockerfile
index 433504f..e976ebd 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -5,7 +5,7 @@ EXPOSE 3000
USER root
RUN curl --silent --location https://deb.nodesource.com/setup_10.x | bash -
-RUN apt-get install -y nodejs python-gdal && npm install -g nodemon && \
+RUN apt-get install -y nodejs python-gdal p7zip-full && npm install -g nodemon && \
ln -s /code/SuperBuild/install/bin/entwine /usr/bin/entwine && \
ln -s /code/SuperBuild/install/bin/pdal /usr/bin/pdal
diff --git a/README.md b/README.md
index a38507d..8702d64 100644
--- a/README.md
+++ b/README.md
@@ -54,11 +54,11 @@ If you are already running [ODM](https://github.com/OpenDroneMap/ODM) on Ubuntu
1) Install Entwine: https://entwine.io/quickstart.html#installation
-2) Install node.js and npm dependencies:
+2) Install node.js, npm dependencies and 7zip:
```bash
sudo curl --silent --location https://deb.nodesource.com/setup_6.x | sudo bash -
-sudo apt-get install -y nodejs python-gdal
+sudo apt-get install -y nodejs python-gdal p7zip-full
git clone https://github.com/OpenDroneMap/NodeODM
cd NodeODM
npm install
diff --git a/config-default.json b/config-default.json
index 2ace042..90f5049 100644
--- a/config-default.json
+++ b/config-default.json
@@ -11,7 +11,7 @@
"port": 3000,
"deamon": false,
- "parallelQueueProcessing": 2,
+ "parallelQueueProcessing": 1,
"cleanupTasksAfter": 2880,
"test": false,
"testSkipOrthophotos": false,
diff --git a/config.js b/config.js
index 481599c..cb38e43 100644
--- a/config.js
+++ b/config.js
@@ -20,6 +20,7 @@ along with this program. If not, see .
let fs = require('fs');
let argv = require('minimist')(process.argv.slice(2));
let utils = require('./libs/utils');
+const spawnSync = require('child_process').spawnSync;
if (argv.help){
console.log(`
@@ -93,7 +94,7 @@ config.logger.logDirectory = fromConfigFile("logger.logDirectory", ''); // Set t
config.port = parseInt(argv.port || argv.p || fromConfigFile("port", process.env.PORT || 3000));
config.deamon = argv.deamonize || argv.d || fromConfigFile("daemon", false);
-config.parallelQueueProcessing = parseInt(argv.parallel_queue_processing || argv.q || fromConfigFile("parallelQueueProcessing", 2));
+config.parallelQueueProcessing = parseInt(argv.parallel_queue_processing || argv.q || fromConfigFile("parallelQueueProcessing", 1));
config.cleanupTasksAfter = parseInt(argv.cleanup_tasks_after || fromConfigFile("cleanupTasksAfter", 2880));
config.cleanupUploadsAfter = parseInt(argv.cleanup_uploads_after || fromConfigFile("cleanupUploadsAfter", 2880));
config.test = argv.test || fromConfigFile("test", false);
@@ -115,4 +116,8 @@ config.s3UploadEverything = argv.s3_upload_everything || fromConfigFile("s3Uploa
config.maxConcurrency = parseInt(argv.max_concurrency || fromConfigFile("maxConcurrency", 0));
config.maxRuntime = parseInt(argv.max_runtime || fromConfigFile("maxRuntime", -1));
+// Detect 7z availability
+const childProcess = spawnSync("7z", ['--help']);
+config.has7z = childProcess.status === 0;
+
module.exports = config;
diff --git a/docs/index.adoc b/docs/index.adoc
index 5bc8ef0..2043827 100644
--- a/docs/index.adoc
+++ b/docs/index.adoc
@@ -8,7 +8,7 @@ REST API to access ODM
=== Version information
[%hardbreaks]
-_Version_ : 1.5.3
+_Version_ : 1.6.0
=== Contact information
@@ -279,6 +279,48 @@ _required_|UUID of the task|string|
|===
+[[_task_list_get]]
+=== GET /task/list
+
+==== Description
+Gets the list of tasks available on this node.
+
+
+==== Parameters
+
+[options="header", cols=".^2,.^3,.^9,.^4,.^2"]
+|===
+|Type|Name|Description|Schema|Default
+|*Query*|*token* +
+_optional_|Token required for authentication (when authentication is required).|string|
+|===
+
+
+==== Responses
+
+[options="header", cols=".^2,.^14,.^4"]
+|===
+|HTTP Code|Description|Schema
+|*200*|Task List|< <<_task_list_get_response_200,Response 200>> > array
+|*default*|Error|<<_error,Error>>
+|===
+
+[[_task_list_get_response_200]]
+*Response 200*
+
+[options="header", cols=".^3,.^11,.^4"]
+|===
+|Name|Description|Schema
+|*uuid* +
+_required_|UUID|string
+|===
+
+
+==== Tags
+
+* task
+
+
[[_task_new_post]]
=== POST /task/new
diff --git a/docs/swagger.json b/docs/swagger.json
index d37bc76..3974d25 100644
--- a/docs/swagger.json
+++ b/docs/swagger.json
@@ -1 +1 @@
-{"info":{"title":"NodeODM","version":"1.5.3","description":"REST API to access ODM","license":{"name":"GPL-3.0"},"contact":{"name":"Piero Toffanin"}},"consumes":["application/json"],"produces":["application/json","application/zip"],"basePath":"/","schemes":["http"],"swagger":"2.0","paths":{"/task/new/init":{"post":{"description":"Initialize the upload of a new task. If successful, a user can start uploading files via /task/new/upload. The task will not start until /task/new/commit is called.","tags":["task"],"parameters":[{"name":"name","in":"formData","description":"An optional name to be associated with the task","required":false,"type":"string"},{"name":"options","in":"formData","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options","required":false,"type":"string"},{"name":"skipPostProcessing","in":"formData","description":"When set, skips generation of map tiles, derivate assets, point cloud tiles.","required":false,"type":"boolean"},{"name":"webhook","in":"formData","description":"Optional URL to call when processing has ended (either successfully or unsuccessfully).","required":false,"type":"string"},{"name":"outputs","in":"formData","description":"An optional serialized JSON string of paths relative to the project directory that should be included in the all.zip result file, overriding the default behavior.","required":false,"type":"string"},{"name":"dateCreated","in":"formData","description":"An optional timestamp overriding the default creation date of the task.","required":false,"type":"integer"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"},{"name":"set-uuid","in":"header","description":"An optional UUID string that will be used as UUID for this task instead of generating a random one.","required":false,"type":"string"}],"responses":{"200":{"description":"Success","schema":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID of the newly created task"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/new/upload/{uuid}":{"post":{"description":"Adds one or more files to the task created via /task/new/init. It does not start the task. To start the task, call /task/new/commit.","tags":["task"],"consumes":["multipart/form-data"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"images","in":"formData","description":"Images to process, plus an optional GCP file (*.txt) and/or an optional seed file (seed.zip). If included, the GCP file should have .txt extension. If included, the seed archive pre-polulates the task directory with its contents.","required":true,"type":"file"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"File Received","schema":{"$ref":"#/definitions/Response"}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/new/commit/{uuid}":{"post":{"description":"Creates a new task for which images have been uploaded via /task/new/upload.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Success","schema":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID of the newly created task"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/new":{"post":{"description":"Creates a new task and places it at the end of the processing queue. For uploading really large tasks, see /task/new/init instead.","tags":["task"],"consumes":["multipart/form-data"],"parameters":[{"name":"images","in":"formData","description":"Images to process, plus an optional GCP file (*.txt) and/or an optional seed file (seed.zip). If included, the GCP file should have .txt extension. If included, the seed archive pre-polulates the task directory with its contents.","required":false,"type":"file"},{"name":"zipurl","in":"formData","description":"URL of the zip file containing the images to process, plus an optional GCP file. If included, the GCP file should have .txt extension","required":false,"type":"string"},{"name":"name","in":"formData","description":"An optional name to be associated with the task","required":false,"type":"string"},{"name":"options","in":"formData","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options","required":false,"type":"string"},{"name":"skipPostProcessing","in":"formData","description":"When set, skips generation of map tiles, derivate assets, point cloud tiles.","required":false,"type":"boolean"},{"name":"webhook","in":"formData","description":"Optional URL to call when processing has ended (either successfully or unsuccessfully).","required":false,"type":"string"},{"name":"outputs","in":"formData","description":"An optional serialized JSON string of paths relative to the project directory that should be included in the all.zip result file, overriding the default behavior.","required":false,"type":"string"},{"name":"dateCreated","in":"formData","description":"An optional timestamp overriding the default creation date of the task.","required":false,"type":"integer"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"},{"name":"set-uuid","in":"header","description":"An optional UUID string that will be used as UUID for this task instead of generating a random one.","required":false,"type":"string"}],"responses":{"200":{"description":"Success","schema":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID of the newly created task"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/info":{"get":{"description":"Gets information about this task, such as name, creation date, processing time, status, command line options and number of images being processed. See schema definition for a full list.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"},{"name":"with_output","in":"query","description":"Optionally retrieve the console output for this task. The parameter specifies the line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. By default no console output is added to the response.","default":0,"required":false,"type":"integer"}],"responses":{"200":{"description":"Task Information","schema":{"title":"TaskInfo","type":"object","required":["uuid","name","dateCreated","processingTime","status","options","imagesCount","progress"],"properties":{"uuid":{"type":"string","description":"UUID"},"name":{"type":"string","description":"Name"},"dateCreated":{"type":"integer","description":"Timestamp"},"processingTime":{"type":"integer","description":"Milliseconds that have elapsed since the task started being processed."},"status":{"type":"object","required":["code"],"properties":{"code":{"type":"integer","description":"Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)","enum":[10,20,30,40,50]}}},"options":{"type":"array","description":"List of options used to process this task","items":{"type":"object","required":["name","value"],"properties":{"name":{"type":"string","description":"Option name (example: \"odm_meshing-octreeDepth\")"},"value":{"type":"string","description":"Value (example: 9)"}}}},"imagesCount":{"type":"integer","description":"Number of images"},"progress":{"type":"float","description":"Percentage progress (estimated) of the task"},"output":{"type":"array","description":"Console output for the task (only if requested via ?output=)","items":{"type":"string"}}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/output":{"get":{"description":"Retrieves the console output of the OpenDroneMap's process. Useful for monitoring execution and to provide updates to the user.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"line","in":"query","description":"Optional line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. Defaults to 0 (retrieve all console output).","default":0,"required":false,"type":"integer"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Console Output","schema":{"type":"string"}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/download/{asset}":{"get":{"description":"Retrieves an asset (the output of OpenDroneMap's processing) associated with a task","tags":["task"],"produces":["application/zip"],"parameters":[{"name":"uuid","in":"path","type":"string","description":"UUID of the task","required":true},{"name":"asset","in":"path","type":"string","description":"Type of asset to download. Use \"all.zip\" for zip file containing all assets.","required":true,"enum":["all.zip","orthophoto.tif"]},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Asset File","schema":{"type":"file"}},"default":{"description":"Error message","schema":{"$ref":"#/definitions/Error"}}}}},"/task/cancel":{"post":{"description":"Cancels a task (stops its execution, or prevents it from being executed)","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/task/remove":{"post":{"description":"Removes a task and deletes all of its assets","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/task/restart":{"post":{"description":"Restarts a task that was previously canceled, that had failed to process or that successfully completed","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"options","in":"body","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options. Overrides the previous options set for this task.","required":false,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/options":{"get":{"description":"Retrieves the command line options that can be passed to process a task","parameters":[{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"tags":["server"],"responses":{"200":{"description":"Options","schema":{"type":"array","items":{"title":"Option","type":"object","required":["name","type","value","domain","help"],"properties":{"name":{"type":"string","description":"Command line option (exactly as it is passed to the OpenDroneMap process, minus the leading '--')"},"type":{"type":"string","description":"Datatype of the value of this option","enum":["int","float","string","bool"]},"value":{"type":"string","description":"Default value of this option"},"domain":{"type":"string","description":"Valid range of values (for example, \"positive integer\" or \"float > 0.0\")"},"help":{"type":"string","description":"Description of what this option does"}}}}}}}},"/info":{"get":{"description":"Retrieves information about this node","parameters":[{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"tags":["server"],"responses":{"200":{"description":"Info","schema":{"type":"object","required":["version","taskQueueCount","maxImages","engineVersion","engine"],"properties":{"version":{"type":"string","description":"Current API version"},"taskQueueCount":{"type":"integer","description":"Number of tasks currently being processed or waiting to be processed"},"availableMemory":{"type":"integer","description":"Amount of RAM available in bytes"},"totalMemory":{"type":"integer","description":"Amount of total RAM in the system in bytes"},"cpuCores":{"type":"integer","description":"Number of CPU cores (virtual)"},"maxImages":{"type":"integer","description":"Maximum number of images allowed for new tasks or null if there's no limit."},"maxParallelTasks":{"type":"integer","description":"Maximum number of tasks that can be processed simultaneously"},"engineVersion":{"type":"string","description":"Current version of processing engine"},"engine":{"type":"string","description":"Lowercase identifier of processing engine"}}}}}}},"/auth/info":{"get":{"description":"Retrieves login information for this node.","tags":["auth"],"responses":{"200":{"description":"LoginInformation","schema":{"type":"object","required":["message","loginUrl","registerUrl"],"properties":{"message":{"type":"string","description":"Message to be displayed to the user prior to login/registration. This might include instructions on how to register or login, or to communicate that authentication is not available."},"loginUrl":{"type":"string","description":"URL (absolute or relative) where to make a POST request to obtain a token, or null if login is disabled."},"registerUrl":{"type":"string","description":"URL (absolute or relative) where to make a POST request to register a user, or null if registration is disabled."}}}}}}},"/auth/login":{"post":{"description":"Retrieve a token from a username/password pair.","parameters":[{"name":"username","in":"body","description":"Username","required":true,"schema":{"type":"string"}},{"name":"password","in":"body","description":"Password","required":true,"type":"string"}],"responses":{"200":{"description":"Login Succeeded","schema":{"type":"object","required":["token"],"properties":{"token":{"type":"string","description":"Token to be passed as a query parameter to other API calls."}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/auth/register":{"post":{"description":"Register a new username/password.","parameters":[{"name":"username","in":"body","description":"Username","required":true,"schema":{"type":"string"}},{"name":"password","in":"body","description":"Password","required":true,"type":"string"}],"responses":{"200":{"description":"Response","schema":{"$ref":"#/definitions/Response"}}}}}},"definitions":{"Error":{"type":"object","required":["error"],"properties":{"error":{"type":"string","description":"Description of the error"}}},"Response":{"type":"object","required":["success"],"properties":{"success":{"type":"boolean","description":"true if the command succeeded, false otherwise"},"error":{"type":"string","description":"Error message if an error occured"}}}},"responses":{},"parameters":{},"securityDefinitions":{},"tags":[]}
\ No newline at end of file
+{"info":{"title":"NodeODM","version":"1.6.0","description":"REST API to access ODM","license":{"name":"GPL-3.0"},"contact":{"name":"Piero Toffanin"}},"consumes":["application/json"],"produces":["application/json","application/zip"],"basePath":"/","schemes":["http"],"swagger":"2.0","paths":{"/task/new/init":{"post":{"description":"Initialize the upload of a new task. If successful, a user can start uploading files via /task/new/upload. The task will not start until /task/new/commit is called.","tags":["task"],"parameters":[{"name":"name","in":"formData","description":"An optional name to be associated with the task","required":false,"type":"string"},{"name":"options","in":"formData","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options","required":false,"type":"string"},{"name":"skipPostProcessing","in":"formData","description":"When set, skips generation of map tiles, derivate assets, point cloud tiles.","required":false,"type":"boolean"},{"name":"webhook","in":"formData","description":"Optional URL to call when processing has ended (either successfully or unsuccessfully).","required":false,"type":"string"},{"name":"outputs","in":"formData","description":"An optional serialized JSON string of paths relative to the project directory that should be included in the all.zip result file, overriding the default behavior.","required":false,"type":"string"},{"name":"dateCreated","in":"formData","description":"An optional timestamp overriding the default creation date of the task.","required":false,"type":"integer"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"},{"name":"set-uuid","in":"header","description":"An optional UUID string that will be used as UUID for this task instead of generating a random one.","required":false,"type":"string"}],"responses":{"200":{"description":"Success","schema":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID of the newly created task"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/new/upload/{uuid}":{"post":{"description":"Adds one or more files to the task created via /task/new/init. It does not start the task. To start the task, call /task/new/commit.","tags":["task"],"consumes":["multipart/form-data"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"images","in":"formData","description":"Images to process, plus an optional GCP file (*.txt) and/or an optional seed file (seed.zip). If included, the GCP file should have .txt extension. If included, the seed archive pre-polulates the task directory with its contents.","required":true,"type":"file"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"File Received","schema":{"$ref":"#/definitions/Response"}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/new/commit/{uuid}":{"post":{"description":"Creates a new task for which images have been uploaded via /task/new/upload.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Success","schema":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID of the newly created task"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/new":{"post":{"description":"Creates a new task and places it at the end of the processing queue. For uploading really large tasks, see /task/new/init instead.","tags":["task"],"consumes":["multipart/form-data"],"parameters":[{"name":"images","in":"formData","description":"Images to process, plus an optional GCP file (*.txt) and/or an optional seed file (seed.zip). If included, the GCP file should have .txt extension. If included, the seed archive pre-polulates the task directory with its contents.","required":false,"type":"file"},{"name":"zipurl","in":"formData","description":"URL of the zip file containing the images to process, plus an optional GCP file. If included, the GCP file should have .txt extension","required":false,"type":"string"},{"name":"name","in":"formData","description":"An optional name to be associated with the task","required":false,"type":"string"},{"name":"options","in":"formData","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options","required":false,"type":"string"},{"name":"skipPostProcessing","in":"formData","description":"When set, skips generation of map tiles, derivate assets, point cloud tiles.","required":false,"type":"boolean"},{"name":"webhook","in":"formData","description":"Optional URL to call when processing has ended (either successfully or unsuccessfully).","required":false,"type":"string"},{"name":"outputs","in":"formData","description":"An optional serialized JSON string of paths relative to the project directory that should be included in the all.zip result file, overriding the default behavior.","required":false,"type":"string"},{"name":"dateCreated","in":"formData","description":"An optional timestamp overriding the default creation date of the task.","required":false,"type":"integer"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"},{"name":"set-uuid","in":"header","description":"An optional UUID string that will be used as UUID for this task instead of generating a random one.","required":false,"type":"string"}],"responses":{"200":{"description":"Success","schema":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID of the newly created task"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/list":{"get":{"description":"Gets the list of tasks available on this node.","tags":["task"],"parameters":[{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Task List","schema":{"title":"TaskList","type":"array","items":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID"}}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/info":{"get":{"description":"Gets information about this task, such as name, creation date, processing time, status, command line options and number of images being processed. See schema definition for a full list.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"},{"name":"with_output","in":"query","description":"Optionally retrieve the console output for this task. The parameter specifies the line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. By default no console output is added to the response.","default":0,"required":false,"type":"integer"}],"responses":{"200":{"description":"Task Information","schema":{"title":"TaskInfo","type":"object","required":["uuid","name","dateCreated","processingTime","status","options","imagesCount","progress"],"properties":{"uuid":{"type":"string","description":"UUID"},"name":{"type":"string","description":"Name"},"dateCreated":{"type":"integer","description":"Timestamp"},"processingTime":{"type":"integer","description":"Milliseconds that have elapsed since the task started being processed."},"status":{"type":"object","required":["code"],"properties":{"code":{"type":"integer","description":"Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)","enum":[10,20,30,40,50]}}},"options":{"type":"array","description":"List of options used to process this task","items":{"type":"object","required":["name","value"],"properties":{"name":{"type":"string","description":"Option name (example: \"odm_meshing-octreeDepth\")"},"value":{"type":"string","description":"Value (example: 9)"}}}},"imagesCount":{"type":"integer","description":"Number of images"},"progress":{"type":"float","description":"Percentage progress (estimated) of the task"},"output":{"type":"array","description":"Console output for the task (only if requested via ?output=)","items":{"type":"string"}}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/output":{"get":{"description":"Retrieves the console output of the OpenDroneMap's process. Useful for monitoring execution and to provide updates to the user.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"line","in":"query","description":"Optional line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. Defaults to 0 (retrieve all console output).","default":0,"required":false,"type":"integer"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Console Output","schema":{"type":"string"}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/download/{asset}":{"get":{"description":"Retrieves an asset (the output of OpenDroneMap's processing) associated with a task","tags":["task"],"produces":["application/zip"],"parameters":[{"name":"uuid","in":"path","type":"string","description":"UUID of the task","required":true},{"name":"asset","in":"path","type":"string","description":"Type of asset to download. Use \"all.zip\" for zip file containing all assets.","required":true,"enum":["all.zip","orthophoto.tif"]},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Asset File","schema":{"type":"file"}},"default":{"description":"Error message","schema":{"$ref":"#/definitions/Error"}}}}},"/task/cancel":{"post":{"description":"Cancels a task (stops its execution, or prevents it from being executed)","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/task/remove":{"post":{"description":"Removes a task and deletes all of its assets","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/task/restart":{"post":{"description":"Restarts a task that was previously canceled, that had failed to process or that successfully completed","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"options","in":"body","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options. Overrides the previous options set for this task.","required":false,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/options":{"get":{"description":"Retrieves the command line options that can be passed to process a task","parameters":[{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"tags":["server"],"responses":{"200":{"description":"Options","schema":{"type":"array","items":{"title":"Option","type":"object","required":["name","type","value","domain","help"],"properties":{"name":{"type":"string","description":"Command line option (exactly as it is passed to the OpenDroneMap process, minus the leading '--')"},"type":{"type":"string","description":"Datatype of the value of this option","enum":["int","float","string","bool"]},"value":{"type":"string","description":"Default value of this option"},"domain":{"type":"string","description":"Valid range of values (for example, \"positive integer\" or \"float > 0.0\")"},"help":{"type":"string","description":"Description of what this option does"}}}}}}}},"/info":{"get":{"description":"Retrieves information about this node","parameters":[{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"tags":["server"],"responses":{"200":{"description":"Info","schema":{"type":"object","required":["version","taskQueueCount","maxImages","engineVersion","engine"],"properties":{"version":{"type":"string","description":"Current API version"},"taskQueueCount":{"type":"integer","description":"Number of tasks currently being processed or waiting to be processed"},"availableMemory":{"type":"integer","description":"Amount of RAM available in bytes"},"totalMemory":{"type":"integer","description":"Amount of total RAM in the system in bytes"},"cpuCores":{"type":"integer","description":"Number of CPU cores (virtual)"},"maxImages":{"type":"integer","description":"Maximum number of images allowed for new tasks or null if there's no limit."},"maxParallelTasks":{"type":"integer","description":"Maximum number of tasks that can be processed simultaneously"},"engineVersion":{"type":"string","description":"Current version of processing engine"},"engine":{"type":"string","description":"Lowercase identifier of processing engine"}}}}}}},"/auth/info":{"get":{"description":"Retrieves login information for this node.","tags":["auth"],"responses":{"200":{"description":"LoginInformation","schema":{"type":"object","required":["message","loginUrl","registerUrl"],"properties":{"message":{"type":"string","description":"Message to be displayed to the user prior to login/registration. This might include instructions on how to register or login, or to communicate that authentication is not available."},"loginUrl":{"type":"string","description":"URL (absolute or relative) where to make a POST request to obtain a token, or null if login is disabled."},"registerUrl":{"type":"string","description":"URL (absolute or relative) where to make a POST request to register a user, or null if registration is disabled."}}}}}}},"/auth/login":{"post":{"description":"Retrieve a token from a username/password pair.","parameters":[{"name":"username","in":"body","description":"Username","required":true,"schema":{"type":"string"}},{"name":"password","in":"body","description":"Password","required":true,"type":"string"}],"responses":{"200":{"description":"Login Succeeded","schema":{"type":"object","required":["token"],"properties":{"token":{"type":"string","description":"Token to be passed as a query parameter to other API calls."}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/auth/register":{"post":{"description":"Register a new username/password.","parameters":[{"name":"username","in":"body","description":"Username","required":true,"schema":{"type":"string"}},{"name":"password","in":"body","description":"Password","required":true,"type":"string"}],"responses":{"200":{"description":"Response","schema":{"$ref":"#/definitions/Response"}}}}}},"definitions":{"Error":{"type":"object","required":["error"],"properties":{"error":{"type":"string","description":"Description of the error"}}},"Response":{"type":"object","required":["success"],"properties":{"success":{"type":"boolean","description":"true if the command succeeded, false otherwise"},"error":{"type":"string","description":"Error message if an error occured"}}}},"responses":{},"parameters":{},"securityDefinitions":{},"tags":[]}
\ No newline at end of file
diff --git a/index.js b/index.js
index 0094c06..9808739 100644
--- a/index.js
+++ b/index.js
@@ -292,6 +292,44 @@ let getTaskFromUuid = (req, res, next) => {
} else res.json({ error: `${req.params.uuid} not found` });
};
+/** @swagger
+ * /task/list:
+ * get:
+ * description: Gets the list of tasks available on this node.
+ * tags: [task]
+ * parameters:
+ * -
+ * name: token
+ * in: query
+ * description: 'Token required for authentication (when authentication is required).'
+ * required: false
+ * type: string
+ * responses:
+ * 200:
+ * description: Task List
+ * schema:
+ * title: TaskList
+ * type: array
+ * items:
+ * type: object
+ * required: [uuid]
+ * properties:
+ * uuid:
+ * type: string
+ * description: UUID
+ * default:
+ * description: Error
+ * schema:
+ * $ref: '#/definitions/Error'
+ */
+app.get('/task/list', authCheck, (req, res) => {
+ const tasks = [];
+ for (let uuid in taskManager.tasks){
+ tasks.push({uuid});
+ }
+ res.json(tasks);
+});
+
/** @swagger
* /task/{uuid}/info:
* get:
@@ -863,6 +901,10 @@ if (config.test) {
if (config.testDropUploads) logger.info("Uploads will drop at random");
}
+if (!config.has7z){
+ logger.warn("The 7z program is not installed, falling back to legacy (zipping will be slower)");
+}
+
let commands = [
cb => odmInfo.initialize(cb),
cb => auth.initialize(cb),
diff --git a/libs/Task.js b/libs/Task.js
index eb78fd7..1fb07d7 100644
--- a/libs/Task.js
+++ b/libs/Task.js
@@ -22,12 +22,10 @@ const async = require('async');
const assert = require('assert');
const logger = require('./logger');
const fs = require('fs');
-const glob = require("glob");
const path = require('path');
const rmdir = require('rimraf');
const odmRunner = require('./odmRunner');
const processRunner = require('./processRunner');
-const archiver = require('archiver');
const Directories = require('./Directories');
const kill = require('tree-kill');
const S3 = require('./S3');
@@ -249,6 +247,40 @@ module.exports = class Task{
const postProcess = () => {
const createZipArchive = (outputFilename, files) => {
+ return (done) => {
+ this.output.push(`Compressing ${outputFilename}\n`);
+
+ const zipFile = path.resolve(this.getAssetsArchivePath(outputFilename));
+ const sourcePath = !config.test ?
+ this.getProjectFolderPath() :
+ path.join("tests", "processing_results");
+
+ const pathsToArchive = [];
+ files.forEach(f => {
+ if (fs.existsSync(path.join(sourcePath, f))){
+ pathsToArchive.push(f);
+ }
+ });
+
+ processRunner.sevenZip({
+ destination: zipFile,
+ pathsToArchive,
+ cwd: sourcePath
+ }, (err, code, _) => {
+ if (err){
+ logger.error(`Could not archive .zip file: ${err.message}`);
+ done(err);
+ }else{
+ if (code === 0){
+ this.updateProgress(97);
+ done();
+ }else done(new Error(`Could not archive .zip file, 7z exited with code ${code}`));
+ }
+ });
+ };
+ };
+
+ const createZipArchiveLegacy = (outputFilename, files) => {
return (done) => {
this.output.push(`Compressing ${outputFilename}\n`);
@@ -327,7 +359,7 @@ module.exports = class Task{
this.runningProcesses.push(
processRunner.runPostProcessingScript({
projectFolderPath: this.getProjectFolderPath()
- }, (err, code, signal) => {
+ }, (err, code, _) => {
if (err) done(err);
else{
if (code === 0){
@@ -388,7 +420,9 @@ module.exports = class Task{
}
if (!this.skipPostProcessing) tasks.push(runPostProcessingScript());
- tasks.push(createZipArchive('all.zip', allPaths));
+
+ const archiveFunc = config.has7z ? createZipArchive : createZipArchiveLegacy;
+ tasks.push(archiveFunc('all.zip', allPaths));
// Upload to S3 all paths + all.zip file (if config says so)
if (S3.enabled()){
diff --git a/libs/processRunner.js b/libs/processRunner.js
index 8d5fa1b..f6b0ab0 100644
--- a/libs/processRunner.js
+++ b/libs/processRunner.js
@@ -25,7 +25,7 @@ let logger = require('./logger');
let utils = require('./utils');
-function makeRunner(command, args, requiredOptions = [], outputTestFile = null){
+function makeRunner(command, args, requiredOptions = [], outputTestFile = null, skipOnTest = true){
return function(options, done, outputReceived){
for (let requiredOption of requiredOptions){
assert(options[requiredOption] !== undefined, `${requiredOption} must be defined`);
@@ -36,14 +36,16 @@ function makeRunner(command, args, requiredOptions = [], outputTestFile = null){
logger.info(`About to run: ${command} ${commandArgs.join(" ")}`);
- if (config.test){
+ if (config.test && skipOnTest){
logger.info("Test mode is on, command will not execute");
if (outputTestFile){
fs.readFile(path.resolve(__dirname, outputTestFile), 'utf8', (err, text) => {
if (!err){
- let lines = text.split("\n");
- lines.forEach(line => outputReceived(line));
+ if (outputReceived !== undefined){
+ let lines = text.split("\n");
+ lines.forEach(line => outputReceived(line));
+ }
done(null, 0, null);
}else{
@@ -62,20 +64,21 @@ function makeRunner(command, args, requiredOptions = [], outputTestFile = null){
const env = utils.clone(process.env);
env.LD_LIBRARY_PATH = path.join(config.odm_path, "SuperBuild", "install", "lib");
- try{
- let childProcess = spawn(command, commandArgs, { env });
- childProcess
- .on('exit', (code, signal) => done(null, code, signal))
- .on('error', done);
-
+ let cwd = undefined;
+ if (options.cwd) cwd = options.cwd;
+
+ let childProcess = spawn(command, commandArgs, { env, cwd });
+
+ childProcess
+ .on('exit', (code, signal) => done(null, code, signal))
+ .on('error', done);
+
+ if (outputReceived !== undefined){
childProcess.stdout.on('data', chunk => outputReceived(chunk.toString()));
childProcess.stderr.on('data', chunk => outputReceived(chunk.toString()));
- return childProcess;
- }catch(e){
- // Catch errors such as ENOMEM
- logger.warn(`Error: ${e.message}`);
- done(e);
}
+
+ return childProcess;
};
}
@@ -84,5 +87,12 @@ module.exports = {
function(options){
return [options.projectFolderPath];
},
- ["projectFolderPath"])
+ ["projectFolderPath"]),
+
+ sevenZip: makeRunner("7z", function(options){
+ return ["a", "-r", "-bd", options.destination].concat(options.pathsToArchive);
+ },
+ ["destination", "pathsToArchive", "cwd"],
+ null,
+ false)
};
diff --git a/libs/taskNew.js b/libs/taskNew.js
index 0e2efd0..af80543 100644
--- a/libs/taskNew.js
+++ b/libs/taskNew.js
@@ -48,6 +48,24 @@ const removeDirectory = function(dir, cb = () => {}){
});
};
+const assureUniqueFilename = (dstPath, filename, cb) => {
+ const dstFile = path.join(dstPath, filename);
+ fs.exists(dstFile, exists => {
+ if (!exists) cb(null, filename);
+ else{
+ const parts = filename.split(".");
+ if (parts.length > 1){
+ assureUniqueFilename(dstPath,
+ `${parts.slice(0, parts.length - 1).join(".")}_.${parts[parts.length - 1]}`,
+ cb);
+ }else{
+ // Filename without extension? Strange..
+ assureUniqueFilename(dstPath, filename + "_", cb);
+ }
+ }
+ });
+};
+
const upload = multer({
storage: multer.diskStorage({
destination: (req, file, cb) => {
@@ -65,7 +83,9 @@ const upload = multer({
filename: (req, file, cb) => {
let filename = utils.sanitize(file.originalname);
if (filename === "body.json") filename = "_body.json";
- cb(null, filename);
+
+ let dstPath = path.join("tmp", req.id);
+ assureUniqueFilename(dstPath, filename, cb);
}
})
});
diff --git a/package.json b/package.json
index 7552c15..0b9c28d 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "NodeODM",
- "version": "1.5.3",
+ "version": "1.6.0",
"description": "REST API to access ODM",
"main": "index.js",
"scripts": {
diff --git a/public/index.html b/public/index.html
index 0eb3f44..13f62e8 100644
--- a/public/index.html
+++ b/public/index.html
@@ -141,7 +141,13 @@
-
No running tasks.
+
+
+ Loading task list...
+
+
No running tasks.
Retrieving ...
@@ -208,7 +214,7 @@
-
+