diff --git a/Dockerfile b/Dockerfile index 433504f..e976ebd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ EXPOSE 3000 USER root RUN curl --silent --location https://deb.nodesource.com/setup_10.x | bash - -RUN apt-get install -y nodejs python-gdal && npm install -g nodemon && \ +RUN apt-get install -y nodejs python-gdal p7zip-full && npm install -g nodemon && \ ln -s /code/SuperBuild/install/bin/entwine /usr/bin/entwine && \ ln -s /code/SuperBuild/install/bin/pdal /usr/bin/pdal diff --git a/README.md b/README.md index a38507d..8702d64 100644 --- a/README.md +++ b/README.md @@ -54,11 +54,11 @@ If you are already running [ODM](https://github.com/OpenDroneMap/ODM) on Ubuntu 1) Install Entwine: https://entwine.io/quickstart.html#installation -2) Install node.js and npm dependencies: +2) Install node.js, npm dependencies and 7zip: ```bash sudo curl --silent --location https://deb.nodesource.com/setup_6.x | sudo bash - -sudo apt-get install -y nodejs python-gdal +sudo apt-get install -y nodejs python-gdal p7zip-full git clone https://github.com/OpenDroneMap/NodeODM cd NodeODM npm install diff --git a/config-default.json b/config-default.json index 2ace042..90f5049 100644 --- a/config-default.json +++ b/config-default.json @@ -11,7 +11,7 @@ "port": 3000, "deamon": false, - "parallelQueueProcessing": 2, + "parallelQueueProcessing": 1, "cleanupTasksAfter": 2880, "test": false, "testSkipOrthophotos": false, diff --git a/config.js b/config.js index 481599c..cb38e43 100644 --- a/config.js +++ b/config.js @@ -20,6 +20,7 @@ along with this program. If not, see . let fs = require('fs'); let argv = require('minimist')(process.argv.slice(2)); let utils = require('./libs/utils'); +const spawnSync = require('child_process').spawnSync; if (argv.help){ console.log(` @@ -93,7 +94,7 @@ config.logger.logDirectory = fromConfigFile("logger.logDirectory", ''); // Set t config.port = parseInt(argv.port || argv.p || fromConfigFile("port", process.env.PORT || 3000)); config.deamon = argv.deamonize || argv.d || fromConfigFile("daemon", false); -config.parallelQueueProcessing = parseInt(argv.parallel_queue_processing || argv.q || fromConfigFile("parallelQueueProcessing", 2)); +config.parallelQueueProcessing = parseInt(argv.parallel_queue_processing || argv.q || fromConfigFile("parallelQueueProcessing", 1)); config.cleanupTasksAfter = parseInt(argv.cleanup_tasks_after || fromConfigFile("cleanupTasksAfter", 2880)); config.cleanupUploadsAfter = parseInt(argv.cleanup_uploads_after || fromConfigFile("cleanupUploadsAfter", 2880)); config.test = argv.test || fromConfigFile("test", false); @@ -115,4 +116,8 @@ config.s3UploadEverything = argv.s3_upload_everything || fromConfigFile("s3Uploa config.maxConcurrency = parseInt(argv.max_concurrency || fromConfigFile("maxConcurrency", 0)); config.maxRuntime = parseInt(argv.max_runtime || fromConfigFile("maxRuntime", -1)); +// Detect 7z availability +const childProcess = spawnSync("7z", ['--help']); +config.has7z = childProcess.status === 0; + module.exports = config; diff --git a/docs/index.adoc b/docs/index.adoc index 5bc8ef0..2043827 100644 --- a/docs/index.adoc +++ b/docs/index.adoc @@ -8,7 +8,7 @@ REST API to access ODM === Version information [%hardbreaks] -_Version_ : 1.5.3 +_Version_ : 1.6.0 === Contact information @@ -279,6 +279,48 @@ _required_|UUID of the task|string| |=== +[[_task_list_get]] +=== GET /task/list + +==== Description +Gets the list of tasks available on this node. + + +==== Parameters + +[options="header", cols=".^2,.^3,.^9,.^4,.^2"] +|=== +|Type|Name|Description|Schema|Default +|*Query*|*token* + +_optional_|Token required for authentication (when authentication is required).|string| +|=== + + +==== Responses + +[options="header", cols=".^2,.^14,.^4"] +|=== +|HTTP Code|Description|Schema +|*200*|Task List|< <<_task_list_get_response_200,Response 200>> > array +|*default*|Error|<<_error,Error>> +|=== + +[[_task_list_get_response_200]] +*Response 200* + +[options="header", cols=".^3,.^11,.^4"] +|=== +|Name|Description|Schema +|*uuid* + +_required_|UUID|string +|=== + + +==== Tags + +* task + + [[_task_new_post]] === POST /task/new diff --git a/docs/swagger.json b/docs/swagger.json index d37bc76..3974d25 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -1 +1 @@ -{"info":{"title":"NodeODM","version":"1.5.3","description":"REST API to access ODM","license":{"name":"GPL-3.0"},"contact":{"name":"Piero Toffanin"}},"consumes":["application/json"],"produces":["application/json","application/zip"],"basePath":"/","schemes":["http"],"swagger":"2.0","paths":{"/task/new/init":{"post":{"description":"Initialize the upload of a new task. If successful, a user can start uploading files via /task/new/upload. The task will not start until /task/new/commit is called.","tags":["task"],"parameters":[{"name":"name","in":"formData","description":"An optional name to be associated with the task","required":false,"type":"string"},{"name":"options","in":"formData","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options","required":false,"type":"string"},{"name":"skipPostProcessing","in":"formData","description":"When set, skips generation of map tiles, derivate assets, point cloud tiles.","required":false,"type":"boolean"},{"name":"webhook","in":"formData","description":"Optional URL to call when processing has ended (either successfully or unsuccessfully).","required":false,"type":"string"},{"name":"outputs","in":"formData","description":"An optional serialized JSON string of paths relative to the project directory that should be included in the all.zip result file, overriding the default behavior.","required":false,"type":"string"},{"name":"dateCreated","in":"formData","description":"An optional timestamp overriding the default creation date of the task.","required":false,"type":"integer"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"},{"name":"set-uuid","in":"header","description":"An optional UUID string that will be used as UUID for this task instead of generating a random one.","required":false,"type":"string"}],"responses":{"200":{"description":"Success","schema":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID of the newly created task"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/new/upload/{uuid}":{"post":{"description":"Adds one or more files to the task created via /task/new/init. It does not start the task. To start the task, call /task/new/commit.","tags":["task"],"consumes":["multipart/form-data"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"images","in":"formData","description":"Images to process, plus an optional GCP file (*.txt) and/or an optional seed file (seed.zip). If included, the GCP file should have .txt extension. If included, the seed archive pre-polulates the task directory with its contents.","required":true,"type":"file"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"File Received","schema":{"$ref":"#/definitions/Response"}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/new/commit/{uuid}":{"post":{"description":"Creates a new task for which images have been uploaded via /task/new/upload.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Success","schema":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID of the newly created task"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/new":{"post":{"description":"Creates a new task and places it at the end of the processing queue. For uploading really large tasks, see /task/new/init instead.","tags":["task"],"consumes":["multipart/form-data"],"parameters":[{"name":"images","in":"formData","description":"Images to process, plus an optional GCP file (*.txt) and/or an optional seed file (seed.zip). If included, the GCP file should have .txt extension. If included, the seed archive pre-polulates the task directory with its contents.","required":false,"type":"file"},{"name":"zipurl","in":"formData","description":"URL of the zip file containing the images to process, plus an optional GCP file. If included, the GCP file should have .txt extension","required":false,"type":"string"},{"name":"name","in":"formData","description":"An optional name to be associated with the task","required":false,"type":"string"},{"name":"options","in":"formData","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options","required":false,"type":"string"},{"name":"skipPostProcessing","in":"formData","description":"When set, skips generation of map tiles, derivate assets, point cloud tiles.","required":false,"type":"boolean"},{"name":"webhook","in":"formData","description":"Optional URL to call when processing has ended (either successfully or unsuccessfully).","required":false,"type":"string"},{"name":"outputs","in":"formData","description":"An optional serialized JSON string of paths relative to the project directory that should be included in the all.zip result file, overriding the default behavior.","required":false,"type":"string"},{"name":"dateCreated","in":"formData","description":"An optional timestamp overriding the default creation date of the task.","required":false,"type":"integer"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"},{"name":"set-uuid","in":"header","description":"An optional UUID string that will be used as UUID for this task instead of generating a random one.","required":false,"type":"string"}],"responses":{"200":{"description":"Success","schema":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID of the newly created task"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/info":{"get":{"description":"Gets information about this task, such as name, creation date, processing time, status, command line options and number of images being processed. See schema definition for a full list.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"},{"name":"with_output","in":"query","description":"Optionally retrieve the console output for this task. The parameter specifies the line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. By default no console output is added to the response.","default":0,"required":false,"type":"integer"}],"responses":{"200":{"description":"Task Information","schema":{"title":"TaskInfo","type":"object","required":["uuid","name","dateCreated","processingTime","status","options","imagesCount","progress"],"properties":{"uuid":{"type":"string","description":"UUID"},"name":{"type":"string","description":"Name"},"dateCreated":{"type":"integer","description":"Timestamp"},"processingTime":{"type":"integer","description":"Milliseconds that have elapsed since the task started being processed."},"status":{"type":"object","required":["code"],"properties":{"code":{"type":"integer","description":"Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)","enum":[10,20,30,40,50]}}},"options":{"type":"array","description":"List of options used to process this task","items":{"type":"object","required":["name","value"],"properties":{"name":{"type":"string","description":"Option name (example: \"odm_meshing-octreeDepth\")"},"value":{"type":"string","description":"Value (example: 9)"}}}},"imagesCount":{"type":"integer","description":"Number of images"},"progress":{"type":"float","description":"Percentage progress (estimated) of the task"},"output":{"type":"array","description":"Console output for the task (only if requested via ?output=)","items":{"type":"string"}}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/output":{"get":{"description":"Retrieves the console output of the OpenDroneMap's process. Useful for monitoring execution and to provide updates to the user.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"line","in":"query","description":"Optional line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. Defaults to 0 (retrieve all console output).","default":0,"required":false,"type":"integer"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Console Output","schema":{"type":"string"}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/download/{asset}":{"get":{"description":"Retrieves an asset (the output of OpenDroneMap's processing) associated with a task","tags":["task"],"produces":["application/zip"],"parameters":[{"name":"uuid","in":"path","type":"string","description":"UUID of the task","required":true},{"name":"asset","in":"path","type":"string","description":"Type of asset to download. Use \"all.zip\" for zip file containing all assets.","required":true,"enum":["all.zip","orthophoto.tif"]},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Asset File","schema":{"type":"file"}},"default":{"description":"Error message","schema":{"$ref":"#/definitions/Error"}}}}},"/task/cancel":{"post":{"description":"Cancels a task (stops its execution, or prevents it from being executed)","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/task/remove":{"post":{"description":"Removes a task and deletes all of its assets","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/task/restart":{"post":{"description":"Restarts a task that was previously canceled, that had failed to process or that successfully completed","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"options","in":"body","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options. Overrides the previous options set for this task.","required":false,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/options":{"get":{"description":"Retrieves the command line options that can be passed to process a task","parameters":[{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"tags":["server"],"responses":{"200":{"description":"Options","schema":{"type":"array","items":{"title":"Option","type":"object","required":["name","type","value","domain","help"],"properties":{"name":{"type":"string","description":"Command line option (exactly as it is passed to the OpenDroneMap process, minus the leading '--')"},"type":{"type":"string","description":"Datatype of the value of this option","enum":["int","float","string","bool"]},"value":{"type":"string","description":"Default value of this option"},"domain":{"type":"string","description":"Valid range of values (for example, \"positive integer\" or \"float > 0.0\")"},"help":{"type":"string","description":"Description of what this option does"}}}}}}}},"/info":{"get":{"description":"Retrieves information about this node","parameters":[{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"tags":["server"],"responses":{"200":{"description":"Info","schema":{"type":"object","required":["version","taskQueueCount","maxImages","engineVersion","engine"],"properties":{"version":{"type":"string","description":"Current API version"},"taskQueueCount":{"type":"integer","description":"Number of tasks currently being processed or waiting to be processed"},"availableMemory":{"type":"integer","description":"Amount of RAM available in bytes"},"totalMemory":{"type":"integer","description":"Amount of total RAM in the system in bytes"},"cpuCores":{"type":"integer","description":"Number of CPU cores (virtual)"},"maxImages":{"type":"integer","description":"Maximum number of images allowed for new tasks or null if there's no limit."},"maxParallelTasks":{"type":"integer","description":"Maximum number of tasks that can be processed simultaneously"},"engineVersion":{"type":"string","description":"Current version of processing engine"},"engine":{"type":"string","description":"Lowercase identifier of processing engine"}}}}}}},"/auth/info":{"get":{"description":"Retrieves login information for this node.","tags":["auth"],"responses":{"200":{"description":"LoginInformation","schema":{"type":"object","required":["message","loginUrl","registerUrl"],"properties":{"message":{"type":"string","description":"Message to be displayed to the user prior to login/registration. This might include instructions on how to register or login, or to communicate that authentication is not available."},"loginUrl":{"type":"string","description":"URL (absolute or relative) where to make a POST request to obtain a token, or null if login is disabled."},"registerUrl":{"type":"string","description":"URL (absolute or relative) where to make a POST request to register a user, or null if registration is disabled."}}}}}}},"/auth/login":{"post":{"description":"Retrieve a token from a username/password pair.","parameters":[{"name":"username","in":"body","description":"Username","required":true,"schema":{"type":"string"}},{"name":"password","in":"body","description":"Password","required":true,"type":"string"}],"responses":{"200":{"description":"Login Succeeded","schema":{"type":"object","required":["token"],"properties":{"token":{"type":"string","description":"Token to be passed as a query parameter to other API calls."}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/auth/register":{"post":{"description":"Register a new username/password.","parameters":[{"name":"username","in":"body","description":"Username","required":true,"schema":{"type":"string"}},{"name":"password","in":"body","description":"Password","required":true,"type":"string"}],"responses":{"200":{"description":"Response","schema":{"$ref":"#/definitions/Response"}}}}}},"definitions":{"Error":{"type":"object","required":["error"],"properties":{"error":{"type":"string","description":"Description of the error"}}},"Response":{"type":"object","required":["success"],"properties":{"success":{"type":"boolean","description":"true if the command succeeded, false otherwise"},"error":{"type":"string","description":"Error message if an error occured"}}}},"responses":{},"parameters":{},"securityDefinitions":{},"tags":[]} \ No newline at end of file +{"info":{"title":"NodeODM","version":"1.6.0","description":"REST API to access ODM","license":{"name":"GPL-3.0"},"contact":{"name":"Piero Toffanin"}},"consumes":["application/json"],"produces":["application/json","application/zip"],"basePath":"/","schemes":["http"],"swagger":"2.0","paths":{"/task/new/init":{"post":{"description":"Initialize the upload of a new task. If successful, a user can start uploading files via /task/new/upload. The task will not start until /task/new/commit is called.","tags":["task"],"parameters":[{"name":"name","in":"formData","description":"An optional name to be associated with the task","required":false,"type":"string"},{"name":"options","in":"formData","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options","required":false,"type":"string"},{"name":"skipPostProcessing","in":"formData","description":"When set, skips generation of map tiles, derivate assets, point cloud tiles.","required":false,"type":"boolean"},{"name":"webhook","in":"formData","description":"Optional URL to call when processing has ended (either successfully or unsuccessfully).","required":false,"type":"string"},{"name":"outputs","in":"formData","description":"An optional serialized JSON string of paths relative to the project directory that should be included in the all.zip result file, overriding the default behavior.","required":false,"type":"string"},{"name":"dateCreated","in":"formData","description":"An optional timestamp overriding the default creation date of the task.","required":false,"type":"integer"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"},{"name":"set-uuid","in":"header","description":"An optional UUID string that will be used as UUID for this task instead of generating a random one.","required":false,"type":"string"}],"responses":{"200":{"description":"Success","schema":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID of the newly created task"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/new/upload/{uuid}":{"post":{"description":"Adds one or more files to the task created via /task/new/init. It does not start the task. To start the task, call /task/new/commit.","tags":["task"],"consumes":["multipart/form-data"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"images","in":"formData","description":"Images to process, plus an optional GCP file (*.txt) and/or an optional seed file (seed.zip). If included, the GCP file should have .txt extension. If included, the seed archive pre-polulates the task directory with its contents.","required":true,"type":"file"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"File Received","schema":{"$ref":"#/definitions/Response"}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/new/commit/{uuid}":{"post":{"description":"Creates a new task for which images have been uploaded via /task/new/upload.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Success","schema":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID of the newly created task"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/new":{"post":{"description":"Creates a new task and places it at the end of the processing queue. For uploading really large tasks, see /task/new/init instead.","tags":["task"],"consumes":["multipart/form-data"],"parameters":[{"name":"images","in":"formData","description":"Images to process, plus an optional GCP file (*.txt) and/or an optional seed file (seed.zip). If included, the GCP file should have .txt extension. If included, the seed archive pre-polulates the task directory with its contents.","required":false,"type":"file"},{"name":"zipurl","in":"formData","description":"URL of the zip file containing the images to process, plus an optional GCP file. If included, the GCP file should have .txt extension","required":false,"type":"string"},{"name":"name","in":"formData","description":"An optional name to be associated with the task","required":false,"type":"string"},{"name":"options","in":"formData","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options","required":false,"type":"string"},{"name":"skipPostProcessing","in":"formData","description":"When set, skips generation of map tiles, derivate assets, point cloud tiles.","required":false,"type":"boolean"},{"name":"webhook","in":"formData","description":"Optional URL to call when processing has ended (either successfully or unsuccessfully).","required":false,"type":"string"},{"name":"outputs","in":"formData","description":"An optional serialized JSON string of paths relative to the project directory that should be included in the all.zip result file, overriding the default behavior.","required":false,"type":"string"},{"name":"dateCreated","in":"formData","description":"An optional timestamp overriding the default creation date of the task.","required":false,"type":"integer"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"},{"name":"set-uuid","in":"header","description":"An optional UUID string that will be used as UUID for this task instead of generating a random one.","required":false,"type":"string"}],"responses":{"200":{"description":"Success","schema":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID of the newly created task"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/list":{"get":{"description":"Gets the list of tasks available on this node.","tags":["task"],"parameters":[{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Task List","schema":{"title":"TaskList","type":"array","items":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID"}}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/info":{"get":{"description":"Gets information about this task, such as name, creation date, processing time, status, command line options and number of images being processed. See schema definition for a full list.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"},{"name":"with_output","in":"query","description":"Optionally retrieve the console output for this task. The parameter specifies the line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. By default no console output is added to the response.","default":0,"required":false,"type":"integer"}],"responses":{"200":{"description":"Task Information","schema":{"title":"TaskInfo","type":"object","required":["uuid","name","dateCreated","processingTime","status","options","imagesCount","progress"],"properties":{"uuid":{"type":"string","description":"UUID"},"name":{"type":"string","description":"Name"},"dateCreated":{"type":"integer","description":"Timestamp"},"processingTime":{"type":"integer","description":"Milliseconds that have elapsed since the task started being processed."},"status":{"type":"object","required":["code"],"properties":{"code":{"type":"integer","description":"Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)","enum":[10,20,30,40,50]}}},"options":{"type":"array","description":"List of options used to process this task","items":{"type":"object","required":["name","value"],"properties":{"name":{"type":"string","description":"Option name (example: \"odm_meshing-octreeDepth\")"},"value":{"type":"string","description":"Value (example: 9)"}}}},"imagesCount":{"type":"integer","description":"Number of images"},"progress":{"type":"float","description":"Percentage progress (estimated) of the task"},"output":{"type":"array","description":"Console output for the task (only if requested via ?output=)","items":{"type":"string"}}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/output":{"get":{"description":"Retrieves the console output of the OpenDroneMap's process. Useful for monitoring execution and to provide updates to the user.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"line","in":"query","description":"Optional line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. Defaults to 0 (retrieve all console output).","default":0,"required":false,"type":"integer"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Console Output","schema":{"type":"string"}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/download/{asset}":{"get":{"description":"Retrieves an asset (the output of OpenDroneMap's processing) associated with a task","tags":["task"],"produces":["application/zip"],"parameters":[{"name":"uuid","in":"path","type":"string","description":"UUID of the task","required":true},{"name":"asset","in":"path","type":"string","description":"Type of asset to download. Use \"all.zip\" for zip file containing all assets.","required":true,"enum":["all.zip","orthophoto.tif"]},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Asset File","schema":{"type":"file"}},"default":{"description":"Error message","schema":{"$ref":"#/definitions/Error"}}}}},"/task/cancel":{"post":{"description":"Cancels a task (stops its execution, or prevents it from being executed)","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/task/remove":{"post":{"description":"Removes a task and deletes all of its assets","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/task/restart":{"post":{"description":"Restarts a task that was previously canceled, that had failed to process or that successfully completed","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"options","in":"body","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options. Overrides the previous options set for this task.","required":false,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/options":{"get":{"description":"Retrieves the command line options that can be passed to process a task","parameters":[{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"tags":["server"],"responses":{"200":{"description":"Options","schema":{"type":"array","items":{"title":"Option","type":"object","required":["name","type","value","domain","help"],"properties":{"name":{"type":"string","description":"Command line option (exactly as it is passed to the OpenDroneMap process, minus the leading '--')"},"type":{"type":"string","description":"Datatype of the value of this option","enum":["int","float","string","bool"]},"value":{"type":"string","description":"Default value of this option"},"domain":{"type":"string","description":"Valid range of values (for example, \"positive integer\" or \"float > 0.0\")"},"help":{"type":"string","description":"Description of what this option does"}}}}}}}},"/info":{"get":{"description":"Retrieves information about this node","parameters":[{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"tags":["server"],"responses":{"200":{"description":"Info","schema":{"type":"object","required":["version","taskQueueCount","maxImages","engineVersion","engine"],"properties":{"version":{"type":"string","description":"Current API version"},"taskQueueCount":{"type":"integer","description":"Number of tasks currently being processed or waiting to be processed"},"availableMemory":{"type":"integer","description":"Amount of RAM available in bytes"},"totalMemory":{"type":"integer","description":"Amount of total RAM in the system in bytes"},"cpuCores":{"type":"integer","description":"Number of CPU cores (virtual)"},"maxImages":{"type":"integer","description":"Maximum number of images allowed for new tasks or null if there's no limit."},"maxParallelTasks":{"type":"integer","description":"Maximum number of tasks that can be processed simultaneously"},"engineVersion":{"type":"string","description":"Current version of processing engine"},"engine":{"type":"string","description":"Lowercase identifier of processing engine"}}}}}}},"/auth/info":{"get":{"description":"Retrieves login information for this node.","tags":["auth"],"responses":{"200":{"description":"LoginInformation","schema":{"type":"object","required":["message","loginUrl","registerUrl"],"properties":{"message":{"type":"string","description":"Message to be displayed to the user prior to login/registration. This might include instructions on how to register or login, or to communicate that authentication is not available."},"loginUrl":{"type":"string","description":"URL (absolute or relative) where to make a POST request to obtain a token, or null if login is disabled."},"registerUrl":{"type":"string","description":"URL (absolute or relative) where to make a POST request to register a user, or null if registration is disabled."}}}}}}},"/auth/login":{"post":{"description":"Retrieve a token from a username/password pair.","parameters":[{"name":"username","in":"body","description":"Username","required":true,"schema":{"type":"string"}},{"name":"password","in":"body","description":"Password","required":true,"type":"string"}],"responses":{"200":{"description":"Login Succeeded","schema":{"type":"object","required":["token"],"properties":{"token":{"type":"string","description":"Token to be passed as a query parameter to other API calls."}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/auth/register":{"post":{"description":"Register a new username/password.","parameters":[{"name":"username","in":"body","description":"Username","required":true,"schema":{"type":"string"}},{"name":"password","in":"body","description":"Password","required":true,"type":"string"}],"responses":{"200":{"description":"Response","schema":{"$ref":"#/definitions/Response"}}}}}},"definitions":{"Error":{"type":"object","required":["error"],"properties":{"error":{"type":"string","description":"Description of the error"}}},"Response":{"type":"object","required":["success"],"properties":{"success":{"type":"boolean","description":"true if the command succeeded, false otherwise"},"error":{"type":"string","description":"Error message if an error occured"}}}},"responses":{},"parameters":{},"securityDefinitions":{},"tags":[]} \ No newline at end of file diff --git a/index.js b/index.js index 0094c06..9808739 100644 --- a/index.js +++ b/index.js @@ -292,6 +292,44 @@ let getTaskFromUuid = (req, res, next) => { } else res.json({ error: `${req.params.uuid} not found` }); }; +/** @swagger + * /task/list: + * get: + * description: Gets the list of tasks available on this node. + * tags: [task] + * parameters: + * - + * name: token + * in: query + * description: 'Token required for authentication (when authentication is required).' + * required: false + * type: string + * responses: + * 200: + * description: Task List + * schema: + * title: TaskList + * type: array + * items: + * type: object + * required: [uuid] + * properties: + * uuid: + * type: string + * description: UUID + * default: + * description: Error + * schema: + * $ref: '#/definitions/Error' + */ +app.get('/task/list', authCheck, (req, res) => { + const tasks = []; + for (let uuid in taskManager.tasks){ + tasks.push({uuid}); + } + res.json(tasks); +}); + /** @swagger * /task/{uuid}/info: * get: @@ -863,6 +901,10 @@ if (config.test) { if (config.testDropUploads) logger.info("Uploads will drop at random"); } +if (!config.has7z){ + logger.warn("The 7z program is not installed, falling back to legacy (zipping will be slower)"); +} + let commands = [ cb => odmInfo.initialize(cb), cb => auth.initialize(cb), diff --git a/libs/Task.js b/libs/Task.js index eb78fd7..1fb07d7 100644 --- a/libs/Task.js +++ b/libs/Task.js @@ -22,12 +22,10 @@ const async = require('async'); const assert = require('assert'); const logger = require('./logger'); const fs = require('fs'); -const glob = require("glob"); const path = require('path'); const rmdir = require('rimraf'); const odmRunner = require('./odmRunner'); const processRunner = require('./processRunner'); -const archiver = require('archiver'); const Directories = require('./Directories'); const kill = require('tree-kill'); const S3 = require('./S3'); @@ -249,6 +247,40 @@ module.exports = class Task{ const postProcess = () => { const createZipArchive = (outputFilename, files) => { + return (done) => { + this.output.push(`Compressing ${outputFilename}\n`); + + const zipFile = path.resolve(this.getAssetsArchivePath(outputFilename)); + const sourcePath = !config.test ? + this.getProjectFolderPath() : + path.join("tests", "processing_results"); + + const pathsToArchive = []; + files.forEach(f => { + if (fs.existsSync(path.join(sourcePath, f))){ + pathsToArchive.push(f); + } + }); + + processRunner.sevenZip({ + destination: zipFile, + pathsToArchive, + cwd: sourcePath + }, (err, code, _) => { + if (err){ + logger.error(`Could not archive .zip file: ${err.message}`); + done(err); + }else{ + if (code === 0){ + this.updateProgress(97); + done(); + }else done(new Error(`Could not archive .zip file, 7z exited with code ${code}`)); + } + }); + }; + }; + + const createZipArchiveLegacy = (outputFilename, files) => { return (done) => { this.output.push(`Compressing ${outputFilename}\n`); @@ -327,7 +359,7 @@ module.exports = class Task{ this.runningProcesses.push( processRunner.runPostProcessingScript({ projectFolderPath: this.getProjectFolderPath() - }, (err, code, signal) => { + }, (err, code, _) => { if (err) done(err); else{ if (code === 0){ @@ -388,7 +420,9 @@ module.exports = class Task{ } if (!this.skipPostProcessing) tasks.push(runPostProcessingScript()); - tasks.push(createZipArchive('all.zip', allPaths)); + + const archiveFunc = config.has7z ? createZipArchive : createZipArchiveLegacy; + tasks.push(archiveFunc('all.zip', allPaths)); // Upload to S3 all paths + all.zip file (if config says so) if (S3.enabled()){ diff --git a/libs/processRunner.js b/libs/processRunner.js index 8d5fa1b..f6b0ab0 100644 --- a/libs/processRunner.js +++ b/libs/processRunner.js @@ -25,7 +25,7 @@ let logger = require('./logger'); let utils = require('./utils'); -function makeRunner(command, args, requiredOptions = [], outputTestFile = null){ +function makeRunner(command, args, requiredOptions = [], outputTestFile = null, skipOnTest = true){ return function(options, done, outputReceived){ for (let requiredOption of requiredOptions){ assert(options[requiredOption] !== undefined, `${requiredOption} must be defined`); @@ -36,14 +36,16 @@ function makeRunner(command, args, requiredOptions = [], outputTestFile = null){ logger.info(`About to run: ${command} ${commandArgs.join(" ")}`); - if (config.test){ + if (config.test && skipOnTest){ logger.info("Test mode is on, command will not execute"); if (outputTestFile){ fs.readFile(path.resolve(__dirname, outputTestFile), 'utf8', (err, text) => { if (!err){ - let lines = text.split("\n"); - lines.forEach(line => outputReceived(line)); + if (outputReceived !== undefined){ + let lines = text.split("\n"); + lines.forEach(line => outputReceived(line)); + } done(null, 0, null); }else{ @@ -62,20 +64,21 @@ function makeRunner(command, args, requiredOptions = [], outputTestFile = null){ const env = utils.clone(process.env); env.LD_LIBRARY_PATH = path.join(config.odm_path, "SuperBuild", "install", "lib"); - try{ - let childProcess = spawn(command, commandArgs, { env }); - childProcess - .on('exit', (code, signal) => done(null, code, signal)) - .on('error', done); - + let cwd = undefined; + if (options.cwd) cwd = options.cwd; + + let childProcess = spawn(command, commandArgs, { env, cwd }); + + childProcess + .on('exit', (code, signal) => done(null, code, signal)) + .on('error', done); + + if (outputReceived !== undefined){ childProcess.stdout.on('data', chunk => outputReceived(chunk.toString())); childProcess.stderr.on('data', chunk => outputReceived(chunk.toString())); - return childProcess; - }catch(e){ - // Catch errors such as ENOMEM - logger.warn(`Error: ${e.message}`); - done(e); } + + return childProcess; }; } @@ -84,5 +87,12 @@ module.exports = { function(options){ return [options.projectFolderPath]; }, - ["projectFolderPath"]) + ["projectFolderPath"]), + + sevenZip: makeRunner("7z", function(options){ + return ["a", "-r", "-bd", options.destination].concat(options.pathsToArchive); + }, + ["destination", "pathsToArchive", "cwd"], + null, + false) }; diff --git a/libs/taskNew.js b/libs/taskNew.js index 0e2efd0..af80543 100644 --- a/libs/taskNew.js +++ b/libs/taskNew.js @@ -48,6 +48,24 @@ const removeDirectory = function(dir, cb = () => {}){ }); }; +const assureUniqueFilename = (dstPath, filename, cb) => { + const dstFile = path.join(dstPath, filename); + fs.exists(dstFile, exists => { + if (!exists) cb(null, filename); + else{ + const parts = filename.split("."); + if (parts.length > 1){ + assureUniqueFilename(dstPath, + `${parts.slice(0, parts.length - 1).join(".")}_.${parts[parts.length - 1]}`, + cb); + }else{ + // Filename without extension? Strange.. + assureUniqueFilename(dstPath, filename + "_", cb); + } + } + }); +}; + const upload = multer({ storage: multer.diskStorage({ destination: (req, file, cb) => { @@ -65,7 +83,9 @@ const upload = multer({ filename: (req, file, cb) => { let filename = utils.sanitize(file.originalname); if (filename === "body.json") filename = "_body.json"; - cb(null, filename); + + let dstPath = path.join("tmp", req.id); + assureUniqueFilename(dstPath, filename, cb); } }) }); diff --git a/package.json b/package.json index 7552c15..0b9c28d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "NodeODM", - "version": "1.5.3", + "version": "1.6.0", "description": "REST API to access ODM", "main": "index.js", "scripts": { diff --git a/public/index.html b/public/index.html index 0eb3f44..13f62e8 100644 --- a/public/index.html +++ b/public/index.html @@ -141,7 +141,13 @@
-

No running tasks.

+
+ +
+
+ Loading task list... +
+

No running tasks.

Retrieving ...

@@ -208,7 +214,7 @@ - + diff --git a/public/js/main.js b/public/js/main.js index 9801987..9b6e53e 100644 --- a/public/js/main.js +++ b/public/js/main.js @@ -205,27 +205,34 @@ $(function() { } function TaskList() { - var uuids = JSON.parse(localStorage.getItem("odmTaskList") || "[]"); - if (Object.prototype.toString.call(uuids) !== "[object Array]") uuids = []; + var self = this; + var url = "/task/list?token=" + token; + this.error = ko.observable(""); + this.loading = ko.observable(true); + this.tasks = ko.observableArray(); - this.tasks = ko.observableArray($.map(uuids, function(uuid) { - return new Task(uuid); - })); + $.get(url) + .done(function(tasksJson) { + if (tasksJson.error){ + self.error(tasksJson.error); + }else{ + for (var i in tasksJson){ + self.tasks.push(new Task(tasksJson[i].uuid)); + } + } + }) + .fail(function() { + self.error(url + " is unreachable."); + }) + .always(function() { self.loading(false); }); } TaskList.prototype.add = function(task) { this.tasks.push(task); - this.saveTaskListToLocalStorage(); - }; - TaskList.prototype.saveTaskListToLocalStorage = function() { - localStorage.setItem("odmTaskList", JSON.stringify($.map(this.tasks(), function(task) { - return task.uuid; - }))); }; TaskList.prototype.remove = function(task) { this.tasks.remove(function(t) { return t === task; }); - this.saveTaskListToLocalStorage(); }; var codes = { diff --git a/scripts/postprocess.sh b/scripts/postprocess.sh index fc8503f..895d582 100755 --- a/scripts/postprocess.sh +++ b/scripts/postprocess.sh @@ -42,19 +42,20 @@ orthophoto_path="odm_orthophoto/odm_orthophoto.tif" if [ -e "$orthophoto_path" ]; then python "$script_path/gdal2tiles.py" $g2t_options $orthophoto_path orthophoto_tiles + + # Check for DEM tiles also + for dem_product in ${dem_products[@]}; do + colored_dem_path="odm_dem/""$dem_product""_colored_hillshade.tif" + if [ -e "$colored_dem_path" ]; then + python "$script_path/gdal2tiles.py" $g2t_options $colored_dem_path "$dem_product""_tiles" + else + echo "No $dem_product found at $colored_dem_path: will skip tiling" + fi + done else echo "No orthophoto found at $orthophoto_path: will skip tiling" fi -for dem_product in ${dem_products[@]}; do - colored_dem_path="odm_dem/""$dem_product""_colored_hillshade.tif" - if [ -e "$colored_dem_path" ]; then - python "$script_path/gdal2tiles.py" $g2t_options $colored_dem_path "$dem_product""_tiles" - else - echo "No $dem_product found at $colored_dem_path: will skip tiling" - fi -done - # Generate point cloud (if entwine or potreeconverter is available) pointcloud_input_path="" for path in "odm_georeferencing/odm_georeferenced_model.laz" \ @@ -71,12 +72,6 @@ for path in "odm_georeferencing/odm_georeferenced_model.laz" \ fi done -# Never generate point cloud tiles with split-merge workflows -if [ -e "submodels" ] && [ -e "entwine_pointcloud" ]; then - pointcloud_input_path="" - echo "Split-merge dataset with point cloud detected. No need to regenerate point cloud tiles." -fi - if [ ! -z "$pointcloud_input_path" ]; then # Convert the failsafe PLY point cloud to laz in odm_georeferencing # if necessary, otherwise it will not get zipped @@ -93,13 +88,12 @@ if [ ! -z "$pointcloud_input_path" ]; then fi if hash entwine 2>/dev/null; then - # Optionally cleanup previous results (from a restart) - if [ -e "entwine_pointcloud" ]; then - rm -fr "entwine_pointcloud" + if [ ! -e "entwine_pointcloud" ]; then + entwine build --threads $(nproc) --tmp "entwine_pointcloud-tmp" -i "$pointcloud_input_path" -o entwine_pointcloud + else + echo "Entwine point cloud is already built." fi - entwine build --threads $(nproc) --tmp "entwine_pointcloud-tmp" -i "$pointcloud_input_path" -o entwine_pointcloud - # Cleanup if [ -e "entwine_pointcloud-tmp" ]; then rm -fr "entwine_pointcloud-tmp"