diff --git a/config.js b/config.js index f86b957..4170e81 100644 --- a/config.js +++ b/config.js @@ -33,6 +33,7 @@ Options: -d, --deamonize Set process to run as a deamon --parallel_queue_processing Number of simultaneous processing tasks (default: 2) --cleanup_tasks_after Number of minutes that elapse before deleting finished and canceled tasks (default: 2880) + --cleanup_uploads_after Number of minutes that elapse before deleting unfinished uploads. Set this value to the maximum time you expect a dataset to be uploaded. (default: 2880) --test Enable test mode. In test mode, no commands are sent to OpenDroneMap. This can be useful during development or testing (default: false) --test_skip_orthophotos If test mode is enabled, skip orthophoto results when generating assets. (default: false) --test_skip_dems If test mode is enabled, skip dems results when generating assets. (default: false) @@ -90,6 +91,7 @@ config.port = parseInt(argv.port || argv.p || fromConfigFile("port", process.env config.deamon = argv.deamonize || argv.d || fromConfigFile("daemon", false); config.parallelQueueProcessing = argv.parallel_queue_processing || fromConfigFile("parallelQueueProcessing", 2); config.cleanupTasksAfter = parseInt(argv.cleanup_tasks_after || fromConfigFile("cleanupTasksAfter", 2880)); +config.cleanupUploadsAfter = parseInt(argv.cleanup_uploads_after || fromConfigFile("cleanupUploadsAfter", 2880)); config.test = argv.test || fromConfigFile("test", false); config.testSkipOrthophotos = argv.test_skip_orthophotos || fromConfigFile("testSkipOrthophotos", false); config.testSkipDems = argv.test_skip_dems || fromConfigFile("testSkipDems", false); diff --git a/docs/index.adoc b/docs/index.adoc index 1060c84..6b0b507 100644 --- a/docs/index.adoc +++ b/docs/index.adoc @@ -8,7 +8,7 @@ REST API to access ODM === Version information [%hardbreaks] -_Version_ : 1.3.1 +_Version_ : 1.4.0 === Contact information @@ -281,7 +281,7 @@ _required_|UUID of the task|string| === POST /task/new ==== Description -Creates a new task and places it at the end of the processing queue +Creates a new task and places it at the end of the processing queue. For uploading really large tasks, see /task/new/init instead. ==== Parameters @@ -301,6 +301,8 @@ _optional_|An optional name to be associated with the task|string| _optional_|Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, …]. For example, [{"name":"cmvs-maxImages","value":"500"},{"name":"time","value":true}]. For a list of all options, call /options|string| |*FormData*|*skipPostProcessing* + _optional_|When set, skips generation of map tiles, derivate assets, point cloud tiles.|boolean| +|*FormData*|*webhook* + +_optional_|Optional URL to call when processing has ended (either successfully or unsuccessfully).|string| |*FormData*|*zipurl* + _optional_|URL of the zip file containing the images to process, plus an optional GCP file. If included, the GCP file should have .txt extension|string| |=== @@ -336,6 +338,143 @@ _required_|UUID of the newly created task|string * task +[[_task_new_commit_uuid_post]] +=== POST /task/new/commit/{uuid} + +==== Description +Creates a new task for which images have been uploaded via /task/new/upload. + + +==== Parameters + +[options="header", cols=".^2,.^3,.^9,.^4,.^2"] +|=== +|Type|Name|Description|Schema|Default +|*Path*|*uuid* + +_required_|UUID of the task|string| +|*Query*|*token* + +_optional_|Token required for authentication (when authentication is required).|string| +|=== + + +==== Responses + +[options="header", cols=".^2,.^14,.^4"] +|=== +|HTTP Code|Description|Schema +|*200*|Success|<<_task_new_commit_uuid_post_response_200,Response 200>> +|*default*|Error|<<_error,Error>> +|=== + +[[_task_new_commit_uuid_post_response_200]] +*Response 200* + +[options="header", cols=".^3,.^11,.^4"] +|=== +|Name|Description|Schema +|*uuid* + +_required_|UUID of the newly created task|string +|=== + + +==== Tags + +* task + + +[[_task_new_init_post]] +=== POST /task/new/init + +==== Description +Initialize the upload of a new task. If successful, a user can start uploading files via /task/new/upload. The task will not start until /task/new/commit is called. + + +==== Parameters + +[options="header", cols=".^2,.^3,.^9,.^4,.^2"] +|=== +|Type|Name|Description|Schema|Default +|*Header*|*set-uuid* + +_optional_|An optional UUID string that will be used as UUID for this task instead of generating a random one.|string| +|*Query*|*token* + +_optional_|Token required for authentication (when authentication is required).|string| +|*FormData*|*name* + +_optional_|An optional name to be associated with the task|string| +|*FormData*|*options* + +_optional_|Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, …]. For example, [{"name":"cmvs-maxImages","value":"500"},{"name":"time","value":true}]. For a list of all options, call /options|string| +|*FormData*|*skipPostProcessing* + +_optional_|When set, skips generation of map tiles, derivate assets, point cloud tiles.|boolean| +|*FormData*|*webhook* + +_optional_|Optional URL to call when processing has ended (either successfully or unsuccessfully).|string| +|=== + + +==== Responses + +[options="header", cols=".^2,.^14,.^4"] +|=== +|HTTP Code|Description|Schema +|*200*|Success|<<_task_new_init_post_response_200,Response 200>> +|*default*|Error|<<_error,Error>> +|=== + +[[_task_new_init_post_response_200]] +*Response 200* + +[options="header", cols=".^3,.^11,.^4"] +|=== +|Name|Description|Schema +|*uuid* + +_required_|UUID of the newly created task|string +|=== + + +==== Tags + +* task + + +[[_task_new_upload_uuid_post]] +=== POST /task/new/upload/{uuid} + +==== Description +Adds one or more files to the task created via /task/new/init. It does not start the task. To start the task, call /task/new/commit. + + +==== Parameters + +[options="header", cols=".^2,.^3,.^9,.^4,.^2"] +|=== +|Type|Name|Description|Schema|Default +|*Path*|*uuid* + +_required_|UUID of the task|string| +|*Query*|*token* + +_optional_|Token required for authentication (when authentication is required).|string| +|*FormData*|*images* + +_required_|Images to process, plus an optional GCP file. If included, the GCP file should have .txt extension|file| +|=== + + +==== Responses + +[options="header", cols=".^2,.^14,.^4"] +|=== +|HTTP Code|Description|Schema +|*200*|File Received|<<_response,Response>> +|*default*|Error|<<_error,Error>> +|=== + + +==== Consumes + +* `multipart/form-data` + + +==== Tags + +* task + + [[_task_remove_post]] === POST /task/remove diff --git a/docs/swagger.json b/docs/swagger.json index 0563f64..b819577 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -1 +1 @@ -{"info":{"title":"node-opendronemap","version":"1.3.1","description":"REST API to access ODM","license":{"name":"GPL-3.0"},"contact":{"name":"Piero Toffanin"}},"consumes":["application/json"],"produces":["application/json","application/zip"],"basePath":"/","schemes":["http"],"swagger":"2.0","paths":{"/task/new":{"post":{"description":"Creates a new task and places it at the end of the processing queue","tags":["task"],"consumes":["multipart/form-data"],"parameters":[{"name":"images","in":"formData","description":"Images to process, plus an optional GCP file. If included, the GCP file should have .txt extension","required":false,"type":"file"},{"name":"zipurl","in":"formData","description":"URL of the zip file containing the images to process, plus an optional GCP file. If included, the GCP file should have .txt extension","required":false,"type":"string"},{"name":"name","in":"formData","description":"An optional name to be associated with the task","required":false,"type":"string"},{"name":"options","in":"formData","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options","required":false,"type":"string"},{"name":"skipPostProcessing","in":"formData","description":"When set, skips generation of map tiles, derivate assets, point cloud tiles.","required":false,"type":"boolean"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"},{"name":"set-uuid","in":"header","description":"An optional UUID string that will be used as UUID for this task instead of generating a random one.","required":false,"type":"string"}],"responses":{"200":{"description":"Success","schema":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID of the newly created task"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/info":{"get":{"description":"Gets information about this task, such as name, creation date, processing time, status, command line options and number of images being processed. See schema definition for a full list.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"options","in":"formData","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options","required":false,"type":"string"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Task Information","schema":{"title":"TaskInfo","type":"object","required":["uuid","name","dateCreated","processingTime","status","options","imagesCount"],"properties":{"uuid":{"type":"string","description":"UUID"},"name":{"type":"string","description":"Name"},"dateCreated":{"type":"integer","description":"Timestamp"},"processingTime":{"type":"integer","description":"Milliseconds that have elapsed since the task started being processed."},"status":{"type":"integer","description":"Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)","enum":[10,20,30,40,50]},"options":{"type":"array","description":"List of options used to process this task","items":{"type":"object","required":["name","value"],"properties":{"name":{"type":"string","description":"Option name (example: \"odm_meshing-octreeDepth\")"},"value":{"type":"string","description":"Value (example: 9)"}}}},"imagesCount":{"type":"integer","description":"Number of images"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/output":{"get":{"description":"Retrieves the console output of the OpenDroneMap's process. Useful for monitoring execution and to provide updates to the user.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"line","in":"query","description":"Optional line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. Defaults to 0 (retrieve all console output).","default":0,"required":false,"type":"integer"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Console Output","schema":{"type":"string"}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/download/{asset}":{"get":{"description":"Retrieves an asset (the output of OpenDroneMap's processing) associated with a task","tags":["task"],"produces":["application/zip"],"parameters":[{"name":"uuid","in":"path","type":"string","description":"UUID of the task","required":true},{"name":"asset","in":"path","type":"string","description":"Type of asset to download. Use \"all.zip\" for zip file containing all assets.","required":true,"enum":["all.zip","orthophoto.tif"]},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Asset File","schema":{"type":"file"}},"default":{"description":"Error message","schema":{"$ref":"#/definitions/Error"}}}}},"/task/cancel":{"post":{"description":"Cancels a task (stops its execution, or prevents it from being executed)","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/task/remove":{"post":{"description":"Removes a task and deletes all of its assets","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/task/restart":{"post":{"description":"Restarts a task that was previously canceled, that had failed to process or that successfully completed","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"options","in":"body","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options. Overrides the previous options set for this task.","required":false,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/options":{"get":{"description":"Retrieves the command line options that can be passed to process a task","parameters":[{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"tags":["server"],"responses":{"200":{"description":"Options","schema":{"type":"array","items":{"title":"Option","type":"object","required":["name","type","value","domain","help"],"properties":{"name":{"type":"string","description":"Command line option (exactly as it is passed to the OpenDroneMap process, minus the leading '--')"},"type":{"type":"string","description":"Datatype of the value of this option","enum":["int","float","string","bool"]},"value":{"type":"string","description":"Default value of this option"},"domain":{"type":"string","description":"Valid range of values (for example, \"positive integer\" or \"float > 0.0\")"},"help":{"type":"string","description":"Description of what this option does"}}}}}}}},"/info":{"get":{"description":"Retrieves information about this node","parameters":[{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"tags":["server"],"responses":{"200":{"description":"Info","schema":{"type":"object","required":["version","taskQueueCount"],"properties":{"version":{"type":"string","description":"Current API version"},"taskQueueCount":{"type":"integer","description":"Number of tasks currently being processed or waiting to be processed"},"availableMemory":{"type":"integer","description":"Amount of RAM available in bytes"},"totalMemory":{"type":"integer","description":"Amount of total RAM in the system in bytes"},"cpuCores":{"type":"integer","description":"Number of CPU cores (virtual)"},"maxImages":{"type":"integer","description":"Maximum number of images allowed for new tasks or null if there's no limit."},"maxParallelTasks":{"type":"integer","description":"Maximum number of tasks that can be processed simultaneously"},"odmVersion":{"type":"string","description":"Current version of ODM"}}}}}}},"/auth/info":{"get":{"description":"Retrieves login information for this node.","tags":["auth"],"responses":{"200":{"description":"LoginInformation","schema":{"type":"object","required":["message","loginUrl","registerUrl"],"properties":{"message":{"type":"string","description":"Message to be displayed to the user prior to login/registration. This might include instructions on how to register or login, or to communicate that authentication is not available."},"loginUrl":{"type":"string","description":"URL (absolute or relative) where to make a POST request to obtain a token, or null if login is disabled."},"registerUrl":{"type":"string","description":"URL (absolute or relative) where to make a POST request to register a user, or null if registration is disabled."}}}}}}},"/auth/login":{"post":{"description":"Retrieve a token from a username/password pair.","parameters":[{"name":"username","in":"body","description":"Username","required":true,"schema":{"type":"string"}},{"name":"password","in":"body","description":"Password","required":true,"type":"string"}],"responses":{"200":{"description":"Login Succeeded","schema":{"type":"object","required":["token"],"properties":{"token":{"type":"string","description":"Token to be passed as a query parameter to other API calls."}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/auth/register":{"post":{"description":"Register a new username/password.","parameters":[{"name":"username","in":"body","description":"Username","required":true,"schema":{"type":"string"}},{"name":"password","in":"body","description":"Password","required":true,"type":"string"}],"responses":{"200":{"description":"Response","schema":{"$ref":"#/definitions/Response"}}}}}},"definitions":{"Error":{"type":"object","required":["error"],"properties":{"error":{"type":"string","description":"Description of the error"}}},"Response":{"type":"object","required":["success"],"properties":{"success":{"type":"boolean","description":"true if the command succeeded, false otherwise"},"error":{"type":"string","description":"Error message if an error occured"}}}},"responses":{},"parameters":{},"securityDefinitions":{},"tags":[]} \ No newline at end of file +{"info":{"title":"node-opendronemap","version":"1.4.0","description":"REST API to access ODM","license":{"name":"GPL-3.0"},"contact":{"name":"Piero Toffanin"}},"consumes":["application/json"],"produces":["application/json","application/zip"],"basePath":"/","schemes":["http"],"swagger":"2.0","paths":{"/task/new/init":{"post":{"description":"Initialize the upload of a new task. If successful, a user can start uploading files via /task/new/upload. The task will not start until /task/new/commit is called.","tags":["task"],"parameters":[{"name":"name","in":"formData","description":"An optional name to be associated with the task","required":false,"type":"string"},{"name":"options","in":"formData","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options","required":false,"type":"string"},{"name":"skipPostProcessing","in":"formData","description":"When set, skips generation of map tiles, derivate assets, point cloud tiles.","required":false,"type":"boolean"},{"name":"webhook","in":"formData","description":"Optional URL to call when processing has ended (either successfully or unsuccessfully).","required":false,"type":"string"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"},{"name":"set-uuid","in":"header","description":"An optional UUID string that will be used as UUID for this task instead of generating a random one.","required":false,"type":"string"}],"responses":{"200":{"description":"Success","schema":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID of the newly created task"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/new/upload/{uuid}":{"post":{"description":"Adds one or more files to the task created via /task/new/init. It does not start the task. To start the task, call /task/new/commit.","tags":["task"],"consumes":["multipart/form-data"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"images","in":"formData","description":"Images to process, plus an optional GCP file. If included, the GCP file should have .txt extension","required":true,"type":"file"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"File Received","schema":{"$ref":"#/definitions/Response"}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/new/commit/{uuid}":{"post":{"description":"Creates a new task for which images have been uploaded via /task/new/upload.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Success","schema":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID of the newly created task"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/new":{"post":{"description":"Creates a new task and places it at the end of the processing queue. For uploading really large tasks, see /task/new/init instead.","tags":["task"],"consumes":["multipart/form-data"],"parameters":[{"name":"images","in":"formData","description":"Images to process, plus an optional GCP file. If included, the GCP file should have .txt extension","required":false,"type":"file"},{"name":"zipurl","in":"formData","description":"URL of the zip file containing the images to process, plus an optional GCP file. If included, the GCP file should have .txt extension","required":false,"type":"string"},{"name":"name","in":"formData","description":"An optional name to be associated with the task","required":false,"type":"string"},{"name":"options","in":"formData","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options","required":false,"type":"string"},{"name":"skipPostProcessing","in":"formData","description":"When set, skips generation of map tiles, derivate assets, point cloud tiles.","required":false,"type":"boolean"},{"name":"webhook","in":"formData","description":"Optional URL to call when processing has ended (either successfully or unsuccessfully).","required":false,"type":"string"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"},{"name":"set-uuid","in":"header","description":"An optional UUID string that will be used as UUID for this task instead of generating a random one.","required":false,"type":"string"}],"responses":{"200":{"description":"Success","schema":{"type":"object","required":["uuid"],"properties":{"uuid":{"type":"string","description":"UUID of the newly created task"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/info":{"get":{"description":"Gets information about this task, such as name, creation date, processing time, status, command line options and number of images being processed. See schema definition for a full list.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"options","in":"formData","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options","required":false,"type":"string"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Task Information","schema":{"title":"TaskInfo","type":"object","required":["uuid","name","dateCreated","processingTime","status","options","imagesCount"],"properties":{"uuid":{"type":"string","description":"UUID"},"name":{"type":"string","description":"Name"},"dateCreated":{"type":"integer","description":"Timestamp"},"processingTime":{"type":"integer","description":"Milliseconds that have elapsed since the task started being processed."},"status":{"type":"integer","description":"Status code (10 = QUEUED, 20 = RUNNING, 30 = FAILED, 40 = COMPLETED, 50 = CANCELED)","enum":[10,20,30,40,50]},"options":{"type":"array","description":"List of options used to process this task","items":{"type":"object","required":["name","value"],"properties":{"name":{"type":"string","description":"Option name (example: \"odm_meshing-octreeDepth\")"},"value":{"type":"string","description":"Value (example: 9)"}}}},"imagesCount":{"type":"integer","description":"Number of images"}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/output":{"get":{"description":"Retrieves the console output of the OpenDroneMap's process. Useful for monitoring execution and to provide updates to the user.","tags":["task"],"parameters":[{"name":"uuid","in":"path","description":"UUID of the task","required":true,"type":"string"},{"name":"line","in":"query","description":"Optional line number that the console output should be truncated from. For example, passing a value of 100 will retrieve the console output starting from line 100. Defaults to 0 (retrieve all console output).","default":0,"required":false,"type":"integer"},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Console Output","schema":{"type":"string"}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/task/{uuid}/download/{asset}":{"get":{"description":"Retrieves an asset (the output of OpenDroneMap's processing) associated with a task","tags":["task"],"produces":["application/zip"],"parameters":[{"name":"uuid","in":"path","type":"string","description":"UUID of the task","required":true},{"name":"asset","in":"path","type":"string","description":"Type of asset to download. Use \"all.zip\" for zip file containing all assets.","required":true,"enum":["all.zip","orthophoto.tif"]},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Asset File","schema":{"type":"file"}},"default":{"description":"Error message","schema":{"$ref":"#/definitions/Error"}}}}},"/task/cancel":{"post":{"description":"Cancels a task (stops its execution, or prevents it from being executed)","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/task/remove":{"post":{"description":"Removes a task and deletes all of its assets","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/task/restart":{"post":{"description":"Restarts a task that was previously canceled, that had failed to process or that successfully completed","parameters":[{"name":"uuid","in":"body","description":"UUID of the task","required":true,"schema":{"type":"string"}},{"name":"options","in":"body","description":"Serialized JSON string of the options to use for processing, as an array of the format: [{name: option1, value: value1}, {name: option2, value: value2}, ...]. For example, [{\"name\":\"cmvs-maxImages\",\"value\":\"500\"},{\"name\":\"time\",\"value\":true}]. For a list of all options, call /options. Overrides the previous options set for this task.","required":false,"schema":{"type":"string"}},{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"responses":{"200":{"description":"Command Received","schema":{"$ref":"#/definitions/Response"}}}}},"/options":{"get":{"description":"Retrieves the command line options that can be passed to process a task","parameters":[{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"tags":["server"],"responses":{"200":{"description":"Options","schema":{"type":"array","items":{"title":"Option","type":"object","required":["name","type","value","domain","help"],"properties":{"name":{"type":"string","description":"Command line option (exactly as it is passed to the OpenDroneMap process, minus the leading '--')"},"type":{"type":"string","description":"Datatype of the value of this option","enum":["int","float","string","bool"]},"value":{"type":"string","description":"Default value of this option"},"domain":{"type":"string","description":"Valid range of values (for example, \"positive integer\" or \"float > 0.0\")"},"help":{"type":"string","description":"Description of what this option does"}}}}}}}},"/info":{"get":{"description":"Retrieves information about this node","parameters":[{"name":"token","in":"query","description":"Token required for authentication (when authentication is required).","required":false,"type":"string"}],"tags":["server"],"responses":{"200":{"description":"Info","schema":{"type":"object","required":["version","taskQueueCount"],"properties":{"version":{"type":"string","description":"Current API version"},"taskQueueCount":{"type":"integer","description":"Number of tasks currently being processed or waiting to be processed"},"availableMemory":{"type":"integer","description":"Amount of RAM available in bytes"},"totalMemory":{"type":"integer","description":"Amount of total RAM in the system in bytes"},"cpuCores":{"type":"integer","description":"Number of CPU cores (virtual)"},"maxImages":{"type":"integer","description":"Maximum number of images allowed for new tasks or null if there's no limit."},"maxParallelTasks":{"type":"integer","description":"Maximum number of tasks that can be processed simultaneously"},"odmVersion":{"type":"string","description":"Current version of ODM"}}}}}}},"/auth/info":{"get":{"description":"Retrieves login information for this node.","tags":["auth"],"responses":{"200":{"description":"LoginInformation","schema":{"type":"object","required":["message","loginUrl","registerUrl"],"properties":{"message":{"type":"string","description":"Message to be displayed to the user prior to login/registration. This might include instructions on how to register or login, or to communicate that authentication is not available."},"loginUrl":{"type":"string","description":"URL (absolute or relative) where to make a POST request to obtain a token, or null if login is disabled."},"registerUrl":{"type":"string","description":"URL (absolute or relative) where to make a POST request to register a user, or null if registration is disabled."}}}}}}},"/auth/login":{"post":{"description":"Retrieve a token from a username/password pair.","parameters":[{"name":"username","in":"body","description":"Username","required":true,"schema":{"type":"string"}},{"name":"password","in":"body","description":"Password","required":true,"type":"string"}],"responses":{"200":{"description":"Login Succeeded","schema":{"type":"object","required":["token"],"properties":{"token":{"type":"string","description":"Token to be passed as a query parameter to other API calls."}}}},"default":{"description":"Error","schema":{"$ref":"#/definitions/Error"}}}}},"/auth/register":{"post":{"description":"Register a new username/password.","parameters":[{"name":"username","in":"body","description":"Username","required":true,"schema":{"type":"string"}},{"name":"password","in":"body","description":"Password","required":true,"type":"string"}],"responses":{"200":{"description":"Response","schema":{"$ref":"#/definitions/Response"}}}}}},"definitions":{"Error":{"type":"object","required":["error"],"properties":{"error":{"type":"string","description":"Description of the error"}}},"Response":{"type":"object","required":["success"],"properties":{"success":{"type":"boolean","description":"true if the command succeeded, false otherwise"},"error":{"type":"string","description":"Error message if an error occured"}}}},"responses":{},"parameters":{},"securityDefinitions":{},"tags":[]} \ No newline at end of file diff --git a/index.js b/index.js index df6fdea..3dc957c 100644 --- a/index.js +++ b/index.js @@ -29,6 +29,7 @@ const express = require('express'); const app = express(); const bodyParser = require('body-parser'); +const multer = require('multer'); const TaskManager = require('./libs/TaskManager'); const odmInfo = require('./libs/odmInfo'); @@ -39,21 +40,10 @@ const auth = require('./libs/auth/factory').fromConfig(config); const authCheck = auth.getMiddleware(); const taskNew = require('./libs/taskNew'); -// zip files -let request = require('request'); - -let download = function(uri, filename, callback) { - request.head(uri, function(err, res, body) { - if (err) callback(err); - else{ - request(uri).pipe(fs.createWriteStream(filename)).on('close', callback); - } - }); -}; - app.use(express.static('public')); app.use('/swagger.json', express.static('docs/swagger.json')); +const formDataParser = multer().none(); const urlEncodedBodyParser = bodyParser.urlencoded({extended: false}); let taskManager; @@ -65,6 +55,7 @@ let server; * description: Initialize the upload of a new task. If successful, a user can start uploading files via /task/new/upload. The task will not start until /task/new/commit is called. * tags: [task] * parameters: + * - * name: name * in: formData * description: An optional name to be associated with the task @@ -115,21 +106,85 @@ let server; * schema: * $ref: '#/definitions/Error' */ -app.post('/task/new/init', authCheck, taskNew.assignUUID, (req, res) => { - -}); +app.post('/task/new/init', authCheck, taskNew.assignUUID, formDataParser, taskNew.handleInit); -app.post('/task/new/upload/:uuid', authCheck, (req, res) => { -}); +/** @swagger + * /task/new/upload/{uuid}: + * post: + * description: Adds one or more files to the task created via /task/new/init. It does not start the task. To start the task, call /task/new/commit. + * tags: [task] + * consumes: + * - multipart/form-data + * parameters: + * - + * name: uuid + * in: path + * description: UUID of the task + * required: true + * type: string + * - + * name: images + * in: formData + * description: Images to process, plus an optional GCP file. If included, the GCP file should have .txt extension + * required: true + * type: file + * - + * name: token + * in: query + * description: 'Token required for authentication (when authentication is required).' + * required: false + * type: string + * responses: + * 200: + * description: File Received + * schema: + * $ref: "#/definitions/Response" + * default: + * description: Error + * schema: + * $ref: '#/definitions/Error' + */ +app.post('/task/new/upload/:uuid', authCheck, taskNew.getUUID, taskNew.uploadImages, taskNew.handleUpload); -app.post('/task/new/commit/:uuid', authCheck, (req, res) => { - -}); +/** @swagger + * /task/new/commit/{uuid}: + * post: + * description: Creates a new task for which images have been uploaded via /task/new/upload. + * tags: [task] + * parameters: + * - + * name: uuid + * in: path + * description: UUID of the task + * required: true + * type: string + * - + * name: token + * in: query + * description: 'Token required for authentication (when authentication is required).' + * required: false + * type: string + * responses: + * 200: + * description: Success + * schema: + * type: object + * required: [uuid] + * properties: + * uuid: + * type: string + * description: UUID of the newly created task + * default: + * description: Error + * schema: + * $ref: '#/definitions/Error' + */ +app.post('/task/new/commit/:uuid', authCheck, taskNew.getUUID, taskNew.handleCommit, taskNew.createTask); /** @swagger * /task/new: * post: - * description: Creates a new task and places it at the end of the processing queue + * description: Creates a new task and places it at the end of the processing queue. For uploading really large tasks, see /task/new/init instead. * tags: [task] * consumes: * - multipart/form-data @@ -198,10 +253,12 @@ app.post('/task/new/commit/:uuid', authCheck, (req, res) => { * $ref: '#/definitions/Error' */ app.post('/task/new', authCheck, taskNew.assignUUID, taskNew.uploadImages, (req, res, next) => { + console.log(req.body); + req.body = req.body || {}; if ((!req.files || req.files.length === 0) && !req.body.zipurl) req.error = "Need at least 1 file or a zip file url."; else if (config.maxImages && req.files && req.files.length > config.maxImages) req.error = `${req.files.length} images uploaded, but this node can only process up to ${config.maxImages}.`; next(); -}, taskNew.handleTaskNew); +}, taskNew.createTask); let getTaskFromUuid = (req, res, next) => { let task = taskManager.find(req.params.uuid); diff --git a/libs/TaskManager.js b/libs/TaskManager.js index db9622c..e01ec48 100644 --- a/libs/TaskManager.js +++ b/libs/TaskManager.js @@ -30,6 +30,7 @@ const Directories = require('./Directories'); const TASKS_DUMP_FILE = path.join(Directories.data, "tasks.json"); const CLEANUP_TASKS_IF_OLDER_THAN = 1000 * 60 * config.cleanupTasksAfter; // minutes +const CLEANUP_STALE_UPLOADS_AFTER = 1000 * 60 * config.cleanupUploadsAfter; // minutes let taskManager; @@ -42,6 +43,7 @@ class TaskManager{ cb => this.restoreTaskListFromDump(cb), cb => this.removeOldTasks(cb), cb => this.removeOrphanedDirectories(cb), + cb => this.removeStaleUploads(cb), cb => { this.processNextTask(); cb(); @@ -51,6 +53,7 @@ class TaskManager{ schedule.scheduleJob('0 * * * *', () => { this.removeOldTasks(); this.dumpTaskList(); + this.removeStaleUploads(); }); cb(); @@ -84,7 +87,6 @@ class TaskManager{ // Removes directories that don't have a corresponding // task associated with it (maybe as a cause of an abrupt exit) - // TODO: do not delete /task/new/init directories!!! removeOrphanedDirectories(done){ logger.info("Checking for orphaned directories to be removed..."); @@ -104,6 +106,30 @@ class TaskManager{ }); } + removeStaleUploads(done){ + logger.info("Checking for stale uploads..."); + fs.readdir("tmp", (err, entries) => { + if (err) done(err); + else{ + const now = new Date(); + async.eachSeries(entries, (entry, cb) => { + let dirPath = path.join("tmp", entry); + if (entry.match(/^[\w\d]+\-[\w\d]+\-[\w\d]+\-[\w\d]+\-[\w\d]+$/)){ + fs.stat(dirPath, (err, stats) => { + if (err) cb(err); + else{ + if (stats.isDirectory() && stats.ctime.getTime() + CLEANUP_STALE_UPLOADS_AFTER < now.getTime()){ + logger.info(`Found stale upload directory: ${entry}, removing...`); + rmdir(dirPath, cb); + }else cb(); + } + }); + }else cb(); + }, done); + } + }); + } + // Load tasks that already exists (if any) restoreTaskListFromDump(done){ fs.readFile(TASKS_DUMP_FILE, (err, data) => { diff --git a/libs/taskNew.js b/libs/taskNew.js index 0d84735..9620ba7 100644 --- a/libs/taskNew.js +++ b/libs/taskNew.js @@ -27,6 +27,25 @@ const Directories = require('./Directories'); const unzip = require('node-unzip-2'); const mv = require('mv'); const Task = require('./Task'); +const async = require('async'); +const odmInfo = require('./odmInfo'); +const request = require('request'); + +const download = function(uri, filename, callback) { + request.head(uri, function(err, res, body) { + if (err) callback(err); + else{ + request(uri).pipe(fs.createWriteStream(filename)).on('close', callback); + } + }); +}; + +const removeDirectory = function(dir, cb = () => {}){ + fs.stat(dir, (err, stats) => { + if (!err && stats.isDirectory()) rmdir(dir, cb); // ignore errors, don't wait + else cb(err); + }); +}; const upload = multer({ storage: multer.diskStorage({ @@ -43,7 +62,9 @@ const upload = multer({ }); }, filename: (req, file, cb) => { - cb(null, file.originalname); + let filename = file.originalname; + if (filename === "body.json") filename = "_body.json"; + cb(null, filename); } }) }); @@ -68,30 +89,115 @@ module.exports = { } }, - uploadImages: upload.array("images"), + getUUID: (req, res, next) => { + req.id = req.params.uuid; + if (!req.id) res.json({error: `Invalid uuid (not set)`}); - setupFiles: (req, res, next) => { - // populate req.id (here or somehwere else) - // populate req.files from directory - // populate req.body from metadata file + const srcPath = path.join("tmp", req.id); + const bodyFile = path.join(srcPath, "body.json"); + fs.access(bodyFile, fs.F_OK, err => { + if (err) res.json({error: `Invalid uuid (not found)`}); + else next(); + }); }, - handleTaskNew: (req, res) => { - // TODO: consider doing the file moving in the background - // and return a response more quickly instead of a long timeout. - req.setTimeout(1000 * 60 * 20); + uploadImages: upload.array("images"), - let srcPath = path.join("tmp", req.id); + handleUpload: (req, res) => { + // IMPROVEMENT: check files count limits ahead of handleTaskNew + if (req.files && req.files.length > 0){ + res.json({success: true}); + }else{ + res.json({error: "Need at least 1 file."}); + } + }, + + handleCommit: (req, res, next) => { + const srcPath = path.join("tmp", req.id); + const bodyFile = path.join(srcPath, "body.json"); + + async.series([ + cb => { + fs.readFile(bodyFile, 'utf8', (err, data) => { + if (err) cb(err); + else{ + try{ + const body = JSON.parse(data); + fs.unlink(bodyFile, err => { + if (err) cb(err); + else cb(null, body); + }); + }catch(e){ + cb("Malformed body.json"); + } + } + }); + }, + cb => fs.readdir(srcPath, cb), + ], (err, [ body, files ]) => { + if (err) res.json({error: err.message}); + else{ + req.body = body; + req.files = files; + next(); + } + }); + }, + + handleInit: (req, res) => { + console.log(req.body); + req.body = req.body || {}; + + const srcPath = path.join("tmp", req.id); + const bodyFile = path.join(srcPath, "body.json"); // Print error message and cleanup const die = (error) => { res.json({error}); + removeDirectory(srcPath); + }; - // Check if tmp/ directory needs to be cleaned - if (fs.stat(srcPath, (err, stats) => { - if (!err && stats.isDirectory()) rmdir(srcPath, () => {}); // ignore errors, don't wait - })); + async.series([ + cb => { + // Check for problems before file uploads + if (req.body && req.body.options){ + odmInfo.filterOptions(req.body.options, err => { + if (err) cb(err); + else cb(); + }); + }else cb(); + }, + cb => { + fs.stat(srcPath, (err, stat) => { + if (err && err.code === 'ENOENT') cb(); + else cb(new Error(`Directory exists (should not have happened: ${err.code})`)); + }); + }, + cb => fs.mkdir(srcPath, undefined, cb), + cb => { + fs.writeFile(bodyFile, JSON.stringify(req.body), {encoding: 'utf8'}, cb); + }, + cb => { + res.json({uuid: req.id}); + cb(); + } + ], err => { + if (err) die(err.message); + }); + }, + + createTask: (req, res) => { + // IMPROVEMENT: consider doing the file moving in the background + // and return a response more quickly instead of a long timeout. + req.setTimeout(1000 * 60 * 20); + + const srcPath = path.join("tmp", req.id); + + // Print error message and cleanup + const die = (error) => { + res.json({error}); + removeDirectory(srcPath); }; if (req.error !== undefined){ @@ -202,7 +308,7 @@ module.exports = { res.json({ uuid: req.id }); cb(); } - }, req.body.options, + }, req.body.options, req.body.webhook, req.body.skipPostProcessing === 'true'); }