Merge pull request #95 from pierotofy/timeout

Added timeout option
2019-10-03 22:10:10 -04:00 · 2019-10-03 22:10:10 -04:00 · 2b0bf47c11
commit 2b0bf47c11
--- a/config.js
+++ b/config.js
@ -51,6 +51,7 @@ Options:
 	--s3_signature_version <version>	S3 signature version. (default: 4)
 	--s3_upload_everything	Upload all task results to S3. (default: upload only .zip archive and orthophoto)
 	--max_concurrency   <number>	Place a cap on the max-concurrency option to use for each task. (default: no limit)
+	--max_runtime	<number> Number of minutes (approximate) that a task is allowed to run before being forcibly canceled (timeout). (default: no limit)
 Log Levels: 
 error | debug | info | verbose | debug | silly 
 `);
@ -112,5 +113,6 @@ config.s3SecretKey = argv.s3_secret_key || fromConfigFile("s3SecretKey", process
 config.s3SignatureVersion = argv.s3_signature_version || fromConfigFile("s3SignatureVersion", "4")
 config.s3UploadEverything = argv.s3_upload_everything || fromConfigFile("s3UploadEverything", false);
 config.maxConcurrency = parseInt(argv.max_concurrency || fromConfigFile("maxConcurrency", 0));
+config.maxRuntime = parseInt(argv.max_runtime || fromConfigFile("maxRuntime", -1));

 module.exports = config;
--- a/libs/Task.js
+++ b/libs/Task.js
@ -44,6 +44,7 @@ module.exports = class Task{
        this.uuid = uuid;
        this.name = name !== "" ? name : "Task of " + (new Date()).toISOString();
        this.dateCreated = isNaN(parseInt(dateCreated)) ? new Date().getTime() : parseInt(dateCreated);
+        this.dateStarted = 0;
        this.processingTime = -1;
        this.setStatus(statusCodes.QUEUED);
        this.options = options;
@ -207,6 +208,10 @@ module.exports = class Task{
        return this.status.code === statusCodes.CANCELED;
    }

+    isRunning(){
+        return this.status.code === statusCodes.RUNNING;
+    }
+
    // Cancels the current task (unless it's already canceled)
    cancel(cb){
        if (this.status.code !== statusCodes.CANCELED){
@ -338,7 +343,9 @@ module.exports = class Task{
            };

            // All paths are relative to the project directory (./data/<uuid>/)
-            let allPaths = ['odm_orthophoto/odm_orthophoto.tif', 'odm_orthophoto/odm_orthophoto.mbtiles',
+            let allPaths = ['odm_orthophoto/odm_orthophoto.tif', 
+                              'odm_orthophoto/odm_orthophoto.png',
+                              'odm_orthophoto/odm_orthophoto.mbtiles',
                              'odm_georeferencing', 'odm_texturing',
                              'odm_dem/dsm.tif', 'odm_dem/dtm.tif', 'dsm_tiles', 'dtm_tiles',
                              'orthophoto_tiles', 'potree_pointcloud', 'entwine_pointcloud', 
@ -416,6 +423,7 @@ module.exports = class Task{

        if (this.status.code === statusCodes.QUEUED){
            this.startTrackingProcessingTime();
+            this.dateStarted = new Date().getTime();
            this.setStatus(statusCodes.RUNNING);

            let runnerOptions = this.options.reduce((result, opt) => {
@ -469,6 +477,7 @@ module.exports = class Task{
        if ([statusCodes.CANCELED, statusCodes.FAILED, statusCodes.COMPLETED].indexOf(this.status.code) !== -1){
            this.setStatus(statusCodes.QUEUED);
            this.dateCreated = new Date().getTime();
+            this.dateStarted = 0;
            this.output = [];
            this.progress = 0;
            this.stopTrackingProcessingTime(true);
@ -563,6 +572,7 @@ module.exports = class Task{
            uuid: this.uuid,
            name: this.name,
            dateCreated: this.dateCreated,
+            dateStarted: this.dateStarted,
            status: this.status,
            options: this.options,
            webhook: this.webhook,
--- a/libs/TaskManager.js
+++ b/libs/TaskManager.js
@ -59,6 +59,13 @@ class TaskManager{
                    this.dumpTaskList();
                    this.removeStaleUploads();
                });
+                
+                if (config.maxRuntime > 0){
+                    // Every minute
+                    schedule.scheduleJob('* * * * *', () => {
+                        this.checkTimeouts();
+                    });
+                }

                cb();
            }
@ -306,6 +313,23 @@ class TaskManager{
        }
        return count;
    }
+
+    checkTimeouts(){
+        if (config.maxRuntime > 0){
+            let now = new Date().getTime();
+
+            for (let uuid in this.tasks){
+                let task = this.tasks[uuid];
+                
+                if (task.isRunning() && task.dateStarted > 0 && (now - task.dateStarted) > config.maxRuntime * 60 * 1000){
+                    task.output.push(`Task timed out after ${Math.ceil(task.processingTime / 60 / 1000)} minutes.\n`);
+                    this.cancel(uuid, () => {
+                        logger.warn(`Task ${uuid} timed out`);
+                    });
+                }
+            }
+        }
+    }
 }

 module.exports = {
--- a/libs/taskNew.js
+++ b/libs/taskNew.js
@ -292,6 +292,14 @@ module.exports = {
                                    if (config.maxImages && files.length > config.maxImages) cb(`${files.length} images uploaded, but this node can only process up to ${config.maxImages}.`);
                                    else cb(err);
                                });
+                            },
+
+                            // Remove
+                            cb => {
+                                fs.exists(seedFileDst, exists => {
+                                    if (exists) fs.unlink(seedFileDst, cb);
+                                    else cb();
+                                });
                            }
                        ], cb);
                    }
--- a/public/index.html
+++ b/public/index.html
@ -4,7 +4,7 @@
 <head>
    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
-    <title>NodeODM</title>
+    <title>NodeODM - Web UI</title>
    <meta name="description" content="">
    <meta name="viewport" content="width=device-width, initial-scale=1">

@ -39,7 +39,7 @@
    <nav class="navbar navbar-inverse navbar-fixed-top" role="navigation">
        <div class="container">
            <div class="navbar-header">
-                <a class="navbar-brand" href="/">NodeODM</a>
+                <a class="navbar-brand" href="/">NodeODM - Web UI</a>
            </div>
        </div>
    </nav>
@ -49,9 +49,6 @@
            <div class="col-md-5">
                <form enctype="multipart/form-data" onsubmit="return false;">
                    <div id="app">
-                        <div class="form-group form-inline">
-                            <label for="taskName">Project Name:</lable> <input type="text" class="form-control" value="" id="taskName" data-bind="attr: {disabled: uploading()}" />
-                        </div>
                        <div id="imagesInput" class="form-group" data-bind="visible: mode() === 'file'">
                            <div id="images">Images and GCP File (optional):</div> <button id="btnSelectFiles" class="btn btn-default btn-sm" data-bind="attr: {disabled: uploading()}">Add Files...</button>
                            <div data-bind="visible: filesCount() && !uploading()">Selected files: <span data-bind="text: filesCount()"></span></div>
@ -89,9 +86,17 @@

                            <div data-bind="visible: showOptions()">
                                <div>
+                                    <label for="taskName">Project Name:</lable> 
+                                    <br/>
+                                    <input type="text" class="form-control" value="" id="taskName" />
+                                    <button type="submit" class="btn glyphicon glyphicon-info-sign btn-info" data-toggle="tooltip" data-placement="top" title="Assign a name to the project." ></button>
+                                    <button id="resetTaskName" type="submit" class="btn glyphicon glyphicon glyphicon-repeat btn-default" data-toggle="tooltip" data-placement="top" title="Reset to default"  ></button>
+                                    <br/><br/>
+
                                    <label for="doPostProcessing">generate 2D and potree point cloud tiles:</label>
                                    <br/>

+                                    
                                    <div class="checkbox">
                                        <label>
                                            <input type="checkbox" id="doPostProcessing"> Enable
@ -101,6 +106,7 @@
                                    <button id="resetDoPostProcessing" type="submit" class="btn glyphicon glyphicon glyphicon-repeat btn-default" data-toggle="tooltip" data-placement="top" title="Reset to default"  ></button>
                                    
                                    <br/><br/>
+                                    
                                </div>

                                <div>
@ -188,11 +194,9 @@
                </div>
            </div>
        </div>
-
-        <hr>
-
        <footer>
-            <p>This software is released under the terms of the <a href="https://www.gnu.org/licenses/gpl-3.0.en.html" target="_blank">GPLv3 License</a>. See <a href="https://github.com/OpenDroneMap/NodeODM" target="_blank">NodeODM</a> on Github for more information.</p>
+        <hr/>
+        This window can be closed after uploading a task. The process will continue running on the server.
        </footer>
    </div>
    <!-- /container -->
@ -204,7 +208,7 @@
    <script src="js/vendor/knockout-3.4.0.js"></script>
    <script src="js/vendor/ko.observableDictionary.js"></script>
    <script src="js/dropzone.js" type="text/javascript"></script>
-    <script src="js/main.js"></script>
+    <script src="js/main.js?t=1"></script>
 </body>

 </html>
--- a/public/js/main.js
+++ b/public/js/main.js
@ -480,14 +480,23 @@ $(function() {
    $('#resetDoPostProcessing').on('click', function(){
        $("#doPostProcessing").prop('checked', false);
    });
+    $('#resetTaskName').on('click', function(){
+        $("#taskName").val('');
+    });

    // Load options
    function Option(properties) {
        this.properties = properties;
-        this.value = ko.observable();
+
+        this.defaultValue = undefined;
+        if (properties.type === 'bool' && properties.value === 'true'){
+            this.defaultValue = true;
+        }
+        
+        this.value = ko.observable(this.defaultValue);
    }
    Option.prototype.resetToDefault = function() {
-        this.value(undefined);
+        this.value(this.defaultValue);
    };

    function OptionsModel() {
--- a/scripts/postprocess.sh
+++ b/scripts/postprocess.sh
@ -55,21 +55,6 @@ for dem_product in ${dem_products[@]}; do
 	fi
 done

-# Generate MBTiles
-if hash gdal_translate 2>/dev/null; then
-	orthophoto_path="odm_orthophoto/odm_orthophoto.tif"
-	orthophoto_mbtiles_path="odm_orthophoto/odm_orthophoto.mbtiles"
-
-	if [ -e "$orthophoto_path" ]; then
-		gdal_translate $orthophoto_path $orthophoto_mbtiles_path -of MBTILES
-		gdaladdo -r bilinear $orthophoto_mbtiles_path 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384
-	else
-		echo "No orthophoto found at $orthophoto_path: will skip MBTiles generation"
-	fi
-else
-	echo "gdal_translate is not installed, will skip MBTiles generation"
-fi
-
 # Generate point cloud (if entwine or potreeconverter is available)
 pointcloud_input_path=""
 for path in "odm_georeferencing/odm_georeferenced_model.laz" \