Started rewriting UI, added dropzone, tweaks, drop uploads test mode

2019-01-31 17:16:37 -05:00 · 2019-01-31 17:16:37 -05:00 · 5be741bef2
commit 5be741bef2
--- a/config.js
+++ b/config.js
@ -37,6 +37,7 @@ Options:
 	--test Enable test mode. In test mode, no commands are sent to OpenDroneMap. This can be useful during development or testing (default: false)
 	--test_skip_orthophotos	If test mode is enabled, skip orthophoto results when generating assets. (default: false) 
 	--test_skip_dems	If test mode is enabled, skip dems results when generating assets. (default: false) 
+	--test_drop_uploads	If test mode is enabled, drop /task/new/upload requests with 50% probability. (default: false)
 	--powercycle	When set, the application exits immediately after powering up. Useful for testing launch and compilation issues.
 	--token <token>	Sets a token that needs to be passed for every request. This can be used to limit access to the node only to token holders. (default: none)
 	--max_images <number>	Specify the maximum number of images that this processing node supports. (default: unlimited)
@ -95,6 +96,7 @@ config.cleanupUploadsAfter = parseInt(argv.cleanup_uploads_after || fromConfigFi
 config.test = argv.test || fromConfigFile("test", false);
 config.testSkipOrthophotos = argv.test_skip_orthophotos || fromConfigFile("testSkipOrthophotos", false);
 config.testSkipDems = argv.test_skip_dems || fromConfigFile("testSkipDems", false);
+config.testDropUploads = argv.test_drop_uploads || fromConfigFile("testDropUploads", false);
 config.powercycle = argv.powercycle || fromConfigFile("powercycle", false);
 config.token = argv.token || fromConfigFile("token", "");
 config.maxImages = parseInt(argv.max_images || fromConfigFile("maxImages", "")) || null;
--- a/index.js
+++ b/index.js
@ -144,7 +144,7 @@ app.post('/task/new/init', authCheck, taskNew.assignUUID, formDataParser, taskNe
 *          schema:
 *            $ref: '#/definitions/Error'
 */
-app.post('/task/new/upload/:uuid', authCheck, taskNew.getUUID, taskNew.uploadImages, taskNew.handleUpload);
+app.post('/task/new/upload/:uuid', authCheck, taskNew.getUUID, taskNew.preUpload, taskNew.uploadImages, taskNew.handleUpload);

 /** @swagger
 *  /task/new/commit/{uuid}:
@ -253,7 +253,6 @@ app.post('/task/new/commit/:uuid', authCheck, taskNew.getUUID, taskNew.handleCom
 *            $ref: '#/definitions/Error'
 */
 app.post('/task/new', authCheck, taskNew.assignUUID, taskNew.uploadImages, (req, res, next) => {
-    console.log(req.body);
    req.body = req.body || {};
    if ((!req.files || req.files.length === 0) && !req.body.zipurl) req.error = "Need at least 1 file or a zip file url.";
    else if (config.maxImages && req.files && req.files.length > config.maxImages) req.error = `${req.files.length} images uploaded, but this node can only process up to ${config.maxImages}.`;
@ -813,7 +812,12 @@ process.on('SIGTERM', gracefulShutdown);
 process.on('SIGINT', gracefulShutdown);

 // Startup
-if (config.test) logger.info("Running in test mode");
+if (config.test) {
+    logger.info("Running in test mode");
+    if (config.testSkipOrthophotos) logger.info("Orthophotos will be skipped");
+    if (config.testSkipDems) logger.info("DEMs will be skipped");
+    if (config.testDropUploads) logger.info("Uploads will drop at random");
+}

 let commands = [
    cb => odmInfo.initialize(cb),
--- a/libs/taskNew.js
+++ b/libs/taskNew.js
@ -102,6 +102,17 @@ module.exports = {
        });
    },

+    preUpload: (req, res, next) => {
+        // Testing stuff
+        if (!config.test) next();
+        else{
+            if (config.testDropUploads){
+                if (Math.random() < 0.5) res.sendStatus(500);
+                else next();
+            }
+        }
+    },
+
    uploadImages: upload.array("images"),

    handleUpload: (req, res) => {
@ -140,13 +151,16 @@ module.exports = {
            else{
                req.body = body;
                req.files = files;
+
+                if (req.files.length === 0){
+                    req.error = "Need at least 1 file.";
+                }
                next();
            }
        });
    },

    handleInit: (req, res) => {
-        console.log(req.body);
        req.body = req.body || {};
        
        const srcPath = path.join("tmp", req.id);
--- a/public/index.html
+++ b/public/index.html
@ -14,8 +14,19 @@
            padding-top: 50px;
            padding-bottom: 20px;
        }
+        .navbar{
+            background-color: #3498db;
+        }
+        a:hover, a:focus, a:active, a{
+            color: #3498db;
+        }
+        #images{
+            font-weight: bold;
+        }
+        #btnSelectFiles, #images{
+            display: inline-block;
+        }
    </style>
-    <link href="css/fileinput.css" media="all" rel="stylesheet" type="text/css" />
    <link rel="stylesheet" href="css/main.css">

    <script src="js/vendor/modernizr-2.8.3.min.js"></script>
@ -29,34 +40,34 @@
        <div class="container">
            <div class="navbar-header">
                <a class="navbar-brand" href="/">NodeODM</a>
-                <p class="navbar-text navbar-right">Open Source Drone Aerial Imagery Processing</a>
-                </p>
            </div>
        </div>
    </nav>

    <div class="container">
-        <!-- Example row of columns -->
        <div class="row">
            <div class="col-md-5">
-                <h2>New Task</h2>
-                <br/>
                <form enctype="multipart/form-data" onsubmit="return false;">
-                    <div class="form-group form-inline">
-                        <label for="taskName">Project Name:</lable> <input type="text" class="form-control" value="" id="taskName" />
+                    <div id="app">
+                        <div class="form-group form-inline">
+                            <label for="taskName">Project Name:</lable> <input type="text" class="form-control" value="" id="taskName" data-bind="attr: {disabled: uploading()}" />
+                        </div>
+                        <div id="imagesInput" class="form-group" data-bind="visible: mode() === 'file'">
+                            <div id="images">Images and GCP File (optional):</div> <button id="btnSelectFiles" class="btn btn-default btn-sm" data-bind="attr: {disabled: uploading()}">Add Files...</button>
+                            <div data-bind="visible: filesCount()">Selected files: <span data-bind="text: filesCount()"></span></div>
+                        </div>
+                        <div id="zipFileInput" class="form-group" data-bind="visible: mode() === 'url'">
+                            <label for="zipurl">URL to zip file with Images and GCP File (optional):</label> <input id="zipurl" name="zipurl" class="form-control" type="text" data-bind="attr: {disabled: uploading()}" >
+                        </div>
+                        <div id="errorBlock" data-bind="visible: error().length > 0, click: dismissError">⚠️ <span data-bind="text: error"></span></div>
+                        <hr/>
+                        <div class="text-right">
+                            <input type="button" class="btn btn-info" data-bind="visible: mode() === 'file', click: toggleMode" value="Switch to URL" />
+                            <input type="button" class="btn btn-info" data-bind="visible: mode() === 'url', click: toggleMode" value="Switch to File Upload" />
+                            
+                            <input type="submit" class="btn btn-success" data-bind="attr: {disabled: uploading()}, value: uploading() ? 'Uploading...' : 'Start Task', click: startTask" />
+                        </div>
                    </div>
-                    <div id="imagesInput" class="form-group">
-                        <label for="images">Aerial Images and GCP List (optional):</label> <input id="images" name="images" multiple accept="image/*|.txt|.zip" type="file">
-                    </div>
-                    <div id="zipFileInput" class="form-group hidden">
-                        <label for="zipurl">URL to zip file with Aerial Images and GCP List (optional):</label> <input id="zipurl" name="zipurl" class="form-control" type="text">
-                    </div>
-                    <div id="errorBlock" class="help-block"></div>
-
-                    <div class="text-right"><input type="button" class="btn btn-info" value="Switch to URL" id="btnShowImport" />
-                    <input type="button" class="btn btn-info hidden" value="Switch to File Upload" id="btnShowUpload" />
-                    <input type="submit" class="btn btn-success" value="Start Task" id="btnUpload" />
-                     <input type="submit" class="btn btn-success hidden" value="Get ZIP" id="btnImport" /></div>
                    <div id="options">
                        <div class="form-inline form-group form-horizontal">
                            <div data-bind="visible: error(), text: error()" class="alert alert-warning" role="alert"></div>
@ -110,7 +121,6 @@
                </form>
            </div>
            <div class="col-md-7" id="taskList">
-                <h2>Current Tasks (<span data-bind="text: tasks().length"></span>)</h2>
                <p data-bind="visible: tasks().length === 0">No running tasks.</p>
                <div data-bind="foreach: tasks">
                    <div class="task" data-bind="css: {pulsePositive: info().status && info().status.code === 40, pulseNegative: info().status && info().status.code === 30}">
@ -168,8 +178,7 @@
        <hr>

        <footer>
-            <p>Links: <a href="https://github.com/OpenDroneMap/NodeODM/blob/master/docs/index.adoc" target="_blank">API Docs</a> | <a href="https://github.com/OpenDroneMap/WebODM" target="_blank">WebODM</a>
-                <p>This software is released under the terms of the <a href="https://www.gnu.org/licenses/gpl-3.0.en.html" target="_blank">GPLv3 License</a>. See <a href="https://github.com/OpenDroneMap/NodeODM" target="_blank">NodeODM</a> on Github for more information.</p>
+            <p>This software is released under the terms of the <a href="https://www.gnu.org/licenses/gpl-3.0.en.html" target="_blank">GPLv3 License</a>. See <a href="https://github.com/OpenDroneMap/NodeODM" target="_blank">NodeODM</a> on Github for more information.</p>
        </footer>
    </div>
    <!-- /container -->
@ -179,8 +188,7 @@
    </script>
    <script src="js/vendor/bootstrap.min.js"></script>
    <script src="js/vendor/knockout-3.4.0.js"></script>
-    <script src="js/fileinput.js" type="text/javascript"></script>
-
+    <script src="js/dropzone.js" type="text/javascript"></script>
    <script src="js/main.js"></script>
 </body>

--- a/public/js/dropzone.js
+++ b/public/js/dropzone.js
--- a/public/js/fileinput.js
+++ b/public/js/fileinput.js
--- a/public/js/fileinput.min.js
+++ b/public/js/fileinput.min.js
--- a/public/js/main.js
+++ b/public/js/main.js
@ -16,6 +16,175 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 $(function() {
+    function App(){
+        this.filesCount = ko.observable(0);
+        this.mode = ko.observable("file");
+        this.error = ko.observable("");
+        this.uploading = ko.observable(false);
+        this.uuid = ko.observable("");
+    }
+    App.prototype.toggleMode = function(){
+        if (this.mode() === 'file') this.mode('url');
+        else this.mode('file');
+    };
+    App.prototype.dismissError = function(){
+        this.error("");
+    };
+    App.prototype.startTask = function(){
+        var self = this;
+        this.uploading(true);
+        this.error("");
+        this.uuid("");
+
+        var die = function(err){
+            self.error(err);
+            self.uploading(false);
+        };
+
+        // validate webhook if exists
+        var webhook = $("#webhook").val();
+        var regex = new RegExp("^(http[s]?:\\/\\/(www\\.)?|ftp:\\/\\/(www\\.)?|www\\.){1}([0-9A-Za-z-\\.@:%_\+~#=]+)+((\\.[a-zA-Z]{2,3})+)(/(.)*)?(\\?(.)*)?");
+        if (webhook.length > 0 && !regex.test(webhook)){
+            die("Invalid webhook URL");
+            return;
+        }
+
+        // Start upload
+        if (this.mode() === 'file'){
+            if (this.filesCount() > 0){
+                var formData = new FormData();
+                formData.append("name", $("#taskName").val());
+                formData.append("webhook", $("#webhook").val());
+                formData.append("skipPostProcessing", !$("#doPostProcessing").prop('checked'));
+                formData.append("options", JSON.stringify(optionsModel.getUserOptions()));
+                $.ajax("/task/new/init", {
+                    type: "POST",
+                    data: formData,
+                    processData: false,
+                    contentType: false
+                }).done(function(result){
+                    if (result.uuid){
+                        self.uuid(result.uuid);
+                        console.log("Proessing");
+                        dz.processQueue();
+                    }else{
+                        die(result.error || result);
+                    }
+                }).fail(function(){
+                    die("Cannot start task. Is the server available and are you connected to the internet?");
+                });
+            }else{
+                die("No files selected");
+            }
+        } else if (this.mode() === 'url'){
+            // TODO
+            // Handle uploads
+            // $("#images").fileinput({
+            //     uploadUrl: '/task/new?token=' + token,
+            //     showPreview: false,
+            //     allowedFileExtensions: ['jpg', 'jpeg', 'txt', 'zip'],
+            //     elErrorContainer: '#errorBlock',
+            //     showUpload: false,
+            //     uploadAsync: false,
+            //     // ajaxSettings: { headers: { 'set-uuid': '8366b2ad-a608-4cd1-bdcb-c3d84a034623' } },
+            //     uploadExtraData: function() {
+            //         return {
+            //             name: $("#taskName").val(),
+            //             zipurl: $("#zipurl").val(),
+            //             webhook: $("#webhook").val(),
+            //             skipPostProcessing: !$("#doPostProcessing").prop('checked'),
+            //             options: JSON.stringify(optionsModel.getUserOptions())
+            //         };
+            //     }
+            // });
+        }
+    }
+
+    app = new App();
+    ko.applyBindings(app, document.getElementById('app'));
+
+    Dropzone.autoDiscover = false;
+
+    var dz = new Dropzone("div#images", {
+        paramName: function(){ return "images"; },
+        url : "/task/new/upload/",
+        parallelUploads: 8,
+        uploadMultiple: false,
+        acceptedFiles: "image/*,text/*",
+        autoProcessQueue: false,
+        createImageThumbnails: false,
+        previewTemplate: '<div style="display:none"></div>',
+        clickable: document.getElementById("btnSelectFiles"),
+        chunkSize: 2147483647,
+        timeout: 2147483647
+    });
+
+    dz.on("processing", function(file){
+        this.options.url = '/task/new/upload/' + app.uuid();
+    })
+    .on("error", function(file){
+        // Retry
+        console.log("Error uploading ", file, " put back in queue...");
+        file.status = Dropzone.QUEUED;
+        dz.processQueue();
+    })
+    .on("totaluploadprogress", function(progress, totalBytes, totalBytesSent){
+        // Limit updates since this gets called a lot
+        var now = (new Date()).getTime();
+
+        // Progress 100 is sent multiple times at the end
+        // this makes it so that we update the state only once.
+        if (progress === 100) now = now + 9999999999;
+
+    //   if (this.state.upload.lastUpdated + 500 < now){
+        //   this.setUploadState({
+        //     progress, totalBytes, totalBytesSent, lastUpdated: now
+        //   });
+    //   }
+    })
+    .on("addedfiles", function(files){
+        app.filesCount(app.filesCount() + files.length);
+    })
+    .on("complete", function(files){
+        // Check
+        var failedCount = 0;
+        for (var i = 0; i < files.length; i++){
+            if (files[i].status !== "success"){
+                failedCount++;
+                dz.enqueueFile(files[i]);
+            }
+        }
+
+        if (failedCount === 0){
+            
+        }else{
+            console.log(failedCount, "files failed to upload, retrying...");
+            dz.processQueue();
+        }
+    })
+    .on("queuecomplete", function(files){
+        // Commit
+        $.ajax("/task/new/commit/" + app.uuid(), {
+            type: "POST",
+        }).done(function(json){
+            if (json.uuid){
+                taskList.add(new Task(json.uuid));
+                dz.removeAllFiles(true);
+                app.filesCount(0);
+                app.uuid("");
+            }else{
+                app.error(json.error || json);
+            }
+            app.uploading(false);
+        }).fail(function(){
+            app.error("Cannot commit task. Is the server available and are you connected to the internet?");
+            app.uploading(false);
+        });
+    })
+    .on("reset", function(){
+        app.filesCount(0);
+    });
+
    function query(key) {
        key = key.replace(/[*+?^$.\[\]{}()|\\\/]/g, "\\$&"); // escape RegEx meta chars
        var match = location.search.match(new RegExp("[?&]"+key+"=([^&]+)(&|$)"));
@ -285,44 +454,6 @@ $(function() {
    var taskList = new TaskList();
    ko.applyBindings(taskList, document.getElementById('taskList'));

-    // Handle uploads
-    $("#images").fileinput({
-        uploadUrl: '/task/new?token=' + token,
-        showPreview: false,
-        allowedFileExtensions: ['jpg', 'jpeg', 'txt', 'zip'],
-        elErrorContainer: '#errorBlock',
-        showUpload: false,
-        uploadAsync: false,
-        // ajaxSettings: { headers: { 'set-uuid': '8366b2ad-a608-4cd1-bdcb-c3d84a034623' } },
-        uploadExtraData: function() {
-            return {
-                name: $("#taskName").val(),
-                zipurl: $("#zipurl").val(),
-                webhook: $("#webhook").val(),
-                skipPostProcessing: !$("#doPostProcessing").prop('checked'),
-                options: JSON.stringify(optionsModel.getUserOptions())
-            };
-        }
-    });
-
-    $("#btnUpload").click(function() {
-        $("#btnUpload").attr('disabled', true)
-            .val("Uploading...");
-
-            // validate webhook if exists
-            var webhook = $("#webhook").val();
-            var regex = new RegExp("^(http[s]?:\\/\\/(www\\.)?|ftp:\\/\\/(www\\.)?|www\\.){1}([0-9A-Za-z-\\.@:%_\+~#=]+)+((\\.[a-zA-Z]{2,3})+)(/(.)*)?(\\?(.)*)?");
-
-            if(webhook.length > 0 && !regex.test(webhook)){
-                $('#errorBlock').text("Webhook url is not valid..");
-                $("#btnUpload").attr('disabled', false).val("Start Task");
-                return;
-            }
-        
-            // Start upload
-            $("#images").fileinput('upload'); 
-    });
-
    $('#resetWebhook').on('click', function(){
        $("#webhook").val('');
    });
@ -331,43 +462,6 @@ $(function() {
        $("#doPostProcessing").prop('checked', false);
    });

-    // zip file control
-    $('#btnShowImport').on('click', function(e){
-        e.preventDefault();
-        $('#zipFileInput').removeClass('hidden');
-        $('#btnShowUpload').removeClass('hidden');
-
-        $('#imagesInput').addClass('hidden');
-        $('#btnShowImport').addClass('hidden');
-
-    });
-
-    $('#btnShowUpload').on('click', function(e){
-        e.preventDefault();
-        $('#imagesInput').removeClass('hidden');
-        $('#btnShowImport').removeClass('hidden');
-
-        $('#zipFileInput').addClass('hidden');
-        $('#btnShowUpload').addClass('hidden');
-        $('#zipurl').val('');
-    });
-    
-
-    var btnUploadLabel = $("#btnUpload").val();
-    $("#images")
-        .on('filebatchuploadsuccess', function(e, params) {
-            $("#images").fileinput('reset');
-
-            if (params.response && params.response.uuid) {
-                taskList.add(new Task(params.response.uuid));
-            }
-        })
-        .on('filebatchuploadcomplete', function() {
-            $("#btnUpload").removeAttr('disabled')
-                .val(btnUploadLabel);
-        })
-        .on('filebatchuploaderror', console.warn);
-
    // Load options
    function Option(properties) {
        this.properties = properties;