Bug fixes, parallel uploads working

libs/TaskManager.js

@@ -107,7 +107,6 @@ class TaskManager{
     }
 
     removeStaleUploads(done){
-        logger.info("Checking for stale uploads...");
         fs.readdir("tmp", (err, entries) => {
             if (err) done(err);
             else{

libs/taskNew.js

@@ -109,6 +109,8 @@ module.exports = {
             if (config.testDropUploads){
                 if (Math.random() < 0.5) res.sendStatus(500);
                 else next();
+            }else{
+                next();
             }
         }
     },

public/index.html

@@ -54,10 +54,24 @@
                         </div>
                         <div id="imagesInput" class="form-group" data-bind="visible: mode() === 'file'">
                             <div id="images">Images and GCP File (optional):</div> <button id="btnSelectFiles" class="btn btn-default btn-sm" data-bind="attr: {disabled: uploading()}">Add Files...</button>
-                            <div data-bind="visible: filesCount()">Selected files: <span data-bind="text: filesCount()"></span></div>
+                            <div data-bind="visible: filesCount() && !uploading()">Selected files: <span data-bind="text: filesCount()"></span></div>
+                            <div data-bind="visible: uploading()" class="progress" style="margin-top: 12px;">
+                                <div class="progress-bar progress-bar-success" role="progressbar" data-bind="text: uploadedFiles() + ' / ' + filesCount() + ' files', style: {width: uploadedPercentage()}">
+                                </div>
+                            </div>
+                            <div data-bind="visible: uploading()" style="min-height: 230px;">
+                                <div data-bind="foreach: fileUploadStatus.items">
+                                    <div class="progress">
+                                        <div class="progress-bar progress-bar-info" role="progressbar" data-bind="text: key() + ': ' + parseInt(value()) + '%', style: {width: value() + '%'}"></div>
+                                    </div>
+                                </div>
+                            </div>
                         </div>
                         <div id="zipFileInput" class="form-group" data-bind="visible: mode() === 'url'">
                             <label for="zipurl">URL to zip file with Images and GCP File (optional):</label> <input id="zipurl" name="zipurl" class="form-control" type="text" data-bind="attr: {disabled: uploading()}" >
+                            <div data-bind="visible: uploading()">
+                                Uploading...
+                            </div>
                         </div>
                         <div id="errorBlock" data-bind="visible: error().length > 0, click: dismissError">⚠️ <span data-bind="text: error"></span></div>
                         <hr/>
@@ -188,6 +202,7 @@
     </script>
     <script src="js/vendor/bootstrap.min.js"></script>
     <script src="js/vendor/knockout-3.4.0.js"></script>
+    <script src="js/vendor/ko.observableDictionary.js"></script>
     <script src="js/dropzone.js" type="text/javascript"></script>
     <script src="js/main.js"></script>
 </body>

public/js/main.js

@@ -17,11 +17,16 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 $(function() {
     function App(){
-        this.filesCount = ko.observable(0);
         this.mode = ko.observable("file");
+        this.filesCount = ko.observable(0);
         this.error = ko.observable("");
         this.uploading = ko.observable(false);
         this.uuid = ko.observable("");
+        this.uploadedFiles = ko.observable(0);
+        this.fileUploadStatus = new ko.observableDictionary({});
+        this.uploadedPercentage = ko.pureComputed(function(){
+            return ((this.uploadedFiles() / this.filesCount()) * 100.0) + "%";
+        }, this);
     }
     App.prototype.toggleMode = function(){
         if (this.mode() === 'file') this.mode('url');
@@ -30,6 +35,15 @@ $(function() {
     App.prototype.dismissError = function(){
         this.error("");
     };
+    App.prototype.resetUpload = function(){
+        this.filesCount(0);
+        this.error("");
+        this.uploading(false);
+        this.uuid("");
+        this.uploadedFiles(0);
+        this.fileUploadStatus.removeAll();
+        dz.removeAllFiles(true);
+    };
     App.prototype.startTask = function(){
         var self = this;
         this.uploading(true);
@@ -50,14 +64,15 @@ $(function() {
         }
 
         // Start upload
-        if (this.mode() === 'file'){
-            if (this.filesCount() > 0){
         var formData = new FormData();
         formData.append("name", $("#taskName").val());
         formData.append("webhook", $("#webhook").val());
         formData.append("skipPostProcessing", !$("#doPostProcessing").prop('checked'));
         formData.append("options", JSON.stringify(optionsModel.getUserOptions()));
-                $.ajax("/task/new/init", {
+
+        if (this.mode() === 'file'){
+            if (this.filesCount() > 0){
+                $.ajax("/task/new/init?token=" + token, {
                     type: "POST",
                     data: formData,
                     processData: false,
@@ -65,7 +80,6 @@ $(function() {
                 }).done(function(result){
                     if (result.uuid){
                         self.uuid(result.uuid);
-                        console.log("Proessing");
                         dz.processQueue();
                     }else{
                         die(result.error || result);
@@ -77,38 +91,33 @@ $(function() {
                 die("No files selected");
             }
         } else if (this.mode() === 'url'){
-            // TODO
+            this.uploading(true);
-            // Handle uploads
+            formData.append("zipurl", $("#zipurl").val());
-            // $("#images").fileinput({
-            //     uploadUrl: '/task/new?token=' + token,
-            //     showPreview: false,
-            //     allowedFileExtensions: ['jpg', 'jpeg', 'txt', 'zip'],
-            //     elErrorContainer: '#errorBlock',
-            //     showUpload: false,
-            //     uploadAsync: false,
-            //     // ajaxSettings: { headers: { 'set-uuid': '8366b2ad-a608-4cd1-bdcb-c3d84a034623' } },
-            //     uploadExtraData: function() {
-            //         return {
-            //             name: $("#taskName").val(),
-            //             zipurl: $("#zipurl").val(),
-            //             webhook: $("#webhook").val(),
-            //             skipPostProcessing: !$("#doPostProcessing").prop('checked'),
-            //             options: JSON.stringify(optionsModel.getUserOptions())
-            //         };
-            //     }
-            // });
-        }
-    }
 
-    app = new App();
+            $.ajax("/task/new?token=" + token, {
-    ko.applyBindings(app, document.getElementById('app'));
+                type: "POST",
+                data: formData,
+                processData: false,
+                contentType: false
+            }).done(function(json){
+                if (json.uuid){
+                    taskList.add(new Task(json.uuid));
+                    self.resetUpload();
+                }else{
+                    die(json.error || result);
+                }
+            }).fail(function(){
+                die("Cannot start task. Is the server available and are you connected to the internet?");
+            });
+        }
+    }
 
     Dropzone.autoDiscover = false;
 
     var dz = new Dropzone("div#images", {
         paramName: function(){ return "images"; },
         url : "/task/new/upload/",
-        parallelUploads: 8,
+        parallelUploads: 8, // http://blog.olamisan.com/max-parallel-http-connections-in-a-browser max parallel connections
         uploadMultiple: false,
         acceptedFiles: "image/*,text/*",
         autoProcessQueue: false,
@@ -120,58 +129,38 @@ $(function() {
     });
 
     dz.on("processing", function(file){
-        this.options.url = '/task/new/upload/' + app.uuid();
+        this.options.url = '/task/new/upload/' + app.uuid() + "?token=" + token;
+        app.fileUploadStatus.set(file.name, 0);
     })
     .on("error", function(file){
         // Retry
         console.log("Error uploading ", file, " put back in queue...");
+        app.error("Upload of " + file.name + " failed, retrying...");
         file.status = Dropzone.QUEUED;
+        app.fileUploadStatus.remove(file.name);
         dz.processQueue();
     })
-    .on("totaluploadprogress", function(progress, totalBytes, totalBytesSent){
+    .on("uploadprogress", function(file, progress){
-        // Limit updates since this gets called a lot
+        app.fileUploadStatus.set(file.name, progress);
-        var now = (new Date()).getTime();
-
-        // Progress 100 is sent multiple times at the end
-        // this makes it so that we update the state only once.
-        if (progress === 100) now = now + 9999999999;
-
-    //   if (this.state.upload.lastUpdated + 500 < now){
-        //   this.setUploadState({
-        //     progress, totalBytes, totalBytesSent, lastUpdated: now
-        //   });
-    //   }
     })
     .on("addedfiles", function(files){
         app.filesCount(app.filesCount() + files.length);
     })
-    .on("complete", function(files){
+    .on("complete", function(file){
-        // Check
+        if (file.status === "success"){
-        var failedCount = 0;
+            app.uploadedFiles(app.uploadedFiles() + 1);
-        for (var i = 0; i < files.length; i++){
-            if (files[i].status !== "success"){
-                failedCount++;
-                dz.enqueueFile(files[i]);
         }
-        }
+        app.fileUploadStatus.remove(file.name);
-
-        if (failedCount === 0){
-            
-        }else{
-            console.log(failedCount, "files failed to upload, retrying...");
         dz.processQueue();
-        }
     })
     .on("queuecomplete", function(files){
         // Commit
-        $.ajax("/task/new/commit/" + app.uuid(), {
+        $.ajax("/task/new/commit/" + app.uuid() + "?token=" + token, {
             type: "POST",
         }).done(function(json){
             if (json.uuid){
                 taskList.add(new Task(json.uuid));
-                dz.removeAllFiles(true);
+                app.resetUpload();
-                app.filesCount(0);
-                app.uuid("");
             }else{
                 app.error(json.error || json);
             }
@@ -185,6 +174,9 @@ $(function() {
         app.filesCount(0);
     });
 
+    app = new App();
+    ko.applyBindings(app, document.getElementById('app'));
+
     function query(key) {
         key = key.replace(/[*+?^$.\[\]{}()|\\\/]/g, "\\$&"); // escape RegEx meta chars
         var match = location.search.match(new RegExp("[?&]"+key+"=([^&]+)(&|$)"));

public/js/vendor/ko.observableDictionary.js

@@ -0,0 +1,224 @@
+// Knockout Observable Dictionary
+// (c) James Foster
+// License: MIT (http://www.opensource.org/licenses/mit-license.php)
+
+(function () {
+    function DictionaryItem(key, value, dictionary) {
+        var observableKey = new ko.observable(key);
+
+        this.value = new ko.observable(value);
+        this.key = new ko.computed({
+            read: observableKey,
+            write: function (newKey) {
+                var current = observableKey();
+
+                if (current == newKey) return;
+
+                // no two items are allowed to share the same key.
+                dictionary.remove(newKey);
+
+                observableKey(newKey);
+            }
+        });
+    }
+
+    ko.observableDictionary = function (dictionary, keySelector, valueSelector) {
+        var result = {};
+
+        result.items = new ko.observableArray();
+
+        result._wrappers = {};
+        result._keySelector = keySelector || function (value, key) { return key; };
+        result._valueSelector = valueSelector || function (value) { return value; };
+
+        if (typeof keySelector == 'string') result._keySelector = function (value) { return value[keySelector]; };
+        if (typeof valueSelector == 'string') result._valueSelector = function (value) { return value[valueSelector]; };
+
+        ko.utils.extend(result, ko.observableDictionary['fn']);
+
+        result.pushAll(dictionary);
+
+        return result;
+    };
+
+    ko.observableDictionary['fn'] = {
+        remove: function (valueOrPredicate) {
+            var predicate = valueOrPredicate;
+
+            if (valueOrPredicate instanceof DictionaryItem) {
+                predicate = function (item) {
+                    return item.key() === valueOrPredicate.key();
+                };
+            }
+            else if (typeof valueOrPredicate != "function") {
+                predicate = function (item) {
+                    return item.key() === valueOrPredicate;
+                };
+            }
+
+            ko.observableArray['fn'].remove.call(this.items, predicate);
+        },
+
+        push: function (key, value) {
+            var item = null;
+
+            if (key instanceof DictionaryItem) {
+                // handle the case where only a DictionaryItem is passed in
+                item = key;
+                value = key.value();
+                key = key.key();
+            }
+
+            if (value === undefined) {
+                value = this._valueSelector(key);
+                key = this._keySelector(value);
+            }
+            else {
+                value = this._valueSelector(value);
+            }
+
+            var current = this.get(key, false);
+            if (current) {
+                // update existing value
+                current(value);
+                return current;
+            }
+
+            if (!item) {
+                item = new DictionaryItem(key, value, this);
+            }
+
+            ko.observableArray['fn'].push.call(this.items, item);
+
+            return value;
+        },
+
+        pushAll: function (dictionary) {
+            var self = this;
+            var items = self.items();
+
+            if (dictionary instanceof Array) {
+                $.each(dictionary, function (index, item) {
+                    var key = self._keySelector(item, index);
+                    var value = self._valueSelector(item);
+                    items.push(new DictionaryItem(key, value, self));
+                });
+            }
+            else {
+                for (var prop in dictionary) {
+                    if (dictionary.hasOwnProperty(prop)) {
+                        var item = dictionary[prop];
+                        var key = self._keySelector(item, prop);
+                        var value = self._valueSelector(item);
+                        items.push(new DictionaryItem(key, value, self));
+                    }
+                }
+            }
+
+            self.items.valueHasMutated();
+        },
+
+        sort: function (method) {
+            if (method === undefined) {
+                method = function (a, b) {
+                    return defaultComparison(a.key(), b.key());
+                };
+            }
+
+            return ko.observableArray['fn'].sort.call(this.items, method);
+        },
+
+        indexOf: function (key) {
+            if (key instanceof DictionaryItem) {
+                return ko.observableArray['fn'].indexOf.call(this.items, key);
+            }
+
+            var underlyingArray = this.items();
+            for (var index = 0; index < underlyingArray.length; index++) {
+                if (underlyingArray[index].key() == key)
+                    return index;
+            }
+            return -1;
+        },
+
+        get: function (key, wrap) {
+            if (wrap == false)
+                return getValue(key, this.items());
+
+            var wrapper = this._wrappers[key];
+
+            if (wrapper == null) {
+                wrapper = this._wrappers[key] = new ko.computed({
+                    read: function () {
+                        var value = getValue(key, this.items());
+                        return value ? value() : null;
+                    },
+                    write: function (newValue) {
+                        var value = getValue(key, this.items());
+
+                        if (value)
+                            value(newValue);
+                        else
+                            this.push(key, newValue);
+                    }
+                }, this);
+            }
+
+            return wrapper;
+        },
+
+        set: function (key, value) {
+            return this.push(key, value);
+        },
+
+        keys: function () {
+            return ko.utils.arrayMap(this.items(), function (item) { return item.key(); });
+        },
+
+        values: function () {
+            return ko.utils.arrayMap(this.items(), function (item) { return item.value(); });
+        },
+        
+        removeAll: function () {
+            this.items.removeAll();
+        },
+
+        toJSON: function () {
+            var result = {};
+            var items = ko.utils.unwrapObservable(this.items);
+
+            ko.utils.arrayForEach(items, function (item) {
+                var key = ko.utils.unwrapObservable(item.key);
+                var value = ko.utils.unwrapObservable(item.value);
+
+                result[key] = value;
+            });
+
+            return result;
+        }
+    };
+
+    function getValue(key, items) {
+        var found = ko.utils.arrayFirst(items, function (item) {
+            return item.key() == key;
+        });
+        return found ? found.value : null;
+    }
+})();
+
+
+// Utility methods
+// ---------------------------------------------
+function isNumeric(n) {
+    return !isNaN(parseFloat(n)) && isFinite(n);
+}
+
+function defaultComparison(a, b) {
+    if (isNumeric(a) && isNumeric(b)) return a - b;
+
+    a = a.toString();
+    b = b.toString();
+
+    return a == b ? 0 : (a < b ? -1 : 1);
+}
+// ---------------------------------------------