kopia lustrzana https://github.com/OpenDroneMap/NodeODM
Bug fixes, parallel uploads working
Piero Toffanin
rodzic
5be741bef2
commit
a489a95d6e
|
|
@ -107,7 +107,6 @@ class TaskManager{
|
|||
}
|
||||
|
||||
removeStaleUploads(done){
|
||||
logger.info("Checking for stale uploads...");
|
||||
fs.readdir("tmp", (err, entries) => {
|
||||
if (err) done(err);
|
||||
else{
|
||||
|
|
|
|||
|
|
@ -109,6 +109,8 @@ module.exports = {
|
|||
if (config.testDropUploads){
|
||||
if (Math.random() < 0.5) res.sendStatus(500);
|
||||
else next();
|
||||
}else{
|
||||
next();
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -54,10 +54,24 @@
|
|||
</div>
|
||||
<div id="imagesInput" class="form-group" data-bind="visible: mode() === 'file'">
|
||||
<div id="images">Images and GCP File (optional):</div> <button id="btnSelectFiles" class="btn btn-default btn-sm" data-bind="attr: {disabled: uploading()}">Add Files...</button>
|
||||
<div data-bind="visible: filesCount()">Selected files: <span data-bind="text: filesCount()"></span></div>
|
||||
<div data-bind="visible: filesCount() && !uploading()">Selected files: <span data-bind="text: filesCount()"></span></div>
|
||||
<div data-bind="visible: uploading()" class="progress" style="margin-top: 12px;">
|
||||
<div class="progress-bar progress-bar-success" role="progressbar" data-bind="text: uploadedFiles() + ' / ' + filesCount() + ' files', style: {width: uploadedPercentage()}">
|
||||
</div>
|
||||
</div>
|
||||
<div data-bind="visible: uploading()" style="min-height: 230px;">
|
||||
<div data-bind="foreach: fileUploadStatus.items">
|
||||
<div class="progress">
|
||||
<div class="progress-bar progress-bar-info" role="progressbar" data-bind="text: key() + ': ' + parseInt(value()) + '%', style: {width: value() + '%'}"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div id="zipFileInput" class="form-group" data-bind="visible: mode() === 'url'">
|
||||
<label for="zipurl">URL to zip file with Images and GCP File (optional):</label> <input id="zipurl" name="zipurl" class="form-control" type="text" data-bind="attr: {disabled: uploading()}" >
|
||||
<div data-bind="visible: uploading()">
|
||||
Uploading...
|
||||
</div>
|
||||
</div>
|
||||
<div id="errorBlock" data-bind="visible: error().length > 0, click: dismissError">⚠️ <span data-bind="text: error"></span></div>
|
||||
<hr/>
|
||||
|
|
@ -188,6 +202,7 @@
|
|||
</script>
|
||||
<script src="js/vendor/bootstrap.min.js"></script>
|
||||
<script src="js/vendor/knockout-3.4.0.js"></script>
|
||||
<script src="js/vendor/ko.observableDictionary.js"></script>
|
||||
<script src="js/dropzone.js" type="text/javascript"></script>
|
||||
<script src="js/main.js"></script>
|
||||
</body>
|
||||
|
|
|
|||
|
|
@ -17,11 +17,16 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|||
*/
|
||||
$(function() {
|
||||
function App(){
|
||||
this.filesCount = ko.observable(0);
|
||||
this.mode = ko.observable("file");
|
||||
this.filesCount = ko.observable(0);
|
||||
this.error = ko.observable("");
|
||||
this.uploading = ko.observable(false);
|
||||
this.uuid = ko.observable("");
|
||||
this.uploadedFiles = ko.observable(0);
|
||||
this.fileUploadStatus = new ko.observableDictionary({});
|
||||
this.uploadedPercentage = ko.pureComputed(function(){
|
||||
return ((this.uploadedFiles() / this.filesCount()) * 100.0) + "%";
|
||||
}, this);
|
||||
}
|
||||
App.prototype.toggleMode = function(){
|
||||
if (this.mode() === 'file') this.mode('url');
|
||||
|
|
@ -30,6 +35,15 @@ $(function() {
|
|||
App.prototype.dismissError = function(){
|
||||
this.error("");
|
||||
};
|
||||
App.prototype.resetUpload = function(){
|
||||
this.filesCount(0);
|
||||
this.error("");
|
||||
this.uploading(false);
|
||||
this.uuid("");
|
||||
this.uploadedFiles(0);
|
||||
this.fileUploadStatus.removeAll();
|
||||
dz.removeAllFiles(true);
|
||||
};
|
||||
App.prototype.startTask = function(){
|
||||
var self = this;
|
||||
this.uploading(true);
|
||||
|
|
@ -50,14 +64,15 @@ $(function() {
|
|||
}
|
||||
|
||||
// Start upload
|
||||
var formData = new FormData();
|
||||
formData.append("name", $("#taskName").val());
|
||||
formData.append("webhook", $("#webhook").val());
|
||||
formData.append("skipPostProcessing", !$("#doPostProcessing").prop('checked'));
|
||||
formData.append("options", JSON.stringify(optionsModel.getUserOptions()));
|
||||
|
||||
if (this.mode() === 'file'){
|
||||
if (this.filesCount() > 0){
|
||||
var formData = new FormData();
|
||||
formData.append("name", $("#taskName").val());
|
||||
formData.append("webhook", $("#webhook").val());
|
||||
formData.append("skipPostProcessing", !$("#doPostProcessing").prop('checked'));
|
||||
formData.append("options", JSON.stringify(optionsModel.getUserOptions()));
|
||||
$.ajax("/task/new/init", {
|
||||
$.ajax("/task/new/init?token=" + token, {
|
||||
type: "POST",
|
||||
data: formData,
|
||||
processData: false,
|
||||
|
|
@ -65,7 +80,6 @@ $(function() {
|
|||
}).done(function(result){
|
||||
if (result.uuid){
|
||||
self.uuid(result.uuid);
|
||||
console.log("Proessing");
|
||||
dz.processQueue();
|
||||
}else{
|
||||
die(result.error || result);
|
||||
|
|
@ -77,38 +91,33 @@ $(function() {
|
|||
die("No files selected");
|
||||
}
|
||||
} else if (this.mode() === 'url'){
|
||||
// TODO
|
||||
// Handle uploads
|
||||
// $("#images").fileinput({
|
||||
// uploadUrl: '/task/new?token=' + token,
|
||||
// showPreview: false,
|
||||
// allowedFileExtensions: ['jpg', 'jpeg', 'txt', 'zip'],
|
||||
// elErrorContainer: '#errorBlock',
|
||||
// showUpload: false,
|
||||
// uploadAsync: false,
|
||||
// // ajaxSettings: { headers: { 'set-uuid': '8366b2ad-a608-4cd1-bdcb-c3d84a034623' } },
|
||||
// uploadExtraData: function() {
|
||||
// return {
|
||||
// name: $("#taskName").val(),
|
||||
// zipurl: $("#zipurl").val(),
|
||||
// webhook: $("#webhook").val(),
|
||||
// skipPostProcessing: !$("#doPostProcessing").prop('checked'),
|
||||
// options: JSON.stringify(optionsModel.getUserOptions())
|
||||
// };
|
||||
// }
|
||||
// });
|
||||
this.uploading(true);
|
||||
formData.append("zipurl", $("#zipurl").val());
|
||||
|
||||
$.ajax("/task/new?token=" + token, {
|
||||
type: "POST",
|
||||
data: formData,
|
||||
processData: false,
|
||||
contentType: false
|
||||
}).done(function(json){
|
||||
if (json.uuid){
|
||||
taskList.add(new Task(json.uuid));
|
||||
self.resetUpload();
|
||||
}else{
|
||||
die(json.error || result);
|
||||
}
|
||||
}).fail(function(){
|
||||
die("Cannot start task. Is the server available and are you connected to the internet?");
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
app = new App();
|
||||
ko.applyBindings(app, document.getElementById('app'));
|
||||
|
||||
Dropzone.autoDiscover = false;
|
||||
|
||||
var dz = new Dropzone("div#images", {
|
||||
paramName: function(){ return "images"; },
|
||||
url : "/task/new/upload/",
|
||||
parallelUploads: 8,
|
||||
parallelUploads: 8, // http://blog.olamisan.com/max-parallel-http-connections-in-a-browser max parallel connections
|
||||
uploadMultiple: false,
|
||||
acceptedFiles: "image/*,text/*",
|
||||
autoProcessQueue: false,
|
||||
|
|
@ -120,58 +129,38 @@ $(function() {
|
|||
});
|
||||
|
||||
dz.on("processing", function(file){
|
||||
this.options.url = '/task/new/upload/' + app.uuid();
|
||||
this.options.url = '/task/new/upload/' + app.uuid() + "?token=" + token;
|
||||
app.fileUploadStatus.set(file.name, 0);
|
||||
})
|
||||
.on("error", function(file){
|
||||
// Retry
|
||||
console.log("Error uploading ", file, " put back in queue...");
|
||||
app.error("Upload of " + file.name + " failed, retrying...");
|
||||
file.status = Dropzone.QUEUED;
|
||||
app.fileUploadStatus.remove(file.name);
|
||||
dz.processQueue();
|
||||
})
|
||||
.on("totaluploadprogress", function(progress, totalBytes, totalBytesSent){
|
||||
// Limit updates since this gets called a lot
|
||||
var now = (new Date()).getTime();
|
||||
|
||||
// Progress 100 is sent multiple times at the end
|
||||
// this makes it so that we update the state only once.
|
||||
if (progress === 100) now = now + 9999999999;
|
||||
|
||||
// if (this.state.upload.lastUpdated + 500 < now){
|
||||
// this.setUploadState({
|
||||
// progress, totalBytes, totalBytesSent, lastUpdated: now
|
||||
// });
|
||||
// }
|
||||
.on("uploadprogress", function(file, progress){
|
||||
app.fileUploadStatus.set(file.name, progress);
|
||||
})
|
||||
.on("addedfiles", function(files){
|
||||
app.filesCount(app.filesCount() + files.length);
|
||||
})
|
||||
.on("complete", function(files){
|
||||
// Check
|
||||
var failedCount = 0;
|
||||
for (var i = 0; i < files.length; i++){
|
||||
if (files[i].status !== "success"){
|
||||
failedCount++;
|
||||
dz.enqueueFile(files[i]);
|
||||
}
|
||||
}
|
||||
|
||||
if (failedCount === 0){
|
||||
|
||||
}else{
|
||||
console.log(failedCount, "files failed to upload, retrying...");
|
||||
dz.processQueue();
|
||||
.on("complete", function(file){
|
||||
if (file.status === "success"){
|
||||
app.uploadedFiles(app.uploadedFiles() + 1);
|
||||
}
|
||||
app.fileUploadStatus.remove(file.name);
|
||||
dz.processQueue();
|
||||
})
|
||||
.on("queuecomplete", function(files){
|
||||
// Commit
|
||||
$.ajax("/task/new/commit/" + app.uuid(), {
|
||||
$.ajax("/task/new/commit/" + app.uuid() + "?token=" + token, {
|
||||
type: "POST",
|
||||
}).done(function(json){
|
||||
if (json.uuid){
|
||||
taskList.add(new Task(json.uuid));
|
||||
dz.removeAllFiles(true);
|
||||
app.filesCount(0);
|
||||
app.uuid("");
|
||||
app.resetUpload();
|
||||
}else{
|
||||
app.error(json.error || json);
|
||||
}
|
||||
|
|
@ -185,6 +174,9 @@ $(function() {
|
|||
app.filesCount(0);
|
||||
});
|
||||
|
||||
app = new App();
|
||||
ko.applyBindings(app, document.getElementById('app'));
|
||||
|
||||
function query(key) {
|
||||
key = key.replace(/[*+?^$.\[\]{}()|\\\/]/g, "\\$&"); // escape RegEx meta chars
|
||||
var match = location.search.match(new RegExp("[?&]"+key+"=([^&]+)(&|$)"));
|
||||
|
|
|
|||
|
|
@ -0,0 +1,224 @@
|
|||
// Knockout Observable Dictionary
|
||||
// (c) James Foster
|
||||
// License: MIT (http://www.opensource.org/licenses/mit-license.php)
|
||||
|
||||
(function () {
|
||||
function DictionaryItem(key, value, dictionary) {
|
||||
var observableKey = new ko.observable(key);
|
||||
|
||||
this.value = new ko.observable(value);
|
||||
this.key = new ko.computed({
|
||||
read: observableKey,
|
||||
write: function (newKey) {
|
||||
var current = observableKey();
|
||||
|
||||
if (current == newKey) return;
|
||||
|
||||
// no two items are allowed to share the same key.
|
||||
dictionary.remove(newKey);
|
||||
|
||||
observableKey(newKey);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
ko.observableDictionary = function (dictionary, keySelector, valueSelector) {
|
||||
var result = {};
|
||||
|
||||
result.items = new ko.observableArray();
|
||||
|
||||
result._wrappers = {};
|
||||
result._keySelector = keySelector || function (value, key) { return key; };
|
||||
result._valueSelector = valueSelector || function (value) { return value; };
|
||||
|
||||
if (typeof keySelector == 'string') result._keySelector = function (value) { return value[keySelector]; };
|
||||
if (typeof valueSelector == 'string') result._valueSelector = function (value) { return value[valueSelector]; };
|
||||
|
||||
ko.utils.extend(result, ko.observableDictionary['fn']);
|
||||
|
||||
result.pushAll(dictionary);
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
ko.observableDictionary['fn'] = {
|
||||
remove: function (valueOrPredicate) {
|
||||
var predicate = valueOrPredicate;
|
||||
|
||||
if (valueOrPredicate instanceof DictionaryItem) {
|
||||
predicate = function (item) {
|
||||
return item.key() === valueOrPredicate.key();
|
||||
};
|
||||
}
|
||||
else if (typeof valueOrPredicate != "function") {
|
||||
predicate = function (item) {
|
||||
return item.key() === valueOrPredicate;
|
||||
};
|
||||
}
|
||||
|
||||
ko.observableArray['fn'].remove.call(this.items, predicate);
|
||||
},
|
||||
|
||||
push: function (key, value) {
|
||||
var item = null;
|
||||
|
||||
if (key instanceof DictionaryItem) {
|
||||
// handle the case where only a DictionaryItem is passed in
|
||||
item = key;
|
||||
value = key.value();
|
||||
key = key.key();
|
||||
}
|
||||
|
||||
if (value === undefined) {
|
||||
value = this._valueSelector(key);
|
||||
key = this._keySelector(value);
|
||||
}
|
||||
else {
|
||||
value = this._valueSelector(value);
|
||||
}
|
||||
|
||||
var current = this.get(key, false);
|
||||
if (current) {
|
||||
// update existing value
|
||||
current(value);
|
||||
return current;
|
||||
}
|
||||
|
||||
if (!item) {
|
||||
item = new DictionaryItem(key, value, this);
|
||||
}
|
||||
|
||||
ko.observableArray['fn'].push.call(this.items, item);
|
||||
|
||||
return value;
|
||||
},
|
||||
|
||||
pushAll: function (dictionary) {
|
||||
var self = this;
|
||||
var items = self.items();
|
||||
|
||||
if (dictionary instanceof Array) {
|
||||
$.each(dictionary, function (index, item) {
|
||||
var key = self._keySelector(item, index);
|
||||
var value = self._valueSelector(item);
|
||||
items.push(new DictionaryItem(key, value, self));
|
||||
});
|
||||
}
|
||||
else {
|
||||
for (var prop in dictionary) {
|
||||
if (dictionary.hasOwnProperty(prop)) {
|
||||
var item = dictionary[prop];
|
||||
var key = self._keySelector(item, prop);
|
||||
var value = self._valueSelector(item);
|
||||
items.push(new DictionaryItem(key, value, self));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
self.items.valueHasMutated();
|
||||
},
|
||||
|
||||
sort: function (method) {
|
||||
if (method === undefined) {
|
||||
method = function (a, b) {
|
||||
return defaultComparison(a.key(), b.key());
|
||||
};
|
||||
}
|
||||
|
||||
return ko.observableArray['fn'].sort.call(this.items, method);
|
||||
},
|
||||
|
||||
indexOf: function (key) {
|
||||
if (key instanceof DictionaryItem) {
|
||||
return ko.observableArray['fn'].indexOf.call(this.items, key);
|
||||
}
|
||||
|
||||
var underlyingArray = this.items();
|
||||
for (var index = 0; index < underlyingArray.length; index++) {
|
||||
if (underlyingArray[index].key() == key)
|
||||
return index;
|
||||
}
|
||||
return -1;
|
||||
},
|
||||
|
||||
get: function (key, wrap) {
|
||||
if (wrap == false)
|
||||
return getValue(key, this.items());
|
||||
|
||||
var wrapper = this._wrappers[key];
|
||||
|
||||
if (wrapper == null) {
|
||||
wrapper = this._wrappers[key] = new ko.computed({
|
||||
read: function () {
|
||||
var value = getValue(key, this.items());
|
||||
return value ? value() : null;
|
||||
},
|
||||
write: function (newValue) {
|
||||
var value = getValue(key, this.items());
|
||||
|
||||
if (value)
|
||||
value(newValue);
|
||||
else
|
||||
this.push(key, newValue);
|
||||
}
|
||||
}, this);
|
||||
}
|
||||
|
||||
return wrapper;
|
||||
},
|
||||
|
||||
set: function (key, value) {
|
||||
return this.push(key, value);
|
||||
},
|
||||
|
||||
keys: function () {
|
||||
return ko.utils.arrayMap(this.items(), function (item) { return item.key(); });
|
||||
},
|
||||
|
||||
values: function () {
|
||||
return ko.utils.arrayMap(this.items(), function (item) { return item.value(); });
|
||||
},
|
||||
|
||||
removeAll: function () {
|
||||
this.items.removeAll();
|
||||
},
|
||||
|
||||
toJSON: function () {
|
||||
var result = {};
|
||||
var items = ko.utils.unwrapObservable(this.items);
|
||||
|
||||
ko.utils.arrayForEach(items, function (item) {
|
||||
var key = ko.utils.unwrapObservable(item.key);
|
||||
var value = ko.utils.unwrapObservable(item.value);
|
||||
|
||||
result[key] = value;
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
};
|
||||
|
||||
function getValue(key, items) {
|
||||
var found = ko.utils.arrayFirst(items, function (item) {
|
||||
return item.key() == key;
|
||||
});
|
||||
return found ? found.value : null;
|
||||
}
|
||||
})();
|
||||
|
||||
|
||||
// Utility methods
|
||||
// ---------------------------------------------
|
||||
function isNumeric(n) {
|
||||
return !isNaN(parseFloat(n)) && isFinite(n);
|
||||
}
|
||||
|
||||
function defaultComparison(a, b) {
|
||||
if (isNumeric(a) && isNumeric(b)) return a - b;
|
||||
|
||||
a = a.toString();
|
||||
b = b.toString();
|
||||
|
||||
return a == b ? 0 : (a < b ? -1 : 1);
|
||||
}
|
||||
// ---------------------------------------------
|
||||
Ładowanie…
Reference in New Issue