mirror
/
OpenDroneMap-WebODM
kopia lustrzana https://github.com/OpenDroneMap/WebODM
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Chunked file upload support, progress bar color fix

pull/688/head
Piero Toffanin 2019-06-26 18:41:09 -04:00
rodzic 4fce6e19c2
commit ac649e7940
7 zmienionych plików z 232 dodań i 68 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

72
app/api/tasks.py
Wyświetl plik

@ -158,28 +158,78 @@ class TaskViewSet(viewsets.ViewSet):
serializer = TaskSerializer(task)
return Response(serializer.data)
def create(self, request, project_pk=None):
project = get_and_check_project(request, project_pk, ('change_project', ))
@detail_route(methods=['post'])
def commit(self, request, pk=None, project_pk=None):
"""
Commit a task after all images have been uploaded
"""
get_and_check_project(request, project_pk, ('change_project', ))
try:
task = self.queryset.get(pk=pk, project=project_pk)
except (ObjectDoesNotExist, ValidationError):
raise exceptions.NotFound()
task.partial = False
task.images_count = models.ImageUpload.objects.filter(task=task).count()
task.save()
worker_tasks.process_task.delay(task.id)
serializer = TaskSerializer(task)
return Response(serializer.data, status=status.HTTP_200_OK)
@detail_route(methods=['post'])
def upload(self, request, pk=None, project_pk=None):
"""
Add images to a task
"""
get_and_check_project(request, project_pk, ('change_project', ))
try:
task = self.queryset.get(pk=pk, project=project_pk)
except (ObjectDoesNotExist, ValidationError):
raise exceptions.NotFound()
files = flatten_files(request.FILES)
if len(files) <= 1:
raise exceptions.ValidationError(detail="Cannot create task, you need at least 2 images")
if len(files) == 0:
raise exceptions.ValidationError(detail="No files uploaded")
with transaction.atomic():
task = models.Task.objects.create(project=project,
pending_action=pending_actions.RESIZE if 'resize_to' in request.data else None)
for image in files:
models.ImageUpload.objects.create(task=task, image=image)
task.images_count = len(files)
# Update other parameters such as processing node, task name, etc.
return Response({'success': True}, status=status.HTTP_200_OK)
def create(self, request, project_pk=None):
project = get_and_check_project(request, project_pk, ('change_project', ))
# If this is a partial task, we're going to upload images later
# for now we just create a placeholder task.
if request.data.get('partial'):
task = models.Task.objects.create(project=project,
pending_action=pending_actions.RESIZE if 'resize_to' in request.data else None)
serializer = TaskSerializer(task, data=request.data, partial=True)
serializer.is_valid(raise_exception=True)
serializer.save()
else:
files = flatten_files(request.FILES)
worker_tasks.process_task.delay(task.id)
if len(files) <= 1:
raise exceptions.ValidationError(detail="Cannot create task, you need at least 2 images")
with transaction.atomic():
task = models.Task.objects.create(project=project,
pending_action=pending_actions.RESIZE if 'resize_to' in request.data else None)
for image in files:
models.ImageUpload.objects.create(task=task, image=image)
task.images_count = len(files)
# Update other parameters such as processing node, task name, etc.
serializer = TaskSerializer(task, data=request.data, partial=True)
serializer.is_valid(raise_exception=True)
serializer.save()
worker_tasks.process_task.delay(task.id)
return Response(serializer.data, status=status.HTTP_201_CREATED)

18
app/migrations/0028_task_partial.py 100644
Wyświetl plik

@ -0,0 +1,18 @@
# Generated by Django 2.1.7 on 2019-06-26 18:31
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('app', '0027_plugin'),
]
operations = [
migrations.AddField(
model_name='task',
name='partial',
field=models.BooleanField(default=False, help_text='A flag indicating whether this task is currently waiting for information or files to be uploaded before being considered for processing.'),
),
]

1
app/models/task.py
Wyświetl plik

@ -213,6 +213,7 @@ class Task(models.Model):
blank=True)
import_url = models.TextField(null=False, default="", blank=True, help_text="URL this task is imported from (only for imported tasks)")
images_count = models.IntegerField(null=False, blank=True, default=0, help_text="Number of images associated with this task")
partial = models.BooleanField(default=False, help_text="A flag indicating whether this task is currently waiting for information or files to be uploaded before being considered for processing.")
def __init__(self, *args, **kwargs):
super(Task, self).__init__(*args, **kwargs)

3
app/static/app/css/theme.scss
Wyświetl plik

@ -72,6 +72,9 @@ body,
a, a:hover, a:focus{
color: theme("tertiary");
}
.progress-bar-success{
background-color: theme("tertiary");
}
/* Button primary */
#navbar-top .navbar-top-links,{

196
app/static/app/js/components/ProjectListItem.jsx
Wyświetl plik

@ -81,6 +81,7 @@ class ProjectListItem extends React.Component {
progress: 0,
files: [],
totalCount: 0,
uploadedCount: 0,
totalBytes: 0,
totalBytesSent: 0,
lastUpdated: 0
@ -109,9 +110,9 @@ class ProjectListItem extends React.Component {
if (this.hasPermission("add")){
this.dz = new Dropzone(this.dropzone, {
paramName: "images",
url : `/api/projects/${this.state.data.id}/tasks/`,
parallelUploads: 2147483647,
uploadMultiple: true,
url : 'TO_BE_CHANGED',
parallelUploads: 10,
uploadMultiple: false,
acceptedFiles: "image/*,text/*",
autoProcessQueue: false,
createImageThumbnails: false,
@ -155,27 +156,39 @@ class ProjectListItem extends React.Component {
}
});
this.dz.on("totaluploadprogress", (progress, totalBytes, totalBytesSent) => {
// Limit updates since this gets called a lot
let now = (new Date()).getTime();
// Progress 100 is sent multiple times at the end
// this makes it so that we update the state only once.
if (progress === 100) now = now + 9999999999;
if (this.state.upload.lastUpdated + 500 < now){
this.setUploadState({
progress, totalBytes, totalBytesSent, lastUpdated: now
});
this.dz.on("addedfiles", files => {
let totalBytes = 0;
for (let i = 0; i < files.length; i++){
totalBytes += files[i].size;
files[i].deltaBytesSent = 0;
files[i].trackedBytesSent = 0;
files[i].retries = 0;
}
})
.on("addedfiles", files => {
this.setUploadState({
editing: true,
totalCount: this.state.upload.totalCount + files.length,
files
files,
totalBytes: this.state.upload.totalBytes + totalBytes
});
})
.on("uploadprogress", (file, progress, bytesSent) => {
const now = new Date().getTime();
if (now - this.state.upload.lastUpdated > 500){
file.deltaBytesSent = bytesSent - file.deltaBytesSent;
file.trackedBytesSent += file.deltaBytesSent;
const totalBytesSent = this.state.upload.totalBytesSent + file.deltaBytesSent;
const progress = totalBytesSent / this.state.upload.totalBytes * 100;
this.setUploadState({
progress,
totalBytesSent,
lastUpdated: now
});
}
})
.on("transformcompleted", (file, total) => {
if (this.dz._resizeMap) this.dz._resizeMap[file.name] = this.dz._taskInfo.resizeSize / Math.max(file.width, file.height);
if (this.dz.options.resizeWidth) this.setUploadState({resizedImages: total});
@ -192,29 +205,89 @@ class ProjectListItem extends React.Component {
.on("transformend", () => {
this.setUploadState({resizing: false, uploading: true});
})
.on("completemultiple", (files) => {
// Check
const invalidFilesCount = files.filter(file => file.status !== "success").length;
let success = files.length > 0 && invalidFilesCount === 0;
.on("complete", (file) => {
// Retry
const retry = () => {
const MAX_RETRIES = 10;
if (file.retries < MAX_RETRIES){
// Update progress
const totalBytesSent = this.state.upload.totalBytesSent - file.trackedBytesSent;
const progress = totalBytesSent / this.state.upload.totalBytes * 100;
this.setUploadState({
progress,
totalBytesSent,
});
file.status = Dropzone.QUEUED;
file.deltaBytesSent = 0;
file.trackedBytesSent = 0;
file.retries++;
this.dz.processQueue();
}else{
throw new Error(`Cannot upload ${file.name}, exceeded max retries (${MAX_RETRIES})`);
}
};
// All files have uploaded!
if (success){
this.setUploadState({uploading: false});
try{
let response = JSON.parse(files[0].xhr.response);
if (!response.id) throw new Error(`Expected id field, but none given (${response})`);
this.newTaskAdded();
if (file.status === "error"){
retry();
}else{
// Check response
let response = JSON.parse(file.xhr.response);
if (response.success){
// Update progress by removing the tracked progress and
// use the file size as the true number of bytes
let totalBytesSent = this.state.upload.totalBytesSent + file.size;
if (file.trackedBytesSent) totalBytesSent -= file.trackedBytesSent;
const progress = totalBytesSent / this.state.upload.totalBytes * 100;
this.setUploadState({
progress,
totalBytesSent,
uploadedCount: this.state.upload.uploadedCount + 1
});
this.dz.processQueue();
}else{
retry();
}
}
}catch(e){
this.setUploadState({error: `Invalid response from server: ${e.message}`, uploading: false})
this.setUploadState({error: `${e.message}`, uploading: false});
this.dz.cancelUpload();
}
})
.on("queuecomplete", () => {
const remainingFilesCount = this.state.upload.totalCount - this.state.upload.uploadedCount;
if (remainingFilesCount === 0){
// All files have uploaded!
this.setUploadState({uploading: false});
$.ajax({
url: `/api/projects/${this.state.data.id}/tasks/${this.dz._taskInfo.id}/commit/`,
contentType: 'application/json',
dataType: 'json',
type: 'POST'
}).done((task) => {
if (task && task.id){
this.newTaskAdded();
}else{
this.setUploadState({error: `Cannot create new task. Invalid response from server: ${JSON.stringify(task)}`});
}
}).fail(() => {
this.setUploadState({error: "Cannot create new task. Please try again later."});
});
}else if (this.dz.getQueuedFiles() === 0){
// Done but didn't upload all?
this.setUploadState({
totalCount: this.state.upload.totalCount - remainingFilesCount,
uploading: false,
error: `${remainingFilesCount} files cannot be uploaded. As a reminder, only images (.jpg, .png) and GCP files (.txt) can be uploaded. Try again.`
});
}
}else{
this.setUploadState({
totalCount: this.state.upload.totalCount - invalidFilesCount,
uploading: false,
error: `${invalidFilesCount} files cannot be uploaded. As a reminder, only images (.jpg, .png) and GCP files (.txt) can be uploaded. Try again.`
});
}
})
.on("reset", () => {
this.resetUploadState();
@ -223,20 +296,6 @@ class ProjectListItem extends React.Component {
if (!this.state.upload.editing){
this.resetUploadState();
}
})
.on("sending", (file, xhr, formData) => {
const taskInfo = this.dz._taskInfo;
// Safari does not have support for has on FormData
// as of December 2017
if (!formData.has || !formData.has("name")) formData.append("name", taskInfo.name);
if (!formData.has || !formData.has("options")) formData.append("options", JSON.stringify(taskInfo.options));
if (!formData.has || !formData.has("processing_node")) formData.append("processing_node", taskInfo.selectedNode.id);
if (!formData.has || !formData.has("auto_processing_node")) formData.append("auto_processing_node", taskInfo.selectedNode.key == "auto");
if (taskInfo.resizeMode === ResizeModes.YES){
if (!formData.has || !formData.has("resize_to")) formData.append("resize_to", taskInfo.resizeSize);
}
});
}
}
@ -303,9 +362,38 @@ class ProjectListItem extends React.Component {
this.setUploadState({uploading: true, editing: false});
}
setTimeout(() => {
this.dz.processQueue();
}, 1);
// Create task
const formData = {
name: taskInfo.name,
options: taskInfo.options,
processing_node: taskInfo.selectedNode.id,
auto_processing_node: taskInfo.selectedNode.key == "auto",
partial: true
};
if (taskInfo.resizeMode === ResizeModes.YES){
formData["resize_to"] = taskInfo.resizeSize
}
$.ajax({
url: `/api/projects/${this.state.data.id}/tasks/`,
contentType: 'application/json',
data: JSON.stringify(formData),
dataType: 'json',
type: 'POST'
}).done((task) => {
if (task && task.id){
console.log(this.dz._taskInfo);
this.dz._taskInfo.id = task.id;
this.dz.options.url = `/api/projects/${this.state.data.id}/tasks/${task.id}/upload/`;
this.dz.processQueue();
}else{
this.setState({error: `Cannot create new task. Invalid response from server: ${JSON.stringify(task)}`});
this.handleTaskCanceled();
}
}).fail(() => {
this.setState({error: "Cannot create new task. Please try again later."});
this.handleTaskCanceled();
});
}
handleTaskCanceled = () => {

3
app/static/app/js/components/TaskListItem.jsx
Wyświetl plik

@ -570,6 +570,9 @@ class TaskListItem extends React.Component {
statusLabel = getStatusLabel("Set a processing node");
statusIcon = "fa fa-hourglass-3";
showEditLink = true;
}else if (task.partial && !task.pending_action){
statusIcon = "fa fa-hourglass-3";
statusLabel = getStatusLabel("Waiting for image upload...");
}else{
let progress = 100;
let type = 'done';

7
worker/tasks.py
Wyświetl plik

@ -124,10 +124,11 @@ def get_pending_tasks():
# Or that need one assigned (via auto)
# or tasks that need a status update
# or tasks that have a pending action
return Task.objects.filter(Q(processing_node__isnull=True, auto_processing_node=True) |
# no partial tasks allowed
return Task.objects.filter(Q(processing_node__isnull=True, auto_processing_node=True, partial=False) |
Q(Q(status=None) | Q(status__in=[status_codes.QUEUED, status_codes.RUNNING]),
processing_node__isnull=False) |
Q(pending_action__isnull=False))
processing_node__isnull=False, partial=False) |
Q(pending_action__isnull=False, partial=False))
@app.task
def process_pending_tasks():