kopia lustrzana https://github.com/OpenDroneMap/WebODM
Chunked import uploads
rodzic
ac78176f2d
commit
e2b7de81d3
|
@ -1,9 +1,11 @@
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
from wsgiref.util import FileWrapper
|
from wsgiref.util import FileWrapper
|
||||||
|
|
||||||
import mimetypes
|
import mimetypes
|
||||||
|
|
||||||
from shutil import copyfileobj
|
from shutil import copyfileobj, move
|
||||||
from django.core.exceptions import ObjectDoesNotExist, SuspiciousFileOperation, ValidationError
|
from django.core.exceptions import ObjectDoesNotExist, SuspiciousFileOperation, ValidationError
|
||||||
from django.core.files.uploadedfile import InMemoryUploadedFile
|
from django.core.files.uploadedfile import InMemoryUploadedFile
|
||||||
from django.db import transaction
|
from django.db import transaction
|
||||||
|
@ -23,7 +25,7 @@ from .common import get_and_check_project, get_asset_download_filename
|
||||||
from .tags import TagsField
|
from .tags import TagsField
|
||||||
from app.security import path_traversal_check
|
from app.security import path_traversal_check
|
||||||
from django.utils.translation import gettext_lazy as _
|
from django.utils.translation import gettext_lazy as _
|
||||||
|
from webodm import settings
|
||||||
|
|
||||||
def flatten_files(request_files):
|
def flatten_files(request_files):
|
||||||
# MultiValueDict in, flat array of files out
|
# MultiValueDict in, flat array of files out
|
||||||
|
@ -420,18 +422,52 @@ class TaskAssetsImport(APIView):
|
||||||
if import_url and len(files) > 0:
|
if import_url and len(files) > 0:
|
||||||
raise exceptions.ValidationError(detail=_("Cannot create task, either specify a URL or upload 1 file."))
|
raise exceptions.ValidationError(detail=_("Cannot create task, either specify a URL or upload 1 file."))
|
||||||
|
|
||||||
|
chunk_index = request.data.get('dzchunkindex')
|
||||||
|
uuid = request.data.get('dzuuid')
|
||||||
|
total_chunk_count = request.data.get('dztotalchunkcount', None)
|
||||||
|
|
||||||
|
# Chunked upload?
|
||||||
|
tmp_upload_file = None
|
||||||
|
if len(files) > 0 and chunk_index is not None and uuid is not None and total_chunk_count is not None:
|
||||||
|
byte_offset = request.data.get('dzchunkbyteoffset', 0)
|
||||||
|
|
||||||
|
try:
|
||||||
|
chunk_index = int(chunk_index)
|
||||||
|
byte_offset = int(byte_offset)
|
||||||
|
total_chunk_count = int(total_chunk_count)
|
||||||
|
except ValueError:
|
||||||
|
raise exceptions.ValidationError(detail="chunkIndex is not an int")
|
||||||
|
uuid = re.sub('[^0-9a-zA-Z-]+', "", uuid)
|
||||||
|
|
||||||
|
tmp_upload_file = os.path.join(settings.FILE_UPLOAD_TEMP_DIR, f"{uuid}.upload")
|
||||||
|
if os.path.isfile(tmp_upload_file) and chunk_index == 0:
|
||||||
|
os.unlink(tmp_upload_file)
|
||||||
|
|
||||||
|
with open(tmp_upload_file, 'ab') as fd:
|
||||||
|
fd.seek(byte_offset)
|
||||||
|
if isinstance(files[0], InMemoryUploadedFile):
|
||||||
|
for chunk in files[0].chunks():
|
||||||
|
fd.write(chunk)
|
||||||
|
else:
|
||||||
|
with open(files[0].temporary_file_path(), 'rb') as file:
|
||||||
|
fd.write(file.read())
|
||||||
|
|
||||||
|
if chunk_index + 1 < total_chunk_count:
|
||||||
|
return Response({'uploaded': True}, status=status.HTTP_200_OK)
|
||||||
|
|
||||||
|
# Ready to import
|
||||||
with transaction.atomic():
|
with transaction.atomic():
|
||||||
task = models.Task.objects.create(project=project,
|
task = models.Task.objects.create(project=project,
|
||||||
auto_processing_node=False,
|
auto_processing_node=False,
|
||||||
name=task_name,
|
name=task_name,
|
||||||
import_url=import_url if import_url else "file://all.zip",
|
import_url=import_url if import_url else "file://all.zip",
|
||||||
status=status_codes.RUNNING,
|
status=status_codes.RUNNING,
|
||||||
pending_action=pending_actions.IMPORT)
|
pending_action=pending_actions.IMPORT)
|
||||||
task.create_task_directories()
|
task.create_task_directories()
|
||||||
|
destination_file = task.assets_path("all.zip")
|
||||||
|
|
||||||
if len(files) > 0:
|
# Non-chunked file import
|
||||||
destination_file = task.assets_path("all.zip")
|
if tmp_upload_file is None and len(files) > 0:
|
||||||
|
|
||||||
with open(destination_file, 'wb+') as fd:
|
with open(destination_file, 'wb+') as fd:
|
||||||
if isinstance(files[0], InMemoryUploadedFile):
|
if isinstance(files[0], InMemoryUploadedFile):
|
||||||
for chunk in files[0].chunks():
|
for chunk in files[0].chunks():
|
||||||
|
@ -439,6 +475,9 @@ class TaskAssetsImport(APIView):
|
||||||
else:
|
else:
|
||||||
with open(files[0].temporary_file_path(), 'rb') as file:
|
with open(files[0].temporary_file_path(), 'rb') as file:
|
||||||
copyfileobj(file, fd)
|
copyfileobj(file, fd)
|
||||||
|
elif tmp_upload_file is not None:
|
||||||
|
# Move
|
||||||
|
shutil.move(tmp_upload_file, destination_file)
|
||||||
|
|
||||||
worker_tasks.process_task.delay(task.id)
|
worker_tasks.process_task.delay(task.id)
|
||||||
|
|
||||||
|
|
|
@ -53,7 +53,8 @@ class ImportTaskPanel extends React.Component {
|
||||||
clickable: this.uploadButton,
|
clickable: this.uploadButton,
|
||||||
chunkSize: 2147483647,
|
chunkSize: 2147483647,
|
||||||
timeout: 2147483647,
|
timeout: 2147483647,
|
||||||
|
chunking: true,
|
||||||
|
chunkSize: 16000000, // 16MB
|
||||||
headers: {
|
headers: {
|
||||||
[csrf.header]: csrf.token
|
[csrf.header]: csrf.token
|
||||||
}
|
}
|
||||||
|
@ -69,6 +70,7 @@ class ImportTaskPanel extends React.Component {
|
||||||
this.setState({uploading: false, progress: 0, totalBytes: 0, totalBytesSent: 0});
|
this.setState({uploading: false, progress: 0, totalBytes: 0, totalBytesSent: 0});
|
||||||
})
|
})
|
||||||
.on("uploadprogress", (file, progress, bytesSent) => {
|
.on("uploadprogress", (file, progress, bytesSent) => {
|
||||||
|
if (progress == 100) return; // Workaround for chunked upload progress bar jumping around
|
||||||
this.setState({
|
this.setState({
|
||||||
progress,
|
progress,
|
||||||
totalBytes: file.size,
|
totalBytes: file.size,
|
||||||
|
|
Ładowanie…
Reference in New Issue