kopia lustrzana https://github.com/OpenDroneMap/WebODM
commit
7bb818dda9
|
@ -1,9 +1,11 @@
|
|||
import os
|
||||
import re
|
||||
import shutil
|
||||
from wsgiref.util import FileWrapper
|
||||
|
||||
import mimetypes
|
||||
|
||||
from shutil import copyfileobj
|
||||
from shutil import copyfileobj, move
|
||||
from django.core.exceptions import ObjectDoesNotExist, SuspiciousFileOperation, ValidationError
|
||||
from django.core.files.uploadedfile import InMemoryUploadedFile
|
||||
from django.db import transaction
|
||||
|
@ -23,7 +25,7 @@ from .common import get_and_check_project, get_asset_download_filename
|
|||
from .tags import TagsField
|
||||
from app.security import path_traversal_check
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
||||
from webodm import settings
|
||||
|
||||
def flatten_files(request_files):
|
||||
# MultiValueDict in, flat array of files out
|
||||
|
@ -420,18 +422,52 @@ class TaskAssetsImport(APIView):
|
|||
if import_url and len(files) > 0:
|
||||
raise exceptions.ValidationError(detail=_("Cannot create task, either specify a URL or upload 1 file."))
|
||||
|
||||
chunk_index = request.data.get('dzchunkindex')
|
||||
uuid = request.data.get('dzuuid')
|
||||
total_chunk_count = request.data.get('dztotalchunkcount', None)
|
||||
|
||||
# Chunked upload?
|
||||
tmp_upload_file = None
|
||||
if len(files) > 0 and chunk_index is not None and uuid is not None and total_chunk_count is not None:
|
||||
byte_offset = request.data.get('dzchunkbyteoffset', 0)
|
||||
|
||||
try:
|
||||
chunk_index = int(chunk_index)
|
||||
byte_offset = int(byte_offset)
|
||||
total_chunk_count = int(total_chunk_count)
|
||||
except ValueError:
|
||||
raise exceptions.ValidationError(detail="Some parameters are not integers")
|
||||
uuid = re.sub('[^0-9a-zA-Z-]+', "", uuid)
|
||||
|
||||
tmp_upload_file = os.path.join(settings.FILE_UPLOAD_TEMP_DIR, f"{uuid}.upload")
|
||||
if os.path.isfile(tmp_upload_file) and chunk_index == 0:
|
||||
os.unlink(tmp_upload_file)
|
||||
|
||||
with open(tmp_upload_file, 'ab') as fd:
|
||||
fd.seek(byte_offset)
|
||||
if isinstance(files[0], InMemoryUploadedFile):
|
||||
for chunk in files[0].chunks():
|
||||
fd.write(chunk)
|
||||
else:
|
||||
with open(files[0].temporary_file_path(), 'rb') as file:
|
||||
fd.write(file.read())
|
||||
|
||||
if chunk_index + 1 < total_chunk_count:
|
||||
return Response({'uploaded': True}, status=status.HTTP_200_OK)
|
||||
|
||||
# Ready to import
|
||||
with transaction.atomic():
|
||||
task = models.Task.objects.create(project=project,
|
||||
auto_processing_node=False,
|
||||
name=task_name,
|
||||
import_url=import_url if import_url else "file://all.zip",
|
||||
status=status_codes.RUNNING,
|
||||
pending_action=pending_actions.IMPORT)
|
||||
auto_processing_node=False,
|
||||
name=task_name,
|
||||
import_url=import_url if import_url else "file://all.zip",
|
||||
status=status_codes.RUNNING,
|
||||
pending_action=pending_actions.IMPORT)
|
||||
task.create_task_directories()
|
||||
destination_file = task.assets_path("all.zip")
|
||||
|
||||
if len(files) > 0:
|
||||
destination_file = task.assets_path("all.zip")
|
||||
|
||||
# Non-chunked file import
|
||||
if tmp_upload_file is None and len(files) > 0:
|
||||
with open(destination_file, 'wb+') as fd:
|
||||
if isinstance(files[0], InMemoryUploadedFile):
|
||||
for chunk in files[0].chunks():
|
||||
|
@ -439,6 +475,9 @@ class TaskAssetsImport(APIView):
|
|||
else:
|
||||
with open(files[0].temporary_file_path(), 'rb') as file:
|
||||
copyfileobj(file, fd)
|
||||
elif tmp_upload_file is not None:
|
||||
# Move
|
||||
shutil.move(tmp_upload_file, destination_file)
|
||||
|
||||
worker_tasks.process_task.delay(task.id)
|
||||
|
||||
|
|
|
@ -3,6 +3,7 @@ import rio_tiler.utils
|
|||
from rasterio.enums import ColorInterp
|
||||
from rasterio.crs import CRS
|
||||
from rasterio.features import bounds as featureBounds
|
||||
from rasterio.errors import NotGeoreferencedWarning
|
||||
import urllib
|
||||
import os
|
||||
from .common import get_asset_download_filename
|
||||
|
@ -16,7 +17,7 @@ from rio_tiler.models import Metadata as RioMetadata
|
|||
from rio_tiler.profiles import img_profiles
|
||||
from rio_tiler.colormap import cmap as colormap, apply_cmap
|
||||
from rio_tiler.io import COGReader
|
||||
from rio_tiler.errors import InvalidColorMapName
|
||||
from rio_tiler.errors import InvalidColorMapName, AlphaBandWarning
|
||||
import numpy as np
|
||||
from .custom_colormaps_helper import custom_colormaps
|
||||
from app.raster_utils import extension_for_export_format, ZOOM_EXTRA_LEVELS
|
||||
|
@ -28,7 +29,13 @@ from rest_framework import exceptions
|
|||
from rest_framework.response import Response
|
||||
from worker.tasks import export_raster, export_pointcloud
|
||||
from django.utils.translation import gettext as _
|
||||
import warnings
|
||||
|
||||
# Disable: NotGeoreferencedWarning: Dataset has no geotransform, gcps, or rpcs. The identity matrix be returned.
|
||||
warnings.filterwarnings("ignore", category=NotGeoreferencedWarning)
|
||||
|
||||
# Disable: Alpha band was removed from the output data array
|
||||
warnings.filterwarnings("ignore", category=AlphaBandWarning)
|
||||
|
||||
for custom_colormap in custom_colormaps:
|
||||
colormap = colormap.register(custom_colormap)
|
||||
|
|
|
@ -53,7 +53,8 @@ class ImportTaskPanel extends React.Component {
|
|||
clickable: this.uploadButton,
|
||||
chunkSize: 2147483647,
|
||||
timeout: 2147483647,
|
||||
|
||||
chunking: true,
|
||||
chunkSize: 16000000, // 16MB
|
||||
headers: {
|
||||
[csrf.header]: csrf.token
|
||||
}
|
||||
|
@ -69,6 +70,7 @@ class ImportTaskPanel extends React.Component {
|
|||
this.setState({uploading: false, progress: 0, totalBytes: 0, totalBytesSent: 0});
|
||||
})
|
||||
.on("uploadprogress", (file, progress, bytesSent) => {
|
||||
if (progress == 100) return; // Workaround for chunked upload progress bar jumping around
|
||||
this.setState({
|
||||
progress,
|
||||
totalBytes: file.size,
|
||||
|
|
|
@ -482,7 +482,11 @@ _('Example:'),
|
|||
});
|
||||
new AddOverlayCtrl().addTo(this.map);
|
||||
|
||||
this.map.fitWorld();
|
||||
this.map.fitBounds([
|
||||
[13.772919746115805,
|
||||
45.664640939831735],
|
||||
[13.772825784981254,
|
||||
45.664591558975154]]);
|
||||
this.map.attributionControl.setPrefix("");
|
||||
|
||||
this.setState({showLoading: true});
|
||||
|
|
|
@ -166,3 +166,54 @@ class TestApiTask(BootTransactionTestCase):
|
|||
self.assertEqual(corrupted_task.status, status_codes.FAILED)
|
||||
self.assertTrue("Invalid" in corrupted_task.last_error)
|
||||
|
||||
# Test chunked upload import
|
||||
assets_file = open(assets_path, 'rb')
|
||||
assets_size = os.path.getsize(assets_path)
|
||||
chunk_1_size = assets_size // 2
|
||||
chunk_1_path = os.path.join(os.path.dirname(assets_path), "1.zip")
|
||||
chunk_2_path = os.path.join(os.path.dirname(assets_path), "2.zip")
|
||||
with open(chunk_1_path, 'wb') as f:
|
||||
assets_file.seek(0)
|
||||
f.write(assets_file.read(chunk_1_size))
|
||||
with open(chunk_2_path, 'wb') as f:
|
||||
f.write(assets_file.read())
|
||||
|
||||
chunk_1 = open(chunk_1_path, 'rb')
|
||||
chunk_2 = open(chunk_2_path, 'rb')
|
||||
assets_file.close()
|
||||
|
||||
res = client.post("/api/projects/{}/tasks/import".format(project.id), {
|
||||
'file': [chunk_1],
|
||||
'dzuuid': 'abc-test',
|
||||
'dzchunkindex': 0,
|
||||
'dztotalchunkcount': 2,
|
||||
'dzchunkbyteoffset': 0
|
||||
}, format="multipart")
|
||||
self.assertEqual(res.status_code, status.HTTP_200_OK)
|
||||
self.assertTrue(res.data['uploaded'])
|
||||
chunk_1.close()
|
||||
|
||||
res = client.post("/api/projects/{}/tasks/import".format(project.id), {
|
||||
'file': [chunk_2],
|
||||
'dzuuid': 'abc-test',
|
||||
'dzchunkindex': 1,
|
||||
'dztotalchunkcount': 2,
|
||||
'dzchunkbyteoffset': chunk_1_size
|
||||
}, format="multipart")
|
||||
self.assertEqual(res.status_code, status.HTTP_201_CREATED)
|
||||
chunk_2.close()
|
||||
|
||||
file_import_task = Task.objects.get(id=res.data['id'])
|
||||
# Wait for completion
|
||||
c = 0
|
||||
while c < 10:
|
||||
worker.tasks.process_pending_tasks()
|
||||
file_import_task.refresh_from_db()
|
||||
if file_import_task.status == status_codes.COMPLETED:
|
||||
break
|
||||
c += 1
|
||||
time.sleep(1)
|
||||
|
||||
self.assertEqual(file_import_task.import_url, "file://all.zip")
|
||||
self.assertEqual(file_import_task.images_count, 1)
|
||||
|
||||
|
|
|
@ -4,10 +4,13 @@ let BundleTracker = require('webpack-bundle-tracker');
|
|||
let ExtractTextPlugin = require('extract-text-webpack-plugin');
|
||||
let LiveReloadPlugin = require('webpack-livereload-plugin');
|
||||
|
||||
module.exports = {
|
||||
mode: 'development',
|
||||
context: __dirname,
|
||||
const mode = process.argv.indexOf("production") !== -1 ? "production" : "development";
|
||||
console.log(`Webpack mode: ${mode}`);
|
||||
|
||||
module.exports = {
|
||||
mode,
|
||||
context: __dirname,
|
||||
|
||||
entry: {
|
||||
main: ['./app/static/app/js/main.jsx'],
|
||||
Console: ['./app/static/app/js/Console.jsx'],
|
||||
|
|
Ładowanie…
Reference in New Issue