diff --git a/app/uploadhandler.py b/app/uploadhandler.py new file mode 100644 index 00000000..f3e02700 --- /dev/null +++ b/app/uploadhandler.py @@ -0,0 +1,60 @@ +import tempfile + +import errno +from django.core.files.uploadedfile import UploadedFile +from django.core.files.uploadhandler import FileUploadHandler + +from django.conf import settings + +""" +Same as Django's TemporaryFileUploadHandler, but closes the file +after the upload is completed as not to hog the number of open fd limits +(see https://github.com/OpenDroneMap/WebODM/issues/233) +""" +class TemporaryFileUploadHandler(FileUploadHandler): + """ + Upload handler that streams data into a temporary file. + """ + def __init__(self, *args, **kwargs): + super(TemporaryFileUploadHandler, self).__init__(*args, **kwargs) + + def new_file(self, *args, **kwargs): + """ + Create the file object to append to as data is coming in. + """ + super(TemporaryFileUploadHandler, self).new_file(*args, **kwargs) + self.file = ClosedTemporaryUploadedFile(self.file_name, self.content_type, 0, self.charset, self.content_type_extra) + + def receive_data_chunk(self, raw_data, start): + self.file.write(raw_data) + + def file_complete(self, file_size): + self.file.seek(0) + self.file.size = file_size + self.file.close() # Close the file as not to hog the number of open files descriptors + return self.file + + +class ClosedTemporaryUploadedFile(UploadedFile): + """ + A file uploaded to a temporary location (i.e. stream-to-disk). + """ + def __init__(self, name, content_type, size, charset, content_type_extra=None): + file = tempfile.NamedTemporaryFile(suffix='.upload', dir=settings.FILE_UPLOAD_TEMP_DIR, delete=False) + super(ClosedTemporaryUploadedFile, self).__init__(file, name, content_type, size, charset, content_type_extra) + + def temporary_file_path(self): + """ + Returns the full path of this file. + """ + return self.file.name + + def close(self): + try: + return self.file.close() + except OSError as e: + if e.errno != errno.ENOENT: + # Means the file was moved or deleted before the tempfile + # could unlink it. Still sets self.file.close_called and + # calls self.file.file.close() before the exception + raise diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 23c4fac6..c23d817b 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -29,7 +29,7 @@ http { server { listen 8000 deferred; - client_max_body_size 20G; + client_max_body_size 0; # set the correct host(s) for your site server_name webodm.localhost; diff --git a/nodeodm/api_client.py b/nodeodm/api_client.py index afecb0f6..2c565b54 100644 --- a/nodeodm/api_client.py +++ b/nodeodm/api_client.py @@ -59,8 +59,16 @@ class ApiClient: :param options: options to be used for processing ([{'name': optionName, 'value': optionValue}, ...]) :return: UUID or error """ + + # Equivalent as passing the open file descriptor, since requests + # eventually calls read(), but this way we make sure to close + # the file prior to reading the next, so we don't run into open file OS limits + def read_file(path): + with open(path, 'rb') as f: + return f.read() + files = [('images', - (os.path.basename(image), open(image, 'rb'), (mimetypes.guess_type(image)[0] or "image/jpg")) + (os.path.basename(image), read_file(image), (mimetypes.guess_type(image)[0] or "image/jpg")) ) for image in images] return requests.post(self.url("/task/new"), files=files, diff --git a/webodm/settings.py b/webodm/settings.py index 97d594a8..4762ff32 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -172,6 +172,11 @@ STATICFILES_DIRS = [ FILE_UPLOAD_MAX_MEMORY_SIZE = 4718592 # 4.5 MB DATA_UPLOAD_MAX_NUMBER_FIELDS = None +FILE_UPLOAD_HANDLERS = [ + 'django.core.files.uploadhandler.MemoryFileUploadHandler', + 'app.uploadhandler.TemporaryFileUploadHandler', # Ours doesn't keep file descriptors open by default +] + # Webpack WEBPACK_LOADER = { 'DEFAULT': {