Merge pull request #1070 from AIDI-solar/import_from_file_system

Added possibility to import file from system.

app/models/task.py

@@ -12,14 +12,14 @@ import piexif
 import re
 
 import zipfile
-
+from shutil import copyfile
 import requests
 from PIL import Image
 from django.contrib.gis.gdal import GDALRaster
 from django.contrib.gis.gdal import OGRGeometry
 from django.contrib.gis.geos import GEOSGeometry
 from django.contrib.postgres import fields
-from django.core.exceptions import ValidationError
+from django.core.exceptions import ValidationError, SuspiciousFileOperation
 from django.db import models
 from django.db import transaction
 from django.db import connection
@@ -31,6 +31,7 @@ from django.contrib.gis.db.models.fields import GeometryField
 
 from app.cogeo import assure_cogeo
 from app.testwatch import testWatch
+from app.api.common import path_traversal_check
 from nodeodm import status_codes
 from nodeodm.models import ProcessingNode
 from pyodm.exceptions import NodeResponseError, NodeConnectionError, NodeServerError, OdmError
@@ -462,35 +463,48 @@ class Task(models.Model):
         self.save()
 
         zip_path = self.assets_path("all.zip")
-
+        # Import assets file from mounted system volume (media-dir)/imports by relative path.
+        # Import file from relative path.
         if self.import_url and not os.path.exists(zip_path):
-            try:
+            if self.import_url.startswith("file://"):
-                # TODO: this is potentially vulnerable to a zip bomb attack
+                imports_folder_path = os.path.join(settings.MEDIA_ROOT, "imports")
-                #       mitigated by the fact that a valid account is needed to
+                unsafe_path_to_import_file = os.path.join(settings.MEDIA_ROOT, "imports", self.import_url.replace("file://", ""))
-                #       import tasks
+                # check is file placed in shared media folder in /imports directory without traversing
-                logger.info("Importing task assets from {} for {}".format(self.import_url, self))
+                try:
-                download_stream = requests.get(self.import_url, stream=True, timeout=10)
+                    checked_path_to_file = path_traversal_check(unsafe_path_to_import_file, imports_folder_path)
-                content_length = download_stream.headers.get('content-length')
+                    if os.path.isfile(checked_path_to_file):
-                total_length = int(content_length) if content_length is not None else None
+                        copyfile(checked_path_to_file, zip_path)
-                downloaded = 0
+                except SuspiciousFileOperation as e:
-                last_update = 0
+                    logger.error("Error due importing assets from {} for {} in cause of path checking error".format(self.import_url, self))
+                    raise NodeServerError(e)
+            else:
+                try:
+                    # TODO: this is potentially vulnerable to a zip bomb attack
+                    #       mitigated by the fact that a valid account is needed to
+                    #       import tasks
+                    logger.info("Importing task assets from {} for {}".format(self.import_url, self))
+                    download_stream = requests.get(self.import_url, stream=True, timeout=10)
+                    content_length = download_stream.headers.get('content-length')
+                    total_length = int(content_length) if content_length is not None else None
+                    downloaded = 0
+                    last_update = 0
 
-                with open(zip_path, 'wb') as fd:
+                    with open(zip_path, 'wb') as fd:
-                    for chunk in download_stream.iter_content(4096):
+                        for chunk in download_stream.iter_content(4096):
-                        downloaded += len(chunk)
+                            downloaded += len(chunk)
 
-                        if time.time() - last_update >= 2:
+                            if time.time() - last_update >= 2:
-                            # Update progress
+                                # Update progress
-                            if total_length is not None:
+                                if total_length is not None:
-                                Task.objects.filter(pk=self.id).update(running_progress=(float(downloaded) / total_length) * 0.9)
+                                    Task.objects.filter(pk=self.id).update(running_progress=(float(downloaded) / total_length) * 0.9)
 
-                            self.check_if_canceled()
+                                self.check_if_canceled()
-                            last_update = time.time()
+                                last_update = time.time()
 
-                        fd.write(chunk)
+                            fd.write(chunk)
 
-            except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, ReadTimeoutError) as e:
+                except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, ReadTimeoutError) as e:
-                raise NodeServerError(e)
+                    raise NodeServerError(e)
 
         self.refresh_from_db()