Changed orthophoto import mechanism, dropped OffDbRaster

app/api/tasks.py

@@ -39,7 +39,7 @@ class TaskSerializer(serializers.ModelSerializer):
 
     class Meta:
         model = models.Task
-        exclude = ('processing_lock', 'console_output', 'orthophoto', )
+        exclude = ('processing_lock', 'console_output', 'orthophoto_extent', )
         read_only_fields = ('processing_time', 'status', 'last_error', 'created_at', 'pending_action', )
 
 class TaskViewSet(viewsets.ViewSet):
@@ -48,7 +48,7 @@ class TaskViewSet(viewsets.ViewSet):
     A task represents a set of images and other input to be sent to a processing node.
     Once a processing node completes processing, results are stored in the task.
     """
-    queryset = models.Task.objects.all().defer('orthophoto', 'console_output')
+    queryset = models.Task.objects.all().defer('orthophoto_extent', 'console_output')
     
     # We don't use object level permissions on tasks, relying on
     # project's object permissions instead (but standard model permissions still apply)
@@ -174,7 +174,7 @@ class TaskViewSet(viewsets.ViewSet):
 
 
 class TaskNestedView(APIView):
-    queryset = models.Task.objects.all().defer('orthophoto', 'console_output')
+    queryset = models.Task.objects.all().defer('orthophoto_extent', 'console_output')
 
     def get_and_check_task(self, request, pk, project_pk, annotate={}):
         get_and_check_project(request, project_pk)
@@ -204,16 +204,14 @@ class TaskTilesJson(TaskNestedView):
         """
         Get tile.json for this tasks's orthophoto
         """
-        task = self.get_and_check_task(request, pk, project_pk, annotate={
-                'orthophoto_area': Envelope(Cast("orthophoto", GeometryField()))
-            })
+        task = self.get_and_check_task(request, pk, project_pk)
 
-        if task.orthophoto_area is None:
+        if task.orthophoto_extent is None:
             raise exceptions.ValidationError("An orthophoto has not been processed for this task. Tiles are not available.")
 
         json = get_tile_json(task.name, [
                 '/api/projects/{}/tasks/{}/tiles/{{z}}/{{x}}/{{y}}.png'.format(task.project.id, task.id)
-            ], task.orthophoto_area.extent)
+            ], task.orthophoto_extent.extent)
         return Response(json)
 
 

app/models.py

@@ -6,6 +6,8 @@ import zipfile
 from django.contrib.auth.models import User
 from django.contrib.gis.gdal import GDALException
 from django.contrib.gis.gdal import GDALRaster
+from django.contrib.gis.gdal import OGRGeometry
+from django.contrib.gis.geos import GEOSGeometry
 from django.contrib.postgres import fields
 from django.core.exceptions import ValidationError
 from django.db import models
@@ -72,7 +74,7 @@ class Project(models.Model):
     def get_tile_json_data(self):
         return [task.get_tile_json_data() for task in self.task_set.filter(
                     status=status_codes.COMPLETED,
-                    orthophoto__isnull=False
+                    orthophoto_extent__isnull=False
                 ).only('id', 'project_id')]
 
     class Meta:
@@ -190,10 +192,6 @@ class Task(models.Model):
                         logger.info("Changing {} to {}".format(prev_name, img))
                         img.save()
 
-                if self.orthophoto is not None:
-                    new_orthophoto_path = os.path.realpath(full_task_directory_path(self.id, new_project_id, "assets", "odm_orthophoto", "odm_orthophoto_4326.tif"))
-                    logger.info("Changing orthophoto path to {}".format(new_orthophoto_path))
-                    self.orthophoto = GDALRaster(new_orthophoto_path, write=True)
             else:
                 logger.warning("Project changed for task {}, but either {} doesn't exist, or {} already exists. This doesn't look right, so we will not move any files.".format(self,
                                                                                                              old_task_folder,
@@ -207,11 +205,7 @@ class Task(models.Model):
             self.__original_project_id = self.project.id
 
         # Autovalidate on save
-        try:
-            self.full_clean()
-        except GDALException as e:
-            logger.warning("Problem while handling GDAL raster: {}. We're going to attempt to remove the reference to it...".format(e))
-            self.orthophoto = None
+        self.full_clean()
 
         super(Task, self).save(*args, **kwargs)
 
@@ -416,17 +410,17 @@ class Task(models.Model):
 
                             logger.info("Extracted all.zip for {}".format(self))
 
-                            # Add to database orthophoto
+                            # Populate orthophoto_extent field
                             orthophoto_path = os.path.realpath(self.assets_path("odm_orthophoto", "odm_orthophoto.tif"))
                             if os.path.exists(orthophoto_path):
-                                orthophoto = GDALRaster(orthophoto_path, write=True)
+                                # Read extent and SRID
+                                orthophoto = GDALRaster(orthophoto_path)
+                                extent = OGRGeometry.from_bbox(orthophoto.extent)
 
-                                # We need to transform to 4326 before we can store it
-                                # as an offdb raster field
-                                orthophoto_4326_path = os.path.realpath(self.assets_path("odm_orthophoto", "odm_orthophoto_4326.tif"))
-                                self.orthophoto = orthophoto.transform(4326, 'GTiff', orthophoto_4326_path)
+                                # It will be implicitly transformed into the SRID of the model’s field
+                                self.orthophoto_extent = GEOSGeometry(extent.wkt, srid=orthophoto.srid)
 
-                                logger.info("Imported orthophoto {} for {}".format(orthophoto_4326_path, self))
+                                logger.info("Populated orthophoto_extent for {}".format(self))
 
                             # Remove old odm_texturing.zip archive (if any)
                             textured_model_archive = self.assets_path(self.get_textured_model_filename())
@@ -485,7 +479,7 @@ class Task(models.Model):
         if self.status == status_codes.COMPLETED:
             assets = list(self.ASSET_DOWNLOADS)
 
-            if self.orthophoto is None:
+            if self.orthophoto_extent is None:
                 assets.remove('geotiff')
                 assets.remove('las')
                 assets.remove('csv')

app/tests/test_api_task.py

@@ -9,6 +9,8 @@ from datetime import timedelta
 
 import requests
 from django.contrib.auth.models import User
+from django.contrib.gis.gdal import GDALRaster
+from django.contrib.gis.gdal import OGRGeometry
 from rest_framework import status
 from rest_framework.test import APIClient
 
@@ -329,8 +331,6 @@ class TestApiTask(BootTransactionTestCase):
 
         # Reassigning the task to another project should move its assets
         self.assertTrue(os.path.exists(full_task_directory_path(task.id, project.id)))
-        self.assertTrue(task.orthophoto is not None)
-        self.assertTrue('project/{}/'.format(project.id) in task.orthophoto.name)
         self.assertTrue(len(task.imageupload_set.all()) == 2)
         for image in task.imageupload_set.all():
             self.assertTrue('project/{}/'.format(project.id) in image.image.path)
@@ -341,8 +341,6 @@ class TestApiTask(BootTransactionTestCase):
         self.assertFalse(os.path.exists(full_task_directory_path(task.id, project.id)))
         self.assertTrue(os.path.exists(full_task_directory_path(task.id, other_project.id)))
 
-        self.assertTrue('project/{}/'.format(other_project.id) in task.orthophoto.name)
-
         for image in task.imageupload_set.all():
             self.assertTrue('project/{}/'.format(other_project.id) in image.image.path)
 
@@ -370,6 +368,9 @@ class TestApiTask(BootTransactionTestCase):
         self.assertFalse(os.path.exists(task.assets_path("odm_orthophoto", "odm_orthophoto.tif")))
         self.assertFalse(os.path.exists(task.assets_path("orthophoto_tiles")))
 
+        # orthophoto_extent should be none
+        self.assertTrue(task.orthophoto_extent is None)
+
         # Available assets should be missing the geotiff type
         # but others such as texturedmodel should be available
         res = client.get("/api/projects/{}/tasks/{}/".format(project.id, task.id))

app/tests/test_db.py

@@ -1,11 +1,7 @@
-from django.contrib.gis.gdal import GDALRaster
 from django.contrib.gis.geos import GEOSGeometry
-from django.db import InternalError
-from django.db import transaction
 
 from .classes import BootTestCase
 from app.models import Task, Project
-import os
 
 class TestApi(BootTestCase):
     def setUp(self):

app/views.py

@@ -49,7 +49,7 @@ def map(request, project_pk=None, task_pk=None):
             raise Http404()
         
         if task_pk is not None:
-            task = get_object_or_404(Task.objects.defer('orthophoto'), pk=task_pk, project=project)
+            task = get_object_or_404(Task.objects.defer('orthophoto_extent'), pk=task_pk, project=project)
             title = task.name
             tiles = [task.get_tile_json_data()]
         else:
@@ -74,7 +74,7 @@ def model_display(request, project_pk=None, task_pk=None):
             raise Http404()
 
         if task_pk is not None:
-            task = get_object_or_404(Task.objects.defer('orthophoto'), pk=task_pk, project=project)
+            task = get_object_or_404(Task.objects.defer('orthophoto_extent'), pk=task_pk, project=project)
             title = task.name
         else:
             raise Http404()