diff --git a/app/migrations/0001_initial.py b/app/migrations/0001_initial.py index 9f86966d..a96fd404 100644 --- a/app/migrations/0001_initial.py +++ b/app/migrations/0001_initial.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals import app.models -import app.postgis from django.conf import settings import django.contrib.postgres.fields.jsonb from django.db import migrations, models @@ -39,9 +38,6 @@ class Migration(migrations.Migration): ('deleting', models.BooleanField(db_index=True, default=False, help_text='Whether this project has been marked for deletion. Projects that have running tasks need to wait for tasks to be properly cleaned up before they can be deleted.')), ('owner', models.ForeignKey(help_text='The person who created the project', on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), ], - options={ - 'permissions': (('view_project', 'Can view project'),), - }, ), migrations.CreateModel( name='ProjectGroupObjectPermission', @@ -80,15 +76,12 @@ class Migration(migrations.Migration): ('options', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default={}, help_text='Options that are being used to process this task', validators=[app.models.validate_task_options])), ('console_output', models.TextField(blank=True, default='', help_text="Console output of the OpenDroneMap's process")), ('ground_control_points', models.FileField(blank=True, help_text='Optional Ground Control Points file to use for processing', null=True, upload_to=app.models.gcp_directory_path)), - ('orthophoto', app.postgis.OffDbRasterField(blank=True, help_text='Orthophoto created by OpenDroneMap', null=True, srid=4326)), + ('orthophoto', django.contrib.gis.db.models.RasterField(blank=True, help_text='Orthophoto created by OpenDroneMap', null=True, srid=4326)), ('created_at', models.DateTimeField(default=django.utils.timezone.now, help_text='Creation date')), ('pending_action', models.IntegerField(blank=True, choices=[(1, 'CANCEL'), (2, 'REMOVE'), (3, 'RESTART')], db_index=True, help_text='A requested action to be performed on the task. The selected action will be performed by the scheduler at the next iteration.', null=True)), ('processing_node', models.ForeignKey(blank=True, help_text='Processing node assigned to this task (or null if this task has not been associated yet)', null=True, on_delete=django.db.models.deletion.CASCADE, to='nodeodm.ProcessingNode')), ('project', models.ForeignKey(help_text='Project that this task belongs to', on_delete=django.db.models.deletion.CASCADE, to='app.Project')), ], - options={ - 'permissions': (('view_task', 'Can view task'),), - }, ), migrations.AddField( model_name='imageupload', diff --git a/app/models/preset.py b/app/models/preset.py index 5a571e03..03d7772c 100644 --- a/app/models/preset.py +++ b/app/models/preset.py @@ -12,7 +12,7 @@ logger = logging.getLogger('app.logger') class Preset(models.Model): owner = models.ForeignKey(User, blank=True, null=True, on_delete=models.CASCADE, help_text="The person who owns this preset") name = models.CharField(max_length=255, blank=False, null=False, help_text="A label used to describe the preset") - options = JSONField(default=list(), blank=True, help_text="Options that define this preset (same format as in a Task's options).", + options = JSONField(default=list, blank=True, help_text="Options that define this preset (same format as in a Task's options).", validators=[validate_task_options]) created_at = models.DateTimeField(default=timezone.now, help_text="Creation date") system = models.BooleanField(db_index=True, default=False, help_text="Whether this preset is available to every user in the system or just to its owner.") diff --git a/app/models/project.py b/app/models/project.py index 800dcbdb..0b488d6f 100644 --- a/app/models/project.py +++ b/app/models/project.py @@ -51,11 +51,6 @@ class Project(models.Model): ).filter(Q(orthophoto_extent__isnull=False) | Q(dsm_extent__isnull=False) | Q(dtm_extent__isnull=False)) .only('id', 'project_id')] - class Meta: - permissions = ( - ('view_project', 'Can view project'), - ) - @receiver(signals.post_save, sender=Project, dispatch_uid="project_post_save") def project_post_save(sender, instance, created, **kwargs): diff --git a/app/models/task.py b/app/models/task.py index 6b75254e..d0cf228a 100644 --- a/app/models/task.py +++ b/app/models/task.py @@ -198,8 +198,8 @@ class Task(models.Model): auto_processing_node = models.BooleanField(default=True, help_text="A flag indicating whether this task should be automatically assigned a processing node") status = models.IntegerField(choices=STATUS_CODES, db_index=True, null=True, blank=True, help_text="Current status of the task") last_error = models.TextField(null=True, blank=True, help_text="The last processing error received") - options = fields.JSONField(default=dict(), blank=True, help_text="Options that are being used to process this task", validators=[validate_task_options]) - available_assets = fields.ArrayField(models.CharField(max_length=80), default=list(), blank=True, help_text="List of available assets to download") + options = fields.JSONField(default=dict, blank=True, help_text="Options that are being used to process this task", validators=[validate_task_options]) + available_assets = fields.ArrayField(models.CharField(max_length=80), default=list, blank=True, help_text="List of available assets to download") console_output = models.TextField(null=False, default="", blank=True, help_text="Console output of the OpenDroneMap's process") orthophoto_extent = GeometryField(null=True, blank=True, srid=4326, help_text="Extent of the orthophoto created by OpenDroneMap") @@ -769,8 +769,3 @@ class Task(models.Model): except subprocess.CalledProcessError as e: logger.warning("Could not resize GCP file {}: {}".format(gcp_path, str(e))) return None - - class Meta: - permissions = ( - ('view_task', 'Can view task'), - ) diff --git a/app/permissions.py b/app/permissions.py deleted file mode 100644 index 39bb0c06..00000000 --- a/app/permissions.py +++ /dev/null @@ -1,15 +0,0 @@ -from rest_framework import permissions - -class GuardianObjectPermissions(permissions.DjangoObjectPermissions): - """ - Similar to `DjangoObjectPermissions`, but adding 'view' permissions. - """ - perms_map = { - 'GET': ['%(app_label)s.view_%(model_name)s'], - 'OPTIONS': ['%(app_label)s.view_%(model_name)s'], - 'HEAD': ['%(app_label)s.view_%(model_name)s'], - 'POST': ['%(app_label)s.add_%(model_name)s'], - 'PUT': ['%(app_label)s.change_%(model_name)s'], - 'PATCH': ['%(app_label)s.change_%(model_name)s'], - 'DELETE': ['%(app_label)s.delete_%(model_name)s'], - } \ No newline at end of file diff --git a/app/postgis.py b/app/postgis.py deleted file mode 100644 index e54f718d..00000000 --- a/app/postgis.py +++ /dev/null @@ -1,218 +0,0 @@ -import binascii -import struct - -from django.contrib.gis.db.backends.postgis.const import GDAL_TO_POSTGIS -from django.contrib.gis.db.backends.postgis.pgraster import ( - GDAL_TO_STRUCT, POSTGIS_HEADER_STRUCTURE, POSTGIS_TO_GDAL, - STRUCT_SIZE, - pack) -from django.contrib.gis.db.backends.postgis.pgraster import chunk, unpack -from django.contrib.gis.db.models.fields import RasterField, BaseSpatialField -from django.contrib.gis.gdal import GDALException -from django.contrib.gis.gdal import GDALRaster -from django.forms import ValidationError -from django.utils.translation import ugettext_lazy as _ - - -class OffDbRasterField(RasterField): - """ - Out-of-db Raster field for GeoDjango -- evaluates into GDALRaster objects. - """ - - description = _("Out-of-db Raster Field") - - def from_db_value(self, value, expression, connection, context): - return from_pgraster(value, True) - - def get_db_prep_save(self, value, connection): - """ - Prepare the value for saving in the database. - """ - if not value: - return None - else: - return to_pgraster(value, True) - - def get_db_prep_value(self, value, connection, prepared=False): - self._check_connection(connection) - # Prepare raster for writing to database. - if not prepared: - value = to_pgraster(value, True) - - # Call RasterField's base class get_db_prep_value - return BaseSpatialField.get_db_prep_value(self, value, connection, prepared) - - def get_raster_prep_value(self, value, is_candidate): - """ - Return a GDALRaster if conversion is successful, otherwise return None. - """ - if isinstance(value, GDALRaster): - return value - elif is_candidate: - try: - return GDALRaster(value) - except GDALException: - pass - elif isinstance(value, (dict, str)): - try: - return GDALRaster(value) - except GDALException: - raise ValueError("Couldn't create spatial object from lookup value '%s'." % value) - - -class POSTGIS_BANDTYPES(object): - BANDTYPE_FLAG_OFFDB = 1 << 7 - BANDTYPE_FLAG_HASNODATA = 1 << 6 - BANDTYPE_FLAG_ISNODATA = 1 << 5 - - -def from_pgraster(data, offdb = False): - """ - Convert a PostGIS HEX String into a dictionary. - """ - if data is None: - return - - # Split raster header from data - header, data = chunk(data, 122) - header = unpack(POSTGIS_HEADER_STRUCTURE, header) - - # Parse band data - bands = [] - pixeltypes = [] - - while data: - # Get pixel type for this band - pixeltype, data = chunk(data, 2) - pixeltype = unpack('B', pixeltype)[0] - - # Check flags - offdb = has_nodata = False - - if POSTGIS_BANDTYPES.BANDTYPE_FLAG_OFFDB & pixeltype == POSTGIS_BANDTYPES.BANDTYPE_FLAG_OFFDB: - offdb = True - pixeltype ^= POSTGIS_BANDTYPES.BANDTYPE_FLAG_OFFDB - if POSTGIS_BANDTYPES.BANDTYPE_FLAG_HASNODATA & pixeltype == POSTGIS_BANDTYPES.BANDTYPE_FLAG_HASNODATA: - has_nodata = True - pixeltype ^= POSTGIS_BANDTYPES.BANDTYPE_FLAG_HASNODATA - if POSTGIS_BANDTYPES.BANDTYPE_FLAG_ISNODATA & pixeltype == POSTGIS_BANDTYPES.BANDTYPE_FLAG_ISNODATA: - raise ValidationError("Band has pixeltype BANDTYPE_FLAG_ISNODATA flag set, but we don't know how to handle it.") - - # Convert datatype from PostGIS to GDAL & get pack type and size - pixeltype = POSTGIS_TO_GDAL[pixeltype] - pack_type = GDAL_TO_STRUCT[pixeltype] - pack_size = 2 * STRUCT_SIZE[pack_type] - - # Parse band nodata value. The nodata value is part of the - # PGRaster string even if the nodata flag is True, so it always - # has to be chunked off the data string. - nodata, data = chunk(data, pack_size) - nodata = unpack(pack_type, nodata)[0] - - if offdb: - # Extract band number - band_num, data = chunk(data, 2) - - # Find NULL byte for end of file path - file_path_length = (binascii.unhexlify(data).find(b'\x00') + 1) * 2 - - # Extract path - file_path, data = chunk(data, file_path_length) - band_result = {'path' : binascii.unhexlify(file_path).decode()[:-1]} # Remove last NULL byte - else: - # Chunk and unpack band data (pack size times nr of pixels) - band, data = chunk(data, pack_size * header[10] * header[11]) - band_result = {'data': binascii.unhexlify(band)} - - # If the nodata flag is True, set the nodata value. - if has_nodata: - band_result['nodata_value'] = nodata - if offdb: - band_result['offdb'] = True - - # Append band data to band list - bands.append(band_result) - - # Store pixeltype of this band in pixeltypes array - pixeltypes.append(pixeltype) - - # Check that all bands have the same pixeltype. - # This is required by GDAL. PostGIS rasters could have different pixeltypes - # for bands of the same raster. - if len(set(pixeltypes)) != 1: - raise ValidationError("Band pixeltypes are not all equal.") - - if offdb and len(bands) > 0: - return bands[0]['path'] - else: - return { - 'srid': int(header[9]), - 'width': header[10], 'height': header[11], - 'datatype': pixeltypes[0], - 'origin': (header[5], header[6]), - 'scale': (header[3], header[4]), - 'skew': (header[7], header[8]), - 'bands': bands, - } - - -def to_pgraster(rast, offdb = False): - """ - Convert a GDALRaster into PostGIS Raster format. - """ - # Return if the raster is null - if rast is None or rast == '': - return - - # Prepare the raster header data as a tuple. The first two numbers are - # the endianness and the PostGIS Raster Version, both are fixed by - # PostGIS at the moment. - rasterheader = ( - 1, 0, len(rast.bands), rast.scale.x, rast.scale.y, - rast.origin.x, rast.origin.y, rast.skew.x, rast.skew.y, - rast.srs.srid, rast.width, rast.height, - ) - - # Hexlify raster header - result = pack(POSTGIS_HEADER_STRUCTURE, rasterheader) - i = 0 - - for band in rast.bands: - # The PostGIS raster band header has exactly two elements, a 8BUI byte - # and the nodata value. - # - # The 8BUI stores both the PostGIS pixel data type and a nodata flag. - # It is composed as the datatype integer plus optional flags for existing - # nodata values, offdb or isnodata: - # 8BUI_VALUE = PG_PIXEL_TYPE (0-11) + FLAGS - # - # For example, if the byte value is 71, then the datatype is - # 71-64 = 7 (32BSI) and the nodata value is True. - structure = 'B' + GDAL_TO_STRUCT[band.datatype()] - - # Get band pixel type in PostGIS notation - pixeltype = GDAL_TO_POSTGIS[band.datatype()] - - # Set the nodata flag - if band.nodata_value is not None: - pixeltype |= POSTGIS_BANDTYPES.BANDTYPE_FLAG_HASNODATA - if offdb: - pixeltype |= POSTGIS_BANDTYPES.BANDTYPE_FLAG_OFFDB - - # Pack band header - bandheader = pack(structure, (pixeltype, band.nodata_value or 0)) - - # Hexlify band data - if offdb: - # Band num | Path | NULL terminator - band_data_hex = binascii.hexlify(struct.Struct('b').pack(i) + rast.name.encode('utf-8') + b'\x00').upper() - else: - band_data_hex = binascii.hexlify(band.data(as_memoryview=True)).upper() - - # Add packed header and band data to result - result += bandheader + band_data_hex - - i += 1 - - # Cast raster to string before passing it to the DB - return result.decode() \ No newline at end of file diff --git a/app/tests/test_api_task.py b/app/tests/test_api_task.py index b14544b4..f492c18e 100644 --- a/app/tests/test_api_task.py +++ b/app/tests/test_api_task.py @@ -320,7 +320,7 @@ class TestApiTask(BootTransactionTestCase): self.assertTrue(res.status_code == status.HTTP_200_OK) # We can stream downloads - res = client.get("/api/projects/{}/tasks/{}/download/{}?_force_stream=1".format(project.id, task.id, task.ASSETS_MAP.keys()[0])) + res = client.get("/api/projects/{}/tasks/{}/download/{}?_force_stream=1".format(project.id, task.id, list(task.ASSETS_MAP.keys())[0])) self.assertTrue(res.status_code == status.HTTP_200_OK) self.assertTrue(res.has_header('_stream')) diff --git a/app/tests/test_gdal.py b/app/tests/test_gdal.py new file mode 100644 index 00000000..9bbfeac4 --- /dev/null +++ b/app/tests/test_gdal.py @@ -0,0 +1,20 @@ +from django.contrib.gis.gdal import GDALRaster + +from .classes import BootTestCase +import os + +class TestApi(BootTestCase): + def setUp(self): + pass + + def tearDown(self): + pass + + def test_gdal_functions(self): + raster = GDALRaster(os.path.join("app", "fixtures", "orthophoto.tif")) + + self.assertTrue(raster.srid == 32615) + self.assertTrue(raster.width == 212) + + + diff --git a/app/tests/test_postgis.py b/app/tests/test_postgis.py deleted file mode 100644 index fae08eae..00000000 --- a/app/tests/test_postgis.py +++ /dev/null @@ -1,38 +0,0 @@ -from django.contrib.gis.gdal import GDALRaster - -from .classes import BootTestCase -from app.postgis import from_pgraster, to_pgraster -import os - -class TestApi(BootTestCase): - def setUp(self): - pass - - def tearDown(self): - pass - - def test_pgraster_functions(self): - # Make sure conversion from PostGIS <---> GDALRaster works - # for out-of-db - raster = GDALRaster(os.path.join("app", "fixtures", "orthophoto.tif")) - - self.assertTrue(raster.srid == 32615) - self.assertTrue(raster.width == 212) - - # Classic - hexwkb = to_pgraster(raster) - deserialized_raster = GDALRaster(from_pgraster(hexwkb)) - self.assertTrue(len(deserialized_raster.bands) == 4) - self.assertTrue(deserialized_raster.srid == raster.srid) - self.assertTrue(deserialized_raster.width == raster.width) - self.assertTrue(deserialized_raster.height == raster.height) - - # Off-db - hexwkb = to_pgraster(raster, True) - deserialized_raster = GDALRaster(from_pgraster(hexwkb, True)) - - self.assertTrue(deserialized_raster.name == raster.name) - self.assertTrue(deserialized_raster.srid == raster.srid) - self.assertTrue(deserialized_raster.width == raster.width) - self.assertTrue(deserialized_raster.height == raster.height) - diff --git a/nodeodm/migrations/0001_initial.py b/nodeodm/migrations/0001_initial.py index 3404c54a..42521dfd 100644 --- a/nodeodm/migrations/0001_initial.py +++ b/nodeodm/migrations/0001_initial.py @@ -29,9 +29,6 @@ class Migration(migrations.Migration): ('queue_count', models.PositiveIntegerField(default=0, help_text='Number of tasks currently being processed by this node (as reported by the node itself)')), ('available_options', django.contrib.postgres.fields.jsonb.JSONField(default={}, help_text='Description of the options that can be used for processing')), ], - options={ - 'permissions': (('view_processingnode', 'Can view processing node'),), - }, ), migrations.CreateModel( name='ProcessingNodeGroupObjectPermission', diff --git a/nodeodm/models.py b/nodeodm/models.py index efd1d691..54a47b22 100644 --- a/nodeodm/models.py +++ b/nodeodm/models.py @@ -40,7 +40,7 @@ class ProcessingNode(models.Model): api_version = models.CharField(max_length=32, null=True, help_text="API version used by the node") last_refreshed = models.DateTimeField(null=True, help_text="When was the information about this node last retrieved?") queue_count = models.PositiveIntegerField(default=0, help_text="Number of tasks currently being processed by this node (as reported by the node itself)") - available_options = fields.JSONField(default=dict(), help_text="Description of the options that can be used for processing") + available_options = fields.JSONField(default=dict, help_text="Description of the options that can be used for processing") token = models.CharField(max_length=1024, blank=True, default="", help_text="Token to use for authentication. If the node doesn't have authentication, you can leave this field blank.") max_images = models.PositiveIntegerField(help_text="Maximum number of images accepted by this node.", blank=True, null=True) odm_version = models.CharField(max_length=32, null=True, help_text="OpenDroneMap version used by the node") @@ -223,12 +223,6 @@ class ProcessingNode(models.Model): plugin_signals.processing_node_removed.send_robust(sender=self.__class__, processing_node_id=pnode_id) - class Meta: - permissions = ( - ('view_processingnode', 'Can view processing node'), - ) - - # First time a processing node is created, automatically try to update @receiver(signals.post_save, sender=ProcessingNode, dispatch_uid="update_processing_node_info") def auto_update_node_info(sender, instance, created, **kwargs): diff --git a/requirements.txt b/requirements.txt index 86d5ca40..089357a0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ django-codemirror2==0.2 django-colorfield==0.1.14 django-compressor==2.2 django-cors-headers==2.2.0 -django-filter==1.1.0 +django-filter==2.0.0 django-guardian==1.4.9 django-imagekit==4.0.1 django-libsass==0.7 diff --git a/webodm/settings.py b/webodm/settings.py index 16cd665a..69d41e0c 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -265,7 +265,7 @@ MESSAGE_TAGS = { # Use Django's standard django.contrib.auth permissions (no anonymous usage) REST_FRAMEWORK = { 'DEFAULT_PERMISSION_CLASSES': [ - 'app.permissions.GuardianObjectPermissions', + 'rest_framework.permissions.DjangoObjectPermissions', ], 'DEFAULT_FILTER_BACKENDS': [ 'rest_framework.filters.DjangoObjectPermissionsFilter',