kopia lustrzana https://github.com/OpenDroneMap/WebODM
Upgraded Django to 2.1
Piero Toffanin
rodzic
a0fbc5021e
commit
0fca052900
|
|
@ -3,7 +3,6 @@
|
|||
from __future__ import unicode_literals
|
||||
|
||||
import app.models
|
||||
import app.postgis
|
||||
from django.conf import settings
|
||||
import django.contrib.postgres.fields.jsonb
|
||||
from django.db import migrations, models
|
||||
|
|
@ -39,9 +38,6 @@ class Migration(migrations.Migration):
|
|||
('deleting', models.BooleanField(db_index=True, default=False, help_text='Whether this project has been marked for deletion. Projects that have running tasks need to wait for tasks to be properly cleaned up before they can be deleted.')),
|
||||
('owner', models.ForeignKey(help_text='The person who created the project', on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)),
|
||||
],
|
||||
options={
|
||||
'permissions': (('view_project', 'Can view project'),),
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='ProjectGroupObjectPermission',
|
||||
|
|
@ -80,15 +76,12 @@ class Migration(migrations.Migration):
|
|||
('options', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default={}, help_text='Options that are being used to process this task', validators=[app.models.validate_task_options])),
|
||||
('console_output', models.TextField(blank=True, default='', help_text="Console output of the OpenDroneMap's process")),
|
||||
('ground_control_points', models.FileField(blank=True, help_text='Optional Ground Control Points file to use for processing', null=True, upload_to=app.models.gcp_directory_path)),
|
||||
('orthophoto', app.postgis.OffDbRasterField(blank=True, help_text='Orthophoto created by OpenDroneMap', null=True, srid=4326)),
|
||||
('orthophoto', django.contrib.gis.db.models.RasterField(blank=True, help_text='Orthophoto created by OpenDroneMap', null=True, srid=4326)),
|
||||
('created_at', models.DateTimeField(default=django.utils.timezone.now, help_text='Creation date')),
|
||||
('pending_action', models.IntegerField(blank=True, choices=[(1, 'CANCEL'), (2, 'REMOVE'), (3, 'RESTART')], db_index=True, help_text='A requested action to be performed on the task. The selected action will be performed by the scheduler at the next iteration.', null=True)),
|
||||
('processing_node', models.ForeignKey(blank=True, help_text='Processing node assigned to this task (or null if this task has not been associated yet)', null=True, on_delete=django.db.models.deletion.CASCADE, to='nodeodm.ProcessingNode')),
|
||||
('project', models.ForeignKey(help_text='Project that this task belongs to', on_delete=django.db.models.deletion.CASCADE, to='app.Project')),
|
||||
],
|
||||
options={
|
||||
'permissions': (('view_task', 'Can view task'),),
|
||||
},
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='imageupload',
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ logger = logging.getLogger('app.logger')
|
|||
class Preset(models.Model):
|
||||
owner = models.ForeignKey(User, blank=True, null=True, on_delete=models.CASCADE, help_text="The person who owns this preset")
|
||||
name = models.CharField(max_length=255, blank=False, null=False, help_text="A label used to describe the preset")
|
||||
options = JSONField(default=list(), blank=True, help_text="Options that define this preset (same format as in a Task's options).",
|
||||
options = JSONField(default=list, blank=True, help_text="Options that define this preset (same format as in a Task's options).",
|
||||
validators=[validate_task_options])
|
||||
created_at = models.DateTimeField(default=timezone.now, help_text="Creation date")
|
||||
system = models.BooleanField(db_index=True, default=False, help_text="Whether this preset is available to every user in the system or just to its owner.")
|
||||
|
|
|
|||
|
|
@ -51,11 +51,6 @@ class Project(models.Model):
|
|||
).filter(Q(orthophoto_extent__isnull=False) | Q(dsm_extent__isnull=False) | Q(dtm_extent__isnull=False))
|
||||
.only('id', 'project_id')]
|
||||
|
||||
class Meta:
|
||||
permissions = (
|
||||
('view_project', 'Can view project'),
|
||||
)
|
||||
|
||||
|
||||
@receiver(signals.post_save, sender=Project, dispatch_uid="project_post_save")
|
||||
def project_post_save(sender, instance, created, **kwargs):
|
||||
|
|
|
|||
|
|
@ -198,8 +198,8 @@ class Task(models.Model):
|
|||
auto_processing_node = models.BooleanField(default=True, help_text="A flag indicating whether this task should be automatically assigned a processing node")
|
||||
status = models.IntegerField(choices=STATUS_CODES, db_index=True, null=True, blank=True, help_text="Current status of the task")
|
||||
last_error = models.TextField(null=True, blank=True, help_text="The last processing error received")
|
||||
options = fields.JSONField(default=dict(), blank=True, help_text="Options that are being used to process this task", validators=[validate_task_options])
|
||||
available_assets = fields.ArrayField(models.CharField(max_length=80), default=list(), blank=True, help_text="List of available assets to download")
|
||||
options = fields.JSONField(default=dict, blank=True, help_text="Options that are being used to process this task", validators=[validate_task_options])
|
||||
available_assets = fields.ArrayField(models.CharField(max_length=80), default=list, blank=True, help_text="List of available assets to download")
|
||||
console_output = models.TextField(null=False, default="", blank=True, help_text="Console output of the OpenDroneMap's process")
|
||||
|
||||
orthophoto_extent = GeometryField(null=True, blank=True, srid=4326, help_text="Extent of the orthophoto created by OpenDroneMap")
|
||||
|
|
@ -769,8 +769,3 @@ class Task(models.Model):
|
|||
except subprocess.CalledProcessError as e:
|
||||
logger.warning("Could not resize GCP file {}: {}".format(gcp_path, str(e)))
|
||||
return None
|
||||
|
||||
class Meta:
|
||||
permissions = (
|
||||
('view_task', 'Can view task'),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,15 +0,0 @@
|
|||
from rest_framework import permissions
|
||||
|
||||
class GuardianObjectPermissions(permissions.DjangoObjectPermissions):
|
||||
"""
|
||||
Similar to `DjangoObjectPermissions`, but adding 'view' permissions.
|
||||
"""
|
||||
perms_map = {
|
||||
'GET': ['%(app_label)s.view_%(model_name)s'],
|
||||
'OPTIONS': ['%(app_label)s.view_%(model_name)s'],
|
||||
'HEAD': ['%(app_label)s.view_%(model_name)s'],
|
||||
'POST': ['%(app_label)s.add_%(model_name)s'],
|
||||
'PUT': ['%(app_label)s.change_%(model_name)s'],
|
||||
'PATCH': ['%(app_label)s.change_%(model_name)s'],
|
||||
'DELETE': ['%(app_label)s.delete_%(model_name)s'],
|
||||
}
|
||||
218
app/postgis.py
218
app/postgis.py
|
|
@ -1,218 +0,0 @@
|
|||
import binascii
|
||||
import struct
|
||||
|
||||
from django.contrib.gis.db.backends.postgis.const import GDAL_TO_POSTGIS
|
||||
from django.contrib.gis.db.backends.postgis.pgraster import (
|
||||
GDAL_TO_STRUCT, POSTGIS_HEADER_STRUCTURE, POSTGIS_TO_GDAL,
|
||||
STRUCT_SIZE,
|
||||
pack)
|
||||
from django.contrib.gis.db.backends.postgis.pgraster import chunk, unpack
|
||||
from django.contrib.gis.db.models.fields import RasterField, BaseSpatialField
|
||||
from django.contrib.gis.gdal import GDALException
|
||||
from django.contrib.gis.gdal import GDALRaster
|
||||
from django.forms import ValidationError
|
||||
from django.utils.translation import ugettext_lazy as _
|
||||
|
||||
|
||||
class OffDbRasterField(RasterField):
|
||||
"""
|
||||
Out-of-db Raster field for GeoDjango -- evaluates into GDALRaster objects.
|
||||
"""
|
||||
|
||||
description = _("Out-of-db Raster Field")
|
||||
|
||||
def from_db_value(self, value, expression, connection, context):
|
||||
return from_pgraster(value, True)
|
||||
|
||||
def get_db_prep_save(self, value, connection):
|
||||
"""
|
||||
Prepare the value for saving in the database.
|
||||
"""
|
||||
if not value:
|
||||
return None
|
||||
else:
|
||||
return to_pgraster(value, True)
|
||||
|
||||
def get_db_prep_value(self, value, connection, prepared=False):
|
||||
self._check_connection(connection)
|
||||
# Prepare raster for writing to database.
|
||||
if not prepared:
|
||||
value = to_pgraster(value, True)
|
||||
|
||||
# Call RasterField's base class get_db_prep_value
|
||||
return BaseSpatialField.get_db_prep_value(self, value, connection, prepared)
|
||||
|
||||
def get_raster_prep_value(self, value, is_candidate):
|
||||
"""
|
||||
Return a GDALRaster if conversion is successful, otherwise return None.
|
||||
"""
|
||||
if isinstance(value, GDALRaster):
|
||||
return value
|
||||
elif is_candidate:
|
||||
try:
|
||||
return GDALRaster(value)
|
||||
except GDALException:
|
||||
pass
|
||||
elif isinstance(value, (dict, str)):
|
||||
try:
|
||||
return GDALRaster(value)
|
||||
except GDALException:
|
||||
raise ValueError("Couldn't create spatial object from lookup value '%s'." % value)
|
||||
|
||||
|
||||
class POSTGIS_BANDTYPES(object):
|
||||
BANDTYPE_FLAG_OFFDB = 1 << 7
|
||||
BANDTYPE_FLAG_HASNODATA = 1 << 6
|
||||
BANDTYPE_FLAG_ISNODATA = 1 << 5
|
||||
|
||||
|
||||
def from_pgraster(data, offdb = False):
|
||||
"""
|
||||
Convert a PostGIS HEX String into a dictionary.
|
||||
"""
|
||||
if data is None:
|
||||
return
|
||||
|
||||
# Split raster header from data
|
||||
header, data = chunk(data, 122)
|
||||
header = unpack(POSTGIS_HEADER_STRUCTURE, header)
|
||||
|
||||
# Parse band data
|
||||
bands = []
|
||||
pixeltypes = []
|
||||
|
||||
while data:
|
||||
# Get pixel type for this band
|
||||
pixeltype, data = chunk(data, 2)
|
||||
pixeltype = unpack('B', pixeltype)[0]
|
||||
|
||||
# Check flags
|
||||
offdb = has_nodata = False
|
||||
|
||||
if POSTGIS_BANDTYPES.BANDTYPE_FLAG_OFFDB & pixeltype == POSTGIS_BANDTYPES.BANDTYPE_FLAG_OFFDB:
|
||||
offdb = True
|
||||
pixeltype ^= POSTGIS_BANDTYPES.BANDTYPE_FLAG_OFFDB
|
||||
if POSTGIS_BANDTYPES.BANDTYPE_FLAG_HASNODATA & pixeltype == POSTGIS_BANDTYPES.BANDTYPE_FLAG_HASNODATA:
|
||||
has_nodata = True
|
||||
pixeltype ^= POSTGIS_BANDTYPES.BANDTYPE_FLAG_HASNODATA
|
||||
if POSTGIS_BANDTYPES.BANDTYPE_FLAG_ISNODATA & pixeltype == POSTGIS_BANDTYPES.BANDTYPE_FLAG_ISNODATA:
|
||||
raise ValidationError("Band has pixeltype BANDTYPE_FLAG_ISNODATA flag set, but we don't know how to handle it.")
|
||||
|
||||
# Convert datatype from PostGIS to GDAL & get pack type and size
|
||||
pixeltype = POSTGIS_TO_GDAL[pixeltype]
|
||||
pack_type = GDAL_TO_STRUCT[pixeltype]
|
||||
pack_size = 2 * STRUCT_SIZE[pack_type]
|
||||
|
||||
# Parse band nodata value. The nodata value is part of the
|
||||
# PGRaster string even if the nodata flag is True, so it always
|
||||
# has to be chunked off the data string.
|
||||
nodata, data = chunk(data, pack_size)
|
||||
nodata = unpack(pack_type, nodata)[0]
|
||||
|
||||
if offdb:
|
||||
# Extract band number
|
||||
band_num, data = chunk(data, 2)
|
||||
|
||||
# Find NULL byte for end of file path
|
||||
file_path_length = (binascii.unhexlify(data).find(b'\x00') + 1) * 2
|
||||
|
||||
# Extract path
|
||||
file_path, data = chunk(data, file_path_length)
|
||||
band_result = {'path' : binascii.unhexlify(file_path).decode()[:-1]} # Remove last NULL byte
|
||||
else:
|
||||
# Chunk and unpack band data (pack size times nr of pixels)
|
||||
band, data = chunk(data, pack_size * header[10] * header[11])
|
||||
band_result = {'data': binascii.unhexlify(band)}
|
||||
|
||||
# If the nodata flag is True, set the nodata value.
|
||||
if has_nodata:
|
||||
band_result['nodata_value'] = nodata
|
||||
if offdb:
|
||||
band_result['offdb'] = True
|
||||
|
||||
# Append band data to band list
|
||||
bands.append(band_result)
|
||||
|
||||
# Store pixeltype of this band in pixeltypes array
|
||||
pixeltypes.append(pixeltype)
|
||||
|
||||
# Check that all bands have the same pixeltype.
|
||||
# This is required by GDAL. PostGIS rasters could have different pixeltypes
|
||||
# for bands of the same raster.
|
||||
if len(set(pixeltypes)) != 1:
|
||||
raise ValidationError("Band pixeltypes are not all equal.")
|
||||
|
||||
if offdb and len(bands) > 0:
|
||||
return bands[0]['path']
|
||||
else:
|
||||
return {
|
||||
'srid': int(header[9]),
|
||||
'width': header[10], 'height': header[11],
|
||||
'datatype': pixeltypes[0],
|
||||
'origin': (header[5], header[6]),
|
||||
'scale': (header[3], header[4]),
|
||||
'skew': (header[7], header[8]),
|
||||
'bands': bands,
|
||||
}
|
||||
|
||||
|
||||
def to_pgraster(rast, offdb = False):
|
||||
"""
|
||||
Convert a GDALRaster into PostGIS Raster format.
|
||||
"""
|
||||
# Return if the raster is null
|
||||
if rast is None or rast == '':
|
||||
return
|
||||
|
||||
# Prepare the raster header data as a tuple. The first two numbers are
|
||||
# the endianness and the PostGIS Raster Version, both are fixed by
|
||||
# PostGIS at the moment.
|
||||
rasterheader = (
|
||||
1, 0, len(rast.bands), rast.scale.x, rast.scale.y,
|
||||
rast.origin.x, rast.origin.y, rast.skew.x, rast.skew.y,
|
||||
rast.srs.srid, rast.width, rast.height,
|
||||
)
|
||||
|
||||
# Hexlify raster header
|
||||
result = pack(POSTGIS_HEADER_STRUCTURE, rasterheader)
|
||||
i = 0
|
||||
|
||||
for band in rast.bands:
|
||||
# The PostGIS raster band header has exactly two elements, a 8BUI byte
|
||||
# and the nodata value.
|
||||
#
|
||||
# The 8BUI stores both the PostGIS pixel data type and a nodata flag.
|
||||
# It is composed as the datatype integer plus optional flags for existing
|
||||
# nodata values, offdb or isnodata:
|
||||
# 8BUI_VALUE = PG_PIXEL_TYPE (0-11) + FLAGS
|
||||
#
|
||||
# For example, if the byte value is 71, then the datatype is
|
||||
# 71-64 = 7 (32BSI) and the nodata value is True.
|
||||
structure = 'B' + GDAL_TO_STRUCT[band.datatype()]
|
||||
|
||||
# Get band pixel type in PostGIS notation
|
||||
pixeltype = GDAL_TO_POSTGIS[band.datatype()]
|
||||
|
||||
# Set the nodata flag
|
||||
if band.nodata_value is not None:
|
||||
pixeltype |= POSTGIS_BANDTYPES.BANDTYPE_FLAG_HASNODATA
|
||||
if offdb:
|
||||
pixeltype |= POSTGIS_BANDTYPES.BANDTYPE_FLAG_OFFDB
|
||||
|
||||
# Pack band header
|
||||
bandheader = pack(structure, (pixeltype, band.nodata_value or 0))
|
||||
|
||||
# Hexlify band data
|
||||
if offdb:
|
||||
# Band num | Path | NULL terminator
|
||||
band_data_hex = binascii.hexlify(struct.Struct('b').pack(i) + rast.name.encode('utf-8') + b'\x00').upper()
|
||||
else:
|
||||
band_data_hex = binascii.hexlify(band.data(as_memoryview=True)).upper()
|
||||
|
||||
# Add packed header and band data to result
|
||||
result += bandheader + band_data_hex
|
||||
|
||||
i += 1
|
||||
|
||||
# Cast raster to string before passing it to the DB
|
||||
return result.decode()
|
||||
|
|
@ -320,7 +320,7 @@ class TestApiTask(BootTransactionTestCase):
|
|||
self.assertTrue(res.status_code == status.HTTP_200_OK)
|
||||
|
||||
# We can stream downloads
|
||||
res = client.get("/api/projects/{}/tasks/{}/download/{}?_force_stream=1".format(project.id, task.id, task.ASSETS_MAP.keys()[0]))
|
||||
res = client.get("/api/projects/{}/tasks/{}/download/{}?_force_stream=1".format(project.id, task.id, list(task.ASSETS_MAP.keys())[0]))
|
||||
self.assertTrue(res.status_code == status.HTTP_200_OK)
|
||||
self.assertTrue(res.has_header('_stream'))
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,20 @@
|
|||
from django.contrib.gis.gdal import GDALRaster
|
||||
|
||||
from .classes import BootTestCase
|
||||
import os
|
||||
|
||||
class TestApi(BootTestCase):
|
||||
def setUp(self):
|
||||
pass
|
||||
|
||||
def tearDown(self):
|
||||
pass
|
||||
|
||||
def test_gdal_functions(self):
|
||||
raster = GDALRaster(os.path.join("app", "fixtures", "orthophoto.tif"))
|
||||
|
||||
self.assertTrue(raster.srid == 32615)
|
||||
self.assertTrue(raster.width == 212)
|
||||
|
||||
|
||||
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
from django.contrib.gis.gdal import GDALRaster
|
||||
|
||||
from .classes import BootTestCase
|
||||
from app.postgis import from_pgraster, to_pgraster
|
||||
import os
|
||||
|
||||
class TestApi(BootTestCase):
|
||||
def setUp(self):
|
||||
pass
|
||||
|
||||
def tearDown(self):
|
||||
pass
|
||||
|
||||
def test_pgraster_functions(self):
|
||||
# Make sure conversion from PostGIS <---> GDALRaster works
|
||||
# for out-of-db
|
||||
raster = GDALRaster(os.path.join("app", "fixtures", "orthophoto.tif"))
|
||||
|
||||
self.assertTrue(raster.srid == 32615)
|
||||
self.assertTrue(raster.width == 212)
|
||||
|
||||
# Classic
|
||||
hexwkb = to_pgraster(raster)
|
||||
deserialized_raster = GDALRaster(from_pgraster(hexwkb))
|
||||
self.assertTrue(len(deserialized_raster.bands) == 4)
|
||||
self.assertTrue(deserialized_raster.srid == raster.srid)
|
||||
self.assertTrue(deserialized_raster.width == raster.width)
|
||||
self.assertTrue(deserialized_raster.height == raster.height)
|
||||
|
||||
# Off-db
|
||||
hexwkb = to_pgraster(raster, True)
|
||||
deserialized_raster = GDALRaster(from_pgraster(hexwkb, True))
|
||||
|
||||
self.assertTrue(deserialized_raster.name == raster.name)
|
||||
self.assertTrue(deserialized_raster.srid == raster.srid)
|
||||
self.assertTrue(deserialized_raster.width == raster.width)
|
||||
self.assertTrue(deserialized_raster.height == raster.height)
|
||||
|
||||
|
|
@ -29,9 +29,6 @@ class Migration(migrations.Migration):
|
|||
('queue_count', models.PositiveIntegerField(default=0, help_text='Number of tasks currently being processed by this node (as reported by the node itself)')),
|
||||
('available_options', django.contrib.postgres.fields.jsonb.JSONField(default={}, help_text='Description of the options that can be used for processing')),
|
||||
],
|
||||
options={
|
||||
'permissions': (('view_processingnode', 'Can view processing node'),),
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='ProcessingNodeGroupObjectPermission',
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ class ProcessingNode(models.Model):
|
|||
api_version = models.CharField(max_length=32, null=True, help_text="API version used by the node")
|
||||
last_refreshed = models.DateTimeField(null=True, help_text="When was the information about this node last retrieved?")
|
||||
queue_count = models.PositiveIntegerField(default=0, help_text="Number of tasks currently being processed by this node (as reported by the node itself)")
|
||||
available_options = fields.JSONField(default=dict(), help_text="Description of the options that can be used for processing")
|
||||
available_options = fields.JSONField(default=dict, help_text="Description of the options that can be used for processing")
|
||||
token = models.CharField(max_length=1024, blank=True, default="", help_text="Token to use for authentication. If the node doesn't have authentication, you can leave this field blank.")
|
||||
max_images = models.PositiveIntegerField(help_text="Maximum number of images accepted by this node.", blank=True, null=True)
|
||||
odm_version = models.CharField(max_length=32, null=True, help_text="OpenDroneMap version used by the node")
|
||||
|
|
@ -223,12 +223,6 @@ class ProcessingNode(models.Model):
|
|||
plugin_signals.processing_node_removed.send_robust(sender=self.__class__, processing_node_id=pnode_id)
|
||||
|
||||
|
||||
class Meta:
|
||||
permissions = (
|
||||
('view_processingnode', 'Can view processing node'),
|
||||
)
|
||||
|
||||
|
||||
# First time a processing node is created, automatically try to update
|
||||
@receiver(signals.post_save, sender=ProcessingNode, dispatch_uid="update_processing_node_info")
|
||||
def auto_update_node_info(sender, instance, created, **kwargs):
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ django-codemirror2==0.2
|
|||
django-colorfield==0.1.14
|
||||
django-compressor==2.2
|
||||
django-cors-headers==2.2.0
|
||||
django-filter==1.1.0
|
||||
django-filter==2.0.0
|
||||
django-guardian==1.4.9
|
||||
django-imagekit==4.0.1
|
||||
django-libsass==0.7
|
||||
|
|
|
|||
|
|
@ -265,7 +265,7 @@ MESSAGE_TAGS = {
|
|||
# Use Django's standard django.contrib.auth permissions (no anonymous usage)
|
||||
REST_FRAMEWORK = {
|
||||
'DEFAULT_PERMISSION_CLASSES': [
|
||||
'app.permissions.GuardianObjectPermissions',
|
||||
'rest_framework.permissions.DjangoObjectPermissions',
|
||||
],
|
||||
'DEFAULT_FILTER_BACKENDS': [
|
||||
'rest_framework.filters.DjangoObjectPermissionsFilter',
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue