kopia lustrzana https://github.com/OpenDroneMap/WebODM
Fixed tests
Piero Toffanin
rodzic
fdebc8f157
commit
134ae42ac5
|
|
@ -39,13 +39,12 @@ def get_extent(task, tile_type):
|
|||
}
|
||||
|
||||
if not tile_type in extent_map:
|
||||
raise exceptions.ValidationError("Type {} is not a valid tile type".format(tile_type))
|
||||
raise exceptions.NotFound()
|
||||
|
||||
extent = extent_map[tile_type]
|
||||
|
||||
if extent is None:
|
||||
raise exceptions.ValidationError(
|
||||
"A {} has not been processed for this task. Tiles are not available.".format(tile_type))
|
||||
raise exceptions.NotFound()
|
||||
|
||||
return extent
|
||||
|
||||
|
|
|
|||
168
app/cogeo.py
168
app/cogeo.py
|
|
@ -1,175 +1,11 @@
|
|||
import os
|
||||
import logging
|
||||
#from rio_cogeo.cogeo import cog_validate
|
||||
import tempfile
|
||||
import shutil
|
||||
import rasterio
|
||||
|
||||
from rio_cogeo.cogeo import cog_translate
|
||||
from rio_cogeo.cogeo import cog_validate, cog_translate
|
||||
from webodm import settings
|
||||
|
||||
# TODO REMOVE
|
||||
from rasterio.env import GDALVersion
|
||||
|
||||
def cog_validate(src_path, strict=False):
|
||||
"""
|
||||
Validate Cloud Optimized Geotiff.
|
||||
Parameters
|
||||
----------
|
||||
src_path : str or PathLike object
|
||||
A dataset path or URL. Will be opened in "r" mode.
|
||||
This script is the rasterio equivalent of
|
||||
https://svn.osgeo.org/gdal/trunk/gdal/swig/python/samples/validate_cloud_optimized_geotiff.py
|
||||
"""
|
||||
errors = []
|
||||
warnings = []
|
||||
details = {}
|
||||
|
||||
if not GDALVersion.runtime().at_least("2.2"):
|
||||
raise Exception("GDAL 2.2 or above required")
|
||||
|
||||
config = dict(GDAL_DISABLE_READDIR_ON_OPEN="FALSE")
|
||||
with rasterio.Env(**config):
|
||||
with rasterio.open(src_path) as src:
|
||||
if not src.driver == "GTiff":
|
||||
raise Exception("The file is not a GeoTIFF")
|
||||
|
||||
filelist = [os.path.basename(f) for f in src.files]
|
||||
src_bname = os.path.basename(src_path)
|
||||
if len(filelist) > 1 and src_bname + ".ovr" in filelist:
|
||||
errors.append(
|
||||
"Overviews found in external .ovr file. They should be internal"
|
||||
)
|
||||
|
||||
overviews = src.overviews(1)
|
||||
if src.width > 512 or src.height > 512:
|
||||
if not src.is_tiled:
|
||||
errors.append(
|
||||
"The file is greater than 512xH or 512xW, but is not tiled"
|
||||
)
|
||||
|
||||
if not overviews:
|
||||
warnings.append(
|
||||
"The file is greater than 512xH or 512xW, it is recommended "
|
||||
"to include internal overviews"
|
||||
)
|
||||
|
||||
ifd_offset = int(src.get_tag_item("IFD_OFFSET", "TIFF", bidx=1))
|
||||
ifd_offsets = [ifd_offset]
|
||||
if ifd_offset not in (8, 16):
|
||||
errors.append(
|
||||
"The offset of the main IFD should be 8 for ClassicTIFF "
|
||||
"or 16 for BigTIFF. It is {} instead".format(ifd_offset)
|
||||
)
|
||||
|
||||
details["ifd_offsets"] = {}
|
||||
details["ifd_offsets"]["main"] = ifd_offset
|
||||
|
||||
if overviews and overviews != sorted(overviews):
|
||||
errors.append("Overviews should be sorted")
|
||||
|
||||
for ix, dec in enumerate(overviews):
|
||||
|
||||
# NOTE: Size check is handled in rasterio `src.overviews` methods
|
||||
# https://github.com/mapbox/rasterio/blob/4ebdaa08cdcc65b141ed3fe95cf8bbdd9117bc0b/rasterio/_base.pyx
|
||||
# We just need to make sure the decimation level is > 1
|
||||
if not dec > 1:
|
||||
errors.append(
|
||||
"Invalid Decimation {} for overview level {}".format(dec, ix)
|
||||
)
|
||||
|
||||
# Check that the IFD of descending overviews are sorted by increasing
|
||||
# offsets
|
||||
ifd_offset = int(src.get_tag_item("IFD_OFFSET", "TIFF", bidx=1, ovr=ix))
|
||||
ifd_offsets.append(ifd_offset)
|
||||
|
||||
details["ifd_offsets"]["overview_{}".format(ix)] = ifd_offset
|
||||
if ifd_offsets[-1] < ifd_offsets[-2]:
|
||||
if ix == 0:
|
||||
errors.append(
|
||||
"The offset of the IFD for overview of index {} is {}, "
|
||||
"whereas it should be greater than the one of the main "
|
||||
"image, which is at byte {}".format(
|
||||
ix, ifd_offsets[-1], ifd_offsets[-2]
|
||||
)
|
||||
)
|
||||
else:
|
||||
errors.append(
|
||||
"The offset of the IFD for overview of index {} is {}, "
|
||||
"whereas it should be greater than the one of index {}, "
|
||||
"which is at byte {}".format(
|
||||
ix, ifd_offsets[-1], ix - 1, ifd_offsets[-2]
|
||||
)
|
||||
)
|
||||
|
||||
block_offset = int(src.get_tag_item("BLOCK_OFFSET_0_0", "TIFF", bidx=1))
|
||||
if not block_offset:
|
||||
errors.append("Missing BLOCK_OFFSET_0_0")
|
||||
|
||||
data_offset = int(block_offset) if block_offset else None
|
||||
data_offsets = [data_offset]
|
||||
details["data_offsets"] = {}
|
||||
details["data_offsets"]["main"] = data_offset
|
||||
|
||||
for ix, dec in enumerate(overviews):
|
||||
data_offset = int(
|
||||
src.get_tag_item("BLOCK_OFFSET_0_0", "TIFF", bidx=1, ovr=ix)
|
||||
)
|
||||
data_offsets.append(data_offset)
|
||||
details["data_offsets"]["overview_{}".format(ix)] = data_offset
|
||||
|
||||
if data_offsets[-1] < ifd_offsets[-1]:
|
||||
if len(overviews) > 0:
|
||||
errors.append(
|
||||
"The offset of the first block of the smallest overview "
|
||||
"should be after its IFD"
|
||||
)
|
||||
else:
|
||||
errors.append(
|
||||
"The offset of the first block of the image should "
|
||||
"be after its IFD"
|
||||
)
|
||||
|
||||
for i in range(len(data_offsets) - 2, 0, -1):
|
||||
if data_offsets[i] < data_offsets[i + 1]:
|
||||
errors.append(
|
||||
"The offset of the first block of overview of index {} should "
|
||||
"be after the one of the overview of index {}".format(i - 1, i)
|
||||
)
|
||||
|
||||
if len(data_offsets) >= 2 and data_offsets[0] < data_offsets[1]:
|
||||
errors.append(
|
||||
"The offset of the first block of the main resolution image "
|
||||
"should be after the one of the overview of index {}".format(
|
||||
len(overviews) - 1
|
||||
)
|
||||
)
|
||||
|
||||
for ix, dec in enumerate(overviews):
|
||||
with rasterio.open(src_path, OVERVIEW_LEVEL=ix) as ovr_dst:
|
||||
if ovr_dst.width >= 512 or ovr_dst.height >= 512:
|
||||
if not ovr_dst.is_tiled:
|
||||
errors.append("Overview of index {} is not tiled".format(ix))
|
||||
|
||||
if warnings:
|
||||
logger.warning("The following warnings were found:")
|
||||
for w in warnings:
|
||||
logger.warning(w)
|
||||
|
||||
if errors:
|
||||
logger.warning("The following errors were found:")
|
||||
for e in errors:
|
||||
logger.warning("- " + e)
|
||||
|
||||
return False
|
||||
|
||||
if warnings and strict:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
# TODO: REMOVE
|
||||
|
||||
logger = logging.getLogger('app.logger')
|
||||
|
||||
def valid_cogeo(src_path):
|
||||
|
|
@ -198,7 +34,7 @@ def assure_cogeo(src_path):
|
|||
return
|
||||
|
||||
# Not a cogeo
|
||||
|
||||
logger.info("Optimizing %s as Cloud Optimized GeoTIFF" % src_path)
|
||||
tmpfile = tempfile.mktemp('_cogeo.tif', dir=settings.MEDIA_TMP)
|
||||
swapfile = tempfile.mktemp('_cogeo_swap.tif', dir=settings.MEDIA_TMP)
|
||||
|
||||
|
|
|
|||
|
|
@ -391,9 +391,10 @@ class TestApiTask(BootTransactionTestCase):
|
|||
# Histogram stats are available (3 bands for orthophoto)
|
||||
self.assertTrue(len(metadata['statistics']) == 3)
|
||||
for b in ['1', '2', '3']:
|
||||
self.assertEqual(len(metadata['statistics'][b]['histogram']), 255) # bins
|
||||
self.assertEqual(metadata['statistics'][b]['min'], 0)
|
||||
self.assertEqual(metadata['statistics'][b]['max'], 255)
|
||||
self.assertEqual(len(metadata['statistics'][b]['histogram']), 2)
|
||||
self.assertEqual(len(metadata['statistics'][b]['histogram'][0]), 255)
|
||||
self.assertTrue('max' in metadata['statistics'][b])
|
||||
self.assertTrue('min' in metadata['statistics'][b])
|
||||
|
||||
# Metadata with invalid formula
|
||||
res = client.get("/api/projects/{}/tasks/{}/orthophoto/metadata?formula=INVALID".format(project.id, task.id))
|
||||
|
|
@ -413,8 +414,8 @@ class TestApiTask(BootTransactionTestCase):
|
|||
self.assertTrue(len(metadata['color_maps']) > 0)
|
||||
|
||||
# Colormap is for algorithms
|
||||
self.assertTrue('rdylgn' in metadata['color_maps'])
|
||||
self.assertFalse('jet_r' in metadata['color_maps'])
|
||||
self.assertEqual(len([x for x in metadata['color_maps'] if x['key'] == 'rdylgn']), 1)
|
||||
self.assertEqual(len([x for x in metadata['color_maps'] if x['key'] == 'jet_r']), 0)
|
||||
|
||||
# Formula parameters are copied to tile URL
|
||||
self.assertTrue(metadata['tiles'][0].endswith('?formula=NDVI&bands=RGN'))
|
||||
|
|
@ -442,23 +443,23 @@ class TestApiTask(BootTransactionTestCase):
|
|||
self.assertTrue(len(metadata['color_maps']) > 0)
|
||||
|
||||
# Colormaps are for elevation
|
||||
self.assertTrue('jet_r' in metadata['color_maps'])
|
||||
self.assertFalse('rdylgn' in metadata['color_maps'])
|
||||
self.assertEqual(len([x for x in metadata['color_maps'] if x['key'] == 'rdylgn']), 0)
|
||||
self.assertEqual(len([x for x in metadata['color_maps'] if x['key'] == 'jet_r']), 1)
|
||||
|
||||
# Algorithms are empty
|
||||
self.assertEqual(len(metadata['algorithms']), 0)
|
||||
|
||||
# Min/max values are what we expect them to be
|
||||
self.assertEqual(len(metadata['statistics']), 1)
|
||||
self.assertEqual(round(metadata['statistics']['1']['min'], 2), 156.91)
|
||||
self.assertEqual(round(metadata['statistics']['1']['max'], 2), 164.94)
|
||||
self.assertEqual(round(metadata['statistics']['1']['min'], 2), 156.92)
|
||||
self.assertEqual(round(metadata['statistics']['1']['max'], 2), 164.88)
|
||||
|
||||
# Can access individual tiles
|
||||
for tile_type in tile_types:
|
||||
res = client.get("/api/projects/{}/tasks/{}/{}/tiles/17/32042/46185.png".format(project.id, task.id, tile_type))
|
||||
self.assertEqual(res.status_code, status.HTTP_200_OK)
|
||||
|
||||
with Image.open(io.BytesIO(res.data)) as i:
|
||||
with Image.open(io.BytesIO(res.content)) as i:
|
||||
self.assertEqual(i.width, 256)
|
||||
self.assertEqual(i.height, 256)
|
||||
|
||||
|
|
@ -467,7 +468,7 @@ class TestApiTask(BootTransactionTestCase):
|
|||
res = client.get("/api/projects/{}/tasks/{}/{}/tiles/17/32042/46185@2x.png".format(project.id, task.id, tile_type))
|
||||
self.assertEqual(res.status_code, status.HTTP_200_OK)
|
||||
|
||||
with Image.open(io.BytesIO(res.data)) as i:
|
||||
with Image.open(io.BytesIO(res.content)) as i:
|
||||
self.assertEqual(i.width, 512)
|
||||
self.assertEqual(i.height, 512)
|
||||
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
Subproject commit 48cf7f01b2ab96b6fdbdcb81d4ddc5828ae66c0e
|
||||
Subproject commit 77b20a68a8a9d238cdce080944f04ec71076dc2b
|
||||
|
|
@ -56,4 +56,4 @@ webcolors==1.5
|
|||
rasterio==1.1.0
|
||||
-e git://github.com/OpenDroneMap/rio-tiler.git#egg=rio-tiler
|
||||
rio-color==1.0.0
|
||||
rio-cogeo==1.1.6
|
||||
rio-cogeo==1.1.7
|
||||
Ładowanie…
Reference in New Issue