mirror
/
OpenDroneMap-WebODM
kopia lustrzana https://github.com/OpenDroneMap/WebODM
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge pull request #1205 from pierotofy/tiler 

Smart Tiles, Faster Tiles
pull/1208/head
Piero Toffanin 2022-07-08 16:35:26 -04:00 zatwierdzone przez GitHub
rodzic 0e9b3d5527 d4938f1d0d
commit a05951b60f
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
8 zmienionych plików z 155 dodań i 113 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

213
app/api/tiler.py
Wyświetl plik

@ -1,5 +1,4 @@
import json
import numpy
import rio_tiler.utils
from rasterio.enums import ColorInterp
from rasterio.crs import CRS
@ -43,13 +42,13 @@ def get_zoom_safe(src_dst):
def get_tile_url(task, tile_type, query_params):
url = '/api/projects/{}/tasks/{}/{}/tiles/{{z}}/{{x}}/{{y}}.png'.format(task.project.id, task.id, tile_type)
url = '/api/projects/{}/tasks/{}/{}/tiles/{{z}}/{{x}}/{{y}}'.format(task.project.id, task.id, tile_type)
params = {}
for k in ['formula', 'bands', 'rescale', 'color_map', 'hillshade']:
if query_params.get(k):
params[k] = query_params.get(k)
if len(params) > 0:
url = url + '?' + urllib.parse.urlencode(params)
@ -172,7 +171,7 @@ class Metadata(TaskNestedView):
data, mask = src.preview(expression=expr, vrt_options={'cutline': boundaries_cutline})
else:
data, mask = src.preview(expression=expr)
data = numpy.ma.array(data)
data = np.ma.array(data)
data.mask = mask == 0
stats = {
str(b + 1): raster_stats(data[b], percentiles=(pmin, pmax), bins=255, range=hrange)
@ -251,52 +250,18 @@ class Metadata(TaskNestedView):
return Response(info)
def get_elevation_tiles(elevation, url, x, y, z, tilesize, nodata, resampling, padding):
tile = np.full((tilesize * 3, tilesize * 3), nodata, dtype=elevation.dtype)
with COGReader(url) as src:
try:
left, _discard_ = src.tile(x - 1, y, z, indexes=1, tilesize=tilesize, nodata=nodata,
resampling_method=resampling, padding=padding)
tile[tilesize:tilesize * 2, 0:tilesize] = left
except TileOutsideBounds:
pass
try:
right, _discard_ = src.tile(x + 1, y, z, indexes=1, tilesize=tilesize, nodata=nodata,
resampling_method=resampling, padding=padding)
tile[tilesize:tilesize * 2, tilesize * 2:tilesize * 3] = right
except TileOutsideBounds:
pass
try:
bottom, _discard_ = src.tile(x, y + 1, z, indexes=1, tilesize=tilesize, nodata=nodata,
resampling_method=resampling, padding=padding)
tile[tilesize * 2:tilesize * 3, tilesize:tilesize * 2] = bottom
except TileOutsideBounds:
pass
try:
top, _discard_ = src.tile(x, y - 1, z, indexes=1, tilesize=tilesize, nodata=nodata,
resampling_method=resampling, padding=padding)
tile[0:tilesize, tilesize:tilesize * 2] = top
except TileOutsideBounds:
pass
tile[tilesize:tilesize * 2, tilesize:tilesize * 2] = elevation
return tile
class Tiles(TaskNestedView):
def get(self, request, pk=None, project_pk=None, tile_type="", z="", x="", y="", scale=1):
def get(self, request, pk=None, project_pk=None, tile_type="", z="", x="", y="", scale=1, ext=None):
"""
Get a tile image
"""
task = self.get_and_check_task(request, pk)
z = int(z)
x = int(x)
y = int(y)
scale = int(scale)
ext = "png"
driver = "jpeg" if ext == "jpg" else ext
indexes = None
nodata = None
@ -307,6 +272,8 @@ class Tiles(TaskNestedView):
rescale = self.request.query_params.get('rescale')
color_map = self.request.query_params.get('color_map')
hillshade = self.request.query_params.get('hillshade')
tilesize = self.request.query_params.get('size')
boundaries_feature = self.request.query_params.get('boundaries')
if boundaries_feature == '':
boundaries_feature = None
@ -321,6 +288,17 @@ class Tiles(TaskNestedView):
if rescale == '': rescale = None
if color_map == '': color_map = None
if hillshade == '' or hillshade == '0': hillshade = None
if tilesize == '' or tilesize is None: tilesize = 256
try:
tilesize = int(tilesize)
if tilesize != 256 and tilesize != 512:
raise ValueError("Invalid size")
if tilesize == 512:
z -= 1
except ValueError:
raise exceptions.ValidationError(_("Invalid tile size parameter"))
try:
expr, _discard_ = lookup_formula(formula, bands)
@ -343,7 +321,7 @@ class Tiles(TaskNestedView):
if nodata is not None:
nodata = np.nan if nodata == "nan" else float(nodata)
tilesize = scale * 256
tilesize = scale * tilesize
url = get_raster_path(task, tile_type)
if not os.path.isfile(url):
raise exceptions.NotFound()
@ -352,7 +330,6 @@ class Tiles(TaskNestedView):
if not src.tile_exists(z, x, y):
raise exceptions.NotFound(_("Outside of bounds"))
with COGReader(url) as src:
minzoom, maxzoom = get_zoom_safe(src)
has_alpha = has_alpha_band(src.dataset)
if z < minzoom - ZOOM_EXTRA_LEVELS or z > maxzoom + ZOOM_EXTRA_LEVELS:
@ -386,93 +363,125 @@ class Tiles(TaskNestedView):
if nodata is None and tile_type == 'orthophoto':
nodata = 0
resampling = "nearest"
padding = 0
if tile_type in ["dsm", "dtm"]:
resampling = "bilinear"
padding = 16
resampling = "nearest"
padding = 0
tile_buffer = None
try:
with COGReader(url) as src:
if tile_type in ["dsm", "dtm"]:
resampling = "bilinear"
padding = 16
# Hillshading is not a local tile operation and
# requires neighbor tiles to be rendered seamlessly
if hillshade is not None:
tile_buffer = tilesize
try:
if expr is not None:
if boundaries_cutline is not None:
tile = src.tile(x, y, z, expression=expr, tilesize=tilesize, nodata=nodata,
padding=padding,
tile_buffer=tile_buffer,
resampling_method=resampling, vrt_options={'cutline': boundaries_cutline})
else:
tile = src.tile(x, y, z, expression=expr, tilesize=tilesize, nodata=nodata,
padding=padding,
tile_buffer=tile_buffer,
resampling_method=resampling)
else:
if boundaries_cutline is not None:
tile = src.tile(x, y, z, tilesize=tilesize, nodata=nodata,
padding=padding,
tile_buffer=tile_buffer,
resampling_method=resampling, vrt_options={'cutline': boundaries_cutline})
else:
tile = src.tile(x, y, z, indexes=indexes, tilesize=tilesize, nodata=nodata,
padding=padding, resampling_method=resampling)
except TileOutsideBounds:
raise exceptions.NotFound(_("Outside of bounds"))
if color_map:
padding=padding,
tile_buffer=tile_buffer,
resampling_method=resampling)
except TileOutsideBounds:
raise exceptions.NotFound(_("Outside of bounds"))
if color_map:
try:
colormap.get(color_map)
except InvalidColorMapName:
raise exceptions.ValidationError(_("Not a valid color_map value"))
intensity = None
try:
colormap.get(color_map)
except InvalidColorMapName:
raise exceptions.ValidationError(_("Not a valid color_map value"))
intensity = None
try:
rescale_arr = list(map(float, rescale.split(",")))
except ValueError:
raise exceptions.ValidationError(_("Invalid rescale value"))
options = img_profiles.get(driver, {})
if hillshade is not None:
try:
hillshade = float(hillshade)
if hillshade <= 0:
hillshade = 1.0
rescale_arr = list(map(float, rescale.split(",")))
except ValueError:
raise exceptions.ValidationError(_("Invalid hillshade value"))
if tile.data.shape[0] != 1:
raise exceptions.ValidationError(
_("Cannot compute hillshade of non-elevation raster (multiple bands found)"))
delta_scale = (maxzoom + ZOOM_EXTRA_LEVELS + 1 - z) * 4
dx = src.dataset.meta["transform"][0] * delta_scale
dy = -src.dataset.meta["transform"][4] * delta_scale
ls = LightSource(azdeg=315, altdeg=45)
# Hillshading is not a local tile operation and
# requires neighbor tiles to be rendered seamlessly
elevation = get_elevation_tiles(tile.data[0], url, x, y, z, tilesize, nodata, resampling, padding)
intensity = ls.hillshade(elevation, dx=dx, dy=dy, vert_exag=hillshade)
intensity = intensity[tilesize:tilesize * 2, tilesize:tilesize * 2]
raise exceptions.ValidationError(_("Invalid rescale value"))
if intensity is not None:
rgb = tile.post_process(in_range=(rescale_arr,))
if colormap:
rgb, _discard_ = apply_cmap(rgb.data, colormap.get(color_map))
if rgb.data.shape[0] != 3:
raise exceptions.ValidationError(
_("Cannot process tile: intensity image provided, but no RGB data was computed."))
intensity = intensity * 255.0
rgb = hsv_blend(rgb, intensity)
if rgb is not None:
# Auto?
if ext is None:
# Check for transparency
if np.equal(tile.mask, 255).all():
ext = "jpg"
else:
if 'image/webp' in request.headers.get('Accept', ''):
ext = "webp"
else:
ext = "png"
driver = "jpeg" if ext == "jpg" else ext
options = img_profiles.get(driver, {})
if hillshade is not None:
try:
hillshade = float(hillshade)
if hillshade <= 0:
hillshade = 1.0
except ValueError:
raise exceptions.ValidationError(_("Invalid hillshade value"))
if tile.data.shape[0] != 1:
raise exceptions.ValidationError(
_("Cannot compute hillshade of non-elevation raster (multiple bands found)"))
delta_scale = (maxzoom + ZOOM_EXTRA_LEVELS + 1 - z) * 4
dx = src.dataset.meta["transform"][0] * delta_scale
dy = -src.dataset.meta["transform"][4] * delta_scale
ls = LightSource(azdeg=315, altdeg=45)
# Remove elevation data from edge buffer tiles
# (to keep intensity uniform across tiles)
elevation = tile.data[0]
elevation[0:tilesize, 0:tilesize] = nodata
elevation[tilesize*2:tilesize*3, 0:tilesize] = nodata
elevation[0:tilesize, tilesize*2:tilesize*3] = nodata
elevation[tilesize*2:tilesize*3, tilesize*2:tilesize*3] = nodata
intensity = ls.hillshade(elevation, dx=dx, dy=dy, vert_exag=hillshade)
intensity = intensity[tilesize:tilesize * 2, tilesize:tilesize * 2]
if intensity is not None:
rgb = tile.post_process(in_range=(rescale_arr,))
rgb_data = rgb.data[:,tilesize:tilesize * 2, tilesize:tilesize * 2]
if colormap:
rgb, _discard_ = apply_cmap(rgb_data, colormap.get(color_map))
if rgb.data.shape[0] != 3:
raise exceptions.ValidationError(
_("Cannot process tile: intensity image provided, but no RGB data was computed."))
intensity = intensity * 255.0
rgb = hsv_blend(rgb, intensity)
if rgb is not None:
mask = tile.mask[tilesize:tilesize * 2, tilesize:tilesize * 2]
return HttpResponse(
render(rgb, mask, img_format=driver, **options),
content_type="image/{}".format(ext)
)
if color_map is not None:
return HttpResponse(
render(rgb, tile.mask, img_format=driver, **options),
tile.post_process(in_range=(rescale_arr,)).render(img_format=driver, colormap=colormap.get(color_map),
**options),
content_type="image/{}".format(ext)
)
if color_map is not None:
return HttpResponse(
tile.post_process(in_range=(rescale_arr,)).render(img_format=driver, colormap=colormap.get(color_map),
**options),
tile.post_process(in_range=(rescale_arr,)).render(img_format=driver, **options),
content_type="image/{}".format(ext)
)
return HttpResponse(
tile.post_process(in_range=(rescale_arr,)).render(img_format=driver, **options),
content_type="image/{}".format(ext)
)
class Export(TaskNestedView):

4
app/api/urls.py
Wyświetl plik

@ -37,8 +37,8 @@ urlpatterns = [
url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/(?P<tile_type>orthophoto|dsm|dtm)/tiles\.json$', TileJson.as_view()),
url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/(?P<tile_type>orthophoto|dsm|dtm)/bounds$', Bounds.as_view()),
url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/(?P<tile_type>orthophoto|dsm|dtm)/metadata$', Metadata.as_view()),
url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/(?P<tile_type>orthophoto|dsm|dtm)/tiles/(?P<z>[\d]+)/(?P<x>[\d]+)/(?P<y>[\d]+)\.png$', Tiles.as_view()),
url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/(?P<tile_type>orthophoto|dsm|dtm)/tiles/(?P<z>[\d]+)/(?P<x>[\d]+)/(?P<y>[\d]+)@(?P<scale>[\d]+)x\.png$', Tiles.as_view()),
url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/(?P<tile_type>orthophoto|dsm|dtm)/tiles/(?P<z>[\d]+)/(?P<x>[\d]+)/(?P<y>[\d]+)\.?(?P<ext>png|jpg|webp)?$', Tiles.as_view()),
url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/(?P<tile_type>orthophoto|dsm|dtm)/tiles/(?P<z>[\d]+)/(?P<x>[\d]+)/(?P<y>[\d]+)@(?P<scale>[\d]+)x\.?(?P<ext>png|jpg|webp)?$', Tiles.as_view()),
url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/(?P<asset_type>orthophoto|dsm|dtm|georeferenced_model)/export$', Export.as_view()),
url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/download/(?P<asset>.+)$', TaskDownloads.as_view()),

3
app/static/app/js/components/LayersControlLayer.jsx
Wyświetl plik

@ -192,7 +192,8 @@ export default class LayersControlLayer extends React.Component {
formula,
bands,
hillshade,
rescale: this.rescale
rescale: this.rescale,
size: 512
};
}

5
app/static/app/js/components/Map.jsx
Wyświetl plik

@ -138,6 +138,7 @@ class Map extends React.Component {
// Build URL
let tileUrl = mres.tiles[0];
const TILESIZE = 512;
// Set rescale
if (statistics){
@ -150,7 +151,10 @@ class Map extends React.Component {
params["rescale"] = encodeURIComponent("-1,1");
}
params["size"] = TILESIZE;
tileUrl = Utils.buildUrlWithQuery(tileUrl, params);
}else{
tileUrl = Utils.buildUrlWithQuery(tileUrl, { size: TILESIZE });
}
const layer = Leaflet.tileLayer(tileUrl, {
@ -158,6 +162,7 @@ class Map extends React.Component {
minZoom: 0,
maxZoom: maxzoom + 99,
maxNativeZoom: maxzoom - 1,
tileSize: TILESIZE,
tms: scheme === 'tms',
opacity: this.state.opacity / 100,
detectRetina: true

4
app/static/app/js/components/TaskListItem.jsx
Wyświetl plik

@ -438,12 +438,12 @@ class TaskListItem extends React.Component {
};
if (task.status === statusCodes.COMPLETED){
if (task.available_assets.indexOf("orthophoto.tif") !== -1){
if (task.available_assets.indexOf("orthophoto.tif") !== -1 || task.available_assets.indexOf("dsm.tif") !== -1){
addActionButton(" " + _("View Map"), "btn-primary", "fa fa-globe", () => {
location.href = `/map/project/${task.project}/task/${task.id}/`;
});
}else{
showOrthophotoMissingWarning = true;
showOrthophotoMissingWarning = task.available_assets.indexOf("orthophoto.tif") === -1;
}
addActionButton(" " + _("View 3D Model"), "btn-primary", "fa fa-cube", () => {

35
app/tests/test_api_task.py
Wyświetl plik

@ -496,7 +496,7 @@ class TestApiTask(BootTransactionTestCase):
self.assertEqual(metadata['scheme'], 'xyz')
# Tiles URL has no extra params
self.assertTrue(metadata['tiles'][0].endswith('.png'))
self.assertTrue(metadata['tiles'][0].endswith('{z}/{x}/{y}'))
# Histogram stats are available (3 bands for orthophoto)
self.assertTrue(len(metadata['statistics']) == 3)
@ -554,6 +554,9 @@ class TestApiTask(BootTransactionTestCase):
'dsm': '18/64083/92370',
'dtm': '18/64083/92370'
}
tile_path_512 = {
'orthophoto': '18/32042/46185'
}
# Metadata for DSM/DTM
for tile_type in ['dsm', 'dtm']:
@ -647,7 +650,8 @@ class TestApiTask(BootTransactionTestCase):
("dtm", "hillshade=0", status.HTTP_200_OK),
("orthophoto", "hillshade=3", status.HTTP_400_BAD_REQUEST),
("orthophoto", "", status.HTTP_200_OK),
("orthophoto", "formula=NDVI&bands=RGN", status.HTTP_200_OK),
("orthophoto", "formula=VARI&bands=RGN", status.HTTP_400_BAD_REQUEST),
("orthophoto", "formula=VARI&bands=RGB", status.HTTP_200_OK),
@ -673,9 +677,32 @@ class TestApiTask(BootTransactionTestCase):
params.append(("orthophoto", "formula={}&bands={}&color_map=rdylgn".format(k, f), status.HTTP_200_OK))
for tile_type, url, sc in params:
res = client.get("/api/projects/{}/tasks/{}/{}/tiles/{}.png?{}".format(project.id, task.id, tile_type, tile_path[tile_type], url))
res = client.get("/api/projects/{}/tasks/{}/{}/tiles/{}?{}".format(project.id, task.id, tile_type, tile_path[tile_type], url))
self.assertEqual(res.status_code, sc)
# Can request PNG/JPG/WEBP tiles explicitely
for ext in ["png", "jpg", "webp"]:
res = client.get("/api/projects/{}/tasks/{}/orthophoto/tiles/{}.{}".format(project.id, task.id, tile_path['orthophoto'], ext))
self.assertEqual(res.status_code, status.HTTP_200_OK)
self.assertEqual(res.get('content-type'), "image/" + ext)
# Size is 256 by default
res = client.get("/api/projects/{}/tasks/{}/orthophoto/tiles/{}.png".format(project.id, task.id, tile_path['orthophoto']))
with Image.open(io.BytesIO(res.content)) as i:
self.assertEqual(i.width, 256)
self.assertEqual(i.height, 256)
# Can request 512 tiles
res = client.get("/api/projects/{}/tasks/{}/orthophoto/tiles/{}.png?size=512".format(project.id, task.id, tile_path_512['orthophoto']))
with Image.open(io.BytesIO(res.content)) as i:
self.assertEqual(i.width, 512)
self.assertEqual(i.height, 512)
# Cannot request invalid tiles sizes
for s in ["1024", "abc", "-1"]:
res = client.get("/api/projects/{}/tasks/{}/orthophoto/tiles/{}.png?size={}".format(project.id, task.id, tile_path['orthophoto'], s))
self.assertEqual(res.status_code, status.HTTP_400_BAD_REQUEST)
# Another user does not have access to the resources
other_client = APIClient()
other_client.login(username="testuser2", password="test1234")

2
locale

@ -1 +1 @@
Subproject commit 3a5fd44968553977618328d06a9b9bd7c0cfd2ba
Subproject commit 4e01fc1ccea78e41494191e082997000ee032c06

2
package.json
Wyświetl plik

@ -1,6 +1,6 @@
{
"name": "WebODM",
"version": "1.9.14",
"version": "1.9.15",
"description": "User-friendly, extendable application and API for processing aerial imagery.",
"main": "index.js",
"scripts": {