diff --git a/app/api/tiler.py b/app/api/tiler.py index 44541a5e..fed221f4 100644 --- a/app/api/tiler.py +++ b/app/api/tiler.py @@ -480,7 +480,7 @@ class Export(TaskNestedView): formula = request.data.get('formula') bands = request.data.get('bands') rescale = request.data.get('rescale') - export_format = request.data.get('format', 'gtiff') + export_format = request.data.get('format', 'laz' if asset_type == 'georeferenced_model' else 'gtiff') epsg = request.data.get('epsg') color_map = request.data.get('color_map') hillshade = request.data.get('hillshade') @@ -499,6 +499,19 @@ class Export(TaskNestedView): if asset_type == 'georeferenced_model' and not export_format in ['laz', 'las', 'ply', 'csv']: raise exceptions.ValidationError(_("Unsupported format: %(value)s") % {'value': export_format}) + # Default color map, hillshade + if asset_type in ['dsm', 'dtm'] and export_format != 'gtiff': + if color_map is None: + color_map = 'viridis' + if hillshade is None: + hillshade = 6 + + if color_map is not None: + try: + colormap.get(color_map) + except InvalidColorMapName: + raise exceptions.ValidationError(_("Not a valid color_map value")) + if epsg is not None: try: epsg = int(epsg) diff --git a/app/static/app/js/components/tests/ExportAssetDialog.test.jsx b/app/static/app/js/components/tests/ExportAssetDialog.test.jsx new file mode 100644 index 00000000..b5729e48 --- /dev/null +++ b/app/static/app/js/components/tests/ExportAssetDialog.test.jsx @@ -0,0 +1,14 @@ +import React from 'react'; +import { shallow } from 'enzyme'; +import ExportAssetDialog from '../ExportAssetDialog'; + +const taskMock = require('../../tests/utils/MockLoader').load("task.json"); + +describe('', () => { + it('renders without exploding', () => { + const wrapper = shallow( {}} + asset={"orthophoto"} + task={taskMock} />); + expect(wrapper.exists()).toBe(true); + }) +}); \ No newline at end of file diff --git a/app/static/app/js/components/tests/ExportAssetPanel.test.jsx b/app/static/app/js/components/tests/ExportAssetPanel.test.jsx new file mode 100644 index 00000000..1c2b3ab8 --- /dev/null +++ b/app/static/app/js/components/tests/ExportAssetPanel.test.jsx @@ -0,0 +1,14 @@ +import React from 'react'; +import { shallow } from 'enzyme'; +import ExportAssetPanel from '../ExportAssetPanel'; + +const taskMock = require('../../tests/utils/MockLoader').load("task.json"); + +describe('', () => { + it('renders without exploding', () => { + const wrapper = shallow(); + expect(wrapper.exists()).toBe(true); + }) +}); \ No newline at end of file diff --git a/app/tests/test_api_export.py b/app/tests/test_api_export.py new file mode 100644 index 00000000..826b3867 --- /dev/null +++ b/app/tests/test_api_export.py @@ -0,0 +1,200 @@ +import os +import time +from worker.celery import app as celery +import logging +import json + +import requests +from PIL import Image +from django.contrib.auth.models import User +from rest_framework import status +from rest_framework.test import APIClient +from app.plugins.signals import task_completed +from app.tests.classes import BootTransactionTestCase +from app.models import Project, Task +from nodeodm.models import ProcessingNode +from nodeodm import status_codes + +import worker +from worker.tasks import TestSafeAsyncResult + +from .utils import start_processing_node, clear_test_media_root, catch_signal + +# We need to test in a TransactionTestCase because +# task processing happens on a separate thread, and normal TestCases +# do not commit changes to the DB, so spawning a new thread will show no +# data in it. +from webodm import settings +logger = logging.getLogger('app.logger') + +DELAY = 2 # time to sleep for during process launch, background processing, etc. + +class TestApiTask(BootTransactionTestCase): + def setUp(self): + super().setUp() + + def tearDown(self): + clear_test_media_root() + + def test_exports(self): + client = APIClient() + + with start_processing_node(): + user = User.objects.get(username="testuser") + self.assertFalse(user.is_superuser) + + other_user = User.objects.get(username="testuser2") + + project = Project.objects.create( + owner=user, + name="test project" + ) + other_project = Project.objects.create( + owner=other_user, + name="another test project" + ) + + # Start processing node + + # Create processing node + pnode = ProcessingNode.objects.create(hostname="localhost", port=11223) + + # task creation via file upload + image1 = open("app/fixtures/tiny_drone_image.jpg", 'rb') + image2 = open("app/fixtures/tiny_drone_image_2.jpg", 'rb') + + client.login(username="testuser", password="test1234") + + # Normal case with images[], name and processing node parameter + res = client.post("/api/projects/{}/tasks/".format(project.id), { + 'images': [image1, image2], + 'name': 'test task', + 'processing_node': pnode.id + }, format="multipart") + self.assertTrue(res.status_code == status.HTTP_201_CREATED) + image1.close() + image2.close() + + # Should have returned the id of the newly created task + task = Task.objects.latest('created_at') + + params = [ + ('orthophoto', {'formula': 'NDVI', 'bands': 'RGN'}, status.HTTP_200_OK), + ('dsm', {'epsg': 4326}, status.HTTP_200_OK), + ('dtm', {'epsg': 4326}, status.HTTP_200_OK), + ('georeferenced_model', {'epsg': 4326}, status.HTTP_200_OK) + ] + + # Cannot export stuff + for p in params: + asset_type, data, _ = p + res = client.post("/api/projects/{}/tasks/{}/{}/export".format(project.id, task.id, asset_type), data) + self.assertEqual(res.status_code, status.HTTP_404_NOT_FOUND) + + # Assign processing node to task via API + res = client.patch("/api/projects/{}/tasks/{}/".format(project.id, task.id), { + 'processing_node': pnode.id + }) + self.assertTrue(res.status_code == status.HTTP_200_OK) + + retry_count = 0 + while task.status != status_codes.COMPLETED: + worker.tasks.process_pending_tasks() + time.sleep(DELAY) + task.refresh_from_db() + retry_count += 1 + if retry_count > 10: + break + + self.assertEqual(task.status, status_codes.COMPLETED) + + # Can export stuff (basic) + for p in params: + asset_type, data, exp_status = p + res = client.post("/api/projects/{}/tasks/{}/{}/export".format(project.id, task.id, asset_type), data) + self.assertEqual(res.status_code, exp_status) + reply = json.loads(res.content.decode("utf-8")) + self.assertTrue("celery_task_id" in reply) + celery_task_id = reply["celery_task_id"] + + # More exhaustive export testing + params = [ + ('orthophoto', {}, True, ".tif", status.HTTP_200_OK), + ('orthophoto', {'format': 'gtiff'}, True, ".tif", status.HTTP_200_OK), + ('orthophoto', {'format': 'gtiff-rgb', 'rescale': "10,100"}, False, ".tif", status.HTTP_200_OK), + ('orthophoto', {'format': 'laz'}, False, ".tif", status.HTTP_400_BAD_REQUEST), + ('orthophoto', {'format': 'jpg', 'epsg': 4326}, False, ".jpg", status.HTTP_200_OK), + ('orthophoto', {'format': 'jpg', 'epsg': 4326, 'rescale': '10,200'}, False, ".jpg", status.HTTP_200_OK), + ('orthophoto', {'format': 'png'}, False, ".png", status.HTTP_200_OK), + ('orthophoto', {'format': 'kmz'}, False, ".kmz", status.HTTP_200_OK), + + ('orthophoto', {'formula': 'NDVI'}, False, "-NDVI.tif", status.HTTP_400_BAD_REQUEST), + ('orthophoto', {'bands': 'RGN'}, False, "-NDVI.tif", status.HTTP_400_BAD_REQUEST), + ('orthophoto', {'bands': 'RGN', 'formula': 'NDVI'}, False, "-NDVI.tif", status.HTTP_200_OK), + + ('dsm', {'format': 'gtiff'}, True, ".tif", status.HTTP_200_OK), + ('dsm', {'epsg': 4326}, False, ".tif", status.HTTP_200_OK), + ('dsm', {'format': 'jpg', 'epsg': 4326}, False, ".jpg", status.HTTP_200_OK), + ('dsm', {'format': 'jpg', 'color_map': 'jet', 'hillshade': 0, 'epsg': 3857}, False, ".jpg", status.HTTP_200_OK), + ('dsm', {'epsg': 4326, 'format': 'jpg'}, False, ".jpg", status.HTTP_200_OK), + ('dsm', {'epsg': 4326, 'format': 'gtiff-rgb'}, False, ".tif", status.HTTP_200_OK), + ('dsm', {'format': 'kmz'}, False, ".kmz", status.HTTP_200_OK), + ('dsm', {'color_map': 'viridis', 'hillshade': 2, 'format': 'png'}, False, ".png", status.HTTP_200_OK), + ('dsm', {'rescale': 'invalid-but-works-cuz-gtiff'}, True, ".tif", status.HTTP_200_OK), + + ('dsm', {'epsg': 'invalid'}, False, ".tif", status.HTTP_400_BAD_REQUEST), + ('dsm', {'format': 'invalid'}, False, ".tif", status.HTTP_400_BAD_REQUEST), + ('dsm', {'hillshade': 'invalid'}, False, ".tif", status.HTTP_400_BAD_REQUEST), + ('dsm', {'color_map': 'invalid'}, False, ".tif", status.HTTP_400_BAD_REQUEST), + ('dsm', {'format': 'gtiff-rgb', 'rescale': 'invalid'}, False, ".tif", status.HTTP_400_BAD_REQUEST), + ('dsm', {'format': 'las'}, False, ".tif", status.HTTP_400_BAD_REQUEST), + + ('dtm', {'format': 'gtiff'}, True, ".tif", status.HTTP_200_OK), + ('dtm', {'epsg': 4326}, False, ".tif", status.HTTP_200_OK), + + ('georeferenced_model', {}, True, ".laz", status.HTTP_200_OK), + ('georeferenced_model', {'format': 'las'}, False, ".las", status.HTTP_200_OK), + ('georeferenced_model', {'format': 'ply'}, False, ".ply", status.HTTP_200_OK), + ('georeferenced_model', {'format': 'csv'}, False, ".csv", status.HTTP_200_OK), + ('georeferenced_model', {'format': 'las', 'epsg': 4326}, False, ".las", status.HTTP_200_OK), + + ('georeferenced_model', {'format': 'tif'}, False, ".laz", status.HTTP_400_BAD_REQUEST), + ] + + for p in params: + asset_type, data, shortcut_link, extension, exp_status = p + logger.info("Testing {}".format(p)) + res = client.post("/api/projects/{}/tasks/{}/{}/export".format(project.id, task.id, asset_type), data) + self.assertEqual(res.status_code, exp_status) + + reply = json.loads(res.content.decode("utf-8")) + + if res.status_code == status.HTTP_200_OK: + self.assertTrue("filename" in reply) + self.assertEqual(reply["filename"], "test-task-" + asset_type + extension) + + if shortcut_link: + self.assertFalse("celery_task_id" in reply) + self.assertTrue("url" in reply) + + # Can download + res = client.get(reply["url"]) + self.assertEqual(res.status_code, status.HTTP_200_OK) + else: + self.assertTrue("celery_task_id" in reply) + self.assertFalse("url" in reply) + + cres = TestSafeAsyncResult(celery_task_id) + c = 0 + while not cres.ready(): + time.sleep(0.2) + c += 1 + if c > 50: + self.assertTrue(False) + break + + res = client.get("/api/workers/get/{}?filename={}".format(celery_task_id, reply["filename"])) + self.assertEqual(res.status_code, status.HTTP_200_OK) + self.assertEqual(res._headers['content-disposition'][1], 'attachment; filename={}'.format(reply["filename"])) + else: + self.assertTrue(len(reply[0]) > 0) # Error message diff --git a/nodeodm/external/NodeODM b/nodeodm/external/NodeODM index 2fb254e3..dc32c0a2 160000 --- a/nodeodm/external/NodeODM +++ b/nodeodm/external/NodeODM @@ -1 +1 @@ -Subproject commit 2fb254e378abfa42ddd04bd0662583b9dbe277bf +Subproject commit dc32c0a232e78fd5ba0ffa0d3837b50679b0a66b