mirror
/
OpenDroneMap-WebODM
kopia lustrzana https://github.com/OpenDroneMap/WebODM
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge pull request #94 from pierotofy/unittest 

Unittest
pull/120/head
Piero Toffanin 2017-02-08 13:53:55 -05:00 zatwierdzone przez GitHub
rodzic 9807521376 eae691cf73
commit 03f07a223f
22 zmienionych plików z 707 dodań i 122 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

1
app/.gitignore vendored 100644
Wyświetl plik

@ -0,0 +1 @@
media_test/

2
app/api/processingnodes.py
Wyświetl plik

@ -1,8 +1,6 @@
import django_filters
from django_filters.rest_framework import FilterSet
from rest_framework import serializers, viewsets
from rest_framework.filters import DjangoFilterBackend
from rest_framework.permissions import DjangoModelPermissions
from nodeodm.models import ProcessingNode

13
app/api/tasks.py
Wyświetl plik

@ -121,8 +121,17 @@ class TaskViewSet(viewsets.ViewSet):
[keys for keys in request.FILES])
for file in filesList]
if len(files) <= 1:
raise exceptions.ValidationError(detail="Cannot create task, you need at least 2 images")
task = models.Task.create_from_images(files, project)
if task is not None:
# Update other parameters such as processing node, task name, etc.
serializer = TaskSerializer(task, data=request.data, partial=True)
serializer.is_valid(raise_exception=True)
serializer.save()
return Response({"id": task.id}, status=status.HTTP_201_CREATED)
else:
raise exceptions.ValidationError(detail="Cannot create task, input provided is not valid.")
@ -182,6 +191,10 @@ class TaskTilesJson(TaskNestedView):
task = self.get_and_check_task(request, pk, project_pk, annotate={
'orthophoto_area': Envelope(Cast("orthophoto", GeometryField()))
})
if task.orthophoto_area is None:
raise exceptions.ValidationError("An orthophoto has not been processed for this task. Tiles are not available yet.")
json = get_tile_json(task.name, [
'/api/projects/{}/tasks/{}/tiles/{{z}}/{{x}}/{{y}}.png'.format(task.project.id, task.id)
], task.orthophoto_area.extent)

41
app/background.py 100644
Wyświetl plik

@ -0,0 +1,41 @@
from threading import Thread
import logging
from django import db
from webodm import settings
from app.testwatch import testWatch
logger = logging.getLogger('app.logger')
def background(func):
"""
Adds background={True|False} param to any function
so that we can call update_nodes_info(background=True) from the outside
"""
def wrapper(*args,**kwargs):
background = kwargs.get('background', False)
if 'background' in kwargs: del kwargs['background']
if background:
if testWatch.hook_pre(func, *args, **kwargs): return
# Create a function that closes all
# db connections at the end of the thread
# This is necessary to make sure we don't leave
# open connections lying around.
def execute_and_close_db():
ret = None
try:
ret = func(*args, **kwargs)
finally:
db.connections.close_all()
testWatch.hook_post(func, *args, **kwargs)
return ret
t = Thread(target=execute_and_close_db)
t.daemon = True
t.start()
return t
else:
return func(*args, **kwargs)
return wrapper

BIN
app/fixtures/tiny_drone_image.jpg 100644
Wyświetl plik

Plik binarny nie jest wyświetlany.
Side by Side

Po

Szerokość:  |  Wysokość:  |  Rozmiar: 17 KiB

BIN
app/fixtures/tiny_drone_image_2.jpg 100644
Wyświetl plik

Plik binarny nie jest wyświetlany.
Side by Side

Po

Szerokość:  |  Wysokość:  |  Rozmiar: 17 KiB

42
app/models.py
Wyświetl plik

@ -2,6 +2,7 @@ import logging
import os
import shutil
import zipfile
import requests
from django.contrib.auth.models import User
from django.contrib.gis.gdal import GDALRaster
@ -146,7 +147,9 @@ class Task(models.Model):
pending_action = models.IntegerField(choices=PENDING_ACTIONS, db_index=True, null=True, blank=True, help_text="A requested action to be performed on the task. The selected action will be performed by the scheduler at the next iteration.")
def __str__(self):
return 'Task ID: {}'.format(self.id)
name = self.name if self.name is not None else "unnamed"
return 'Task {} ({})'.format(name, self.id)
def save(self, *args, **kwargs):
# Autovalidate on save
@ -189,8 +192,8 @@ class Task(models.Model):
try:
if self.processing_node:
# Need to process some images (UUID not yet set and task not marked for deletion)?
if not self.uuid and self.pending_action != pending_actions.REMOVE:
# Need to process some images (UUID not yet set and task doesn't have pending actions)?
if not self.uuid and self.pending_action is None:
logger.info("Processing... {}".format(self))
images = [image.path() for image in self.imageupload_set.all()]
@ -217,29 +220,31 @@ class Task(models.Model):
if self.processing_node and self.uuid:
self.processing_node.cancel_task(self.uuid)
self.pending_action = None
self.status = None
self.save()
else:
raise ProcessingException("Cannot cancel a task that has no processing node or UUID")
elif self.pending_action == pending_actions.RESTART:
logger.info("Restarting task {}".format(self))
if self.processing_node and self.uuid:
logger.info("Restarting {}".format(self))
if self.processing_node:
# Check if the UUID is still valid, as processing nodes purge
# results after a set amount of time, the UUID might have eliminated.
try:
info = self.processing_node.get_task_info(self.uuid)
uuid_still_exists = info['uuid'] == self.uuid
except ProcessingException:
uuid_still_exists = False
uuid_still_exists = False
if self.uuid:
try:
info = self.processing_node.get_task_info(self.uuid)
uuid_still_exists = info['uuid'] == self.uuid
except ProcessingException:
pass
if uuid_still_exists:
# Good to go
self.processing_node.restart_task(self.uuid)
else:
# Task has been purged (or processing node is offline)
# TODO: what if processing node went offline?
# Process this as a new task
# Removing its UUID will cause the scheduler
# to process this the next tick
@ -341,11 +346,10 @@ class Task(models.Model):
self.save()
except ProcessingException as e:
self.set_failure(str(e))
except ConnectionRefusedError as e:
logger.warning("Task {} cannot communicate with processing node: {}".format(self, str(e)))
# In the future we might want to retry instead of just failing
#self.set_failure(str(e))
except (ConnectionRefusedError, ConnectionError) as e:
logger.warning("{} cannot communicate with processing node: {}".format(self, str(e)))
except requests.exceptions.ConnectTimeout as e:
logger.warning("{} timed out with error: {}. We'll try reprocessing at the next tick.".format(self, str(e)))
def get_tile_path(self, z, x, y):
@ -373,11 +377,11 @@ class Task(models.Model):
try:
shutil.rmtree(directory_to_delete)
except FileNotFoundError as e:
logger.warn(e)
logger.warning(e)
def set_failure(self, error_message):
logger.error("{} ERROR: {}".format(self, error_message))
logger.error("FAILURE FOR {}: {}".format(self, error_message))
self.last_error = error_message
self.status = status_codes.FAILED
self.save()

56
app/scheduler.py
Wyświetl plik

@ -1,16 +1,18 @@
import logging
import traceback
from multiprocessing.dummy import Pool as ThreadPool
from threading import Lock
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.schedulers import SchedulerAlreadyRunningError, SchedulerNotRunningError
from threading import Thread, Lock
from multiprocessing.dummy import Pool as ThreadPool
from nodeodm.models import ProcessingNode
from app.models import Task, Project
from django.db.models import Q, Count
from apscheduler.schedulers.background import BackgroundScheduler
from django import db
from django.db.models import Q, Count
from webodm import settings
from app.models import Task, Project
from nodeodm import status_codes
import random
from nodeodm.models import ProcessingNode
from app.background import background
logger = logging.getLogger('app.logger')
scheduler = BackgroundScheduler({
@ -18,43 +20,12 @@ scheduler = BackgroundScheduler({
'apscheduler.job_defaults.max_instances': '3',
})
def background(func):
"""
Adds background={True|False} param to any function
so that we can call update_nodes_info(background=True) from the outside
"""
def wrapper(*args,**kwargs):
background = kwargs.get('background', False)
if 'background' in kwargs: del kwargs['background']
if background:
# Create a function that closes all
# db connections at the end of the thread
# This is necessary to make sure we don't leave
# open connections lying around.
def execute_and_close_db():
ret = None
try:
ret = func(*args, **kwargs)
finally:
db.connections.close_all()
return ret
t = Thread(target=execute_and_close_db)
t.start()
return t
else:
return func(*args, **kwargs)
return wrapper
@background
def update_nodes_info():
processing_nodes = ProcessingNode.objects.all()
for processing_node in processing_nodes:
processing_node.update_node_info()
tasks_mutex = Lock()
@background
@ -81,13 +52,16 @@ def process_pending_tasks():
def process(task):
try:
task.process()
except Exception as e:
logger.error("Uncaught error! This is potentially bad. Please report it to http://github.com/OpenDroneMap/WebODM/issues: {} {}".format(e, traceback.format_exc()))
if settings.TESTING: raise e
finally:
# Might have been deleted
if task.pk is not None:
task.processing_lock = False
task.save()
except Exception as e:
logger.error("Uncaught error: {} {}".format(e, traceback.format_exc()))
db.connections.close_all()
if tasks.count() > 0:
pool = ThreadPool(tasks.count())

3
app/static/app/js/ModelView.jsx
Wyświetl plik

@ -46,10 +46,11 @@ class ModelView extends React.Component {
}
objFilePath(){
return this.texturedModelDirectoryPath() + 'odm_textured_model.obj';
return this.texturedModelDirectoryPath() + 'odm_textured_model_geo.obj';
}
mtlFilename(){
// For some reason, loading odm_textured_model_geo.mtl does not load textures properly
return 'odm_textured_model.mtl';
}

5
app/static/app/js/css/ModelView.scss
Wyświetl plik

@ -4,6 +4,11 @@
.model-view{
height: 80%;
canvas{
width: 100% !important;
height: 100% !important;
}
.container{
background: rgb(79,79,79);
background: -moz-radial-gradient(center, ellipse cover, rgba(79,79,79,1) 0%, rgba(22,22,22,1) 100%);

79
app/tests/classes.py
Wyświetl plik

@ -1,10 +1,41 @@
from django import db
from django.contrib.auth.models import User
from django.test import TestCase
from django.test import TransactionTestCase
from app.boot import boot
from app.models import Project
def setupUsers():
User.objects.create_superuser(username='testsuperuser',
email='superuser@test.com',
password='test1234')
User.objects.create_user(username='testuser',
email='user@test.com',
password='test1234')
User.objects.create_user(username='testuser2',
email='user2@test.com',
password='test1234')
def setupProjects():
Project.objects.create(
owner=User.objects.get(username="testsuperuser"),
name="Super User Test Project",
description="This is a test project"
)
Project.objects.create(
owner=User.objects.get(username="testuser"),
name="User Test Project",
description="This is a test project"
)
Project.objects.create(
owner=User.objects.get(username="testuser2"),
name="User 2 Test Project",
description="This is a test project"
)
class BootTestCase(TestCase):
'''
This class provides optional default mock data as well as
@ -15,36 +46,8 @@ class BootTestCase(TestCase):
for some models, which doesn't play well with them.
'''
@classmethod
def setUpClass(cls):
def setupUsers():
User.objects.create_superuser(username='testsuperuser',
email='superuser@test.com',
password='test1234')
User.objects.create_user(username='testuser',
email='user@test.com',
password='test1234')
User.objects.create_user(username='testuser2',
email='user2@test.com',
password='test1234')
def setupProjects():
Project.objects.create(
owner=User.objects.get(username="testsuperuser"),
name="Super User Test Project",
description="This is a test project"
)
Project.objects.create(
owner=User.objects.get(username="testuser"),
name="User Test Project",
description="This is a test project"
)
Project.objects.create(
owner=User.objects.get(username="testuser2"),
name="User 2 Test Project",
description="This is a test project"
)
super(BootTestCase, cls).setUpClass()
def setUpTestData(cls):
super(BootTestCase, cls).setUpTestData()
boot()
setupUsers()
setupProjects()
@ -52,3 +55,19 @@ class BootTestCase(TestCase):
@classmethod
def tearDownClass(cls):
super(BootTestCase, cls).tearDownClass()
class BootTransactionTestCase(TransactionTestCase):
'''
Same as above, but inherits from TransactionTestCase
'''
@classmethod
def setUpClass(cls):
super(BootTransactionTestCase, cls).setUpClass()
boot()
setupUsers()
setupProjects()
@classmethod
def tearDownClass(cls):
super(BootTransactionTestCase, cls).tearDownClass()

41
app/tests/test_api.py
Wyświetl plik

@ -1,15 +1,20 @@
import datetime
import subprocess
from guardian.shortcuts import assign_perm
from app import pending_actions
from nodeodm import status_codes
from .classes import BootTestCase
from rest_framework.test import APIClient
from rest_framework import status
import datetime
import time, os
from app.models import Project, Task
from app.models import Project, Task, ImageUpload
from nodeodm.models import ProcessingNode
from django.contrib.auth.models import User
class TestApi(BootTestCase):
def setUp(self):
pass
@ -23,12 +28,14 @@ class TestApi(BootTestCase):
user = User.objects.get(username="testuser")
self.assertFalse(user.is_superuser)
other_user = User.objects.get(username="testuser2")
project = Project.objects.create(
owner=user,
name="test project"
)
other_project = Project.objects.create(
owner=User.objects.get(username="testuser2"),
owner=other_user,
name="another test project"
)
@ -191,15 +198,27 @@ class TestApi(BootTestCase):
self.assertTrue(task.last_error is None)
self.assertTrue(task.pending_action == pending_actions.REMOVE)
# Can delete project that we we own
temp_project = Project.objects.create(owner=user)
res = client.delete('/api/projects/{}/'.format(temp_project.id))
self.assertTrue(res.status_code == status.HTTP_204_NO_CONTENT)
self.assertTrue(Project.objects.filter(id=temp_project.id).count() == 0) # Really deleted
# Cannot delete a project we don't own
other_temp_project = Project.objects.create(owner=other_user)
res = client.delete('/api/projects/{}/'.format(other_temp_project.id))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Can't delete a project for which we just have view permissions
assign_perm('view_project', user, other_temp_project)
res = client.delete('/api/projects/{}/'.format(other_temp_project.id))
self.assertTrue(res.status_code == status.HTTP_403_FORBIDDEN)
# Can delete a project for which we have delete permissions
assign_perm('delete_project', user, other_temp_project)
res = client.delete('/api/projects/{}/'.format(other_temp_project.id))
self.assertTrue(res.status_code == status.HTTP_204_NO_CONTENT)
# TODO test:
# - tiles.json requests
# - task creation via file upload
# - scheduler processing steps
# - tiles API urls (permissions, 404s)
# - assets download (aliases)
# - assets raw downloads
# - project deletion
def test_processingnodes(self):
client = APIClient()

326
app/tests/test_api_task.py 100644
Wyświetl plik

@ -0,0 +1,326 @@
import os
import subprocess
import time
import shutil
import logging
import requests
from django.contrib.auth.models import User
from rest_framework import status
from rest_framework.test import APIClient
from app import pending_actions
from app import scheduler
from app.models import Project, Task, ImageUpload, task_directory_path
from app.tests.classes import BootTransactionTestCase
from nodeodm import status_codes
from nodeodm.models import ProcessingNode
from app.testwatch import testWatch
# We need to test the task API in a TransactionTestCase because
# task processing happens on a separate thread, and normal TestCases
# do not commit changes to the DB, so spawning a new thread will show no
# data in it.
from webodm import settings
logger = logging.getLogger('app.logger')
DELAY = 1 # time to sleep for during process launch, background processing, etc.
def start_processing_node():
current_dir = os.path.dirname(os.path.realpath(__file__))
node_odm = subprocess.Popen(['node', 'index.js', '--port', '11223', '--test'], shell=False,
cwd=os.path.join(current_dir, "..", "..", "nodeodm", "external", "node-OpenDroneMap"))
time.sleep(DELAY) # Wait for the server to launch
return node_odm
class TestApiTask(BootTransactionTestCase):
def setUp(self):
# We need to clear previous media_root content
# This points to the test directory, but just in case
# we double check that the directory is indeed a test directory
if "_test" in settings.MEDIA_ROOT:
if os.path.exists(settings.MEDIA_ROOT):
logger.info("Cleaning up {}".format(settings.MEDIA_ROOT))
shutil.rmtree(settings.MEDIA_ROOT)
else:
logger.warning("We did not remove MEDIA_ROOT because we couldn't find a _test suffix in its path.")
def test_task(self):
client = APIClient()
node_odm = start_processing_node()
user = User.objects.get(username="testuser")
self.assertFalse(user.is_superuser)
other_user = User.objects.get(username="testuser2")
project = Project.objects.create(
owner=user,
name="test project"
)
other_project = Project.objects.create(
owner=other_user,
name="another test project"
)
other_task = Task.objects.create(project=other_project)
# Start processing node
# Create processing node
pnode = ProcessingNode.objects.create(hostname="localhost", port=11223)
# Verify that it's working
self.assertTrue(pnode.api_version is not None)
# task creation via file upload
image1 = open("app/fixtures/tiny_drone_image.jpg", 'rb')
image2 = open("app/fixtures/tiny_drone_image_2.jpg", 'rb')
# Not authenticated?
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2]
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_403_FORBIDDEN);
client.login(username="testuser", password="test1234")
# Cannot create a task for a project that does not exist
res = client.post("/api/projects/0/tasks/", {
'images': [image1, image2]
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot create a task for a project for which we have no access to
res = client.post("/api/projects/{}/tasks/".format(other_project.id), {
'images': [image1, image2]
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot create a task without images
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': []
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST)
# Cannot create a task with just 1 image
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': image1
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST)
# Normal case with images[], name and processing node parameter
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2],
'name': 'test_task',
'processing_node': pnode.id
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
multiple_param_task = Task.objects.latest('created_at')
self.assertTrue(multiple_param_task.name == 'test_task')
self.assertTrue(multiple_param_task.processing_node.id == pnode.id)
# Cannot create a task with images[], name, but invalid processing node parameter
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2],
'name': 'test_task',
'processing_node': 9999
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST)
# Normal case with just images[] parameter
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2]
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
# Should have returned the id of the newly created task
task = Task.objects.latest('created_at')
self.assertTrue('id' in res.data)
self.assertTrue(task.id == res.data['id'])
# Two images should have been uploaded
self.assertTrue(ImageUpload.objects.filter(task=task).count() == 2)
# No processing node is set
self.assertTrue(task.processing_node is None)
# tiles.json should not be accessible at this point
res = client.get("/api/projects/{}/tasks/{}/tiles.json".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_400_BAD_REQUEST)
# Neither should an individual tile
# Z/X/Y coords are choosen based on node-odm test dataset for orthophoto_tiles/
res = client.get("/api/projects/{}/tasks/{}/tiles/16/16020/42443.png".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot access a tiles.json we have no access to
res = client.get("/api/projects/{}/tasks/{}/tiles.json".format(other_project.id, other_task.id))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot access an individual tile we have no access to
res = client.get("/api/projects/{}/tasks/{}/tiles/16/16020/42443.png".format(other_project.id, other_task.id))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot download assets (they don't exist yet)
assets = ["all", "geotiff", "las", "csv", "ply"]
for asset in assets:
res = client.get("/api/projects/{}/tasks/{}/download/{}/".format(project.id, task.id, asset))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot access raw assets (they don't exist yet)
res = client.get("/api/projects/{}/tasks/{}/assets/odm_orthophoto/odm_orthophoto.tif".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot assign processing node to a task we have no access to
res = client.patch("/api/projects/{}/tasks/{}/".format(other_project.id, other_task.id), {
'processing_node': pnode.id
})
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
testWatch.clear()
# No UUID at this point
self.assertTrue(len(task.uuid) == 0)
# Assign processing node to task via API
res = client.patch("/api/projects/{}/tasks/{}/".format(project.id, task.id), {
'processing_node': pnode.id
})
self.assertTrue(res.status_code == status.HTTP_200_OK)
# On update scheduler.processing_pending_tasks should have been called in the background
testWatch.wait_until_call("app.scheduler.process_pending_tasks", timeout=5)
# Processing should have started and a UUID is assigned
task.refresh_from_db()
self.assertTrue(task.status == status_codes.RUNNING)
self.assertTrue(len(task.uuid) > 0)
time.sleep(DELAY)
# Calling process pending tasks should finish the process
scheduler.process_pending_tasks()
task.refresh_from_db()
self.assertTrue(task.status == status_codes.COMPLETED)
# Can download assets
for asset in assets:
res = client.get("/api/projects/{}/tasks/{}/download/{}/".format(project.id, task.id, asset))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# Can download raw assets
res = client.get("/api/projects/{}/tasks/{}/assets/odm_orthophoto/odm_orthophoto.tif".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# Can access tiles.json and individual tiles
res = client.get("/api/projects/{}/tasks/{}/tiles.json".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
res = client.get("/api/projects/{}/tasks/{}/tiles/16/16020/42443.png".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# Restart a task
testWatch.clear()
res = client.post("/api/projects/{}/tasks/{}/restart/".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
testWatch.wait_until_call("app.scheduler.process_pending_tasks", timeout=5)
task.refresh_from_db()
self.assertTrue(task.status in [status_codes.RUNNING, status_codes.COMPLETED])
# Cancel a task
testWatch.clear()
res = client.post("/api/projects/{}/tasks/{}/cancel/".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
testWatch.wait_until_call("app.scheduler.process_pending_tasks", timeout=5)
# Should have been canceled
task.refresh_from_db()
self.assertTrue(task.status == status_codes.CANCELED)
# Remove a task
res = client.post("/api/projects/{}/tasks/{}/remove/".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
testWatch.wait_until_call("app.scheduler.process_pending_tasks", 2, timeout=5)
# Has been removed along with assets
self.assertFalse(Task.objects.filter(pk=task.id).exists())
self.assertFalse(ImageUpload.objects.filter(task=task).exists())
task_assets_path = os.path.join(settings.MEDIA_ROOT, task_directory_path(task.id, task.project.id))
self.assertFalse(os.path.exists(task_assets_path))
testWatch.clear()
testWatch.intercept("app.scheduler.process_pending_tasks")
# Create a task, then kill the processing node
res = client.post("/api/projects/{}/tasks/".format(project.id), {
'images': [image1, image2],
'name': 'test_task_offline',
'processing_node': pnode.id
}, format="multipart")
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
task = Task.objects.get(pk=res.data['id'])
# Stop processing node
node_odm.terminate()
task.refresh_from_db()
self.assertTrue(task.last_error is None)
scheduler.process_pending_tasks()
# Processing should fail and set an error
task.refresh_from_db()
self.assertTrue(task.last_error is not None)
self.assertTrue(task.status == status_codes.FAILED)
# Now bring it back online
node_odm = start_processing_node()
# Restart
res = client.post("/api/projects/{}/tasks/{}/restart/".format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
task.refresh_from_db()
self.assertTrue(task.pending_action == pending_actions.RESTART)
# After processing, the task should have restarted, and have no UUID or status
scheduler.process_pending_tasks()
task.refresh_from_db()
self.assertTrue(task.status is None)
self.assertTrue(len(task.uuid) == 0)
# Another step and it should have acquired a UUID
scheduler.process_pending_tasks()
task.refresh_from_db()
self.assertTrue(task.status is status_codes.RUNNING)
self.assertTrue(len(task.uuid) > 0)
# Another step and it should be completed
time.sleep(DELAY)
scheduler.process_pending_tasks()
task.refresh_from_db()
self.assertTrue(task.status == status_codes.COMPLETED)
# Test connection, timeout errors
res = client.post("/api/projects/{}/tasks/{}/restart/".format(project.id, task.id))
def connTimeout(*args, **kwargs):
raise requests.exceptions.ConnectTimeout("Simulated timeout")
testWatch.intercept("nodeodm.api_client.task_output", connTimeout)
scheduler.process_pending_tasks()
# Timeout errors should be handled by retrying again at a later time
# and not fail
task.refresh_from_db()
self.assertTrue(task.last_error is None)
image1.close()
image2.close()
node_odm.terminate()

38
app/tests/test_app.py
Wyświetl plik

@ -1,5 +1,6 @@
from django.contrib.auth.models import User, Group
from django.test import Client
from rest_framework import status
from app.models import Project, Task
from .classes import BootTestCase
@ -57,6 +58,12 @@ class TestApp(BootTestCase):
res = c.get('/processingnode/1/', follow=True)
self.assertRedirects(res, '/login/?next=/processingnode/1/')
res = c.get('/map/project/1/', follow=True)
self.assertRedirects(res, '/login/?next=/map/project/1/')
res = c.get('/3d/project/1/task/1/', follow=True)
self.assertRedirects(res, '/login/?next=/3d/project/1/task/1/')
# Login
c.post('/login/', data=self.credentials, follow=True)
@ -84,8 +91,35 @@ class TestApp(BootTestCase):
res = c.get('/processingnode/abc/')
self.assertTrue(res.status_code == 404)
# TODO:
# - test /map/ urls
# /map/ and /3d/ views
user = User.objects.get(username="testuser")
other_user = User.objects.get(username="testuser2")
project = Project.objects.create(owner=user)
task = Task.objects.create(project=project)
other_project = Project.objects.create(owner=other_user)
other_task = Task.objects.create(project=other_project)
# Cannot access a project that we have no access to, or that does not exist
for project_id in [other_project.id, 99999]:
res = c.get('/map/project/{}/'.format(project_id))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# We can access a project that we have access to
res = c.get('/map/project/{}/'.format(project.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
# 3D views need project and task parameters
res = c.get('/3d/project/{}/'.format(project.id))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Cannot access a 3d view for a task we have no access to
res = c.get('/3d/project/{}/task/{}/'.format(other_project.id, other_task.id))
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
# Can access 3d view for task we have access to
res = c.get('/3d/project/{}/task/{}/'.format(project.id, task.id))
self.assertTrue(res.status_code == status.HTTP_200_OK)
def test_default_group(self):
# It exists

57
app/tests/test_testwatch.py 100644
Wyświetl plik

@ -0,0 +1,57 @@
from django.test import TestCase
from app.testwatch import TestWatch
def test(a, b):
return a + b
class TestTestWatch(TestCase):
def test_methods(self):
tw = TestWatch()
self.assertTrue(tw.get_calls_count("app.tests.test_testwatch.test") == 0)
self.assertTrue(tw.get_calls_count("app.tests.test_testwatch.nonexistant") == 0)
# Test watch count
tw.hook_pre(test, 1, 2)
test(1, 2)
tw.hook_post(test, 1, 2)
self.assertTrue(tw.get_calls_count("app.tests.test_testwatch.test") == 1)
tw.hook_pre(test, 1, 2)
test(1, 2)
tw.hook_post(test, 1, 2)
self.assertTrue(tw.get_calls_count("app.tests.test_testwatch.test") == 2)
@TestWatch.watch(testWatch=tw)
def test2(d):
d['flag'] = not d['flag']
# Test intercept
tw.intercept("app.tests.test_testwatch.test2")
d = {'flag': True}
test2(d)
self.assertTrue(d['flag'])
# Test function replacement intercept
d = {
'a': False,
'b': False
}
@TestWatch.watch(testWatch=tw)
def test3(d):
d['a'] = True
def replacement(d):
d['b'] = True
tw.intercept("app.tests.test_testwatch.test3", replacement)
test3(d)
self.assertFalse(d['a'])
self.assertTrue(d['b'])

83
app/testwatch.py 100644
Wyświetl plik

@ -0,0 +1,83 @@
import time
import logging
from webodm import settings
logger = logging.getLogger('app.logger')
class TestWatch:
def __init__(self):
self.clear()
def clear(self):
self._calls = {}
self._intercept_list = {}
def func_to_name(f):
return "{}.{}".format(f.__module__, f.__name__)
def intercept(self, fname, f = None):
self._intercept_list[fname] = f if f is not None else True
def execute_intercept_function_replacement(self, fname, *args, **kwargs):
if fname in self._intercept_list and callable(self._intercept_list[fname]):
(self._intercept_list[fname])(*args, **kwargs)
def should_prevent_execution(self, func):
return TestWatch.func_to_name(func) in self._intercept_list
def get_calls(self, fname):
return self._calls[fname] if fname in self._calls else []
def get_calls_count(self, fname):
return len(self.get_calls(fname))
def wait_until_call(self, fname, count = 1, timeout = 30):
SLEEP_INTERVAL = 0.125
TIMEOUT_LIMIT = timeout / SLEEP_INTERVAL
c = 0
while self.get_calls_count(fname) < count and c < TIMEOUT_LIMIT:
time.sleep(SLEEP_INTERVAL)
c += 1
if c >= TIMEOUT_LIMIT:
raise TimeoutError("wait_until_call has timed out waiting for {}".format(fname))
return self.get_calls(fname)
def log_call(self, func, *args, **kwargs):
fname = TestWatch.func_to_name(func)
logger.info("{} called".format(fname))
list = self._calls[fname] if fname in self._calls else []
list.append({'f': fname, 'args': args, 'kwargs': kwargs})
self._calls[fname] = list
def hook_pre(self, func, *args, **kwargs):
if settings.TESTING and self.should_prevent_execution(func):
fname = TestWatch.func_to_name(func)
logger.info(fname + " intercepted")
self.execute_intercept_function_replacement(fname, *args, **kwargs)
self.log_call(func, *args, **kwargs)
return True # Intercept
return False # Do not intercept
def hook_post(self, func, *args, **kwargs):
if settings.TESTING:
self.log_call(func, *args, **kwargs)
def watch(**kwargs):
"""
Decorator that adds pre/post hook calls
"""
tw = kwargs.get('testWatch', testWatch)
def outer(func):
def wrapper(*args, **kwargs):
if tw.hook_pre(func, *args, **kwargs): return
ret = func(*args, **kwargs)
tw.hook_post(func, *args, **kwargs)
return ret
return wrapper
return outer
testWatch = TestWatch()

4
app/views.py
Wyświetl plik

@ -34,7 +34,7 @@ def map(request, project_pk=None, task_pk=None):
if project_pk is not None:
project = get_object_or_404(Project, pk=project_pk)
if not request.user.has_perm('projects.view_project', project):
if not request.user.has_perm('app.view_project', project):
raise Http404()
if task_pk is not None:
@ -59,7 +59,7 @@ def model_display(request, project_pk=None, task_pk=None):
if project_pk is not None:
project = get_object_or_404(Project, pk=project_pk)
if not request.user.has_perm('projects.view_project', project):
if not request.user.has_perm('app.view_project', project):
raise Http404()
if task_pk is not None:

17
nodeodm/api_client.py
Wyświetl plik

@ -7,7 +7,9 @@ import mimetypes
import json
import os
from urllib.parse import urlunparse
from app.testwatch import TestWatch
TIMEOUT = 30
class ApiClient:
def __init__(self, host, port):
@ -21,25 +23,26 @@ class ApiClient:
return urlunparse(('http', netloc, url, '', '', ''))
def info(self):
return requests.get(self.url('/info')).json()
return requests.get(self.url('/info'), timeout=TIMEOUT).json()
def options(self):
return requests.get(self.url('/options')).json()
return requests.get(self.url('/options'), timeout=TIMEOUT).json()
def task_info(self, uuid):
return requests.get(self.url('/task/{}/info').format(uuid)).json()
return requests.get(self.url('/task/{}/info').format(uuid), timeout=TIMEOUT).json()
@TestWatch.watch()
def task_output(self, uuid, line = 0):
return requests.get(self.url('/task/{}/output?line={}').format(uuid, line)).json()
return requests.get(self.url('/task/{}/output?line={}').format(uuid, line), timeout=TIMEOUT).json()
def task_cancel(self, uuid):
return requests.post(self.url('/task/cancel'), data={'uuid': uuid}).json()
return requests.post(self.url('/task/cancel'), data={'uuid': uuid}, timeout=TIMEOUT).json()
def task_remove(self, uuid):
return requests.post(self.url('/task/remove'), data={'uuid': uuid}).json()
return requests.post(self.url('/task/remove'), data={'uuid': uuid}, timeout=TIMEOUT).json()
def task_restart(self, uuid):
return requests.post(self.url('/task/restart'), data={'uuid': uuid}).json()
return requests.post(self.url('/task/restart'), data={'uuid': uuid}, timeout=TIMEOUT).json()
def task_download(self, uuid, asset):
res = requests.get(self.url('/task/{}/download/{}').format(uuid, asset), stream=True)

2
nodeodm/external/node-OpenDroneMap vendored

@ -1 +1 @@
Subproject commit 254ce04f55db521acbae1b38294d2ec65e3c8a09
Subproject commit a25e688bcb31972671f3735efe17c0b1c32e6da4

11
nodeodm/models.py
Wyświetl plik

@ -58,7 +58,7 @@ class ProcessingNode(models.Model):
self.last_refreshed = timezone.now()
self.save()
return True
except:
except (ConnectionError, json.decoder.JSONDecodeError, simplejson.JSONDecodeError):
return False
def api_client(self):
@ -91,10 +91,12 @@ class ProcessingNode(models.Model):
except requests.exceptions.ConnectionError as e:
raise ProcessingException(e)
if result['uuid']:
if isinstance(result, dict) and 'uuid' in result:
return result['uuid']
elif result['error']:
elif isinstance(result, dict) and 'error' in result:
raise ProcessingException(result['error'])
else:
raise ProcessingException("Unexpected answer from server: {}".format(result))
@api
def get_task_info(self, uuid):
@ -196,5 +198,6 @@ def auto_update_node_info(sender, instance, created, **kwargs):
class ProcessingNodeUserObjectPermission(UserObjectPermissionBase):
content_object = models.ForeignKey(ProcessingNode)
class ProcessingNodeGroupObjectPermission(GroupObjectPermissionBase):
content_object = models.ForeignKey(ProcessingNode)
content_object = models.ForeignKey(ProcessingNode)

6
requirements.txt
Wyświetl plik

@ -1,8 +1,8 @@
anyjson==0.3.3
appdirs==1.4.0
APScheduler==3.2.0
coreapi==2.0.9
Django==1.10
django-common-helpers==0.8.0
Django==1.10.5
django-debug-toolbar==1.6
django-filter==0.15.3
django-guardian==1.4.6
@ -15,9 +15,11 @@ futures==3.0.5
itypes==1.1.0
Markdown==2.6.7
openapi-codec==1.1.7
packaging==16.8
Pillow==3.3.1
pip-autoremove==0.9.0
psycopg2==2.6.2
pyparsing==2.1.10
pytz==2016.6.1
requests==2.11.1
rfc3987==1.3.7

2
webodm/settings.py
Wyświetl plik

@ -224,6 +224,8 @@ REST_FRAMEWORK = {
}
TESTING = sys.argv[1:2] == ['test']
if TESTING:
MEDIA_ROOT = os.path.join(BASE_DIR, 'app', 'media_test')
try:
from .local_settings import *