diff --git a/Dockerfile b/Dockerfile
index d827fbe9..4ca48d4a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -10,13 +10,8 @@ WORKDIR /webodm
# Install pip reqs
ADD requirements.txt /webodm/
-RUN pip install --upgrade git+https://github.com/pierotofy/django-knockout
RUN pip install -r requirements.txt
-# swagger_spec_validator is not up to date, fetch directly from github
-# also install django-knockout
-RUN pip install --upgrade git+https://github.com/Yelp/swagger_spec_validator
-
ADD . /webodm/
RUN git submodule init
diff --git a/README.md b/README.md
index f62a9066..6718f39a 100644
--- a/README.md
+++ b/README.md
@@ -62,7 +62,6 @@ Then:
```
pip install -r requirements.txt
-pip install --upgrade git+git://github.com/Yelp/swagger_spec_validator
npm install -g webpack
npm install
webpack
diff --git a/app/api/processingnodes.py b/app/api/processingnodes.py
index 2a956ecb..6e53e9f5 100644
--- a/app/api/processingnodes.py
+++ b/app/api/processingnodes.py
@@ -3,12 +3,35 @@ from rest_framework.response import Response
from rest_framework.decorators import permission_classes
from rest_framework.permissions import DjangoModelPermissions
from rest_framework.filters import DjangoFilterBackend
+from django_filters.rest_framework import FilterSet
from nodeodm.models import ProcessingNode
+import django_filters
+from django.utils import timezone
+from datetime import timedelta
+from django.db.models import Q
+
class ProcessingNodeSerializer(serializers.ModelSerializer):
class Meta:
model = ProcessingNode
+ fields = '__all__'
+class ProcessingNodeFilter(FilterSet):
+ online = django_filters.MethodFilter()
+
+ def filter_online(self, queryset, value):
+ online_threshold = timezone.now() - timedelta(minutes=5)
+
+ if value.lower() in ['true', '1']:
+ return queryset.filter(last_refreshed__isnull=False, last_refreshed__gte=online_threshold)
+ elif value.lower() in ['false', '0']:
+ return queryset.filter(Q(last_refreshed__isnull=True) | Q(last_refreshed__lt=online_threshold))
+
+ return queryset
+
+ class Meta:
+ model = ProcessingNode
+ fields = ['online', 'id', 'hostname', 'port', 'api_version', 'queue_count', ]
class ProcessingNodeViewSet(viewsets.ModelViewSet):
"""
@@ -18,7 +41,10 @@ class ProcessingNodeViewSet(viewsets.ModelViewSet):
# Don't need a "view node" permission. If you are logged-in, you can view nodes.
permission_classes = (DjangoModelPermissions, )
+
filter_backends = (DjangoFilterBackend, )
+ filter_class = ProcessingNodeFilter
+
pagination_class = None
serializer_class = ProcessingNodeSerializer
- queryset = ProcessingNode.objects.all()
+ queryset = ProcessingNode.objects.all()
\ No newline at end of file
diff --git a/app/api/projects.py b/app/api/projects.py
index d2e54077..20ceb38c 100644
--- a/app/api/projects.py
+++ b/app/api/projects.py
@@ -10,6 +10,7 @@ class ProjectSerializer(serializers.ModelSerializer):
class Meta:
model = models.Project
+ fields = '__all__'
class ProjectViewSet(viewsets.ModelViewSet):
diff --git a/app/api/tasks.py b/app/api/tasks.py
index 1a936af1..dedaeaea 100644
--- a/app/api/tasks.py
+++ b/app/api/tasks.py
@@ -3,7 +3,7 @@ from django.core.exceptions import ObjectDoesNotExist
from rest_framework import status, serializers, viewsets, filters, exceptions, permissions, parsers
from rest_framework.response import Response
from rest_framework.decorators import parser_classes, api_view
-from app import models
+from app import models, scheduler
from nodeodm.models import ProcessingNode
class TaskIDsSerializer(serializers.BaseSerializer):
@@ -16,7 +16,7 @@ class TaskSerializer(serializers.ModelSerializer):
class Meta:
model = models.Task
-
+ fields = '__all__'
class TaskViewSet(viewsets.ViewSet):
"""
@@ -83,6 +83,10 @@ class TaskViewSet(viewsets.ViewSet):
serializer = TaskSerializer(task, data=request.data, partial=partial)
serializer.is_valid(raise_exception=True)
serializer.save()
+
+ # Call the scheduler (speed things up)
+ scheduler.process_pending_tasks(background=True)
+
return Response(serializer.data)
def partial_update(self, request, *args, **kwargs):
diff --git a/app/apps.py b/app/apps.py
index 1d26e888..25a43c7e 100644
--- a/app/apps.py
+++ b/app/apps.py
@@ -1,11 +1,7 @@
from __future__ import unicode_literals
from django.apps import AppConfig
-from .boot import boot
class MainConfig(AppConfig):
name = 'app'
verbose_name = 'Application'
-
- def ready(self):
- boot()
\ No newline at end of file
diff --git a/app/boot.py b/app/boot.py
index 721a6f8a..5c765265 100644
--- a/app/boot.py
+++ b/app/boot.py
@@ -1,11 +1,13 @@
-def boot():
- from django.contrib.contenttypes.models import ContentType
- from django.contrib.auth.models import Permission
- from django.contrib.auth.models import User, Group
- from django.db.utils import ProgrammingError
- from . import signals
- import logging
+from django.contrib.contenttypes.models import ContentType
+from django.contrib.auth.models import Permission
+from django.contrib.auth.models import User, Group
+from django.db.utils import ProgrammingError
+from . import signals, scheduler
+import logging, os
+from .models import Task
+from webodm import settings
+def boot():
logger = logging.getLogger('app.logger')
# Check default group
@@ -24,5 +26,15 @@ def boot():
if User.objects.filter(is_superuser=True).count() == 0:
User.objects.create_superuser('admin', 'admin@example.com', 'admin')
logger.info("Created superuser")
+
+ # Unlock any Task that might have been locked
+ Task.objects.filter(processing_lock=True).update(processing_lock=False)
+
+ if not settings.TESTING:
+ # Setup and start scheduler
+ scheduler.setup()
+
+ scheduler.update_nodes_info(background=True)
+
except ProgrammingError:
- logger.warn("Could not create default group/user. If running a migration, this is expected.")
+ logger.warn("Could not touch the database. If running a migration, this is expected.")
\ No newline at end of file
diff --git a/app/models.py b/app/models.py
index 3fe8c097..0cdc8a46 100644
--- a/app/models.py
+++ b/app/models.py
@@ -6,10 +6,12 @@ from django.utils import timezone
from django.contrib.auth.models import User
from django.contrib.postgres import fields
from nodeodm.models import ProcessingNode
-from django.dispatch import receiver
from guardian.shortcuts import get_perms_for_model, assign_perm
from guardian.models import UserObjectPermissionBase
from guardian.models import GroupObjectPermissionBase
+from django.core.exceptions import ValidationError
+from django.dispatch import receiver
+from nodeodm.exceptions import ProcessingException
from django.db import transaction
def assets_directory_path(taskId, projectId, filename):
@@ -55,6 +57,19 @@ class ProjectGroupObjectPermission(GroupObjectPermissionBase):
def gcp_directory_path(task, filename):
return assets_directory_path(task.id, task.project.id, filename)
+def validate_task_options(value):
+ """
+ Make sure that the format of this options field is valid
+ """
+ if len(value) == 0: return
+
+ try:
+ for option in value:
+ if not option['name']: raise ValidationError("Name key not found in option")
+ if not option['value']: raise ValidationError("Value key not found in option")
+ except:
+ raise ValidationError("Invalid options")
+
class Task(models.Model):
STATUS_CODES = (
(10, 'QUEUED'),
@@ -64,15 +79,18 @@ class Task(models.Model):
(50, 'CANCELED')
)
- uuid = models.CharField(max_length=255, null=True, blank=True, help_text="Identifier of the task (as returned by OpenDroneMap's REST API)")
+ uuid = models.CharField(max_length=255, db_index=True, null=True, blank=True, help_text="Identifier of the task (as returned by OpenDroneMap's REST API)")
project = models.ForeignKey(Project, on_delete=models.CASCADE, help_text="Project that this task belongs to")
name = models.CharField(max_length=255, null=True, blank=True, help_text="A label for the task")
+ processing_lock = models.BooleanField(default=False, help_text="A flag indicating whether this task is currently locked for processing. When this flag is turned on, the task is in the middle of a processing step.")
processing_time = models.IntegerField(default=-1, help_text="Number of milliseconds that elapsed since the beginning of this task (-1 indicates that no information is available)")
processing_node = models.ForeignKey(ProcessingNode, null=True, blank=True, help_text="Processing node assigned to this task (or null if this task has not been associated yet)")
- status = models.IntegerField(choices=STATUS_CODES, null=True, blank=True, help_text="Current status of the task")
- options = fields.JSONField(default=dict(), blank=True, help_text="Options that are being used to process this task")
+ status = models.IntegerField(choices=STATUS_CODES, db_index=True, null=True, blank=True, help_text="Current status of the task")
+ last_error = models.TextField(null=True, blank=True, help_text="The last processing error received")
+ options = fields.JSONField(default=dict(), blank=True, help_text="Options that are being used to process this task", validators=[validate_task_options])
console_output = models.TextField(null=True, blank=True, help_text="Console output of the OpenDroneMap's process")
ground_control_points = models.FileField(null=True, blank=True, upload_to=gcp_directory_path, help_text="Optional Ground Control Points file to use for processing")
+
# georeferenced_model
# orthophoto
# textured_model
@@ -82,6 +100,11 @@ class Task(models.Model):
def __str__(self):
return 'Task ID: {}'.format(self.id)
+ def save(self, *args, **kwargs):
+ # Autovalidate on save
+ self.full_clean()
+ super(Task, self).save(*args, **kwargs)
+
@staticmethod
def create_from_images(images, project):
'''
@@ -100,6 +123,58 @@ class Task(models.Model):
# In case of error
return None
+ def process(self):
+ # Nothing to do if we don't have a processing node...
+ if not self.processing_node: return
+
+ # Need to process some images (UUID not yet set)?
+ if not self.uuid:
+ print("Processing... {}".format(self))
+
+ images = [image.path() for image in self.imageupload_set.all()]
+
+ try:
+ self.uuid = self.processing_node.process_new_task(images, self.name, self.options)
+ self.save()
+
+ # TODO: log process has started processing
+
+ except ProcessingException, e:
+ print("TASK ERROR: " + e.message)
+
+ # Need to update status (first time, queued or running?)
+ if self.uuid and self.status in [None, 10, 20]:
+ print("Have UUID: {}".format(self.uuid))
+
+ # Update task info from processing node
+ try:
+ info = self.processing_node.get_task_info(self.uuid)
+
+ self.processing_time = info["processingTime"]
+ self.status = info["status"]["code"]
+
+ if "errorMessage" in info["status"]:
+ self.last_error = info["status"]["errorMessage"]
+
+ # Has the task just been canceled, failed, or completed?
+ # Note that we don't save the status code right away,
+ # if the assets retrieval fails we want to retry again.
+ if self.status in [30, 40, 50]:
+ print("ALMOST DONE: " + str(self.status))
+
+ # Completed?
+ if self.status == 40:
+ # TODO: retrieve assets
+ pass
+ else:
+ self.save()
+ else:
+ # Still waiting...
+ self.save()
+ except ProcessingException, e:
+ print("TASK ERROR 2: " + e.message)
+
+
class Meta:
permissions = (
('view_task', 'Can view task'),
@@ -115,3 +190,6 @@ class ImageUpload(models.Model):
def __str__(self):
return self.image.name
+
+ def path(self):
+ return self.image.path
diff --git a/app/scheduler.py b/app/scheduler.py
new file mode 100644
index 00000000..06acb337
--- /dev/null
+++ b/app/scheduler.py
@@ -0,0 +1,81 @@
+import logging
+from apscheduler.schedulers.background import BackgroundScheduler
+from apscheduler.schedulers import SchedulerAlreadyRunningError, SchedulerNotRunningError
+from threading import Thread, Lock
+from multiprocessing.dummy import Pool as ThreadPool
+from nodeodm.models import ProcessingNode
+from app.models import Task
+from django.db.models import Q
+import random
+
+logger = logging.getLogger('app.logger')
+scheduler = BackgroundScheduler()
+
+def job(func):
+ """
+ Adds background={True|False} param to any function
+ so that we can call update_nodes_info(background=True) from the outside
+ """
+ def wrapper(*args,**kwargs):
+ background = kwargs.get('background', False)
+ if background:
+ t = Thread(target=func)
+ t.start()
+ return t
+ else:
+ return func(*args, **kwargs)
+ return wrapper
+
+
+@job
+def update_nodes_info():
+ processing_nodes = ProcessingNode.objects.all()
+ for processing_node in processing_nodes:
+ processing_node.update_node_info()
+
+
+tasks_mutex = Lock()
+
+@job
+def process_pending_tasks():
+ tasks = []
+ try:
+ tasks_mutex.acquire()
+
+ # All tasks that have a processing node assigned
+ # but don't have a UUID
+ # and that are not locked (being processed by another thread)
+ tasks = Task.objects.filter(Q(uuid=None) | Q(status=10) | Q(status=20)).exclude(Q(processing_node=None) | Q(processing_lock=True))
+ for task in tasks:
+ logger.info("Acquiring lock: {}".format(task))
+ task.processing_lock = True
+ task.save()
+ finally:
+ tasks_mutex.release()
+
+ def process(task):
+ task.process()
+ task.processing_lock = False
+ task.save()
+
+ if tasks.count() > 0:
+ pool = ThreadPool(tasks.count())
+ pool.map(process, tasks, chunksize=1)
+ pool.close()
+ pool.join()
+
+def setup():
+ logger.info("Starting background scheduler...")
+ try:
+ scheduler.start()
+ scheduler.add_job(update_nodes_info, 'interval', seconds=30)
+ scheduler.add_job(process_pending_tasks, 'interval', seconds=5)
+ except SchedulerAlreadyRunningError:
+ logger.warn("Scheduler already running (this is OK while testing)")
+
+def teardown():
+ logger.info("Stopping scheduler...")
+ try:
+ scheduler.shutdown()
+ except SchedulerNotRunningError:
+ logger.warn("Scheduler not running")
diff --git a/app/static/app/js/components/EditTaskPanel.jsx b/app/static/app/js/components/EditTaskPanel.jsx
index 0c8dadf0..9b9e2893 100644
--- a/app/static/app/js/components/EditTaskPanel.jsx
+++ b/app/static/app/js/components/EditTaskPanel.jsx
@@ -10,6 +10,7 @@ class EditTaskPanel extends React.Component {
this.state = {
name: "",
+ error: "",
advancedOptions: false,
loadedProcessingNodes: false,
selectedNode: null,
@@ -36,8 +37,14 @@ class EditTaskPanel extends React.Component {
setTimeout(loadProcessingNodes, 1000);
}
- this.nodesRequest = $.getJSON("/api/processingnodes/", json => {
+ this.nodesRequest =
+ $.getJSON("/api/processingnodes/?online=True", json => {
if (Array.isArray(json)){
+ // All nodes offline?
+ if (json.length === 0){
+ this.setState({error: "There are no processing nodes online. Make sure at least one of them is reachable."});
+ return;
+ }
let nodes = json.map(node => {
return {
@@ -138,6 +145,12 @@ class EditTaskPanel extends React.Component {
}
render() {
+ if (this.state.error){
+ return (
+ {this.state.error}
+
);
+ }
+
if (this.state.editing){
let processingNodesOptions = "";
if (this.state.loadedProcessingNodes){
diff --git a/app/static/app/js/components/ProjectList.jsx b/app/static/app/js/components/ProjectList.jsx
index c6bec728..85fe6aba 100644
--- a/app/static/app/js/components/ProjectList.jsx
+++ b/app/static/app/js/components/ProjectList.jsx
@@ -16,7 +16,8 @@ class ProjectList extends React.Component {
componentDidMount(){
// Load projects from API
- this.serverRequest = $.getJSON(this.props.source, json => {
+ this.serverRequest =
+ $.getJSON(this.props.source, json => {
if (json.results){
this.setState({
projects: json.results,
diff --git a/app/static/app/js/components/ProjectListItem.jsx b/app/static/app/js/components/ProjectListItem.jsx
index 07040959..c602fe5e 100644
--- a/app/static/app/js/components/ProjectListItem.jsx
+++ b/app/static/app/js/components/ProjectListItem.jsx
@@ -1,7 +1,7 @@
import '../css/ProjectListItem.scss';
import React from 'react';
import update from 'react-addons-update';
-import ProjectListItemPanel from './ProjectListItemPanel';
+import TaskList from './TaskList';
import EditTaskPanel from './EditTaskPanel';
import UploadProgressBar from './UploadProgressBar';
import Dropzone from '../vendor/dropzone';
@@ -13,12 +13,12 @@ class ProjectListItem extends React.Component {
super(props);
this.state = {
- showPanel: false,
+ showTaskList: false,
updatingTask: false,
upload: this.getDefaultUploadState()
};
- this.togglePanel = this.togglePanel.bind(this);
+ this.toggleTaskList = this.toggleTaskList.bind(this);
this.handleUpload = this.handleUpload.bind(this);
this.closeUploadError = this.closeUploadError.bind(this);
this.cancelUpload = this.cancelUpload.bind(this);
@@ -133,7 +133,8 @@ class ProjectListItem extends React.Component {
this.setUploadState({showEditTask: false});
this.setState({updatingTask: true});
- this.updateTaskRequest = $.ajax({
+ this.updateTaskRequest =
+ $.ajax({
url: `/api/projects/${this.props.data.id}/tasks/${this.state.upload.taskId}/`,
contentType: 'application/json',
data: JSON.stringify({
@@ -144,10 +145,10 @@ class ProjectListItem extends React.Component {
dataType: 'json',
type: 'PATCH'
}).done(() => {
- if (this.state.showPanel){
- this.projectListItemPanel.refresh();
+ if (this.state.showTaskList){
+ this.taskList.refresh();
}else{
- this.setState({showPanel: true});
+ this.setState({showTaskList: true});
}
}).fail(() => {
this.setUploadState({error: "Could not update task information. Plese try again."});
@@ -162,9 +163,9 @@ class ProjectListItem extends React.Component {
}
}
- togglePanel(){
+ toggleTaskList(){
this.setState({
- showPanel: !this.state.showPanel
+ showTaskList: !this.state.showTaskList
});
}
@@ -222,10 +223,18 @@ class ProjectListItem extends React.Component {
-
-
+
{this.props.data.name}
-
+
+
+ {this.props.data.description}
+
+
@@ -233,7 +242,7 @@ class ProjectListItem extends React.Component {
- {this.state.showPanel ? : ""}
+ {this.state.showTaskList ? : ""}
{this.state.upload.showEditTask ? : ""}
diff --git a/app/static/app/js/components/ProjectListItemPanel.jsx b/app/static/app/js/components/ProjectListItemPanel.jsx
deleted file mode 100644
index 42d611f1..00000000
--- a/app/static/app/js/components/ProjectListItemPanel.jsx
+++ /dev/null
@@ -1,25 +0,0 @@
-import React from 'react';
-
-class ProjectListItemPanel extends React.Component {
- constructor(){
- super();
- }
-
- componentDidMount(){
- console.log("DISPLAY");
- }
-
- refresh(){
- console.log("REFRESH");
- }
-
- render() {
- return (
-
- TODO
-
- );
- }
-}
-
-export default ProjectListItemPanel;
diff --git a/app/static/app/js/components/TaskList.jsx b/app/static/app/js/components/TaskList.jsx
new file mode 100644
index 00000000..6cae4e70
--- /dev/null
+++ b/app/static/app/js/components/TaskList.jsx
@@ -0,0 +1,26 @@
+import React from 'react';
+import '../css/TaskList.scss';
+
+class TaskList extends React.Component {
+ constructor(){
+ super();
+ }
+
+ componentDidMount(){
+ console.log("DISPLAY");
+ }
+
+ refresh(){
+ console.log("REFRESH");
+ }
+
+ render() {
+ return (
+
+ Updating task list...
+
+ );
+ }
+}
+
+export default TaskList;
diff --git a/app/static/app/js/css/Dashboard.scss b/app/static/app/js/css/Dashboard.scss
index b4797e4a..8aa2d3a0 100644
--- a/app/static/app/js/css/Dashboard.scss
+++ b/app/static/app/js/css/Dashboard.scss
@@ -1,8 +1,4 @@
#dashboard-app{
- .project-list-item-panel{
- min-height: 100px;
- }
-
.row{
&.no-margin{
margin: 0;
diff --git a/app/static/app/js/css/ProjectListItem.scss b/app/static/app/js/css/ProjectListItem.scss
index d9eb7b35..848abf56 100644
--- a/app/static/app/js/css/ProjectListItem.scss
+++ b/app/static/app/js/css/ProjectListItem.scss
@@ -1,5 +1,16 @@
.project-list-item{
min-height: 60px;
+
+ .project-name{
+ font-weight: bold;
+ }
+
+ .project-links{
+ font-size: 90%;
+ i{
+ margin-right: 4px;
+ }
+ }
.dz-preview{
display: none;
diff --git a/app/static/app/js/css/TaskList.scss b/app/static/app/js/css/TaskList.scss
new file mode 100644
index 00000000..e8bc078a
--- /dev/null
+++ b/app/static/app/js/css/TaskList.scss
@@ -0,0 +1,5 @@
+.task-list{
+ background-color: #ecf0f1;
+ padding: 10px;
+ min-height: 100px;
+}
\ No newline at end of file
diff --git a/app/tests/classes.py b/app/tests/classes.py
index 3f89e63e..981c3275 100644
--- a/app/tests/classes.py
+++ b/app/tests/classes.py
@@ -10,7 +10,7 @@ class BootTestCase(TestCase):
module should derive from this class instead of TestCase.
We don't use fixtures because we have signal initialization login
- for some models, which doesn't play well with them, and this: http://blog.namis.me/2012/04/21/burn-your-fixtures/
+ for some models, which doesn't play well with them.
'''
@classmethod
def setUpClass(cls):
@@ -46,3 +46,7 @@ class BootTestCase(TestCase):
boot()
setupUsers()
setupProjects()
+
+ @classmethod
+ def tearDownClass(cls):
+ super(BootTestCase, cls).tearDownClass()
diff --git a/app/tests/test_api.py b/app/tests/test_api.py
index 27d4b1d3..b65e1d9e 100644
--- a/app/tests/test_api.py
+++ b/app/tests/test_api.py
@@ -137,11 +137,26 @@ class TestApi(BootTestCase):
self.assertTrue(len(res.data) == 1)
self.assertTrue(res.data[0]["hostname"] == "localhost")
+ # Can use filters
+ res = client.get('/api/processingnodes/?id={}'.format(pnode.id))
+ self.assertEqual(res.status_code, status.HTTP_200_OK)
+ self.assertTrue(len(res.data) == 1)
+
+ # Can filter online
+ res = client.get('/api/processingnodes/?online=true')
+ self.assertEqual(res.status_code, status.HTTP_200_OK)
+ self.assertTrue(len(res.data) == 0)
+
+ res = client.get('/api/processingnodes/?online=false')
+ self.assertEqual(res.status_code, status.HTTP_200_OK)
+ self.assertTrue(len(res.data) == 1)
+
# Can get single processing node as normal user
res = client.get('/api/processingnodes/{}/'.format(pnode.id))
self.assertEqual(res.status_code, status.HTTP_200_OK)
self.assertTrue(res.data["hostname"] == "localhost")
+
# Cannot delete a processing node as normal user
res = client.delete('/api/processingnodes/{}/'.format(pnode.id))
self.assertTrue(res.status_code, status.HTTP_403_FORBIDDEN)
diff --git a/app/tests/test_app.py b/app/tests/test_app.py
index cc4fc3ae..55af1773 100644
--- a/app/tests/test_app.py
+++ b/app/tests/test_app.py
@@ -4,6 +4,8 @@ from django.test import Client
from app.models import Project, Task
from .classes import BootTestCase
+from app import scheduler
+from django.core.exceptions import ValidationError
class TestApp(BootTestCase):
fixtures = ['test_processingnodes', ]
@@ -119,3 +121,30 @@ class TestApp(BootTestCase):
# Should not have permission
self.assertFalse(anotherUser.has_perm("delete_project", p))
+
+ def test_tasks(self):
+ # Create a new task
+ p = Project.objects.create(owner=User.objects.get(username="testuser"), name="test")
+ task = Task.objects.create(project=p)
+
+ # Test options validation
+ task.options = [{'name': 'test', 'value': 1}]
+ self.assertTrue(task.save() == None)
+
+ task.options = {'test': 1}
+ self.assertRaises(ValidationError, task.save)
+
+ task.options = [{'name': 'test', 'value': 1}, {"invalid": 1}]
+ self.assertRaises(ValidationError, task.save)
+
+
+ def test_scheduler(self):
+ self.assertTrue(scheduler.setup() == None)
+
+ # Can call update_nodes_info()
+ self.assertTrue(scheduler.update_nodes_info() == None)
+
+ # Can call function in background
+ self.assertTrue(scheduler.update_nodes_info(background=True).join() == None)
+
+ self.assertTrue(scheduler.teardown() == None)
diff --git a/app/urls.py b/app/urls.py
index 52daa503..822d0953 100644
--- a/app/urls.py
+++ b/app/urls.py
@@ -1,5 +1,7 @@
from django.conf.urls import url, include
from . import views
+from app.boot import boot
+from webodm import settings
urlpatterns = [
url(r'^$', views.index, name='index'),
@@ -7,4 +9,8 @@ urlpatterns = [
url(r'^processingnode/([\d]+)/$', views.processing_node, name='processing_node'),
url(r'^api/', include("app.api.urls")),
-]
\ No newline at end of file
+]
+
+# Test cases call boot() independently
+if not settings.TESTING:
+ boot()
\ No newline at end of file
diff --git a/nodeodm/api_client.py b/nodeodm/api_client.py
index 983443f5..13d0f926 100644
--- a/nodeodm/api_client.py
+++ b/nodeodm/api_client.py
@@ -1,35 +1,45 @@
"""
-A wrapper around Bravado to communicate with a node-OpenDroneMap node.
+An interface to node-OpenDroneMap's API
+https://github.com/pierotofy/node-OpenDroneMap/blob/master/docs/index.adoc
"""
-from bravado.client import SwaggerClient
-from bravado.exception import HTTPError
-from requests import ConnectionError
+import requests
+import mimetypes
+import json
+import os
+from urlparse import urlunparse
class ApiClient:
- def check_client(func):
- def check(self, *args, **kwargs):
- """
- Makes sure that the client has been instantiated.
- Sometimes this will fail (rest endpoint might be offline),
- so we need to handle it gracefully...
- """
- if not hasattr(self, 'client'):
- try:
- self.client = SwaggerClient.from_url('http://{}:{}/swagger.json'.format(self.host, self.port))
- except (ConnectionError, HTTPError) as err:
- return None
-
- return func(self, *args, **kwargs)
- return check
-
def __init__(self, host, port):
self.host = host
self.port = port
- @check_client
- def info(self):
- return self.client.server.get_info().result()
+ def url(self, url):
+ netloc = self.host if self.port == 80 else "{}:{}".format(self.host, self.port)
+
+ # TODO: https support
+ return urlunparse(('http', netloc, url, '', '', ''))
+
+ def info(self):
+ return requests.get(self.url('/info')).json()
- @check_client
def options(self):
- return self.client.server.get_options().result()
\ No newline at end of file
+ return requests.get(self.url('/options')).json()
+
+ def task_info(self, uuid):
+ return requests.get(self.url('/task/{}/info').format(uuid)).json()
+
+ def new_task(self, images, name=None, options=[]):
+ """
+ Starts processing of a new task
+ :param images: list of path images
+ :param name: name of the task
+ :param options: options to be used for processing ([{'name': optionName, 'value': optionValue}, ...])
+ :return: UUID or error
+ """
+
+ files = [('images',
+ (os.path.basename(image), open(image, 'rb'), (mimetypes.guess_type(image)[0] or "image/jpg"))
+ ) for image in images]
+ return requests.post(self.url("/task/new"),
+ files=files,
+ data={'name': name, 'options': json.dumps(options)}).json()
\ No newline at end of file
diff --git a/nodeodm/exceptions.py b/nodeodm/exceptions.py
new file mode 100644
index 00000000..77c36115
--- /dev/null
+++ b/nodeodm/exceptions.py
@@ -0,0 +1,2 @@
+class ProcessingException(Exception):
+ pass
diff --git a/nodeodm/fixtures/test_images/DJI_0176.JPG b/nodeodm/fixtures/test_images/DJI_0176.JPG
new file mode 100644
index 00000000..ea3b9932
Binary files /dev/null and b/nodeodm/fixtures/test_images/DJI_0176.JPG differ
diff --git a/nodeodm/fixtures/test_images/DJI_0177.JPG b/nodeodm/fixtures/test_images/DJI_0177.JPG
new file mode 100644
index 00000000..474585a1
Binary files /dev/null and b/nodeodm/fixtures/test_images/DJI_0177.JPG differ
diff --git a/nodeodm/fixtures/test_images/DJI_0178.JPG b/nodeodm/fixtures/test_images/DJI_0178.JPG
new file mode 100644
index 00000000..9ec037a5
Binary files /dev/null and b/nodeodm/fixtures/test_images/DJI_0178.JPG differ
diff --git a/nodeodm/fixtures/test_images/DJI_0179.JPG b/nodeodm/fixtures/test_images/DJI_0179.JPG
new file mode 100644
index 00000000..3aaa0f6d
Binary files /dev/null and b/nodeodm/fixtures/test_images/DJI_0179.JPG differ
diff --git a/nodeodm/fixtures/test_images/DJI_0180.JPG b/nodeodm/fixtures/test_images/DJI_0180.JPG
new file mode 100644
index 00000000..0e0e0446
Binary files /dev/null and b/nodeodm/fixtures/test_images/DJI_0180.JPG differ
diff --git a/nodeodm/models.py b/nodeodm/models.py
index 50c7ae38..d5e263a3 100644
--- a/nodeodm/models.py
+++ b/nodeodm/models.py
@@ -3,8 +3,12 @@ from __future__ import unicode_literals
from django.db import models
from django.contrib.postgres import fields
from django.utils import timezone
+from django.dispatch import receiver
from .api_client import ApiClient
import json
+from django.db.models import signals
+from requests.exceptions import ConnectionError
+from .exceptions import ProcessingException
class ProcessingNode(models.Model):
hostname = models.CharField(max_length=255, help_text="Hostname where the node is located (can be an internal hostname as well)")
@@ -14,12 +18,6 @@ class ProcessingNode(models.Model):
queue_count = models.PositiveIntegerField(default=0, help_text="Number of tasks currently being processed by this node (as reported by the node itself)")
available_options = fields.JSONField(default=dict(), help_text="Description of the options that can be used for processing")
- def __init__(self, *args, **kwargs):
- super(ProcessingNode, self).__init__(*args, **kwargs)
-
- # Initialize api client
- self.api_client = ApiClient(self.hostname, self.port)
-
def __str__(self):
return '{}:{}'.format(self.hostname, self.port)
@@ -30,22 +28,64 @@ class ProcessingNode(models.Model):
:returns: True if information could be updated, False otherwise
"""
- info = self.api_client.info()
- if info != None:
+ api_client = self.api_client()
+ try:
+ info = api_client.info()
self.api_version = info['version']
self.queue_count = info['taskQueueCount']
- options = self.api_client.options()
- if options != None:
- self.available_options = options
- self.last_refreshed = timezone.now()
+ options = api_client.options()
+ self.available_options = options
+ self.last_refreshed = timezone.now()
+ self.save()
+ return True
+ except ConnectionError:
+ return False
- self.save()
- return True
- return False
+ def api_client(self):
+ return ApiClient(self.hostname, self.port)
def get_available_options_json(self):
"""
:returns available options in JSON string format
"""
return json.dumps(self.available_options)
+
+ def process_new_task(self, images, name=None, options=[]):
+ """
+ Sends a set of images (and optional GCP file) via the API
+ to start processing.
+
+ :param images: list of path images
+ :param name: name of the task
+ :param options: options to be used for processing ([{'name': optionName, 'value': optionValue}, ...])
+
+ :returns UUID of the newly created task
+ """
+ if len(images) < 2: raise ProcessingException("Need at least 2 images")
+
+ api_client = self.api_client()
+ result = api_client.new_task(images, name, options)
+ if result['uuid']:
+ return result['uuid']
+ elif result['error']:
+ raise ProcessingException(result['error'])
+
+ def get_task_info(self, uuid):
+ """
+ Gets information about this task, such as name, creation date,
+ processing time, status, command line options and number of
+ images being processed.
+ """
+ api_client = self.api_client()
+ result = api_client.task_info(uuid)
+ if result['uuid']:
+ return result
+ elif result['error']:
+ raise ProcessingException(result['error'])
+
+# First time a processing node is created, automatically try to update
+@receiver(signals.post_save, sender=ProcessingNode, dispatch_uid="update_processing_node_info")
+def auto_update_node_info(sender, instance, created, **kwargs):
+ if created:
+ instance.update_node_info()
diff --git a/nodeodm/tests.py b/nodeodm/tests.py
index 46e1bd77..39e1288a 100644
--- a/nodeodm/tests.py
+++ b/nodeodm/tests.py
@@ -4,6 +4,7 @@ import subprocess, time
from os import path
from .models import ProcessingNode
from .api_client import ApiClient
+from requests.exceptions import ConnectionError
current_dir = path.dirname(path.realpath(__file__))
@@ -31,8 +32,8 @@ class TestClientApi(TestCase):
def test_offline_api(self):
api = ApiClient("offline-host", 3000)
- self.assertTrue(api.info() == None)
- self.assertTrue(api.options() == None)
+ self.assertRaises(ConnectionError, api.info)
+ self.assertRaises(ConnectionError, api.options)
def test_info(self):
info = self.api_client.info()
@@ -46,12 +47,12 @@ class TestClientApi(TestCase):
def test_online_processing_node(self):
online_node = ProcessingNode.objects.get(pk=1)
self.assertTrue(str(online_node) == "localhost:11223", "Formatting string works")
- self.assertTrue(online_node.last_refreshed != 0, "Last refreshed not yet set")
+ self.assertTrue(online_node.last_refreshed == None, "Last refreshed not yet set")
self.assertTrue(len(online_node.available_options) == 0, "Available options not yet set")
self.assertTrue(online_node.api_version == "", "API version is not set")
self.assertTrue(online_node.update_node_info(), "Could update info")
- self.assertTrue(online_node.last_refreshed != 0, "Last refreshed is set")
+ self.assertTrue(online_node.last_refreshed != None, "Last refreshed is set")
self.assertTrue(len(online_node.available_options) > 0, "Available options are set")
self.assertTrue(online_node.api_version != "", "API version is set")
@@ -61,3 +62,28 @@ class TestClientApi(TestCase):
offline_node = ProcessingNode.objects.get(pk=2)
self.assertFalse(offline_node.update_node_info(), "Could not update info (offline)")
self.assertTrue(offline_node.api_version == "", "API version is not set")
+
+ def test_auto_update_node_info(self):
+ online_node = ProcessingNode.objects.create(hostname="localhost", port=11223)
+ self.assertTrue(online_node.last_refreshed != None, "Last refreshed info is here (update_node_info() was called)")
+
+ def test_client_api(self):
+ api = ApiClient("localhost", 11223)
+
+ # Can call info(), options()
+ self.assertTrue(type(api.info()['version']) in [str, unicode])
+ self.assertTrue(len(api.options()) > 0)
+
+ # Can call new_task()
+ import glob
+ res = api.new_task(
+ glob.glob("nodeodm/fixtures/test_images/*.JPG"),
+ "test",
+ [{'name': 'cmvs-maxImages', 'value': 5}])
+ uuid = res['uuid']
+ self.assertTrue(uuid != None)
+
+ # Can call task_info()
+ task_info = api.task_info(uuid)
+ self.assertTrue(isinstance(task_info['dateCreated'], (int, long)))
+ self.assertTrue(isinstance(task_info['uuid'], (str, unicode)))
diff --git a/requirements.txt b/requirements.txt
index 49f0de41..b0629648 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,39 +1,22 @@
anyjson==0.3.3
-attrs==16.2.0
-bravado==8.3.0
-bravado-core==4.5.0
-cffi==1.8.3
-crochet==1.5.0
-cryptography==1.5
+APScheduler==3.2.0
Django==1.10
django-common-helpers==0.8.0
-django-filter==0.15.2
+django-filter==0.15.3
django-guardian==1.4.6
django-webpack-loader==0.3.3
-djangorestframework==3.4.7
+djangorestframework==3.5.1
drf-nested-routers==0.11.1
-enum34==1.1.6
-fido==3.2.0
-functools32==3.2.3.post2
-idna==2.1
-ipaddress==1.0.17
-jsonschema==2.5.1
+funcsigs==1.0.2
+futures==3.0.5
Markdown==2.6.7
pillow==3.3.1
+pip-autoremove==0.9.0
psycopg2==2.6.2
-pyasn1==0.1.9
-pyasn1-modules==0.0.8
-pycparser==2.14
-pyOpenSSL==16.1.0
-python-dateutil==2.5.3
pytz==2016.6.1
-PyYAML==3.12
requests==2.11.1
-service-identity==16.0.0
-simplejson==3.8.2
+rfc3987==1.3.7
six==1.10.0
-swagger-spec-validator==2.0.2
-Twisted==16.4.1
-yelp-bytes==0.3.0
-yelp-encodings==0.1.3
-zope.interface==4.3.2
+strict-rfc3339==0.7
+tzlocal==1.3
+webcolors==1.5
diff --git a/webodm/settings.py b/webodm/settings.py
index c5f05ddc..c81b6a51 100644
--- a/webodm/settings.py
+++ b/webodm/settings.py
@@ -10,7 +10,7 @@ For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.10/ref/settings/
"""
-import os
+import os, sys
from django.contrib.messages import constants as messages
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@@ -181,6 +181,10 @@ LOGGING = {
'app.logger': {
'handlers': ['console'],
'level': 'INFO',
+ },
+ 'apscheduler.executors.default': {
+ 'handlers': ['console'],
+ 'level': 'WARN',
}
}
}
@@ -212,6 +216,8 @@ REST_FRAMEWORK = {
'PAGE_SIZE': 10,
}
+TESTING = sys.argv[1:2] == ['test']
+
try:
from .local_settings import *
except ImportError: