PostGIS, Python 3.5, raster datatype inclusion in task, db tests

.gitignore

@@ -91,3 +91,4 @@ ENV/
 node_modules/
 webpack-stats.json
 pip-selfcheck.json
+.idea/

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:2.7
+FROM python:3.5
 MAINTAINER Piero Toffanin <pt@masseranolabs.com>
 
 ENV PYTHONUNBUFFERED 1
@@ -18,7 +18,7 @@ RUN git submodule init
 RUN git submodule update
 
 # Install Node.js + npm requirements for testing node-OpenDroneMap and React
-RUN curl --silent --location https://deb.nodesource.com/setup_6.x | bash -
+RUN curl --silent --location https://deb.nodesource.com/setup_7.x | bash -
 RUN apt-get install -y nodejs
 
 WORKDIR /webodm/nodeodm/external/node-OpenDroneMap

README.md

@@ -2,7 +2,7 @@
 
 [![Build Status](https://travis-ci.org/OpenDroneMap/WebODM.svg?branch=master)](https://travis-ci.org/OpenDroneMap/WebODM)
 
-An open source solution for drone image processing. The long term vision includes a web interface, API and Mission Planner.
+A free, user-friendly application and API for drone image processing.
 
 ![Alt text](/screenshots/ui-mockup.png?raw=true "WebODM")
 
@@ -35,7 +35,8 @@ Linux users can connect to 127.0.0.1.
 
 If you want to run WebODM natively, you will need to install:
  * PostgreSQL (>= 9.5)
- * Python 2.7
+ * PostGIS 2.3
+ * Python 3.5
 
 Then these steps should be sufficient to get you up and running:
 
@@ -48,7 +49,7 @@ Create a `WebODM\webodm\local_settings.py` file containing:
 ```
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.postgresql',
+        'ENGINE': 'django.contrib.gis.db.backends.postgis',
         'NAME': 'webodm_dev',
         'USER': 'postgres',
         'PASSWORD': 'postgres',
@@ -68,15 +69,26 @@ webpack
 chmod +x start.sh && ./start.sh
 ```
 
+If you are getting a `rt_raster_gdal_warp: Could not create GDAL transformation object for output dataset creation`, make sure that your PostGIS installation has PROJ support:
+
+```
+SELECT PostGIS_Full_Version();
+```
+
+You may also need to set the environment variable PROJSO to the .so or .dll projection library your PostGIS is using. This just needs to have the name of the file. So for example on Windows, you would in Control Panel -> System -> Environment Variables add a system variable called PROJSO and set it to libproj.dll (if you are using proj 4.6.1). You'll have to restart your PostgreSQL service/daemon after this change. [http://postgis.net/docs/manual-2.0/RT_ST_Transform.html](http://postgis.net/docs/manual-2.0/RT_ST_Transform.html)
+
 ## Roadmap
 - [X] User Registration / Authentication
 - [X] UI mockup
-- [ ] Task Processing
+- [X] Task Processing
 - [ ] Model display (using Cesium/Leaflet) for both 2D and 3D outputs.
 - [X] Cluster management and setup.
 - [ ] Mission Planner
 - [X] API
 - [ ] Documentation
+- [ ] Android Mobile App
+- [ ] iOS Mobile App
+- [ ] Processing Nodes Volunteer Network
 - [X] Unit Testing
 
 ## Terminology

app/api/tasks.py

@@ -14,10 +14,10 @@ class TaskIDsSerializer(serializers.BaseSerializer):
 class TaskSerializer(serializers.ModelSerializer):
     project = serializers.PrimaryKeyRelatedField(queryset=models.Project.objects.all())
     processing_node = serializers.PrimaryKeyRelatedField(queryset=ProcessingNode.objects.all()) 
-    images_count = serializers.IntegerField(
-            source='imageupload_set.count',
-            read_only=True
-        )
+    images_count = serializers.SerializerMethodField()
+
+    def get_images_count(self, obj):
+        return obj.imageupload_set.count()
 
     class Meta:
         model = models.Task

app/apps.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from django.apps import AppConfig
 
 class MainConfig(AppConfig):

app/models.py

@@ -1,8 +1,6 @@
-from __future__ import unicode_literals
-
 import time, os
-import traceback
 
+from django.contrib.gis.gdal import GDALRaster
 from django.db import models
 from django.db.models import signals
 from django.contrib.gis.db import models as gismodels
@@ -154,143 +152,144 @@ class Task(models.Model):
         ready to be processed execute some logic. This could be communication
         with a processing node or executing a pending action.
         """
-        try:
-            if self.processing_node:
-                # Need to process some images (UUID not yet set)?
-                if not self.uuid:
-                    logger.info("Processing... {}".format(self))
 
-                    images = [image.path() for image in self.imageupload_set.all()]
+        if self.processing_node:
+            # Need to process some images (UUID not yet set)?
+            if not self.uuid:
+                logger.info("Processing... {}".format(self))
 
-                    try:
-                        # This takes a while
-                        uuid = self.processing_node.process_new_task(images, self.name, self.options)
+                images = [image.path() for image in self.imageupload_set.all()]
 
-                        # Refresh task object before committing change
-                        self.refresh_from_db()
-                        self.uuid = uuid
-                        self.save()
-
-                        # TODO: log process has started processing
-
-                    except ProcessingException as e:
-                        self.set_failure(e.message)
-
-
-            if self.pending_action is not None:
                 try:
-                    if self.pending_action == self.PendingActions.CANCEL:
-                        # Do we need to cancel the task on the processing node?
-                        logger.info("Canceling task {}".format(self))
-                        if self.processing_node and self.uuid:
-                            self.processing_node.cancel_task(self.uuid)
-                            self.pending_action = None
-                            self.save()
-                        else:
-                            raise ProcessingException("Cannot cancel a task that has no processing node or UUID")
+                    # This takes a while
+                    uuid = self.processing_node.process_new_task(images, self.name, self.options)
 
-                    elif self.pending_action == self.PendingActions.RESTART:
-                        logger.info("Restarting task {}".format(self))
-                        if self.processing_node and self.uuid:
-
-                            # Check if the UUID is still valid, as processing nodes purge
-                            # results after a set amount of time, the UUID might have eliminated.
-                            try:
-                                info = self.processing_node.get_task_info(self.uuid)
-                                uuid_still_exists = info['uuid'] == self.uuid
-                            except ProcessingException:
-                                uuid_still_exists = False
-
-                            if uuid_still_exists:
-                                # Good to go
-                                self.processing_node.restart_task(self.uuid)
-                            else:
-                                # Task has been purged (or processing node is offline)
-                                # TODO: what if processing node went offline?
-
-                                # Process this as a new task
-                                # Removing its UUID will cause the scheduler
-                                # to process this the next tick
-                                self.uuid = None
-
-                            self.console_output = ""
-                            self.processing_time = -1
-                            self.status = None
-                            self.last_error = None
-                            self.pending_action = None
-                            self.save()
-                        else:
-                            raise ProcessingException("Cannot restart a task that has no processing node or UUID")
-
-                    elif self.pending_action == self.PendingActions.REMOVE:
-                        logger.info("Removing task {}".format(self))
-                        if self.processing_node and self.uuid:
-                            # Attempt to delete the resources on the processing node
-                            # We don't care if this fails, as resources on processing nodes
-                            # Are expected to be purged on their own after a set amount of time anyway
-                            try:
-                                self.processing_node.remove_task(self.uuid)
-                            except ProcessingException:
-                                pass
-
-                        # What's more important is that we delete our task properly here
-                        self.delete()
-
-                        # Stop right here!
-                        return
-
-                except ProcessingException as e:
-                    self.last_error = e.message
+                    # Refresh task object before committing change
+                    self.refresh_from_db()
+                    self.uuid = uuid
                     self.save()
 
+                    # TODO: log process has started processing
 
-            if self.processing_node:
-                # Need to update status (first time, queued or running?)
-                if self.uuid and self.status in [None, status_codes.QUEUED, status_codes.RUNNING]:
-                    # Update task info from processing node
-                    try:
-                        info = self.processing_node.get_task_info(self.uuid)
+                except ProcessingException as e:
+                    self.set_failure(str(e))
 
-                        self.processing_time = info["processingTime"]
-                        self.status = info["status"]["code"]
 
-                        current_lines_count = len(self.console_output.split("\n")) - 1
-                        self.console_output += self.processing_node.get_task_console_output(self.uuid, current_lines_count)
+        if self.pending_action is not None:
+            try:
+                if self.pending_action == self.PendingActions.CANCEL:
+                    # Do we need to cancel the task on the processing node?
+                    logger.info("Canceling task {}".format(self))
+                    if self.processing_node and self.uuid:
+                        self.processing_node.cancel_task(self.uuid)
+                        self.pending_action = None
+                        self.save()
+                    else:
+                        raise ProcessingException("Cannot cancel a task that has no processing node or UUID")
 
-                        if "errorMessage" in info["status"]:
-                            self.last_error = info["status"]["errorMessage"]
+                elif self.pending_action == self.PendingActions.RESTART:
+                    logger.info("Restarting task {}".format(self))
+                    if self.processing_node and self.uuid:
 
-                        # Has the task just been canceled, failed, or completed?
-                        if self.status in [status_codes.FAILED, status_codes.COMPLETED, status_codes.CANCELED]:
-                            logger.info("Processing status: {} for {}".format(self.status, self))
+                        # Check if the UUID is still valid, as processing nodes purge
+                        # results after a set amount of time, the UUID might have eliminated.
+                        try:
+                            info = self.processing_node.get_task_info(self.uuid)
+                            uuid_still_exists = info['uuid'] == self.uuid
+                        except ProcessingException:
+                            uuid_still_exists = False
 
-                            if self.status == status_codes.COMPLETED:
-                                try:
-                                    orthophoto_stream = self.processing_node.download_task_asset(self.uuid, "orthophoto.tif")
-                                    orthophoto_filename = "orthophoto_{}.tif".format(int(time.time()))
-                                    orthophoto_path = os.path.join(settings.MEDIA_ROOT,
-                                                        assets_directory_path(self.id, self.project.id, orthophoto_filename))
-
-                                    # Save to disk
-                                    with open(orthophoto_path, 'wb') as fd:
-                                        for chunk in orthophoto_stream.iter_content(4096):
-                                            fd.write(chunk)
-
-                                    # Create raster layer
-                                    self.orthophoto = raster_models.RasterLayer.objects.create(rasterfile=orthophoto_path)
-                                    self.save()
-                                except ProcessingException as e:
-                                    self.set_failure(e.message)
-                            else:
-                                # FAILED, CANCELED
-                                self.save()
+                        if uuid_still_exists:
+                            # Good to go
+                            self.processing_node.restart_task(self.uuid)
                         else:
-                            # Still waiting...
+                            # Task has been purged (or processing node is offline)
+                            # TODO: what if processing node went offline?
+
+                            # Process this as a new task
+                            # Removing its UUID will cause the scheduler
+                            # to process this the next tick
+                            self.uuid = None
+
+                        self.console_output = ""
+                        self.processing_time = -1
+                        self.status = None
+                        self.last_error = None
+                        self.pending_action = None
+                        self.save()
+                    else:
+                        raise ProcessingException("Cannot restart a task that has no processing node or UUID")
+
+                elif self.pending_action == self.PendingActions.REMOVE:
+                    logger.info("Removing task {}".format(self))
+                    if self.processing_node and self.uuid:
+                        # Attempt to delete the resources on the processing node
+                        # We don't care if this fails, as resources on processing nodes
+                        # Are expected to be purged on their own after a set amount of time anyway
+                        try:
+                            self.processing_node.remove_task(self.uuid)
+                        except ProcessingException:
+                            pass
+
+                    # What's more important is that we delete our task properly here
+                    self.delete()
+
+                    # Stop right here!
+                    return
+
+            except ProcessingException as e:
+                self.last_error = str(e)
+                self.save()
+
+
+        if self.processing_node:
+            # Need to update status (first time, queued or running?)
+            if self.uuid and self.status in [None, status_codes.QUEUED, status_codes.RUNNING]:
+                # Update task info from processing node
+                try:
+                    info = self.processing_node.get_task_info(self.uuid)
+
+                    self.processing_time = info["processingTime"]
+                    self.status = info["status"]["code"]
+
+                    current_lines_count = len(self.console_output.split("\n")) - 1
+                    self.console_output += self.processing_node.get_task_console_output(self.uuid, current_lines_count)
+
+                    if "errorMessage" in info["status"]:
+                        self.last_error = info["status"]["errorMessage"]
+
+                    # Has the task just been canceled, failed, or completed?
+                    if self.status in [status_codes.FAILED, status_codes.COMPLETED, status_codes.CANCELED]:
+                        logger.info("Processing status: {} for {}".format(self.status, self))
+
+                        if self.status == status_codes.COMPLETED:
+                            try:
+                                orthophoto_stream = self.processing_node.download_task_asset(self.uuid, "orthophoto.tif")
+                                orthophoto_path = os.path.join(settings.MEDIA_ROOT,
+                                                                assets_directory_path(self.id, self.project.id, "orthophoto.tif"))
+
+                                # Save to disk original photo
+                                with open(orthophoto_path, 'wb') as fd:
+                                    for chunk in orthophoto_stream.iter_content(4096):
+                                        fd.write(chunk)
+
+                                # Add to database another copy
+                                self.orthophoto = GDALRaster(orthophoto_path, write=True)
+
+                                # TODO: Create tiles
+
+                                self.save()
+                            except ProcessingException as e:
+                                self.set_failure(str(e))
+                        else:
+                            # FAILED, CANCELED
                             self.save()
-                    except ProcessingException as e:
-                        self.set_failure(e.message)
-        except Exception as e:
-            logger.error("Uncaught error: {} {}".format(e.message, traceback.format_exc()))
+                    else:
+                        # Still waiting...
+                        self.save()
+                except ProcessingException as e:
+                    self.set_failure(str(e))
+
 
     def set_failure(self, error_message):
         logger.error("{} ERROR: {}".format(self, error_message))

app/scheduler.py

@@ -1,4 +1,4 @@
-import logging
+import logging, traceback
 from apscheduler.schedulers.background import BackgroundScheduler
 from apscheduler.schedulers import SchedulerAlreadyRunningError, SchedulerNotRunningError
 from threading import Thread, Lock
@@ -74,12 +74,15 @@ def process_pending_tasks():
         tasks_mutex.release()
 
     def process(task):
-        task.process()
+        try:
+            task.process()
 
-        # Might have been deleted
-        if task.pk is not None:
-            task.processing_lock = False
-            task.save()
+            # Might have been deleted
+            if task.pk is not None:
+                task.processing_lock = False
+                task.save()
+        except Exception as e:
+            logger.error("Uncaught error: {} {}".format(e, traceback.format_exc()))
 
     if tasks.count() > 0:
         pool = ThreadPool(tasks.count())

app/static/app/js/components/TaskListItem.jsx

@@ -127,7 +127,8 @@ class TaskListItem extends React.Component {
       const doAction = () => {
         this.setState({actionButtonsDisabled: true});
 
-        $.post(`/api/projects/${this.state.task.project}/tasks/${this.state.task.id}/${action}/`,
+        let url = `/api/projects/${this.state.task.project}/tasks/${this.state.task.id}/${action}/`;
+        $.post(url,
           {
             uuid: this.state.task.uuid
           }

app/tests/test_db.py

@@ -0,0 +1,20 @@
+from django.contrib.gis.gdal import GDALRaster
+
+from .classes import BootTestCase
+from app.models import Task, Project
+import os
+
+class TestApi(BootTestCase):
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def test_db(self):
+        # Make sure we can use PostGIS raster type
+        task = Task.objects.create(project=Project.objects.get(pk=1),
+                                   orthophoto=GDALRaster(os.path.join("app", "fixtures", "orthophoto.tif"), write=True))
+        task.refresh_from_db()
+        self.assertTrue(task.orthophoto.srid == 4326)
+        self.assertTrue(task.orthophoto.width == 252) # not original size, warp happened

db/Dockerfile

@@ -1,4 +1,15 @@
 FROM postgres:9.5
 MAINTAINER Piero Toffanin <pt@masseranolabs.com>
+
+ENV POSTGIS_MAJOR 2.3
+ENV POSTGIS_VERSION 2.3.0+dfsg-2.pgdg80+1
+
+RUN apt-get update \
+      && apt-get install -y --no-install-recommends \
+           postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR=$POSTGIS_VERSION \
+           postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts=$POSTGIS_VERSION \
+           postgis=$POSTGIS_VERSION \
+      && rm -rf /var/lib/apt/lists/*
+
 EXPOSE 5432
 COPY init.sql /docker-entrypoint-initdb.d/init-db.sql

db/init.sql

@@ -1,2 +1,3 @@
 ALTER USER postgres PASSWORD 'postgres';
-CREATE DATABASE webodm_dev;
+CREATE DATABASE webodm_dev;
+ALTER DATABASE webodm_dev SET postgis.gdal_enabled_drivers TO 'GTiff';

nodeodm/tests.py

@@ -1,3 +1,4 @@
+import requests
 from django.test import TestCase
 from django.utils import six
 import subprocess, time
@@ -6,7 +7,7 @@ from .models import ProcessingNode
 from .api_client import ApiClient
 from requests.exceptions import ConnectionError
 from .exceptions import ProcessingException
-import status_codes
+from . import status_codes
 
 current_dir = path.dirname(path.realpath(__file__))
 
@@ -76,7 +77,7 @@ class TestClientApi(TestCase):
         online_node = ProcessingNode.objects.get(pk=1)
 
         # Can call info(), options()
-        self.assertTrue(type(api.info()['version']) in [str, unicode])
+        self.assertTrue(type(api.info()['version']) == str)
         self.assertTrue(len(api.options()) > 0)
         
         # Can call new_task()
@@ -90,8 +91,8 @@ class TestClientApi(TestCase):
 
         # Can call task_info()
         task_info = api.task_info(uuid)
-        self.assertTrue(isinstance(task_info['dateCreated'], (int, long)))
-        self.assertTrue(isinstance(task_info['uuid'], (str, unicode)))
+        self.assertTrue(isinstance(task_info['dateCreated'], int))
+        self.assertTrue(isinstance(task_info['uuid'], str))
 
         # Can download assets?
         # Here we are waiting for the task to be completed
@@ -101,7 +102,7 @@ class TestClientApi(TestCase):
                 task_info = api.task_info(uuid)
                 if task_info['status']['code'] == status_codes.COMPLETED:
                     asset = api.task_download(uuid, "all.zip")
-                    self.assertTrue(isinstance(asset, (str, unicode))) # Binary content, really
+                    self.assertTrue(isinstance(asset, requests.Response)) # Binary content, really
                     break
             except ProcessingException:
                 pass
@@ -114,7 +115,7 @@ class TestClientApi(TestCase):
 
         # task_output
         self.assertTrue(isinstance(api.task_output(uuid, 0), list))
-        self.assertTrue(isinstance(online_node.get_task_console_output(uuid, 0), (str, unicode)))
+        self.assertTrue(isinstance(online_node.get_task_console_output(uuid, 0), str))
 
         self.assertRaises(ProcessingException, online_node.get_task_console_output, "wrong-uuid", 0)