Merge pull request #668 from dronemapper-io/node-micmac-integration

initial node-micmac integration

app/models/task.py

@@ -133,7 +133,6 @@ class Task(models.Model):
     ASSETS_MAP = {
             'all.zip': 'all.zip',
             'orthophoto.tif': os.path.join('odm_orthophoto', 'odm_orthophoto.tif'),
-            'orthophoto.png': os.path.join('odm_orthophoto', 'odm_orthophoto.png'),
             'orthophoto.mbtiles': os.path.join('odm_orthophoto', 'odm_orthophoto.mbtiles'),
             'georeferenced_model.las': os.path.join('odm_georeferencing', 'odm_georeferenced_model.las'),
             'georeferenced_model.laz': os.path.join('odm_georeferencing', 'odm_georeferenced_model.laz'),

app/tests/test_api_task.py

@@ -324,7 +324,7 @@ class TestApiTask(BootTransactionTestCase):
         # Can download assets
         for asset in list(task.ASSETS_MAP.keys()):
             res = client.get("/api/projects/{}/tasks/{}/download/{}".format(project.id, task.id, asset))
-            self.assertTrue(res.status_code == status.HTTP_200_OK)
+            self.assertEqual(res.status_code, status.HTTP_200_OK)
 
         # We can stream downloads
         res = client.get("/api/projects/{}/tasks/{}/download/{}?_force_stream=1".format(project.id, task.id, list(task.ASSETS_MAP.keys())[0]))

docker-compose.nodemicmac.yml

@@ -0,0 +1,17 @@
+# Chaining this file to the main docker-compose file adds
+# a default processing node instance. This is best for users
+# who are just getting started with WebODM.
+
+version: '2.1'
+services:
+  webapp:
+    entrypoint: /bin/bash -c "chmod +x /webodm/*.sh && /bin/bash -c \"/webodm/wait-for-postgres.sh db /webodm/wait-for-it.sh -t 0 broker:6379 -- /webodm/start.sh --create-micmac-pnode\""
+    depends_on:
+      - node-micmac-1
+  node-micmac-1:
+    image: dronemapper/node-micmac
+    container_name: node-micmac-1
+    ports:
+      - "3000"
+    restart: on-failure:10
+    oom_score_adj: 500

nodeodm/external/NodeODM

@@ -1 +1 @@
-Subproject commit d0e1e1424e90c21fee401ce6e5a7dd1de7b32112
+Subproject commit 2a5d73ae2dfc89fc69f24095baebe31f0f0ac340

nodeodm/models.py

@@ -6,6 +6,7 @@ from django.utils import timezone
 from django.dispatch import receiver
 from guardian.models import GroupObjectPermissionBase
 from guardian.models import UserObjectPermissionBase
+from webodm import settings
 
 import json
 from pyodm import Node
@@ -115,7 +116,15 @@ class ProcessingNode(models.Model):
 
         opts = self.options_list_to_dict(options)
 
-        task = api_client.create_task(images, opts, name, progress_callback)
+        if not settings.TESTING:
+            task = api_client.create_task(images, opts, name, progress_callback)
+        else:
+            # The create_task function uses multi-threaded parallel uploads
+            # but Django tests cannot cope with DB updates from different threads
+            # (and progress_callback often updates the DB). So during testing
+            # we use the fallback function equivalent which is single-threaded
+            task = api_client.create_task_fallback(images, opts, name, progress_callback)
+
         return task.uuid
 
     def get_task_info(self, uuid, with_output=None):

start.sh

@@ -61,6 +61,10 @@ if [[ "$1" = "--create-default-pnode" ]]; then
    echo "from nodeodm.models import ProcessingNode; ProcessingNode.objects.update_or_create(hostname='node-odm-1', defaults={'hostname': 'node-odm-1', 'port': 3000})" | python manage.py shell
 fi
 
+if [[ "$1" = "--create-micmac-pnode" ]]; then
+   echo "from nodeodm.models import ProcessingNode; ProcessingNode.objects.update_or_create(hostname='node-micmac-1', defaults={'hostname': 'node-micmac-1', 'port': 3000})" | python manage.py shell
+fi
+
 export WO_HOST="${WO_HOST:=localhost}"
 export WO_PORT="${WO_PORT:=8000}"
 

webodm.sh

@@ -89,6 +89,10 @@ case $key in
     load_default_node=false
     shift # past argument
     ;;
+    --with-micmac)
+    load_micmac_node=true
+    shift # past argument
+    ;;
     *)    # unknown option
     POSITIONAL+=("$1") # save it in an array for later
     shift # past argument
@@ -117,6 +121,7 @@ usage(){
   echo "	--hostname	<hostname>	Set the hostname that WebODM will be accessible from (default: $DEFAULT_HOST)"
   echo "	--media-dir	<path>	Path where processing results will be stored to (default: $DEFAULT_MEDIA_DIR (docker named volume))"
   echo "	--no-default-node	Do not create a default NodeODM node attached to WebODM on startup (default: disabled)"
+  echo "	--with-micmac	Create a NodeMICMAC node attached to WebODM on startup. Experimental! (default: disabled)"
   echo "	--ssl	Enable SSL and automatically request and install a certificate from letsencrypt.org. (default: $DEFAULT_SSL)"
   echo "	--ssl-key	<path>	Manually specify a path to the private key file (.pem) to use with nginx to enable SSL (default: None)"
   echo "	--ssl-cert	<path>	Manually specify a path to the certificate file (.pem) to use with nginx to enable SSL (default: None)"
@@ -194,6 +199,10 @@ start(){
         command+=" -f docker-compose.nodeodm.yml"
     fi
 
+    if [[ $load_micmac_node = true ]]; then
+        command+=" -f docker-compose.nodemicmac.yml"
+    fi
+
     if [[ $dev_mode = true ]]; then
         command+=" -f docker-compose.dev.yml"
     fi
@@ -240,7 +249,7 @@ start(){
 }
 
 down(){
-	run "docker-compose -f docker-compose.yml -f docker-compose.nodeodm.yml down --remove-orphans"
+	run "docker-compose -f docker-compose.yml -f docker-compose.nodeodm.yml -f docker-compose.nodemicmac.yml down --remove-orphans"
 }
 
 rebuild(){
@@ -298,7 +307,7 @@ if [[ $1 = "start" ]]; then
 elif [[ $1 = "stop" ]]; then
 	environment_check
 	echo "Stopping WebODM..."
-	run "docker-compose -f docker-compose.yml -f docker-compose.nodeodm.yml stop"
+	run "docker-compose -f docker-compose.yml -f docker-compose.nodeodm.yml -f docker-compose.nodemicmac.yml stop"
 elif [[ $1 = "restart" ]]; then
 	environment_check
 	echo "Restarting WebODM..."
@@ -316,9 +325,19 @@ elif [[ $1 = "update" ]]; then
 	down
 	echo "Updating WebODM..."
 	run "git pull origin master"
-	run "docker pull opendronemap/nodeodm"
-	run "docker pull opendronemap/webodm_db"
-	run "docker pull opendronemap/webodm_webapp"
+
+	command="docker-compose -f docker-compose.yml"
+
+	if [[ $load_default_node = true ]]; then
+		command+=" -f docker-compose.nodeodm.yml"
+	fi
+
+	if [[ $load_micmac_node = true ]]; then
+		command+=" -f docker-compose.nodemicmac.yml"
+	fi
+
+	command+=" pull"
+	run "$command"
 	echo -e "\033[1mDone!\033[0m You can now start WebODM by running $0 start"
 elif [[ $1 = "checkenv" ]]; then
 	environment_check