Proof of concept celery worker working with redis

.env

@@ -6,3 +6,4 @@ WO_SSL_KEY=
 WO_SSL_CERT=
 WO_SSL_INSECURE_PORT_REDIRECT=80
 WO_DEBUG=YES
+WO_BROKER=redis://broker

docker-compose.yml

@@ -36,6 +36,8 @@ services:
   worker:
     image: opendronemap/webodm_worker
     container_name: worker
+    entrypoint: /bin/bash -c \"/webodm/wait-for-it.sh broker:6379 -- /broker/start.sh\""
     depends_on:
       - broker
-
+    environment:
+      - WO_BROKER

requirements.txt

@@ -1,6 +1,9 @@
+amqp==2.2.2
 anyjson==0.3.3
 appdirs==1.4.0
 APScheduler==3.2.0
+billiard==3.5.0.3
+celery==4.1.0
 coreapi==2.0.9
 Django==1.11.7
 django-appconf==1.0.2
@@ -21,8 +24,10 @@ funcsigs==1.0.2
 futures==3.0.5
 gunicorn==19.7.1
 itypes==1.1.0
+kombu==4.1.0
 libsass==0.13.3
 Markdown==2.6.7
+olefile==0.44
 openapi-codec==1.1.7
 packaging==16.8
 pilkit==2.0
@@ -31,8 +36,9 @@ pip-autoremove==0.9.0
 psycopg2==2.6.2
 PyJWT==1.5.3
 pyparsing==2.1.10
-pytz==2017.2
+pytz==2018.3
 rcssmin==1.0.6
+redis==2.10.6
 requests==2.11.1
 rfc3987==1.3.7
 rjsmin==1.0.12
@@ -42,4 +48,5 @@ sqlparse==0.2.2
 strict-rfc3339==0.7
 tzlocal==1.3
 uritemplate==3.0.0
+vine==1.1.4
 webcolors==1.5

worker/Dockerfile

@@ -1,6 +1,8 @@
 FROM ubuntu:16.04
 MAINTAINER Piero Toffanin <pt@masseranolabs.com>
-	
+
+WO_BROKER=redis://broker
+
 RUN apt-get update && \
 	apt-get install -y software-properties-common && \
 	add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable && \
@@ -8,6 +10,7 @@ RUN apt-get update && \
 	apt-get install -y grass-core python-pip
 
 COPY requirements.txt /worker/
+COPY ../wait-for-it.sh /worker/
 WORKDIR /worker
 
 RUN pip install -U pip && pip install -r requirements.txt

worker/celery.py

@@ -0,0 +1,8 @@
+from celery import Celery
+import os
+
+app = Celery('tasks')
+app.config_from_object('worker.celeryconfig');
+
+if __name__ == '__main__':
+    app.start()

worker/celeryconfig.py

@@ -0,0 +1,9 @@
+import os
+
+broker_url = os.environ.get('WO_BROKER', 'redis://localhost')
+result_backend = os.environ.get('WO_BROKER', 'redis://localhost')
+
+task_serializer = 'json'
+result_serializer = 'json'
+accept_content = ['json']
+include=['worker.tasks']

worker/requirements.txt

@@ -1 +1,2 @@
 celery
+redis

worker/start.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+set -eo pipefail
+__dirname=$(cd $(dirname "$0"); pwd -P)
+cd ${__dirname}
+
+check_command(){
+	check_msg_prefix="Checking for $1... "
+	check_msg_result="\033[92m\033[1m OK\033[0m\033[39m"
+
+	hash $1 2>/dev/null || not_found=true 
+	if [[ $not_found ]]; then
+		
+		# Can we attempt to install it?
+		if [[ ! -z "$3" ]]; then
+			echo -e "$check_msg_prefix \033[93mnot found, we'll attempt to install\033[39m"
+			run "$3 || sudo $3"
+
+			# Recurse, but don't pass the install command
+			check_command "$1" "$2"	
+		else
+			check_msg_result="\033[91m can't find $1! Check that the program is installed and that you have added the proper path to the program to your PATH environment variable before launching WebODM. If you change your PATH environment variable, remember to close and reopen your terminal. $2\033[39m"
+		fi
+	fi
+
+	echo -e "$check_msg_prefix $check_msg_result"
+	if [[ $not_found ]]; then
+		return 1
+	fi
+}
+
+environment_check(){
+	check_command "celery" "Run \033[1msudo pip install -U celery\033[0m" "pip install -U celery"
+	if [[ -z "$WO_BROKER" ]]; then
+		echo -e "\033[91mWO_BROKER environment variable is not set. Defaulting to redis://localhost\033[39m"
+		export WO_BROKER=redis://localhost
+	fi
+}
+
+environment_check
+echo "Starting worker using broker at $WO_BROKER"
+
+# Switch to parent directory
+# so that celery recognizes the package name
+cd ${__dirname}/../
+
+celery -A worker worker --loglevel=info

worker/tasks.py

@@ -0,0 +1,15 @@
+from .celery import app
+
+@app.task
+def add(x, y):
+    return x + y
+
+
+@app.task
+def mul(x, y):
+    return x * y
+
+
+@app.task
+def xsum(numbers):
+    return sum(numbers)