From d9f4f1527c659cdb6776dfe76efe4ea2269f9816 Mon Sep 17 00:00:00 2001
From: Piero Toffanin <pt@masseranolabs.com>
Date: Wed, 14 Feb 2018 17:13:32 -0500
Subject: [PATCH] Proof of concept celery worker working with redis

---
 .env                    |  1 +
 docker-compose.yml      |  4 +++-
 requirements.txt        |  9 +++++++-
 worker/Dockerfile       |  5 ++++-
 worker/__init__.py      |  0
 worker/celery.py        |  8 +++++++
 worker/celeryconfig.py  |  9 ++++++++
 worker/requirements.txt |  1 +
 worker/start.sh         | 46 +++++++++++++++++++++++++++++++++++++++++
 worker/tasks.py         | 15 ++++++++++++++
 10 files changed, 95 insertions(+), 3 deletions(-)
 create mode 100644 worker/__init__.py
 create mode 100644 worker/celery.py
 create mode 100644 worker/celeryconfig.py
 create mode 100755 worker/start.sh
 create mode 100644 worker/tasks.py

diff --git a/.env b/.env
index c62a75b4..c074bc8e 100644
--- a/.env
+++ b/.env
@@ -6,3 +6,4 @@ WO_SSL_KEY=
 WO_SSL_CERT=
 WO_SSL_INSECURE_PORT_REDIRECT=80
 WO_DEBUG=YES
+WO_BROKER=redis://broker
diff --git a/docker-compose.yml b/docker-compose.yml
index 378d34a1..efe5d9bf 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -36,6 +36,8 @@ services:
   worker:
     image: opendronemap/webodm_worker
     container_name: worker
+    entrypoint: /bin/bash -c \"/webodm/wait-for-it.sh broker:6379 -- /broker/start.sh\""
     depends_on:
       - broker
-
+    environment:
+      - WO_BROKER
diff --git a/requirements.txt b/requirements.txt
index 8880e45a..14959fb4 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,9 @@
+amqp==2.2.2
 anyjson==0.3.3
 appdirs==1.4.0
 APScheduler==3.2.0
+billiard==3.5.0.3
+celery==4.1.0
 coreapi==2.0.9
 Django==1.11.7
 django-appconf==1.0.2
@@ -21,8 +24,10 @@ funcsigs==1.0.2
 futures==3.0.5
 gunicorn==19.7.1
 itypes==1.1.0
+kombu==4.1.0
 libsass==0.13.3
 Markdown==2.6.7
+olefile==0.44
 openapi-codec==1.1.7
 packaging==16.8
 pilkit==2.0
@@ -31,8 +36,9 @@ pip-autoremove==0.9.0
 psycopg2==2.6.2
 PyJWT==1.5.3
 pyparsing==2.1.10
-pytz==2017.2
+pytz==2018.3
 rcssmin==1.0.6
+redis==2.10.6
 requests==2.11.1
 rfc3987==1.3.7
 rjsmin==1.0.12
@@ -42,4 +48,5 @@ sqlparse==0.2.2
 strict-rfc3339==0.7
 tzlocal==1.3
 uritemplate==3.0.0
+vine==1.1.4
 webcolors==1.5
diff --git a/worker/Dockerfile b/worker/Dockerfile
index 82fc7216..c49abc01 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -1,6 +1,8 @@
 FROM ubuntu:16.04
 MAINTAINER Piero Toffanin <pt@masseranolabs.com>
-	
+
+WO_BROKER=redis://broker
+
 RUN apt-get update && \
 	apt-get install -y software-properties-common && \
 	add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable && \
@@ -8,6 +10,7 @@ RUN apt-get update && \
 	apt-get install -y grass-core python-pip
 
 COPY requirements.txt /worker/
+COPY ../wait-for-it.sh /worker/
 WORKDIR /worker
 
 RUN pip install -U pip && pip install -r requirements.txt
diff --git a/worker/__init__.py b/worker/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/worker/celery.py b/worker/celery.py
new file mode 100644
index 00000000..7a3fd59d
--- /dev/null
+++ b/worker/celery.py
@@ -0,0 +1,8 @@
+from celery import Celery
+import os
+
+app = Celery('tasks')
+app.config_from_object('worker.celeryconfig');
+
+if __name__ == '__main__':
+    app.start()
\ No newline at end of file
diff --git a/worker/celeryconfig.py b/worker/celeryconfig.py
new file mode 100644
index 00000000..64372f79
--- /dev/null
+++ b/worker/celeryconfig.py
@@ -0,0 +1,9 @@
+import os
+
+broker_url = os.environ.get('WO_BROKER', 'redis://localhost')
+result_backend = os.environ.get('WO_BROKER', 'redis://localhost')
+
+task_serializer = 'json'
+result_serializer = 'json'
+accept_content = ['json']
+include=['worker.tasks']
diff --git a/worker/requirements.txt b/worker/requirements.txt
index 74f9e8fe..d3323c83 100644
--- a/worker/requirements.txt
+++ b/worker/requirements.txt
@@ -1 +1,2 @@
 celery
+redis
diff --git a/worker/start.sh b/worker/start.sh
new file mode 100755
index 00000000..55510c7f
--- /dev/null
+++ b/worker/start.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+set -eo pipefail
+__dirname=$(cd $(dirname "$0"); pwd -P)
+cd ${__dirname}
+
+check_command(){
+	check_msg_prefix="Checking for $1... "
+	check_msg_result="\033[92m\033[1m OK\033[0m\033[39m"
+
+	hash $1 2>/dev/null || not_found=true 
+	if [[ $not_found ]]; then
+		
+		# Can we attempt to install it?
+		if [[ ! -z "$3" ]]; then
+			echo -e "$check_msg_prefix \033[93mnot found, we'll attempt to install\033[39m"
+			run "$3 || sudo $3"
+
+			# Recurse, but don't pass the install command
+			check_command "$1" "$2"	
+		else
+			check_msg_result="\033[91m can't find $1! Check that the program is installed and that you have added the proper path to the program to your PATH environment variable before launching WebODM. If you change your PATH environment variable, remember to close and reopen your terminal. $2\033[39m"
+		fi
+	fi
+
+	echo -e "$check_msg_prefix $check_msg_result"
+	if [[ $not_found ]]; then
+		return 1
+	fi
+}
+
+environment_check(){
+	check_command "celery" "Run \033[1msudo pip install -U celery\033[0m" "pip install -U celery"
+	if [[ -z "$WO_BROKER" ]]; then
+		echo -e "\033[91mWO_BROKER environment variable is not set. Defaulting to redis://localhost\033[39m"
+		export WO_BROKER=redis://localhost
+	fi
+}
+
+environment_check
+echo "Starting worker using broker at $WO_BROKER"
+
+# Switch to parent directory
+# so that celery recognizes the package name
+cd ${__dirname}/../
+
+celery -A worker worker --loglevel=info
\ No newline at end of file
diff --git a/worker/tasks.py b/worker/tasks.py
new file mode 100644
index 00000000..0af84c19
--- /dev/null
+++ b/worker/tasks.py
@@ -0,0 +1,15 @@
+from .celery import app
+
+@app.task
+def add(x, y):
+    return x + y
+
+
+@app.task
+def mul(x, y):
+    return x * y
+
+
+@app.task
+def xsum(numbers):
+    return sum(numbers)
\ No newline at end of file