kopia lustrzana https://github.com/OpenDroneMap/WebODM
Proof of concept celery worker working with redis
rodzic
9329e345d1
commit
d9f4f1527c
1
.env
1
.env
|
@ -6,3 +6,4 @@ WO_SSL_KEY=
|
||||||
WO_SSL_CERT=
|
WO_SSL_CERT=
|
||||||
WO_SSL_INSECURE_PORT_REDIRECT=80
|
WO_SSL_INSECURE_PORT_REDIRECT=80
|
||||||
WO_DEBUG=YES
|
WO_DEBUG=YES
|
||||||
|
WO_BROKER=redis://broker
|
||||||
|
|
|
@ -36,6 +36,8 @@ services:
|
||||||
worker:
|
worker:
|
||||||
image: opendronemap/webodm_worker
|
image: opendronemap/webodm_worker
|
||||||
container_name: worker
|
container_name: worker
|
||||||
|
entrypoint: /bin/bash -c \"/webodm/wait-for-it.sh broker:6379 -- /broker/start.sh\""
|
||||||
depends_on:
|
depends_on:
|
||||||
- broker
|
- broker
|
||||||
|
environment:
|
||||||
|
- WO_BROKER
|
||||||
|
|
|
@ -1,6 +1,9 @@
|
||||||
|
amqp==2.2.2
|
||||||
anyjson==0.3.3
|
anyjson==0.3.3
|
||||||
appdirs==1.4.0
|
appdirs==1.4.0
|
||||||
APScheduler==3.2.0
|
APScheduler==3.2.0
|
||||||
|
billiard==3.5.0.3
|
||||||
|
celery==4.1.0
|
||||||
coreapi==2.0.9
|
coreapi==2.0.9
|
||||||
Django==1.11.7
|
Django==1.11.7
|
||||||
django-appconf==1.0.2
|
django-appconf==1.0.2
|
||||||
|
@ -21,8 +24,10 @@ funcsigs==1.0.2
|
||||||
futures==3.0.5
|
futures==3.0.5
|
||||||
gunicorn==19.7.1
|
gunicorn==19.7.1
|
||||||
itypes==1.1.0
|
itypes==1.1.0
|
||||||
|
kombu==4.1.0
|
||||||
libsass==0.13.3
|
libsass==0.13.3
|
||||||
Markdown==2.6.7
|
Markdown==2.6.7
|
||||||
|
olefile==0.44
|
||||||
openapi-codec==1.1.7
|
openapi-codec==1.1.7
|
||||||
packaging==16.8
|
packaging==16.8
|
||||||
pilkit==2.0
|
pilkit==2.0
|
||||||
|
@ -31,8 +36,9 @@ pip-autoremove==0.9.0
|
||||||
psycopg2==2.6.2
|
psycopg2==2.6.2
|
||||||
PyJWT==1.5.3
|
PyJWT==1.5.3
|
||||||
pyparsing==2.1.10
|
pyparsing==2.1.10
|
||||||
pytz==2017.2
|
pytz==2018.3
|
||||||
rcssmin==1.0.6
|
rcssmin==1.0.6
|
||||||
|
redis==2.10.6
|
||||||
requests==2.11.1
|
requests==2.11.1
|
||||||
rfc3987==1.3.7
|
rfc3987==1.3.7
|
||||||
rjsmin==1.0.12
|
rjsmin==1.0.12
|
||||||
|
@ -42,4 +48,5 @@ sqlparse==0.2.2
|
||||||
strict-rfc3339==0.7
|
strict-rfc3339==0.7
|
||||||
tzlocal==1.3
|
tzlocal==1.3
|
||||||
uritemplate==3.0.0
|
uritemplate==3.0.0
|
||||||
|
vine==1.1.4
|
||||||
webcolors==1.5
|
webcolors==1.5
|
||||||
|
|
|
@ -1,6 +1,8 @@
|
||||||
FROM ubuntu:16.04
|
FROM ubuntu:16.04
|
||||||
MAINTAINER Piero Toffanin <pt@masseranolabs.com>
|
MAINTAINER Piero Toffanin <pt@masseranolabs.com>
|
||||||
|
|
||||||
|
WO_BROKER=redis://broker
|
||||||
|
|
||||||
RUN apt-get update && \
|
RUN apt-get update && \
|
||||||
apt-get install -y software-properties-common && \
|
apt-get install -y software-properties-common && \
|
||||||
add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable && \
|
add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable && \
|
||||||
|
@ -8,6 +10,7 @@ RUN apt-get update && \
|
||||||
apt-get install -y grass-core python-pip
|
apt-get install -y grass-core python-pip
|
||||||
|
|
||||||
COPY requirements.txt /worker/
|
COPY requirements.txt /worker/
|
||||||
|
COPY ../wait-for-it.sh /worker/
|
||||||
WORKDIR /worker
|
WORKDIR /worker
|
||||||
|
|
||||||
RUN pip install -U pip && pip install -r requirements.txt
|
RUN pip install -U pip && pip install -r requirements.txt
|
||||||
|
|
|
@ -0,0 +1,8 @@
|
||||||
|
from celery import Celery
|
||||||
|
import os
|
||||||
|
|
||||||
|
app = Celery('tasks')
|
||||||
|
app.config_from_object('worker.celeryconfig');
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
app.start()
|
|
@ -0,0 +1,9 @@
|
||||||
|
import os
|
||||||
|
|
||||||
|
broker_url = os.environ.get('WO_BROKER', 'redis://localhost')
|
||||||
|
result_backend = os.environ.get('WO_BROKER', 'redis://localhost')
|
||||||
|
|
||||||
|
task_serializer = 'json'
|
||||||
|
result_serializer = 'json'
|
||||||
|
accept_content = ['json']
|
||||||
|
include=['worker.tasks']
|
|
@ -1 +1,2 @@
|
||||||
celery
|
celery
|
||||||
|
redis
|
||||||
|
|
|
@ -0,0 +1,46 @@
|
||||||
|
#!/bin/bash
|
||||||
|
set -eo pipefail
|
||||||
|
__dirname=$(cd $(dirname "$0"); pwd -P)
|
||||||
|
cd ${__dirname}
|
||||||
|
|
||||||
|
check_command(){
|
||||||
|
check_msg_prefix="Checking for $1... "
|
||||||
|
check_msg_result="\033[92m\033[1m OK\033[0m\033[39m"
|
||||||
|
|
||||||
|
hash $1 2>/dev/null || not_found=true
|
||||||
|
if [[ $not_found ]]; then
|
||||||
|
|
||||||
|
# Can we attempt to install it?
|
||||||
|
if [[ ! -z "$3" ]]; then
|
||||||
|
echo -e "$check_msg_prefix \033[93mnot found, we'll attempt to install\033[39m"
|
||||||
|
run "$3 || sudo $3"
|
||||||
|
|
||||||
|
# Recurse, but don't pass the install command
|
||||||
|
check_command "$1" "$2"
|
||||||
|
else
|
||||||
|
check_msg_result="\033[91m can't find $1! Check that the program is installed and that you have added the proper path to the program to your PATH environment variable before launching WebODM. If you change your PATH environment variable, remember to close and reopen your terminal. $2\033[39m"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo -e "$check_msg_prefix $check_msg_result"
|
||||||
|
if [[ $not_found ]]; then
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
environment_check(){
|
||||||
|
check_command "celery" "Run \033[1msudo pip install -U celery\033[0m" "pip install -U celery"
|
||||||
|
if [[ -z "$WO_BROKER" ]]; then
|
||||||
|
echo -e "\033[91mWO_BROKER environment variable is not set. Defaulting to redis://localhost\033[39m"
|
||||||
|
export WO_BROKER=redis://localhost
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
environment_check
|
||||||
|
echo "Starting worker using broker at $WO_BROKER"
|
||||||
|
|
||||||
|
# Switch to parent directory
|
||||||
|
# so that celery recognizes the package name
|
||||||
|
cd ${__dirname}/../
|
||||||
|
|
||||||
|
celery -A worker worker --loglevel=info
|
|
@ -0,0 +1,15 @@
|
||||||
|
from .celery import app
|
||||||
|
|
||||||
|
@app.task
|
||||||
|
def add(x, y):
|
||||||
|
return x + y
|
||||||
|
|
||||||
|
|
||||||
|
@app.task
|
||||||
|
def mul(x, y):
|
||||||
|
return x * y
|
||||||
|
|
||||||
|
|
||||||
|
@app.task
|
||||||
|
def xsum(numbers):
|
||||||
|
return sum(numbers)
|
Ładowanie…
Reference in New Issue