From 38c8db00373b1187e2ab8d69ee0d56c284a21a7d Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 19 Apr 2018 14:45:33 -0400 Subject: [PATCH 1/8] Changed DEBUG default, added restart on failure on all services --- .env | 2 +- docker-compose.nodeodm.yml | 1 + docker-compose.yml | 3 +++ webodm/settings.py | 1 + 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.env b/.env index c074bc8e..ddbb88a9 100644 --- a/.env +++ b/.env @@ -5,5 +5,5 @@ WO_SSL=NO WO_SSL_KEY= WO_SSL_CERT= WO_SSL_INSECURE_PORT_REDIRECT=80 -WO_DEBUG=YES +WO_DEBUG=NO WO_BROKER=redis://broker diff --git a/docker-compose.nodeodm.yml b/docker-compose.nodeodm.yml index 7f3c848a..3de13079 100644 --- a/docker-compose.nodeodm.yml +++ b/docker-compose.nodeodm.yml @@ -13,3 +13,4 @@ services: container_name: node-odm-1 ports: - "3000" + restart: on-failure:10 diff --git a/docker-compose.yml b/docker-compose.yml index d73bf8c0..4609f14b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,7 @@ services: - "5432" volumes: - dbdata:/var/lib/postgresql/data + restart: on-failure:10 webapp: image: opendronemap/webodm_webapp container_name: webapp @@ -33,6 +34,7 @@ services: broker: image: redis container_name: broker + restart: on-failure:10 worker: image: opendronemap/webodm_webapp container_name: worker @@ -45,3 +47,4 @@ services: environment: - WO_BROKER - WO_DEBUG + restart: on-failure:10 diff --git a/webodm/settings.py b/webodm/settings.py index e81d0110..a0bf4e24 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -326,6 +326,7 @@ CELERY_ACCEPT_CONTENT = ['json'] CELERY_INCLUDE=['worker.tasks'] CELERY_WORKER_REDIRECT_STDOUTS = False CELERY_WORKER_HIJACK_ROOT_LOGGER = False +CELERYD_MAX_TASKS_PER_CHILD = 1000 if TESTING: CELERY_TASK_ALWAYS_EAGER = True From 480a20d9c773f4b33f1258d7ed3afc46745ff5fc Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 19 Apr 2018 14:55:09 -0400 Subject: [PATCH 2/8] Added max-tasks-per-child to workers --- webodm/settings.py | 2 +- worker.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/webodm/settings.py b/webodm/settings.py index a0bf4e24..a4f02666 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -326,7 +326,7 @@ CELERY_ACCEPT_CONTENT = ['json'] CELERY_INCLUDE=['worker.tasks'] CELERY_WORKER_REDIRECT_STDOUTS = False CELERY_WORKER_HIJACK_ROOT_LOGGER = False -CELERYD_MAX_TASKS_PER_CHILD = 1000 + if TESTING: CELERY_TASK_ALWAYS_EAGER = True diff --git a/worker.sh b/worker.sh index 93507582..98980159 100755 --- a/worker.sh +++ b/worker.sh @@ -52,7 +52,7 @@ start(){ action=$1 echo "Starting worker using broker at $WO_BROKER" - celery -A worker worker --loglevel=warn > /dev/null + celery -A worker worker --max-tasks-per-child 1000 --loglevel=warn > /dev/null } start_scheduler(){ From f802a4aaf851757e5bc1490bdde7f87ed1ab730e Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 19 Apr 2018 15:39:43 -0400 Subject: [PATCH 3/8] removed pmvs suggestion for exit code 1 --- app/static/app/js/components/TaskListItem.jsx | 1 - worker.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app/static/app/js/components/TaskListItem.jsx b/app/static/app/js/components/TaskListItem.jsx index f7431acd..93e051b5 100644 --- a/app/static/app/js/components/TaskListItem.jsx +++ b/app/static/app/js/components/TaskListItem.jsx @@ -500,7 +500,6 @@ class TaskListItem extends React.Component { "Process exited with code 1" means that part of the processing failed. Try tweaking the Task Options as follows:
  • Increase the min-num-features option, especially if your images have lots of vegetation
    • -
  • Enable the use-pmvs option.
Still not working? Upload your images somewhere like Dropbox or Google Drive and open a topic on our community forum, making sure to include a copy of your task's output (the one you see above , click to download it). Our awesome contributors will try to help you! diff --git a/worker.sh b/worker.sh index 98980159..147b1b21 100755 --- a/worker.sh +++ b/worker.sh @@ -52,7 +52,7 @@ start(){ action=$1 echo "Starting worker using broker at $WO_BROKER" - celery -A worker worker --max-tasks-per-child 1000 --loglevel=warn > /dev/null + celery -A worker worker --concurrency 1 --max-tasks-per-child 1000 --loglevel=warn > /dev/null } start_scheduler(){ From 9a162841f15f775ac7aa01b3f2d49fe85bf64139 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sun, 22 Apr 2018 11:24:17 -0400 Subject: [PATCH 4/8] Debug disabled by default --- README.md | 10 ---------- devenv.sh | 2 ++ webodm.sh | 6 +++--- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 7743bc55..dbe8f658 100644 --- a/README.md +++ b/README.md @@ -84,16 +84,6 @@ Adding more processing nodes will allow you to run multiple jobs in parallel. You **will not be able to distribute a single job across multiple processing nodes**. We are actively working to bring this feature to reality, but we're not there yet. -### Security - -If you want to run WebODM in production, make sure to pass the `--no-debug` flag while starting WebODM: - -```bash -./webodm.sh restart --no-debug -``` - -This will disable the `DEBUG` flag from `webodm/settings.py` within the docker container. This is [really important](https://docs.djangoproject.com/en/1.11/ref/settings/#std:setting-DEBUG). - ### Enable SSL WebODM has the ability to automatically request and install a SSL certificate via [Let’s Encrypt](https://letsencrypt.org/), or you can manually specify your own key/certificate pair. diff --git a/devenv.sh b/devenv.sh index 976f645b..813097ab 100755 --- a/devenv.sh +++ b/devenv.sh @@ -4,6 +4,8 @@ __dirname=$(cd $(dirname "$0"); pwd -P) ${__dirname}/webodm.sh checkenv +export WO_DEBUG=YES + usage(){ echo "Usage: $0 [options]" echo diff --git a/webodm.sh b/webodm.sh index 062c6192..bea096fb 100755 --- a/webodm.sh +++ b/webodm.sh @@ -68,8 +68,8 @@ case $key in shift # past argument shift # past value ;; - --no-debug) - export WO_DEBUG=NO + --debug) + export WO_DEBUG=YES shift # past argument ;; --broker) @@ -108,7 +108,7 @@ usage(){ echo " --ssl-key Manually specify a path to the private key file (.pem) to use with nginx to enable SSL (default: None)" echo " --ssl-cert Manually specify a path to the certificate file (.pem) to use with nginx to enable SSL (default: None)" echo " --ssl-insecure-port-redirect Insecure port number to redirect from when SSL is enabled (default: $DEFAULT_SSL_INSECURE_PORT_REDIRECT)" - echo " --no-debug Disable debug for production environments (default: disabled)" + echo " --debug Enable debug for development environments (default: disabled)" echo " --broker Set the URL used to connect to the celery broker (default: $DEFAULT_BROKER)" exit } From 8317283b8767a5d55002b6fcc96563aad9940fdd Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sun, 22 Apr 2018 14:49:20 -0400 Subject: [PATCH 5/8] Faster timeout on update_node_info(), oom preferences --- docker-compose.nodeodm.yml | 1 + docker-compose.yml | 6 ++++++ nodeodm/api_client.py | 19 +++++++++---------- nodeodm/models.py | 6 +++--- webodm/settings.py | 1 + worker.sh | 2 +- worker/tasks.py | 1 - 7 files changed, 21 insertions(+), 15 deletions(-) diff --git a/docker-compose.nodeodm.yml b/docker-compose.nodeodm.yml index 3de13079..0b2a4d6b 100644 --- a/docker-compose.nodeodm.yml +++ b/docker-compose.nodeodm.yml @@ -14,3 +14,4 @@ services: ports: - "3000" restart: on-failure:10 + oom_score_adj: 500 diff --git a/docker-compose.yml b/docker-compose.yml index 4609f14b..84c6f2e9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,6 +13,7 @@ services: volumes: - dbdata:/var/lib/postgresql/data restart: on-failure:10 + oom_score_adj: -100 webapp: image: opendronemap/webodm_webapp container_name: webapp @@ -31,10 +32,14 @@ services: - WO_DEBUG - WO_BROKER restart: on-failure:10 + oom_score_adj: 0 broker: image: redis container_name: broker restart: on-failure:10 + mem_limit: 256m + oom_kill_disable: true + oom_score_adj: -500 worker: image: opendronemap/webodm_webapp container_name: worker @@ -48,3 +53,4 @@ services: - WO_BROKER - WO_DEBUG restart: on-failure:10 + oom_score_adj: 250 diff --git a/nodeodm/api_client.py b/nodeodm/api_client.py index 42285c44..82d07ead 100644 --- a/nodeodm/api_client.py +++ b/nodeodm/api_client.py @@ -9,12 +9,11 @@ import os from urllib.parse import urlunparse from app.testwatch import TestWatch -TIMEOUT = 30 - class ApiClient: - def __init__(self, host, port): + def __init__(self, host, port, timeout=30): self.host = host self.port = port + self.timeout = timeout def url(self, url): netloc = self.host if self.port == 80 else "{}:{}".format(self.host, self.port) @@ -23,28 +22,28 @@ class ApiClient: return urlunparse(('http', netloc, url, '', '', '')) def info(self): - return requests.get(self.url('/info'), timeout=TIMEOUT).json() + return requests.get(self.url('/info'), timeout=self.timeout).json() def options(self): - return requests.get(self.url('/options'), timeout=TIMEOUT).json() + return requests.get(self.url('/options'), timeout=self.timeout).json() def task_info(self, uuid): - return requests.get(self.url('/task/{}/info').format(uuid), timeout=TIMEOUT).json() + return requests.get(self.url('/task/{}/info').format(uuid), timeout=self.timeout).json() @TestWatch.watch() def task_output(self, uuid, line = 0): - return requests.get(self.url('/task/{}/output?line={}').format(uuid, line), timeout=TIMEOUT).json() + return requests.get(self.url('/task/{}/output?line={}').format(uuid, line), timeout=self.timeout).json() def task_cancel(self, uuid): - return requests.post(self.url('/task/cancel'), data={'uuid': uuid}, timeout=TIMEOUT).json() + return requests.post(self.url('/task/cancel'), data={'uuid': uuid}, timeout=self.timeout).json() def task_remove(self, uuid): - return requests.post(self.url('/task/remove'), data={'uuid': uuid}, timeout=TIMEOUT).json() + return requests.post(self.url('/task/remove'), data={'uuid': uuid}, timeout=self.timeout).json() def task_restart(self, uuid, options = None): data = {'uuid': uuid} if options is not None: data['options'] = json.dumps(options) - return requests.post(self.url('/task/restart'), data=data, timeout=TIMEOUT).json() + return requests.post(self.url('/task/restart'), data=data, timeout=self.timeout).json() def task_download(self, uuid, asset): res = requests.get(self.url('/task/{}/download/{}').format(uuid, asset), stream=True) diff --git a/nodeodm/models.py b/nodeodm/models.py index 7ccad56b..514054f7 100644 --- a/nodeodm/models.py +++ b/nodeodm/models.py @@ -66,7 +66,7 @@ class ProcessingNode(models.Model): :returns: True if information could be updated, False otherwise """ - api_client = self.api_client() + api_client = self.api_client(timeout=5) try: info = api_client.info() self.api_version = info['version'] @@ -80,8 +80,8 @@ class ProcessingNode(models.Model): except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, json.decoder.JSONDecodeError, simplejson.JSONDecodeError): return False - def api_client(self): - return ApiClient(self.hostname, self.port) + def api_client(self, timeout=30): + return ApiClient(self.hostname, self.port, timeout) def get_available_options_json(self, pretty=False): """ diff --git a/webodm/settings.py b/webodm/settings.py index a4f02666..4465a39b 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -326,6 +326,7 @@ CELERY_ACCEPT_CONTENT = ['json'] CELERY_INCLUDE=['worker.tasks'] CELERY_WORKER_REDIRECT_STDOUTS = False CELERY_WORKER_HIJACK_ROOT_LOGGER = False +CELERY_TASK_RESULT_EXPIRES = 60 if TESTING: CELERY_TASK_ALWAYS_EAGER = True diff --git a/worker.sh b/worker.sh index 147b1b21..d471b623 100755 --- a/worker.sh +++ b/worker.sh @@ -52,7 +52,7 @@ start(){ action=$1 echo "Starting worker using broker at $WO_BROKER" - celery -A worker worker --concurrency 1 --max-tasks-per-child 1000 --loglevel=warn > /dev/null + celery -A worker worker --autoscale $(grep -c '^processor' /proc/cpuinfo),2 --max-tasks-per-child 1000 --loglevel=warn > /dev/null } start_scheduler(){ diff --git a/worker/tasks.py b/worker/tasks.py index 13228557..35359543 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -79,7 +79,6 @@ def get_pending_tasks(): @app.task def process_pending_tasks(): tasks = get_pending_tasks() - for task in tasks: process_task.delay(task.id) From e619f8217387fab2409809601321a0cf44f65d4f Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sun, 22 Apr 2018 15:00:16 -0400 Subject: [PATCH 6/8] Removed task result expiries key --- webodm/settings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/webodm/settings.py b/webodm/settings.py index 4465a39b..a4f02666 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -326,7 +326,6 @@ CELERY_ACCEPT_CONTENT = ['json'] CELERY_INCLUDE=['worker.tasks'] CELERY_WORKER_REDIRECT_STDOUTS = False CELERY_WORKER_HIJACK_ROOT_LOGGER = False -CELERY_TASK_RESULT_EXPIRES = 60 if TESTING: CELERY_TASK_ALWAYS_EAGER = True From 271d62ddac1d7e28c1ec5aa206822ed07ad8b107 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sun, 22 Apr 2018 15:08:11 -0400 Subject: [PATCH 7/8] Removed unsupported docker-compose configs --- docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 84c6f2e9..4d71e107 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,8 +37,6 @@ services: image: redis container_name: broker restart: on-failure:10 - mem_limit: 256m - oom_kill_disable: true oom_score_adj: -500 worker: image: opendronemap/webodm_webapp From ee2c6a67c2a3e4ac29463f39cf8e0c1823379b01 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sun, 22 Apr 2018 15:22:54 -0400 Subject: [PATCH 8/8] Updated README --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index dbe8f658..fef617fa 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,6 @@ A free, user-friendly, extendable application and [API](http://docs.webodm.org) * [Getting Started](#getting-started) * [Add More Processing Nodes](#add-more-processing-nodes) - * [Security](#security) * [Enable SSL](#enable-ssl) * [Where Are My Files Stored?](#where-are-my-files-stored) * [Common Troubleshooting](#common-troubleshooting)