From 8317283b8767a5d55002b6fcc96563aad9940fdd Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sun, 22 Apr 2018 14:49:20 -0400 Subject: [PATCH] Faster timeout on update_node_info(), oom preferences --- docker-compose.nodeodm.yml | 1 + docker-compose.yml | 6 ++++++ nodeodm/api_client.py | 19 +++++++++---------- nodeodm/models.py | 6 +++--- webodm/settings.py | 1 + worker.sh | 2 +- worker/tasks.py | 1 - 7 files changed, 21 insertions(+), 15 deletions(-) diff --git a/docker-compose.nodeodm.yml b/docker-compose.nodeodm.yml index 3de13079..0b2a4d6b 100644 --- a/docker-compose.nodeodm.yml +++ b/docker-compose.nodeodm.yml @@ -14,3 +14,4 @@ services: ports: - "3000" restart: on-failure:10 + oom_score_adj: 500 diff --git a/docker-compose.yml b/docker-compose.yml index 4609f14b..84c6f2e9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,6 +13,7 @@ services: volumes: - dbdata:/var/lib/postgresql/data restart: on-failure:10 + oom_score_adj: -100 webapp: image: opendronemap/webodm_webapp container_name: webapp @@ -31,10 +32,14 @@ services: - WO_DEBUG - WO_BROKER restart: on-failure:10 + oom_score_adj: 0 broker: image: redis container_name: broker restart: on-failure:10 + mem_limit: 256m + oom_kill_disable: true + oom_score_adj: -500 worker: image: opendronemap/webodm_webapp container_name: worker @@ -48,3 +53,4 @@ services: - WO_BROKER - WO_DEBUG restart: on-failure:10 + oom_score_adj: 250 diff --git a/nodeodm/api_client.py b/nodeodm/api_client.py index 42285c44..82d07ead 100644 --- a/nodeodm/api_client.py +++ b/nodeodm/api_client.py @@ -9,12 +9,11 @@ import os from urllib.parse import urlunparse from app.testwatch import TestWatch -TIMEOUT = 30 - class ApiClient: - def __init__(self, host, port): + def __init__(self, host, port, timeout=30): self.host = host self.port = port + self.timeout = timeout def url(self, url): netloc = self.host if self.port == 80 else "{}:{}".format(self.host, self.port) @@ -23,28 +22,28 @@ class ApiClient: return urlunparse(('http', netloc, url, '', '', '')) def info(self): - return requests.get(self.url('/info'), timeout=TIMEOUT).json() + return requests.get(self.url('/info'), timeout=self.timeout).json() def options(self): - return requests.get(self.url('/options'), timeout=TIMEOUT).json() + return requests.get(self.url('/options'), timeout=self.timeout).json() def task_info(self, uuid): - return requests.get(self.url('/task/{}/info').format(uuid), timeout=TIMEOUT).json() + return requests.get(self.url('/task/{}/info').format(uuid), timeout=self.timeout).json() @TestWatch.watch() def task_output(self, uuid, line = 0): - return requests.get(self.url('/task/{}/output?line={}').format(uuid, line), timeout=TIMEOUT).json() + return requests.get(self.url('/task/{}/output?line={}').format(uuid, line), timeout=self.timeout).json() def task_cancel(self, uuid): - return requests.post(self.url('/task/cancel'), data={'uuid': uuid}, timeout=TIMEOUT).json() + return requests.post(self.url('/task/cancel'), data={'uuid': uuid}, timeout=self.timeout).json() def task_remove(self, uuid): - return requests.post(self.url('/task/remove'), data={'uuid': uuid}, timeout=TIMEOUT).json() + return requests.post(self.url('/task/remove'), data={'uuid': uuid}, timeout=self.timeout).json() def task_restart(self, uuid, options = None): data = {'uuid': uuid} if options is not None: data['options'] = json.dumps(options) - return requests.post(self.url('/task/restart'), data=data, timeout=TIMEOUT).json() + return requests.post(self.url('/task/restart'), data=data, timeout=self.timeout).json() def task_download(self, uuid, asset): res = requests.get(self.url('/task/{}/download/{}').format(uuid, asset), stream=True) diff --git a/nodeodm/models.py b/nodeodm/models.py index 7ccad56b..514054f7 100644 --- a/nodeodm/models.py +++ b/nodeodm/models.py @@ -66,7 +66,7 @@ class ProcessingNode(models.Model): :returns: True if information could be updated, False otherwise """ - api_client = self.api_client() + api_client = self.api_client(timeout=5) try: info = api_client.info() self.api_version = info['version'] @@ -80,8 +80,8 @@ class ProcessingNode(models.Model): except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, json.decoder.JSONDecodeError, simplejson.JSONDecodeError): return False - def api_client(self): - return ApiClient(self.hostname, self.port) + def api_client(self, timeout=30): + return ApiClient(self.hostname, self.port, timeout) def get_available_options_json(self, pretty=False): """ diff --git a/webodm/settings.py b/webodm/settings.py index a4f02666..4465a39b 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -326,6 +326,7 @@ CELERY_ACCEPT_CONTENT = ['json'] CELERY_INCLUDE=['worker.tasks'] CELERY_WORKER_REDIRECT_STDOUTS = False CELERY_WORKER_HIJACK_ROOT_LOGGER = False +CELERY_TASK_RESULT_EXPIRES = 60 if TESTING: CELERY_TASK_ALWAYS_EAGER = True diff --git a/worker.sh b/worker.sh index 147b1b21..d471b623 100755 --- a/worker.sh +++ b/worker.sh @@ -52,7 +52,7 @@ start(){ action=$1 echo "Starting worker using broker at $WO_BROKER" - celery -A worker worker --concurrency 1 --max-tasks-per-child 1000 --loglevel=warn > /dev/null + celery -A worker worker --autoscale $(grep -c '^processor' /proc/cpuinfo),2 --max-tasks-per-child 1000 --loglevel=warn > /dev/null } start_scheduler(){ diff --git a/worker/tasks.py b/worker/tasks.py index 13228557..35359543 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -79,7 +79,6 @@ def get_pending_tasks(): @app.task def process_pending_tasks(): tasks = get_pending_tasks() - for task in tasks: process_task.delay(task.id)