diff --git a/app/plugins/worker.py b/app/plugins/worker.py index 4e4a92e3..159cd01c 100644 --- a/app/plugins/worker.py +++ b/app/plugins/worker.py @@ -1,5 +1,6 @@ import inspect from worker.celery import app +from webodm import settings task = app.task @@ -15,7 +16,7 @@ def run_function_async(func, *args, **kwargs): return eval_async.delay(source, func.__name__, *args, **kwargs) -@app.task(bind=True) +@app.task(bind=True, time_limit=settings.WORKERS_MAX_TIME_LIMIT) def eval_async(self, source, funcname, *args, **kwargs): """ Run Python code asynchronously using Celery. diff --git a/webodm/settings.py b/webodm/settings.py index 5b4e3fdf..435fbf9f 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -398,6 +398,9 @@ CLEANUP_EMPTY_PROJECTS = None # Maximum number of threads that a worker should use for processing WORKERS_MAX_THREADS = 1 +# Maximum number of seconds a worker task should take before being terminated +WORKERS_MAX_TIME_LIMIT = None + # Link to GCP docs GCP_DOCS_LINK = "https://docs.opendronemap.org/gcp/#gcp-file-format" diff --git a/worker/tasks.py b/worker/tasks.py index 1cf5d8fc..cd7c5a35 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -123,7 +123,7 @@ def setInterval(interval, func, *args): t.start() return stopped.set -@app.task(ignore_result=True) +@app.task(ignore_result=True, time_limit=settings.WORKERS_MAX_TIME_LIMIT) def process_task(taskId): lock_id = 'task_lock_{}'.format(taskId) cancel_monitor = None @@ -190,7 +190,7 @@ def process_pending_tasks(): process_task.delay(task.id) -@app.task(bind=True) +@app.task(bind=True, time_limit=settings.WORKERS_MAX_TIME_LIMIT) def export_raster(self, input, **opts): try: logger.info("Exporting raster {} with options: {}".format(input, json.dumps(opts))) @@ -210,7 +210,7 @@ def export_raster(self, input, **opts): logger.error(str(e)) return {'error': str(e)} -@app.task(bind=True) +@app.task(bind=True, time_limit=settings.WORKERS_MAX_TIME_LIMIT) def export_pointcloud(self, input, **opts): try: logger.info("Exporting point cloud {} with options: {}".format(input, json.dumps(opts)))