Add ability to set maximum worker time limits

app/plugins/worker.py

@@ -1,5 +1,6 @@
 import inspect
 from worker.celery import app
+from webodm import settings
 
 task = app.task
 
@@ -15,7 +16,7 @@ def run_function_async(func, *args, **kwargs):
     return eval_async.delay(source, func.__name__, *args, **kwargs)
 
 
-@app.task(bind=True)
+@app.task(bind=True, time_limit=settings.WORKERS_MAX_TIME_LIMIT)
 def eval_async(self, source, funcname, *args, **kwargs):
     """
     Run Python code asynchronously using Celery.

webodm/settings.py

@@ -398,6 +398,9 @@ CLEANUP_EMPTY_PROJECTS = None
 # Maximum number of threads that a worker should use for processing
 WORKERS_MAX_THREADS = 1
 
+# Maximum number of seconds a worker task should take before being terminated
+WORKERS_MAX_TIME_LIMIT = 2
+
 # Link to GCP docs
 GCP_DOCS_LINK = "https://docs.opendronemap.org/gcp/#gcp-file-format"
 

worker/tasks.py

@@ -123,7 +123,7 @@ def setInterval(interval, func, *args):
     t.start()
     return stopped.set
 
-@app.task(ignore_result=True)
+@app.task(ignore_result=True, time_limit=settings.WORKERS_MAX_TIME_LIMIT)
 def process_task(taskId):
     lock_id = 'task_lock_{}'.format(taskId)
     cancel_monitor = None
@@ -190,7 +190,7 @@ def process_pending_tasks():
         process_task.delay(task.id)
 
 
-@app.task(bind=True)
+@app.task(bind=True, time_limit=settings.WORKERS_MAX_TIME_LIMIT)
 def export_raster(self, input, **opts):
     try:
         logger.info("Exporting raster {} with options: {}".format(input, json.dumps(opts)))
@@ -210,7 +210,7 @@ def export_raster(self, input, **opts):
         logger.error(str(e))
         return {'error': str(e)}
 
-@app.task(bind=True)
+@app.task(bind=True, time_limit=settings.WORKERS_MAX_TIME_LIMIT)
 def export_pointcloud(self, input, **opts):
     try:
         logger.info("Exporting point cloud {} with options: {}".format(input, json.dumps(opts)))