kopia lustrzana https://github.com/OpenDroneMap/WebODM
Task locking monitor
rodzic
9558cc804c
commit
5a5ba195e9
|
@ -3,6 +3,7 @@ import shutil
|
||||||
import traceback
|
import traceback
|
||||||
|
|
||||||
import time
|
import time
|
||||||
|
from threading import Event, Thread
|
||||||
from celery.utils.log import get_task_logger
|
from celery.utils.log import get_task_logger
|
||||||
from django.core.exceptions import ObjectDoesNotExist
|
from django.core.exceptions import ObjectDoesNotExist
|
||||||
from django.db.models import Count
|
from django.db.models import Count
|
||||||
|
@ -58,21 +59,42 @@ def cleanup_tmp_directory():
|
||||||
logger.info('Cleaned up: %s (%s)' % (f, modified))
|
logger.info('Cleaned up: %s (%s)' % (f, modified))
|
||||||
|
|
||||||
|
|
||||||
|
# Based on https://stackoverflow.com/questions/22498038/improve-current-implementation-of-a-setinterval-python/22498708#22498708
|
||||||
|
def setInterval(interval, func, *args):
|
||||||
|
stopped = Event()
|
||||||
|
def loop():
|
||||||
|
while not stopped.wait(interval):
|
||||||
|
func(*args)
|
||||||
|
t = Thread(target=loop)
|
||||||
|
t.daemon = True
|
||||||
|
t.start()
|
||||||
|
return stopped.set
|
||||||
|
|
||||||
@app.task
|
@app.task
|
||||||
def process_task(taskId):
|
def process_task(taskId):
|
||||||
have_lock = False
|
lock_id = 'task_lock_{}'.format(taskId)
|
||||||
|
cancel_monitor = None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
lock = redis_client.lock('task_lock_{}'.format(taskId))
|
task_lock_last_update = redis_client.getset(lock_id, time.time())
|
||||||
have_lock = lock.acquire(blocking=False)
|
if task_lock_last_update is not None:
|
||||||
|
# Check if lock has expired
|
||||||
|
if time.time() - float(task_lock_last_update) <= 30:
|
||||||
|
# Locked
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
# Expired
|
||||||
|
logger.warning("Task {} has an expired lock! This could mean that WebODM is running out of memory. Check your server configuration.")
|
||||||
|
|
||||||
if not have_lock:
|
# Set lock
|
||||||
return
|
def update_lock():
|
||||||
|
redis_client.set(lock_id, time.time())
|
||||||
|
cancel_monitor = setInterval(10, update_lock)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
task = Task.objects.get(pk=taskId)
|
task = Task.objects.get(pk=taskId)
|
||||||
except ObjectDoesNotExist:
|
except ObjectDoesNotExist:
|
||||||
logger.info("Task id {} has already been deleted.".format(taskId))
|
logger.info("Task {} has already been deleted.".format(taskId))
|
||||||
return
|
return
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
@ -84,12 +106,14 @@ def process_task(taskId):
|
||||||
if settings.TESTING: raise e
|
if settings.TESTING: raise e
|
||||||
finally:
|
finally:
|
||||||
try:
|
try:
|
||||||
if have_lock:
|
redis_client.delete(lock_id)
|
||||||
lock.release()
|
except redis.exceptions.RedisError:
|
||||||
except redis.exceptions.LockError:
|
# Ignore errors, the lock will expire at some point
|
||||||
# A lock could have expired
|
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
if cancel_monitor is not None:
|
||||||
|
cancel_monitor()
|
||||||
|
|
||||||
def get_pending_tasks():
|
def get_pending_tasks():
|
||||||
# All tasks that have a processing node assigned
|
# All tasks that have a processing node assigned
|
||||||
# Or that need one assigned (via auto)
|
# Or that need one assigned (via auto)
|
||||||
|
|
Ładowanie…
Reference in New Issue