kopia lustrzana https://github.com/hholzgra/maposmatic/
add support for multiple render queues (Issue #91)
* maprenderingjob model now has a "queue" name column, defaulting to "default" * methods dealing with the rendering queue now all either use the jobs queue column value if already available or have an additional optional "queue_name" parameter with default value "default" * a systemd template service file was added to be able to start individual per-queue render services * the progress bar in the map request detail view was finally fixed to show actual position-in-queue progresssite-osm-baustelle
rodzic
7bafbe8aaa
commit
e05711be43
|
@ -63,8 +63,9 @@ class MapOSMaticDaemon:
|
||||||
stalls the queue.
|
stalls the queue.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, frequency=_DEFAULT_POLL_FREQUENCY):
|
def __init__(self, frequency=_DEFAULT_POLL_FREQUENCY, queue_name="default"):
|
||||||
self.frequency = frequency
|
self.frequency = frequency
|
||||||
|
self.queue_name = queue_name
|
||||||
LOG.info("MapOSMatic rendering daemon started.")
|
LOG.info("MapOSMatic rendering daemon started.")
|
||||||
self.rollback_orphaned_jobs()
|
self.rollback_orphaned_jobs()
|
||||||
|
|
||||||
|
@ -89,7 +90,7 @@ class MapOSMaticDaemon:
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
job = MapRenderingJob.objects.to_render()[0]
|
job = MapRenderingJob.objects.to_render(self.queue_name)[0]
|
||||||
self.dispatch(job)
|
self.dispatch(job)
|
||||||
|
|
||||||
# check disk space after rendering
|
# check disk space after rendering
|
||||||
|
@ -144,8 +145,8 @@ class MapOSMaticDaemon:
|
||||||
|
|
||||||
class ForkingMapOSMaticDaemon(MapOSMaticDaemon):
|
class ForkingMapOSMaticDaemon(MapOSMaticDaemon):
|
||||||
|
|
||||||
def __init__(self, frequency=_DEFAULT_POLL_FREQUENCY):
|
def __init__(self, frequency=_DEFAULT_POLL_FREQUENCY, queue_name="default"):
|
||||||
MapOSMaticDaemon.__init__(self, frequency)
|
MapOSMaticDaemon.__init__(self, frequency, queue_name)
|
||||||
LOG.info('This is the forking daemon. Will fork to process each job.')
|
LOG.info('This is the forking daemon. Will fork to process each job.')
|
||||||
|
|
||||||
def get_renderer(self, job, prefix):
|
def get_renderer(self, job, prefix):
|
||||||
|
@ -273,7 +274,10 @@ if __name__ == '__main__':
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
daemon = ForkingMapOSMaticDaemon()
|
if len(sys.argv) == 2:
|
||||||
|
daemon = ForkingMapOSMaticDaemon(queue_name=sys.argv[1])
|
||||||
|
else:
|
||||||
|
daemon = ForkingMapOSMaticDaemon()
|
||||||
daemon.serve()
|
daemon.serve()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
LOG.exception('Fatal error during daemon execution!')
|
LOG.exception('Fatal error during daemon execution!')
|
||||||
|
|
|
@ -0,0 +1,18 @@
|
||||||
|
Copy both .service files to /etc/systemd/system and edit them to change the
|
||||||
|
path to the wrapper.py script to the actual script location on your system.
|
||||||
|
|
||||||
|
The maposmatic-render.service takes render requests from the "default" queue.
|
||||||
|
|
||||||
|
You can start this service by simply invoking
|
||||||
|
|
||||||
|
systemctl start maposmatic-render
|
||||||
|
|
||||||
|
and enable it permanently using
|
||||||
|
|
||||||
|
systemctl enable maposmatic-render
|
||||||
|
|
||||||
|
Jobs from additional queues, like e.g. "api", can be rendered using the
|
||||||
|
maposmatic-render@.service template. To start a service for a specific
|
||||||
|
queue add the queue name after the @ sign, e.g.:
|
||||||
|
|
||||||
|
systemctl start maposmatic-render@api
|
|
@ -0,0 +1,14 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Maposmatic render daemon processing queue: %I
|
||||||
|
After=multi-user.target
|
||||||
|
Wants=postgresql.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=idle
|
||||||
|
User=maposmatic
|
||||||
|
ExecStart=/usr/bin/python /path/to/maposmatic/scripts/wrapper.py scripts/daemon.py %i
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
|
||||||
|
|
|
@ -45,11 +45,11 @@ def get_poi_file_path(instance, filename):
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
class MapRenderingJobManager(models.Manager):
|
class MapRenderingJobManager(models.Manager):
|
||||||
def to_render(self):
|
def to_render(self, queue_name = 'default'):
|
||||||
return MapRenderingJob.objects.filter(status=0).order_by('submission_time')
|
return MapRenderingJob.objects.filter(status=0).filter(queue=queue_name).order_by('submission_time')
|
||||||
|
|
||||||
def queue_size(self):
|
def queue_size(self, queue_name = 'default'):
|
||||||
return MapRenderingJob.objects.filter(status=0).count()
|
return MapRenderingJob.objects.filter(status=0).filter(queue=queue_name).count()
|
||||||
|
|
||||||
def get_by_filename(self, name):
|
def get_by_filename(self, name):
|
||||||
"""Tries to find the parent MapRenderingJob of a given file from its
|
"""Tries to find the parent MapRenderingJob of a given file from its
|
||||||
|
@ -314,17 +314,7 @@ class MapRenderingJob(models.Model):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def current_position_in_queue(self):
|
def current_position_in_queue(self):
|
||||||
return MapRenderingJob.objects.filter(status=0).filter(id__lte=self.id).count()
|
return MapRenderingJob.objects.filter(status=0).filter(queue=self.queue).filter(id__lte=self.id).count()
|
||||||
|
|
||||||
# Estimate the date at which the rendering will be started
|
|
||||||
def rendering_estimated_start_time(self):
|
|
||||||
waiting_time = datetime.now() - self.submission_time
|
|
||||||
progression = self.index_queue_at_submission - self.current_position_in_queue()
|
|
||||||
if progression == 0:
|
|
||||||
return datetime.now()
|
|
||||||
mean_job_rendering_time = waiting_time // progression
|
|
||||||
estimated_time_left = mean_job_rendering_time * self.current_position_in_queue()
|
|
||||||
return datetime.now() + estimated_time_left
|
|
||||||
|
|
||||||
def get_absolute_url(self):
|
def get_absolute_url(self):
|
||||||
return reverse('map-by-id', args=[self.id])
|
return reverse('map-by-id', args=[self.id])
|
||||||
|
|
|
@ -40,10 +40,11 @@
|
||||||
<div style="height: 2em"></div>
|
<div style="height: 2em"></div>
|
||||||
|
|
||||||
{% if map.needs_waiting %}
|
{% if map.needs_waiting %}
|
||||||
<div id="queue-progress" class="progress progress-striped active">
|
<div class="progress">
|
||||||
<div class="bar" style="width: {{ progress }}%; text-align: right; padding-right: 10px;">
|
<div id="queue-progress" class="progress-bar progress-bar-striped progress-bar-animated"" role="progressbar"
|
||||||
{{ map.current_position_in_queue }} / {{ queue_size }}
|
style="width: {{ progress }}%" aria-valuenow="{{ progress }}" aria-valuemin="0" aria-valuemax="100">
|
||||||
</div>
|
<span class="text-dark font-weight-bold">{{ map.current_position_in_queue }} {% trans "jobs ahead of us" %}</span>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
|
|
|
@ -167,7 +167,7 @@ def new(request):
|
||||||
job.submitteremail = form.cleaned_data.get('submitteremail')
|
job.submitteremail = form.cleaned_data.get('submitteremail')
|
||||||
job.map_language = form.cleaned_data.get('map_language')
|
job.map_language = form.cleaned_data.get('map_language')
|
||||||
job.index_queue_at_submission = (models.MapRenderingJob.objects
|
job.index_queue_at_submission = (models.MapRenderingJob.objects
|
||||||
.queue_size())
|
.queue_size() + 1)
|
||||||
job.nonce = helpers.generate_nonce(models.MapRenderingJob.NONCE_SIZE)
|
job.nonce = helpers.generate_nonce(models.MapRenderingJob.NONCE_SIZE)
|
||||||
|
|
||||||
job.save()
|
job.save()
|
||||||
|
@ -252,11 +252,11 @@ def map_full(request, id, nonce=None):
|
||||||
isredirected = request.session.get('redirected', False)
|
isredirected = request.session.get('redirected', False)
|
||||||
request.session.pop('redirected', None)
|
request.session.pop('redirected', None)
|
||||||
|
|
||||||
queue_size = models.MapRenderingJob.objects.queue_size()
|
queue_size = job.index_queue_at_submission
|
||||||
progress = 100
|
progress = 100
|
||||||
if queue_size:
|
if queue_size:
|
||||||
progress = 20 + int(80 * (queue_size -
|
progress = int(100 * (queue_size -
|
||||||
job.current_position_in_queue()) / float(queue_size))
|
job.current_position_in_queue()) / float(queue_size))
|
||||||
|
|
||||||
refresh = job.is_rendering() and \
|
refresh = job.is_rendering() and \
|
||||||
www.settings.REFRESH_JOB_RENDERING or \
|
www.settings.REFRESH_JOB_RENDERING or \
|
||||||
|
@ -340,7 +340,7 @@ def recreate(request):
|
||||||
newjob.submittermail = None # TODO
|
newjob.submittermail = None # TODO
|
||||||
newjob.map_language = job.map_language
|
newjob.map_language = job.map_language
|
||||||
newjob.index_queue_at_submission = (models.MapRenderingJob.objects
|
newjob.index_queue_at_submission = (models.MapRenderingJob.objects
|
||||||
.queue_size())
|
.queue_size() + 1)
|
||||||
newjob.nonce = helpers.generate_nonce(models.MapRenderingJob.NONCE_SIZE)
|
newjob.nonce = helpers.generate_nonce(models.MapRenderingJob.NONCE_SIZE)
|
||||||
|
|
||||||
newjob.save()
|
newjob.save()
|
||||||
|
@ -635,11 +635,11 @@ def api_rendering_status(request, id, nonce=None):
|
||||||
isredirected = request.session.get('redirected', False)
|
isredirected = request.session.get('redirected', False)
|
||||||
request.session.pop('redirected', None)
|
request.session.pop('redirected', None)
|
||||||
|
|
||||||
queue_size = models.MapRenderingJob.objects.queue_size()
|
queue_size = job.index_queue_at_submission
|
||||||
progress = 100
|
progress = 100
|
||||||
if queue_size:
|
if queue_size:
|
||||||
progress = 20 + int(80 * (queue_size -
|
progress = int(100 * (queue_size -
|
||||||
job.current_position_in_queue()) / float(queue_size))
|
job.current_position_in_queue()) / float(queue_size))
|
||||||
|
|
||||||
refresh = job.is_rendering() and \
|
refresh = job.is_rendering() and \
|
||||||
www.settings.REFRESH_JOB_RENDERING or \
|
www.settings.REFRESH_JOB_RENDERING or \
|
||||||
|
|
Ładowanie…
Reference in New Issue