add support for multiple render queues (Issue #91)

* maprenderingjob model now has a "queue" name column,
  defaulting to "default"
* methods dealing with the rendering queue now all either
  use the jobs queue column value if already available
  or have an additional optional "queue_name" parameter
  with default value "default"
* a systemd template service file was added to be able to start
  individual per-queue render services
* the progress bar in the map request detail view was finally
  fixed to show actual position-in-queue progress

scripts/daemon.py

@@ -63,8 +63,9 @@ class MapOSMaticDaemon:
     stalls the queue.
     """
 
-    def __init__(self, frequency=_DEFAULT_POLL_FREQUENCY):
+    def __init__(self, frequency=_DEFAULT_POLL_FREQUENCY, queue_name="default"):
         self.frequency = frequency
+        self.queue_name = queue_name
         LOG.info("MapOSMatic rendering daemon started.")
         self.rollback_orphaned_jobs()
 
@@ -89,7 +90,7 @@ class MapOSMaticDaemon:
 
         while True:
             try:
-                job = MapRenderingJob.objects.to_render()[0]
+                job = MapRenderingJob.objects.to_render(self.queue_name)[0]
                 self.dispatch(job)
 
                 # check disk space after rendering 
@@ -144,8 +145,8 @@ class MapOSMaticDaemon:
 
 class ForkingMapOSMaticDaemon(MapOSMaticDaemon):
 
-    def __init__(self, frequency=_DEFAULT_POLL_FREQUENCY):
-        MapOSMaticDaemon.__init__(self, frequency)
+    def __init__(self, frequency=_DEFAULT_POLL_FREQUENCY, queue_name="default"):
+        MapOSMaticDaemon.__init__(self, frequency, queue_name)
         LOG.info('This is the forking daemon. Will fork to process each job.')
 
     def get_renderer(self, job, prefix):
@@ -273,7 +274,10 @@ if __name__ == '__main__':
         sys.exit(1)
 
     try:
-        daemon = ForkingMapOSMaticDaemon()
+        if len(sys.argv) == 2:
+            daemon = ForkingMapOSMaticDaemon(queue_name=sys.argv[1])
+        else:
+            daemon = ForkingMapOSMaticDaemon()
         daemon.serve()
     except Exception as e:
         LOG.exception('Fatal error during daemon execution!')

support/systemd/README.md

@@ -0,0 +1,18 @@
+Copy both .service files to /etc/systemd/system and edit them to change the 
+path to the wrapper.py script to the actual script location on your system.
+
+The maposmatic-render.service takes render requests from the "default" queue.
+
+You can start this service by simply invoking
+
+  systemctl start maposmatic-render
+
+and enable it permanently using
+
+  systemctl enable maposmatic-render
+
+Jobs from additional queues, like e.g. "api", can be rendered using the
+maposmatic-render@.service template. To start a service for a specific
+queue add the queue name after the @ sign, e.g.:
+
+  systemctl start maposmatic-render@api

support/systemd/maposmatic-render@.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Maposmatic render daemon processing queue: %I
+After=multi-user.target
+Wants=postgresql.service
+
+[Service]
+Type=idle
+User=maposmatic
+ExecStart=/usr/bin/python /path/to/maposmatic/scripts/wrapper.py scripts/daemon.py %i
+
+[Install]
+WantedBy=multi-user.target
+
+

www/maposmatic/models.py

@@ -45,11 +45,11 @@ def get_poi_file_path(instance, filename):
     return ""
 
 class MapRenderingJobManager(models.Manager):
-    def to_render(self):
-        return MapRenderingJob.objects.filter(status=0).order_by('submission_time')
+    def to_render(self, queue_name = 'default'):
+        return MapRenderingJob.objects.filter(status=0).filter(queue=queue_name).order_by('submission_time')
 
-    def queue_size(self):
-        return MapRenderingJob.objects.filter(status=0).count()
+    def queue_size(self, queue_name = 'default'):
+        return MapRenderingJob.objects.filter(status=0).filter(queue=queue_name).count()
 
     def get_by_filename(self, name):
         """Tries to find the parent MapRenderingJob of a given file from its
@@ -314,17 +314,7 @@ class MapRenderingJob(models.Model):
         return None
 
     def current_position_in_queue(self):
-        return MapRenderingJob.objects.filter(status=0).filter(id__lte=self.id).count()
-
-    # Estimate the date at which the rendering will be started
-    def rendering_estimated_start_time(self):
-        waiting_time = datetime.now() - self.submission_time
-        progression = self.index_queue_at_submission - self.current_position_in_queue()
-        if progression == 0:
-            return datetime.now()
-        mean_job_rendering_time = waiting_time // progression
-        estimated_time_left = mean_job_rendering_time * self.current_position_in_queue()
-        return datetime.now() + estimated_time_left
+        return MapRenderingJob.objects.filter(status=0).filter(queue=self.queue).filter(id__lte=self.id).count()
 
     def get_absolute_url(self):
         return reverse('map-by-id', args=[self.id])

www/maposmatic/templates/maposmatic/map-full-parts/rendering-status.html

@@ -40,10 +40,11 @@
       <div style="height: 2em"></div>
 
     {% if map.needs_waiting %}
-      <div id="queue-progress" class="progress progress-striped active">
-        <div class="bar" style="width: {{ progress }}%; text-align: right; padding-right: 10px;">
-          {{ map.current_position_in_queue }} / {{ queue_size }}
-        </div>
+      <div class="progress">
+        <div id="queue-progress" class="progress-bar progress-bar-striped progress-bar-animated"" role="progressbar"
+	     style="width: {{ progress }}%" aria-valuenow="{{ progress }}" aria-valuemin="0" aria-valuemax="100">
+	  <span class="text-dark font-weight-bold">{{ map.current_position_in_queue }} {% trans "jobs ahead of us" %}</span>
+	</div>
       </div>
     {% endif %}
 

www/maposmatic/views.py

@@ -167,7 +167,7 @@ def new(request):
             job.submitteremail = form.cleaned_data.get('submitteremail')
             job.map_language = form.cleaned_data.get('map_language')
             job.index_queue_at_submission = (models.MapRenderingJob.objects
-                                             .queue_size())
+                                             .queue_size() + 1)
             job.nonce = helpers.generate_nonce(models.MapRenderingJob.NONCE_SIZE)
 
             job.save()
@@ -252,11 +252,11 @@ def map_full(request, id, nonce=None):
     isredirected = request.session.get('redirected', False)
     request.session.pop('redirected', None)
 
-    queue_size = models.MapRenderingJob.objects.queue_size()
+    queue_size = job.index_queue_at_submission
     progress = 100
     if queue_size:
-        progress = 20 + int(80 * (queue_size -
-            job.current_position_in_queue()) / float(queue_size))
+       progress = int(100 * (queue_size -
+           job.current_position_in_queue()) / float(queue_size))
 
     refresh = job.is_rendering() and \
         www.settings.REFRESH_JOB_RENDERING or \
@@ -340,7 +340,7 @@ def recreate(request):
             newjob.submittermail = None # TODO
             newjob.map_language = job.map_language
             newjob.index_queue_at_submission = (models.MapRenderingJob.objects
-                                               .queue_size())
+                                                .queue_size() + 1)
             newjob.nonce = helpers.generate_nonce(models.MapRenderingJob.NONCE_SIZE)
 
             newjob.save()
@@ -635,11 +635,11 @@ def api_rendering_status(request, id, nonce=None):
     isredirected = request.session.get('redirected', False)
     request.session.pop('redirected', None)
 
-    queue_size = models.MapRenderingJob.objects.queue_size()
+    queue_size = job.index_queue_at_submission
     progress = 100
     if queue_size:
-        progress = 20 + int(80 * (queue_size -
-            job.current_position_in_queue()) / float(queue_size))
+       progress = int(100 * (queue_size -
+           job.current_position_in_queue()) / float(queue_size))
 
     refresh = job.is_rendering() and \
         www.settings.REFRESH_JOB_RENDERING or \