diff --git a/scripts/daemon.py b/scripts/daemon.py index e982e3a5..0e04462a 100755 --- a/scripts/daemon.py +++ b/scripts/daemon.py @@ -63,8 +63,9 @@ class MapOSMaticDaemon: stalls the queue. """ - def __init__(self, frequency=_DEFAULT_POLL_FREQUENCY): + def __init__(self, frequency=_DEFAULT_POLL_FREQUENCY, queue_name="default"): self.frequency = frequency + self.queue_name = queue_name LOG.info("MapOSMatic rendering daemon started.") self.rollback_orphaned_jobs() @@ -89,7 +90,7 @@ class MapOSMaticDaemon: while True: try: - job = MapRenderingJob.objects.to_render()[0] + job = MapRenderingJob.objects.to_render(self.queue_name)[0] self.dispatch(job) # check disk space after rendering @@ -144,8 +145,8 @@ class MapOSMaticDaemon: class ForkingMapOSMaticDaemon(MapOSMaticDaemon): - def __init__(self, frequency=_DEFAULT_POLL_FREQUENCY): - MapOSMaticDaemon.__init__(self, frequency) + def __init__(self, frequency=_DEFAULT_POLL_FREQUENCY, queue_name="default"): + MapOSMaticDaemon.__init__(self, frequency, queue_name) LOG.info('This is the forking daemon. Will fork to process each job.') def get_renderer(self, job, prefix): @@ -273,7 +274,10 @@ if __name__ == '__main__': sys.exit(1) try: - daemon = ForkingMapOSMaticDaemon() + if len(sys.argv) == 2: + daemon = ForkingMapOSMaticDaemon(queue_name=sys.argv[1]) + else: + daemon = ForkingMapOSMaticDaemon() daemon.serve() except Exception as e: LOG.exception('Fatal error during daemon execution!') diff --git a/support/systemd/README.md b/support/systemd/README.md new file mode 100644 index 00000000..6e315ba3 --- /dev/null +++ b/support/systemd/README.md @@ -0,0 +1,18 @@ +Copy both .service files to /etc/systemd/system and edit them to change the +path to the wrapper.py script to the actual script location on your system. + +The maposmatic-render.service takes render requests from the "default" queue. + +You can start this service by simply invoking + + systemctl start maposmatic-render + +and enable it permanently using + + systemctl enable maposmatic-render + +Jobs from additional queues, like e.g. "api", can be rendered using the +maposmatic-render@.service template. To start a service for a specific +queue add the queue name after the @ sign, e.g.: + + systemctl start maposmatic-render@api diff --git a/support/systemd-maposmatic-render-service-template b/support/systemd/maposmatic-render.service similarity index 100% rename from support/systemd-maposmatic-render-service-template rename to support/systemd/maposmatic-render.service diff --git a/support/systemd/maposmatic-render@.service b/support/systemd/maposmatic-render@.service new file mode 100644 index 00000000..d99e46b2 --- /dev/null +++ b/support/systemd/maposmatic-render@.service @@ -0,0 +1,14 @@ +[Unit] +Description=Maposmatic render daemon processing queue: %I +After=multi-user.target +Wants=postgresql.service + +[Service] +Type=idle +User=maposmatic +ExecStart=/usr/bin/python /path/to/maposmatic/scripts/wrapper.py scripts/daemon.py %i + +[Install] +WantedBy=multi-user.target + + diff --git a/www/maposmatic/models.py b/www/maposmatic/models.py index adad44c3..bae9edec 100644 --- a/www/maposmatic/models.py +++ b/www/maposmatic/models.py @@ -45,11 +45,11 @@ def get_poi_file_path(instance, filename): return "" class MapRenderingJobManager(models.Manager): - def to_render(self): - return MapRenderingJob.objects.filter(status=0).order_by('submission_time') + def to_render(self, queue_name = 'default'): + return MapRenderingJob.objects.filter(status=0).filter(queue=queue_name).order_by('submission_time') - def queue_size(self): - return MapRenderingJob.objects.filter(status=0).count() + def queue_size(self, queue_name = 'default'): + return MapRenderingJob.objects.filter(status=0).filter(queue=queue_name).count() def get_by_filename(self, name): """Tries to find the parent MapRenderingJob of a given file from its @@ -314,17 +314,7 @@ class MapRenderingJob(models.Model): return None def current_position_in_queue(self): - return MapRenderingJob.objects.filter(status=0).filter(id__lte=self.id).count() - - # Estimate the date at which the rendering will be started - def rendering_estimated_start_time(self): - waiting_time = datetime.now() - self.submission_time - progression = self.index_queue_at_submission - self.current_position_in_queue() - if progression == 0: - return datetime.now() - mean_job_rendering_time = waiting_time // progression - estimated_time_left = mean_job_rendering_time * self.current_position_in_queue() - return datetime.now() + estimated_time_left + return MapRenderingJob.objects.filter(status=0).filter(queue=self.queue).filter(id__lte=self.id).count() def get_absolute_url(self): return reverse('map-by-id', args=[self.id]) diff --git a/www/maposmatic/templates/maposmatic/map-full-parts/rendering-status.html b/www/maposmatic/templates/maposmatic/map-full-parts/rendering-status.html index 087ec4a1..b5e0ad75 100644 --- a/www/maposmatic/templates/maposmatic/map-full-parts/rendering-status.html +++ b/www/maposmatic/templates/maposmatic/map-full-parts/rendering-status.html @@ -40,10 +40,11 @@
{% if map.needs_waiting %} -
-
- {{ map.current_position_in_queue }} / {{ queue_size }} -
+
+
+ {{ map.current_position_in_queue }} {% trans "jobs ahead of us" %} +
{% endif %} diff --git a/www/maposmatic/views.py b/www/maposmatic/views.py index d0db71fd..13816d79 100644 --- a/www/maposmatic/views.py +++ b/www/maposmatic/views.py @@ -167,7 +167,7 @@ def new(request): job.submitteremail = form.cleaned_data.get('submitteremail') job.map_language = form.cleaned_data.get('map_language') job.index_queue_at_submission = (models.MapRenderingJob.objects - .queue_size()) + .queue_size() + 1) job.nonce = helpers.generate_nonce(models.MapRenderingJob.NONCE_SIZE) job.save() @@ -252,11 +252,11 @@ def map_full(request, id, nonce=None): isredirected = request.session.get('redirected', False) request.session.pop('redirected', None) - queue_size = models.MapRenderingJob.objects.queue_size() + queue_size = job.index_queue_at_submission progress = 100 if queue_size: - progress = 20 + int(80 * (queue_size - - job.current_position_in_queue()) / float(queue_size)) + progress = int(100 * (queue_size - + job.current_position_in_queue()) / float(queue_size)) refresh = job.is_rendering() and \ www.settings.REFRESH_JOB_RENDERING or \ @@ -340,7 +340,7 @@ def recreate(request): newjob.submittermail = None # TODO newjob.map_language = job.map_language newjob.index_queue_at_submission = (models.MapRenderingJob.objects - .queue_size()) + .queue_size() + 1) newjob.nonce = helpers.generate_nonce(models.MapRenderingJob.NONCE_SIZE) newjob.save() @@ -635,11 +635,11 @@ def api_rendering_status(request, id, nonce=None): isredirected = request.session.get('redirected', False) request.session.pop('redirected', None) - queue_size = models.MapRenderingJob.objects.queue_size() + queue_size = job.index_queue_at_submission progress = 100 if queue_size: - progress = 20 + int(80 * (queue_size - - job.current_position_in_queue()) / float(queue_size)) + progress = int(100 * (queue_size - + job.current_position_in_queue()) / float(queue_size)) refresh = job.is_rendering() and \ www.settings.REFRESH_JOB_RENDERING or \