diff --git a/scripts/render.py b/scripts/render.py index 7f3dd68a..2aaa8779 100755 --- a/scripts/render.py +++ b/scripts/render.py @@ -40,6 +40,8 @@ from www.maposmatic.models import MapRenderingJob from www.settings import ADMINS, OCITYSMAP_CFG_PATH, MEDIA_ROOT from www.settings import RENDERING_RESULT_PATH, RENDERING_RESULT_FORMATS from www.settings import DAEMON_ERRORS_SMTP_HOST, DAEMON_ERRORS_SMTP_PORT +from www.settings import DAEMON_ERRORS_SMTP_ENCRYPT +from www.settings import DAEMON_ERRORS_SMTP_USER, DAEMON_ERRORS_SMTP_PASSWORD from www.settings import DAEMON_ERRORS_EMAIL_FROM from www.settings import DAEMON_ERRORS_EMAIL_REPLY_TO from www.settings import DAEMON_ERRORS_JOB_URL @@ -73,6 +75,76 @@ You can view the job page at <%(url)s>. MapOSMatic """ +SUCCESS_EMAIL_TEMPLATE = """From: MapOSMatic rendering daemon <%(from)s> +Reply-To: %(replyto)s +To: $(to)s +Content-Type: text/plain; charset=utf-8 +Content-Transfer-Encoding: 8bit +Subject: Rendering of job #%(jobid)d succeeded +Date: %(date)s + +Hello %(to)s, + +your map rendering request for + + %(title)s + +has successfully been processed now, and the results can be downloaded +from the rendering jobs detail pages: + + %(url)s + +-- +MapOSMatic""" + + +FAILURE_EMAIL_TEMPLATE = """From: MapOSMatic rendering daemon <%(from)s> +Reply-To: %(replyto)s +To: $(to)s +Content-Type: text/plain; charset=utf-8 +Content-Transfer-Encoding: 8bit +Subject: Rendering of job #%(jobid)d failed +Date: %(date)s + +Hello %(to)s, + +unfortunately your map rendering request for + + %(title)s + +has failed. + +You can check for failure details on the request detail page: + + %(url)s + +-- +MapOSMatic""" + + +TIMEOUT_EMAIL_TEMPLATE = """From: MapOSMatic rendering daemon <%(from)s> +Reply-To: %(replyto)s +To: $(to)s +Content-Type: text/plain; charset=utf-8 +Content-Transfer-Encoding: 8bit +Subject: Rendering of job #%(jobid)d timed out +Date: %(date)s + +Hello %(to)s, + +unfortunately your map rendering request for + + %(title)s + +has been runnning for more than %(timeout)d minutes and had to be cancelled. + +You may want to retry with a smaller map area or with a less complex map +style or less map overlays. + +-- +MapOSMatic""" + + l = logging.getLogger('maposmatic') class ThreadingJobRenderer: @@ -95,6 +167,45 @@ class ThreadingJobRenderer: self.__timeout = timeout self.__thread = JobRenderer(job, prefix) + def _email_timeout(self): + """Send a notification about timeouts to the request submitter""" + + if not DAEMON_ERRORS_SMTP_HOST or not self.__job.submittermail: + return + + try: + l.info("Emailing timeout message to %s via %s:%d..." % + (self.__job.submittermail, + DAEMON_ERRORS_SMTP_HOST, + DAEMON_ERRORS_SMTP_PORT)) + + if DAEMON_ERRORS_SMTP_ENCRYPT == "SSL": + mailer = smtplib.SMTP_SSL() + else: + mailer = smtplib.SMTP() + mailer.connect(DAEMON_ERRORS_SMTP_HOST, DAEMON_ERRORS_SMTP_PORT) + if DAEMON_ERRORS_SMTP_ENCRYPT == "TLS": + mailer.starttls() + if DAEMON_ERRORS_SMTP_USER and DAEMON_ERRORS_SMTP_PASSWORD: + mailer.login(DAEMON_ERRORS_SMTP_USER, DAEMON_ERRORS_SMTP_PASSWORD) + + msg = TIMEOUT_EMAIL_TEMPLATE % \ + { 'from': DAEMON_ERRORS_EMAIL_FROM, + 'replyto': DAEMON_ERRORS_EMAIL_REPLY_TO, + 'to': self.__job.submittermail, + 'jobid': self.__job.id, + 'date': datetime.datetime.now().strftime('%a, %d %b %Y %H:%M:%S %Z'), + 'url': DAEMON_ERRORS_JOB_URL % self.__job.id, + 'title': self.__job.maptitle, + 'timeout': self.__timeout / 60 + } + + mailer.sendmail(DAEMON_ERRORS_EMAIL_FROM, + [admin[1] for admin in ADMINS], msg) + l.info("Email notification sent.") + except Exception, e: + l.exception("Could not send notification email to the submitter!") + def run(self): """Renders the job using a JobRendered, encapsulating all processing errors and exceptions, with the addition here of a processing timeout. @@ -102,6 +213,8 @@ class ThreadingJobRenderer: Returns one of the RESULT_ constants. """ + l.info("Timeout is %d" % self.__timeout) + self.__thread.start() self.__thread.join(self.__timeout) @@ -122,6 +235,8 @@ class ThreadingJobRenderer: # Remove the job files self.__job.remove_all_files() + self._email_timeout() + l.debug("Worker removed.") return RESULT_TIMEOUT_REACHED @@ -134,6 +249,45 @@ class ForkingJobRenderer: self.__renderer = JobRenderer(job, prefix) self.__process = multiprocessing.Process(target=self._wrap) + def _email_timeout(self): + """Send a notification about timeouts to the request submitter""" + + if not DAEMON_ERRORS_SMTP_HOST or not self.__job.submittermail: + return + + try: + l.info("Emailing timeout message to %s via %s:%d..." % + (self.__job.submittermail, + DAEMON_ERRORS_SMTP_HOST, + DAEMON_ERRORS_SMTP_PORT)) + + if DAEMON_ERRORS_SMTP_ENCRYPT == "SSL": + mailer = smtplib.SMTP_SSL() + else: + mailer = smtplib.SMTP() + mailer.connect(DAEMON_ERRORS_SMTP_HOST, DAEMON_ERRORS_SMTP_PORT) + if DAEMON_ERRORS_SMTP_ENCRYPT == "TLS": + mailer.starttls() + if DAEMON_ERRORS_SMTP_USER and DAEMON_ERRORS_SMTP_PASSWORD: + mailer.login(DAEMON_ERRORS_SMTP_USER, DAEMON_ERRORS_SMTP_PASSWORD) + + msg = TIMEOUT_EMAIL_TEMPLATE % \ + { 'from': DAEMON_ERRORS_EMAIL_FROM, + 'replyto': DAEMON_ERRORS_EMAIL_REPLY_TO, + 'to': self.__job.submittermail, + 'jobid': self.__job.id, + 'date': datetime.datetime.now().strftime('%a, %d %b %Y %H:%M:%S %Z'), + 'url': DAEMON_ERRORS_JOB_URL % self.__job.id, + 'title': self.__job.maptitle, + 'timeout': self.__timeout / 60 + } + + mailer.sendmail(DAEMON_ERRORS_EMAIL_FROM, + [admin[1] for admin in ADMINS], msg) + l.info("Email notification sent.") + except Exception, e: + l.exception("Could not send notification email to the submitter!") + def run(self): self.__process.start() self.__process.join(self.__timeout) @@ -161,6 +315,8 @@ class ForkingJobRenderer: # Remove job files self.__job.remove_all_files() + self._email_timeout() + l.debug("Process terminated.") return RESULT_TIMEOUT_REACHED @@ -205,6 +361,45 @@ class JobRenderer(threading.Thread): ctypes.pythonapi.PyThreadState_SetAsyncExc(self.__get_my_tid(), 0) raise SystemError("PyThreadState_SetAsync failed") + def _email_submitter(self, template): + """Send a notification with status and result URL to the request submitter""" + + if not DAEMON_ERRORS_SMTP_HOST or not self.job.submittermail: + return + + try: + l.info("Emailing success/failure message to %s via %s:%d..." % + (self.job.submittermail, + DAEMON_ERRORS_SMTP_HOST, + DAEMON_ERRORS_SMTP_PORT)) + + if DAEMON_ERRORS_SMTP_ENCRYPT == "SSL": + mailer = smtplib.SMTP_SSL() + else: + mailer = smtplib.SMTP() + mailer.connect(DAEMON_ERRORS_SMTP_HOST, DAEMON_ERRORS_SMTP_PORT) + if DAEMON_ERRORS_SMTP_ENCRYPT == "TLS": + mailer.starttls() + if DAEMON_ERRORS_SMTP_USER and DAEMON_ERRORS_SMTP_PASSWORD: + mailer.login(DAEMON_ERRORS_SMTP_USER, DAEMON_ERRORS_SMTP_PASSWORD) + + msg = template % \ + { 'from': DAEMON_ERRORS_EMAIL_FROM, + 'replyto': DAEMON_ERRORS_EMAIL_REPLY_TO, + 'to': self.job.submittermail, + 'jobid': self.job.id, + 'date': datetime.datetime.now().strftime('%a, %d %b %Y %H:%M:%S %Z'), + 'url': DAEMON_ERRORS_JOB_URL % self.job.id, + 'title': self.job.maptitle + } + + mailer.sendmail(DAEMON_ERRORS_EMAIL_FROM, + [admin[1] for admin in ADMINS], msg) + l.info("Email notification sent.") + except Exception, e: + l.exception("Could not send notification email to the submitter!") + + def _email_exception(self, e): """This method can be used to send the given exception by email to the configured admins in the project's settings.""" @@ -218,8 +413,15 @@ class JobRenderer(threading.Thread): DAEMON_ERRORS_SMTP_HOST, DAEMON_ERRORS_SMTP_PORT)) - mailer = smtplib.SMTP() + if DAEMON_ERRORS_SMTP_ENCRYPT == "SSL": + mailer = smtplib.SMTP_SSL() + else: + mailer = smtplib.SMTP() mailer.connect(DAEMON_ERRORS_SMTP_HOST, DAEMON_ERRORS_SMTP_PORT) + if DAEMON_ERRORS_SMTP_ENCRYPT == "TLS": + mailer.starttls() + if DAEMON_ERRORS_SMTP_USER and DAEMON_ERRORS_SMTP_PASSWORD: + mailer.login(DAEMON_ERRORS_SMTP_USER, DAEMON_ERRORS_SMTP_PASSWORD) jobinfo = [] for k in sorted(self.job.__dict__.keys()): @@ -245,6 +447,8 @@ class JobRenderer(threading.Thread): except Exception, e: l.exception("Could not send error email to the admins!") + self._email_submitter(FAILURE_EMAIL_TEMPLATE) + def _gen_thumbnail(self, prefix, paper_width_mm, paper_height_mm): l.info('Creating map thumbnail...') @@ -271,6 +475,7 @@ class JobRenderer(threading.Thread): elif 'png' in RENDERING_RESULT_FORMATS: img = Image.open(prefix + '.png') + img.save(prefix + '.jpg', quality=50) img.thumbnail((200, 200), Image.ANTIALIAS) img.save(prefix + THUMBNAIL_SUFFIX) @@ -368,6 +573,7 @@ class JobRenderer(threading.Thread): except KeyboardInterrupt: self.result = RESULT_KEYBOARD_INTERRUPT l.info("Rendering of job #%d interrupted!" % self.job.id) + return self.result except Exception, e: self.result = RESULT_RENDERING_EXCEPTION l.exception("Rendering of job #%d failed (exception occurred during" @@ -377,6 +583,9 @@ class JobRenderer(threading.Thread): traceback.print_exc(file=fp) fp.close() self._email_exception(e) + return self.result + + self._email_submitter(SUCCESS_EMAIL_TEMPLATE) return self.result diff --git a/www/maposmatic/forms.py b/www/maposmatic/forms.py index 11e2d88b..5c444cd0 100644 --- a/www/maposmatic/forms.py +++ b/www/maposmatic/forms.py @@ -54,7 +54,7 @@ class MapRenderingJobForm(forms.ModelForm): fields = ('maptitle', 'administrative_city', 'lat_upper_left', 'lon_upper_left', 'lat_bottom_right', 'lon_bottom_right', - 'track', 'track_bbox_mode') + 'track', 'track_bbox_mode','submittermail') MODES = (('admin', _('Administrative boundary')), ('bbox', _('Bounding box'))) diff --git a/www/maposmatic/migrations/0010_maprenderingjob_submittermail.py b/www/maposmatic/migrations/0010_maprenderingjob_submittermail.py new file mode 100644 index 00000000..4192e224 --- /dev/null +++ b/www/maposmatic/migrations/0010_maprenderingjob_submittermail.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('maposmatic', '0009_auto_20170701_1412'), + ] + + operations = [ + migrations.AddField( + model_name='maprenderingjob', + name='submittermail', + field=models.EmailField(max_length=254, null=True), + ), + ] diff --git a/www/maposmatic/models.py b/www/maposmatic/models.py index 8b150f90..548d300a 100644 --- a/www/maposmatic/models.py +++ b/www/maposmatic/models.py @@ -111,6 +111,7 @@ class MapRenderingJob(models.Model): endofrendering_time = models.DateTimeField(null=True) resultmsg = models.CharField(max_length=256, null=True) submitterip = models.GenericIPAddressField() + submittermail = models.EmailField(null=True) index_queue_at_submission = models.IntegerField() map_language = models.CharField(max_length=16) @@ -203,7 +204,9 @@ class MapRenderingJob(models.Model): allfiles = {'maps': {}, 'indeces': {}, 'thumbnail': [], 'errorlog': []} - for format in www.settings.RENDERING_RESULT_FORMATS: + formats = www.settings.RENDERING_RESULT_FORMATS + formats.append('jpg') + for format in formats: map_path = self.get_map_filepath(format) if format != 'csv' and os.path.exists(map_path): # Map files (all formats but CSV) diff --git a/www/maposmatic/views.py b/www/maposmatic/views.py index 0c72759d..b0820b7a 100644 --- a/www/maposmatic/views.py +++ b/www/maposmatic/views.py @@ -93,6 +93,7 @@ def new(request): job.paper_height_mm = form.cleaned_data.get('paper_height_mm') job.status = 0 # Submitted job.submitterip = request.META['REMOTE_ADDR'] + job.submitteremail = form.cleaned_data.get('submitteremail') job.map_language = form.cleaned_data.get('map_language') job.index_queue_at_submission = (models.MapRenderingJob.objects .queue_size()) @@ -201,6 +202,7 @@ def recreate(request): newjob.status = 0 # Submitted newjob.submitterip = request.META['REMOTE_ADDR'] + newjob.submittermail = None # TODO newjob.map_language = job.map_language newjob.index_queue_at_submission = (models.MapRenderingJob.objects .queue_size()) diff --git a/www/settings_local.py.dist b/www/settings_local.py.dist index 4f1fd81c..94cfaede 100644 --- a/www/settings_local.py.dist +++ b/www/settings_local.py.dist @@ -103,9 +103,23 @@ MAPOSMATIC_RSS_FEED = 'http://blog.osm-baustelle.de/index.php/feed/?cat=2' # defined. DAEMON_ERRORS_SMTP_HOST = None DAEMON_ERRORS_SMTP_PORT = 25 -DAEMON_ERRORS_EMAIL_FROM = 'daemon@domain.com' -DAEMON_ERRORS_EMAIL_REPLY_TO = 'noreply@domain.com' -DAEMON_ERRORS_JOB_URL = 'http://domain.com/jobs/%d' +DAEMON_ERRORS_SMTP_ENCRYPTION = None +DAEMON_ERRORS_SMTP_USER = None +DAEMON_ERRORS_SMTP_PASSOWRD = None +DAEMON_ERRORS_EMAIL_FROM = 'daemon@example.com' +DAEMON_ERRORS_EMAIL_REPLY_TO = 'noreply@example.com' +DAEMON_ERRORS_JOB_URL = 'http://example.com/jobs/%d' + +# example email settings for using a Google Mail account +# DAEMON_ERRORS_SMTP_HOST = 'smtp.googlemail.com' +# DAEMON_ERRORS_SMTP_PORT = 587 +# DAEMON_ERRORS_SMTP_ENCRYPT = 'TLS' +# DAEMON_ERRORS_SMTP_USER = '...@gmail.com' +# DAEMON_ERRORS_SMTP_PASSWORD = "..." +# DAEMON_ERRORS_EMAIL_FROM = '...@gmail.com' +# DAEMON_ERRORS_EMAIL_REPLY_TO = '...@gmail.com' +# DAEMON_ERRORS_JOB_URL = 'http://example.com/jobs/%d' + # Absolute path to the directory that holds media. # Example: "/home/media/media.lawrence.com/" diff --git a/www/templates/maposmatic/new.html b/www/templates/maposmatic/new.html index e5712173..d03c97a3 100644 --- a/www/templates/maposmatic/new.html +++ b/www/templates/maposmatic/new.html @@ -242,6 +242,15 @@ will be visible on the map.{% endblocktrans %} +
+
+
+ {% trans "Your Email address (for notifications, optional)" %} + {{ form.submittermail }} +
+
+
+