mirror
/
maposmatic
kopia lustrzana https://github.com/hholzgra/maposmatic/
1
0
Forkuj 0
Kod Zgłoszenia Packages Projekty Wydania Wiki Aktywność

Merge branch 'site-osm-baustelle' of https://github.com/hholzgra/maposmatic into site-osm-baustelle

pull/18/head
Hartmut Holzgraefe 2017-07-16 16:01:58 +02:00
rodzic 5c2117fa60 a66157f218
commit dae417f15b
7 zmienionych plików z 262 dodań i 6 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

211
scripts/render.py
Wyświetl plik

@ -40,6 +40,8 @@ from www.maposmatic.models import MapRenderingJob
from www.settings import ADMINS, OCITYSMAP_CFG_PATH, MEDIA_ROOT
from www.settings import RENDERING_RESULT_PATH, RENDERING_RESULT_FORMATS
from www.settings import DAEMON_ERRORS_SMTP_HOST, DAEMON_ERRORS_SMTP_PORT
from www.settings import DAEMON_ERRORS_SMTP_ENCRYPT
from www.settings import DAEMON_ERRORS_SMTP_USER, DAEMON_ERRORS_SMTP_PASSWORD
from www.settings import DAEMON_ERRORS_EMAIL_FROM
from www.settings import DAEMON_ERRORS_EMAIL_REPLY_TO
from www.settings import DAEMON_ERRORS_JOB_URL
@ -73,6 +75,76 @@ You can view the job page at <%(url)s>.
MapOSMatic
"""
SUCCESS_EMAIL_TEMPLATE = """From: MapOSMatic rendering daemon <%(from)s>
Reply-To: %(replyto)s
To: $(to)s
Content-Type: text/plain; charset=utf-8
Content-Transfer-Encoding: 8bit
Subject: Rendering of job #%(jobid)d succeeded
Date: %(date)s
Hello %(to)s,
your map rendering request for
%(title)s
has successfully been processed now, and the results can be downloaded
from the rendering jobs detail pages:
%(url)s
--
MapOSMatic"""
FAILURE_EMAIL_TEMPLATE = """From: MapOSMatic rendering daemon <%(from)s>
Reply-To: %(replyto)s
To: $(to)s
Content-Type: text/plain; charset=utf-8
Content-Transfer-Encoding: 8bit
Subject: Rendering of job #%(jobid)d failed
Date: %(date)s
Hello %(to)s,
unfortunately your map rendering request for
%(title)s
has failed.
You can check for failure details on the request detail page:
%(url)s
--
MapOSMatic"""
TIMEOUT_EMAIL_TEMPLATE = """From: MapOSMatic rendering daemon <%(from)s>
Reply-To: %(replyto)s
To: $(to)s
Content-Type: text/plain; charset=utf-8
Content-Transfer-Encoding: 8bit
Subject: Rendering of job #%(jobid)d timed out
Date: %(date)s
Hello %(to)s,
unfortunately your map rendering request for
%(title)s
has been runnning for more than %(timeout)d minutes and had to be cancelled.
You may want to retry with a smaller map area or with a less complex map
style or less map overlays.
--
MapOSMatic"""
l = logging.getLogger('maposmatic')
class ThreadingJobRenderer:
@ -95,6 +167,45 @@ class ThreadingJobRenderer:
self.__timeout = timeout
self.__thread = JobRenderer(job, prefix)
def _email_timeout(self):
"""Send a notification about timeouts to the request submitter"""
if not DAEMON_ERRORS_SMTP_HOST or not self.__job.submittermail:
return
try:
l.info("Emailing timeout message to %s via %s:%d..." %
(self.__job.submittermail,
DAEMON_ERRORS_SMTP_HOST,
DAEMON_ERRORS_SMTP_PORT))
if DAEMON_ERRORS_SMTP_ENCRYPT == "SSL":
mailer = smtplib.SMTP_SSL()
else:
mailer = smtplib.SMTP()
mailer.connect(DAEMON_ERRORS_SMTP_HOST, DAEMON_ERRORS_SMTP_PORT)
if DAEMON_ERRORS_SMTP_ENCRYPT == "TLS":
mailer.starttls()
if DAEMON_ERRORS_SMTP_USER and DAEMON_ERRORS_SMTP_PASSWORD:
mailer.login(DAEMON_ERRORS_SMTP_USER, DAEMON_ERRORS_SMTP_PASSWORD)
msg = TIMEOUT_EMAIL_TEMPLATE % \
{ 'from': DAEMON_ERRORS_EMAIL_FROM,
'replyto': DAEMON_ERRORS_EMAIL_REPLY_TO,
'to': self.__job.submittermail,
'jobid': self.__job.id,
'date': datetime.datetime.now().strftime('%a, %d %b %Y %H:%M:%S %Z'),
'url': DAEMON_ERRORS_JOB_URL % self.__job.id,
'title': self.__job.maptitle,
'timeout': self.__timeout / 60
}
mailer.sendmail(DAEMON_ERRORS_EMAIL_FROM,
[admin[1] for admin in ADMINS], msg)
l.info("Email notification sent.")
except Exception, e:
l.exception("Could not send notification email to the submitter!")
def run(self):
"""Renders the job using a JobRendered, encapsulating all processing
errors and exceptions, with the addition here of a processing timeout.
@ -102,6 +213,8 @@ class ThreadingJobRenderer:
Returns one of the RESULT_ constants.
"""
l.info("Timeout is %d" % self.__timeout)
self.__thread.start()
self.__thread.join(self.__timeout)
@ -122,6 +235,8 @@ class ThreadingJobRenderer:
# Remove the job files
self.__job.remove_all_files()
self._email_timeout()
l.debug("Worker removed.")
return RESULT_TIMEOUT_REACHED
@ -134,6 +249,45 @@ class ForkingJobRenderer:
self.__renderer = JobRenderer(job, prefix)
self.__process = multiprocessing.Process(target=self._wrap)
def _email_timeout(self):
"""Send a notification about timeouts to the request submitter"""
if not DAEMON_ERRORS_SMTP_HOST or not self.__job.submittermail:
return
try:
l.info("Emailing timeout message to %s via %s:%d..." %
(self.__job.submittermail,
DAEMON_ERRORS_SMTP_HOST,
DAEMON_ERRORS_SMTP_PORT))
if DAEMON_ERRORS_SMTP_ENCRYPT == "SSL":
mailer = smtplib.SMTP_SSL()
else:
mailer = smtplib.SMTP()
mailer.connect(DAEMON_ERRORS_SMTP_HOST, DAEMON_ERRORS_SMTP_PORT)
if DAEMON_ERRORS_SMTP_ENCRYPT == "TLS":
mailer.starttls()
if DAEMON_ERRORS_SMTP_USER and DAEMON_ERRORS_SMTP_PASSWORD:
mailer.login(DAEMON_ERRORS_SMTP_USER, DAEMON_ERRORS_SMTP_PASSWORD)
msg = TIMEOUT_EMAIL_TEMPLATE % \
{ 'from': DAEMON_ERRORS_EMAIL_FROM,
'replyto': DAEMON_ERRORS_EMAIL_REPLY_TO,
'to': self.__job.submittermail,
'jobid': self.__job.id,
'date': datetime.datetime.now().strftime('%a, %d %b %Y %H:%M:%S %Z'),
'url': DAEMON_ERRORS_JOB_URL % self.__job.id,
'title': self.__job.maptitle,
'timeout': self.__timeout / 60
}
mailer.sendmail(DAEMON_ERRORS_EMAIL_FROM,
[admin[1] for admin in ADMINS], msg)
l.info("Email notification sent.")
except Exception, e:
l.exception("Could not send notification email to the submitter!")
def run(self):
self.__process.start()
self.__process.join(self.__timeout)
@ -161,6 +315,8 @@ class ForkingJobRenderer:
# Remove job files
self.__job.remove_all_files()
self._email_timeout()
l.debug("Process terminated.")
return RESULT_TIMEOUT_REACHED
@ -205,6 +361,45 @@ class JobRenderer(threading.Thread):
ctypes.pythonapi.PyThreadState_SetAsyncExc(self.__get_my_tid(), 0)
raise SystemError("PyThreadState_SetAsync failed")
def _email_submitter(self, template):
"""Send a notification with status and result URL to the request submitter"""
if not DAEMON_ERRORS_SMTP_HOST or not self.job.submittermail:
return
try:
l.info("Emailing success/failure message to %s via %s:%d..." %
(self.job.submittermail,
DAEMON_ERRORS_SMTP_HOST,
DAEMON_ERRORS_SMTP_PORT))
if DAEMON_ERRORS_SMTP_ENCRYPT == "SSL":
mailer = smtplib.SMTP_SSL()
else:
mailer = smtplib.SMTP()
mailer.connect(DAEMON_ERRORS_SMTP_HOST, DAEMON_ERRORS_SMTP_PORT)
if DAEMON_ERRORS_SMTP_ENCRYPT == "TLS":
mailer.starttls()
if DAEMON_ERRORS_SMTP_USER and DAEMON_ERRORS_SMTP_PASSWORD:
mailer.login(DAEMON_ERRORS_SMTP_USER, DAEMON_ERRORS_SMTP_PASSWORD)
msg = template % \
{ 'from': DAEMON_ERRORS_EMAIL_FROM,
'replyto': DAEMON_ERRORS_EMAIL_REPLY_TO,
'to': self.job.submittermail,
'jobid': self.job.id,
'date': datetime.datetime.now().strftime('%a, %d %b %Y %H:%M:%S %Z'),
'url': DAEMON_ERRORS_JOB_URL % self.job.id,
'title': self.job.maptitle
}
mailer.sendmail(DAEMON_ERRORS_EMAIL_FROM,
[admin[1] for admin in ADMINS], msg)
l.info("Email notification sent.")
except Exception, e:
l.exception("Could not send notification email to the submitter!")
def _email_exception(self, e):
"""This method can be used to send the given exception by email to the
configured admins in the project's settings."""
@ -218,8 +413,15 @@ class JobRenderer(threading.Thread):
DAEMON_ERRORS_SMTP_HOST,
DAEMON_ERRORS_SMTP_PORT))
mailer = smtplib.SMTP()
if DAEMON_ERRORS_SMTP_ENCRYPT == "SSL":
mailer = smtplib.SMTP_SSL()
else:
mailer = smtplib.SMTP()
mailer.connect(DAEMON_ERRORS_SMTP_HOST, DAEMON_ERRORS_SMTP_PORT)
if DAEMON_ERRORS_SMTP_ENCRYPT == "TLS":
mailer.starttls()
if DAEMON_ERRORS_SMTP_USER and DAEMON_ERRORS_SMTP_PASSWORD:
mailer.login(DAEMON_ERRORS_SMTP_USER, DAEMON_ERRORS_SMTP_PASSWORD)
jobinfo = []
for k in sorted(self.job.__dict__.keys()):
@ -245,6 +447,8 @@ class JobRenderer(threading.Thread):
except Exception, e:
l.exception("Could not send error email to the admins!")
self._email_submitter(FAILURE_EMAIL_TEMPLATE)
def _gen_thumbnail(self, prefix, paper_width_mm, paper_height_mm):
l.info('Creating map thumbnail...')
@ -271,6 +475,7 @@ class JobRenderer(threading.Thread):
elif 'png' in RENDERING_RESULT_FORMATS:
img = Image.open(prefix + '.png')
img.save(prefix + '.jpg', quality=50)
img.thumbnail((200, 200), Image.ANTIALIAS)
img.save(prefix + THUMBNAIL_SUFFIX)
@ -368,6 +573,7 @@ class JobRenderer(threading.Thread):
except KeyboardInterrupt:
self.result = RESULT_KEYBOARD_INTERRUPT
l.info("Rendering of job #%d interrupted!" % self.job.id)
return self.result
except Exception, e:
self.result = RESULT_RENDERING_EXCEPTION
l.exception("Rendering of job #%d failed (exception occurred during"
@ -377,6 +583,9 @@ class JobRenderer(threading.Thread):
traceback.print_exc(file=fp)
fp.close()
self._email_exception(e)
return self.result
self._email_submitter(SUCCESS_EMAIL_TEMPLATE)
return self.result

2
www/maposmatic/forms.py
Wyświetl plik

@ -54,7 +54,7 @@ class MapRenderingJobForm(forms.ModelForm):
fields = ('maptitle', 'administrative_city',
'lat_upper_left', 'lon_upper_left',
'lat_bottom_right', 'lon_bottom_right',
'track', 'track_bbox_mode')
'track', 'track_bbox_mode','submittermail')
MODES = (('admin', _('Administrative boundary')),
('bbox', _('Bounding box')))

19
www/maposmatic/migrations/0010_maprenderingjob_submittermail.py 100644
Wyświetl plik

@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('maposmatic', '0009_auto_20170701_1412'),
]
operations = [
migrations.AddField(
model_name='maprenderingjob',
name='submittermail',
field=models.EmailField(max_length=254, null=True),
),
]

5
www/maposmatic/models.py
Wyświetl plik

@ -111,6 +111,7 @@ class MapRenderingJob(models.Model):
endofrendering_time = models.DateTimeField(null=True)
resultmsg = models.CharField(max_length=256, null=True)
submitterip = models.GenericIPAddressField()
submittermail = models.EmailField(null=True)
index_queue_at_submission = models.IntegerField()
map_language = models.CharField(max_length=16)
@ -203,7 +204,9 @@ class MapRenderingJob(models.Model):
allfiles = {'maps': {}, 'indeces': {}, 'thumbnail': [], 'errorlog': []}
for format in www.settings.RENDERING_RESULT_FORMATS:
formats = www.settings.RENDERING_RESULT_FORMATS
formats.append('jpg')
for format in formats:
map_path = self.get_map_filepath(format)
if format != 'csv' and os.path.exists(map_path):
# Map files (all formats but CSV)

2
www/maposmatic/views.py
Wyświetl plik

@ -93,6 +93,7 @@ def new(request):
job.paper_height_mm = form.cleaned_data.get('paper_height_mm')
job.status = 0 # Submitted
job.submitterip = request.META['REMOTE_ADDR']
job.submitteremail = form.cleaned_data.get('submitteremail')
job.map_language = form.cleaned_data.get('map_language')
job.index_queue_at_submission = (models.MapRenderingJob.objects
.queue_size())
@ -201,6 +202,7 @@ def recreate(request):
newjob.status = 0 # Submitted
newjob.submitterip = request.META['REMOTE_ADDR']
newjob.submittermail = None # TODO
newjob.map_language = job.map_language
newjob.index_queue_at_submission = (models.MapRenderingJob.objects
.queue_size())

20
www/settings_local.py.dist
Wyświetl plik

@ -103,9 +103,23 @@ MAPOSMATIC_RSS_FEED = 'http://blog.osm-baustelle.de/index.php/feed/?cat=2'
# defined.
DAEMON_ERRORS_SMTP_HOST = None
DAEMON_ERRORS_SMTP_PORT = 25
DAEMON_ERRORS_EMAIL_FROM = 'daemon@domain.com'
DAEMON_ERRORS_EMAIL_REPLY_TO = 'noreply@domain.com'
DAEMON_ERRORS_JOB_URL = 'http://domain.com/jobs/%d'
DAEMON_ERRORS_SMTP_ENCRYPTION = None
DAEMON_ERRORS_SMTP_USER = None
DAEMON_ERRORS_SMTP_PASSOWRD = None
DAEMON_ERRORS_EMAIL_FROM = 'daemon@example.com'
DAEMON_ERRORS_EMAIL_REPLY_TO = 'noreply@example.com'
DAEMON_ERRORS_JOB_URL = 'http://example.com/jobs/%d'
# example email settings for using a Google Mail account
# DAEMON_ERRORS_SMTP_HOST = 'smtp.googlemail.com'
# DAEMON_ERRORS_SMTP_PORT = 587
# DAEMON_ERRORS_SMTP_ENCRYPT = 'TLS'
# DAEMON_ERRORS_SMTP_USER = '...@gmail.com'
# DAEMON_ERRORS_SMTP_PASSWORD = "..."
# DAEMON_ERRORS_EMAIL_FROM = '...@gmail.com'
# DAEMON_ERRORS_EMAIL_REPLY_TO = '...@gmail.com'
# DAEMON_ERRORS_JOB_URL = 'http://example.com/jobs/%d'
# Absolute path to the directory that holds media.
# Example: "/home/media/media.lawrence.com/"

9
www/templates/maposmatic/new.html
Wyświetl plik

@ -242,6 +242,15 @@ will be visible on the map.{% endblocktrans %}
</div>
</div>
<div class="row-fluid">
<div class="span12">
<fieldset>
<legend>{% trans "Your Email address (for notifications, optional)" %}</legend>
{{ form.submittermail }}
</fieldset>
</div>
</div>
<div class="row-fluid" style="margin-top: 30px;">
<div class="span12">
<fieldset id="summary">