add "delete after rendering" option and processing (#116 / #90, WiP)

django-3.2
Hartmut Holzgraefe 2023-09-22 04:33:30 +00:00
rodzic efe09e6d27
commit 4080469a5c
6 zmienionych plików z 127 dodań i 84 usunięć

Wyświetl plik

@ -265,6 +265,10 @@ class RenderingsGarbageCollector:
pass
# records = UploadFile.objects.filter(keep_until__lte = datetime.now())
# for record in records:
if __name__ == '__main__':
if (not os.path.exists(RENDERING_RESULT_PATH)
or not os.path.isdir(RENDERING_RESULT_PATH)):

Wyświetl plik

@ -518,16 +518,13 @@ class JobRenderer(threading.Thread):
else:
self.result = RESULT_RENDERING_EXCEPTION
LOG.exception("Rendering of job #%d faild, no output files generated" % self.job.id)
return self.result
except KeyboardInterrupt:
self.result = RESULT_KEYBOARD_INTERRUPT
LOG.info("Rendering of job #%d interrupted!" % self.job.id)
return self.result
except MemoryError:
self.result = RESULT_MEMORY_EXCEEDED
LOG.exception("Not enough memory to render job #%d" % self.job.id)
self._email_exception(sys.exc_info())
return self.result
except Exception as e:
self.result = RESULT_RENDERING_EXCEPTION
LOG.exception("Rendering of job #%d failed (exception occurred during"
@ -537,9 +534,13 @@ class JobRenderer(threading.Thread):
traceback.print_exc(file=fp)
fp.close()
self._email_exception(sys.exc_info())
return self.result
self._email_submitter("render_email_success.txt")
if self.result == RESULT_SUCCESS:
self._email_submitter("render_email_success.txt")
for file in self.job.uploads.all():
if file.keep_until is None:
os.remove(os.path.join(MEDIA_ROOT, file.uploaded_file.name))
return self.result

Wyświetl plik

@ -74,6 +74,7 @@ class MapRenderingJobForm(forms.ModelForm):
fields=(forms.FloatField(), forms.FloatField(),
forms.FloatField(), forms.FloatField()))
uploadfile = MultiFileField(required=False)
delete_files_after_rendering = forms.BooleanField(required=False)
map_lang_flag_list = []
for lang_key, lang_name in www.settings.MAP_LANGUAGES_LIST:

Wyświetl plik

@ -2,88 +2,116 @@
{% load l10n %}
{% load extratags %}
<div class="col-lg-12" >
<div class="col-lg-12" >
<div class="tab" style="display: block" id="wizard-step-location">
<div class="tab" style="display: block" id="wizard-step-location">
<ul class="nav nav-tabs" id="locTabs">
<li class="nav-item" role="presentation">
<button class="nav-link active" id="step-location-bbox-tab" data-bs-toggle="tab" data-bs-target="#step-location-bbox" type="button" role="tab" aria-controls="bt-step-location-bbox" aria-selected="true"><i class="fas fa-globe-africa"></i> {% trans "Geographic area" %}</button>
</li>
<li class="nav-item" role="presentation">
<button class="nav-link" id="step-location-admin-tab" data-bs-toggle="tab" data-bs-target="#step-location-admin" type="button" role="tab" aria-controls="bt-step-location-bbox" aria-selected="false"><i class="fas fa-search-location"></i> {% trans "City search" %}</button>
</li>
<li class="nav-item" role="presentation">
<button class="nav-link" id="step-location-file-tab" data-bs-toggle="tab" data-bs-target="#step-location-file" type="button" role="tab" aria-controls="bt-step-location-file" aria-selected="false"><i class="fas fa-upload"></i> {% trans "File upload" %}</button>
</li>
</ul>
<ul class="nav nav-tabs" id="locTabs">
<li class="nav-item" role="presentation">
<button class="nav-link active" id="step-location-bbox-tab" data-bs-toggle="tab" data-bs-target="#step-location-bbox" type="button" role="tab" aria-controls="bt-step-location-bbox" aria-selected="true"><i class="fas fa-globe-africa"></i> {% trans "Geographic area" %}</button>
</li>
<li class="nav-item" role="presentation">
<button class="nav-link" id="step-location-admin-tab" data-bs-toggle="tab" data-bs-target="#step-location-admin" type="button" role="tab" aria-controls="bt-step-location-bbox" aria-selected="false"><i class="fas fa-search-location"></i> {% trans "City search" %}</button>
</li>
<li class="nav-item" role="presentation">
<button class="nav-link" id="step-location-file-tab" data-bs-toggle="tab" data-bs-target="#step-location-file" type="button" role="tab" aria-controls="bt-step-location-file" aria-selected="false"><i class="fas fa-upload"></i> {% trans "File upload" %}</button>
</li>
</ul>
<div class="tab-content" id="step-location-tabs">
<div class="tab-pane active" id="step-location-bbox" role="tabpanel">
{{ form.bbox }}
</div>
<div class="tab-pane" id="step-location-admin" role="tabpanel">
<div style="position: relative;">
{{ form.administrative_city }}
<i class="fas fa-redo suggest-icon" id="loading-icon"></i>
<i class="fas fa-exclamation-triangle suggest-icon" id="error-icon"></i>
<ul id="suggest" class="dropdown-menu" role="menu"
aria-labelledby="id_administrative_city">
</ul>
</div>
{{ form.administrative_osmid }}
</div>
<div class="tab-pane" id="step-location-file" role="tabpanel">
<div class="col-lg-12">
<fieldset id="track-file">
<legend><i class="fas fa-file-upload"></i> {% trans "Upload files" %}</legend>
{{ form.uploadfile }}
<tt id="file-list">
</tt>
</fieldset>
<div style='height: 2ex'></div>
<div class="alert alert-secondary">
{% blocktrans trimmed %}
Upload GPX, Umap, or general GeoJSON files here.
<br/>
You can select multiple files at once. If you re-open the
file selection dialog to select files, your selection will
replace the previous one.
{% endblocktrans %}
</div>
<div style='height: 2ex'></div>
<div class="alert alert-danger">
{% blocktrans trimmed %}
<b>Note:</b> The uploaded files will not be made available to anyone, the map created using this
file <b>will be publicly visible</b> though.<br/>Do <b>NOT</b> upload any data you don't want to be publicly
visible.
{% endblocktrans %}
</div>
</div>
</div>
</div>
<div class="tab-content" id="step-location-tabs">
<div class="tab-pane active" id="step-location-bbox" role="tabpanel">
{{ form.bbox }}
<div class="alert alert-info">
<h4>{% trans "Geographic area selection" %}</h4>
<p>
{% blocktrans trimmed %}
Start by choosing the city or geographic area you want to
render. Suggestions will appear as you start typing. If you can't find the city
you want, or if you prefer to choose the exact area you want to render
yourself, select the <em>Geographic area</em> tab and use the mini-map to set
the limits of the rendered area.
{% endblocktrans %}
{% blocktrans trimmed %}
Start by choosing the geographic area you want to
render. Use <em>Select area</em> to select a specific bounding box,
or just use the visible area.
{% endblocktrans %}
</p>
</div>
</div>
<div class="tab-pane" id="step-location-admin" role="tabpanel">
<div style="position: relative;">
{{ form.administrative_city }}
<i class="fas fa-redo suggest-icon" id="loading-icon"></i>
<i class="fas fa-exclamation-triangle suggest-icon" id="error-icon"></i>
<ul id="suggest" class="dropdown-menu" role="menu"
aria-labelledby="id_administrative_city">
</ul>
</div>
{{ form.administrative_osmid }}
<div class="alert alert-info">
<h4>{% trans "City search" %}</h4>
<p>
{% blocktrans trimmed %}
Start by choosing the city or geographic area you want to
render. Suggestions will appear as you start typing, based on what
is found in the local database.
{% endblocktrans %}
</p>
</div>
</div>
<div class="tab-pane" id="step-location-file" role="tabpanel">
<div class="col-lg-12">
<div class="container">
<div class="row">
<div class="col">
<fieldset id="track-file">
<legend><i class="fas fa-file-upload"></i> {% trans "Upload files" %}</legend>
{{ form.uploadfile }}
<tt id="file-list">
</tt>
</fieldset>
<fieldset>
{{ form.delete_files_after_rendering }}
<label>{% trans "Delete files after rendering" %}</label>
</fieldset>
</div>
<div class="col">
<div class="alert alert-danger">
{% blocktrans trimmed %}
<b>Note:</b>
The uploaded files will not be made available to anyone,
the map created using this
file <b>will be publicly visible</b> though.<br/>Do <b>NOT</b> upload any data you don't want to be publicly
visible.
{% endblocktrans %}
</div>
</div>
</div>
</div>
<div class="alert alert-info">
<h4>{% trans "File upload" %}</h4>
<p>
{% blocktrans trimmed %}
Upload GPX, Umap, or general GeoJSON files here.
<br/>
You can select multiple files at once.
<br/>
If you re-open the file selection dialog to select files
your selection will replace the previous one.
{% endblocktrans %}
</p>
</div>
</div>
</div>
</div>
</div>
</div>

Wyświetl plik

@ -131,14 +131,16 @@ def donate_thanks(request):
"""The thanks for donation page."""
return render(request, 'maposmatic/donate-thanks.html')
def create_upload_file(job, file):
def create_upload_file(job, file, keep_until = None):
first_line = file.readline().decode("utf-8-sig")
LOG.info("firstline type %s" % type(first_line))
if first_line.startswith(u'<?xml'):
file_type = 'gpx'
else:
file_type = 'umap'
file_instance = models.UploadFile(uploaded_file = file, file_type = file_type)
file_instance = models.UploadFile(uploaded_file = file,
file_type = file_type,
keep_until = keep_until)
file_instance.save()
file_instance.job.add(job)
@ -188,8 +190,12 @@ def new(request):
job.save()
files = request.FILES.getlist('uploadfile')
if form.cleaned_data.get('delete_files_after_rendering'):
keep_until = None
else:
keep_until = '1999-01-01'
for file in files:
create_upload_file(job, file)
create_upload_file(job, file, keep_until)
return HttpResponseRedirect(reverse('map-by-id-and-nonce',
args=[job.id, job.nonce]))

Wyświetl plik

@ -350,10 +350,13 @@ logconfig.setup_maposmatic_logging(
LOG = logging.getLogger('maposmatic')
# File upload settings
# make sure files and dirs are group-writable
# render user should be in www-data group to be able to clean up
# make sure that files that exceed FILE_UPLOAD_MAX_MEMORY_SIZE
# are still readable
FILE_UPLOAD_PERMISSIONS = 0o644
FILE_UPLOAD_PERMISSIONS = 0o664
FILE_UPLOAD_DIRECTORY_PERMISSIONS = 0o664
# Maintenance mode settings
MAINTENANCE_MODE = False # True or False, *NOT* None
MAINTENANCE_MODE_IGNORE_IP_ADDRESSES = ('217.146.146.90',)