mirror
/
OpenDroneMap-WebODM
kopia lustrzana https://github.com/OpenDroneMap/WebODM
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge pull request #1426 from pierotofy/upimp 

Cancel upload improvements, automatic task cleanup, configurable doc links
pierotofy-patch-2
Piero Toffanin 2023-11-06 12:45:26 -05:00 zatwierdzone przez GitHub
rodzic bd183c6455 ce9cd0a8b9
commit 9ea217c218
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
14 zmienionych plików z 93 dodań i 13 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

4
app/admin.py
Wyświetl plik

@ -40,9 +40,9 @@ class TaskAdmin(admin.ModelAdmin):
def has_add_permission(self, request):
return False
list_display = ('id', 'project', 'processing_node', 'created_at', 'status', 'last_error')
list_display = ('id', 'name', 'project', 'processing_node', 'created_at', 'status', 'last_error')
list_filter = ('status', 'project',)
search_fields = ('id', 'project__name')
search_fields = ('id', 'name', 'project__name')
admin.site.register(Task, TaskAdmin)

9
app/static/app/js/components/ProcessingNodeOption.jsx
Wyświetl plik

@ -106,6 +106,13 @@ class ProcessingNodeOption extends React.Component {
}
}
handleHelp = e => {
e.preventDefault();
if (window.__taskOptionsDocsLink){
window.open(window.__taskOptionsDocsLink + "#" + encodeURIComponent(this.props.name), "task-options")
}
}
render() {
let inputControl = "";
let warningMsg = "";
@ -168,7 +175,7 @@ class ProcessingNodeOption extends React.Component {
return (
<div className="processing-node-option form-inline form-group form-horizontal" ref={this.setTooltips}>
<label>{this.props.name} {(!this.isEnumType() && this.props.domain ? `(${this.props.domain})` : "")} <i data-toggle="tooltip" data-placement="bottom" title={this.props.help} onClick={e => e.preventDefault()} className="fa fa-info-circle info-button"></i></label><br/>
<label>{this.props.name} {(!this.isEnumType() && this.props.domain ? `(${this.props.domain})` : "")} <i data-toggle="tooltip" data-placement="bottom" title={this.props.help} onClick={this.handleHelp} className="fa fa-info-circle info-button help-button"></i></label><br/>
{inputControl}
{loadFileControl}

33
app/static/app/js/components/ProjectListItem.jsx
Wyświetl plik

@ -60,6 +60,7 @@ class ProjectListItem extends React.Component {
this.toggleTaskList = this.toggleTaskList.bind(this);
this.closeUploadError = this.closeUploadError.bind(this);
this.cancelUpload = this.cancelUpload.bind(this);
this.handleCancel = this.handleCancel.bind(this);
this.handleTaskSaved = this.handleTaskSaved.bind(this);
this.viewMap = this.viewMap.bind(this);
this.handleDelete = this.handleDelete.bind(this);
@ -248,13 +249,19 @@ class ProjectListItem extends React.Component {
}
}
}catch(e){
this.setUploadState({error: `${e.message}`, uploading: false});
this.dz.cancelUpload();
if (this.manuallyCanceled){
// Manually canceled, ignore error
this.setUploadState({uploading: false});
}else{
this.setUploadState({error: `${e.message}`, uploading: false});
}
if (this.dz.files.length) this.dz.cancelUpload();
}
})
.on("queuecomplete", () => {
const remainingFilesCount = this.state.upload.totalCount - this.state.upload.uploadedCount;
if (remainingFilesCount === 0){
if (remainingFilesCount === 0 && this.state.upload.uploadedCount > 0){
// All files have uploaded!
this.setUploadState({uploading: false});
@ -332,10 +339,26 @@ class ProjectListItem extends React.Component {
this.setUploadState({error: ""});
}
cancelUpload(e){
cancelUpload(){
this.dz.removeAllFiles(true);
}
handleCancel(){
this.manuallyCanceled = true;
this.cancelUpload();
if (this.dz._taskInfo && this.dz._taskInfo.id !== undefined){
$.ajax({
url: `/api/projects/${this.state.data.id}/tasks/${this.dz._taskInfo.id}/remove/`,
contentType: 'application/json',
dataType: 'json',
type: 'POST'
});
}
setTimeout(() => {
this.manuallyCanceled = false;
}, 500);
}
taskDeleted(){
this.refresh();
}
@ -628,7 +651,7 @@ class ProjectListItem extends React.Component {
<button disabled={this.state.upload.error !== ""}
type="button"
className={"btn btn-danger btn-sm " + (!this.state.upload.uploading ? "hide" : "")}
onClick={this.cancelUpload}>
onClick={this.handleCancel}>
<i className="glyphicon glyphicon-remove-circle"></i>
Cancel Upload
</button>

4
app/static/app/js/components/TaskListItem.jsx
Wyświetl plik

@ -605,14 +605,14 @@ class TaskListItem extends React.Component {
<div className="task-warning"><i className="fa fa-warning"></i> <span>{_("An orthophoto could not be generated. To generate one, make sure GPS information is embedded in the EXIF tags of your images, or use a Ground Control Points (GCP) file.")}</span></div> : ""}
{showMemoryErrorWarning ?
<div className="task-warning"><i className="fa fa-support"></i> <Trans params={{ memlink: `<a href="${memoryErrorLink}" target='_blank'>${_("enough RAM allocated")}</a>`, cloudlink: `<a href='https://www.opendronemap.org/webodm/lightning/' target='_blank'>${_("cloud processing node")}</a>` }}>{_("It looks like your processing node ran out of memory. If you are using docker, make sure that your docker environment has %(memlink)s. Alternatively, make sure you have enough physical RAM, reduce the number of images, make your images smaller, or reduce the max-concurrency parameter from the task's options. You can also try to use a %(cloudlink)s.")}</Trans></div> : ""}
<div className="task-warning"><i className="fa fa-support"></i> <Trans params={{ memlink: `<a href="${memoryErrorLink}" target='_blank'>${_("enough RAM allocated")}</a>`, cloudlink: `<a href='https://webodm.net' target='_blank'>${_("cloud processing node")}</a>` }}>{_("It looks like your processing node ran out of memory. If you are using docker, make sure that your docker environment has %(memlink)s. Alternatively, make sure you have enough physical RAM, reduce the number of images, make your images smaller, or reduce the max-concurrency parameter from the task's options. You can also try to use a %(cloudlink)s.")}</Trans></div> : ""}
{showTaskWarning ?
<div className="task-warning"><i className="fa fa-support"></i> <span dangerouslySetInnerHTML={{__html: this.state.friendlyTaskError}} /></div> : ""}
{showExitedWithCodeOneHints ?
<div className="task-warning"><i className="fa fa-info-circle"></i> <div className="inline">
<Trans params={{link: `<a href="https://docs.opendronemap.org" target="_blank">docs.opendronemap.org</a>` }}>{_("\"Process exited with code 1\" means that part of the processing failed. Sometimes it's a problem with the dataset, sometimes it can be solved by tweaking the Task Options. Check the documentation at %(link)")}</Trans>
<Trans params={{link: `<a href="${window.__taskOptionsDocsLink}" target="_blank">${window.__taskOptionsDocsLink.replace("https://", "")}</a>` }}>{_("\"Process exited with code 1\" means that part of the processing failed. Sometimes it's a problem with the dataset, sometimes it can be solved by tweaking the Task Options. Check the documentation at %(link)s")}</Trans>
</div>
</div>
: ""}

4
app/static/app/js/css/ProcessingNodeOption.scss
Wyświetl plik

@ -29,4 +29,8 @@
padding: 2px 4px 2px 4px;
margin-top: 12px;
}
.help-button:hover{
cursor: pointer;
}
}

2
app/static/app/js/vendor/dropzone.js vendored
Wyświetl plik

@ -2199,6 +2199,8 @@ var Dropzone = function (_Emitter) {
}, {
key: "cancelUpload",
value: function cancelUpload(file) {
if (file === undefined) return;
if (file.status === Dropzone.UPLOADING) {
var groupedFiles = this._getFilesWithXhr(file.xhr);
for (var _iterator19 = groupedFiles, _isArray19 = true, _i20 = 0, _iterator19 = _isArray19 ? _iterator19 : _iterator19[Symbol.iterator]();;) {

3
app/templates/app/base.html
Wyświetl plik

@ -115,6 +115,9 @@
</body>
<script src="{% static 'app/js/vendor/metisMenu.min.js' %}"></script>
<script>
{% task_options_docs_link as to_link %}
window.__taskOptionsDocsLink = "{{ to_link|safe }}";
$(function(){
$('#side-menu').metisMenu();

2
app/templates/app/dashboard.html
Wyświetl plik

@ -38,7 +38,7 @@
<li>{% trans 'You need at least 5 images, but 16-32 is typically the minimum.' %}</li>
<li>{% trans 'Images must overlap by 65% or more. Aim for 70-72%' %}</li>
<li>{% trans 'For great 3D, images must overlap by 83%' %}</li>
<li>{% blocktrans with link_start='<a href="https://github.com/OpenDroneMap/OpenDroneMap/wiki/Running-OpenDroneMap#running-odm-with-ground-control" target="_blank">' link_end='</a>' %}A {{link_start}}GCP File{{link_end}} is optional, but can increase georeferencing accuracy{% endblocktrans %}</li>
<li>{% gcp_docs_link as gcp_link %}{% blocktrans with link_start=gcp_link|safe link_end='</a>' %}A {{link_start}}GCP File{{link_end}} is optional, but can increase georeferencing accuracy{% endblocktrans %}</li>
</ul>
</p>
{% endif %}

8
app/templatetags/settings.py
Wyświetl plik

@ -9,6 +9,14 @@ from django.utils.translation import gettext as _
register = template.Library()
logger = logging.getLogger('app.logger')
@register.simple_tag
def task_options_docs_link():
return settings.TASK_OPTIONS_DOCS_LINK
@register.simple_tag
def gcp_docs_link():
return '<a href="%s" target="_blank">' % settings.GCP_DOCS_LINK
@register.simple_tag
def reset_password_link():
return settings.RESET_PASSWORD_LINK

2
app/views/app.py
Wyświetl plik

@ -39,7 +39,7 @@ def dashboard(request):
no_tasks = Task.objects.filter(project__owner=request.user).count() == 0
no_projects = Project.objects.filter(owner=request.user).count() == 0
# Create first project automatically
if no_projects and request.user.has_perm('app.add_project'):
Project.objects.create(owner=request.user, name=_("First Project"))

2
package.json
Wyświetl plik

@ -1,6 +1,6 @@
{
"name": "WebODM",
"version": "2.2.0",
"version": "2.2.1",
"description": "User-friendly, extendable application and API for processing aerial imagery.",
"main": "index.js",
"scripts": {

13
webodm/settings.py
Wyświetl plik

@ -411,6 +411,19 @@ QUOTA_EXCEEDED_GRACE_PERIOD = 8
# Maximum number of processing nodes to show in "Processing Nodes" menus/dropdowns
UI_MAX_PROCESSING_NODES = None
# Number of hours before partial tasks
# are removed (or None to disable)
CLEANUP_PARTIAL_TASKS = 72
# Link to GCP docs
GCP_DOCS_LINK = "https://docs.opendronemap.org/gcp/#gcp-file-format"
# Link to general docs
DOCS_LINK = "https://docs.opendronemap.org"
# Link to task options docs
TASK_OPTIONS_DOCS_LINK = "https://docs.opendronemap.org/arguments/"
if TESTING or FLUSHING:
CELERY_TASK_ALWAYS_EAGER = True
EXTERNAL_AUTH_ENDPOINT = 'http://0.0.0.0:5555/auth'

8
worker/celery.py
Wyświetl plik

@ -28,6 +28,14 @@ app.conf.beat_schedule = {
'retry': False
}
},
'cleanup-tasks': {
'task': 'worker.tasks.cleanup_tasks',
'schedule': 3600,
'options': {
'expires': 1799,
'retry': False
}
},
'cleanup-tmp-directory': {
'task': 'worker.tasks.cleanup_tmp_directory',
'schedule': 3600,

12
worker/tasks.py
Wyświetl plik

@ -23,6 +23,8 @@ import worker
from .celery import app
from app.raster_utils import export_raster as export_raster_sync, extension_for_export_format
from app.pointcloud_utils import export_pointcloud as export_pointcloud_sync
from django.utils import timezone
from datetime import timedelta
import redis
logger = get_task_logger("app.logger")
@ -67,6 +69,16 @@ def cleanup_projects():
if total > 0 and 'app.Project' in count_dict:
logger.info("Deleted {} projects".format(count_dict['app.Project']))
@app.task(ignore_result=True)
def cleanup_tasks():
# Delete tasks that are older than
if settings.CLEANUP_PARTIAL_TASKS is None:
return
tasks_to_delete = Task.objects.filter(partial=True, created_at__lte=timezone.now() - timedelta(hours=settings.CLEANUP_PARTIAL_TASKS))
for t in tasks_to_delete:
logger.info("Cleaning up partial task {}".format(t))
t.delete()
@app.task(ignore_result=True)
def cleanup_tmp_directory():