mirror
/
OpenDroneMap-WebODM
kopia lustrzana https://github.com/OpenDroneMap/WebODM
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Import task functionality poc

pull/625/head
Piero Toffanin 2019-02-20 16:42:20 -05:00
rodzic d2b4941213
commit 55712f0d58
13 zmienionych plików z 361 dodań i 52 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

51
app/api/tasks.py
Wyświetl plik

@ -2,6 +2,8 @@ import os
from wsgiref.util import FileWrapper
import mimetypes
import datetime
from django.core.exceptions import ObjectDoesNotExist, SuspiciousFileOperation, ValidationError
from django.db import transaction
from django.http import FileResponse
@ -13,11 +15,19 @@ from rest_framework.response import Response
from rest_framework.views import APIView
from app import models, pending_actions
from nodeodm import status_codes
from nodeodm.models import ProcessingNode
from worker import tasks as worker_tasks
from .common import get_and_check_project, get_tile_json, path_traversal_check
def flatten_files(request_files):
# MultiValueDict in, flat array of files out
return [file for filesList in map(
lambda key: request_files.getlist(key),
[keys for keys in request_files])
for file in filesList]
class TaskIDsSerializer(serializers.BaseSerializer):
def to_representation(self, obj):
return obj.id
@ -36,6 +46,7 @@ class TaskSerializer(serializers.ModelSerializer):
return None
def get_images_count(self, obj):
# TODO: create a field in the model for this
return obj.imageupload_set.count()
def get_can_rerun_from(self, obj):
@ -142,11 +153,7 @@ class TaskViewSet(viewsets.ViewSet):
def create(self, request, project_pk=None):
project = get_and_check_project(request, project_pk, ('change_project', ))
# MultiValueDict in, flat array of files out
files = [file for filesList in map(
lambda key: request.FILES.getlist(key),
[keys for keys in request.FILES])
for file in filesList]
files = flatten_files(request.FILES)
if len(files) <= 1:
raise exceptions.ValidationError(detail="Cannot create task, you need at least 2 images")
@ -322,3 +329,37 @@ class TaskAssets(TaskNestedView):
raise exceptions.NotFound("Asset does not exist")
return download_file_response(request, asset_path, 'inline')
"""
Task assets import
"""
class TaskAssetsImport(APIView):
permission_classes = (permissions.AllowAny,)
parser_classes = (parsers.MultiPartParser, parsers.JSONParser, parsers.FormParser,)
def post(self, request, project_pk=None):
project = get_and_check_project(request, project_pk, ('change_project',))
files = flatten_files(request.FILES)
if len(files) != 1:
raise exceptions.ValidationError(detail="Cannot create task, you need to upload 1 file")
with transaction.atomic():
task = models.Task.objects.create(project=project,
auto_processing_node=False,
name="Imported Task",
import_url="file://all.zip",
status=status_codes.RUNNING,
pending_action=pending_actions.IMPORT)
task.create_task_directories()
destination_file = task.assets_path("all.zip")
with open(destination_file, 'wb+') as fd:
for chunk in files[0].chunks():
fd.write(chunk)
worker_tasks.process_task.delay(task.id)
serializer = TaskSerializer(task)
return Response(serializer.data, status=status.HTTP_201_CREATED)

5
app/api/urls.py
Wyświetl plik

@ -3,7 +3,7 @@ from django.conf.urls import url, include
from app.api.presets import PresetViewSet
from app.plugins import get_api_url_patterns
from .projects import ProjectViewSet
from .tasks import TaskViewSet, TaskTiles, TaskTilesJson, TaskDownloads, TaskAssets
from .tasks import TaskViewSet, TaskTiles, TaskTilesJson, TaskDownloads, TaskAssets, TaskAssetsImport
from .processingnodes import ProcessingNodeViewSet, ProcessingNodeOptionsView
from rest_framework_nested import routers
from rest_framework_jwt.views import obtain_jwt_token
@ -25,10 +25,9 @@ urlpatterns = [
url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/(?P<tile_type>orthophoto|dsm|dtm)/tiles/(?P<z>[\d]+)/(?P<x>[\d]+)/(?P<y>[\d]+)\.png$', TaskTiles.as_view()),
url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/(?P<tile_type>orthophoto|dsm|dtm)/tiles\.json$', TaskTilesJson.as_view()),
url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/download/(?P<asset>.+)$', TaskDownloads.as_view()),
url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/assets/(?P<unsafe_asset_path>.+)$', TaskAssets.as_view()),
url(r'projects/(?P<project_pk>[^/.]+)/tasks/import$', TaskAssetsImport.as_view()),
url(r'^auth/', include('rest_framework.urls')),
url(r'^token-auth/', obtain_jwt_token),

52
app/migrations/0025_auto_20190220_1854.py 100644
Wyświetl plik

@ -0,0 +1,52 @@
# Generated by Django 2.1.5 on 2019-02-20 18:54
import app.models.task
import colorfield.fields
import django.contrib.postgres.fields
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('app', '0024_update_task_assets'),
]
operations = [
migrations.AddField(
model_name='task',
name='import_url',
field=models.TextField(blank=True, default='', help_text='URL this task is imported from (only for imported tasks)'),
),
migrations.AlterField(
model_name='preset',
name='options',
field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=list, help_text="Options that define this preset (same format as in a Task's options).", validators=[app.models.task.validate_task_options]),
),
migrations.AlterField(
model_name='task',
name='available_assets',
field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=80), blank=True, default=list, help_text='List of available assets to download', size=None),
),
migrations.AlterField(
model_name='task',
name='options',
field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict, help_text='Options that are being used to process this task', validators=[app.models.task.validate_task_options]),
),
migrations.AlterField(
model_name='task',
name='pending_action',
field=models.IntegerField(blank=True, choices=[(1, 'CANCEL'), (2, 'REMOVE'), (3, 'RESTART'), (4, 'RESIZE'), (5, 'IMPORT')], db_index=True, help_text='A requested action to be performed on the task. The selected action will be performed by the worker at the next iteration.', null=True),
),
migrations.AlterField(
model_name='theme',
name='header_background',
field=colorfield.fields.ColorField(default='#3498db', help_text="Background color of the site's header.", max_length=18),
),
migrations.AlterField(
model_name='theme',
name='tertiary',
field=colorfield.fields.ColorField(default='#3498db', help_text='Navigation links.', max_length=18),
),
]

110
app/models/task.py
Wyświetl plik

@ -8,6 +8,7 @@ import uuid as uuid_module
import json
from shlex import quote
import errno
import piexif
import re
@ -170,6 +171,7 @@ class Task(models.Model):
(pending_actions.REMOVE, 'REMOVE'),
(pending_actions.RESTART, 'RESTART'),
(pending_actions.RESIZE, 'RESIZE'),
(pending_actions.IMPORT, 'IMPORT'),
)
# Not an exact science
@ -223,6 +225,8 @@ class Task(models.Model):
running_progress = models.FloatField(default=0.0,
help_text="Value between 0 and 1 indicating the running progress (estimated) of this task",
blank=True)
import_url = models.TextField(null=False, default="", blank=True, help_text="URL this task is imported from (only for imported tasks)")
def __init__(self, *args, **kwargs):
super(Task, self).__init__(*args, **kwargs)
@ -333,6 +337,14 @@ class Task(models.Model):
else:
raise FileNotFoundError("{} is not a valid asset".format(asset))
def handle_import(self):
self.console_output += "Importing assets...\n"
self.save()
self.extract_assets_and_complete()
self.pending_action = None
self.processing_time = 0
self.save()
def process(self):
"""
This method contains the logic for processing tasks asynchronously
@ -342,6 +354,9 @@ class Task(models.Model):
"""
try:
if self.pending_action == pending_actions.IMPORT:
self.handle_import()
if self.pending_action == pending_actions.RESIZE:
resized_images = self.resize_images()
self.refresh_from_db()
@ -571,42 +586,7 @@ class Task(models.Model):
self.console_output += "Extracting results. This could take a few minutes...\n";
self.save()
# Extract from zip
with zipfile.ZipFile(zip_path, "r") as zip_h:
zip_h.extractall(assets_dir)
logger.info("Extracted all.zip for {}".format(self))
# Populate *_extent fields
extent_fields = [
(os.path.realpath(self.assets_path("odm_orthophoto", "odm_orthophoto.tif")),
'orthophoto_extent'),
(os.path.realpath(self.assets_path("odm_dem", "dsm.tif")),
'dsm_extent'),
(os.path.realpath(self.assets_path("odm_dem", "dtm.tif")),
'dtm_extent'),
]
for raster_path, field in extent_fields:
if os.path.exists(raster_path):
# Read extent and SRID
raster = GDALRaster(raster_path)
extent = OGRGeometry.from_bbox(raster.extent)
# It will be implicitly transformed into the SRID of the model’s field
# self.field = GEOSGeometry(...)
setattr(self, field, GEOSGeometry(extent.wkt, srid=raster.srid))
logger.info("Populated extent field with {} for {}".format(raster_path, self))
self.update_available_assets_field()
self.running_progress = 1.0
self.console_output += "Done!\n"
self.status = status_codes.COMPLETED
self.save()
from app.plugins import signals as plugin_signals
plugin_signals.task_completed.send_robust(sender=self.__class__, task_id=self.id)
self.extract_assets_and_complete()
else:
# FAILED, CANCELED
self.save()
@ -624,6 +604,51 @@ class Task(models.Model):
# Task was interrupted during image resize / upload
logger.warning("{} interrupted".format(self, str(e)))
def extract_assets_and_complete(self):
"""
Extracts assets/all.zip and populates task fields where required.
:return:
"""
assets_dir = self.assets_path("")
zip_path = self.assets_path("all.zip")
# Extract from zip
with zipfile.ZipFile(zip_path, "r") as zip_h:
zip_h.extractall(assets_dir)
logger.info("Extracted all.zip for {}".format(self))
# Populate *_extent fields
extent_fields = [
(os.path.realpath(self.assets_path("odm_orthophoto", "odm_orthophoto.tif")),
'orthophoto_extent'),
(os.path.realpath(self.assets_path("odm_dem", "dsm.tif")),
'dsm_extent'),
(os.path.realpath(self.assets_path("odm_dem", "dtm.tif")),
'dtm_extent'),
]
for raster_path, field in extent_fields:
if os.path.exists(raster_path):
# Read extent and SRID
raster = GDALRaster(raster_path)
extent = OGRGeometry.from_bbox(raster.extent)
# It will be implicitly transformed into the SRID of the model’s field
# self.field = GEOSGeometry(...)
setattr(self, field, GEOSGeometry(extent.wkt, srid=raster.srid))
logger.info("Populated extent field with {} for {}".format(raster_path, self))
self.update_available_assets_field()
self.running_progress = 1.0
self.console_output += "Done!\n"
self.status = status_codes.COMPLETED
self.save()
from app.plugins import signals as plugin_signals
plugin_signals.task_completed.send_robust(sender=self.__class__, task_id=self.id)
def get_tile_path(self, tile_type, z, x, y):
return self.assets_path("{}_tiles".format(tile_type), z, x, "{}.png".format(y))
@ -783,3 +808,16 @@ class Task(models.Model):
except subprocess.CalledProcessError as e:
logger.warning("Could not resize GCP file {}: {}".format(gcp_path, str(e)))
return None
def create_task_directories(self):
"""
Create directories for this task (if they don't exist already)
"""
assets_dir = self.assets_path("")
try:
os.makedirs(assets_dir)
except OSError as exc: # Python >2.5
if exc.errno == errno.EEXIST and os.path.isdir(assets_dir):
pass
else:
raise

1
app/pending_actions.py
Wyświetl plik

@ -2,3 +2,4 @@ CANCEL = 1
REMOVE = 2
RESTART = 3
RESIZE = 4
IMPORT = 5

7
app/static/app/js/classes/PendingActions.js
Wyświetl plik

@ -1,7 +1,8 @@
const CANCEL = 1,
REMOVE = 2,
RESTART = 3,
RESIZE = 4;
RESIZE = 4,
IMPORT = 5;
let pendingActions = {
[CANCEL]: {
@ -15,6 +16,9 @@ let pendingActions = {
},
[RESIZE]: {
descr: "Resizing images..."
},
[IMPORT]: {
descr: "Importing..."
}
};
@ -23,6 +27,7 @@ export default {
REMOVE: REMOVE,
RESTART: RESTART,
RESIZE: RESIZE,
IMPORT: IMPORT,
description: function(pendingAction) {
if (pendingActions[pendingAction]) return pendingActions[pendingAction].descr;

96
app/static/app/js/components/ImportTaskPanel.jsx 100644
Wyświetl plik

@ -0,0 +1,96 @@
import '../css/ImportTaskPanel.scss';
import React from 'react';
import PropTypes from 'prop-types';
import Dropzone from '../vendor/dropzone';
import csrf from '../django/csrf';
class ImportTaskPanel extends React.Component {
static defaultProps = {
};
static propTypes = {
// onSave: PropTypes.func.isRequired,
onCancel: PropTypes.func,
projectId: PropTypes.number.isRequired
};
constructor(props){
super(props);
this.state = {
};
}
componentDidMount(){
Dropzone.autoDiscover = false;
this.dz = new Dropzone(this.dropzone, {
paramName: "file",
url : `/api/projects/${this.props.projectId}/tasks/import`,
parallelUploads: 1,
uploadMultiple: false,
acceptedFiles: "application/zip",
autoProcessQueue: true,
createImageThumbnails: false,
previewTemplate: '<div style="display:none"></div>',
clickable: this.uploadButton,
chunkSize: 2147483647,
timeout: 2147483647,
headers: {
[csrf.header]: csrf.token
}
});
this.dz.on("error", function(file){
// Show
})
.on("uploadprogress", function(file, progress){
console.log(progress);
})
.on("complete", function(file){
if (file.status === "success"){
}else{
// error
}
});
}
cancel = (e) => {
this.props.onCancel();
}
setRef = (prop) => {
return (domNode) => {
if (domNode != null) this[prop] = domNode;
}
}
render() {
return (
<div ref={this.setRef("dropzone")} className="import-task-panel theme-background-highlight">
<div className="form-horizontal">
<button type="button" className="close theme-color-primary" aria-label="Close" onClick={this.cancel}><span aria-hidden="true">&times;</span></button>
<h4>Import Existing Assets</h4>
<p>You can import .zip files that have been exported from existing tasks via Download Assets <i className="glyphicon glyphicon-arrow-right"></i> All Assets.</p>
<button type="button"
className="btn btn-primary"
onClick={this.handleUpload}
ref={this.setRef("uploadButton")}>
<i className="glyphicon glyphicon-upload"></i>
Upload a File
</button>
<button type="button"
className="btn btn-primary"
onClick={this.handleImportFromUrl}
ref={this.setRef("importFromUrlButton")}>
<i className="glyphicon glyphicon-cloud-download"></i>
Import From URL
</button>
</div>
</div>
);
}
}
export default ImportTaskPanel;

29
app/static/app/js/components/ProjectListItem.jsx
Wyświetl plik

@ -3,6 +3,7 @@ import React from 'react';
import update from 'immutability-helper';
import TaskList from './TaskList';
import NewTaskPanel from './NewTaskPanel';
import ImportTaskPanel from './ImportTaskPanel';
import UploadProgressBar from './UploadProgressBar';
import ProgressBar from './ProgressBar';
import ErrorMessage from './ErrorMessage';
@ -32,7 +33,8 @@ class ProjectListItem extends React.Component {
upload: this.getDefaultUploadState(),
error: "",
data: props.data,
refreshing: false
refreshing: false,
importing: false
};
this.toggleTaskList = this.toggleTaskList.bind(this);
@ -335,6 +337,14 @@ class ProjectListItem extends React.Component {
location.href = `/map/project/${this.state.data.id}/`;
}
handleImportTask = () => {
this.setState({importing: true});
}
handleCancelImportTask = () => {
this.setState({importing: false});
}
render() {
const { refreshing, data } = this.state;
const numTasks = data.tasks.length;
@ -361,13 +371,17 @@ class ProjectListItem extends React.Component {
<ErrorMessage bind={[this, 'error']} />
<div className="btn-group pull-right">
{this.hasPermission("add") ?
<button type="button"
className={"btn btn-primary btn-sm " + (this.state.upload.uploading ? "hide" : "")}
<div className={"asset-download-buttons btn-group " + (this.state.upload.uploading ? "hide" : "")}>
<button type="button"
className="btn btn-primary btn-sm"
onClick={this.handleUpload}
ref={this.setRef("uploadButton")}>
<i className="glyphicon glyphicon-upload"></i>
Select Images and GCP
</button>
</button><button type="button" className="btn btn-sm dropdown-toggle btn-primary" data-toggle="dropdown"><span className="caret"></span></button>
<ul className="dropdown-menu">
<li><a href="javascript:void(0);" onClick={this.handleImportTask}><i className="glyphicon glyphicon-import"></i> Import Existing Assets</a></li>
</ul></div>
: ""}
<button disabled={this.state.upload.error !== ""}
@ -432,6 +446,13 @@ class ProjectListItem extends React.Component {
/>
: ""}
{this.state.importing ?
<ImportTaskPanel
onCancel={this.handleCancelImportTask}
projectId={this.state.data.id}
/>
: ""}
{this.state.showTaskList ?
<TaskList
ref={this.setRef("taskList")}

6
app/static/app/js/components/TaskListItem.jsx
Wyświetl plik

@ -354,13 +354,15 @@ class TaskListItem extends React.Component {
render() {
const task = this.state.task;
const name = task.name !== null ? task.name : `Task #${task.id}`;
const imported = task.import_url !== "";
let status = statusCodes.description(task.status);
if (status === "") status = "Uploading images to processing node";
if (!task.processing_node) status = "Waiting for a node...";
if (!task.processing_node && !imported) status = "Waiting for a node...";
if (task.pending_action !== null) status = pendingActions.description(task.pending_action);
let expanded = "";
if (this.state.expanded){
let showOrthophotoMissingWarning = false,
@ -572,7 +574,7 @@ class TaskListItem extends React.Component {
if (task.last_error){
statusLabel = getStatusLabel(task.last_error, 'error');
}else if (!task.processing_node){
}else if (!task.processing_node && !imported){
statusLabel = getStatusLabel("Set a processing node");
statusIcon = "fa fa-hourglass-3";
showEditLink = true;

10
app/static/app/js/components/tests/ImportTaskPanel.test.jsx 100644
Wyświetl plik

@ -0,0 +1,10 @@
import React from 'react';
import { shallow } from 'enzyme';
import ImportTaskPanel from '../ImportTaskPanel';
describe('<ImportTaskPanel />', () => {
it('renders without exploding', () => {
const wrapper = shallow(<ImportTaskPanel projectId={0} />);
expect(wrapper.exists()).toBe(true);
})
});

20
app/static/app/js/css/ImportTaskPanel.scss 100644
Wyświetl plik

@ -0,0 +1,20 @@
.import-task-panel{
padding: 10px;
margin-top: 10px;
margin-bottom: 10px;
text-align: center;
button{
margin-right: 12px;
margin-bottom: 8px;
}
.glyphicon-arrow-right{
font-size: 80%;
}
.close:hover, .close:focus{
color: inherit;
}
}

24
nodeodm/migrations/0006_auto_20190220_1842.py 100644
Wyświetl plik

@ -0,0 +1,24 @@
# Generated by Django 2.1.5 on 2019-02-20 18:42
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('nodeodm', '0005_auto_20190115_1346'),
]
operations = [
migrations.AlterField(
model_name='processingnode',
name='available_options',
field=django.contrib.postgres.fields.jsonb.JSONField(default=dict, help_text='Description of the options that can be used for processing'),
),
migrations.AlterField(
model_name='processingnode',
name='label',
field=models.CharField(blank=True, default='', help_text='Optional label for this node. When set, this label will be shown instead of the hostname:port name.', max_length=255),
),
]

2
nodeodm/models.py
Wyświetl plik

@ -43,7 +43,7 @@ class ProcessingNode(models.Model):
available_options = fields.JSONField(default=dict, help_text="Description of the options that can be used for processing")
token = models.CharField(max_length=1024, blank=True, default="", help_text="Token to use for authentication. If the node doesn't have authentication, you can leave this field blank.")
max_images = models.PositiveIntegerField(help_text="Maximum number of images accepted by this node.", blank=True, null=True)
odm_version = models.CharField(max_length=32, null=True, help_text="OpenDroneMap version used by the node")
odm_version = models.CharField(max_length=32, null=True, help_text="ODM version used by the node.")
label = models.CharField(max_length=255, default="", blank=True, help_text="Optional label for this node. When set, this label will be shown instead of the hostname:port name.")
def __str__(self):