mirror
/
OpenDroneMap-WebODM
kopia lustrzana https://github.com/OpenDroneMap/WebODM
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Import tasks working

pull/625/head
Piero Toffanin 2019-02-21 14:16:48 -05:00
rodzic 55712f0d58
commit 39589611f7
8 zmienionych plików z 233 dodań i 42 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

26
app/api/tasks.py
Wyświetl plik

@ -36,7 +36,6 @@ class TaskSerializer(serializers.ModelSerializer):
project = serializers.PrimaryKeyRelatedField(queryset=models.Project.objects.all())
processing_node = serializers.PrimaryKeyRelatedField(queryset=ProcessingNode.objects.all())
processing_node_name = serializers.SerializerMethodField()
images_count = serializers.SerializerMethodField()
can_rerun_from = serializers.SerializerMethodField()
def get_processing_node_name(self, obj):
@ -45,10 +44,6 @@ class TaskSerializer(serializers.ModelSerializer):
else:
return None
def get_images_count(self, obj):
# TODO: create a field in the model for this
return obj.imageupload_set.count()
def get_can_rerun_from(self, obj):
"""
When a task has been associated with a processing node
@ -164,6 +159,7 @@ class TaskViewSet(viewsets.ViewSet):
for image in files:
models.ImageUpload.objects.create(task=task, image=image)
task.images_count = len(files)
# Update other parameters such as processing node, task name, etc.
serializer = TaskSerializer(task, data=request.data, partial=True)
@ -341,23 +337,29 @@ class TaskAssetsImport(APIView):
project = get_and_check_project(request, project_pk, ('change_project',))
files = flatten_files(request.FILES)
import_url = request.data.get('url', None)
task_name = request.data.get('name', 'Imported Task')
if len(files) != 1:
if not import_url and len(files) != 1:
raise exceptions.ValidationError(detail="Cannot create task, you need to upload 1 file")
if import_url and len(files) > 0:
raise exceptions.ValidationError(detail="Cannot create task, either specify a URL or upload 1 file.")
with transaction.atomic():
task = models.Task.objects.create(project=project,
auto_processing_node=False,
name="Imported Task",
import_url="file://all.zip",
name=task_name,
import_url=import_url if import_url else "file://all.zip",
status=status_codes.RUNNING,
pending_action=pending_actions.IMPORT)
task.create_task_directories()
destination_file = task.assets_path("all.zip")
with open(destination_file, 'wb+') as fd:
for chunk in files[0].chunks():
fd.write(chunk)
if len(files) > 0:
destination_file = task.assets_path("all.zip")
with open(destination_file, 'wb+') as fd:
for chunk in files[0].chunks():
fd.write(chunk)
worker_tasks.process_task.delay(task.id)

29
app/migrations/0025_auto_20190220_1854.py
Wyświetl plik

@ -17,36 +17,53 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='task',
name='import_url',
field=models.TextField(blank=True, default='', help_text='URL this task is imported from (only for imported tasks)'),
field=models.TextField(blank=True, default='',
help_text='URL this task is imported from (only for imported tasks)'),
),
migrations.AlterField(
model_name='preset',
name='options',
field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=list, help_text="Options that define this preset (same format as in a Task's options).", validators=[app.models.task.validate_task_options]),
field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=list,
help_text="Options that define this preset (same format as in a Task's options).",
validators=[app.models.task.validate_task_options]),
),
migrations.AlterField(
model_name='task',
name='available_assets',
field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=80), blank=True, default=list, help_text='List of available assets to download', size=None),
field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=80), blank=True,
default=list,
help_text='List of available assets to download',
size=None),
),
migrations.AlterField(
model_name='task',
name='options',
field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict, help_text='Options that are being used to process this task', validators=[app.models.task.validate_task_options]),
field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict,
help_text='Options that are being used to process this task',
validators=[app.models.task.validate_task_options]),
),
migrations.AlterField(
model_name='task',
name='pending_action',
field=models.IntegerField(blank=True, choices=[(1, 'CANCEL'), (2, 'REMOVE'), (3, 'RESTART'), (4, 'RESIZE'), (5, 'IMPORT')], db_index=True, help_text='A requested action to be performed on the task. The selected action will be performed by the worker at the next iteration.', null=True),
field=models.IntegerField(blank=True, choices=[(1, 'CANCEL'), (2, 'REMOVE'), (3, 'RESTART'), (4, 'RESIZE'),
(5, 'IMPORT')], db_index=True,
help_text='A requested action to be performed on the task. The selected action will be performed by the worker at the next iteration.',
null=True),
),
migrations.AlterField(
model_name='theme',
name='header_background',
field=colorfield.fields.ColorField(default='#3498db', help_text="Background color of the site's header.", max_length=18),
field=colorfield.fields.ColorField(default='#3498db', help_text="Background color of the site's header.",
max_length=18),
),
migrations.AlterField(
model_name='theme',
name='tertiary',
field=colorfield.fields.ColorField(default='#3498db', help_text='Navigation links.', max_length=18),
),
migrations.AddField(
model_name='task',
name='images_count',
field=models.IntegerField(blank=True, default=0, help_text='Number of images associated with this task'),
),
]

57
app/models/task.py
Wyświetl plik

@ -226,7 +226,7 @@ class Task(models.Model):
help_text="Value between 0 and 1 indicating the running progress (estimated) of this task",
blank=True)
import_url = models.TextField(null=False, default="", blank=True, help_text="URL this task is imported from (only for imported tasks)")
images_count = models.IntegerField(null=False, blank=True, default=0, help_text="Number of images associated with this task")
def __init__(self, *args, **kwargs):
super(Task, self).__init__(*args, **kwargs)
@ -340,7 +340,50 @@ class Task(models.Model):
def handle_import(self):
self.console_output += "Importing assets...\n"
self.save()
zip_path = self.assets_path("all.zip")
if self.import_url and not os.path.exists(zip_path):
try:
# TODO: this is potentially vulnerable to a zip bomb attack
# mitigated by the fact that a valid account is needed to
# import tasks
download_stream = requests.get(self.import_url, stream=True, timeout=10)
content_length = download_stream.headers.get('content-length')
total_length = int(content_length) if content_length is not None else None
downloaded = 0
last_update = 0
with open(zip_path, 'wb') as fd:
for chunk in download_stream.iter_content(4096):
downloaded += len(chunk)
if time.time() - last_update >= 2:
# Update progress
if total_length is not None:
Task.objects.filter(pk=self.id).update(running_progress=(float(downloaded) / total_length) * 0.9)
self.check_if_canceled()
last_update = time.time()
fd.write(chunk)
except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, ReadTimeoutError) as e:
raise ProcessingError(e)
self.refresh_from_db()
self.extract_assets_and_complete()
images_json = self.assets_path("images.json")
if os.path.exists(images_json):
try:
with open(images_json) as f:
images = json.load(f)
self.images_count = len(images)
except:
logger.warning("Cannot read images count from imported task {}".format(self))
pass
self.pending_action = None
self.processing_time = 0
self.save()
@ -441,6 +484,11 @@ class Task(models.Model):
except ProcessingException:
logger.warning("Could not cancel {} on processing node. We'll proceed anyway...".format(self))
self.status = status_codes.CANCELED
self.pending_action = None
self.save()
elif self.import_url:
# Imported tasks need no special action
self.status = status_codes.CANCELED
self.pending_action = None
self.save()
@ -613,8 +661,11 @@ class Task(models.Model):
zip_path = self.assets_path("all.zip")
# Extract from zip
with zipfile.ZipFile(zip_path, "r") as zip_h:
zip_h.extractall(assets_dir)
try:
with zipfile.ZipFile(zip_path, "r") as zip_h:
zip_h.extractall(assets_dir)
except zipfile.BadZipFile:
raise ProcessingError("Corrupted zip file")
logger.info("Extracted all.zip for {}".format(self))

127
app/static/app/js/components/ImportTaskPanel.jsx
Wyświetl plik

@ -3,13 +3,15 @@ import React from 'react';
import PropTypes from 'prop-types';
import Dropzone from '../vendor/dropzone';
import csrf from '../django/csrf';
import ErrorMessage from './ErrorMessage';
import UploadProgressBar from './UploadProgressBar';
class ImportTaskPanel extends React.Component {
static defaultProps = {
};
static propTypes = {
// onSave: PropTypes.func.isRequired,
onImported: PropTypes.func.isRequired,
onCancel: PropTypes.func,
projectId: PropTypes.number.isRequired
};
@ -18,9 +20,20 @@ class ImportTaskPanel extends React.Component {
super(props);
this.state = {
error: "",
typeUrl: false,
uploading: false,
importingFromUrl: false,
progress: 0,
bytesSent: 0,
importUrl: ""
};
}
defaultTaskName = () => {
return `Task of ${new Date().toISOString()}`;
}
componentDidMount(){
Dropzone.autoDiscover = false;
@ -42,24 +55,90 @@ class ImportTaskPanel extends React.Component {
}
});
this.dz.on("error", function(file){
// Show
this.dz.on("error", (file) => {
if (this.state.uploading) this.setState({error: "Cannot upload file. Check your internet connection and try again."});
})
.on("uploadprogress", function(file, progress){
console.log(progress);
.on("sending", () => {
this.setState({typeUrl: false, uploading: true, totalCount: 1});
})
.on("complete", function(file){
if (file.status === "success"){
}else{
// error
.on("reset", () => {
this.setState({uploading: false, progress: 0, totalBytes: 0, totalBytesSent: 0});
})
.on("uploadprogress", (file, progress, bytesSent) => {
this.setState({
progress,
totalBytes: file.size,
totalBytesSent: bytesSent
});
})
.on("sending", (file, xhr, formData) => {
// Safari does not have support for has on FormData
// as of December 2017
if (!formData.has || !formData.has("name")) formData.append("name", this.defaultTaskName());
})
.on("complete", (file) => {
if (file.status === "success"){
this.setState({uploading: false});
try{
let response = JSON.parse(file.xhr.response);
if (!response.id) throw new Error(`Expected id field, but none given (${response})`);
this.props.onImported();
}catch(e){
this.setState({error: `Invalid response from server: ${e.message}`});
}
}else if (this.state.uploading){
this.setState({uploading: false, error: "An error occured while uploading the file. Please try again."});
}
});
}
cancel = (e) => {
this.cancelUpload();
this.props.onCancel();
}
cancelUpload = (e) => {
this.setState({uploading: false});
setTimeout(() => {
this.dz.removeAllFiles(true);
}, 0);
}
handleImportFromUrl = () => {
this.setState({typeUrl: !this.state.typeUrl});
}
handleCancelImportFromURL = () => {
this.setState({typeUrl: false});
}
handleChangeImportUrl = (e) => {
this.setState({importUrl: e.target.value});
}
handleConfirmImportUrl = () => {
this.setState({importingFromUrl: true});
$.post(`/api/projects/${this.props.projectId}/tasks/import`,
{
url: this.state.importUrl,
name: this.defaultTaskName()
}
).done(json => {
if (json.id){
this.props.onImported();
}else{
this.setState({error: json.error || `Cannot import from URL, server responded: ${JSON.stringify(json)}`});
}
})
.fail(() => {
this.setState({error: "Cannot import from URL. Check your internet connection."});
})
.always(() => {
this.setState({importingFromUrl: false});
});
}
setRef = (prop) => {
return (domNode) => {
if (domNode != null) this[prop] = domNode;
@ -70,23 +149,47 @@ class ImportTaskPanel extends React.Component {
return (
<div ref={this.setRef("dropzone")} className="import-task-panel theme-background-highlight">
<div className="form-horizontal">
<ErrorMessage bind={[this, 'error']} />
<button type="button" className="close theme-color-primary" aria-label="Close" onClick={this.cancel}><span aria-hidden="true">&times;</span></button>
<h4>Import Existing Assets</h4>
<p>You can import .zip files that have been exported from existing tasks via Download Assets <i className="glyphicon glyphicon-arrow-right"></i> All Assets.</p>
<button type="button"
<button disabled={this.state.uploading}
type="button"
className="btn btn-primary"
onClick={this.handleUpload}
ref={this.setRef("uploadButton")}>
<i className="glyphicon glyphicon-upload"></i>
Upload a File
</button>
<button type="button"
<button disabled={this.state.uploading}
type="button"
className="btn btn-primary"
onClick={this.handleImportFromUrl}
ref={this.setRef("importFromUrlButton")}>
<i className="glyphicon glyphicon-cloud-download"></i>
Import From URL
</button>
{this.state.typeUrl ?
<div className="form-inline">
<div className="form-group">
<input disabled={this.state.importingFromUrl} onChange={this.handleChangeImportUrl} size="45" type="text" className="form-control" placeholder="http://" value={this.state.importUrl} />
<button onClick={this.handleConfirmImportUrl}
disabled={this.state.importUrl.length < 4 || this.state.importingFromUrl}
className="btn-import btn btn-primary"><i className="glyphicon glyphicon-cloud-download"></i> Import</button>
</div>
</div> : ""}
{this.state.uploading ? <div>
<UploadProgressBar {...this.state}/>
<button type="button"
className="btn btn-danger btn-sm"
onClick={this.cancelUpload}>
<i className="glyphicon glyphicon-remove-circle"></i>
Cancel Upload
</button>
</div> : ""}
</div>
</div>
);

21
app/static/app/js/components/ProjectListItem.jsx
Wyświetl plik

@ -204,13 +204,7 @@ class ProjectListItem extends React.Component {
let response = JSON.parse(files[0].xhr.response);
if (!response.id) throw new Error(`Expected id field, but none given (${response})`);
if (this.state.showTaskList){
this.taskList.refresh();
}else{
this.setState({showTaskList: true});
}
this.resetUploadState();
this.refresh();
this.newTaskAdded();
}catch(e){
this.setUploadState({error: `Invalid response from server: ${e.message}`, uploading: false})
}
@ -247,6 +241,18 @@ class ProjectListItem extends React.Component {
}
}
newTaskAdded = () => {
this.setState({importing: false});
if (this.state.showTaskList){
this.taskList.refresh();
}else{
this.setState({showTaskList: true});
}
this.resetUploadState();
this.refresh();
}
setRef(prop){
return (domNode) => {
if (domNode != null) this[prop] = domNode;
@ -448,6 +454,7 @@ class ProjectListItem extends React.Component {
{this.state.importing ?
<ImportTaskPanel
onImported={this.newTaskAdded}
onCancel={this.handleCancelImportTask}
projectId={this.state.data.id}
/>

5
app/static/app/js/components/TaskListItem.jsx
Wyświetl plik

@ -406,12 +406,13 @@ class TaskListItem extends React.Component {
}
if ([statusCodes.QUEUED, statusCodes.RUNNING, null].indexOf(task.status) !== -1 &&
task.processing_node){
(task.processing_node || imported)){
addActionButton("Cancel", "btn-primary", "glyphicon glyphicon-remove-circle", this.genActionApiCall("cancel", {defaultError: "Cannot cancel task."}));
}
if ([statusCodes.FAILED, statusCodes.COMPLETED, statusCodes.CANCELED].indexOf(task.status) !== -1 &&
task.processing_node){
task.processing_node &&
!imported){
// By default restart reruns every pipeline
// step from the beginning
const rerunFrom = task.can_rerun_from.length > 1 ?

9
app/static/app/js/css/ImportTaskPanel.scss
Wyświetl plik

@ -17,4 +17,13 @@
.close:hover, .close:focus{
color: inherit;
}
.upload-progress-bar{
margin-top: 12px;
}
.btn-import{
margin-top: 8px;
margin-left: 8px;
}
}

1
app/static/app/js/css/TaskListItem.scss
Wyświetl plik

@ -15,6 +15,7 @@
.name{
padding-left: 0;
margin-top: 4px;
}
.details{