kopia lustrzana https://github.com/OpenDroneMap/WebODM
Changed available_assets code to be faster, easier to maintain, still need to write unit tests
Piero Toffanin
rodzic
b413966202
commit
9039dca53d
|
|
@ -29,18 +29,14 @@ class TaskSerializer(serializers.ModelSerializer):
|
|||
project = serializers.PrimaryKeyRelatedField(queryset=models.Project.objects.all())
|
||||
processing_node = serializers.PrimaryKeyRelatedField(queryset=ProcessingNode.objects.all())
|
||||
images_count = serializers.SerializerMethodField()
|
||||
available_assets = serializers.SerializerMethodField()
|
||||
|
||||
def get_images_count(self, obj):
|
||||
return obj.imageupload_set.count()
|
||||
|
||||
def get_available_assets(self, obj):
|
||||
return obj.get_available_assets()
|
||||
|
||||
class Meta:
|
||||
model = models.Task
|
||||
exclude = ('processing_lock', 'console_output', 'orthophoto_extent', )
|
||||
read_only_fields = ('processing_time', 'status', 'last_error', 'created_at', 'pending_action', )
|
||||
read_only_fields = ('processing_time', 'status', 'last_error', 'created_at', 'pending_action', 'available_assets', )
|
||||
|
||||
class TaskViewSet(viewsets.ViewSet):
|
||||
"""
|
||||
|
|
@ -240,7 +236,7 @@ class TaskDownloads(TaskNestedView):
|
|||
file = open(asset_path, "rb")
|
||||
response = HttpResponse(FileWrapper(file),
|
||||
content_type=(mimetypes.guess_type(asset_filename)[0] or "application/zip"))
|
||||
response['Content-Disposition'] = "attachment; filename={}".format(asset_filename)
|
||||
response['Content-Disposition'] = "attachment; filename={}".format(asset)
|
||||
return response
|
||||
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -20,7 +20,8 @@ urlpatterns = [
|
|||
|
||||
url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/tiles/(?P<z>[\d]+)/(?P<x>[\d]+)/(?P<y>[\d]+)\.png$', TaskTiles.as_view()),
|
||||
url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/tiles\.json$', TaskTilesJson.as_view()),
|
||||
url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/download/(?P<asset>[^/.]+)/$', TaskDownloads.as_view()),
|
||||
url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/download/(?P<asset>.+)$', TaskDownloads.as_view()),
|
||||
|
||||
url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/assets/(?P<unsafe_asset_path>.+)$', TaskAssets.as_view()),
|
||||
|
||||
url(r'^auth/', include('rest_framework.urls')),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Generated by Django 1.11.1 on 2017-07-07 18:05
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import django.contrib.postgres.fields
|
||||
from django.db import migrations, models
|
||||
from app.models import Task
|
||||
|
||||
|
||||
def detect_available_assets(apps, schema_editor):
|
||||
for t in Task.objects.all():
|
||||
print("Updating {}".format(t))
|
||||
t.update_available_assets_field(True)
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('app', '0005_auto_20170707_1014'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='task',
|
||||
name='available_assets',
|
||||
field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=80), blank=True, default=[], help_text='List of available assets to download', size=None),
|
||||
),
|
||||
migrations.RunPython(detect_available_assets),
|
||||
]
|
||||
124
app/models.py
124
app/models.py
|
|
@ -120,7 +120,18 @@ def validate_task_options(value):
|
|||
|
||||
|
||||
class Task(models.Model):
|
||||
ASSET_DOWNLOADS = ("all", "geotiff", "texturedmodel", "las", "csv", "ply",)
|
||||
ASSETS_MAP = {
|
||||
'all.zip': 'all.zip',
|
||||
'orthophoto.tif': os.path.join('odm_orthophoto', 'odm_orthophoto.tif'),
|
||||
'orthophoto.png': os.path.join('odm_orthophoto', 'odm_orthophoto.png'),
|
||||
'georeferenced_model.las': os.path.join('odm_georeferencing', 'odm_georeferenced_model.las'),
|
||||
'georeferenced_model.ply': os.path.join('odm_georeferencing', 'odm_georeferenced_model.ply'),
|
||||
'georeferenced_model.csv': os.path.join('odm_georeferencing', 'odm_georeferenced_model.csv'),
|
||||
'textured_model.zip': {
|
||||
'deferred_path': 'textured_model.zip',
|
||||
'deferred_compress_dir': 'odm_texturing'
|
||||
}
|
||||
}
|
||||
|
||||
STATUS_CODES = (
|
||||
(status_codes.QUEUED, 'QUEUED'),
|
||||
|
|
@ -146,6 +157,7 @@ class Task(models.Model):
|
|||
status = models.IntegerField(choices=STATUS_CODES, db_index=True, null=True, blank=True, help_text="Current status of the task")
|
||||
last_error = models.TextField(null=True, blank=True, help_text="The last processing error received")
|
||||
options = fields.JSONField(default=dict(), blank=True, help_text="Options that are being used to process this task", validators=[validate_task_options])
|
||||
available_assets = fields.ArrayField(models.CharField(max_length=80), default=list(), blank=True, help_text="List of available assets to download")
|
||||
console_output = models.TextField(null=False, default="", blank=True, help_text="Console output of the OpenDroneMap's process")
|
||||
ground_control_points = models.FileField(null=True, blank=True, upload_to=gcp_directory_path, help_text="Optional Ground Control Points file to use for processing")
|
||||
|
||||
|
|
@ -196,7 +208,7 @@ class Task(models.Model):
|
|||
logger.warning("Project changed for task {}, but either {} doesn't exist, or {} already exists. This doesn't look right, so we will not move any files.".format(self,
|
||||
old_task_folder,
|
||||
new_task_folder))
|
||||
except (shutil.Error, GDALException) as e:
|
||||
except shutil.Error as e:
|
||||
logger.warning("Could not move assets folder for task {}. We're going to proceed anyway, but you might experience issues: {}".format(self, e))
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
|
|
@ -218,35 +230,46 @@ class Task(models.Model):
|
|||
"assets",
|
||||
*args)
|
||||
|
||||
def is_asset_available_slow(self, asset):
|
||||
"""
|
||||
Checks whether a particular asset is available in the file system
|
||||
Generally this should never be used directly, as it's slow. Use the available_assets field
|
||||
in the database instead.
|
||||
:param asset: one of ASSETS_MAP keys
|
||||
:return: boolean
|
||||
"""
|
||||
if asset in self.ASSETS_MAP:
|
||||
value = self.ASSETS_MAP[asset]
|
||||
if isinstance(value, str):
|
||||
return os.path.exists(self.assets_path(value))
|
||||
elif isinstance(value, dict):
|
||||
if 'deferred_compress_dir' in value:
|
||||
return os.path.exists(self.assets_path(value['deferred_compress_dir']))
|
||||
|
||||
return False
|
||||
|
||||
def get_asset_download_path(self, asset):
|
||||
"""
|
||||
Get the path to an asset download
|
||||
:param asset: one of ASSET_DOWNLOADS
|
||||
:param asset: one of ASSETS_MAP keys
|
||||
:return: path
|
||||
"""
|
||||
if asset == 'texturedmodel':
|
||||
return self.assets_path(os.path.basename(self.get_textured_model_archive()))
|
||||
else:
|
||||
map = {
|
||||
'all': 'all.zip',
|
||||
'geotiff': os.path.join('odm_orthophoto', 'odm_orthophoto.tif'),
|
||||
'las': os.path.join('odm_georeferencing', 'odm_georeferenced_model.las'),
|
||||
'ply': os.path.join('odm_georeferencing', 'odm_georeferenced_model.ply'),
|
||||
'csv': os.path.join('odm_georeferencing', 'odm_georeferenced_model.csv')
|
||||
}
|
||||
|
||||
# BEGIN MIGRATION
|
||||
# Temporary check for naming migration from *model.ply.las to *model.las
|
||||
# This can be deleted at some point in the future
|
||||
if asset == 'las' and not os.path.exists(self.assets_path(map['las'])):
|
||||
logger.info("migration: using odm_georeferenced_model.ply.las instead of odm_georeferenced_model.las")
|
||||
map['las'] = os.path.join('odm_georeferencing', 'odm_georeferenced_model.ply.las')
|
||||
# END MIGRATION
|
||||
if asset in self.ASSETS_MAP:
|
||||
value = self.ASSETS_MAP[asset]
|
||||
if isinstance(value, str):
|
||||
return self.assets_path(value)
|
||||
|
||||
elif isinstance(value, dict):
|
||||
if 'deferred_path' in value and 'deferred_compress_dir' in value:
|
||||
return self.generate_deferred_asset(value['deferred_path'], value['deferred_compress_dir'])
|
||||
else:
|
||||
raise FileNotFoundError("{} is not a valid asset (invalid dict values)".format(asset))
|
||||
|
||||
if asset in map:
|
||||
return self.assets_path(map[asset])
|
||||
else:
|
||||
raise FileNotFoundError("{} is not a valid asset".format(asset))
|
||||
raise FileNotFoundError("{} is not a valid asset (invalid map)".format(asset))
|
||||
else:
|
||||
raise FileNotFoundError("{} is not a valid asset".format(asset))
|
||||
|
||||
def process(self):
|
||||
"""
|
||||
|
|
@ -390,8 +413,13 @@ class Task(models.Model):
|
|||
|
||||
if self.status == status_codes.COMPLETED:
|
||||
assets_dir = self.assets_path("")
|
||||
if not os.path.exists(assets_dir):
|
||||
os.makedirs(assets_dir)
|
||||
|
||||
# Remove previous assets directory
|
||||
if os.path.exists(assets_dir):
|
||||
logger.info("Removing old assets directory: {} for {}".format(assets_dir, self))
|
||||
shutil.rmtree(assets_dir)
|
||||
|
||||
os.makedirs(assets_dir)
|
||||
|
||||
logger.info("Downloading all.zip for {}".format(self))
|
||||
|
||||
|
|
@ -422,11 +450,6 @@ class Task(models.Model):
|
|||
|
||||
logger.info("Populated orthophoto_extent for {}".format(self))
|
||||
|
||||
# Remove old odm_texturing.zip archive (if any)
|
||||
textured_model_archive = self.assets_path(self.get_textured_model_filename())
|
||||
if os.path.exists(textured_model_archive):
|
||||
os.remove(textured_model_archive)
|
||||
|
||||
self.save()
|
||||
else:
|
||||
# FAILED, CANCELED
|
||||
|
|
@ -458,35 +481,32 @@ class Task(models.Model):
|
|||
}
|
||||
}
|
||||
|
||||
def get_textured_model_filename(self):
|
||||
return "odm_texturing.zip"
|
||||
def generate_deferred_asset(self, archive, directory):
|
||||
"""
|
||||
:param archive: path of the destination .zip file (relative to /assets/ directory)
|
||||
:param directory: path of the source directory to compress (relative to /assets/ directory)
|
||||
:return: full path of the generated archive
|
||||
"""
|
||||
archive_path = self.assets_path(archive)
|
||||
directory_path = self.assets_path(directory)
|
||||
|
||||
def get_textured_model_archive(self):
|
||||
archive_path = self.assets_path(self.get_textured_model_filename())
|
||||
textured_model_directory = self.assets_path("odm_texturing")
|
||||
|
||||
if not os.path.exists(textured_model_directory):
|
||||
raise FileNotFoundError("{} does not exist".format(textured_model_directory))
|
||||
if not os.path.exists(directory_path):
|
||||
raise FileNotFoundError("{} does not exist".format(directory_path))
|
||||
|
||||
if not os.path.exists(archive_path):
|
||||
shutil.make_archive(os.path.splitext(archive_path)[0], 'zip', textured_model_directory)
|
||||
shutil.make_archive(os.path.splitext(archive_path)[0], 'zip', directory_path)
|
||||
|
||||
return archive_path
|
||||
|
||||
def get_available_assets(self):
|
||||
# We make some assumptions for the sake of speed
|
||||
# as checking the filesystem would be slow
|
||||
if self.status == status_codes.COMPLETED:
|
||||
assets = list(self.ASSET_DOWNLOADS)
|
||||
def update_available_assets_field(self, commit=False):
|
||||
"""
|
||||
Updates the available_assets field with the actual types of assets available
|
||||
:param commit: when True also saves the model, otherwise the user should manually call save()
|
||||
"""
|
||||
all_assets = list(self.ASSETS_MAP.keys())
|
||||
self.available_assets = [asset for asset in all_assets if self.is_asset_available_slow(asset)]
|
||||
if commit: self.save()
|
||||
|
||||
if self.orthophoto_extent is None:
|
||||
assets.remove('geotiff')
|
||||
assets.remove('las')
|
||||
assets.remove('csv')
|
||||
|
||||
return assets
|
||||
else:
|
||||
return []
|
||||
|
||||
def delete(self, using=None, keep_parents=False):
|
||||
directory_to_delete = os.path.join(settings.MEDIA_ROOT,
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ class ModelView extends React.Component {
|
|||
}
|
||||
|
||||
hasGeoreferencedAssets(){
|
||||
return this.props.task.available_assets.indexOf('geotiff') !== -1;
|
||||
return this.props.task.available_assets.indexOf('orthophoto.tif') !== -1;
|
||||
}
|
||||
|
||||
objFilePath(){
|
||||
|
|
@ -169,7 +169,7 @@ class ModelView extends React.Component {
|
|||
|
||||
// React render
|
||||
render(){
|
||||
const showSwitchModeButton = this.props.task.available_assets.indexOf('geotiff') !== -1;
|
||||
const showSwitchModeButton = this.hasGeoreferencedAssets();
|
||||
const hideWithTexturedModel = {display: this.state.showTexturedModel ? "none" : "block"};
|
||||
|
||||
return (<div className="model-view">
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ class AssetDownload{
|
|||
}
|
||||
|
||||
downloadUrl(project_id, task_id){
|
||||
return `/api/projects/${project_id}/tasks/${task_id}/download/${this.asset}/`;
|
||||
return `/api/projects/${project_id}/tasks/${task_id}/download/${this.asset}`;
|
||||
}
|
||||
|
||||
get separator(){
|
||||
|
|
@ -31,13 +31,14 @@ class AssetDownloadSeparator extends AssetDownload{
|
|||
const api = {
|
||||
all: function() {
|
||||
return [
|
||||
new AssetDownload("GeoTIFF","geotiff","fa fa-map-o"),
|
||||
new AssetDownload("Textured Model","texturedmodel","fa fa-connectdevelop"),
|
||||
new AssetDownload("LAS","las","fa fa-cube"),
|
||||
new AssetDownload("PLY","ply","fa fa-cube"),
|
||||
new AssetDownload("CSV","csv","fa fa-cube"),
|
||||
new AssetDownload("Orthophoto (GeoTIFF)","orthophoto.tif","fa fa-map-o"),
|
||||
new AssetDownload("Orthophoto (PNG)","orthophoto.png","fa fa-picture-o"),
|
||||
new AssetDownload("Point Cloud (LAS)","georeferenced_model.las","fa fa-cube"),
|
||||
new AssetDownload("Point Cloud (PLY)","georeferenced_model.ply","fa fa-cube"),
|
||||
new AssetDownload("Point Cloud (CSV)","georeferenced_model.csv","fa fa-cube"),
|
||||
new AssetDownload("Textured Model","textured_model.zip","fa fa-connectdevelop"),
|
||||
new AssetDownloadSeparator(),
|
||||
new AssetDownload("All Assets","all","fa fa-file-archive-o")
|
||||
new AssetDownload("All Assets","all.zip","fa fa-file-archive-o")
|
||||
];
|
||||
},
|
||||
|
||||
|
|
|
|||
|
|
@ -257,7 +257,7 @@ class TaskListItem extends React.Component {
|
|||
|
||||
let expanded = "";
|
||||
if (this.state.expanded){
|
||||
let showGeotiffMissingWarning = false,
|
||||
let showOrthophotoMissingWarning = false,
|
||||
showMemoryErrorWarning = this.state.memoryError && task.status == statusCodes.FAILED,
|
||||
showExitedWithCodeOneHints = task.last_error === "Process exited with code 1" && !showMemoryErrorWarning && task.status == statusCodes.FAILED,
|
||||
memoryErrorLink = this.isMacOS() ? "http://stackoverflow.com/a/39720010" : "https://docs.docker.com/docker-for-windows/#advanced";
|
||||
|
|
@ -270,12 +270,12 @@ class TaskListItem extends React.Component {
|
|||
};
|
||||
|
||||
if (task.status === statusCodes.COMPLETED){
|
||||
if (task.available_assets.indexOf("geotiff") !== -1){
|
||||
if (task.available_assets.indexOf("orthophoto.tif") !== -1){
|
||||
addActionButton(" View Orthophoto", "btn-primary", "fa fa-globe", () => {
|
||||
location.href = `/map/project/${task.project}/task/${task.id}/`;
|
||||
});
|
||||
}else{
|
||||
showGeotiffMissingWarning = true;
|
||||
showOrthophotoMissingWarning = true;
|
||||
}
|
||||
|
||||
addActionButton(" View 3D Model", "btn-primary", "fa fa-cube", () => {
|
||||
|
|
@ -347,8 +347,8 @@ class TaskListItem extends React.Component {
|
|||
: ""}
|
||||
{/* TODO: List of images? */}
|
||||
|
||||
{showGeotiffMissingWarning ?
|
||||
<div className="task-warning"><i className="fa fa-warning"></i> <span>An orthophoto could not be generated. To generate one, make sure GPS information is embedded in the EXIF tags of your images.</span></div> : ""}
|
||||
{showOrthophotoMissingWarning ?
|
||||
<div className="task-warning"><i className="fa fa-warning"></i> <span>An orthophoto could not be generated. To generate one, make sure GPS information is embedded in the EXIF tags of your images, or use a Ground Control Points (GCP) file.</span></div> : ""}
|
||||
|
||||
</div>
|
||||
<div className="col-md-8">
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ def model_display(request, project_pk=None, task_pk=None):
|
|||
'task': json.dumps({
|
||||
'id': task.id,
|
||||
'project': project.id,
|
||||
'available_assets': task.get_available_assets()
|
||||
'available_assets': task.available_assets
|
||||
})
|
||||
}.items()
|
||||
})
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue