mirror
/
OpenDroneMap-WebODM
kopia lustrzana https://github.com/OpenDroneMap/WebODM
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Changed available_assets code to be faster, easier to maintain, still need to write unit tests

pull/230/head
Piero Toffanin 2017-07-07 15:34:02 -04:00
rodzic b413966202
commit 9039dca53d
8 zmienionych plików z 121 dodań i 74 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

8
app/api/tasks.py
Wyświetl plik

@ -29,18 +29,14 @@ class TaskSerializer(serializers.ModelSerializer):
project = serializers.PrimaryKeyRelatedField(queryset=models.Project.objects.all()) project = serializers.PrimaryKeyRelatedField(queryset=models.Project.objects.all())
processing_node = serializers.PrimaryKeyRelatedField(queryset=ProcessingNode.objects.all()) processing_node = serializers.PrimaryKeyRelatedField(queryset=ProcessingNode.objects.all())
images_count = serializers.SerializerMethodField() images_count = serializers.SerializerMethodField()
available_assets = serializers.SerializerMethodField()
def get_images_count(self, obj): def get_images_count(self, obj):
return obj.imageupload_set.count() return obj.imageupload_set.count()
def get_available_assets(self, obj):
return obj.get_available_assets()
class Meta: class Meta:
model = models.Task model = models.Task
exclude = ('processing_lock', 'console_output', 'orthophoto_extent', ) exclude = ('processing_lock', 'console_output', 'orthophoto_extent', )
read_only_fields = ('processing_time', 'status', 'last_error', 'created_at', 'pending_action', ) read_only_fields = ('processing_time', 'status', 'last_error', 'created_at', 'pending_action', 'available_assets', )
class TaskViewSet(viewsets.ViewSet): class TaskViewSet(viewsets.ViewSet):
""" """
@ -240,7 +236,7 @@ class TaskDownloads(TaskNestedView):
file = open(asset_path, "rb") file = open(asset_path, "rb")
response = HttpResponse(FileWrapper(file), response = HttpResponse(FileWrapper(file),
content_type=(mimetypes.guess_type(asset_filename)[0] or "application/zip")) content_type=(mimetypes.guess_type(asset_filename)[0] or "application/zip"))
response['Content-Disposition'] = "attachment; filename={}".format(asset_filename) response['Content-Disposition'] = "attachment; filename={}".format(asset)
return response return response
""" """

3
app/api/urls.py
Wyświetl plik

@ -20,7 +20,8 @@ urlpatterns = [
url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/tiles/(?P<z>[\d]+)/(?P<x>[\d]+)/(?P<y>[\d]+)\.png$', TaskTiles.as_view()), url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/tiles/(?P<z>[\d]+)/(?P<x>[\d]+)/(?P<y>[\d]+)\.png$', TaskTiles.as_view()),
url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/tiles\.json$', TaskTilesJson.as_view()), url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/tiles\.json$', TaskTilesJson.as_view()),
url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/download/(?P<asset>[^/.]+)/$', TaskDownloads.as_view()), url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/download/(?P<asset>.+)$', TaskDownloads.as_view()),
url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/assets/(?P<unsafe_asset_path>.+)$', TaskAssets.as_view()), url(r'projects/(?P<project_pk>[^/.]+)/tasks/(?P<pk>[^/.]+)/assets/(?P<unsafe_asset_path>.+)$', TaskAssets.as_view()),
url(r'^auth/', include('rest_framework.urls')), url(r'^auth/', include('rest_framework.urls')),

29
app/migrations/0006_task_available_assets.py 100644
Wyświetl plik

@ -0,0 +1,29 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.1 on 2017-07-07 18:05
from __future__ import unicode_literals
import django.contrib.postgres.fields
from django.db import migrations, models
from app.models import Task
def detect_available_assets(apps, schema_editor):
for t in Task.objects.all():
print("Updating {}".format(t))
t.update_available_assets_field(True)
class Migration(migrations.Migration):
dependencies = [
('app', '0005_auto_20170707_1014'),
]
operations = [
migrations.AddField(
model_name='task',
name='available_assets',
field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=80), blank=True, default=[], help_text='List of available assets to download', size=None),
),
migrations.RunPython(detect_available_assets),
]

124
app/models.py
Wyświetl plik

@ -120,7 +120,18 @@ def validate_task_options(value):
class Task(models.Model): class Task(models.Model):
ASSET_DOWNLOADS = ("all", "geotiff", "texturedmodel", "las", "csv", "ply",) ASSETS_MAP = {
'all.zip': 'all.zip',
'orthophoto.tif': os.path.join('odm_orthophoto', 'odm_orthophoto.tif'),
'orthophoto.png': os.path.join('odm_orthophoto', 'odm_orthophoto.png'),
'georeferenced_model.las': os.path.join('odm_georeferencing', 'odm_georeferenced_model.las'),
'georeferenced_model.ply': os.path.join('odm_georeferencing', 'odm_georeferenced_model.ply'),
'georeferenced_model.csv': os.path.join('odm_georeferencing', 'odm_georeferenced_model.csv'),
'textured_model.zip': {
'deferred_path': 'textured_model.zip',
'deferred_compress_dir': 'odm_texturing'
}
}
STATUS_CODES = ( STATUS_CODES = (
(status_codes.QUEUED, 'QUEUED'), (status_codes.QUEUED, 'QUEUED'),
@ -146,6 +157,7 @@ class Task(models.Model):
status = models.IntegerField(choices=STATUS_CODES, db_index=True, null=True, blank=True, help_text="Current status of the task") status = models.IntegerField(choices=STATUS_CODES, db_index=True, null=True, blank=True, help_text="Current status of the task")
last_error = models.TextField(null=True, blank=True, help_text="The last processing error received") last_error = models.TextField(null=True, blank=True, help_text="The last processing error received")
options = fields.JSONField(default=dict(), blank=True, help_text="Options that are being used to process this task", validators=[validate_task_options]) options = fields.JSONField(default=dict(), blank=True, help_text="Options that are being used to process this task", validators=[validate_task_options])
available_assets = fields.ArrayField(models.CharField(max_length=80), default=list(), blank=True, help_text="List of available assets to download")
console_output = models.TextField(null=False, default="", blank=True, help_text="Console output of the OpenDroneMap's process") console_output = models.TextField(null=False, default="", blank=True, help_text="Console output of the OpenDroneMap's process")
ground_control_points = models.FileField(null=True, blank=True, upload_to=gcp_directory_path, help_text="Optional Ground Control Points file to use for processing") ground_control_points = models.FileField(null=True, blank=True, upload_to=gcp_directory_path, help_text="Optional Ground Control Points file to use for processing")
@ -196,7 +208,7 @@ class Task(models.Model):
logger.warning("Project changed for task {}, but either {} doesn't exist, or {} already exists. This doesn't look right, so we will not move any files.".format(self, logger.warning("Project changed for task {}, but either {} doesn't exist, or {} already exists. This doesn't look right, so we will not move any files.".format(self,
old_task_folder, old_task_folder,
new_task_folder)) new_task_folder))
except (shutil.Error, GDALException) as e: except shutil.Error as e:
logger.warning("Could not move assets folder for task {}. We're going to proceed anyway, but you might experience issues: {}".format(self, e)) logger.warning("Could not move assets folder for task {}. We're going to proceed anyway, but you might experience issues: {}".format(self, e))
def save(self, *args, **kwargs): def save(self, *args, **kwargs):
@ -218,35 +230,46 @@ class Task(models.Model):
"assets", "assets",
*args) *args)
def is_asset_available_slow(self, asset):
"""
Checks whether a particular asset is available in the file system
Generally this should never be used directly, as it's slow. Use the available_assets field
in the database instead.
:param asset: one of ASSETS_MAP keys
:return: boolean
"""
if asset in self.ASSETS_MAP:
value = self.ASSETS_MAP[asset]
if isinstance(value, str):
return os.path.exists(self.assets_path(value))
elif isinstance(value, dict):
if 'deferred_compress_dir' in value:
return os.path.exists(self.assets_path(value['deferred_compress_dir']))
return False
def get_asset_download_path(self, asset): def get_asset_download_path(self, asset):
""" """
Get the path to an asset download Get the path to an asset download
:param asset: one of ASSET_DOWNLOADS :param asset: one of ASSETS_MAP keys
:return: path :return: path
""" """
if asset == 'texturedmodel':
return self.assets_path(os.path.basename(self.get_textured_model_archive()))
else:
map = {
'all': 'all.zip',
'geotiff': os.path.join('odm_orthophoto', 'odm_orthophoto.tif'),
'las': os.path.join('odm_georeferencing', 'odm_georeferenced_model.las'),
'ply': os.path.join('odm_georeferencing', 'odm_georeferenced_model.ply'),
'csv': os.path.join('odm_georeferencing', 'odm_georeferenced_model.csv')
}
# BEGIN MIGRATION if asset in self.ASSETS_MAP:
# Temporary check for naming migration from *model.ply.las to *model.las value = self.ASSETS_MAP[asset]
# This can be deleted at some point in the future if isinstance(value, str):
if asset == 'las' and not os.path.exists(self.assets_path(map['las'])): return self.assets_path(value)
logger.info("migration: using odm_georeferenced_model.ply.las instead of odm_georeferenced_model.las")
map['las'] = os.path.join('odm_georeferencing', 'odm_georeferenced_model.ply.las') elif isinstance(value, dict):
# END MIGRATION if 'deferred_path' in value and 'deferred_compress_dir' in value:
return self.generate_deferred_asset(value['deferred_path'], value['deferred_compress_dir'])
else:
raise FileNotFoundError("{} is not a valid asset (invalid dict values)".format(asset))
if asset in map:
return self.assets_path(map[asset])
else: else:
raise FileNotFoundError("{} is not a valid asset".format(asset)) raise FileNotFoundError("{} is not a valid asset (invalid map)".format(asset))
else:
raise FileNotFoundError("{} is not a valid asset".format(asset))
def process(self): def process(self):
""" """
@ -390,8 +413,13 @@ class Task(models.Model):
if self.status == status_codes.COMPLETED: if self.status == status_codes.COMPLETED:
assets_dir = self.assets_path("") assets_dir = self.assets_path("")
if not os.path.exists(assets_dir):
os.makedirs(assets_dir) # Remove previous assets directory
if os.path.exists(assets_dir):
logger.info("Removing old assets directory: {} for {}".format(assets_dir, self))
shutil.rmtree(assets_dir)
os.makedirs(assets_dir)
logger.info("Downloading all.zip for {}".format(self)) logger.info("Downloading all.zip for {}".format(self))
@ -422,11 +450,6 @@ class Task(models.Model):
logger.info("Populated orthophoto_extent for {}".format(self)) logger.info("Populated orthophoto_extent for {}".format(self))
# Remove old odm_texturing.zip archive (if any)
textured_model_archive = self.assets_path(self.get_textured_model_filename())
if os.path.exists(textured_model_archive):
os.remove(textured_model_archive)
self.save() self.save()
else: else:
# FAILED, CANCELED # FAILED, CANCELED
@ -458,35 +481,32 @@ class Task(models.Model):
} }
} }
def get_textured_model_filename(self): def generate_deferred_asset(self, archive, directory):
return "odm_texturing.zip" """
:param archive: path of the destination .zip file (relative to /assets/ directory)
:param directory: path of the source directory to compress (relative to /assets/ directory)
:return: full path of the generated archive
"""
archive_path = self.assets_path(archive)
directory_path = self.assets_path(directory)
def get_textured_model_archive(self): if not os.path.exists(directory_path):
archive_path = self.assets_path(self.get_textured_model_filename()) raise FileNotFoundError("{} does not exist".format(directory_path))
textured_model_directory = self.assets_path("odm_texturing")
if not os.path.exists(textured_model_directory):
raise FileNotFoundError("{} does not exist".format(textured_model_directory))
if not os.path.exists(archive_path): if not os.path.exists(archive_path):
shutil.make_archive(os.path.splitext(archive_path)[0], 'zip', textured_model_directory) shutil.make_archive(os.path.splitext(archive_path)[0], 'zip', directory_path)
return archive_path return archive_path
def get_available_assets(self): def update_available_assets_field(self, commit=False):
# We make some assumptions for the sake of speed """
# as checking the filesystem would be slow Updates the available_assets field with the actual types of assets available
if self.status == status_codes.COMPLETED: :param commit: when True also saves the model, otherwise the user should manually call save()
assets = list(self.ASSET_DOWNLOADS) """
all_assets = list(self.ASSETS_MAP.keys())
self.available_assets = [asset for asset in all_assets if self.is_asset_available_slow(asset)]
if commit: self.save()
if self.orthophoto_extent is None:
assets.remove('geotiff')
assets.remove('las')
assets.remove('csv')
return assets
else:
return []
def delete(self, using=None, keep_parents=False): def delete(self, using=None, keep_parents=False):
directory_to_delete = os.path.join(settings.MEDIA_ROOT, directory_to_delete = os.path.join(settings.MEDIA_ROOT,

4
app/static/app/js/ModelView.jsx
Wyświetl plik

@ -49,7 +49,7 @@ class ModelView extends React.Component {
} }
hasGeoreferencedAssets(){ hasGeoreferencedAssets(){
return this.props.task.available_assets.indexOf('geotiff') !== -1; return this.props.task.available_assets.indexOf('orthophoto.tif') !== -1;
} }
objFilePath(){ objFilePath(){
@ -169,7 +169,7 @@ class ModelView extends React.Component {
// React render // React render
render(){ render(){
const showSwitchModeButton = this.props.task.available_assets.indexOf('geotiff') !== -1; const showSwitchModeButton = this.hasGeoreferencedAssets();
const hideWithTexturedModel = {display: this.state.showTexturedModel ? "none" : "block"}; const hideWithTexturedModel = {display: this.state.showTexturedModel ? "none" : "block"};
return (<div className="model-view"> return (<div className="model-view">

15
app/static/app/js/classes/AssetDownloads.js
Wyświetl plik

@ -6,7 +6,7 @@ class AssetDownload{
} }
downloadUrl(project_id, task_id){ downloadUrl(project_id, task_id){
return `/api/projects/${project_id}/tasks/${task_id}/download/${this.asset}/`; return `/api/projects/${project_id}/tasks/${task_id}/download/${this.asset}`;
} }
get separator(){ get separator(){
@ -31,13 +31,14 @@ class AssetDownloadSeparator extends AssetDownload{
const api = { const api = {
all: function() { all: function() {
return [ return [
new AssetDownload("GeoTIFF","geotiff","fa fa-map-o"), new AssetDownload("Orthophoto (GeoTIFF)","orthophoto.tif","fa fa-map-o"),
new AssetDownload("Textured Model","texturedmodel","fa fa-connectdevelop"), new AssetDownload("Orthophoto (PNG)","orthophoto.png","fa fa-picture-o"),
new AssetDownload("LAS","las","fa fa-cube"), new AssetDownload("Point Cloud (LAS)","georeferenced_model.las","fa fa-cube"),
new AssetDownload("PLY","ply","fa fa-cube"), new AssetDownload("Point Cloud (PLY)","georeferenced_model.ply","fa fa-cube"),
new AssetDownload("CSV","csv","fa fa-cube"), new AssetDownload("Point Cloud (CSV)","georeferenced_model.csv","fa fa-cube"),
new AssetDownload("Textured Model","textured_model.zip","fa fa-connectdevelop"),
new AssetDownloadSeparator(), new AssetDownloadSeparator(),
new AssetDownload("All Assets","all","fa fa-file-archive-o") new AssetDownload("All Assets","all.zip","fa fa-file-archive-o")
]; ];
}, },

10
app/static/app/js/components/TaskListItem.jsx
Wyświetl plik

@ -257,7 +257,7 @@ class TaskListItem extends React.Component {
let expanded = ""; let expanded = "";
if (this.state.expanded){ if (this.state.expanded){
let showGeotiffMissingWarning = false, let showOrthophotoMissingWarning = false,
showMemoryErrorWarning = this.state.memoryError && task.status == statusCodes.FAILED, showMemoryErrorWarning = this.state.memoryError && task.status == statusCodes.FAILED,
showExitedWithCodeOneHints = task.last_error === "Process exited with code 1" && !showMemoryErrorWarning && task.status == statusCodes.FAILED, showExitedWithCodeOneHints = task.last_error === "Process exited with code 1" && !showMemoryErrorWarning && task.status == statusCodes.FAILED,
memoryErrorLink = this.isMacOS() ? "http://stackoverflow.com/a/39720010" : "https://docs.docker.com/docker-for-windows/#advanced"; memoryErrorLink = this.isMacOS() ? "http://stackoverflow.com/a/39720010" : "https://docs.docker.com/docker-for-windows/#advanced";
@ -270,12 +270,12 @@ class TaskListItem extends React.Component {
}; };
if (task.status === statusCodes.COMPLETED){ if (task.status === statusCodes.COMPLETED){
if (task.available_assets.indexOf("geotiff") !== -1){ if (task.available_assets.indexOf("orthophoto.tif") !== -1){
addActionButton(" View Orthophoto", "btn-primary", "fa fa-globe", () => { addActionButton(" View Orthophoto", "btn-primary", "fa fa-globe", () => {
location.href = `/map/project/${task.project}/task/${task.id}/`; location.href = `/map/project/${task.project}/task/${task.id}/`;
}); });
}else{ }else{
showGeotiffMissingWarning = true; showOrthophotoMissingWarning = true;
} }
addActionButton(" View 3D Model", "btn-primary", "fa fa-cube", () => { addActionButton(" View 3D Model", "btn-primary", "fa fa-cube", () => {
@ -347,8 +347,8 @@ class TaskListItem extends React.Component {
: ""} : ""}
{/* TODO: List of images? */} {/* TODO: List of images? */}
{showGeotiffMissingWarning ? {showOrthophotoMissingWarning ?
<div className="task-warning"><i className="fa fa-warning"></i> <span>An orthophoto could not be generated. To generate one, make sure GPS information is embedded in the EXIF tags of your images.</span></div> : ""} <div className="task-warning"><i className="fa fa-warning"></i> <span>An orthophoto could not be generated. To generate one, make sure GPS information is embedded in the EXIF tags of your images, or use a Ground Control Points (GCP) file.</span></div> : ""}
</div> </div>
<div className="col-md-8"> <div className="col-md-8">

2
app/views.py
Wyświetl plik

@ -85,7 +85,7 @@ def model_display(request, project_pk=None, task_pk=None):
'task': json.dumps({ 'task': json.dumps({
'id': task.id, 'id': task.id,
'project': project.id, 'project': project.id,
'available_assets': task.get_available_assets() 'available_assets': task.available_assets
}) })
}.items() }.items()
}) })