kopia lustrzana https://github.com/OpenDroneMap/WebODM
Merge pull request #118 from pierotofy/partial
Test task processing without orthophoto results, addition of availabl…pull/123/head
Piero Toffanin
GitHub
commit
49ebd60a3f
|
|
@ -12,6 +12,8 @@ from rest_framework import status, serializers, viewsets, filters, exceptions, p
|
|||
from rest_framework.response import Response
|
||||
from rest_framework.decorators import detail_route
|
||||
from rest_framework.views import APIView
|
||||
|
||||
from nodeodm import status_codes
|
||||
from .common import get_and_check_project, get_tile_json, path_traversal_check
|
||||
|
||||
from app import models, scheduler, pending_actions
|
||||
|
|
@ -27,10 +29,14 @@ class TaskSerializer(serializers.ModelSerializer):
|
|||
project = serializers.PrimaryKeyRelatedField(queryset=models.Project.objects.all())
|
||||
processing_node = serializers.PrimaryKeyRelatedField(queryset=ProcessingNode.objects.all())
|
||||
images_count = serializers.SerializerMethodField()
|
||||
available_assets = serializers.SerializerMethodField()
|
||||
|
||||
def get_images_count(self, obj):
|
||||
return obj.imageupload_set.count()
|
||||
|
||||
def get_available_assets(self, obj):
|
||||
return obj.get_available_assets()
|
||||
|
||||
class Meta:
|
||||
model = models.Task
|
||||
exclude = ('processing_lock', 'console_output', 'orthophoto', )
|
||||
|
|
@ -203,7 +209,7 @@ class TaskTilesJson(TaskNestedView):
|
|||
})
|
||||
|
||||
if task.orthophoto_area is None:
|
||||
raise exceptions.ValidationError("An orthophoto has not been processed for this task. Tiles are not available yet.")
|
||||
raise exceptions.ValidationError("An orthophoto has not been processed for this task. Tiles are not available.")
|
||||
|
||||
json = get_tile_json(task.name, [
|
||||
'/api/projects/{}/tasks/{}/tiles/{{z}}/{{x}}/{{y}}.png'.format(task.project.id, task.id)
|
||||
|
|
@ -222,38 +228,22 @@ class TaskDownloads(TaskNestedView):
|
|||
"""
|
||||
task = self.get_and_check_task(request, pk, project_pk)
|
||||
|
||||
allowed_assets = {
|
||||
'all': 'all.zip',
|
||||
'geotiff': os.path.join('odm_orthophoto', 'odm_orthophoto.tif'),
|
||||
'texturedmodel': '_SEE_PATH_BELOW_',
|
||||
'las': os.path.join('odm_georeferencing', 'odm_georeferenced_model.ply.las'),
|
||||
'ply': os.path.join('odm_georeferencing', 'odm_georeferenced_model.ply'),
|
||||
'csv': os.path.join('odm_georeferencing', 'odm_georeferenced_model.csv')
|
||||
}
|
||||
|
||||
# Generate textured mesh if requested
|
||||
# Check and download
|
||||
try:
|
||||
if asset == 'texturedmodel':
|
||||
allowed_assets[asset] = os.path.basename(task.get_textured_model_archive())
|
||||
asset_path = task.get_asset_download_path(asset)
|
||||
except FileNotFoundError:
|
||||
raise exceptions.NotFound("Asset does not exist")
|
||||
|
||||
# Check and download
|
||||
if asset in allowed_assets:
|
||||
asset_path = task.assets_path(allowed_assets[asset])
|
||||
if not os.path.exists(asset_path):
|
||||
raise exceptions.NotFound("Asset does not exist")
|
||||
|
||||
if not os.path.exists(asset_path):
|
||||
raise exceptions.NotFound("Asset does not exist")
|
||||
asset_filename = os.path.basename(asset_path)
|
||||
|
||||
asset_filename = os.path.basename(asset_path)
|
||||
|
||||
file = open(asset_path, "rb")
|
||||
response = HttpResponse(FileWrapper(file),
|
||||
content_type=(mimetypes.guess_type(asset_filename)[0] or "application/zip"))
|
||||
response['Content-Disposition'] = "attachment; filename={}".format(asset_filename)
|
||||
return response
|
||||
else:
|
||||
raise exceptions.NotFound()
|
||||
file = open(asset_path, "rb")
|
||||
response = HttpResponse(FileWrapper(file),
|
||||
content_type=(mimetypes.guess_type(asset_filename)[0] or "application/zip"))
|
||||
response['Content-Disposition'] = "attachment; filename={}".format(asset_filename)
|
||||
return response
|
||||
|
||||
"""
|
||||
Raw access to the task's asset folder resources
|
||||
|
|
|
|||
|
|
@ -70,7 +70,8 @@ class Project(models.Model):
|
|||
|
||||
def get_tile_json_data(self):
|
||||
return [task.get_tile_json_data() for task in self.task_set.filter(
|
||||
status=status_codes.COMPLETED
|
||||
status=status_codes.COMPLETED,
|
||||
orthophoto__isnull=False
|
||||
).only('id', 'project_id')]
|
||||
|
||||
class Meta:
|
||||
|
|
@ -116,6 +117,8 @@ def validate_task_options(value):
|
|||
|
||||
|
||||
class Task(models.Model):
|
||||
ASSET_DOWNLOADS = ("all", "geotiff", "texturedmodel", "las", "csv", "ply",)
|
||||
|
||||
STATUS_CODES = (
|
||||
(status_codes.QUEUED, 'QUEUED'),
|
||||
(status_codes.RUNNING, 'RUNNING'),
|
||||
|
|
@ -221,6 +224,28 @@ class Task(models.Model):
|
|||
"assets",
|
||||
*args)
|
||||
|
||||
def get_asset_download_path(self, asset):
|
||||
"""
|
||||
Get the path to an asset download
|
||||
:param asset: one of ASSET_DOWNLOADS
|
||||
:return: path
|
||||
"""
|
||||
if asset == 'texturedmodel':
|
||||
return self.assets_path(os.path.basename(self.get_textured_model_archive()))
|
||||
else:
|
||||
map = {
|
||||
'all': 'all.zip',
|
||||
'geotiff': os.path.join('odm_orthophoto', 'odm_orthophoto.tif'),
|
||||
'las': os.path.join('odm_georeferencing', 'odm_georeferenced_model.ply.las'),
|
||||
'ply': os.path.join('odm_georeferencing', 'odm_georeferenced_model.ply'),
|
||||
'csv': os.path.join('odm_georeferencing', 'odm_georeferenced_model.csv')
|
||||
}
|
||||
|
||||
if asset in map:
|
||||
return self.assets_path(map[asset])
|
||||
else:
|
||||
raise FileNotFoundError("{} is not a valid asset".format(asset))
|
||||
|
||||
def process(self):
|
||||
"""
|
||||
This method contains the logic for processing tasks asynchronously
|
||||
|
|
@ -439,6 +464,21 @@ class Task(models.Model):
|
|||
|
||||
return archive_path
|
||||
|
||||
def get_available_assets(self):
|
||||
# We make some assumptions for the sake of speed
|
||||
# as checking the filesystem would be slow
|
||||
if self.status == status_codes.COMPLETED:
|
||||
assets = list(self.ASSET_DOWNLOADS)
|
||||
|
||||
if self.orthophoto is None:
|
||||
assets.remove('geotiff')
|
||||
assets.remove('ply')
|
||||
assets.remove('csv')
|
||||
|
||||
return assets
|
||||
else:
|
||||
return []
|
||||
|
||||
def delete(self, using=None, keep_parents=False):
|
||||
directory_to_delete = os.path.join(settings.MEDIA_ROOT,
|
||||
task_directory_path(self.id, self.project.id))
|
||||
|
|
|
|||
|
|
@ -45,8 +45,16 @@ class ModelView extends React.Component {
|
|||
return this.assetsPath() + '/odm_texturing/';
|
||||
}
|
||||
|
||||
hasGeoreferencedAssets(){
|
||||
return this.props.task.available_assets.indexOf('geotiff') !== -1;
|
||||
}
|
||||
|
||||
objFilePath(){
|
||||
return this.texturedModelDirectoryPath() + 'odm_textured_model_geo.obj';
|
||||
let file = this.hasGeoreferencedAssets() ?
|
||||
'odm_textured_model_geo.obj' :
|
||||
'odm_textured_model.obj';
|
||||
|
||||
return this.texturedModelDirectoryPath() + file;
|
||||
}
|
||||
|
||||
mtlFilename(){
|
||||
|
|
@ -64,7 +72,7 @@ class ModelView extends React.Component {
|
|||
sizeType: "Adaptive", // options: "Fixed", "Attenuated", "Adaptive"
|
||||
quality: "Interpolation", // options: "Squares", "Circles", "Interpolation", "Splats"
|
||||
fov: 75, // field of view in degree
|
||||
material: "RGB", // options: "RGB", "Height", "Intensity", "Classification"
|
||||
material: this.hasGeoreferencedAssets() ? "RGB" : "Elevation", // options: "RGB", "Height", "Intensity", "Classification"
|
||||
pointLimit: 1, // max number of points in millions
|
||||
navigation: "Orbit", // options: "Earth", "Orbit", "Flight"
|
||||
pointSize: 1.2
|
||||
|
|
@ -1046,6 +1054,8 @@ class ModelView extends React.Component {
|
|||
|
||||
// React render
|
||||
render(){
|
||||
const showSwitchModeButton = this.props.task.available_assets.indexOf('geotiff') !== -1;
|
||||
|
||||
return (<div className="model-view">
|
||||
<ErrorMessage bind={[this, "error"]} />
|
||||
<div
|
||||
|
|
@ -1057,9 +1067,10 @@ class ModelView extends React.Component {
|
|||
message="Loading textured model..."
|
||||
ref={(domNode) => { this.texturedModelStandby = domNode; }}
|
||||
/>
|
||||
<SwitchModeButton
|
||||
task={this.props.task}
|
||||
type="modelToMap" />
|
||||
{showSwitchModeButton ?
|
||||
<SwitchModeButton
|
||||
task={this.props.task}
|
||||
type="modelToMap" /> : ""}
|
||||
</div>
|
||||
<AssetDownloadButtons task={this.props.task} direction="up" />
|
||||
</div>);
|
||||
|
|
|
|||
|
|
@ -43,6 +43,11 @@ const api = {
|
|||
|
||||
excludeSeparators: function(){
|
||||
return api.all().filter(asset => !asset.separator);
|
||||
},
|
||||
|
||||
// @param assets {String[]} list of assets (example: ['geotiff', 'las']))
|
||||
only: function(assets){
|
||||
return api.all().filter(asset => assets.indexOf(asset.asset) !== -1);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ class AssetDownloadButtons extends React.Component {
|
|||
}
|
||||
|
||||
render(){
|
||||
const assetDownloads = AssetDownloads.all();
|
||||
const assetDownloads = AssetDownloads.only(this.props.task.available_assets);
|
||||
|
||||
return (<div className={"asset-download-buttons btn-group " + (this.props.direction === "up" ? "dropup" : "")}>
|
||||
<button type="button" className="btn btn-sm btn-primary" disabled={this.props.disabled} data-toggle="dropdown">
|
||||
|
|
|
|||
|
|
@ -150,9 +150,8 @@ class Map extends React.Component {
|
|||
})
|
||||
.fail((_, __, err) => done(err))
|
||||
);
|
||||
|
||||
}, err => {
|
||||
if (err) this.setState({error: err.message});
|
||||
if (err) this.setState({error: err.message || JSON.stringify(err)});
|
||||
else{
|
||||
this.map.fitBounds(this.mapBounds);
|
||||
|
||||
|
|
|
|||
|
|
@ -224,6 +224,7 @@ class TaskListItem extends React.Component {
|
|||
if (!task.processing_node) status = "";
|
||||
if (task.pending_action !== null) status = pendingActions.description(task.pending_action);
|
||||
|
||||
let showGeotiffMissingWarning = false;
|
||||
let expanded = "";
|
||||
if (this.state.expanded){
|
||||
let actionButtons = [];
|
||||
|
|
@ -234,9 +235,14 @@ class TaskListItem extends React.Component {
|
|||
};
|
||||
|
||||
if (task.status === statusCodes.COMPLETED){
|
||||
addActionButton(" View Orthophoto", "btn-primary", "fa fa-globe", () => {
|
||||
location.href = `/map/project/${task.project}/task/${task.id}/`;
|
||||
});
|
||||
if (task.available_assets.indexOf("geotiff") !== -1){
|
||||
addActionButton(" View Orthophoto", "btn-primary", "fa fa-globe", () => {
|
||||
location.href = `/map/project/${task.project}/task/${task.id}/`;
|
||||
});
|
||||
}else{
|
||||
showGeotiffMissingWarning = true;
|
||||
}
|
||||
|
||||
addActionButton(" View 3D Model", "btn-primary", "fa fa-cube", () => {
|
||||
location.href = `/3d/project/${task.project}/task/${task.id}/`;
|
||||
});
|
||||
|
|
@ -302,6 +308,9 @@ class TaskListItem extends React.Component {
|
|||
</div>
|
||||
: ""}
|
||||
{/* TODO: List of images? */}
|
||||
|
||||
{showGeotiffMissingWarning ?
|
||||
<div className="geotiff-warning"><i className="fa fa-warning"></i> <span>An orthophoto could not be generated. To generate one, make sure GPS information is embedded in the EXIF tags of your images.</span></div> : ""}
|
||||
</div>
|
||||
<div className="col-md-8">
|
||||
<Console
|
||||
|
|
|
|||
|
|
@ -59,6 +59,16 @@
|
|||
padding-bottom: 16px;
|
||||
padding-left: 16px;
|
||||
|
||||
.geotiff-warning{
|
||||
margin-top: 16px;
|
||||
i.fa{
|
||||
color: #ff8000;
|
||||
}
|
||||
span{
|
||||
font-size: 13px;
|
||||
}
|
||||
}
|
||||
|
||||
.action-buttons{
|
||||
button{
|
||||
margin-right: 4px;
|
||||
|
|
|
|||
|
|
@ -30,9 +30,9 @@ logger = logging.getLogger('app.logger')
|
|||
|
||||
DELAY = 2 # time to sleep for during process launch, background processing, etc.
|
||||
|
||||
def start_processing_node():
|
||||
def start_processing_node(*args):
|
||||
current_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
node_odm = subprocess.Popen(['node', 'index.js', '--port', '11223', '--test'], shell=False,
|
||||
node_odm = subprocess.Popen(['node', 'index.js', '--port', '11223', '--test'] + list(args), shell=False,
|
||||
cwd=os.path.join(current_dir, "..", "..", "nodeodm", "external", "node-OpenDroneMap"))
|
||||
time.sleep(DELAY) # Wait for the server to launch
|
||||
return node_odm
|
||||
|
|
@ -170,9 +170,7 @@ class TestApiTask(BootTransactionTestCase):
|
|||
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
|
||||
|
||||
# Cannot download assets (they don't exist yet)
|
||||
assets = ["all", "geotiff", "texturedmodel", "las", "csv", "ply"]
|
||||
|
||||
for asset in assets:
|
||||
for asset in task.ASSET_DOWNLOADS:
|
||||
res = client.get("/api/projects/{}/tasks/{}/download/{}/".format(project.id, task.id, asset))
|
||||
self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND)
|
||||
|
||||
|
|
@ -202,7 +200,7 @@ class TestApiTask(BootTransactionTestCase):
|
|||
|
||||
# Processing should have started and a UUID is assigned
|
||||
task.refresh_from_db()
|
||||
self.assertTrue(task.status == status_codes.RUNNING)
|
||||
self.assertTrue(task.status in [status_codes.RUNNING, status_codes.COMPLETED]) # Sometimes the task finishes and we can't test for RUNNING state
|
||||
self.assertTrue(len(task.uuid) > 0)
|
||||
|
||||
time.sleep(DELAY)
|
||||
|
|
@ -213,7 +211,7 @@ class TestApiTask(BootTransactionTestCase):
|
|||
self.assertTrue(task.status == status_codes.COMPLETED)
|
||||
|
||||
# Can download assets
|
||||
for asset in assets:
|
||||
for asset in task.ASSET_DOWNLOADS:
|
||||
res = client.get("/api/projects/{}/tasks/{}/download/{}/".format(project.id, task.id, asset))
|
||||
self.assertTrue(res.status_code == status.HTTP_200_OK)
|
||||
|
||||
|
|
@ -348,6 +346,36 @@ class TestApiTask(BootTransactionTestCase):
|
|||
for image in task.imageupload_set.all():
|
||||
self.assertTrue('project/{}/'.format(other_project.id) in image.image.path)
|
||||
|
||||
node_odm.terminate()
|
||||
|
||||
# Restart node-odm as to not generate orthophotos
|
||||
testWatch.clear()
|
||||
node_odm = start_processing_node("--test_skip_orthophotos")
|
||||
res = client.post("/api/projects/{}/tasks/".format(project.id), {
|
||||
'images': [image1, image2],
|
||||
'name': 'test_task_no_orthophoto',
|
||||
'processing_node': pnode.id,
|
||||
'auto_processing_node': 'false'
|
||||
}, format="multipart")
|
||||
self.assertTrue(res.status_code == status.HTTP_201_CREATED)
|
||||
|
||||
scheduler.process_pending_tasks()
|
||||
time.sleep(DELAY)
|
||||
scheduler.process_pending_tasks()
|
||||
|
||||
task = Task.objects.get(pk=res.data['id'])
|
||||
self.assertTrue(task.status == status_codes.COMPLETED)
|
||||
|
||||
# Orthophoto files/directories should be missing
|
||||
self.assertFalse(os.path.exists(task.assets_path("odm_orthophoto", "odm_orthophoto.tif")))
|
||||
self.assertFalse(os.path.exists(task.assets_path("orthophoto_tiles")))
|
||||
|
||||
# Available assets should be missing the geotiff type
|
||||
# but others such as texturedmodel should be available
|
||||
res = client.get("/api/projects/{}/tasks/{}/".format(project.id, task.id))
|
||||
self.assertFalse('geotiff' in res.data['available_assets'])
|
||||
self.assertTrue('texturedmodel' in res.data['available_assets'])
|
||||
|
||||
image1.close()
|
||||
image2.close()
|
||||
node_odm.terminate()
|
||||
|
|
|
|||
|
|
@ -73,7 +73,8 @@ def model_display(request, project_pk=None, task_pk=None):
|
|||
'params': {
|
||||
'task': json.dumps({
|
||||
'id': task.id,
|
||||
'project': project.id
|
||||
'project': project.id,
|
||||
'available_assets': task.get_available_assets()
|
||||
})
|
||||
}.items()
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
Subproject commit a25e688bcb31972671f3735efe17c0b1c32e6da4
|
||||
Subproject commit 6ddbe1feba77fff0901a990f6a391bc48525ba74
|
||||
|
|
@ -8,6 +8,14 @@
|
|||
"project": 27,
|
||||
"processing_node": 10,
|
||||
"images_count": 48,
|
||||
"available_assets": [
|
||||
"all",
|
||||
"geotiff",
|
||||
"texturedmodel",
|
||||
"las",
|
||||
"csv",
|
||||
"ply"
|
||||
],
|
||||
"uuid": "4338d684-91b4-49a2-b907-8ba171894393",
|
||||
"name": "Task Name",
|
||||
"processing_time": 2197417,
|
||||
|
|
@ -34,6 +42,7 @@ id | int | Unique identifier
|
|||
project | int | [Project](#project) ID the task belongs to
|
||||
processing_node | int | The ID of the [Processing Node](#processing-node) this task has been assigned to, or `null` if no [Processing Node](#processing-node) has been assigned.
|
||||
images_count | int | Number of images
|
||||
available_assets | string[] | List of [assets](#download-assets) available for download
|
||||
uuid | string | Unique identifier assigned by a [Processing Node](#processing-node) once processing has started.
|
||||
name | string | User defined name for the task
|
||||
processing_time | int | Milliseconds that have elapsed since the start of processing, or `-1` if no information is available. Useful for displaying a time status report to the user.
|
||||
|
|
@ -109,7 +118,7 @@ Retrieves all [Task](#task) items associated with `project_id`.
|
|||
|
||||
`GET /api/projects/{project_id}/tasks/{task_id}/download/{asset}/`
|
||||
|
||||
After a task has been successfully processed, the user can download several assets from this URL.
|
||||
After a task has been successfully processed, the user can download several assets from this URL. Not all assets are always available. For example if GPS information is missing from the input images, the `geotiff` asset will be missing. You can check the `available_assets` property of a [Task](#task) to see which assets are available for download.
|
||||
|
||||
Asset | Description
|
||||
----- | -----------
|
||||
|
|
@ -189,7 +198,7 @@ RESTART | 3 | [Task](#task) is being restarted
|
|||
|
||||
Status | Code | Description
|
||||
----- | ---- | -----------
|
||||
QUEUED | 10 | [Task](#task)'s files have been uploaded to a [#processing-node](#processing-node) and are waiting to be processed.
|
||||
QUEUED | 10 | [Task](#task)'s files have been uploaded to a [Processing Node](#processing-node) and are waiting to be processed.
|
||||
RUNNING | 20 | [Task](#task) is currently being processed.
|
||||
FAILED | 30 | [Task](#task) has failed for some reason (not enough images, out of memory, Piero forgot to close a parenthesis, etc.)
|
||||
COMPLETED | 40 | [Task](#task) has completed. Assets are be ready to be downloaded.
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue