From 299d67b1c5e9ac470cdb37954a8d7e83cc5e6063 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CBrandon?= Date: Tue, 16 Jul 2019 13:25:49 -0400 Subject: [PATCH 1/2] Texture model prune Texture models duplicate their internal data so files are now trimmed based off the availability of geo refrenced files or it will fallback to default files --- plugins/cesium-ion/api_views.py | 17 +++++++- plugins/cesium-ion/model_tools.py | 67 +++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 plugins/cesium-ion/model_tools.py diff --git a/plugins/cesium-ion/api_views.py b/plugins/cesium-ion/api_views.py index 533b8e0d..dc261421 100644 --- a/plugins/cesium-ion/api_views.py +++ b/plugins/cesium-ion/api_views.py @@ -4,6 +4,7 @@ import logging import requests from os import path from enum import Enum +from shutil import rmtree from itertools import chain as iter_chain from app.plugins.views import TaskView @@ -21,7 +22,7 @@ from rest_framework import status from rest_framework import serializers from .globals import PROJECT_NAME, ION_API_URL - +from .model_tools import to_ion_texture_model pluck = lambda dic, *keys: [dic[k] if k in dic else None for k in keys] @@ -369,6 +370,7 @@ def upload_to_ion( asset_logger = LoggerAdapter(prefix=f"Task {task_id} {asset_type}", logger=logger) asset_type = AssetType[asset_type] asset_info = get_asset_info(task_id, asset_type) + del_directory = None try: import boto3 @@ -380,6 +382,16 @@ def upload_to_ion( import boto3 try: + # Update asset_path based off + if asset_type == AssetType.TEXTURED_MODEL: + try: + asset_path, del_directory = to_ion_texture_model(asset_path) + logger.info("Created ion texture model!") + except Exception as e: + logger.warning("Failed to convert to ion texture model") + logger.warning(e) + pass + headers = {"Authorization": f"Bearer {token}"} data = { "name": name, @@ -467,4 +479,7 @@ def upload_to_ion( asset_info["error"] = str(e) asset_logger.error(e) + if del_directory != None: + rmtree(del_directory) + set_asset_info(task_id, asset_type, asset_info) diff --git a/plugins/cesium-ion/model_tools.py b/plugins/cesium-ion/model_tools.py new file mode 100644 index 00000000..7925fc8b --- /dev/null +++ b/plugins/cesium-ion/model_tools.py @@ -0,0 +1,67 @@ +from os import path, mkdir, walk, remove as removeFile +from zipfile import ZipFile, ZIP_DEFLATED +from shutil import rmtree +from tempfile import mkdtemp + + +DELETE_EXTENSIONS = (".conf", ".vec", ".spt") +OBJ_FILE_EXTENSION = ".obj" +MTL_FILE_EXTENSION = ".mtl" + + +def file_walk(directory): + for root, _, file_names in walk(directory): + for file_name in file_names: + yield path.join(root, file_name) + + +def zip_dir(zip_name, directory, destructive=False): + with ZipFile(zip_name, mode="w", compression=ZIP_DEFLATED) as zipfile: + for file_path in file_walk(directory): + relpath = path.relpath(file_path, directory) + zipfile.write(file_path, relpath) + if destructive: + removeFile(file_path) + + +def to_ion_texture_model(texture_model_path, dest_directory=None, minimize_space=True): + is_tmp = False + if dest_directory is None: + is_tmp = True + dest_directory = mkdtemp() + dest_file = path.join(dest_directory, path.basename(texture_model_path)) + try: + unzip_dir = path.join(dest_directory, "_tmp") + mkdir(unzip_dir) + with ZipFile(texture_model_path) as zipfile: + zipfile.extractall(unzip_dir) + + files_to_delete = set() + found_geo = False + for file_name in file_walk(unzip_dir): + if file_name.endswith(DELETE_EXTENSIONS): + files_to_delete.add(file_name) + if file_name.endswith(".obj"): + if "_geo" in file_name: + found_geo = True + else: + file_name = path.splitext(file_name)[0] + files_to_delete.add(file_name + OBJ_FILE_EXTENSION) + files_to_delete.add(file_name + MTL_FILE_EXTENSION) + + if not found_geo: + raise Exception("Unable to find geo file") + + for file_name in files_to_delete: + if not path.isfile(file_name): + continue + removeFile(file_name) + + zip_dir(dest_file, unzip_dir, destructive=minimize_space) + rmtree(unzip_dir) + except Exception as e: + if is_tmp: + rmtree(dest_directory) + raise e + + return dest_file, dest_directory From 275fae846a543c343dbda7679f4beb3d4ab27efa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CBrandon?= Date: Tue, 16 Jul 2019 13:40:30 -0400 Subject: [PATCH 2/2] Silent failure + Bump Version --- plugins/cesium-ion/api_views.py | 5 +++-- plugins/cesium-ion/manifest.json | 2 +- plugins/cesium-ion/model_tools.py | 10 +++++++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/plugins/cesium-ion/api_views.py b/plugins/cesium-ion/api_views.py index dc261421..a727ac99 100644 --- a/plugins/cesium-ion/api_views.py +++ b/plugins/cesium-ion/api_views.py @@ -22,7 +22,7 @@ from rest_framework import status from rest_framework import serializers from .globals import PROJECT_NAME, ION_API_URL -from .model_tools import to_ion_texture_model +from .model_tools import to_ion_texture_model, IonInvalidZip pluck = lambda dic, *keys: [dic[k] if k in dic else None for k in keys] @@ -387,10 +387,11 @@ def upload_to_ion( try: asset_path, del_directory = to_ion_texture_model(asset_path) logger.info("Created ion texture model!") + except IonInvalidZip as e: + logger.info("Non geo-referenced texture model, using default file.") except Exception as e: logger.warning("Failed to convert to ion texture model") logger.warning(e) - pass headers = {"Authorization": f"Bearer {token}"} data = { diff --git a/plugins/cesium-ion/manifest.json b/plugins/cesium-ion/manifest.json index a9f94777..083447c3 100644 --- a/plugins/cesium-ion/manifest.json +++ b/plugins/cesium-ion/manifest.json @@ -2,7 +2,7 @@ "name": "Cesium ion", "webodmMinVersion": "0.6.0", "description": "Upload and tile ODM assets with Cesium ion.", - "version": "1.2.0", + "version": "1.2.1", "author": "Cesium GS, Inc", "email": "hello@cesium.com", "repository": "", diff --git a/plugins/cesium-ion/model_tools.py b/plugins/cesium-ion/model_tools.py index 7925fc8b..56f5c113 100644 --- a/plugins/cesium-ion/model_tools.py +++ b/plugins/cesium-ion/model_tools.py @@ -9,6 +9,10 @@ OBJ_FILE_EXTENSION = ".obj" MTL_FILE_EXTENSION = ".mtl" +class IonInvalidZip(Exception): + pass + + def file_walk(directory): for root, _, file_names in walk(directory): for file_name in file_names: @@ -41,8 +45,8 @@ def to_ion_texture_model(texture_model_path, dest_directory=None, minimize_space for file_name in file_walk(unzip_dir): if file_name.endswith(DELETE_EXTENSIONS): files_to_delete.add(file_name) - if file_name.endswith(".obj"): - if "_geo" in file_name: + elif file_name.endswith(".obj"): + if "_geo" in path.basename(file_name): found_geo = True else: file_name = path.splitext(file_name)[0] @@ -50,7 +54,7 @@ def to_ion_texture_model(texture_model_path, dest_directory=None, minimize_space files_to_delete.add(file_name + MTL_FILE_EXTENSION) if not found_geo: - raise Exception("Unable to find geo file") + raise IonInvalidZip("Unable to find geo file") for file_name in files_to_delete: if not path.isfile(file_name):