Merge pull request #702 from ProjectBarks/master

[cesium-ion] Texture model prune

plugins/cesium-ion/api_views.py

@@ -4,6 +4,7 @@ import logging
 import requests
 from os import path
 from enum import Enum
+from shutil import rmtree
 from itertools import chain as iter_chain
 
 from app.plugins.views import TaskView
@@ -21,7 +22,7 @@ from rest_framework import status
 from rest_framework import serializers
 
 from .globals import PROJECT_NAME, ION_API_URL
-
+from .model_tools import to_ion_texture_model, IonInvalidZip
 
 pluck = lambda dic, *keys: [dic[k] if k in dic else None for k in keys]
 
@@ -369,6 +370,7 @@ def upload_to_ion(
     asset_logger = LoggerAdapter(prefix=f"Task {task_id} {asset_type}", logger=logger)
     asset_type = AssetType[asset_type]
     asset_info = get_asset_info(task_id, asset_type)
+    del_directory = None
 
     try:
         import boto3
@@ -380,6 +382,17 @@ def upload_to_ion(
         import boto3
 
     try:
+        # Update asset_path based off
+        if asset_type == AssetType.TEXTURED_MODEL:
+            try:
+                asset_path, del_directory = to_ion_texture_model(asset_path)
+                logger.info("Created ion texture model!")
+            except IonInvalidZip as e:
+                logger.info("Non geo-referenced texture model, using default file.")
+            except Exception as e:
+                logger.warning("Failed to convert to ion texture model")
+                logger.warning(e)
+
         headers = {"Authorization": f"Bearer {token}"}
         data = {
             "name": name,
@@ -467,4 +480,7 @@ def upload_to_ion(
         asset_info["error"] = str(e)
         asset_logger.error(e)
 
+    if del_directory != None:
+        rmtree(del_directory)
+
     set_asset_info(task_id, asset_type, asset_info)

plugins/cesium-ion/manifest.json

@@ -2,7 +2,7 @@
 	"name": "Cesium ion",
 	"webodmMinVersion": "0.6.0",
 	"description": "Upload and tile ODM assets with Cesium ion.",
-	"version": "1.2.0",
+	"version": "1.2.1",
 	"author": "Cesium GS, Inc",
 	"email": "hello@cesium.com",
 	"repository": "",

plugins/cesium-ion/model_tools.py

@@ -0,0 +1,71 @@
+from os import path, mkdir, walk, remove as removeFile
+from zipfile import ZipFile, ZIP_DEFLATED
+from shutil import rmtree
+from tempfile import mkdtemp
+
+
+DELETE_EXTENSIONS = (".conf", ".vec", ".spt")
+OBJ_FILE_EXTENSION = ".obj"
+MTL_FILE_EXTENSION = ".mtl"
+
+
+class IonInvalidZip(Exception):
+    pass
+
+
+def file_walk(directory):
+    for root, _, file_names in walk(directory):
+        for file_name in file_names:
+            yield path.join(root, file_name)
+
+
+def zip_dir(zip_name, directory, destructive=False):
+    with ZipFile(zip_name, mode="w", compression=ZIP_DEFLATED) as zipfile:
+        for file_path in file_walk(directory):
+            relpath = path.relpath(file_path, directory)
+            zipfile.write(file_path, relpath)
+            if destructive:
+                removeFile(file_path)
+
+
+def to_ion_texture_model(texture_model_path, dest_directory=None, minimize_space=True):
+    is_tmp = False
+    if dest_directory is None:
+        is_tmp = True
+        dest_directory = mkdtemp()
+    dest_file = path.join(dest_directory, path.basename(texture_model_path))
+    try:
+        unzip_dir = path.join(dest_directory, "_tmp")
+        mkdir(unzip_dir)
+        with ZipFile(texture_model_path) as zipfile:
+            zipfile.extractall(unzip_dir)
+
+        files_to_delete = set()
+        found_geo = False
+        for file_name in file_walk(unzip_dir):
+            if file_name.endswith(DELETE_EXTENSIONS):
+                files_to_delete.add(file_name)
+            elif file_name.endswith(".obj"):
+                if "_geo" in path.basename(file_name):
+                    found_geo = True
+                else:
+                    file_name = path.splitext(file_name)[0]
+                    files_to_delete.add(file_name + OBJ_FILE_EXTENSION)
+                    files_to_delete.add(file_name + MTL_FILE_EXTENSION)
+
+        if not found_geo:
+            raise IonInvalidZip("Unable to find geo file")
+
+        for file_name in files_to_delete:
+            if not path.isfile(file_name):
+                continue
+            removeFile(file_name)
+
+        zip_dir(dest_file, unzip_dir, destructive=minimize_space)
+        rmtree(unzip_dir)
+    except Exception as e:
+        if is_tmp:
+            rmtree(dest_directory)
+        raise e
+
+    return dest_file, dest_directory