diff --git a/app/models/task.py b/app/models/task.py index 367134cb..351eed60 100644 --- a/app/models/task.py +++ b/app/models/task.py @@ -1224,6 +1224,7 @@ class Task(models.Model): directory_to_delete = os.path.join(settings.MEDIA_ROOT, task_directory_path(self.id, self.project.id)) + self.clear_task_assets_cache() super(Task, self).delete(using, keep_parents) @@ -1232,7 +1233,6 @@ class Task(models.Model): shutil.rmtree(directory_to_delete) except FileNotFoundError as e: logger.warning(e) - self.clear_task_assets_cache() self.project.owner.profile.clear_used_quota_cache() @@ -1457,10 +1457,15 @@ class Task(models.Model): def get_task_assets_cache(self): + if self.id is None: + return None return os.path.join(settings.MEDIA_CACHE, "task_assets", str(self.id)) def clear_task_assets_cache(self): d = self.get_task_assets_cache() + if d is None: + return + if os.path.isdir(d): try: shutil.rmtree(d) @@ -1472,18 +1477,21 @@ class Task(models.Model): if input_glb is None or (not 'textured_model.glb' in self.available_assets): raise FileNotFoundError("GLB asset does not exist") - size = os.path.getsize(input_glb) - if size <= max_size_mb * 1024 * 1024: - return input_glb - - p, ext = os.path.splitext(input_glb) - base = os.path.basename(p) - cache_dir = self.get_task_assets_cache() - rescale = 1 + if settings.TESTING: + rescale = 2 + else: + size = os.path.getsize(input_glb) + if size <= max_size_mb * 1024 * 1024: + return input_glb + + p, ext = os.path.splitext(input_glb) + base = os.path.basename(p) + cache_dir = self.get_task_assets_cache() + rescale = 1 - while size > max_size_mb * 1024 * 1024: - rescale *= 2 - size = size // 2.6 # Texture size reduction factor (not science) + while size > max_size_mb * 1024 * 1024: + rescale *= 2 + size = size // 2.6 # Texture size reduction factor (not science) output_glb = os.path.join(cache_dir, f"{base}-{rescale}{ext}") if os.path.isfile(output_glb): @@ -1519,10 +1527,13 @@ class Task(models.Model): glbopti_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../scripts/glbopti.js")) output_glb_tmp = output_glb + ".tmp.glb" - subprocess.run(["node", glbopti_path, + params = ["node", glbopti_path, "--input", quote(input_glb), "--output", quote(output_glb_tmp), - "--texture-rescale", str(rescale)], timeout=180) + "--texture-rescale", str(rescale)] + if settings.TESTING: + params += ["--test"] + subprocess.run(params, timeout=180) if not os.path.isfile(output_glb_tmp): raise FileNotFoundError("GLB generation failed") diff --git a/app/scripts/glbopti.js b/app/scripts/glbopti.js index 99a3f5c5..e5decda2 100644 --- a/app/scripts/glbopti.js +++ b/app/scripts/glbopti.js @@ -2,9 +2,9 @@ const fs = require('fs'); const path = require('path'); const { NodeIO, Extension } = require('@gltf-transform/core'); const { KHRONOS_EXTENSIONS } = require('@gltf-transform/extensions'); -const { textureCompress, simplify, weld, draco } = require('@gltf-transform/functions'); -const { MeshoptSimplifier } = require('meshoptimizer'); +const { textureCompress, draco } = require('@gltf-transform/functions'); const draco3d = require('draco3dgltf'); +const encoder = require('sharp'); class CesiumRTC extends Extension { extensionName = 'CESIUM_RTC'; @@ -37,8 +37,8 @@ async function main() { let inputFile = ''; let outputFile = ''; let textureSize = 512; - let simplifyRatio = 1; let textureRescale = null; + let testMode = false; for (let i = 0; i < args.length; i++) { if (args[i] === '--input' && i + 1 < args.length) { @@ -54,13 +54,6 @@ async function main() { process.exit(1); } i++; - } else if (args[i] === '--simplify-ratio' && i + 1 < args.length) { - simplifyRatio = parseFloat(args[i + 1]); - if (isNaN(simplifyRatio) || simplifyRatio < 0 || simplifyRatio > 1){ - console.log(`Invalid simplify ratio: ${args[i + 1]}`); - process.exit(1); - } - i++; } else if (args[i] === '--texture-rescale' && i + 1 < args.length) { textureRescale = parseInt(args[i + 1]); if (isNaN(textureRescale) || textureRescale < 1 || (textureRescale & (textureRescale - 1)) !== 0){ @@ -68,15 +61,24 @@ async function main() { process.exit(1); } i++; + } else if (args[i] === '--test') { + testMode = true; + i++; } } if (!inputFile || !outputFile){ - console.log('Usage: node glb_optimize.js --input --output [--texture-size |--texture-rescale ] [--simplify-ratio ]'); + console.log('Usage: node glb_optimize.js --input --output [--texture-size |--texture-rescale ]'); process.exit(1); } + if (testMode){ + console.log("Test mode, writing empty test file"); + fs.writeFileSync(outputFile, "test", "utf8"); + process.exit(0); + } + const document = await io.read(inputFile); if (textureRescale !== null){ @@ -90,36 +92,18 @@ async function main() { if (dimension === 0) dimension = 512; } - const encoder = require('sharp'); - let transforms = []; - if (simplifyRatio < 1){ - transforms.push(weld()); - transforms.push( - simplify({ - simplifier: MeshoptSimplifier, - error: 0.0001, - ratio: simplifyRatio, - lockBorder: false, - }), - ); - } - - transforms.push( + let transforms = [ textureCompress({ encoder, resize: [textureSize, textureSize], targetFormat: undefined, limitInputPixels: true, - }) - ); - - transforms.push( + }), draco({ quantizationVolume: "scene" }) - ); + ]; - await document.transform(...transforms); const outputDir = path.dirname(outputFile); diff --git a/app/tests/test_api_task.py b/app/tests/test_api_task.py index c0e7be99..007a7ced 100644 --- a/app/tests/test_api_task.py +++ b/app/tests/test_api_task.py @@ -813,6 +813,19 @@ class TestApiTask(BootTransactionTestCase): res = client.get("/api/projects/{}/tasks/{}/orthophoto/tiles/{}.png?size={}".format(project.id, task.id, tile_path['orthophoto'], s)) self.assertEqual(res.status_code, status.HTTP_400_BAD_REQUEST) + # This task's assets cache should not exist + ta_cache_dir = task.get_task_assets_cache() + self.assertFalse(os.path.isdir(ta_cache_dir)) + + # Can access the safe textured model endpoint + res = client.get("/api/projects/{}/tasks/{}/textured_model/".format(project.id, task.id)) + self.assertEqual(res.status_code, status.HTTP_200_OK) + + # The resulting GLB cache should have been created + self.assertTrue(os.path.isdir(ta_cache_dir)) + self.assertTrue(os.path.isfile(os.path.join(ta_cache_dir, "odm_textured_model_geo-2.glb"))) + + # Another user does not have access to the resources other_client = APIClient() other_client.login(username="testuser2", password="test1234") @@ -955,6 +968,9 @@ class TestApiTask(BootTransactionTestCase): task_assets_path = os.path.join(settings.MEDIA_ROOT, task_directory_path(task.id, task.project.id)) self.assertFalse(os.path.exists(task_assets_path)) + # Assets cache should also be removed + self.assertFalse(os.path.isdir(ta_cache_dir)) + # Create a task res = client.post("/api/projects/{}/tasks/".format(project.id), { diff --git a/app/tests/test_worker.py b/app/tests/test_worker.py index b5c40441..0ed99008 100644 --- a/app/tests/test_worker.py +++ b/app/tests/test_worker.py @@ -51,9 +51,46 @@ class TestWorker(BootTestCase): self.assertTrue(Task.objects.filter(pk=task.id).exists()) self.assertTrue(Project.objects.filter(pk=project.id).exists()) + # Generate some mock cached assets + ta_cache_dir = task.get_task_assets_cache() + self.assertFalse(os.path.isdir(ta_cache_dir)) + os.makedirs(ta_cache_dir) + mock_asset = os.path.join(ta_cache_dir, "test.txt") + with open(mock_asset, 'w', encoding='utf-8') as f: + f.write("test") + + # Set modified date + st = os.stat(ta_cache_dir) + atime = st[ST_ATIME] + mtime = st[ST_MTIME] + new_mtime = mtime - (29 * 24 * 3600) # 29 days ago + os.utime(ta_cache_dir, (atime, new_mtime)) + worker.tasks.cleanup_cache_directory() + + # File should still be there + self.assertTrue(os.path.isfile(mock_asset)) + self.assertTrue(os.path.isdir(ta_cache_dir)) + + new_mtime = mtime - (31 * 24 * 3600) # 31 days ago + os.utime(ta_cache_dir, (atime, new_mtime)) + worker.tasks.cleanup_cache_directory() + + # File and cache dirs should be gone + self.assertFalse(os.path.isfile(mock_asset)) + self.assertFalse(os.path.isdir(ta_cache_dir)) + + # Regenerate... + os.makedirs(ta_cache_dir) + mock_asset = os.path.join(ta_cache_dir, "asset.txt") + with open(mock_asset, 'w', encoding='utf-8') as f: + f.write("1") + # Remove task task.delete() + # Cache dir should be gone + self.assertFalse(os.path.isdir(ta_cache_dir)) + worker.tasks.cleanup_projects() # Task and project should have been removed (now that task count is zero) diff --git a/package.json b/package.json index 53b1efab..539f3ce1 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,6 @@ "json-loader": "^0.5.4", "leaflet": "1.3.1", "leaflet-fullscreen": "^1.0.2", - "meshoptimizer": "^0.25.0", "mini-css-extract-plugin": "1.6.2", "object.values": "^1.0.3", "proj4": "^2.4.3", diff --git a/worker/tasks.py b/worker/tasks.py index 222b3e4a..f3390f45 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -109,7 +109,7 @@ def cleanup_tmp_directory(): else: shutil.rmtree(filepath, ignore_errors=True) - logger.info('Cleaned up: %s (%s)' % (f, modified)) + logger.info('Cleaned up: %s (%s)' % (filepath, modified)) @app.task(ignore_result=True) @@ -129,7 +129,7 @@ def cleanup_cache_directory(): else: shutil.rmtree(filepath, ignore_errors=True) - logger.info('Cleaned up: %s (%s)' % (f, modified)) + logger.info('Cleaned up: %s (%s)' % (filepath, modified)) # Based on https://stackoverflow.com/questions/22498038/improve-current-implementation-of-a-setinterval-python/22498708#22498708 def setInterval(interval, func, *args):