Add unit tests, fixes

2025-09-19 16:39:02 -04:00 · 2025-09-19 16:39:02 -04:00 · 61ee41851c
commit 61ee41851c
--- a/app/models/task.py
+++ b/app/models/task.py
@ -1224,6 +1224,7 @@ class Task(models.Model):

        directory_to_delete = os.path.join(settings.MEDIA_ROOT,
                                           task_directory_path(self.id, self.project.id))
+        self.clear_task_assets_cache()

        super(Task, self).delete(using, keep_parents)

@ -1232,7 +1233,6 @@ class Task(models.Model):
            shutil.rmtree(directory_to_delete)
        except FileNotFoundError as e:
            logger.warning(e)
-        self.clear_task_assets_cache()

        self.project.owner.profile.clear_used_quota_cache()

@ -1457,10 +1457,15 @@ class Task(models.Model):


    def get_task_assets_cache(self):
+        if self.id is None:
+            return None
        return os.path.join(settings.MEDIA_CACHE, "task_assets", str(self.id))
    
    def clear_task_assets_cache(self):
        d = self.get_task_assets_cache()
+        if d is None:
+            return
+        
        if os.path.isdir(d):
            try:
                shutil.rmtree(d)
@ -1472,18 +1477,21 @@ class Task(models.Model):
        if input_glb is None or (not 'textured_model.glb' in self.available_assets):
            raise FileNotFoundError("GLB asset does not exist")
        
-        size = os.path.getsize(input_glb)
-        if size <= max_size_mb * 1024 * 1024:
-            return input_glb
-        
-        p, ext = os.path.splitext(input_glb)
-        base = os.path.basename(p)
-        cache_dir = self.get_task_assets_cache()
-        rescale = 1
+        if settings.TESTING:
+            rescale = 2
+        else:
+            size = os.path.getsize(input_glb)
+            if size <= max_size_mb * 1024 * 1024:
+                return input_glb
+            
+            p, ext = os.path.splitext(input_glb)
+            base = os.path.basename(p)
+            cache_dir = self.get_task_assets_cache()
+            rescale = 1

-        while size > max_size_mb * 1024 * 1024:
-            rescale *= 2
-            size = size // 2.6  # Texture size reduction factor (not science)
+            while size > max_size_mb * 1024 * 1024:
+                rescale *= 2
+                size = size // 2.6  # Texture size reduction factor (not science)

        output_glb = os.path.join(cache_dir, f"{base}-{rescale}{ext}")
        if os.path.isfile(output_glb):
@ -1519,10 +1527,13 @@ class Task(models.Model):
            glbopti_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../scripts/glbopti.js"))
            output_glb_tmp = output_glb + ".tmp.glb"

-            subprocess.run(["node", glbopti_path,
+            params = ["node", glbopti_path,
                            "--input", quote(input_glb), 
                            "--output", quote(output_glb_tmp),
-                            "--texture-rescale", str(rescale)], timeout=180)
+                            "--texture-rescale", str(rescale)]
+            if settings.TESTING:
+                params += ["--test"]
+            subprocess.run(params, timeout=180)

            if not os.path.isfile(output_glb_tmp):
                raise FileNotFoundError("GLB generation failed")
--- a/app/scripts/glbopti.js
+++ b/app/scripts/glbopti.js
@ -2,9 +2,9 @@ const fs = require('fs');
 const path = require('path');
 const { NodeIO, Extension } = require('@gltf-transform/core');
 const { KHRONOS_EXTENSIONS } = require('@gltf-transform/extensions');
-const { textureCompress, simplify, weld, draco } = require('@gltf-transform/functions');
-const { MeshoptSimplifier } = require('meshoptimizer');
+const { textureCompress, draco } = require('@gltf-transform/functions');
 const draco3d = require('draco3dgltf');
+const encoder = require('sharp');

 class CesiumRTC extends Extension {
 	extensionName = 'CESIUM_RTC';
@ -37,8 +37,8 @@ async function main() {
    let inputFile = '';
    let outputFile = '';
    let textureSize = 512;
-    let simplifyRatio = 1;
    let textureRescale = null;
+    let testMode = false;

    for (let i = 0; i < args.length; i++) {
        if (args[i] === '--input' && i + 1 < args.length) {
@ -54,13 +54,6 @@ async function main() {
                process.exit(1);
            }
            i++;
-        } else if (args[i] === '--simplify-ratio' && i + 1 < args.length) {
-            simplifyRatio = parseFloat(args[i + 1]);
-            if (isNaN(simplifyRatio) || simplifyRatio < 0 || simplifyRatio > 1){
-                console.log(`Invalid simplify ratio: ${args[i + 1]}`);
-                process.exit(1);
-            }
-            i++;
        } else if (args[i] === '--texture-rescale' && i + 1 < args.length) {
            textureRescale = parseInt(args[i + 1]);
            if (isNaN(textureRescale) || textureRescale < 1 || (textureRescale & (textureRescale - 1)) !== 0){
@ -68,15 +61,24 @@ async function main() {
                process.exit(1);
            }
            i++;
+        } else if (args[i] === '--test') {
+            testMode = true;
+            i++;
        }

    }

    if (!inputFile || !outputFile){
-        console.log('Usage: node glb_optimize.js --input <input.glb> --output <output.glb> [--texture-size <size>|--texture-rescale <factor>] [--simplify-ratio <ratio>]');
+        console.log('Usage: node glb_optimize.js --input <input.glb> --output <output.glb> [--texture-size <size>|--texture-rescale <factor>]');
        process.exit(1);
    }

+    if (testMode){
+        console.log("Test mode, writing empty test file");
+        fs.writeFileSync(outputFile, "test", "utf8");
+        process.exit(0);
+    }
+
    const document = await io.read(inputFile);

    if (textureRescale !== null){
@ -90,36 +92,18 @@ async function main() {
        if (dimension === 0) dimension = 512;
    }
    
-    const encoder = require('sharp');
-    let transforms = [];
-    if (simplifyRatio < 1){
-        transforms.push(weld());
-        transforms.push(
-            simplify({
-                simplifier: MeshoptSimplifier,
-                error: 0.0001,
-                ratio: simplifyRatio,
-                lockBorder: false,
-            }),
-        );
-    }
-
-    transforms.push(
+    let transforms = [
        textureCompress({
                encoder,
                resize: [textureSize, textureSize],
                targetFormat: undefined,
                limitInputPixels: true,
-        })
-    );
-
-    transforms.push(
+        }),
        draco({
            quantizationVolume: "scene"
        })
-    );
+    ];

-    
    await document.transform(...transforms);

    const outputDir = path.dirname(outputFile);
--- a/app/tests/test_api_task.py
+++ b/app/tests/test_api_task.py
@ -813,6 +813,19 @@ class TestApiTask(BootTransactionTestCase):
                res = client.get("/api/projects/{}/tasks/{}/orthophoto/tiles/{}.png?size={}".format(project.id, task.id, tile_path['orthophoto'], s))
                self.assertEqual(res.status_code, status.HTTP_400_BAD_REQUEST)
            
+            # This task's assets cache should not exist
+            ta_cache_dir = task.get_task_assets_cache()
+            self.assertFalse(os.path.isdir(ta_cache_dir))
+
+            # Can access the safe textured model endpoint
+            res = client.get("/api/projects/{}/tasks/{}/textured_model/".format(project.id, task.id))
+            self.assertEqual(res.status_code, status.HTTP_200_OK)
+
+            # The resulting GLB cache should have been created
+            self.assertTrue(os.path.isdir(ta_cache_dir))
+            self.assertTrue(os.path.isfile(os.path.join(ta_cache_dir, "odm_textured_model_geo-2.glb")))
+            
+
            # Another user does not have access to the resources
            other_client = APIClient()
            other_client.login(username="testuser2", password="test1234")
@ -955,6 +968,9 @@ class TestApiTask(BootTransactionTestCase):
            task_assets_path = os.path.join(settings.MEDIA_ROOT, task_directory_path(task.id, task.project.id))
            self.assertFalse(os.path.exists(task_assets_path))

+            # Assets cache should also be removed
+            self.assertFalse(os.path.isdir(ta_cache_dir))
+

        # Create a task
        res = client.post("/api/projects/{}/tasks/".format(project.id), {
--- a/app/tests/test_worker.py
+++ b/app/tests/test_worker.py
@ -51,9 +51,46 @@ class TestWorker(BootTestCase):
            self.assertTrue(Task.objects.filter(pk=task.id).exists())
            self.assertTrue(Project.objects.filter(pk=project.id).exists())

+            # Generate some mock cached assets
+            ta_cache_dir = task.get_task_assets_cache()
+            self.assertFalse(os.path.isdir(ta_cache_dir))
+            os.makedirs(ta_cache_dir)
+            mock_asset = os.path.join(ta_cache_dir, "test.txt")
+            with open(mock_asset, 'w', encoding='utf-8') as f:
+                f.write("test")
+            
+            # Set modified date
+            st = os.stat(ta_cache_dir)
+            atime = st[ST_ATIME]
+            mtime = st[ST_MTIME]
+            new_mtime = mtime - (29 * 24 * 3600)  # 29 days ago
+            os.utime(ta_cache_dir, (atime, new_mtime))
+            worker.tasks.cleanup_cache_directory()
+
+            # File should still be there
+            self.assertTrue(os.path.isfile(mock_asset))
+            self.assertTrue(os.path.isdir(ta_cache_dir))
+
+            new_mtime = mtime - (31 * 24 * 3600)  # 31 days ago
+            os.utime(ta_cache_dir, (atime, new_mtime))
+            worker.tasks.cleanup_cache_directory()
+
+            # File and cache dirs should be gone
+            self.assertFalse(os.path.isfile(mock_asset))
+            self.assertFalse(os.path.isdir(ta_cache_dir))
+
+            # Regenerate...
+            os.makedirs(ta_cache_dir)
+            mock_asset = os.path.join(ta_cache_dir, "asset.txt")
+            with open(mock_asset, 'w', encoding='utf-8') as f:
+                f.write("1")
+
            # Remove task
            task.delete()

+            # Cache dir should be gone
+            self.assertFalse(os.path.isdir(ta_cache_dir))
+
            worker.tasks.cleanup_projects()

            # Task and project should have been removed (now that task count is zero)
--- a/package.json
+++ b/package.json
@ -52,7 +52,6 @@
    "json-loader": "^0.5.4",
    "leaflet": "1.3.1",
    "leaflet-fullscreen": "^1.0.2",
-    "meshoptimizer": "^0.25.0",
    "mini-css-extract-plugin": "1.6.2",
    "object.values": "^1.0.3",
    "proj4": "^2.4.3",
--- a/worker/tasks.py
+++ b/worker/tasks.py
@ -109,7 +109,7 @@ def cleanup_tmp_directory():
            else:
                shutil.rmtree(filepath, ignore_errors=True)

-            logger.info('Cleaned up: %s (%s)' % (f, modified))
+            logger.info('Cleaned up: %s (%s)' % (filepath, modified))


@app.task(ignore_result=True)
@ -129,7 +129,7 @@ def cleanup_cache_directory():
                else:
                    shutil.rmtree(filepath, ignore_errors=True)

-                logger.info('Cleaned up: %s (%s)' % (f, modified))
+                logger.info('Cleaned up: %s (%s)' % (filepath, modified))

 # Based on https://stackoverflow.com/questions/22498038/improve-current-implementation-of-a-setinterval-python/22498708#22498708
 def setInterval(interval, func, *args):